From 0ca7316a24f6e4a5ee60bea2ed8f62f6314f0c90 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Fri, 14 Aug 2026 14:57:50 +0300 Subject: [PATCH] wip(k1): checkpoint connection recovery rewrite Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes. --- README.md | 5 +- apps/control-station/README.md | 10 +- apps/control-station/src/App.tsx | 15 +- .../src/components/MseFmp4WebSocketPlayer.tsx | 399 +- .../src/components/ObservationSources.tsx | 9 +- .../src/components/RerunViewport.tsx | 257 +- .../components/useApplicationPanelActions.ts | 44 +- .../core/device-plugins/DevicePluginHost.tsx | 73 +- .../core/observation/liveCameraRecovery.ts | 204 + .../core/observation/liveReceiverWatchdog.ts | 257 +- .../core/observation/liveViewerDiagnostics.ts | 373 +- .../core/observation/useObservationLayout.ts | 144 +- .../src/core/runtime/contracts.ts | 11 + apps/control-station/src/presentation.ts | 8 + apps/control-station/src/productModel.ts | 4 +- .../control-station/src/styles/responsive.css | 98 + .../src/workspaces/ContourHealthWorkspace.tsx | 42 +- .../src/workspaces/DeviceWorkspace.tsx | 5 +- .../src/workspaces/Workspaces.tsx | 7 +- .../test/devicePluginContracts.test.mjs | 3410 +- .../devicePluginFrontendBoundary.test.mjs | 979 +- .../test/devicePluginHostPersistence.test.mjs | 219 + .../test/k1ActiveStreamRecovery.test.mjs | 977 + .../test/k1SupervisorPresentation.test.mjs | 6966 ++++ .../test/liveReceiverWatchdog.test.mjs | 224 + .../test/liveViewerDiagnostics.test.mjs | 216 + .../test/observationSources.test.mjs | 922 +- .../rerunViewportAtomicAdmission.test.mjs | 64 +- .../test/runtimeStateOrdering.test.mjs | 107 + .../test/workspaceLayout.test.mjs | 95 + docs/04_K1_WIFI_PROVISIONING_PROFILE.md | 65 +- docs/06_K1_LIVE_VIEWER.md | 7 +- docs/20_K1_CONNECTION_SUPERVISION_CANON.md | 1141 + ...3-device-plugin-ui-and-runtime-boundary.md | 2 +- docs/adr/0013-k1-local-connection-matrix.md | 128 +- .../adr/0014-k1-macos-association-observer.md | 177 + docs/adr/0015-k1-physical-state-recovery.md | 309 + docs/k1-connection-acceptance.manifest.json | 93 + docs/runbooks/K1_CONNECTION_RECOVERY.md | 393 + .../worker/run_e15_shadow_inference.py | 152 +- plugins/xgrids-k1/README.md | 16 +- plugins/xgrids-k1/frontend/README.md | 14 +- .../frontend/src/XgridsK1Connection.tsx | 349 +- .../frontend/src/activeStreamRecovery.ts | 278 + plugins/xgrids-k1/frontend/src/api.ts | 1514 +- .../ActiveStreamRecoverySurface.tsx | 172 + .../src/components/K1AcquisitionPipeline.tsx | 437 +- .../frontend/src/components/K1Diagnostics.tsx | 33 +- .../frontend/src/components/K1Metrics.tsx | 10 +- .../src/components/K1OperatorError.tsx | 225 + .../src/components/K1ProvisioningPipeline.tsx | 3580 +- .../src/components/K1SpatialControls.tsx | 251 +- .../xgrids-k1/frontend/src/configuration.ts | 8 +- .../frontend/src/controlSessionCas.ts | 72 + .../src/hostDiagnosticPresentation.ts | 132 + plugins/xgrids-k1/frontend/src/lifecycle.ts | 1527 +- plugins/xgrids-k1/frontend/src/manifest.ts | 28 + plugins/xgrids-k1/frontend/src/messages.ts | 1 - .../frontend/src/observationSources.ts | 274 +- .../frontend/src/operatorIntentGeneration.ts | 9 + .../src/physicalCommandConfirmation.ts | 430 + .../xgrids-k1/frontend/src/presentation.ts | 240 +- plugins/xgrids-k1/frontend/src/projectName.ts | 21 + .../xgrids-k1/frontend/src/runtimeContext.tsx | 43 +- .../xgrids-k1/frontend/src/stateOrdering.ts | 71 + plugins/xgrids-k1/frontend/src/styles.css | 621 +- .../frontend/src/useXgridsK1Runtime.ts | 1817 +- plugins/xgrids-k1/macos/associate_wifi.swift | 578 +- plugins/xgrids-k1/plugin.manifest.json | 11 +- scripts/build_m4_worker_shadow_artifact.py | 2 +- src/k1link/artifacts.py | 14 +- .../l34a_assisted_yolox_error_audit.py | 5 +- src/k1link/compute/live_perception.py | 65 +- .../active_acquisition_recovery_checkpoint.py | 5368 +++ .../application_control_process_lease.py | 265 + .../xgrids_k1/ble/ap_activation.py | 179 +- .../device_plugins/xgrids_k1/ble/gatt.py | 105 +- .../device_plugins/xgrids_k1/ble/reader.py | 73 +- .../xgrids_k1/ble/runtime_arbiter.py | 745 + .../device_plugins/xgrids_k1/ble/scanner.py | 1077 +- .../xgrids_k1/ble/wifi_provisioning.py | 759 +- src/k1link/device_plugins/xgrids_k1/camera.py | 983 +- src/k1link/device_plugins/xgrids_k1/cli.py | 442 +- .../xgrids_k1/connection_supervisor.py | 2181 ++ .../xgrids_k1/device_identity_pin_store.py | 641 + src/k1link/device_plugins/xgrids_k1/facade.py | 30310 +++++++++++++++- .../xgrids_k1/host_diagnostics.py | 429 + .../device_plugins/xgrids_k1/legacy_api.py | 28 +- .../xgrids_k1/live_perception_shadow.py | 42 +- .../device_plugins/xgrids_k1/mqtt/capture.py | 582 +- .../xgrids_k1/network_mutation_ledger.py | 958 + ...etwork_provisioning_idempotency_journal.py | 1153 + .../device_plugins/xgrids_k1/observation.py | 11 +- .../xgrids_k1/physical_command_coordinator.py | 1115 + .../xgrids_k1/physical_command_ledger.py | 6914 ++++ .../protocol/application_acceptance.py | 258 +- .../protocol/application_authority.py | 32 +- .../xgrids_k1/protocol/application_mqtt.py | 812 +- .../xgrids_k1/protocol/application_session.py | 1145 +- .../xgrids_k1/semantic_topology_store.py | 667 + .../xgrids_k1/viewer/messages.py | 5 + .../xgrids_k1/viewer/runtime.py | 1124 +- src/k1link/host_network/__init__.py | 8 + src/k1link/host_network/wifi.py | 800 +- src/k1link/sessions/active.py | 26 +- src/k1link/viewer/rerun_bridge.py | 16 +- src/k1link/web/app.py | 66 +- src/k1link/web/device_lifecycle.py | 55 + src/k1link/web/frontend_assets.py | 49 + src/k1link/web/plugin_runtime.py | 21 +- src/k1link/web/runtime_diagnostics.py | 4 + src/k1link/web/viewer_diagnostics_api.py | 91 +- tests/conftest.py | 50 + tests/test_active_session_lease.py | 66 + tests/test_artifacts.py | 22 + tests/test_ble_scanner.py | 1477 +- tests/test_canonical_pipeline.py | 1239 +- tests/test_cli.py | 414 + tests/test_connection_supervisor.py | 2324 ++ tests/test_device_lifecycle.py | 14 +- tests/test_frontend_assets.py | 60 + tests/test_frontend_plugin_boundaries.py | 31 +- .../test_k1_connection_acceptance_manifest.py | 47 + tests/test_live_perception.py | 39 + tests/test_live_perception_shadow.py | 111 +- tests/test_mqtt_capture.py | 467 + tests/test_plugin_catalog.py | 4 +- tests/test_plugin_runtime.py | 752 +- tests/test_pytest_runtime_isolation.py | 51 + tests/test_rerun_bridge.py | 186 +- tests/test_session_store.py | 2 + tests/test_viewer_diagnostics_api.py | 97 +- tests/test_web_console.py | 8 +- tests/test_web_validation_security.py | 117 + tests/test_wifi_provisioning.py | 948 +- tests/test_xgrids_acquisition_lifecycle.py | 28711 ++++++++++++++- ...tive_acquisition_checkpoint_integration.py | 1917 + ..._active_acquisition_recovery_checkpoint.py | 3095 ++ ..._active_acquisition_restart_rehydration.py | 2386 ++ ...grids_active_acquisition_restart_safety.py | 1269 + tests/test_xgrids_ap_activation.py | 192 +- tests/test_xgrids_application_acceptance.py | 125 +- tests/test_xgrids_application_authority.py | 17 + ...grids_application_control_process_lease.py | 821 + tests/test_xgrids_application_mqtt.py | 1006 +- tests/test_xgrids_application_session.py | 1681 +- tests/test_xgrids_ble_runtime_arbiter.py | 823 + tests/test_xgrids_camera_gateway.py | 1232 +- .../test_xgrids_connection_scenario_reset.py | 2886 ++ .../test_xgrids_device_identity_pin_store.py | 485 + tests/test_xgrids_host_diagnostics.py | 367 + tests/test_xgrids_macos_wifi.py | 956 +- tests/test_xgrids_network_mutation_ledger.py | 781 + ...etwork_provisioning_idempotency_journal.py | 815 + ...est_xgrids_physical_command_coordinator.py | 1393 + tests/test_xgrids_physical_command_ledger.py | 5978 +++ tests/test_xgrids_semantic_topology_store.py | 538 + 157 files changed, 152962 insertions(+), 4036 deletions(-) create mode 100644 apps/control-station/src/core/observation/liveCameraRecovery.ts create mode 100644 apps/control-station/test/devicePluginHostPersistence.test.mjs create mode 100644 apps/control-station/test/k1ActiveStreamRecovery.test.mjs create mode 100644 apps/control-station/test/k1SupervisorPresentation.test.mjs create mode 100644 apps/control-station/test/liveViewerDiagnostics.test.mjs create mode 100644 docs/20_K1_CONNECTION_SUPERVISION_CANON.md create mode 100644 docs/adr/0014-k1-macos-association-observer.md create mode 100644 docs/adr/0015-k1-physical-state-recovery.md create mode 100644 docs/k1-connection-acceptance.manifest.json create mode 100644 docs/runbooks/K1_CONNECTION_RECOVERY.md create mode 100644 plugins/xgrids-k1/frontend/src/activeStreamRecovery.ts create mode 100644 plugins/xgrids-k1/frontend/src/components/ActiveStreamRecoverySurface.tsx create mode 100644 plugins/xgrids-k1/frontend/src/components/K1OperatorError.tsx create mode 100644 plugins/xgrids-k1/frontend/src/controlSessionCas.ts create mode 100644 plugins/xgrids-k1/frontend/src/hostDiagnosticPresentation.ts create mode 100644 plugins/xgrids-k1/frontend/src/physicalCommandConfirmation.ts create mode 100644 src/k1link/device_plugins/xgrids_k1/active_acquisition_recovery_checkpoint.py create mode 100644 src/k1link/device_plugins/xgrids_k1/application_control_process_lease.py create mode 100644 src/k1link/device_plugins/xgrids_k1/ble/runtime_arbiter.py create mode 100644 src/k1link/device_plugins/xgrids_k1/connection_supervisor.py create mode 100644 src/k1link/device_plugins/xgrids_k1/device_identity_pin_store.py create mode 100644 src/k1link/device_plugins/xgrids_k1/host_diagnostics.py create mode 100644 src/k1link/device_plugins/xgrids_k1/network_mutation_ledger.py create mode 100644 src/k1link/device_plugins/xgrids_k1/network_provisioning_idempotency_journal.py create mode 100644 src/k1link/device_plugins/xgrids_k1/physical_command_coordinator.py create mode 100644 src/k1link/device_plugins/xgrids_k1/physical_command_ledger.py create mode 100644 src/k1link/device_plugins/xgrids_k1/semantic_topology_store.py create mode 100644 src/k1link/web/frontend_assets.py create mode 100644 tests/conftest.py create mode 100644 tests/test_connection_supervisor.py create mode 100644 tests/test_frontend_assets.py create mode 100644 tests/test_k1_connection_acceptance_manifest.py create mode 100644 tests/test_pytest_runtime_isolation.py create mode 100644 tests/test_xgrids_active_acquisition_checkpoint_integration.py create mode 100644 tests/test_xgrids_active_acquisition_recovery_checkpoint.py create mode 100644 tests/test_xgrids_active_acquisition_restart_rehydration.py create mode 100644 tests/test_xgrids_active_acquisition_restart_safety.py create mode 100644 tests/test_xgrids_application_control_process_lease.py create mode 100644 tests/test_xgrids_ble_runtime_arbiter.py create mode 100644 tests/test_xgrids_connection_scenario_reset.py create mode 100644 tests/test_xgrids_device_identity_pin_store.py create mode 100644 tests/test_xgrids_host_diagnostics.py create mode 100644 tests/test_xgrids_network_mutation_ledger.py create mode 100644 tests/test_xgrids_network_provisioning_idempotency_journal.py create mode 100644 tests/test_xgrids_physical_command_coordinator.py create mode 100644 tests/test_xgrids_physical_command_ledger.py create mode 100644 tests/test_xgrids_semantic_topology_store.py diff --git a/README.md b/README.md index 1fa777e..0b45d90 100644 --- a/README.md +++ b/README.md @@ -383,8 +383,9 @@ telemetry and the stop action. Operator-manual acquisition still finalizes only local reception; plugin-commanded v0.5.0 acquisition uses the separately gated canonical K1 START/STOP dialogue. -Plugin v0.6.0 makes the local connection direction explicit. Bridge remains the -default and accepted product path; Direct Connect sends the reviewed station +Plugin v0.7.0 supervises the v0.6.0 local connection matrix with separate +desired, configured and active modes plus exact DeviceInfo-backed Ready. +Bridge remains the default and accepted product path; Direct Connect sends the reviewed station provisioning frame for an already-running controller hotspot. Quick Connect sends one separately reviewed AP-enable frame and can associate a prepared Mac through CoreWLAN. Its device activation and prepared-host association were diff --git a/apps/control-station/README.md b/apps/control-station/README.md index 2b2efd4..d076462 100644 --- a/apps/control-station/README.md +++ b/apps/control-station/README.md @@ -321,10 +321,12 @@ facts и старые presentation-поля `phase`, `message`, `devices`, активным между сессиями; для его закрытия нужно остановить `k1link serve`. - Пароль Wi-Fi находится только в React memory, передаётся в JSON POST body, очищается после успешного ответа и не сохраняется в URL/local storage. -- Bridge и Direct Connect выполняют только отдельно рассмотренную provisioning- - запись; Quick Connect не пишет в GATT и передаёт пароль короткоживущему - CoreWLAN helper только через stdin. Случайные GATT writes и автоматические - повторы запрещены. +- Bridge и Direct Connect выполняют только отдельно рассмотренную 99-байтную + provisioning-запись. Quick Connect выполняет отдельную рассмотренную + 100-байтную AP-enable запись и после AP-ready допускает ровно одну системную + ассоциацию с заранее материализованным exact-firmware профилем. Пароль не + попадает в браузер/API/argv; системный Wi-Fi Keychain, post-write prompt и + автоматические повторы запрещены. - MQTT data live/replay остаётся subscribe-only. Команды изолированы в plugin-owned canonical control session и доступны только через отдельные operator-present UI checkpoints. diff --git a/apps/control-station/src/App.tsx b/apps/control-station/src/App.tsx index 23f7571..e8063b9 100644 --- a/apps/control-station/src/App.tsx +++ b/apps/control-station/src/App.tsx @@ -63,7 +63,7 @@ import { workspacesForRoot, type RootId, } from "./productModel"; -import { backendLabel, phaseLabel, phaseTone } from "./presentation"; +import { backendLabel, localConnectionPhaseLabel, phaseTone } from "./presentation"; import { defaultSceneSettings, type PointColorMode, @@ -595,8 +595,8 @@ export default function App() { setRecordedReplayLabel(null); setSourceUrl(""); setSourceDraft(""); - }, []); - + observationLayout.activateAutomaticDefaults(); + }, [observationLayout.activateAutomaticDefaults]); useEffect(() => { if (!["recordings", "lab-archive"].includes(activeDefinition?.kind ?? "")) { setReplayTransitioning(false); @@ -638,6 +638,8 @@ export default function App() { const contentActions = useApplicationPanelActions({ definition: activeDefinition, refreshRuntime: runtime.refresh, + resetConnectionScenario: runtime.resetConnectionScenario, + connectionScenarioResetting: runtime.pendingAction === "mode", saveWorkspaceLayout, workspaceLayoutSaving: workspaceLayoutProfile.state === "saving", systemUtilityActions: computeContourSettings.utilityActions, @@ -728,10 +730,7 @@ export default function App() { { id: "local-contour", label: "Локальный контур", - description: - runtime.state?.activeDevice?.endpointLabel || - selection?.model.displayName || - "Модель не выбрана", + description: selection ? "Подключение" : "Модель не выбрана", icon: , active: runtime.backendStatus !== "offline" @@ -777,7 +776,7 @@ export default function App() { headerTools={ activeDefinition.kind === "device" ? ( - {phaseLabel(runtime.state?.phase)} + {localConnectionPhaseLabel(runtime.state?.phase)} ) : activeDefinition.kind === "spatial" ? (
diff --git a/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx b/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx index 3588e62..ef69bb6 100644 --- a/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx +++ b/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx @@ -2,6 +2,14 @@ import { useEffect, useRef, useState } from "react"; import { Icon } from "@nodedc/ui-react"; import type { ObservationSourceDelivery } from "../core/runtime/contracts"; +import { + cameraBrowserTransportIdentity, + cameraTransportCallbackIsCurrent, + initialCameraPlaybackRecoveryState, + reduceCameraPlaybackRecovery, + type CameraPlaybackRecoveryEvent, +} from "../core/observation/liveCameraRecovery"; +import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics"; type PlayerStatus = "connecting" | "buffering" | "playing" | "error"; @@ -10,7 +18,92 @@ export interface CameraLeaseRetryBudget { count: number; } -const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000] as const; +const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000, 5_000] as const; +export const CAMERA_FIRST_MEDIA_TIMEOUT_MS = 8_000; +export const CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS = 8_000; +// The gateway may release an 8 MiB / 64-fragment slow-reader backlog after a +// main-thread stall. Keep one bounded append margin above that complete batch; +// crossing either limit replaces the MSE epoch instead of dropping fragments. +export const CAMERA_PENDING_QUEUE_MAX_BYTES = 12 * 1024 * 1024; +export const CAMERA_PENDING_QUEUE_MAX_SEGMENTS = 96; + +export type CameraStartupWatchdogStage = "first-media" | "first-playable-frame"; + +export interface CameraStartupWatchdog { + armFirstMedia: () => void; + markMediaReceived: () => void; + markPlaying: () => void; + clear: () => void; + pendingStage: () => CameraStartupWatchdogStage | null; +} + +/** + * Supervise only the disposable browser transport. One fixed first-media + * deadline covers both an MSE that never opens and an open-but-silent socket. + * The first non-empty fragment then starts a separate first-playable-frame + * deadline; later fragments deliberately do not extend it. + */ +export function createCameraStartupWatchdog({ + schedule, + cancel, + onTimeout, + firstMediaTimeoutMs = CAMERA_FIRST_MEDIA_TIMEOUT_MS, + firstPlayableFrameTimeoutMs = CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS, +}: { + schedule: (callback: () => void, timeoutMs: number) => number; + cancel: (handle: number) => void; + onTimeout: (stage: CameraStartupWatchdogStage) => void; + firstMediaTimeoutMs?: number; + firstPlayableFrameTimeoutMs?: number; +}): CameraStartupWatchdog { + let timer: number | undefined; + let stage: CameraStartupWatchdogStage | null = null; + let mediaReceived = false; + let playing = false; + + const clearTimer = () => { + if (timer !== undefined) cancel(timer); + timer = undefined; + stage = null; + }; + const arm = (nextStage: CameraStartupWatchdogStage, timeoutMs: number) => { + clearTimer(); + stage = nextStage; + timer = schedule(() => { + if (stage !== nextStage || playing) return; + timer = undefined; + stage = null; + onTimeout(nextStage); + }, timeoutMs); + }; + + return { + armFirstMedia() { + if (mediaReceived || playing || stage !== null) return; + arm("first-media", firstMediaTimeoutMs); + }, + markMediaReceived() { + if (mediaReceived || playing) return; + mediaReceived = true; + arm("first-playable-frame", firstPlayableFrameTimeoutMs); + }, + markPlaying() { + if (playing) return; + playing = true; + clearTimer(); + }, + clear: clearTimer, + pendingStage: () => stage, + }; +} + +export function cameraStartupWatchdogRecoveryMessage( + stage: CameraStartupWatchdogStage, +): string { + return stage === "first-media" + ? "Камера подключена, но не передаёт медиаданные; восстанавливаем browser-preview." + : "Медиаданные поступают, но первый кадр не воспроизводится; пересоздаём decoder."; +} export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetryBudget { return { deliveryId, count: 0 }; @@ -19,15 +112,60 @@ export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetr export function consumeCameraLeaseRetry( current: CameraLeaseRetryBudget, deliveryId: string, -): { budget: CameraLeaseRetryBudget; delay: number | null } { +): { budget: CameraLeaseRetryBudget; delay: number } { const count = current.deliveryId === deliveryId ? current.count : 0; - const delay = CAMERA_LEASE_RETRY_DELAYS[count] ?? null; + const delayIndex = Math.min(count, CAMERA_LEASE_RETRY_DELAYS.length - 1); + const delay = CAMERA_LEASE_RETRY_DELAYS[delayIndex]; return { - budget: { deliveryId, count: delay === null ? count : count + 1 }, + budget: { + deliveryId, + count: Math.min(count + 1, CAMERA_LEASE_RETRY_DELAYS.length), + }, delay, }; } +export function cameraTransportRecoveryIsCurrent( + activeAuthorityIdentity: string | null, + expectedAuthorityIdentity: string | null, + activeEpoch: number, + expectedEpoch: number, + disposed: boolean, +): boolean { + return Boolean( + expectedAuthorityIdentity + && activeAuthorityIdentity === expectedAuthorityIdentity + && cameraTransportCallbackIsCurrent(activeEpoch, expectedEpoch, disposed), + ); +} + +export function cameraTransportCanOpen(uiBuildStale: boolean): boolean { + return !uiBuildStale; +} + +export function cameraPendingQueueCanAccept( + queuedBytes: number, + queuedSegments: number, + incomingBytes: number, +): boolean { + return incomingBytes > 0 + && queuedBytes + incomingBytes <= CAMERA_PENDING_QUEUE_MAX_BYTES + && queuedSegments + 1 <= CAMERA_PENDING_QUEUE_MAX_SEGMENTS; +} + +export function cameraTransportCloseRecoveryMessage(code: number): string { + if (code === 4_008) { + return "Browser-reader отстал от эфира; восстанавливаем текущую камеру."; + } + if (code === 1_008) { + return "Camera adapter освобождает прежний browser-reader; переподключаемся."; + } + if (code === 1_000) { + return "Browser-preview завершился; восстанавливаем текущую камеру."; + } + return "Browser-preview прерван; восстанавливаем текущую камеру."; +} + function websocketUrl(path: string): string { const url = new URL(path, window.location.href); if (url.protocol === "http:") url.protocol = "ws:"; @@ -41,61 +179,149 @@ function websocketUrl(path: string): string { export function MseFmp4WebSocketPlayer({ delivery, label, + recoveryAuthorityIdentity, }: { delivery: ObservationSourceDelivery & { kind: "mse-fmp4-websocket" }; label: string; + recoveryAuthorityIdentity: string | null; }) { const videoRef = useRef(null); const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id)); + const activeAuthorityRef = useRef(recoveryAuthorityIdentity); + const recoveryPendingAuthorityRef = useRef(null); + const transportEpochRef = useRef(0); + const transportAuthorityRef = useRef(recoveryAuthorityIdentity); + const activeTransportDisposeRef = useRef<(() => void) | null>(null); + const uiBuildStaleRef = useRef(false); const [attempt, setAttempt] = useState(0); + const [uiBuildStale, setUiBuildStale] = useState(false); const [status, setStatus] = useState("connecting"); const [message, setMessage] = useState("Подключение к локальному видеопотоку"); + activeAuthorityRef.current = recoveryAuthorityIdentity; + const transportIdentity = cameraBrowserTransportIdentity( + delivery, + recoveryAuthorityIdentity, + ); + + useEffect(() => subscribeToLiveViewerBuildFence(() => { + // Setting this local fence tears down the effect-owned WebSocket and MSE + // buffer. It intentionally never invokes a plugin or device command. + uiBuildStaleRef.current = true; + activeTransportDisposeRef.current?.(); + setUiBuildStale(true); + }), []); useEffect(() => { const video = videoRef.current; if (!video) return; + if (uiBuildStale || !cameraTransportCanOpen(uiBuildStaleRef.current)) { + video.pause(); + video.removeAttribute("src"); + video.load(); + return; + } + + if (transportAuthorityRef.current !== recoveryAuthorityIdentity) { + transportAuthorityRef.current = recoveryAuthorityIdentity; + leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id); + } let disposed = false; + const transportEpoch = transportEpochRef.current + 1; + transportEpochRef.current = transportEpoch; + const transportIsCurrent = () => cameraTransportCallbackIsCurrent( + transportEpochRef.current, + transportEpoch, + disposed || uiBuildStaleRef.current, + ); let socket: WebSocket | null = null; let sourceBuffer: SourceBuffer | null = null; + let onBufferUpdateEnd: (() => void) | null = null; + let onBufferError: (() => void) | null = null; let objectUrl = ""; let retryTimer: number | undefined; let receivedMedia = false; let failed = false; + let startupWatchdog: CameraStartupWatchdog | null = null; + const recovering = Boolean( + activeAuthorityRef.current + && recoveryPendingAuthorityRef.current === activeAuthorityRef.current, + ); const queue: ArrayBuffer[] = []; let queuedBytes = 0; const fail = (copy: string) => { - if (disposed || failed) return; + if (!transportIsCurrent() || failed) return; + startupWatchdog?.clear(); failed = true; + recoveryPendingAuthorityRef.current = null; setStatus("error"); setMessage(copy); try { if (socket && socket.readyState < WebSocket.CLOSING) { - socket.close(1011, "Live video buffer reset"); + socket.close(4_000, "Live video buffer reset"); } } catch { // The manual reconnect button will create a fresh transport and MSE buffer. } }; - const retryLease = () => { - const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id); - leaseRetryRef.current = retry.budget; - if (retry.delay === null) { - fail("Camera adapter ещё занят предыдущим окном. Подключитесь повторно."); + const retryTransport = (copy: string) => { + if (!transportIsCurrent() || failed) return; + if (!cameraTransportRecoveryIsCurrent( + activeAuthorityRef.current, + recoveryAuthorityIdentity, + transportEpochRef.current, + transportEpoch, + disposed || uiBuildStaleRef.current, + )) { + fail(copy); return; } + startupWatchdog?.clear(); + failed = true; + queue.length = 0; + queuedBytes = 0; + const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id); + leaseRetryRef.current = retry.budget; + recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity; setStatus("connecting"); - setMessage("Освобождение предыдущего окна камеры"); + setMessage(copy); + try { + if (socket && socket.readyState < WebSocket.CLOSING) { + socket.close(4_001, "Live video transport recovery"); + } + } catch { + // The replacement effect still fences and releases this transport. + } retryTimer = window.setTimeout(() => { - if (!disposed) setAttempt((value) => value + 1); + if (cameraTransportRecoveryIsCurrent( + activeAuthorityRef.current, + recoveryAuthorityIdentity, + transportEpochRef.current, + transportEpoch, + disposed || uiBuildStaleRef.current, + )) { + setAttempt((value) => value + 1); + } }, retry.delay); }; + if (recoveryAuthorityIdentity) { + startupWatchdog = createCameraStartupWatchdog({ + schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs), + cancel: (handle) => window.clearTimeout(handle), + onTimeout: (stage) => { + retryTransport(cameraStartupWatchdogRecoveryMessage(stage)); + }, + }); + } + const onPlaying = () => { - if (disposed) return; + if (!transportIsCurrent() || failed) return; + startupWatchdog?.markPlaying(); leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id); + recoveryPendingAuthorityRef.current = null; setStatus("playing"); setMessage(""); }; @@ -103,25 +329,32 @@ export function MseFmp4WebSocketPlayer({ video.addEventListener("playing", onPlaying); const appendNext = () => { - if (disposed || !sourceBuffer || sourceBuffer.updating || queue.length === 0) return; + if ( + !transportIsCurrent() + || failed + || !sourceBuffer + || sourceBuffer.updating + || queue.length === 0 + ) return; const chunk = queue.shift(); if (!chunk) return; queuedBytes -= chunk.byteLength; try { sourceBuffer.appendBuffer(chunk); } catch (error) { - fail(error instanceof DOMException && error.name === "QuotaExceededError" - ? "Live-буфер переполнен и сброшен, чтобы не накапливать задержку." - : "Не удалось добавить видеосегмент. Повторите подключение."); + retryTransport(error instanceof DOMException && error.name === "QuotaExceededError" + ? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки." + : "Не удалось добавить видеосегмент; восстанавливаем browser-preview."); } }; const enqueue = (chunk: ArrayBuffer) => { - if (disposed || chunk.byteLength === 0) return; + if (!transportIsCurrent() || failed || chunk.byteLength === 0) return; + startupWatchdog?.markMediaReceived(); // Never drop arbitrary fMP4 fragments: the following samples may depend // on them. A bounded reset is safer and keeps live latency deterministic. - if (queuedBytes + chunk.byteLength > 2 * 1024 * 1024) { - fail("Видеодекодер не успевает за эфиром. Live-буфер сброшен."); + if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) { + retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер."); return; } queue.push(chunk); @@ -142,13 +375,16 @@ export function MseFmp4WebSocketPlayer({ } setStatus("connecting"); - setMessage("Подключение к локальному видеопотоку"); + setMessage(recovering + ? "Восстановление камеры после разрыва браузерного канала" + : "Подключение к локальному видеопотоку"); const mediaSource = new MediaSource(); objectUrl = URL.createObjectURL(mediaSource); video.src = objectUrl; + startupWatchdog?.armFirstMedia(); const onSourceOpen = () => { - if (disposed) return; + if (!transportIsCurrent() || failed) return; try { sourceBuffer = mediaSource.addSourceBuffer(mediaType); } catch { @@ -156,8 +392,8 @@ export function MseFmp4WebSocketPlayer({ return; } - sourceBuffer.addEventListener("updateend", () => { - if (disposed || !sourceBuffer) return; + onBufferUpdateEnd = () => { + if (!transportIsCurrent() || failed || !sourceBuffer) return; const buffered = sourceBuffer.buffered; if (buffered.length > 0) { const end = buffered.end(buffered.length - 1); @@ -166,7 +402,9 @@ export function MseFmp4WebSocketPlayer({ if (!receivedMedia) { receivedMedia = true; setStatus("buffering"); - setMessage("Запуск первого декодированного кадра"); + setMessage(recovering + ? "Запуск первого кадра восстановленной камеры" + : "Запуск первого декодированного кадра"); void video.play().catch(() => undefined); } const removeBefore = end - 3; @@ -180,10 +418,12 @@ export function MseFmp4WebSocketPlayer({ } } appendNext(); - }); - sourceBuffer.addEventListener("error", () => { - fail("MSE сообщил об ошибке декодирования видеосегмента."); - }); + }; + onBufferError = () => { + retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder."); + }; + sourceBuffer.addEventListener("updateend", onBufferUpdateEnd); + sourceBuffer.addEventListener("error", onBufferError); try { socket = new WebSocket(websocketUrl(delivery.url)); @@ -193,41 +433,49 @@ export function MseFmp4WebSocketPlayer({ } socket.binaryType = "arraybuffer"; socket.addEventListener("open", () => { - if (disposed) return; + if (!transportIsCurrent() || failed) return; + startupWatchdog?.armFirstMedia(); setStatus("buffering"); - setMessage("Ожидание первого видеокадра"); + setMessage(recovering + ? "Ожидание первого кадра после восстановления" + : "Ожидание первого видеокадра"); }); socket.addEventListener("message", (event) => { + if (!transportIsCurrent() || failed) return; if (event.data instanceof ArrayBuffer) { enqueue(event.data); } else if (event.data instanceof Blob) { void event.data.arrayBuffer().then(enqueue).catch(() => { - fail("Получен повреждённый видеосегмент."); + retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал."); }); } }); socket.addEventListener("error", () => { - fail("Соединение с локальным video adapter потеряно."); + retryTransport("Связь с локальным video adapter потеряна; переподключаемся."); }); socket.addEventListener("close", (event) => { - if (!disposed && !failed && event.code === 1008) { - retryLease(); - } else if (!disposed) { - fail(event.code === 1000 - ? "Видеопоток завершён. Можно подключиться повторно." - : "Видеопоток прерван. Проверьте устройство и повторите подключение."); - } + if (!transportIsCurrent() || failed) return; + retryTransport(cameraTransportCloseRecoveryMessage(event.code)); }); }; mediaSource.addEventListener("sourceopen", onSourceOpen, { once: true }); - return () => { + const disposeTransport = () => { + if (disposed) return; disposed = true; + startupWatchdog?.clear(); if (retryTimer !== undefined) window.clearTimeout(retryTimer); queue.length = 0; - socket?.close(1000, "Источник скрыт оператором"); + socket?.close(1000, "Browser preview transport replaced or hidden"); video.removeEventListener("playing", onPlaying); + mediaSource.removeEventListener("sourceopen", onSourceOpen); + if (sourceBuffer && onBufferUpdateEnd) { + sourceBuffer.removeEventListener("updateend", onBufferUpdateEnd); + } + if (sourceBuffer && onBufferError) { + sourceBuffer.removeEventListener("error", onBufferError); + } try { if (sourceBuffer?.updating) sourceBuffer.abort(); } catch { @@ -243,7 +491,69 @@ export function MseFmp4WebSocketPlayer({ video.load(); if (objectUrl) URL.revokeObjectURL(objectUrl); }; - }, [attempt, delivery.id, delivery.mediaType, delivery.url]); + activeTransportDisposeRef.current = disposeTransport; + return () => { + if (activeTransportDisposeRef.current === disposeTransport) { + activeTransportDisposeRef.current = null; + } + disposeTransport(); + }; + }, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]); + + useEffect(() => { + if ( + !recoveryAuthorityIdentity + || uiBuildStale + || !cameraTransportCanOpen(uiBuildStaleRef.current) + ) return; + + let recovery = initialCameraPlaybackRecoveryState( + recoveryAuthorityIdentity, + Date.now(), + ); + const dispatchRecovery = (event: CameraPlaybackRecoveryEvent) => { + if (uiBuildStaleRef.current) return; + const decision = reduceCameraPlaybackRecovery(recovery, event, { + activeAuthorityIdentity: activeAuthorityRef.current, + now: Date.now(), + documentVisible: document.visibilityState === "visible", + networkOnline: navigator.onLine !== false, + }); + recovery = decision.state; + if (!decision.reopen) return; + leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id); + recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity; + setStatus("connecting"); + setMessage("Восстановление камеры после разрыва браузерного канала"); + // The transport effect cleans up the old WebSocket and MSE object before + // opening their replacement. No plugin action or device command occurs. + setAttempt((value) => value + 1); + }; + const onVisibilityChange = () => { + dispatchRecovery({ + type: document.visibilityState === "hidden" + ? "document-hidden" + : "document-visible", + }); + }; + const onOnline = () => dispatchRecovery({ type: "network-online" }); + const onPageShow = (event: PageTransitionEvent) => { + dispatchRecovery({ type: "page-restore", persisted: event.persisted }); + }; + const heartbeat = window.setInterval(() => { + dispatchRecovery({ type: "heartbeat" }); + }, 1_000); + + document.addEventListener("visibilitychange", onVisibilityChange); + window.addEventListener("online", onOnline); + window.addEventListener("pageshow", onPageShow); + return () => { + window.clearInterval(heartbeat); + document.removeEventListener("visibilitychange", onVisibilityChange); + window.removeEventListener("online", onOnline); + window.removeEventListener("pageshow", onPageShow); + }; + }, [delivery.id, recoveryAuthorityIdentity, uiBuildStale]); return (
@@ -265,6 +575,7 @@ export function MseFmp4WebSocketPlayer({ type="button" onClick={() => { leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id); + recoveryPendingAuthorityRef.current = null; setAttempt((value) => value + 1); }} > diff --git a/apps/control-station/src/components/ObservationSources.tsx b/apps/control-station/src/components/ObservationSources.tsx index 21a3395..13ce610 100644 --- a/apps/control-station/src/components/ObservationSources.tsx +++ b/apps/control-station/src/components/ObservationSources.tsx @@ -14,6 +14,7 @@ import type { RecordedAdmissionPhase, RecordedCameraAdmissionState, } from "../core/observation/recordedSessionAdmission"; +import { liveCameraPlaybackAuthorityIdentity } from "../core/observation/liveCameraRecovery"; const sourceIcon: Record = { "point-cloud": "globe", @@ -63,7 +64,13 @@ export function ObservationMedia({ source.delivery?.kind === "mse-fmp4-websocket" && source.modality === "video" ) { - return ; + return ( + + ); } if ( diff --git a/apps/control-station/src/components/RerunViewport.tsx b/apps/control-station/src/components/RerunViewport.tsx index 0acb3bb..a53cf9f 100644 --- a/apps/control-station/src/components/RerunViewport.tsx +++ b/apps/control-station/src/components/RerunViewport.tsx @@ -2,14 +2,22 @@ import { useEffect, useRef, useState } from "react"; import type { SceneSettings } from "../sceneSettings"; import { + advanceLiveReceiverOpenWatchdog, advanceLiveReceiverWatchdog, + initialLiveReceiverOpenWatchdogState, initialLiveReceiverRecoveryState, initialLiveReceiverWatchdogState, LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS, + LIVE_RECEIVER_OPEN_MAX_AGE_MS, requestLiveReceiverRecovery, } from "../core/observation/liveReceiverWatchdog"; -import { postLiveViewerDiagnostic } from "../core/observation/liveViewerDiagnostics"; -import type { LiveViewerFailureStage } from "../core/observation/liveViewerDiagnostics"; +import { + createLiveViewerDiagnosticLifecycle, + createLiveViewerInstanceId, + createLiveViewerLineage, + subscribeToLiveViewerBuildFence, + type LiveViewerFailureStage, +} from "../core/observation/liveViewerDiagnostics"; import { fetchPerceptionPreparationStatus, perceptionPreparationMessage, @@ -81,6 +89,7 @@ export interface RerunViewportProps { followLive?: boolean; liveActivitySequence?: number | null; liveStreamId?: string | null; + liveRecoveryAuthorityIdentity?: string | null; autoplayWhenReady?: boolean; presentationGate?: RecordedAdmissionPhase; expectedTimelineStartSeconds?: number; @@ -197,6 +206,51 @@ export function createRecordedOpenWatchdog({ }; } +export function createReentrantViewerDisposer( + cleanupOnce: () => void, + releaseNativeViewer: () => void, +): () => void { + let cleanupComplete = false; + return () => { + try { + if (!cleanupComplete) { + cleanupComplete = true; + cleanupOnce(); + } + } finally { + // `viewer.start()` can resolve after an earlier pre-ready stop. Reapply + // native release on every disposal boundary so that a stale viewer can + // never reopen after React and diagnostics have already unmounted it. + releaseNativeViewer(); + } + }; +} + +interface ActiveLiveViewerOwner { + release: () => void; +} + +let activeLiveViewerOwner: ActiveLiveViewerOwner | null = null; + +/** + * Own exactly one native live receiver per application document. + * + * React route/StrictMode transitions can overlap two mounted workspaces for a + * render turn. Rerun keeps each native gRPC receiver alive independently, so + * the overlap used to consume the bounded live replay slots and leave the + * operator's visible canvas black. Claiming the next owner synchronously + * retires the previous native receiver before the next one starts. + */ +export function claimExclusiveLiveViewer(release: () => void): () => void { + const owner = { release }; + const previous = activeLiveViewerOwner; + activeLiveViewerOwner = owner; + previous?.release(); + return () => { + if (activeLiveViewerOwner === owner) activeLiveViewerOwner = null; + }; +} + export interface RecordedPlaybackBufferState { bufferedEndNs: number | null; expectedStartNs: number | null; @@ -813,6 +867,7 @@ export function RerunViewport({ followLive = false, liveActivitySequence = null, liveStreamId = null, + liveRecoveryAuthorityIdentity = null, autoplayWhenReady = false, presentationGate = "ready", expectedTimelineStartSeconds, @@ -845,7 +900,14 @@ export function RerunViewport({ liveActivitySequenceRef.current = liveActivitySequence; const liveStreamIdRef = useRef(liveStreamId); liveStreamIdRef.current = liveStreamId; + const liveRecoveryAuthorityRef = useRef(liveRecoveryAuthorityIdentity); + liveRecoveryAuthorityRef.current = liveRecoveryAuthorityIdentity; const liveRecoveryRef = useRef(initialLiveReceiverRecoveryState()); + const liveViewerInstanceIdRef = useRef(null); + liveViewerInstanceIdRef.current ??= createLiveViewerInstanceId(); + const liveViewerLifecycleGenerationRef = useRef(0); + const activeViewerLifecycleRef = useRef<(() => void) | null>(null); + const uiBuildStaleRef = useRef(false); const blueprintChannelRef = useRef(null); const perceptionChannelRef = useRef(null); const loadedPerceptionChannelRef = useRef(null); @@ -871,14 +933,22 @@ export function RerunViewport({ recordedArtifact !== null, ); + useEffect(() => subscribeToLiveViewerBuildFence(() => { + uiBuildStaleRef.current = true; + // A stale document may only release its own browser transports. Device + // START/STOP remains owned by the acquisition authority while the fresh + // application document is loaded. + activeViewerLifecycleRef.current?.(); + }), []); + useEffect(() => { liveRecoveryRef.current = initialLiveReceiverRecoveryState(); - }, [followLive, liveStreamId, sourceUrl]); + }, [followLive, liveRecoveryAuthorityIdentity, liveStreamId, sourceUrl]); useEffect(() => { const normalizedSource = sourceUrl.trim(); const host = hostRef.current; - if (!normalizedSource || !host) { + if (!normalizedSource || !host || uiBuildStaleRef.current) { setStatus("idle"); setRecordingBufferProgress(null); onStatusChange?.("idle"); @@ -887,6 +957,7 @@ export function RerunViewport({ onPlaybackControllerChange?.(null); return; } + const isRecordedSource = RECORDED_RRD_PATH.test(normalizedSource); let resolvedSource: string; try { @@ -912,18 +983,29 @@ export function RerunViewport({ return; } + liveViewerLifecycleGenerationRef.current += 1; + const diagnosticLifecycle = createLiveViewerDiagnosticLifecycle({ + lineage: createLiveViewerLineage( + liveViewerInstanceIdRef.current!, + liveViewerLifecycleGenerationRef.current, + ), + }); + diagnosticLifecycle.verifyBuild(); + let disposed = false; let disposeViewer: (() => void) | undefined; - let recordingOpenTimer: number | undefined; - let liveRecordingDiscoveryTimer: number | undefined; let recordedOpenWatchdog: { arm: () => void; clear: () => void; pending: () => boolean; } | null = null; let playbackRangeTimer: number | undefined; + let liveRecoveryRetryTimer: number | undefined; let recordingOpened = false; let recordingOpenTimedOut = false; + let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState( + liveActivitySequenceRef.current, + ); let viewerStartResolved = false; let latestLiveRangeMaxNs: number | null = null; let liveWatchdog = initialLiveReceiverWatchdogState(); @@ -948,22 +1030,68 @@ export function RerunViewport({ } }; const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear(); - const clearLiveRecordingOpenTimer = () => { - if (recordingOpenTimer !== undefined) { - window.clearTimeout(recordingOpenTimer); - recordingOpenTimer = undefined; - } + const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout; + function refreshOpeningLiveReceiver(openForMs: number) { + if (disposed || recordingOpened) return; + diagnosticLifecycle.post({ + eventCode: "live_receiver_restart_requested", + failureStage: "recording-open-timeout", + streamId: liveStreamIdRef.current, + backendActivitySequence: liveActivitySequenceRef.current, + viewerRangeMaxNs: latestLiveRangeMaxNs, + stalledForMs: Math.round(openForMs), + recoveryAttempt: liveRecoveryRef.current.attempts || null, + }); + setStatus("loading"); + onStatusChange?.( + "loading", + "Живой визуализатор обновляет приёмник продолжающегося потока.", + ); + disposeViewer?.(); + setRetryNonce((nonce) => nonce + 1); + } + const armLiveRecordingOpenTimer = () => { + clearLiveRecordingOpenTimer(); + diagnosticLifecycle.armAdmissionTimeout(() => { + if (disposed || recordingOpened) return; + const observed = advanceLiveReceiverOpenWatchdog( + liveOpenWatchdog, + liveRecoveryRef.current, + liveActivitySequenceRef.current, + Date.now(), + ); + liveOpenWatchdog = observed.state; + liveRecoveryRef.current = observed.recoveryState; + if (observed.signal === "wait-for-store") { + // The backend is still publishing this exact acquisition. Preserve + // the receiver and its partially replayed store instead of throwing + // away startup work on every fixed timeout. + armLiveRecordingOpenTimer(); + return; + } + if (observed.signal === "refresh-receiver") { + // Publication is healthy, so this is presentation-only maintenance: + // refresh the aged native receiver without spending (or clearing) + // recovery debt. + recordingOpenTimedOut = true; + refreshOpeningLiveReceiver(observed.openForMs); + return; + } + recordingOpenTimedOut = true; + requestLiveRecovery("recording-open-timeout"); + }, LIVE_RECEIVER_OPEN_MAX_AGE_MS); }; - const clearLiveRecordingDiscoveryTimer = () => { - if (liveRecordingDiscoveryTimer !== undefined) { - window.clearInterval(liveRecordingDiscoveryTimer); - liveRecordingDiscoveryTimer = undefined; - } + const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval; + const clearLiveRecoveryRetryTimer = () => { + if (liveRecoveryRetryTimer === undefined) return; + window.clearTimeout(liveRecoveryRetryTimer); + liveRecoveryRetryTimer = undefined; }; const clearRecordingTimers = () => { clearRecordedAdmissionWatchdog(); clearLiveRecordingOpenTimer(); clearLiveRecordingDiscoveryTimer(); + clearLiveRecoveryRetryTimer(); }; const clearPlaybackRangeTimer = () => { if (playbackRangeTimer === undefined) return; @@ -989,7 +1117,7 @@ export function RerunViewport({ const reportError = (message: string, failureStage?: LiveViewerFailureStage) => { if (disposed) return; if (followLive) { - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_error", failureStage, streamId: liveStreamIdRef.current, @@ -1008,7 +1136,7 @@ export function RerunViewport({ ) => { if (!followLive || disposed) return false; if (emitErrorEvent) { - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_error", failureStage, streamId: liveStreamIdRef.current, @@ -1018,10 +1146,20 @@ export function RerunViewport({ recoveryAttempt: liveRecoveryRef.current.attempts || null, }); } - const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current); + const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current, { + activeAuthorityIdentity: liveRecoveryAuthorityRef.current, + expectedAuthorityIdentity: liveRecoveryAuthorityIdentity, + disposed, + }); liveRecoveryRef.current = recovery.state; + if (recovery.signal === "stale") { + // A newer runtime snapshot owns the next effect. This retired viewer + // may release itself but cannot schedule a receiver for that lineage. + disposeViewer?.(); + return true; + } if (recovery.signal === "exhausted") { - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_recovery_exhausted", failureStage, streamId: liveStreamIdRef.current, @@ -1037,7 +1175,7 @@ export function RerunViewport({ ); return true; } - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_restart_requested", failureStage, streamId: liveStreamIdRef.current, @@ -1052,7 +1190,15 @@ export function RerunViewport({ "Живой визуализатор переподключается к продолжающемуся потоку.", ); disposeViewer?.(); - setRetryNonce((nonce) => nonce + 1); + clearLiveRecoveryRetryTimer(); + liveRecoveryRetryTimer = window.setTimeout(() => { + liveRecoveryRetryTimer = undefined; + if ( + disposed + || liveRecoveryAuthorityRef.current !== liveRecoveryAuthorityIdentity + ) return; + setRetryNonce((nonce) => nonce + 1); + }, recovery.delayMs ?? 0); return true; }; const observeLiveReceiver = (viewerRangeMaxNs: number | null) => { @@ -1072,7 +1218,7 @@ export function RerunViewport({ (receiverOpenedAfterRecovery || observed.signal === "receiver-advanced") && liveRecoveryRef.current.awaitingRecovery ) { - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_recovered", streamId: liveStreamIdRef.current, backendActivitySequence: liveActivitySequenceRef.current, @@ -1084,7 +1230,7 @@ export function RerunViewport({ } if (observed.signal !== "stalled") return; - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_stalled", failureStage: "receiver-stalled", streamId: liveStreamIdRef.current, @@ -1093,12 +1239,30 @@ export function RerunViewport({ stalledForMs: Math.round(observed.stalledForMs), recoveryAttempt: Math.min( liveRecoveryRef.current.attempts + 1, - LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS, + liveRecoveryAuthorityIdentity + ? Number.MAX_SAFE_INTEGER + : LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS, ), }); requestLiveRecovery("receiver-stalled", observed.stalledForMs, false); }; + let relinquishLiveViewerOwnership: () => void = () => {}; + const disposeActiveViewerLifecycle = () => { + if (disposed) return; + disposed = true; + relinquishLiveViewerOwnership(); + diagnosticLifecycle.dispose(); + disposeViewer?.(); + host.replaceChildren(); + }; + if (followLive) { + relinquishLiveViewerOwnership = claimExclusiveLiveViewer( + disposeActiveViewerLifecycle, + ); + } + activeViewerLifecycleRef.current = disposeActiveViewerLifecycle; + host.replaceChildren(); appliedPointColorKeyRef.current = null; setStatus("loading"); @@ -1113,10 +1277,7 @@ export function RerunViewport({ if (disposed) return; const viewer = new WebViewer(); - let viewerDisposed = false; - disposeViewer = () => { - if (viewerDisposed) return; - viewerDisposed = true; + disposeViewer = createReentrantViewerDisposer(() => { clearRecordingTimers(); clearPlaybackRangeTimer(); playbackTimeUpdates.cancel(); @@ -1150,6 +1311,7 @@ export function RerunViewport({ } catch { // The viewer may already have closed all auxiliary channels. } + }, () => { try { if (viewer.ready) viewer.close(resolvedSource); } catch { @@ -1162,7 +1324,7 @@ export function RerunViewport({ // startup failure. } host.replaceChildren(); - }; + }); if (isRecordedSource && recordedArtifact) { recordedOpenWatchdog = createRecordedOpenWatchdog({ byteLength: recordedArtifact.byteLength, @@ -1188,8 +1350,17 @@ export function RerunViewport({ ) return; recordingOpened = true; if (!isRecordedSource) { - clearLiveRecordingOpenTimer(); - clearLiveRecordingDiscoveryTimer(); + diagnosticLifecycle.markAdmitted(); + if (liveRecoveryRef.current.awaitingRecovery) { + diagnosticLifecycle.post({ + eventCode: "live_receiver_recovered", + streamId: liveStreamIdRef.current, + backendActivitySequence: liveActivitySequenceRef.current, + viewerRangeMaxNs: latestLiveRangeMaxNs, + recoveryAttempt: liveRecoveryRef.current.attempts, + }); + } + liveRecoveryRef.current = initialLiveReceiverRecoveryState(); } if ( recordedBlueprintUrl && @@ -1386,7 +1557,7 @@ export function RerunViewport({ // Rerun 0.34.1 may ingest an SDK gRPC store without forwarding its // recording_open event to the JavaScript wrapper. The active store // is the authoritative fallback and avoids hiding a ready canvas. - postLiveViewerDiagnostic({ + diagnosticLifecycle.post({ eventCode: "live_receiver_active_store_admitted", streamId: liveStreamIdRef.current, backendActivitySequence: liveActivitySequenceRef.current, @@ -1489,16 +1660,19 @@ export function RerunViewport({ } if (!isRecordedSource) { + // Measure native store admission from the resolved viewer start, + // not from dynamic module import or React effect setup time. + liveOpenWatchdog = initialLiveReceiverOpenWatchdogState( + liveActivitySequenceRef.current, + Date.now(), + ); discoverActiveLiveRecording(); if (!recordingOpened) { - liveRecordingDiscoveryTimer = window.setInterval( + diagnosticLifecycle.armAdmissionInterval( discoverActiveLiveRecording, 100, ); - recordingOpenTimer = window.setTimeout(() => { - recordingOpenTimedOut = true; - requestLiveRecovery("recording-open-timeout"); - }, 12_000); + armLiveRecordingOpenTimer(); } } } catch { @@ -1520,12 +1694,14 @@ export function RerunViewport({ }); return () => { - disposed = true; + if (activeViewerLifecycleRef.current === disposeActiveViewerLifecycle) { + activeViewerLifecycleRef.current = null; + } + disposeActiveViewerLifecycle(); clearRecordingTimers(); clearPlaybackRangeTimer(); playbackTimeUpdates.cancel(); unsubscribeAll(); - disposeViewer?.(); onSelectionChange?.(null); onPlaybackControllerChange?.(null); onPlaybackChange?.(null); @@ -1537,6 +1713,7 @@ export function RerunViewport({ followLive, initialPlaybackStartSeconds, liveStreamId, + liveRecoveryAuthorityIdentity, onPlaybackChange, onPlaybackControllerChange, onSelectionChange, diff --git a/apps/control-station/src/components/useApplicationPanelActions.ts b/apps/control-station/src/components/useApplicationPanelActions.ts index 6465e70..bdd127d 100644 --- a/apps/control-station/src/components/useApplicationPanelActions.ts +++ b/apps/control-station/src/components/useApplicationPanelActions.ts @@ -6,14 +6,46 @@ import type { WorkspaceDefinition } from "../productModel"; interface ApplicationPanelActionsOptions { definition: WorkspaceDefinition | null; refreshRuntime: () => void; + resetConnectionScenario?: () => Promise; + connectionScenarioResetting: boolean; saveWorkspaceLayout: () => Promise; workspaceLayoutSaving: boolean; systemUtilityActions: readonly ApplicationPanelUtilityAction[]; } +export function deviceRuntimeUtilityAction({ + refreshRuntime, + resetConnectionScenario, + connectionScenarioResetting, +}: Pick< + ApplicationPanelActionsOptions, + "refreshRuntime" | "resetConnectionScenario" | "connectionScenarioResetting" +>): ApplicationPanelUtilityAction { + return { + label: resetConnectionScenario + ? connectionScenarioResetting + ? "Сбрасываем подключение" + : "Сбросить подключение" + : "Обновить состояние локального контура", + icon: resetConnectionScenario && connectionScenarioResetting + ? "activity" + : "refresh", + disabled: resetConnectionScenario && connectionScenarioResetting + ? true + : undefined, + onClick: resetConnectionScenario + ? connectionScenarioResetting + ? () => undefined + : () => void resetConnectionScenario() + : refreshRuntime, + }; +} + export function useApplicationPanelActions({ definition, refreshRuntime, + resetConnectionScenario, + connectionScenarioResetting, saveWorkspaceLayout, workspaceLayoutSaving, systemUtilityActions, @@ -21,11 +53,11 @@ export function useApplicationPanelActions({ return useMemo(() => { const actions: ApplicationPanelUtilityAction[] = []; if (definition?.kind === "device") { - actions.push({ - label: "Обновить состояние локального контура", - icon: "refresh", - onClick: refreshRuntime, - }); + actions.push(deviceRuntimeUtilityAction({ + refreshRuntime, + resetConnectionScenario, + connectionScenarioResetting, + })); } if (definition && ["spatial", "recordings"].includes(definition.kind)) { actions.push({ @@ -40,6 +72,8 @@ export function useApplicationPanelActions({ }, [ definition, refreshRuntime, + resetConnectionScenario, + connectionScenarioResetting, saveWorkspaceLayout, systemUtilityActions, workspaceLayoutSaving, diff --git a/apps/control-station/src/core/device-plugins/DevicePluginHost.tsx b/apps/control-station/src/core/device-plugins/DevicePluginHost.tsx index f7dde3a..1a843aa 100644 --- a/apps/control-station/src/core/device-plugins/DevicePluginHost.tsx +++ b/apps/control-station/src/core/device-plugins/DevicePluginHost.tsx @@ -23,6 +23,64 @@ interface DevicePluginHostValue { const DevicePluginHostContext = createContext(null); +export const DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY = + "nodedc.mission-core.device-model-selection.v1"; + +export interface DeviceModelSelectionStorage { + getItem: (key: string) => string | null; + setItem: (key: string, value: string) => void; + removeItem: (key: string) => void; +} + +function browserDeviceModelSelectionStorage(): DeviceModelSelectionStorage | null { + if (typeof window === "undefined") return null; + try { + return window.localStorage; + } catch { + return null; + } +} + +/** + * Restore only a model that is present in the current reviewed registry. + * Removed/renamed models and malformed browser values fail closed to the + * picker and are cleared so a later remount cannot keep retrying stale state. + */ +export function restorePersistedDeviceModelId( + registry: DevicePluginRegistry, + storage: DeviceModelSelectionStorage | null, +): string | null { + if (!storage) return null; + try { + const stored = storage.getItem(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY); + const modelId = stored?.trim() || null; + if (modelId && registry.resolveModel(modelId)) return modelId; + if (stored !== null) { + storage.removeItem(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY); + } + return null; + } catch { + return null; + } +} + +/** Persist only an already-admitted host transition; browser storage is never authority. */ +export function commitPersistedDeviceModelId( + modelId: string | null, + storage: DeviceModelSelectionStorage | null, +): void { + if (!storage) return; + try { + if (modelId === null) { + storage.removeItem(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY); + return; + } + storage.setItem(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY, modelId); + } catch { + // A denied/full localStorage must not block the in-memory host transition. + } +} + export function DevicePluginHostProvider({ plugins, children, @@ -31,7 +89,10 @@ export function DevicePluginHostProvider({ children: ReactNode; }) { const registry = useMemo(() => createDevicePluginRegistry(plugins), [plugins]); - const [selectedModelId, setSelectedModelId] = useState(null); + const selectionStorage = useMemo(browserDeviceModelSelectionStorage, []); + const [selectedModelId, setSelectedModelId] = useState(() => + restorePersistedDeviceModelId(registry, selectionStorage) + ); const [selectionTransitionPending, setSelectionTransitionPending] = useState(false); const [selectionTransitionError, setSelectionTransitionError] = useState(null); const transitionInFlight = useRef(false); @@ -56,7 +117,12 @@ export function DevicePluginHostProvider({ if (nextModelId !== null && !registry.resolveModel(nextModelId)) { throw new Error(`Модель устройства не зарегистрирована: ${nextModelId}.`); } - if (nextModelId === selectedModelId) return true; + if (nextModelId === selectedModelId) { + // `clearSelection()` must clear a stale persisted value even when the + // current in-memory selection is already empty. + commitPersistedDeviceModelId(nextModelId, selectionStorage); + return true; + } transitionInFlight.current = true; setSelectionTransitionPending(true); @@ -88,13 +154,14 @@ export function DevicePluginHostProvider({ } } setSelectedModelId(nextModelId); + commitPersistedDeviceModelId(nextModelId, selectionStorage); return true; } finally { transitionInFlight.current = false; setSelectionTransitionPending(false); } }, - [registry, selectedModelId], + [registry, selectedModelId, selectionStorage], ); const deactivationRegistrars = useMemo( diff --git a/apps/control-station/src/core/observation/liveCameraRecovery.ts b/apps/control-station/src/core/observation/liveCameraRecovery.ts new file mode 100644 index 0000000..7f3988a --- /dev/null +++ b/apps/control-station/src/core/observation/liveCameraRecovery.ts @@ -0,0 +1,204 @@ +import type { + ObservationSourceDelivery, + ObservationSourceDescriptor, +} from "../runtime/contracts"; + +export const CAMERA_WAKE_GAP_MS = 5_000; +export const CAMERA_HIDDEN_REOPEN_MS = 1_000; +export const CAMERA_REOPEN_COOLDOWN_MS = 1_000; + +export type CameraPlaybackRecoveryEvent = + | { type: "document-hidden" } + | { type: "document-visible" } + | { type: "network-online" } + | { type: "page-restore"; persisted: boolean } + | { type: "heartbeat" }; + +export interface CameraPlaybackRecoveryState { + authorityIdentity: string; + hiddenAt: number | null; + lastObservedAt: number; + lastReopenAt: number | null; +} + +export interface CameraPlaybackRecoveryContext { + activeAuthorityIdentity: string | null; + now: number; + documentVisible: boolean; + networkOnline: boolean; +} + +export interface CameraPlaybackRecoveryDecision { + state: CameraPlaybackRecoveryState; + reopen: boolean; +} + +export function cameraTransportCallbackIsCurrent( + activeEpoch: number, + callbackEpoch: number, + disposed: boolean, +): boolean { + return !disposed && activeEpoch === callbackEpoch; +} + +export function cameraBrowserTransportIdentity( + delivery: ObservationSourceDelivery & { kind: "mse-fmp4-websocket" }, + authorityIdentity: string | null, +): string { + return JSON.stringify([ + delivery.id, + delivery.url, + delivery.mediaType, + authorityIdentity, + ]); +} + +function trimmedString(value: unknown): string { + return typeof value === "string" ? value.trim() : ""; +} + +function positiveInteger(value: unknown): value is number { + return Number.isInteger(value) && (value as number) > 0; +} + +/** + * Bind browser-only recovery to one exact, server-authoritative live camera. + * A point-cloud recovery, retained delivery or selected camera without the + * active acquisition/session tuple must not reopen a decoder. + */ +export function liveCameraPlaybackAuthorityIdentity( + source: ObservationSourceDescriptor, +): string | null { + const delivery = source.delivery; + const activation = source.activation; + const deviceId = trimmedString(source.binding?.deviceId); + const deviceSessionId = trimmedString(source.binding?.deviceSessionId); + const acquisitionId = trimmedString(source.binding?.acquisitionId); + const sourceId = trimmedString(source.sourceId); + const descriptorId = trimmedString(source.id); + const activationGroupId = trimmedString(activation?.groupId); + const deliveryId = trimmedString(delivery?.id); + const deliveryUrl = trimmedString(delivery?.url); + const presentationLease = source.presentationLease; + const recoveryPresentation = presentationLease?.kind === "active-stream-recovery"; + const recoveryLeaseValid = Boolean( + recoveryPresentation + && trimmedString(presentationLease.runtimeId) + && trimmedString(presentationLease.acquisitionId) === acquisitionId + && positiveInteger(presentationLease.acquisitionStateRevision) + && positiveInteger(presentationLease.producerGeneration) + && positiveInteger(presentationLease.recoveryGeneration), + ); + const availabilityAuthoritative = source.availability === "streaming" + || ( + recoveryLeaseValid + && (source.availability === "connecting" || source.availability === "degraded") + ); + const mediaType = delivery?.kind === "mse-fmp4-websocket" + ? trimmedString(delivery.mediaType) + : ""; + if ( + source.modality !== "video" + || !availabilityAuthoritative + || (presentationLease != null && !recoveryLeaseValid) + || delivery?.kind !== "mse-fmp4-websocket" + || activation?.selected !== true + || activation.maxActive !== 1 + || !activationGroupId + || !descriptorId + || !sourceId + || !deliveryId + || !deliveryUrl + || !/^video\/mp4(?:\s*;|$)/i.test(mediaType) + || !deviceId + || !deviceSessionId + || !acquisitionId + ) { + return null; + } + return JSON.stringify([ + descriptorId, + sourceId, + deviceId, + deviceSessionId, + acquisitionId, + activationGroupId, + activation.maxActive, + deliveryId, + deliveryUrl, + mediaType, + recoveryLeaseValid ? [ + presentationLease?.runtimeId, + presentationLease?.acquisitionStateRevision, + presentationLease?.producerGeneration, + presentationLease?.recoveryGeneration, + ] : null, + ]); +} + +export function initialCameraPlaybackRecoveryState( + authorityIdentity: string, + now: number, +): CameraPlaybackRecoveryState { + return { + authorityIdentity, + hiddenAt: null, + lastObservedAt: now, + lastReopenAt: null, + }; +} + +/** + * Reduce browser lifecycle signals without performing I/O. `reopen=true` + * means replace the current browser WebSocket + MSE pair; it never means a + * device START/STOP, camera selection or network mutation. + */ +export function reduceCameraPlaybackRecovery( + current: CameraPlaybackRecoveryState, + event: CameraPlaybackRecoveryEvent, + context: CameraPlaybackRecoveryContext, +): CameraPlaybackRecoveryDecision { + const { now } = context; + if (event.type === "document-hidden") { + return { + state: { + ...current, + hiddenAt: now, + lastObservedAt: now, + }, + reopen: false, + }; + } + + const authorityCurrent = Boolean( + context.activeAuthorityIdentity + && context.activeAuthorityIdentity === current.authorityIdentity, + ); + const documentReady = context.documentVisible && context.networkOnline; + let candidate = false; + let hiddenAt = current.hiddenAt; + + if (event.type === "document-visible") { + candidate = hiddenAt !== null && now - hiddenAt >= CAMERA_HIDDEN_REOPEN_MS; + hiddenAt = null; + } else if (event.type === "network-online") { + candidate = true; + } else if (event.type === "page-restore") { + candidate = event.persisted; + } else if (event.type === "heartbeat") { + candidate = now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS; + } + + const outsideCooldown = current.lastReopenAt === null + || now - current.lastReopenAt >= CAMERA_REOPEN_COOLDOWN_MS; + const reopen = candidate && authorityCurrent && documentReady && outsideCooldown; + return { + state: { + ...current, + hiddenAt, + lastObservedAt: now, + lastReopenAt: reopen ? now : current.lastReopenAt, + }, + reopen, + }; +} diff --git a/apps/control-station/src/core/observation/liveReceiverWatchdog.ts b/apps/control-station/src/core/observation/liveReceiverWatchdog.ts index dc4c9da..9cb045e 100644 --- a/apps/control-station/src/core/observation/liveReceiverWatchdog.ts +++ b/apps/control-station/src/core/observation/liveReceiverWatchdog.ts @@ -1,3 +1,8 @@ +import type { + ObservationSourceDescriptor, + SpatialSourceDescriptor, +} from "../runtime/contracts"; + export interface LiveReceiverWatchdogState { lastBackendActivitySequence: number | null; lastViewerRangeMaxNs: number | null; @@ -19,16 +24,163 @@ export interface LiveReceiverRecoveryState { awaitingRecovery: boolean; } -export type LiveReceiverRecoverySignal = "retry" | "exhausted"; +export interface LiveReceiverOpenWatchdogState { + lastBackendActivitySequence: number | null; + openedAtMs: number; +} + +export type LiveReceiverOpenWatchdogSignal = + | "wait-for-store" + | "refresh-receiver" + | "restart-receiver"; + +export interface LiveReceiverOpenWatchdogResult { + state: LiveReceiverOpenWatchdogState; + recoveryState: LiveReceiverRecoveryState; + signal: LiveReceiverOpenWatchdogSignal; + openForMs: number; +} + +export type LiveReceiverRecoverySignal = "retry" | "exhausted" | "stale"; export interface LiveReceiverRecoveryResult { state: LiveReceiverRecoveryState; signal: LiveReceiverRecoverySignal; attempt: number; + delayMs: number | null; +} + +export interface LiveReceiverRecoveryRequest { + activeAuthorityIdentity: string | null; + expectedAuthorityIdentity: string | null; + disposed?: boolean; + maxAttempts?: number; } export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000; export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3; +// The bridge publishes its URL only after StoreInfo, blueprint and static +// scene data have been flushed. A receiver that still has not admitted that +// store after one operator-visible four-second window is wedged, not merely +// slow; keeping it for 48 seconds made a healthy live scan look blank. +export const LIVE_RECEIVER_OPEN_MAX_AGE_MS = 4_000; +const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const; + +export function liveReceiverRecoveryRetryDelay(attempt: number): number { + const normalizedAttempt = Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1; + return LIVE_RECEIVER_RECOVERY_DELAYS_MS[ + Math.min(normalizedAttempt - 1, LIVE_RECEIVER_RECOVERY_DELAYS_MS.length - 1) + ]; +} + +function trimmedString(value: unknown): string { + return typeof value === "string" ? value.trim() : ""; +} + +function positiveInteger(value: unknown): value is number { + return Number.isInteger(value) && (value as number) > 0; +} + +/** + * Bind durable browser-only Rerun recovery to one exact authoritative spatial + * presentation. A healthy live descriptor is fenced by its acquisition and + * session tuple; a reconnecting/degraded descriptor additionally requires the + * server-issued active-stream-recovery generation lease. + */ +export function liveRerunRecoveryAuthorityIdentity( + source: ObservationSourceDescriptor | null | undefined, + spatialSource: SpatialSourceDescriptor | null | undefined, +): string | null { + if (!source || !spatialSource) return null; + const descriptorId = trimmedString(source.id); + const sourceId = trimmedString(source.sourceId); + const semanticChannelId = trimmedString(source.semanticChannelId); + const previewUrl = trimmedString(source.previewUrl); + const spatialId = trimmedString(spatialSource.id); + const spatialUrl = trimmedString(spatialSource.url); + const deviceId = trimmedString(source.binding?.deviceId); + const deviceSessionId = trimmedString(source.binding?.deviceSessionId); + const acquisitionId = trimmedString(source.binding?.acquisitionId); + const pluginId = trimmedString(source.provider?.pluginId); + const pluginVersion = trimmedString(source.provider?.pluginVersion); + const modelId = trimmedString(source.provider?.modelId); + const compatibilityProfileId = trimmedString(source.provider?.compatibilityProfileId); + const clockId = trimmedString(source.capabilities?.clockId); + const presentationLease = source.presentationLease; + const recoveryPresentation = presentationLease?.kind === "active-stream-recovery"; + const recoveryLeaseValid = Boolean( + recoveryPresentation + && trimmedString(presentationLease.runtimeId) + && trimmedString(presentationLease.acquisitionId) === acquisitionId + && positiveInteger(presentationLease.acquisitionStateRevision) + && positiveInteger(presentationLease.producerGeneration) + && positiveInteger(presentationLease.recoveryGeneration), + ); + const availabilityAuthoritative = source.availability === "streaming" + || ( + recoveryLeaseValid + && (source.availability === "connecting" || source.availability === "degraded") + ); + if ( + source.modality !== "point-cloud" + || source.transport !== "rerun-grpc" + || spatialSource.kind !== "rerun-grpc" + || source.capabilities.timelineMode !== "live-only" + || source.capabilities.spatialRegistration !== "native" + || source.delivery != null + || !availabilityAuthoritative + || (presentationLease != null && !recoveryLeaseValid) + || !descriptorId + || !sourceId + || !semanticChannelId + || !previewUrl + || previewUrl !== spatialUrl + || !spatialId + || spatialId !== acquisitionId + || !deviceId + || !deviceSessionId + || !acquisitionId + || clockId !== acquisitionId + || !pluginId + || !pluginVersion + || !modelId + || !compatibilityProfileId + ) { + return null; + } + return JSON.stringify([ + descriptorId, + sourceId, + semanticChannelId, + deviceId, + deviceSessionId, + acquisitionId, + spatialId, + spatialUrl, + pluginId, + pluginVersion, + modelId, + compatibilityProfileId, + recoveryLeaseValid ? [ + presentationLease?.runtimeId, + presentationLease?.acquisitionStateRevision, + presentationLease?.producerGeneration, + presentationLease?.recoveryGeneration, + ] : null, + ]); +} + +export function liveReceiverRecoveryAuthorityIsCurrent( + activeAuthorityIdentity: string | null, + expectedAuthorityIdentity: string | null, + disposed = false, +): boolean { + return Boolean( + !disposed + && expectedAuthorityIdentity + && activeAuthorityIdentity === expectedAuthorityIdentity, + ); +} export function initialLiveReceiverWatchdogState(): LiveReceiverWatchdogState { return { @@ -47,16 +199,104 @@ export function initialLiveReceiverRecoveryState(): LiveReceiverRecoveryState { }; } +function validBackendActivitySequence(value: number | null | undefined): number | null { + return Number.isSafeInteger(value) && (value ?? -1) >= 0 ? (value as number) : null; +} + +export function initialLiveReceiverOpenWatchdogState( + backendActivitySequence: number | null = null, + openedAtMs = Date.now(), +): LiveReceiverOpenWatchdogState { + return { + lastBackendActivitySequence: validBackendActivitySequence(backendActivitySequence), + openedAtMs: Number.isFinite(openedAtMs) ? openedAtMs : Date.now(), + }; +} + +/** + * Keep one still-opening Rerun receiver alive while the backend is proving + * fresh publication progress. Recreating the WASM receiver on a fixed timer + * can repeatedly discard an otherwise healthy late StoreInfo replay. Rolling + * patience is nevertheless bounded: a receiver that has not admitted a store + * by the absolute open-age limit is refreshed without consuming the recovery + * budget. A true lack of backend progress delegates to the bounded restart + * policy. Recovery debt is cleared only after viewer admission, never merely + * because the backend counter advanced. + */ +export function advanceLiveReceiverOpenWatchdog( + current: LiveReceiverOpenWatchdogState, + recoveryState: LiveReceiverRecoveryState, + backendActivitySequence: number | null, + nowMs = Date.now(), + maxOpenAgeMs = LIVE_RECEIVER_OPEN_MAX_AGE_MS, +): LiveReceiverOpenWatchdogResult { + const sequence = validBackendActivitySequence(backendActivitySequence); + const previous = current.lastBackendActivitySequence; + const backendAdvanced = sequence !== null && ( + (previous === null && sequence > 0) || + (previous !== null && sequence > previous) + ); + const state = { + lastBackendActivitySequence: sequence === null + ? previous + : Math.max(sequence, previous ?? 0), + openedAtMs: current.openedAtMs, + }; + const openForMs = Math.max(0, nowMs - current.openedAtMs); + if (backendAdvanced) { + return { + state, + recoveryState, + signal: openForMs >= maxOpenAgeMs + ? "refresh-receiver" + : "wait-for-store", + openForMs, + }; + } + return { + state, + recoveryState, + signal: "restart-receiver", + openForMs, + }; +} + /** * Bound viewer-only restarts independently from scanner and acquisition - * lifecycle. The caller may dispose and recreate the browser receiver, but - * must never issue START/STOP or reconnect the physical device. + * lifecycle unless the caller proves that one exact live/recovery authority + * is still current. Under that fence retries remain durable and use a capped + * delay while the attempt counter stays truthful. The caller may dispose and + * recreate only the browser receiver; it must never issue START/STOP or + * reconnect the physical device. */ export function requestLiveReceiverRecovery( current: LiveReceiverRecoveryState, - maxAttempts = LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS, + request?: LiveReceiverRecoveryRequest, ): LiveReceiverRecoveryResult { - if (current.attempts >= maxAttempts) { + const maxAttempts = positiveInteger(request?.maxAttempts) + ? request.maxAttempts + : LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS; + const expectedAuthorityIdentity = request?.expectedAuthorityIdentity ?? null; + const exactAuthorityCurrent = liveReceiverRecoveryAuthorityIsCurrent( + request?.activeAuthorityIdentity ?? null, + expectedAuthorityIdentity, + request?.disposed === true, + ); + if ( + request + && ( + request.disposed === true + || request.activeAuthorityIdentity !== expectedAuthorityIdentity + ) + ) { + return { + state: current, + signal: "stale", + attempt: current.attempts, + delayMs: null, + }; + } + if (current.attempts >= maxAttempts && !exactAuthorityCurrent) { return { state: { attempts: current.attempts, @@ -64,6 +304,7 @@ export function requestLiveReceiverRecovery( }, signal: "exhausted", attempt: current.attempts, + delayMs: null, }; } const attempt = current.attempts + 1; @@ -74,6 +315,7 @@ export function requestLiveReceiverRecovery( }, signal: "retry", attempt, + delayMs: exactAuthorityCurrent ? liveReceiverRecoveryRetryDelay(attempt) : 0, }; } @@ -91,10 +333,7 @@ export function advanceLiveReceiverWatchdog( }, thresholdMs = LIVE_RECEIVER_STALL_THRESHOLD_MS, ): LiveReceiverWatchdogResult { - const backendSequence = Number.isSafeInteger(sample.backendActivitySequence) && - (sample.backendActivitySequence ?? -1) >= 0 - ? sample.backendActivitySequence - : null; + const backendSequence = validBackendActivitySequence(sample.backendActivitySequence); const viewerRange = Number.isFinite(sample.viewerRangeMaxNs) && (sample.viewerRangeMaxNs ?? -1) >= 0 ? sample.viewerRangeMaxNs diff --git a/apps/control-station/src/core/observation/liveViewerDiagnostics.ts b/apps/control-station/src/core/observation/liveViewerDiagnostics.ts index 0424bb9..df50b7a 100644 --- a/apps/control-station/src/core/observation/liveViewerDiagnostics.ts +++ b/apps/control-station/src/core/observation/liveViewerDiagnostics.ts @@ -22,7 +22,66 @@ export interface LiveViewerDiagnostic { recoveryAttempt?: number | null; } +export interface LiveViewerLineage { + uiBuildId: string; + documentInstanceId: string; + viewerInstanceId: string; + lifecycleGeneration: number; +} + +export interface LiveViewerDiagnosticScheduler { + setTimeout(callback: () => void, delayMilliseconds: number): number; + clearTimeout(handle: number): void; + setInterval(callback: () => void, delayMilliseconds: number): number; + clearInterval(handle: number): void; +} + +export interface LiveViewerDiagnosticLifecycle { + readonly lineage: LiveViewerLineage; + readonly signal: AbortSignal; + active(): boolean; + admitted(): boolean; + post(event: LiveViewerDiagnostic): void; + verifyBuild(): void; + armAdmissionTimeout(callback: () => void, delayMilliseconds: number): void; + armAdmissionInterval(callback: () => void, delayMilliseconds: number): void; + clearAdmissionTimeout(): void; + clearAdmissionInterval(): void; + markAdmitted(): void; + dispose(): void; +} + +export interface StaleUiBuild { + loadedUiBuildId: string; + expectedUiBuildId: string; +} + +export interface UiBuildStaleCoordinator { + subscribe(listener: (event: StaleUiBuild) => void): () => void; + report(event: StaleUiBuild): void; + stale(): boolean; +} + +const LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2"; +const LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA = "missioncore.live-viewer-client-contract/v1"; +const UI_BUILD_HEADER = "x-missioncore-ui-build"; +const DEVELOPMENT_UI_BUILD_ID = "development"; const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; +const HASHED_UI_BUILD_ID = /^\/assets\/[A-Za-z0-9._/-]+-[A-Za-z0-9_-]{8,}\.js$/; +const UI_BUILD_CHECK_INTERVAL_MILLISECONDS = 15_000; +const UI_BUILD_RELOAD_DELAY_MILLISECONDS = 50; + +let documentInstanceId: string | null = null; +let sharedUiBuildCoordinator: UiBuildStaleCoordinator | null = null; +let buildMonitorSubscribers = 0; +let buildMonitorInterval: number | null = null; +let buildMonitorAbort: AbortController | null = null; +let buildMonitorOnlineListener: (() => void) | null = null; +let buildMonitorVisibilityListener: (() => void) | null = null; + +function randomInstanceId(): string { + return globalThis.crypto.randomUUID(); +} function safeInteger(value: number | null | undefined): number | undefined { return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 @@ -30,11 +89,75 @@ function safeInteger(value: number | null | undefined): number | undefined { : undefined; } -export function postLiveViewerDiagnostic(event: LiveViewerDiagnostic): void { +function browserScheduler(): LiveViewerDiagnosticScheduler { + return { + setTimeout: (callback, delayMilliseconds) => window.setTimeout(callback, delayMilliseconds), + clearTimeout: (handle) => window.clearTimeout(handle), + setInterval: (callback, delayMilliseconds) => window.setInterval(callback, delayMilliseconds), + clearInterval: (handle) => window.clearInterval(handle), + }; +} + +export function uiBuildIdFromModuleScripts( + scriptSources: readonly string[], + baseUrl: string, +): string { + for (const source of scriptSources) { + try { + const pathname = new URL(source, baseUrl).pathname; + if (HASHED_UI_BUILD_ID.test(pathname)) return pathname; + } catch { + // A malformed non-entry script is not the running application build. + } + } + return DEVELOPMENT_UI_BUILD_ID; +} + +export function currentUiBuildId(): string { + const scripts = Array.from( + document.querySelectorAll('script[type="module"][src]'), + (script) => script.src, + ); + return uiBuildIdFromModuleScripts(scripts, window.location.href); +} + +export function liveViewerDocumentInstanceId(): string { + documentInstanceId ??= randomInstanceId(); + return documentInstanceId; +} + +export function createLiveViewerInstanceId(): string { + return randomInstanceId(); +} + +export function createLiveViewerLineage( + viewerInstanceId: string, + lifecycleGeneration: number, + overrides: Partial> = {}, +): LiveViewerLineage { + if (!Number.isSafeInteger(lifecycleGeneration) || lifecycleGeneration < 1) { + throw new Error("Live viewer lifecycle generation must be a positive safe integer"); + } + return { + uiBuildId: overrides.uiBuildId ?? currentUiBuildId(), + documentInstanceId: overrides.documentInstanceId ?? liveViewerDocumentInstanceId(), + viewerInstanceId, + lifecycleGeneration, + }; +} + +export function liveViewerDiagnosticBody( + event: LiveViewerDiagnostic, + lineage: LiveViewerLineage, +): Record { const streamId = event.streamId?.trim(); - const body = { - schema_version: "missioncore.live-viewer-diagnostic/v1", + return { + schema_version: LIVE_VIEWER_DIAGNOSTIC_SCHEMA, event_code: event.eventCode, + ui_build_id: lineage.uiBuildId, + document_instance_id: lineage.documentInstanceId, + viewer_instance_id: lineage.viewerInstanceId, + lifecycle_generation: lineage.lifecycleGeneration, ...(event.failureStage ? { failure_stage: event.failureStage } : {}), ...(streamId && SAFE_STREAM_ID.test(streamId) ? { stream_id: streamId } : {}), ...(safeInteger(event.backendActivitySequence) === undefined @@ -50,12 +173,252 @@ export function postLiveViewerDiagnostic(event: LiveViewerDiagnostic): void { ? {} : { recovery_attempt: safeInteger(event.recoveryAttempt) }), }; +} + +export function createUiBuildStaleCoordinator({ + scheduleReload, + reload, +}: { + scheduleReload: (callback: () => void, delayMilliseconds: number) => void; + reload: () => void; +}): UiBuildStaleCoordinator { + const listeners = new Set<(event: StaleUiBuild) => void>(); + let staleEvent: StaleUiBuild | null = null; + let reloadScheduled = false; + return { + subscribe(listener) { + listeners.add(listener); + if (staleEvent) listener(staleEvent); + return () => listeners.delete(listener); + }, + report(event) { + if (event.loadedUiBuildId === event.expectedUiBuildId || staleEvent) return; + staleEvent = event; + for (const listener of [...listeners]) listener(event); + if (reloadScheduled) return; + reloadScheduled = true; + scheduleReload(reload, UI_BUILD_RELOAD_DELAY_MILLISECONDS); + }, + stale() { + return staleEvent !== null; + }, + }; +} + +function browserUiBuildCoordinator(): UiBuildStaleCoordinator { + sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator({ + scheduleReload: (callback, delayMilliseconds) => { + window.setTimeout(callback, delayMilliseconds); + }, + reload: () => window.location.reload(), + }); + return sharedUiBuildCoordinator; +} + +function inspectUiBuildResponse( + response: Response, + loadedUiBuildId: string, + signal?: AbortSignal, +): void { + if (signal?.aborted || loadedUiBuildId === DEVELOPMENT_UI_BUILD_ID) return; + const expectedUiBuildId = response.headers.get(UI_BUILD_HEADER); + if ( + expectedUiBuildId && + expectedUiBuildId !== loadedUiBuildId && + (response.status === 409 || response.ok) + ) { + browserUiBuildCoordinator().report({ loadedUiBuildId, expectedUiBuildId }); + } +} + +export function postLiveViewerDiagnostic( + event: LiveViewerDiagnostic, + lineage: LiveViewerLineage, + signal?: AbortSignal, +): void { + if (signal?.aborted) return; void fetch("/api/v1/viewer/live-diagnostics", { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify(body), + body: JSON.stringify(liveViewerDiagnosticBody(event, lineage)), keepalive: true, + signal, + }).then((response) => { + inspectUiBuildResponse(response, lineage.uiBuildId, signal); }).catch(() => { - // Diagnostics must never interfere with the live receiver recovery path. + // Diagnostics and build fencing must never interfere with receiver recovery. }); } + +export function verifyLiveViewerClientBuild( + lineage: Pick, + signal?: AbortSignal, +): void { + if (signal?.aborted || lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID) return; + void fetch("/api/v1/viewer/client-contract", { + method: "GET", + headers: { Accept: "application/json" }, + cache: "no-store", + signal, + }).then(async (response) => { + if (signal?.aborted) return; + inspectUiBuildResponse(response, lineage.uiBuildId, signal); + if (!response.ok) return; + const contract = await response.json() as unknown; + if ( + signal?.aborted || + typeof contract !== "object" || + contract === null || + !("schema_version" in contract) || + contract.schema_version !== LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA || + !("ui_build_id" in contract) || + typeof contract.ui_build_id !== "string" || + contract.ui_build_id === lineage.uiBuildId + ) return; + browserUiBuildCoordinator().report({ + loadedUiBuildId: lineage.uiBuildId, + expectedUiBuildId: contract.ui_build_id, + }); + }).catch(() => { + // Offline periods are handled by the existing live-stream recovery path. + }); +} + +function startBuildMonitor(): void { + if (buildMonitorAbort || typeof window === "undefined") return; + const lineage = createLiveViewerLineage(createLiveViewerInstanceId(), 1); + const controller = new AbortController(); + buildMonitorAbort = controller; + const verify = createAbortFencedBuildVerifier( + controller.signal, + (signal) => verifyLiveViewerClientBuild(lineage, signal), + ); + buildMonitorInterval = window.setInterval( + verify, + UI_BUILD_CHECK_INTERVAL_MILLISECONDS, + ); + buildMonitorOnlineListener = verify; + buildMonitorVisibilityListener = () => { + if (document.visibilityState === "visible") verify(); + }; + window.addEventListener("online", buildMonitorOnlineListener); + document.addEventListener("visibilitychange", buildMonitorVisibilityListener); + verify(); +} + +export function createAbortFencedBuildVerifier( + signal: AbortSignal, + verifier: (signal: AbortSignal) => void, +): () => void { + return () => { + if (signal.aborted) return; + verifier(signal); + }; +} + +function stopBuildMonitor(): void { + buildMonitorAbort?.abort(); + buildMonitorAbort = null; + if (buildMonitorInterval !== null) window.clearInterval(buildMonitorInterval); + buildMonitorInterval = null; + if (buildMonitorOnlineListener) { + window.removeEventListener("online", buildMonitorOnlineListener); + } + if (buildMonitorVisibilityListener) { + document.removeEventListener("visibilitychange", buildMonitorVisibilityListener); + } + buildMonitorOnlineListener = null; + buildMonitorVisibilityListener = null; +} + +export function subscribeToLiveViewerBuildFence( + listener: (event: StaleUiBuild) => void, +): () => void { + const unsubscribe = browserUiBuildCoordinator().subscribe(listener); + buildMonitorSubscribers += 1; + if (buildMonitorSubscribers === 1) startBuildMonitor(); + return () => { + unsubscribe(); + buildMonitorSubscribers = Math.max(0, buildMonitorSubscribers - 1); + if (buildMonitorSubscribers === 0) stopBuildMonitor(); + }; +} + +export function createLiveViewerDiagnosticLifecycle({ + lineage, + scheduler = browserScheduler(), + diagnosticPoster = postLiveViewerDiagnostic, + buildVerifier = verifyLiveViewerClientBuild, +}: { + lineage: LiveViewerLineage; + scheduler?: LiveViewerDiagnosticScheduler; + diagnosticPoster?: ( + event: LiveViewerDiagnostic, + lineage: LiveViewerLineage, + signal?: AbortSignal, + ) => void; + buildVerifier?: ( + lineage: Pick, + signal?: AbortSignal, + ) => void; +}): LiveViewerDiagnosticLifecycle { + const abort = new AbortController(); + let isActive = true; + let isAdmitted = false; + let admissionTimeout: number | null = null; + let admissionInterval: number | null = null; + const clearAdmissionTimeout = () => { + if (admissionTimeout === null) return; + scheduler.clearTimeout(admissionTimeout); + admissionTimeout = null; + }; + const clearAdmissionInterval = () => { + if (admissionInterval === null) return; + scheduler.clearInterval(admissionInterval); + admissionInterval = null; + }; + return { + lineage, + signal: abort.signal, + active: () => isActive, + admitted: () => isAdmitted, + post(event) { + if (!isActive) return; + diagnosticPoster(event, lineage, abort.signal); + }, + verifyBuild() { + if (!isActive) return; + buildVerifier(lineage, abort.signal); + }, + armAdmissionTimeout(callback, delayMilliseconds) { + clearAdmissionTimeout(); + if (!isActive || isAdmitted) return; + admissionTimeout = scheduler.setTimeout(() => { + admissionTimeout = null; + if (isActive && !isAdmitted) callback(); + }, delayMilliseconds); + }, + armAdmissionInterval(callback, delayMilliseconds) { + clearAdmissionInterval(); + if (!isActive || isAdmitted) return; + admissionInterval = scheduler.setInterval(() => { + if (isActive && !isAdmitted) callback(); + }, delayMilliseconds); + }, + clearAdmissionTimeout, + clearAdmissionInterval, + markAdmitted() { + if (!isActive) return; + isAdmitted = true; + clearAdmissionTimeout(); + clearAdmissionInterval(); + }, + dispose() { + if (!isActive) return; + isActive = false; + abort.abort(); + clearAdmissionTimeout(); + clearAdmissionInterval(); + }, + }; +} diff --git a/apps/control-station/src/core/observation/useObservationLayout.ts b/apps/control-station/src/core/observation/useObservationLayout.ts index 64ea7cc..d527bfa 100644 --- a/apps/control-station/src/core/observation/useObservationLayout.ts +++ b/apps/control-station/src/core/observation/useObservationLayout.ts @@ -54,6 +54,7 @@ export interface ObservationLayoutController { setFloatingMaximized: (sourceId: string, maximized: boolean) => void; setWindowRect: (sourceId: string, rect: ObservationWindowRect) => void; setViewportSize: (size: ObservationViewportSize) => void; + activateAutomaticDefaults: () => void; snapshot: () => ObservationLayoutSnapshot | null; restore: (snapshot: ObservationLayoutSnapshot) => void; } @@ -68,6 +69,66 @@ function canOpenByDefault(source: ObservationSourceDescriptor): boolean { ); } +/** + * One exact live presentation lease. The stable source id is deliberately not + * sufficient: the same live camera and even the same browser delivery generation + * can be reused by a later acquisition. + */ +export function automaticLivePresentationIdentity( + source: ObservationSourceDescriptor, +): string | null { + const acquisitionId = source.binding.acquisitionId?.trim(); + const deliveryId = source.delivery?.id?.trim(); + if ( + !acquisitionId + || !deliveryId + || !canOpenByDefault(source) + || source.activation?.selected !== true + ) return null; + return JSON.stringify([source.id, acquisitionId, deliveryId]); +} + +/** A deliberate close fences every delivery generation in that acquisition. */ +export function livePresentationCloseFence( + source: ObservationSourceDescriptor, +): string | null { + const acquisitionId = source.binding.acquisitionId?.trim(); + return acquisitionId ? JSON.stringify([source.id, acquisitionId]) : null; +} + +export interface LiveDefaultPresentationAdmission { + visibleIds: string[]; + removedIds: string[]; + admittedIdentities: string[]; +} + +export function admitLiveDefaultPresentations( + currentIds: readonly string[], + sources: readonly ObservationSourceDescriptor[], + admittedIdentities: ReadonlySet, + closedAcquisitionSources: ReadonlySet, +): LiveDefaultPresentationAdmission { + let change = { visibleIds: [...currentIds], removedIds: [] as string[] }; + const removed = new Set(); + const admitted: string[] = []; + for (const source of sources) { + const presentationIdentity = automaticLivePresentationIdentity(source); + if ( + !presentationIdentity + || admittedIdentities.has(presentationIdentity) + || closedAcquisitionSources.has(livePresentationCloseFence(source) ?? "") + ) continue; + change = openObservationSource(change.visibleIds, source.id, sources); + change.removedIds.forEach((sourceId) => removed.add(sourceId)); + admitted.push(presentationIdentity); + } + return { + visibleIds: change.visibleIds, + removedIds: [...removed], + admittedIdentities: admitted, + }; +} + function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string { return sources .map((source) => [ @@ -107,6 +168,8 @@ export function useObservationLayout( const viewportSizeRef = useRef(null); const desiredSnapshotRef = useRef(null); const restoredLayoutAuthorityRef = useRef(false); + const admittedLivePresentationIdentitiesRef = useRef(new Set()); + const closedLiveAcquisitionSourcesRef = useRef(new Set()); const initializedCatalog = useRef(null); const sourceIdList = sources.map((source) => source.id).sort(); const sourceIdsIdentity = sourceIdList.join("\u0000"); @@ -284,31 +347,25 @@ export function useObservationLayout( sources, ]); - const selectedDeliveryIdentity = sources - .filter((source) => source.capabilities.defaultVisible && source.activation?.selected && source.delivery) - .map((source) => [ - source.id, - source.delivery?.id, - source.activation?.groupId, - source.activation?.maxActive, - ].join(":")) - .sort() - .join("|"); + const selectedDeliveryIdentity = JSON.stringify(sources + .map(automaticLivePresentationIdentity) + .filter((candidate): candidate is string => candidate !== null) + .sort()); useEffect(() => { if (restoredLayoutAuthorityRef.current) return; - const selected = sources.filter( - (source) => source.capabilities.defaultVisible && source.activation?.selected && source.delivery, + const admission = admitLiveDefaultPresentations( + visibleIdsRef.current, + sources, + admittedLivePresentationIdentitiesRef.current, + closedLiveAcquisitionSourcesRef.current, ); - if (!selected.length) return; - let change = { visibleIds: visibleIdsRef.current, removedIds: [] as string[] }; - const removed = new Set(); - for (const source of selected) { - change = openObservationSource(change.visibleIds, source.id, sources); - change.removedIds.forEach((sourceId) => removed.add(sourceId)); - } - commitVisibleIds(change.visibleIds); - clearPresentation([...removed], false); + if (!admission.admittedIdentities.length) return; + admission.admittedIdentities.forEach((presentationIdentity) => { + admittedLivePresentationIdentitiesRef.current.add(presentationIdentity); + }); + commitVisibleIds(admission.visibleIds); + clearPresentation(admission.removedIds, false); persistLiveLayout(); }, [clearPresentation, commitVisibleIds, persistLiveLayout, selectedDeliveryIdentity]); @@ -334,6 +391,8 @@ export function useObservationLayout( markPending(source, false); } } + const closeFence = livePresentationCloseFence(source); + if (closeFence) closedLiveAcquisitionSourcesRef.current.add(closeFence); restoredLayoutAuthorityRef.current = false; const change = closeObservationSource(visibleIdsRef.current, sourceId); commitVisibleIds(change.visibleIds); @@ -356,6 +415,8 @@ export function useObservationLayout( markPending(source, false); } } + const closeFence = livePresentationCloseFence(source); + if (closeFence) closedLiveAcquisitionSourcesRef.current.delete(closeFence); restoredLayoutAuthorityRef.current = false; const change = openObservationSource(visibleIdsRef.current, sourceId, sources); commitVisibleIds(change.visibleIds); @@ -427,6 +488,46 @@ export function useObservationLayout( } }, [applyDesiredSnapshot, persistLiveLayout]); + const activateAutomaticDefaults = useCallback(() => { + // A new operator-started live acquisition owns its initial presentation. + // Keep saved geometry, but do not let an older layout suppress a camera + // that the device plugin has just selected and delivered automatically. + restoredLayoutAuthorityRef.current = false; + const currentSources = sourcesRef.current; + let nextVisibleIds = visibleIdsRef.current; + for (const source of currentSources.filter(canOpenByDefault)) { + if (automaticLivePresentationIdentity(source)) continue; + nextVisibleIds = openObservationSource( + nextVisibleIds, + source.id, + currentSources, + ).visibleIds; + } + const admission = admitLiveDefaultPresentations( + nextVisibleIds, + currentSources, + admittedLivePresentationIdentitiesRef.current, + closedLiveAcquisitionSourcesRef.current, + ); + admission.admittedIdentities.forEach((presentationIdentity) => { + admittedLivePresentationIdentitiesRef.current.add(presentationIdentity); + }); + nextVisibleIds = admission.visibleIds; + commitVisibleIds(nextVisibleIds); + clearPresentation(admission.removedIds, false); + const firstFloatingDefault = currentSources.find( + (source) => ( + canOpenByDefault(source) + && source.capabilities.overlay + && nextVisibleIds.includes(source.id) + ), + ); + if (firstFloatingDefault) { + commitActiveFloatingSourceId(firstFloatingDefault.id); + } + persistLiveLayout(); + }, [clearPresentation, commitActiveFloatingSourceId, commitVisibleIds, persistLiveLayout]); + const snapshot = useCallback((): ObservationLayoutSnapshot | null => { if (!viewportSizeRef.current) return null; persistLiveLayout(); @@ -464,6 +565,7 @@ export function useObservationLayout( setFloatingMaximized, setWindowRect, setViewportSize, + activateAutomaticDefaults, snapshot, restore, }; diff --git a/apps/control-station/src/core/runtime/contracts.ts b/apps/control-station/src/core/runtime/contracts.ts index cdf6deb..f415e7f 100644 --- a/apps/control-station/src/core/runtime/contracts.ts +++ b/apps/control-station/src/core/runtime/contracts.ts @@ -154,6 +154,15 @@ export interface ObservationSourceActivation { controllable: boolean; } +export interface ObservationSourcePresentationLease { + kind: "active-stream-recovery"; + runtimeId: string; + acquisitionId: string; + acquisitionStateRevision: number; + producerGeneration: number; + recoveryGeneration: number; +} + export interface ObservationSourceDescriptor { id: string; sourceId: string; @@ -168,6 +177,7 @@ export interface ObservationSourceDescriptor { previewUrl?: string | null; delivery?: ObservationSourceDelivery | null; activation?: ObservationSourceActivation | null; + presentationLease?: ObservationSourcePresentationLease | null; provider: ObservationSourceProvider; binding: ObservationSourceBinding; capabilities: ObservationSourceCapabilities; @@ -201,6 +211,7 @@ export interface MissionRuntimeController { backendStatus: BackendStatus; pendingAction: string | null; refresh: () => void | Promise; + resetConnectionScenario?: () => Promise; updateViewerSettings: (settings: ViewerSettings) => Promise; setObservationSourceActive?: (sourceId: string, active: boolean) => Promise; } diff --git a/apps/control-station/src/presentation.ts b/apps/control-station/src/presentation.ts index 120d6e2..e692c23 100644 --- a/apps/control-station/src/presentation.ts +++ b/apps/control-station/src/presentation.ts @@ -23,6 +23,14 @@ export function phaseLabel(phase: RuntimePhase | null | undefined): string { return phase ? phaseLabels[phase] : "Нет состояния"; } +export function localConnectionPhaseLabel( + phase: RuntimePhase | null | undefined, +): string { + if (phase === "configuring") return "Подключение"; + if (phase === "connected") return "Подключение установлено"; + return phaseLabel(phase); +} + export function phaseTone(phase: RuntimePhase | null | undefined): StatusTone { if (!phase || phase === "unconfigured" || phase === "idle") return "neutral"; if (phase === "error") return "danger"; diff --git a/apps/control-station/src/productModel.ts b/apps/control-station/src/productModel.ts index 16ec44c..b8aaabd 100644 --- a/apps/control-station/src/productModel.ts +++ b/apps/control-station/src/productModel.ts @@ -190,8 +190,8 @@ export const workspaces: WorkspaceDefinition[] = [ { id: "local-device", root: "fleet", - label: "Локальное устройство", - title: "Локальное устройство", + label: "Подключение", + title: "Подключение", eyebrow: "ПАРК / ТЕКУЩИЙ АДАПТЕР", description: "Выбор модели, сценарий установленного плагина и запуск доступного потока.", icon: "network", diff --git a/apps/control-station/src/styles/responsive.css b/apps/control-station/src/styles/responsive.css index febc29b..b924e9f 100644 --- a/apps/control-station/src/styles/responsive.css +++ b/apps/control-station/src/styles/responsive.css @@ -204,7 +204,105 @@ } } +/* The shared shell switches to its compact overlay at 760 px, but Mission + Core's 20.75 rem navigation leaves less than the plugin's usable 32 rem + workspace before that point. Bridge the structural breakpoint locally so + 761--929 px never renders a full sidebar beside a sub-32-rem device UI. */ +@media (min-width: 761px) and (max-width: 929px) { + .nodedc-header-shell { + width: 100%; + padding: 0.85rem 1rem 0.65rem; + background: var(--nodedc-canvas); + } + + .nodedc-header { + min-height: 8.6rem; + } + + .nodedc-header__row { + min-height: 8.6rem; + grid-template-columns: minmax(0, 1fr) auto; + align-content: start; + row-gap: 0.65rem; + } + + .nodedc-header__center { + order: 3; + grid-column: 1 / -1; + justify-content: stretch; + overflow-x: auto; + } + + .nodedc-header__center .nodedc-segmented, + .nodedc-header__center .nodedc-header-navigation { + min-width: max-content; + } + + .nodedc-app-shell { + --nodedc-app-header-height: 10.25rem; + --nodedc-app-page-pad: 0.7rem; + --nodedc-app-panel-gap: 0.7rem; + } + + .nodedc-app-shell__stage { + padding: 0 var(--nodedc-app-page-pad) var(--nodedc-app-page-pad); + } + + .nodedc-app-shell__navigation, + .nodedc-app-shell__content { + right: var(--nodedc-app-page-pad); + bottom: var(--nodedc-app-page-pad); + left: var(--nodedc-app-page-pad); + width: auto; + } + + .nodedc-app-shell[data-navigation-open="true"] .nodedc-app-shell__stage, + .nodedc-app-shell[data-content-open="true"] .nodedc-app-shell__stage { + padding-left: var(--nodedc-app-page-pad); + pointer-events: none; + transform: translateX(calc(100vw + var(--nodedc-app-page-pad))); + } + + .nodedc-app-shell[data-content-open="true"] .nodedc-app-shell__navigation { + opacity: 0; + pointer-events: none; + } + + .nodedc-application-panel, + .nodedc-admin-panel { + width: 100%; + max-width: none; + } + + .nodedc-application-panel { + padding: 0.75rem; + } + + .nodedc-application-panel__titles p { + display: none; + } + + .nodedc-application-panel__action:first-child { + display: none; + } +} + @media (max-width: 760px) { + /* The shared application panel is a grid. Its header has three fixed-size + controls, so the grid item's automatic min-content width can widen the + body past the panel padding on narrow phones. Keep both grid rows owned + by the panel's content box; long plugin content must wrap inside it. */ + .nodedc-application-panel__head, + .nodedc-application-panel__body { + width: auto; + min-width: 0; + max-width: 100%; + } + + .nodedc-application-panel__head { + grid-template-columns: minmax(0, 1fr) auto; + } + .polygon-review-lead, .polygon-review-player__header, .polygon-review-order, diff --git a/apps/control-station/src/workspaces/ContourHealthWorkspace.tsx b/apps/control-station/src/workspaces/ContourHealthWorkspace.tsx index f64821e..c991e23 100644 --- a/apps/control-station/src/workspaces/ContourHealthWorkspace.tsx +++ b/apps/control-station/src/workspaces/ContourHealthWorkspace.tsx @@ -37,6 +37,25 @@ interface ContourSnapshot { error: string | null; } +export function contourRuntimeAuthorityPresentation( + state: MissionRuntimeState | null, +) { + const deviceControlConnectivity = state?.deviceSession?.connectivity ?? null; + return { + aiActive: Boolean( + state?.sourceMode === "live" + && state.phase === "streaming" + && state.metrics?.aiFrameRateHz + && state.metrics.aiFrameRateHz > 0 + ), + deviceControlConnectivity, + controlledDevice: state?.activeDevice + && (deviceControlConnectivity === "connected" || deviceControlConnectivity === "degraded") + ? state.activeDevice + : null, + }; +} + function validControlPlaneHealth(value: unknown): value is ControlPlaneHealth { if (!value || typeof value !== "object") return false; const item = value as Partial; @@ -155,16 +174,16 @@ export function ContourHealthWorkspace({ return () => window.clearInterval(timer); }, [refresh]); - const aiActive = Boolean( - state?.metrics?.aiFrameRateHz - && state.metrics.aiFrameRateHz > 0, - ); + const { + aiActive, + deviceControlConnectivity, + controlledDevice, + } = contourRuntimeAuthorityPresentation(state); const simulationWorker = snapshot?.simulationWorker ?? null; const controlPlaneReady = Boolean(snapshot?.controlPlane?.ok); const runtimeReady = snapshot?.pluginRuntimes.filter( (runtime) => runtime.status === "ready", ).length ?? 0; - const connectedDevice = state?.activeDevice ?? null; const processCount = (snapshot?.pluginRuntimes.length ?? 0) + 2; const readyProcessCount = runtimeReady + (controlPlaneReady ? 1 : 0) @@ -325,11 +344,16 @@ export function ContourHealthWorkspace({
- {connectedDevice ? ( + {controlledDevice ? (
- ПОДКЛЮЧЁННОЕ УСТРОЙСТВО - {connectedDevice.displayName} - {connectedDevice.endpointLabel ?? connectedDevice.modelId} + ПОДТВЕРЖДЁННАЯ УПРАВЛЯЮЩАЯ СЕССИЯ + {controlledDevice.displayName} + + {controlledDevice.endpointLabel ?? controlledDevice.modelId} + {deviceControlConnectivity === "degraded" + ? " · управление подтверждено, поток данных нарушен" + : " · управление подтверждено"} +
) : null} diff --git a/apps/control-station/src/workspaces/DeviceWorkspace.tsx b/apps/control-station/src/workspaces/DeviceWorkspace.tsx index dc1411d..bc8c57b 100644 --- a/apps/control-station/src/workspaces/DeviceWorkspace.tsx +++ b/apps/control-station/src/workspaces/DeviceWorkspace.tsx @@ -101,9 +101,8 @@ export function DeviceWorkspace({
- АКТИВНАЯ МОДЕЛЬ - {selection.model.displayName} - {selection.plugin.manifest.metadata.displayName} · v{selection.plugin.manifest.metadata.version} + СЦЕНАРИЙ ПОДКЛЮЧЕНИЯ + Модель выбрана {selectionTransitionError ? ( {selectionTransitionError} diff --git a/apps/control-station/src/workspaces/Workspaces.tsx b/apps/control-station/src/workspaces/Workspaces.tsx index 642606a..ddd9aec 100644 --- a/apps/control-station/src/workspaces/Workspaces.tsx +++ b/apps/control-station/src/workspaces/Workspaces.tsx @@ -16,6 +16,7 @@ import type { RecordedAdmissionPhase, RecordedCameraAdmissionState, } from "../core/observation/recordedSessionAdmission"; +import { liveRerunRecoveryAuthorityIdentity } from "../core/observation/liveReceiverWatchdog"; import type { ObservationSourceDescriptor } from "../core/runtime/contracts"; import { RerunViewport, @@ -490,12 +491,14 @@ function SpatialWorkspace({ data-primary-focused={pointCloudFocused ? "true" : undefined} data-media-maximized={floatingSourceMaximized ? "true" : undefined} > - {sourceUrl.trim() && pointCloudVisible ? ( + {sourceUrl.trim() && pointCloudVisible && !intentionalSourceEnd ? ( { server = await createServer({ @@ -57,15 +75,40 @@ before(async () => { compatibility = await server.ssrLoadModule( "@xgrids-k1/frontend/compatibility.ts", ); - ({ xgridsK1Api, ApiError } = await server.ssrLoadModule( + ({ + xgridsK1Api, + ApiError, + ApiRequestTimeoutError, + requestTimeoutMsForAction, + isXgridsConnectionVerification, + isXgridsConnectionReconfiguration, + isXgridsConnectionPolicy, + isXgridsConnectionPolicyDecision, + isXgridsHostFailureDiagnostic, + XGRIDS_CONNECTION_VERIFICATION_STATUSES, + XGRIDS_CONNECTION_VERIFICATION_LEASE_STATES, + XGRIDS_CONNECTION_POLICY_ACTIONS, + XGRIDS_CONNECTION_POLICY_TARGET_SOURCES, + XGRIDS_CONNECTION_ATTEMPT_PHASES, + isXgridsConnectionAttemptPhase, + } = await server.ssrLoadModule( "@xgrids-k1/frontend/api.ts", )); ({ localizeRuntimeMessage } = await server.ssrLoadModule( "@xgrids-k1/frontend/messages.ts", )); - ({ networkProvisionFailureMessage } = await server.ssrLoadModule( - "@xgrids-k1/frontend/useXgridsK1Runtime.ts", - )); + ({ + networkProvisionFailureMessage, + discoveryScanFailureMessage, + connectionVerificationFailureMessage, + operationById, + } = await server.ssrLoadModule("@xgrids-k1/frontend/useXgridsK1Runtime.ts")); + physicalCommandConfirmation = await server.ssrLoadModule( + "@xgrids-k1/frontend/physicalCommandConfirmation.ts", + ); + controlSessionCas = await server.ssrLoadModule( + "@xgrids-k1/frontend/controlSessionCas.ts", + ); }); after(async () => { @@ -129,6 +172,163 @@ function uiPlugin(manifest) { }; } +function supervisedConnectionState({ + mode = "bridge", + leaseState = "reachable", + generation = 1, + intentId = `intent-${mode}-1`, + hostPathEpoch = 3, + target = mode === "quick-connect" + ? { ipv4: "192.168.56.1", port: 1883 } + : { ipv4: "192.168.68.50", port: 1883 }, + controlAllowed = leaseState === "reachable", + dataAuthoritative = false, + lastKnown = null, + deviceNetworkState = "applied", + hostAvailable = true, + endpointState = "reachable", +} = {}) { + const observedAt = "2026-08-06T12:00:00Z"; + const identityVerified = leaseState === "reachable"; + const connectionReady = Boolean( + controlAllowed + && identityVerified + && leaseState === "reachable" + && deviceNetworkState === "applied" + && hostAvailable + && endpointState === "reachable", + ); + return { + snapshot_runtime_id: "runtime-a", + network_write_reconciliation: null, + connection_lifecycle: { + schema_version: "missioncore.xgrids-k1-connection-lifecycle/v1", + revision: generation, + desired_mode: mode, + configured_mode: deviceNetworkState === "applied" ? mode : null, + active_mode: connectionReady ? mode : null, + mode_change: { + state: connectionReady ? "ready" : "awaiting-control", + from: deviceNetworkState === "applied" ? mode : null, + to: mode, + }, + mode_selection: { + allowed: true, + reason_codes: [], + automatic_retry: false, + }, + active_binding_key: connectionReady ? `binding-${intentId}-${hostPathEpoch}` : null, + active_binding: connectionReady + ? { + binding_key: `binding-${intentId}-${hostPathEpoch}`, + intent_id: intentId, + transport_ref: "ble-k1-001", + connection_mode: mode, + target_ipv4: target.ipv4, + target_port: target.port, + host_path_epoch: hostPathEpoch, + control_session_id: "control-session-001", + logical_device_id: "device-k1-001", + } + : null, + connection_ready: connectionReady, + ready_to_start: connectionReady, + operation: null, + allowed_actions: connectionReady + ? ["start-acquisition", "select-connection-mode"] + : ["select-connection-mode"], + automatic_retry: false, + }, + connection_supervisor: { + schema_version: "missioncore.k1-connection-supervisor/v1", + revision: generation, + closed: false, + intent: { + intent_id: intentId, + requested_mode: mode, + expected_device_id: "device-k1-001", + requested_at: observedAt, + }, + observed: { + device_network: { + state: deviceNetworkState, + intent_id: deviceNetworkState === "applied" ? intentId : null, + transport_ref: deviceNetworkState === "applied" ? "ble-k1-001" : null, + connection_mode: deviceNetworkState === "applied" ? mode : null, + target: deviceNetworkState === "applied" ? target : null, + source: deviceNetworkState === "applied" ? "ble-read-only-status" : null, + observed_at: deviceNetworkState === "applied" ? observedAt : null, + }, + host_path: { + epoch: hostPathEpoch, + available: hostAvailable, + fingerprint: `en0:${hostPathEpoch}`, + interface: "en0", + source_ipv4: "192.168.68.10", + route_class: hostAvailable ? "direct" : "unavailable", + reason_code: null, + observed_at: observedAt, + }, + endpoint: { + target, + tcp_state: endpointState, + intent_id: intentId, + host_path_epoch: hostPathEpoch, + reason_code: null, + observed_at: observedAt, + }, + device_identity: { + state: identityVerified ? "verified" : "unverified", + intent_id: identityVerified ? intentId : null, + logical_device_id: identityVerified ? "device-k1-001" : null, + compatibility_profile_id: identityVerified + ? "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2" + : null, + connection_mode: identityVerified ? mode : null, + source: identityVerified ? "mqtt-device-info" : null, + host_path_epoch: identityVerified ? hostPathEpoch : null, + observed_at: identityVerified ? observedAt : null, + }, + control_plane: { + state: controlAllowed ? "healthy" : "idle", + session_id: controlAllowed ? "control-session-001" : null, + host_path_epoch: controlAllowed ? hostPathEpoch : null, + reason_code: null, + observed_at: controlAllowed ? observedAt : null, + }, + data_plane: { + state: dataAuthoritative ? "healthy" : "idle", + session_id: dataAuthoritative ? "data-session-001" : null, + host_path_epoch: dataAuthoritative ? hostPathEpoch : null, + reason_code: null, + observed_at: dataAuthoritative ? observedAt : null, + }, + }, + lease: { + state: leaseState, + generation, + intent_id: intentId, + host_path_epoch: hostPathEpoch, + connection_mode: mode, + target, + logical_device_id: identityVerified ? "device-k1-001" : null, + reason_code: null, + observed_at: observedAt, + }, + authority: { + network_mutation_allowed: false, + control_allowed: controlAllowed, + acquisition_start_allowed: controlAllowed, + data_ingest_authoritative: dataAuthoritative, + physical_motion_allowed: false, + reason_codes: [], + }, + last_known: lastKnown, + allowed_actions: [], + }, + }; +} + test("parser accepts reviewed v1alpha1 and v1alpha2 shapes", () => { const legacy = parseDevicePluginManifest(manifestDocument()); const current = parseDevicePluginManifest( @@ -221,6 +421,10 @@ test("installed XGRIDS frontend manifest exposes the semantic v1alpha2 actions", assert.equal(xgridsK1Actions.acquisitionStart, "acquisition.start"); assert.equal(xgridsK1Actions.acquisitionStop, "acquisition.stop"); assert.equal(xgridsK1Actions.connectionVerify, "connection.verify"); + assert.equal( + xgridsK1Actions.configuredEndpointProbe, + "connection.endpoint-probe", + ); }); test("one operator action can open at most one K1 control session", () => { @@ -299,6 +503,39 @@ test("stale K1 intent cannot continue past an await after runtime reactivation", assert.deepEqual(writes, ["fresh-checkpoint"]); }); +test("canonical K1 intent cannot cross from snapshot runtime A into B", async () => { + const generation = new operatorIntentGeneration.OperatorIntentGeneration(); + generation.activateRuntime(); + const intent = generation.beginOperatorIntent(); + assert.ok(intent); + + const actionRuntimeId = "snapshot-runtime-a"; + let currentRuntimeId = actionRuntimeId; + const writes = []; + const assertActionCurrent = () => { + if ( + !generation.isOperatorIntentCurrent(intent) + || !operatorIntentGeneration.isSnapshotRuntimeCurrent( + actionRuntimeId, + currentRuntimeId, + ) + ) { + throw new Error("snapshot runtime changed"); + } + }; + + const continuation = operatorIntentGeneration.awaitWhileIntentCurrent( + assertActionCurrent, + async () => { + currentRuntimeId = "snapshot-runtime-b"; + return { phase: "connection-ready" }; + }, + ).then(() => writes.push("next-mutation")); + + await assert.rejects(continuation, /snapshot runtime changed/); + assert.deepEqual(writes, []); +}); + test("a fresh explicit K1 intent supersedes the previous intent in one runtime", () => { const generation = new operatorIntentGeneration.OperatorIntentGeneration(); generation.activateRuntime(); @@ -313,7 +550,7 @@ test("a fresh explicit K1 intent supersedes the previous intent in one runtime", assert.equal(generation.isOperatorIntentCurrent(second), true); }); -test("canonical K1 launch wires the generation guard through every async stage", async () => { +test("canonical K1 preparation stops before START and guards every async stage", async () => { const hookSource = await readFile( new URL( "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", @@ -321,26 +558,758 @@ test("canonical K1 launch wires the generation guard through every async stage", ), "utf8", ); - const canonicalStart = hookSource.slice( - hookSource.indexOf("const startCanonicalAcquisition"), + const canonicalPreparation = hookSource.slice( + hookSource.indexOf("const prepareCanonicalAcquisition"), hookSource.indexOf("const prepareAcquisition"), ); - const pollingLoop = hookSource.slice( + const finalStart = hookSource.slice( + hookSource.indexOf("const startPreparedAcquisition"), + hookSource.indexOf("const startReplay"), + ); + const controlPollingLoop = hookSource.slice( hookSource.indexOf("async function waitForControlPhase"), + hookSource.indexOf("async function waitForPhysicalReconciliationProof"), + ); + const reconciliationPollingLoop = hookSource.slice( + hookSource.indexOf("async function waitForPhysicalReconciliationProof"), hookSource.indexOf("function messageFor"), ); assert.doesNotMatch(hookSource, /mounted\.current/); - assert.match(canonicalStart, /beginOperatorIntent\(\)/); - assert.match(canonicalStart, /isOperatorIntentCurrent\(intentToken\)/); - assert.doesNotMatch(canonicalStart, /await xgridsK1Api\./); - assert.doesNotMatch(canonicalStart, /await waitForControlPhase\(/); + assert.match(canonicalPreparation, /beginOperatorIntent\(\)/); + assert.match(canonicalPreparation, /isOperatorIntentCurrent\(intentToken\)/); + assert.match( + canonicalPreparation, + /const actionSnapshotRuntimeId = expectedSnapshotRuntimeId\(\)/, + ); + assert.match( + canonicalPreparation, + /isSnapshotRuntimeCurrent\(actionSnapshotRuntimeId\)/, + ); + assert.equal( + canonicalPreparation.match( + /expected_snapshot_runtime_id: actionSnapshotRuntimeId/g, + )?.length, + 3, + ); + assert.match(canonicalPreparation, /xgridsK1Api\.reconcilePhysicalCommand\(\{/); + assert.doesNotMatch(canonicalPreparation, /await xgridsK1Api\./); + assert.doesNotMatch(canonicalPreparation, /await waitForControlPhase\(/); + assert.doesNotMatch(canonicalPreparation, /xgridsK1Api\.startAcquisition/); + assert.match(finalStart, /xgridsK1Api\.startAcquisition/); assert.ok( - canonicalStart.match(/awaitWhileIntentCurrent\(/g)?.length >= 8, - "each canonical REST/checkpoint boundary must use the intent guard", + canonicalPreparation.match(/awaitWhileIntentCurrent\(/g)?.length >= 7, + "each preparation REST/checkpoint boundary must use the intent guard", + ); + assert.match(controlPollingLoop, /for \(;;\) \{\s*assertOperatorIntentCurrent\(\)/); + assert.equal(controlPollingLoop.match(/awaitWhileIntentCurrent\(/g)?.length, 2); + assert.match( + reconciliationPollingLoop, + /for \(;;\) \{\s*assertOperatorIntentCurrent\(\)/, + ); + assert.equal( + reconciliationPollingLoop.match(/awaitWhileIntentCurrent\(/g)?.length, + 2, + ); +}); + +test("K1 control-session CAS is exact, integer-only and mapped for each mutation family", () => { + const state = { + application_control_session: { + mode: "interactive-canonical", + state: "workspace-ready", + session_generation: 17, + state_revision: 43, + }, + }; + assert.deepEqual( + controlSessionCas.exactApplicationControlCas(state, "ENTER"), + { + expected_session_generation: 17, + expected_state_revision: 43, + }, + ); + assert.deepEqual( + controlSessionCas.exactAcquisitionControlCas(state, "PREPARE"), + { + expected_control_session_generation: 17, + expected_control_state_revision: 43, + }, + ); + assert.equal(controlSessionCas.acquisitionMutationUsesControlSession(state), true); + assert.equal( + controlSessionCas.acquisitionMutationUsesControlSession({ + application_control_session: { ...state.application_control_session, state: "closed" }, + }), + false, + ); + + for (const [field, value] of [ + ["session_generation", undefined], + ["session_generation", 1.5], + ["session_generation", -1], + ["session_generation", Number.NaN], + ["state_revision", undefined], + ["state_revision", 2.25], + ["state_revision", -1], + ["state_revision", Number.POSITIVE_INFINITY], + ]) { + assert.throws( + () => controlSessionCas.exactApplicationControlCas({ + application_control_session: { + ...state.application_control_session, + [field]: value, + }, + }, "TEST"), + (error) => error instanceof ApiError && /не отправлена/.test(error.message), + `${field}=${String(value)}`, + ); + } +}); + +test("K1 frontend reads CAS only from latest accepted state immediately before API mutations", async () => { + const apiSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/api.ts", + import.meta.url, + ), + "utf8", + ); + const hookSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), + "utf8", + ); + + assert.match(apiSource, /session_generation:\s*number/); + assert.match(apiSource, /state_revision:\s*number/); + assert.match(apiSource, /expected_session_generation:\s*number/); + assert.match(apiSource, /expected_control_session_generation:\s*number/); + assert.match(apiSource, /closeApplicationControlSession\(\s*body:/); + assert.doesNotMatch(hookSource, /closeApplicationControlSession\(\)/); + assert.ok( + [...hookSource.matchAll(/exactApplicationControlCas\(\s*latestState\.current/g)].length >= 3, + "standalone ENTER, CLOSE and canonical ENTER must use latest accepted state", + ); + assert.ok( + [...hookSource.matchAll(/exactAcquisitionControlCas\(\s*latestState\.current/g)].length >= 4, + "canonical PREPARE, generic PREPARE, START and STOP must use latest accepted state", + ); + assert.match( + hookSource, + /nextState = latestState\.current \?\? nextState;[\s\S]*?const phase = controlPhase\(nextState\)/, + ); + assert.match( + hookSource, + /mode:\s*"graceful",[\s\S]*?\.\.\.newMutationContext\("acquisition\.stop"\),[\s\S]*?\.\.\.controlCas/, + ); + assert.match(hookSource, /mode:\s*"capture-only"/); + + const stopMutation = hookSource.slice( + hookSource.indexOf("const stop = useCallback"), + hookSource.indexOf("const stopLocalReceiver = useCallback"), + ); + const localReceiverCleanup = hookSource.slice( + hookSource.indexOf("const stopLocalReceiver = useCallback"), + hookSource.indexOf("const abort = useCallback"), + ); + const mutationGate = stopMutation.indexOf( + "physicalStopIntentCheckpoint(dispatchState)", + ); + const exactCas = stopMutation.indexOf("exactAcquisitionControlCas(", mutationGate); + const spentFence = stopMutation.indexOf("spendPhysicalStopIntent(checkpoint)", exactCas); + const physicalCall = stopMutation.indexOf("xgridsK1Api.stopAcquisition({", spentFence); + assert.ok( + mutationGate >= 0 + && exactCas > mutationGate + && spentFence > exactCas + && physicalCall > spentFence, + "physical STOP must re-read exact authority and spend it synchronously before its API call", + ); + assert.match(stopMutation, /physicalStopPresentationOwner\.current === stopPresentationOwner/); + const captureOnlyMarker = stopMutation.indexOf('mode: "capture-only"'); + const captureOnlyPolicyGate = stopMutation.lastIndexOf( + 'connectionPolicyAllows(currentState, "stop-local-receiver")', + captureOnlyMarker, + ); + const compatibilityCall = stopMutation.indexOf( + "xgridsK1Api.stopSessionCompatibility({", + captureOnlyMarker, + ); + const compatibilityPolicyGate = stopMutation.lastIndexOf( + 'connectionPolicyAllows(currentState, "stop-local-receiver")', + compatibilityCall, + ); + assert.ok( + captureOnlyPolicyGate >= 0 + && captureOnlyPolicyGate < captureOnlyMarker + && compatibilityPolicyGate > captureOnlyMarker + && compatibilityPolicyGate < compatibilityCall, + "generic capture-only and compatibility STOP must fail closed on the current local-stop policy before API dispatch", + ); + const localReceiverPolicyGate = localReceiverCleanup.indexOf( + 'connectionPolicyAllows(currentState, "stop-local-receiver")', + ); + const localReceiverFirstApiCall = localReceiverCleanup.indexOf("xgridsK1Api."); + assert.ok( + localReceiverPolicyGate >= 0 + && localReceiverPolicyGate < localReceiverFirstApiCall, + "a direct or stale local cleanup call must be denied before either API path", + ); + assert.match( + localReceiverCleanup, + /const stopPlan = localReceiverStopPlan\(currentState\)/, + ); + assert.match( + localReceiverCleanup, + /stopPlan\.kind === "acquisition"/, + ); + assert.match( + localReceiverCleanup, + /xgridsK1Api\.stopAcquisition\(\{[\s\S]*?acquisition_id: stopPlan\.acquisitionId,[\s\S]*?mode: "capture-only"[\s\S]*?expected_snapshot_runtime_id: snapshotRuntimeId/, + ); + assert.match( + localReceiverCleanup, + /xgridsK1Api\.stopSessionCompatibility\(\{[\s\S]*?expected_snapshot_runtime_id: snapshotRuntimeId/, + ); + assert.doesNotMatch( + localReceiverCleanup, + /exactAcquisitionControlCas|physical_acceptance|spendPhysicalStopIntent/, + ); +}); + +test("K1 API forwards runtime fences and exact CAS fields without dropping them", async () => { + const originalFetch = globalThis.fetch; + const calls = []; + globalThis.fetch = async (path, init) => { + calls.push({ path, init }); + return new Response(JSON.stringify({ state: { source_mode: "idle" } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + + try { + await xgridsK1Api.openApplicationControlSession({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operator_present: true, + owner_controlled_device: true, + lixelgo_closed: true, + battery_storage_confirmed: true, + expected_physical_state_confirmed: true, + timezone_name: "Europe/Moscow", + }); + await xgridsK1Api.enterApplicationWorkspace({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operator_confirmed: true, + expected_session_generation: 7, + expected_state_revision: 8, + }); + await xgridsK1Api.closeApplicationControlSession({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + expected_session_generation: 9, + expected_state_revision: 10, + }); + await xgridsK1Api.reconcilePhysicalCommand({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + reconciliation_id: "reconciliation-cas", + expected_session_generation: 9, + expected_state_revision: 10, + }); + await xgridsK1Api.prepareAcquisition({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000101", + idempotency_key: "acquisition.prepare:op-00000000-0000-4000-8000-000000000101", + project_name: "CAS01", + mount_type: "handheld", + gnss_mode: "none", + compatibility_attestation: { + firmware_version: "3.0.2", + topology: "direct-lan", + verification: "live-device-info", + }, + expected_control_session_generation: 11, + expected_control_state_revision: 12, + }); + await xgridsK1Api.startAcquisition({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000102", + idempotency_key: "acquisition.start:op-00000000-0000-4000-8000-000000000102", + acquisition_id: "acquisition-cas", + expected_control_session_generation: 13, + expected_control_state_revision: 14, + }); + await xgridsK1Api.stopAcquisition({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000103", + idempotency_key: "acquisition.stop:op-00000000-0000-4000-8000-000000000103", + acquisition_id: "acquisition-cas", + mode: "graceful", + expected_control_session_generation: 15, + expected_control_state_revision: 16, + }); + await xgridsK1Api.abortAcquisition({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000104", + idempotency_key: "acquisition.abort:op-00000000-0000-4000-8000-000000000104", + acquisition_id: "acquisition-cas", + expected_control_session_generation: 17, + expected_control_state_revision: 18, + }); + await xgridsK1Api.stopSessionCompatibility({ + expected_snapshot_runtime_id: "snapshot-runtime-cas", + }); + } finally { + globalThis.fetch = originalFetch; + } + + assert.deepEqual( + calls.map(({ init }) => JSON.parse(init.body).input), + [ + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operator_present: true, + owner_controlled_device: true, + lixelgo_closed: true, + battery_storage_confirmed: true, + expected_physical_state_confirmed: true, + timezone_name: "Europe/Moscow", + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operator_confirmed: true, + expected_session_generation: 7, + expected_state_revision: 8, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + expected_session_generation: 9, + expected_state_revision: 10, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + reconciliation_id: "reconciliation-cas", + expected_session_generation: 9, + expected_state_revision: 10, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000101", + idempotency_key: "acquisition.prepare:op-00000000-0000-4000-8000-000000000101", + project_name: "CAS01", + mount_type: "handheld", + gnss_mode: "none", + compatibility_attestation: { + firmware_version: "3.0.2", + topology: "direct-lan", + verification: "live-device-info", + }, + expected_control_session_generation: 11, + expected_control_state_revision: 12, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000102", + idempotency_key: "acquisition.start:op-00000000-0000-4000-8000-000000000102", + acquisition_id: "acquisition-cas", + expected_control_session_generation: 13, + expected_control_state_revision: 14, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000103", + idempotency_key: "acquisition.stop:op-00000000-0000-4000-8000-000000000103", + acquisition_id: "acquisition-cas", + mode: "graceful", + expected_control_session_generation: 15, + expected_control_state_revision: 16, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + operation_id: "op-00000000-0000-4000-8000-000000000104", + idempotency_key: "acquisition.abort:op-00000000-0000-4000-8000-000000000104", + acquisition_id: "acquisition-cas", + expected_control_session_generation: 17, + expected_control_state_revision: 18, + }, + { + expected_snapshot_runtime_id: "snapshot-runtime-cas", + }, + ], + ); +}); + +test("K1 lifecycle mutation context is unique and binds one key to one operation", () => { + const first = lifecycle.newMutationContext("acquisition.start"); + const second = lifecycle.newMutationContext("acquisition.start"); + + assert.match(first.operation_id, /^op-[0-9a-f-]{36}$/); + assert.equal(first.idempotency_key, `acquisition.start:${first.operation_id}`); + assert.notEqual(second.operation_id, first.operation_id); + assert.equal(second.idempotency_key, `acquisition.start:${second.operation_id}`); + assert.throws( + () => lifecycle.newMutationContext(" "), + /безопасный ключ не создан/, + ); +}); + +test("K1 API uses bounded no-retry timeout classes per operation family", async () => { + const apiSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/api.ts", + import.meta.url, + ), + "utf8", + ); + assert.equal(requestTimeoutMsForAction(xgridsK1Actions.stateRead), 8_000); + assert.equal(requestTimeoutMsForAction(xgridsK1Actions.discoveryScan), 70_000); + assert.equal(requestTimeoutMsForAction(xgridsK1Actions.networkProvision), 300_000); + assert.equal(requestTimeoutMsForAction(xgridsK1Actions.connectionVerify), 150_000); + assert.equal(requestTimeoutMsForAction(xgridsK1Actions.configuredEndpointProbe), 15_000); + for (const action of [ + xgridsK1Actions.applicationControlSessionOpen, + xgridsK1Actions.applicationControlWorkspaceEnter, + xgridsK1Actions.applicationControlSessionClose, + xgridsK1Actions.acquisitionPrepare, + xgridsK1Actions.acquisitionStart, + xgridsK1Actions.acquisitionStop, + ]) { + assert.equal(requestTimeoutMsForAction(action), 120_000, action); + } + const timeout = new ApiRequestTimeoutError("acquisition.start", 120_000); + assert.equal(timeout.outcomeUnknown, true); + assert.equal(timeout.automaticRetry, false); + assert.equal(timeout.transportUnavailable, false); + assert.match(timeout.message, /автоматический повтор запрещён/); + assert.match(apiSource, /const controller = new AbortController\(\)/); + assert.match(apiSource, /setTimeout\(\(\) => \{\s*timedOut = true;\s*controller\.abort\(\)/); + assert.match(apiSource, /finally \{\s*clearTimeout\(timeout\)/); + assert.doesNotMatch(apiSource, /automaticRetry\s*=\s*true/); +}); + +test("one explicit K1 operator action supplies the backend physical acceptance", () => { + assert.deepEqual(physicalCommandConfirmation.operatorActionPhysicalAcceptance(), { + operator_present: true, + owner_controlled_device: true, + lixelgo_closed: true, + battery_storage_confirmed: true, + expected_physical_state_confirmed: true, + }); +}); + +test("final K1 START target requires exact prepared project and fresh READY authority", () => { + const state = { + ...supervisedConnectionState(), + phase: "ready", + source_mode: "idle", + selected_device_id: "ble-k1-001", + device_ref: { + device_id: "device-k1-001", + }, + device_session: { + device_session_id: "device-session-001", + device_id: "device-k1-001", + connectivity: "connected", + }, + compatibility: { + permitted_mode: "active-control", + vendor_writes_enabled: true, + }, + acquisition: { + acquisition_id: "acquisition-001", + device_id: "device-k1-001", + device_session_id: "device-session-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_mode: "plugin-commanded", + project_name: "RAVNOVES01", + requested_streams: [], + target_host: "127.0.0.1", + duration_seconds: null, + evidence_policy: "required", + state: "prepared", + state_revision: 4, + }, + application_control_session: { + session_generation: 5, + state_revision: 8, + state: "project-ready", + control_socket_open: true, + can_start: true, + verified_control: { + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_session_id: "control-session-001", + source: "mqtt-device-info", + intent_id: "intent-bridge-1", + transport_ref: "ble-k1-001", + host_path_epoch: 3, + target_ipv4: "192.168.68.50", + target_port: 1883, + connection_mode: "bridge", + control_proof_revision: 9, + control_proof_source: "device-status", + control_proof_fresh: true, + }, + transport: { + latest_device_session_state: "ready", + latest_device_project_bound: true, + latest_device_init_ready: false, + }, + }, + }; + + const target = physicalCommandConfirmation.preparedStartTarget(state); + assert.ok(target); + const { fence, ...displayTarget } = target; + assert.deepEqual(displayTarget, { + deviceId: "device-k1-001", + connection: "bridge · 192.168.68.50:1883", + projectName: "RAVNOVES01", + acquisitionId: "acquisition-001", + deviceState: "READY · проект привязан · инициализация не запущена", + }); + assert.equal(fence.kind, "start"); + assert.equal(fence.controlSessionGeneration, 5); + assert.equal(fence.controlStateRevision, 8); + assert.equal(fence.connectionIntentId, "intent-bridge-1"); + assert.equal(fence.transportRef, "ble-k1-001"); + assert.equal(fence.acquisitionStateRevision, 4); + + for (const mutate of [ + (candidate) => { candidate.application_control_session.can_start = false; }, + (candidate) => { candidate.application_control_session.transport.latest_device_session_state = "scanning"; }, + (candidate) => { candidate.application_control_session.transport.latest_device_project_bound = false; }, + (candidate) => { candidate.application_control_session.transport.latest_device_init_ready = true; }, + (candidate) => { candidate.application_control_session.verified_control.control_proof_fresh = false; }, + (candidate) => { candidate.connection_supervisor.authority.acquisition_start_allowed = false; }, + (candidate) => { candidate.acquisition.device_id = "another-device"; }, + (candidate) => { candidate.application_control_session.verified_control.intent_id = "stale-intent"; }, + (candidate) => { candidate.application_control_session.verified_control.host_path_epoch = 2; }, + (candidate) => { candidate.application_control_session.verified_control.transport_ref = "stale-transport"; }, + (candidate) => { candidate.application_control_session.verified_control.target_port = 1884; }, + (candidate) => { candidate.connection_lifecycle.ready_to_start = false; }, + ]) { + const stale = structuredClone(state); + mutate(stale); + assert.equal(physicalCommandConfirmation.preparedStartTarget(stale), null); + } +}); + +test("physical K1 modal checkpoint stays stable on refresh and invalidates on topology, CAS or state changes", () => { + const state = { + ...supervisedConnectionState(), + phase: "ready", + source_mode: "idle", + selected_device_id: "ble-k1-001", + device_ref: { device_id: "device-k1-001" }, + device_session: { + device_session_id: "device-session-001", + device_id: "device-k1-001", + connectivity: "connected", + }, + acquisition: { + acquisition_id: "acquisition-fence-1", + device_id: "device-k1-001", + device_session_id: "device-session-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_mode: "plugin-commanded", + project_name: "FENCE01", + requested_streams: [], + target_host: "127.0.0.1", + duration_seconds: null, + evidence_policy: "required", + state: "prepared", + state_revision: 21, + }, + application_control_session: { + session_generation: 13, + state_revision: 34, + state: "project-ready", + control_socket_open: true, + can_start: true, + verified_control: { + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_session_id: "control-session-001", + source: "mqtt-device-info", + intent_id: "intent-bridge-1", + transport_ref: "ble-k1-001", + host_path_epoch: 3, + target_ipv4: "192.168.68.50", + target_port: 1883, + connection_mode: "bridge", + control_proof_revision: 55, + control_proof_source: "device-status", + control_proof_fresh: true, + }, + transport: { + latest_device_session_state: "ready", + latest_device_project_bound: true, + latest_device_init_ready: false, + }, + }, + }; + const openingTarget = physicalCommandConfirmation.preparedStartTarget(state); + assert.ok(openingTarget); + const checkpoint = physicalCommandConfirmation.createPhysicalCommandCheckpoint( + "start", + openingTarget, + ); + + const readOnlyRefresh = structuredClone(state); + readOnlyRefresh.snapshot_revision = 999; + // The supervisor revision is an observation counter, not a topology + // generation: a semantically identical host/MQTT poll increments it. + // Such a refresh must not destroy an operator confirmation in progress. + readOnlyRefresh.connection_supervisor.revision += 1; + readOnlyRefresh.snapshot_runtime_started_at_utc = "2026-08-06T12:01:00Z"; + readOnlyRefresh.connection_supervisor.observed.host_path.observed_at = + "2026-08-06T12:01:00Z"; + readOnlyRefresh.connection_supervisor.observed.endpoint.observed_at = + "2026-08-06T12:01:00Z"; + const unchangedTarget = physicalCommandConfirmation.preparedStartTarget(readOnlyRefresh); + assert.ok(unchangedTarget); + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "start", + unchangedTarget, + ), + true, + ); + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "stop", + unchangedTarget, + ), + false, + ); + assert.equal(Object.isFrozen(checkpoint), true); + assert.equal(Object.isFrozen(checkpoint.fence), true); + assert.equal(Object.isFrozen(checkpoint.target), true); + + const topologyChanged = structuredClone(state); + topologyChanged.connection_supervisor.revision += 1; + topologyChanged.connection_supervisor.intent.intent_id = "intent-bridge-2"; + topologyChanged.connection_supervisor.observed.device_network.intent_id = "intent-bridge-2"; + topologyChanged.connection_supervisor.observed.endpoint.intent_id = "intent-bridge-2"; + topologyChanged.connection_supervisor.observed.device_identity.intent_id = "intent-bridge-2"; + topologyChanged.connection_supervisor.lease.intent_id = "intent-bridge-2"; + topologyChanged.connection_lifecycle.active_binding.intent_id = "intent-bridge-2"; + topologyChanged.connection_lifecycle.active_binding.binding_key = "binding-intent-bridge-2-3"; + topologyChanged.connection_lifecycle.active_binding_key = "binding-intent-bridge-2-3"; + topologyChanged.application_control_session.verified_control.intent_id = "intent-bridge-2"; + const topologyTarget = physicalCommandConfirmation.preparedStartTarget(topologyChanged); + assert.ok(topologyTarget); + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "start", + topologyTarget, + ), + false, + ); + + const casChanged = structuredClone(state); + casChanged.application_control_session.session_generation += 1; + casChanged.application_control_session.state_revision += 1; + const casTarget = physicalCommandConfirmation.preparedStartTarget(casChanged); + assert.ok(casTarget); + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "start", + casTarget, + ), + false, + ); + + const runtimeStateChanged = structuredClone(state); + runtimeStateChanged.phase = "degraded"; + const runtimeTarget = physicalCommandConfirmation.preparedStartTarget(runtimeStateChanged); + assert.ok(runtimeTarget); + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "start", + runtimeTarget, + ), + false, + ); + + assert.equal(checkpoint.target.projectName, "FENCE01"); + const changedDisplay = structuredClone(openingTarget); + changedDisplay.projectName = "FENCE02"; + assert.equal( + physicalCommandConfirmation.physicalCommandCheckpointMatches( + checkpoint, + "start", + changedDisplay, + ), + false, + ); + assert.equal(checkpoint.target.projectName, "FENCE01"); +}); + +test("K1 STOP target is bound to the active acquisition instead of START constants", () => { + const state = { + compatibility: { + permitted_mode: "active-control", + vendor_writes_enabled: true, + }, + acquisition: { + acquisition_id: "acquisition-active-77", + device_id: "device-k1-active", + device_session_id: "device-session-active", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_mode: "plugin-commanded", + project_name: "ROUTE77", + requested_streams: [], + target_host: "127.0.0.1", + duration_seconds: null, + evidence_policy: "required", + state: "acquiring", + state_revision: 12, + }, + application_control_session: { + state: "scanning", + verified_control: { + logical_device_id: "device-k1-active", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + connection_mode: "quick-connect", + target_ipv4: "192.168.56.1", + target_port: 1883, + }, + transport: { + latest_device_session_state: "scanning", + latest_device_project_bound: true, + latest_device_init_ready: true, + }, + }, + }; + + const target = physicalCommandConfirmation.activeStopTarget(state); + assert.ok(target); + const { fence, ...displayTarget } = target; + assert.deepEqual(displayTarget, { + deviceId: "device-k1-active", + connection: "quick-connect · 192.168.56.1:1883", + projectName: "ROUTE77", + acquisitionId: "acquisition-active-77", + deviceState: "SCANNING · проект привязан · инициализация завершена", + }); + assert.equal(fence.kind, "stop"); + assert.equal(fence.acquisitionStateRevision, 12); + assert.equal( + physicalCommandConfirmation.activeStopTarget({ + ...state, + acquisition: { ...state.acquisition, control_mode: "operator-manual" }, + }), + null, ); - assert.match(pollingLoop, /for \(;;\) \{\s*assertOperatorIntentCurrent\(\)/); - assert.equal(pollingLoop.match(/awaitWhileIntentCurrent\(/g)?.length, 2); }); test("K1 control errors stay informative and only fetch failures mark transport unavailable", async () => { @@ -435,6 +1404,7 @@ test("live source is confirmed only by an acquiring acquisition", () => { acquisition: { state: "awaiting_external_start" }, }; const acquiring = { + ...supervisedConnectionState({ dataAuthoritative: true }), ...waiting, acquisition: { state: "acquiring" }, }; @@ -445,6 +1415,9 @@ test("live source is confirmed only by an acquiring acquisition", () => { assert.equal(lifecycle.isConfirmedLiveState(acquiring), true); assert.equal(lifecycle.confirmedRuntimeSourceMode(acquiring), "live"); assert.equal(lifecycle.liveStartPlan(acquiring), "already-running"); + const staleDataEpoch = structuredClone(acquiring); + staleDataEpoch.connection_supervisor.observed.data_plane.host_path_epoch = 2; + assert.equal(lifecycle.isConfirmedLiveState(staleDataEpoch), false); assert.equal( lifecycle.controlSessionEntryPlan("scanning", false, false), "continue", @@ -558,6 +1531,98 @@ test("spatial K1 action failures have an explicit retry-safe presentation", () = }), false); }); +test("physical-command guidance projects backend policy without exposing reason codes", () => { + const denied = { + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["stop-local-receiver"], + recommended_action: "stop-local-receiver", + actions: { + "stop-acquisition": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "connection-supervisor", + required_transport_ref: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + }; + const guidance = presentation.connectionPolicyOperatorGuidance( + denied, + "stop-acquisition", + ); + assert.deepEqual(guidance, { + reason: "Результат предыдущей физической команды K1 не подтверждён.", + nextAction: "Завершите локальный приём; физическое состояние K1 проверьте вручную.", + }); + assert.doesNotMatch(`${guidance.reason} ${guidance.nextAction}`, /physical-command|stop-local/); + + const allowed = structuredClone(denied); + allowed.connection_policy.allowed_actions.push("stop-acquisition"); + allowed.connection_policy.actions["stop-acquisition"].allowed = true; + allowed.connection_policy.actions["stop-acquisition"].reason_codes = []; + assert.equal( + presentation.connectionPolicyOperatorGuidance(allowed, "stop-acquisition"), + null, + ); +}); + +test("restart-recovery guidance describes automatic recovery without protocol ceremony", () => { + const deniedFresh = { + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["observe-configured-device-network"], + recommended_action: "observe-configured-device-network", + actions: { + "observe-fresh-device-network": { + allowed: false, + reason_codes: ["reconciliation-target-not-observed"], + target_source: "fresh-scan", + required_transport_ref: "exact-durable-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "exact-durable-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + }; + + const durableGuidance = presentation.connectionPolicyOperatorGuidance( + deniedFresh, + "observe-fresh-device-network", + ); + assert.deepEqual(durableGuidance, { + reason: "В свежем Bluetooth-поиске не найден K1, связанный с незавершённой записью.", + nextAction: "Дождитесь автоматического восстановления сохранённого подключения K1.", + }); + assert.doesNotMatch( + `${durableGuidance.reason} ${durableGuidance.nextAction}`, + /observe-configured|durable-configured/, + ); + + deniedFresh.connection_policy.recommended_action = "observe-current-device-network"; + assert.equal( + presentation.connectionPolicyOperatorGuidance( + deniedFresh, + "observe-fresh-device-network", + ).nextAction, + "Дождитесь автоматического восстановления связи с тем же K1.", + ); +}); + test("replay ignores a stale failed live acquisition", () => { const replay = { source_mode: "replay", @@ -632,9 +1697,199 @@ test("a provisioned address is not presented as a verified device connection", ( assert.equal(lifecycle.normalizeRuntimePhase({ phase: "connected", application_control_session: { state: "connection-ready" }, + }), "configuring"); + assert.equal(lifecycle.normalizeRuntimePhase({ + ...supervisedConnectionState({ + leaseState: "configured-unverified", + controlAllowed: false, + }), + phase: "connected", }), "connected"); }); +test("a released acquisition failure stays exact in the acquisition panel without poisoning global runtime", () => { + const releasedFailure = { + phase: "error", + source_mode: "idle", + acquisition: { + state: "failed", + cleanup_pending: false, + }, + application_control_session: { + state: "idle", + failure: { + network_change_admissible: true, + message: "control connection was lost after acknowledged stop", + }, + }, + }; + + assert.equal(lifecycle.isReleasedTerminalAcquisitionFailure(releasedFailure), true); + assert.equal(lifecycle.normalizeRuntimePhase(releasedFailure), "idle"); + assert.equal(lifecycle.shouldSurfaceRuntimeActionError("live", releasedFailure), false); + assert.equal(lifecycle.shouldSurfaceRuntimeActionError("control", releasedFailure), false); + assert.equal(lifecycle.shouldSurfaceRuntimeActionError("scan", releasedFailure), true); + assert.equal( + lifecycle.authoritativeStateSupersedesRuntimeError( + { action: "stop", runtimeId: "runtime-a", leaseGeneration: 4 }, + releasedFailure, + ), + true, + ); + assert.equal( + lifecycle.authoritativeStateSupersedesRuntimeError( + { action: "scan", runtimeId: "runtime-a", leaseGeneration: 4 }, + releasedFailure, + ), + false, + ); + assert.equal( + lifecycle.isReleasedTerminalAcquisitionFailure({ + ...releasedFailure, + acquisition: { state: "failed", cleanup_pending: true }, + }), + false, + ); + assert.equal( + lifecycle.isReleasedTerminalAcquisitionFailure({ + ...releasedFailure, + application_control_session: { + state: "failed", + failure: { network_change_admissible: false }, + }, + }), + false, + ); +}); + +test("poll and WebSocket state acceptance clear stale runtime banners after local release", async () => { + const runtimeSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), + "utf8", + ); + + assert.match(runtimeSource, /authoritativeStateSupersedesRuntimeError\(\s*errorCorrelation\.current,\s*acceptedState/); + assert.match(runtimeSource, /openEventSocket\(acceptState/); +}); + +test("observed SCANNING recovery is a successful STOP-only connection outcome", async () => { + const recoveredScanning = { + ...supervisedConnectionState({ mode: "bridge" }), + connection_mode: "bridge", + active_connection_mode: "bridge", + application_control_session: { + state: "scanning", + can_stop: true, + physical_command: { + requires_reconciliation: false, + resolved_active_recovery_required: true, + observed_session_state: "scanning", + }, + }, + }; + + assert.equal(lifecycle.isRecoveredPhysicalScanning(recoveredScanning, "bridge"), true); + assert.equal( + lifecycle.isRecoveredPhysicalScanning(recoveredScanning, "quick-connect"), + false, + ); + const replayDuringPhysicalRecovery = { + ...recoveredScanning, + source_mode: "replay", + compatibility: { + vendor_writes_enabled: true, + permitted_mode: "active-control", + }, + acquisition: { + acquisition_id: "terminal-live-acquisition", + device_id: "original-k1", + device_session_id: "fresh-verified-session", + project_name: null, + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_mode: "plugin-commanded", + requested_streams: [], + target_host: "192.168.68.52", + evidence_policy: "disabled", + state: "failed", + state_revision: 9, + cleanup_pending: false, + result: { + recovery_only: true, + device_state: "scanning", + automatic_replay_allowed: false, + }, + }, + }; + assert.equal( + lifecycle.requiresCanonicalStopAfterTerminalLocalFailure(replayDuringPhysicalRecovery), + true, + ); + const recoveredStopTarget = physicalCommandConfirmation.activeStopTarget( + replayDuringPhysicalRecovery, + ); + assert.ok(recoveredStopTarget); + assert.equal(recoveredStopTarget.acquisitionId, "terminal-live-acquisition"); + assert.equal(recoveredStopTarget.deviceId, "original-k1"); + assert.equal(recoveredStopTarget.fence.kind, "stop"); + + const runtimeSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), + "utf8", + ); + const provisioningSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx", + import.meta.url, + ), + "utf8", + ); + const acquisitionSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx", + import.meta.url, + ), + "utf8", + ); + assert.match(runtimeSource, /requireExactReadOnlyVerificationOutcome/); + assert.match(runtimeSource, /isRecoveredPhysicalScanning\(failedState, requestedConnectionMode\)/); + assert.match(provisioningSource, /isPhysicalStopRecoverySettling\(state\)/); + assert.match( + provisioningSource, + /const canScan = backendScanAllowed && !physicalStopRecoverySettling/, + ); + assert.doesNotMatch( + provisioningSource, + /K1 продолжает сканирование|Завершаем остановку K1/, + ); + assert.match(acquisitionSource, /recoveredPhysicalStop\s*\? "Требуется остановка"/); + assert.match(acquisitionSource, /terminalPhysicalStopObserved\s*\? "warning"/); + assert.match(acquisitionSource, /recoveredPhysicalStop \? "Сканирование продолжается"/); + assert.match(acquisitionSource, /Нажмите «Остановить сканирование» ниже или в пространственной сцене/); + assert.match( + acquisitionSource, + /canIssueCanonicalStop\(state, physicalStopIntentSpent\)/, + ); + assert.match( + acquisitionSource, + /physicalStopInFlight\s*\? "Останавливаем устройство…"/, + ); + assert.match( + acquisitionSource, + /: physicalStopPresented \? recoveredPhysicalStop \? "Остановить сканирование" : "Остановить устройство и запись"/, + ); + assert.ok( + acquisitionSource.indexOf("terminalPhysicalStopObserved ? (") + < acquisitionSource.indexOf("
"), + "recovered SCANNING must render the STOP-only branch before project/START controls", + ); +}); + test("K1 provisioning mutations require the operator's fresh BLE candidate", () => { const backendLease = { selected_device_id: "stale-backend-lease-device", @@ -662,46 +1917,1280 @@ test("K1 provisioning mutations require the operator's fresh BLE candidate", () assert.equal(lifecycle.canSubmitProvisioningMutation({ devices: backendLease.devices, selectedDeviceId: "expired-scan-candidate", - powerConfirmed: true, credentialsReady: true, isBusy: false, }), false); assert.equal(lifecycle.canSubmitProvisioningMutation({ devices: backendLease.devices, selectedDeviceId: "fresh-scan-candidate", - powerConfirmed: true, credentialsReady: true, isBusy: false, }), true); }); test("K1 connection status is green only for a reachable matching lease", () => { - const unreachableLease = { + const legacyReachableLease = { k1_ip: "192.168.68.50", connection_mode: "bridge", - connection_verification: { - lease_state: "disconnected", - network_reachability: "unreachable", - }, - }; - const reachableLease = { - ...unreachableLease, connection_verification: { lease_state: "reachable", network_reachability: "reachable", }, }; + const reachableLease = supervisedConnectionState(); - assert.equal(lifecycle.isReachableConnectionLease(unreachableLease, "bridge"), false); + assert.equal(lifecycle.isReachableConnectionLease(legacyReachableLease, "bridge"), false); assert.equal(lifecycle.isReachableConnectionLease(reachableLease, "bridge"), true); assert.equal(lifecycle.isReachableConnectionLease(reachableLease, "quick-connect"), false); assert.equal(lifecycle.isReachableConnectionLease({ k1_ip: "192.168.68.50", connection_mode: "bridge", }, "bridge"), false); + assert.equal(lifecycle.isReachableConnectionLease({ + k1_ip: "192.168.56.1", + connection_mode: "quick-connect", + connection_verification: { + status: "control-transport-lost", + lease_state: "disconnected", + network_reachability: "unreachable", + }, + }, "quick-connect"), false); + + for (const mutate of [ + (state) => { state.connection_supervisor.observed.device_network.state = "unconfigured"; }, + (state) => { state.connection_supervisor.observed.device_network.intent_id = "stale-intent"; }, + (state) => { state.connection_supervisor.observed.device_network.transport_ref = null; }, + (state) => { + state.connection_supervisor.observed.device_network.target = { + ...state.connection_supervisor.observed.device_network.target, + ipv4: "192.168.68.99", + }; + }, + (state) => { state.connection_supervisor.observed.endpoint.intent_id = "stale-intent"; }, + (state) => { state.connection_supervisor.observed.endpoint.host_path_epoch = 2; }, + (state) => { state.connection_supervisor.observed.host_path.route_class = "default"; }, + (state) => { state.connection_supervisor.observed.device_identity.intent_id = "stale-intent"; }, + (state) => { state.connection_supervisor.observed.device_identity.host_path_epoch = 2; }, + (state) => { state.connection_supervisor.observed.control_plane.host_path_epoch = 2; }, + (state) => { state.connection_supervisor.observed.control_plane.session_id = null; }, + (state) => { + state.connection_supervisor.observed.endpoint.target = { + ...state.connection_supervisor.observed.endpoint.target, + ipv4: "192.168.68.99", + }; + }, + ]) { + const stale = structuredClone(reachableLease); + mutate(stale); + assert.equal(lifecycle.isReachableConnectionLease(stale, "bridge"), false); + } }); -test("provisioning intent keeps one idempotency key and exposes unsafe outcomes", () => { +test("frontend connection authority fails closed without the backend lifecycle projection", () => { + const reachable = supervisedConnectionState(); + assert.equal(lifecycle.isReachableConnectionLease(reachable, "bridge"), true); + + const omitted = structuredClone(reachable); + delete omitted.connection_lifecycle; + assert.equal(lifecycle.isReachableConnectionLease(omitted, "bridge"), false); + assert.equal( + lifecycle.currentAppliedConnectionTopology(omitted, "bridge")?.status, + "configured-unverified", + ); + + const drifted = structuredClone(reachable); + drifted.connection_lifecycle.active_mode = "quick-connect"; + assert.equal(lifecycle.isReachableConnectionLease(drifted, "bridge"), false); +}); + +test("backend topology distinguishes reachable authority from last-known address", () => { + const target = { ipv4: "192.168.56.1", port: 1883 }; + const lastKnown = supervisedConnectionState({ + mode: "quick-connect", + target, + leaseState: "lost", + controlAllowed: false, + deviceNetworkState: "unconfigured", + lastKnown: { + connection_mode: "quick-connect", + target, + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + verified_at: "2026-08-06T11:59:00Z", + }, + }); + const configured = supervisedConnectionState({ + mode: "quick-connect", + target, + leaseState: "configured-unverified", + controlAllowed: false, + }); + const active = supervisedConnectionState({ mode: "quick-connect", target }); + + assert.deepEqual(lifecycle.backendConnectionTopology(lastKnown), { + connectionMode: "quick-connect", + status: "configured-offline", + source: "last-known", + endpoint: "192.168.56.1", + }); + assert.deepEqual(lifecycle.backendConnectionTopology(configured), { + connectionMode: "quick-connect", + status: "configured-unverified", + source: "applied", + endpoint: "192.168.56.1", + }); + assert.deepEqual(lifecycle.backendConnectionTopology(active), { + connectionMode: "quick-connect", + status: "active", + source: "applied", + endpoint: "192.168.56.1", + }); + assert.equal(lifecycle.activeConnectionEndpointLabel(lastKnown), null); + assert.equal( + lifecycle.activeConnectionEndpointLabel(active), + "192.168.56.1", + ); + assert.equal(lifecycle.backendConnectionTopology({}), null); +}); + +test("current BLE topology remains visible offline and supersedes durable history", () => { + const currentOffline = supervisedConnectionState({ + mode: "bridge", + leaseState: "lost", + controlAllowed: false, + hostAvailable: false, + endpointState: "unreachable", + }); + currentOffline.semantic_topology_store = { + status: "available", + configured_offline_evidence: true, + live_connection_authority: false, + reason_code: null, + record: { + schema_version: "missioncore.xgrids-k1-semantic-topology/v1", + revision: 2, + transport_ref: "ble-k1-old", + connection_mode: "quick-connect", + ipv4: "192.168.56.1", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + firmware_version: "3.0.2", + source: "ble-read-only-status", + observed_at_utc: "2026-08-06T11:00:00Z", + }, + }; + + assert.deepEqual(lifecycle.backendConnectionTopology(currentOffline), { + connectionMode: "bridge", + status: "configured-offline", + source: "applied", + endpoint: "192.168.68.50", + }); + assert.equal(lifecycle.backendConnectionTopology(currentOffline, "quick-connect"), null); + assert.equal(lifecycle.isReachableConnectionLease(currentOffline, "bridge"), false); + assert.equal(lifecycle.isConfiguredConnectionLease(currentOffline, "bridge"), true); + assert.equal(lifecycle.hasControlAuthority(currentOffline), false); + assert.equal( + lifecycle.normalizeRuntimePhase({ ...currentOffline, phase: "connected" }), + "configuring", + ); + assert.equal(lifecycle.canonicalDeviceConnectivity(currentOffline), "offline"); +}); + +test("durable semantic topology is configured-offline evidence, never authority", () => { + const durable = { + semantic_topology_store: { + status: "available", + configured_offline_evidence: true, + live_connection_authority: false, + reason_code: null, + record: { + schema_version: "missioncore.xgrids-k1-semantic-topology/v1", + revision: 4, + transport_ref: "ble-k1-001", + connection_mode: "quick-connect", + ipv4: "192.168.56.1", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + firmware_version: "3.0.2", + source: "ble-post-write-status", + observed_at_utc: "2026-08-06T12:00:00Z", + }, + }, + }; + assert.deepEqual(lifecycle.backendConnectionTopology(durable), { + connectionMode: "quick-connect", + status: "configured-offline", + source: "durable", + endpoint: "192.168.56.1", + }); + assert.equal(lifecycle.isConfiguredConnectionLease(durable, "quick-connect"), true); + assert.equal(lifecycle.isReachableConnectionLease(durable, "quick-connect"), false); + assert.equal(lifecycle.hasControlAuthority(durable), false); + assert.equal(lifecycle.normalizeRuntimePhase({ ...durable, phase: "connected" }), "configuring"); + assert.equal(lifecycle.canonicalDeviceConnectivity(durable), "offline"); + + const hostProbed = structuredClone(durable); + hostProbed.configured_endpoint_probe = { + schema_version: "missioncore.xgrids-k1-configured-endpoint-probe/v1", + status: "reachable", + target_source: "durable-semantic-topology", + connection_mode: "quick-connect", + endpoint: "192.168.56.1", + transport_ref: "ble-k1-001", + intent_id: null, + semantic_revision: 4, + host_route_available: true, + host_route_class: "direct", + tcp_reachable: true, + identity_validation: "not-performed", + control_authority_granted: false, + ble_operation_performed: false, + network_mutation_performed: false, + automatic_retry: false, + observed_at: "2026-08-08T12:30:19Z", + reason_code: null, + }; + assert.deepEqual(lifecycle.backendConnectionTopology(hostProbed), { + connectionMode: "quick-connect", + status: "configured-unverified", + source: "durable", + endpoint: "192.168.56.1", + }); + assert.equal(lifecycle.isReachableConnectionLease(hostProbed, "quick-connect"), false); + assert.equal(lifecycle.hasControlAuthority(hostProbed), false); + + for (const status of ["empty", "corrupt"]) { + const unavailable = structuredClone(durable); + unavailable.semantic_topology_store.status = status; + unavailable.semantic_topology_store.record = null; + unavailable.semantic_topology_store.configured_offline_evidence = false; + assert.equal(lifecycle.backendConnectionTopology(unavailable), null); + } +}); + +test("unresolved or corrupt durable mutation state remains a provisioning barrier", () => { + assert.equal(lifecycle.hasUnresolvedNetworkMutation({}), false); + assert.equal(lifecycle.hasUnresolvedNetworkMutation({ network_write_reconciliation: {} }), true); + assert.equal(lifecycle.hasUnresolvedNetworkMutation({ + network_mutation_ledger: { status: "unresolved", mutation_allowed: false }, + }), true); + assert.equal(lifecycle.hasUnresolvedNetworkMutation({ + network_mutation_ledger: { status: "corrupt", mutation_allowed: false }, + }), true); + assert.equal(lifecycle.hasUnresolvedNetworkMutation({ + network_mutation_ledger: { status: "resolved", mutation_allowed: true }, + }), false); +}); + +test("retained BLE context is never presence but can carry one backend-authorized recovery", () => { + const activeQuick = { + current_device_recovery: { + transport_ref: "exact-session-handle", + connection_mode: "quick-connect", + handle_available: true, + handle_retained: true, + advertised_now: false, + gatt_validated_recently: true, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["recover-current-device-network"], + actions: { + "recover-current-device-network": { + allowed: true, + reason_codes: [], + target_source: "retained-current-process", + required_transport_ref: "exact-session-handle", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { + retained_context_is_presence: false, + }, + }, + }; + + assert.equal( + lifecycle.currentDeviceTransportRef(activeQuick), + "exact-session-handle", + ); + assert.deepEqual(lifecycle.retainedBleRecoveryTarget(activeQuick), { + transportRef: "exact-session-handle", + connectionMode: "quick-connect", + gattValidatedRecently: true, + }); + assert.equal( + lifecycle.provisioningCandidateById([], "exact-session-handle"), + null, + ); + assert.equal( + lifecycle.canSubmitProvisioningMutation({ + devices: [], + selectedDeviceId: "exact-session-handle", + credentialsReady: true, + isBusy: false, + }), + false, + ); + assert.equal( + lifecycle.connectionPolicyAllows( + activeQuick, + "recover-current-device-network", + ), + true, + ); + assert.equal( + lifecycle.connectionPolicyDecision( + activeQuick, + "recover-current-device-network", + ).requires_live_gatt_validation, + true, + ); + + const freshlyObserved = { + device_id: "exact-session-handle", + name: "Lixel K1", + connectable: true, + }; + assert.equal( + lifecycle.provisioningCandidateById( + [freshlyObserved], + "exact-session-handle", + ), + freshlyObserved, + ); + assert.equal( + lifecycle.canSubmitProvisioningMutation({ + devices: [freshlyObserved], + selectedDeviceId: "exact-session-handle", + credentialsReady: true, + isBusy: false, + }), + true, + ); + assert.equal(lifecycle.retainedBleRecoveryTarget({ + current_device_recovery: { + ...activeQuick.current_device_recovery, + advertised_now: true, + }, + }), null); +}); + +test("browser refresh never adopts an existing backend K1 session as selection", () => { + const backendSession = { + selected_device_id: "ble-k1-001", + connection_mode: "bridge", + device_session: { + device_session_id: "device-session-001", + device_id: "logical-k1-001", + }, + current_device_recovery: { + transport_ref: "ble-k1-001", + connection_mode: "bridge", + handle_retained: true, + }, + }; + + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget(backendSession, null, "bridge"), + null, + ); + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget( + backendSession, + "another-device", + "bridge", + ), + null, + ); + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget( + backendSession, + "ble-k1-001", + "quick-connect", + ), + null, + ); +}); + +test("a pending local connect cannot bind the old matching backend session", () => { + const oldBackendSession = { + selected_device_id: "ble-k1-001", + connection_mode: "bridge", + device_session: { + device_session_id: "device-session-A", + device_id: "logical-k1-001", + }, + current_device_recovery: { + transport_ref: "ble-k1-001", + connection_mode: "bridge", + handle_retained: true, + }, + }; + + const oldTarget = lifecycle.bleSessionTargetForTransport( + oldBackendSession, + "ble-k1-001", + "bridge", + ); + assert.equal(oldTarget?.key, "device-session-A:bridge:ble-k1-001"); + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget( + oldBackendSession, + null, + "bridge", + ), + null, + "pending is not a successful local intent and grants no bind authority", + ); + assert.equal( + lifecycle.acceptedBleSessionKeyAfterConnect( + oldBackendSession, + "ble-k1-001", + "bridge", + oldTarget.key, + ), + null, + "the session that predated the click is not an accepted result", + ); +}); + +test("a successful local connect binds only the new matching backend session", () => { + const backendSession = { + selected_device_id: "ble-k1-001", + connection_mode: "bridge", + device_session: { + device_session_id: "device-session-B", + device_id: "logical-k1-001", + }, + current_device_recovery: { + transport_ref: "ble-k1-001", + connection_mode: "bridge", + handle_retained: true, + }, + }; + + const acceptedSessionKey = lifecycle.acceptedBleSessionKeyAfterConnect( + backendSession, + "ble-k1-001", + "bridge", + "device-session-A:bridge:ble-k1-001", + ); + assert.equal(acceptedSessionKey, "device-session-B:bridge:ble-k1-001"); + + assert.deepEqual( + lifecycle.locallyInitiatedBleSessionTarget( + backendSession, + "ble-k1-001", + "bridge", + { requiredSessionKey: acceptedSessionKey }, + ), + { + transportRef: "ble-k1-001", + connectionMode: "bridge", + deviceSessionId: "device-session-B", + key: "device-session-B:bridge:ble-k1-001", + }, + ); + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget( + { + ...backendSession, + device_session: { + ...backendSession.device_session, + device_session_id: "device-session-C", + }, + }, + "ble-k1-001", + "bridge", + { requiredSessionKey: acceptedSessionKey }, + ), + null, + "a later session cannot silently replace the accepted session B", + ); + assert.equal( + lifecycle.locallyInitiatedBleSessionTarget( + { ...backendSession, device_session: null }, + "ble-k1-001", + "bridge", + { requiredSessionKey: acceptedSessionKey }, + ), + null, + ); + assert.equal( + lifecycle.acceptedBleSessionKeyAfterConnect( + { ...backendSession, device_session: null }, + "ble-k1-001", + "bridge", + "device-session-A:bridge:ble-k1-001", + ), + null, + "success without an exact returned session key is a clean reset", + ); + assert.equal( + lifecycle.canAdmitProvisioningConnection({ + policyAllowed: true, + targetSource: "fresh-scan", + hasSuccessfulLocalConnect: false, + localPrerequisitesReady: true, + }), + true, + "after the reset a fresh explicit scan selection can connect", + ); +}); + +test("an arbitrary BLE result set never selects or connects a device", () => { + const devices = Array.from({ length: 20 }, (_, index) => ({ + device_id: `ble-device-${String(index + 1).padStart(2, "0")}`, + name: `BLE device ${index + 1}`, + connectable: true, + })); + + assert.equal(lifecycle.provisioningCandidateById(devices, ""), null); + assert.equal( + lifecycle.canSubmitProvisioningMutation({ + devices, + selectedDeviceId: "", + credentialsReady: true, + isBusy: false, + }), + false, + ); +}); + +test("connection policy remains authoritative when historical operations are present", () => { + const state = { + operations: [{ + action: "network.provision", + status: "running", + operation_id: "historical-operation", + }], + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["provision-fresh-device"], + actions: { + "provision-fresh-device": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + }, + }; + + assert.equal( + lifecycle.connectionPolicyAllows(state, "provision-fresh-device"), + true, + ); + assert.equal( + lifecycle.canAdmitProvisioningConnection({ + policyAllowed: true, + targetSource: "fresh-scan", + hasSuccessfulLocalConnect: false, + localPrerequisitesReady: true, + }), + true, + ); +}); + +test("legacy backend recovery evidence keeps exact UUID and mode outside UI admission", () => { + const decision = (allowed, target_source, required_transport_ref, required_connection_mode) => ({ + allowed, + reason_codes: allowed ? [] : ["not-selected"], + target_source, + required_transport_ref, + required_connection_mode, + requires_live_gatt_validation: true, + automatic_retry: false, + }); + const state = { + ble_discovery_generation: 7, + devices: [{ + device_id: "fresh-policy-k1", + name: "Lixel K1", + connectable: true, + }], + network_mutation_ledger: { + status: "unresolved", + mutation_allowed: false, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: [ + "observe-fresh-device-network", + "observe-current-device-network", + "observe-configured-device-network", + ], + actions: { + "observe-fresh-device-network": decision( + true, + "fresh-scan", + "fresh-policy-k1", + "direct-connect", + ), + "observe-current-device-network": decision( + true, + "retained-current-process", + "retained-policy-k1", + "quick-connect", + ), + "observe-configured-device-network": decision( + true, + "durable-configured-state", + "durable-policy-k1", + "bridge", + ), + }, + facts: { retained_context_is_presence: false }, + }, + }; + + assert.deepEqual( + lifecycle.readOnlyConnectionObservationTarget( + state, + "different-browser-selection", + "bridge", + ), + { + action: "observe-fresh-device-network", + deviceId: "fresh-policy-k1", + connectionMode: "direct-connect", + source: "fresh-scan", + serverBound: true, + expectedDiscoveryGeneration: 7, + }, + ); + + state.devices = []; + state.connection_policy.allowed_actions = [ + "observe-current-device-network", + "observe-configured-device-network", + ]; + state.connection_policy.actions["observe-fresh-device-network"] = decision( + false, + "fresh-scan", + "fresh-policy-k1", + "direct-connect", + ); + assert.deepEqual( + lifecycle.readOnlyConnectionObservationTarget( + state, + "stale-browser-selection", + "direct-connect", + ), + { + action: "observe-current-device-network", + deviceId: "retained-policy-k1", + connectionMode: "quick-connect", + source: "retained-current-process", + serverBound: true, + expectedDiscoveryGeneration: null, + }, + ); + assert.deepEqual( + lifecycle.serverBoundAppliedNetworkObservationTarget(state, "quick-connect"), + { + action: "observe-current-device-network", + deviceId: "retained-policy-k1", + connectionMode: "quick-connect", + source: "retained-current-process", + serverBound: true, + expectedDiscoveryGeneration: null, + }, + ); + + state.network_mutation_ledger = { + status: "resolved", + mutation_allowed: true, + }; + state.devices = [{ + device_id: "durable-policy-k1", + name: "Lixel K1", + connectable: true, + }]; + state.connection_policy.allowed_actions = [ + "observe-fresh-device-network", + "observe-configured-device-network", + ]; + state.connection_policy.actions["observe-fresh-device-network"] = decision( + true, + "fresh-scan", + null, + null, + ); + state.connection_policy.actions["observe-current-device-network"] = decision( + false, + "retained-current-process", + "retained-policy-k1", + "quick-connect", + ); + assert.deepEqual( + lifecycle.readOnlyConnectionObservationTarget( + state, + "stale-browser-selection", + "direct-connect", + ), + { + action: "observe-configured-device-network", + deviceId: "durable-policy-k1", + connectionMode: "bridge", + source: "durable-configured-state", + serverBound: true, + expectedDiscoveryGeneration: null, + }, + ); + assert.deepEqual( + lifecycle.serverBoundAppliedNetworkObservationTarget(state, "bridge"), + { + action: "observe-configured-device-network", + deviceId: "durable-policy-k1", + connectionMode: "bridge", + source: "durable-configured-state", + serverBound: true, + expectedDiscoveryGeneration: null, + }, + ); +}); + +test("legacy unresolved backend evidence never falls back to stale browser state", () => { + const state = { + devices: [], + network_mutation_ledger: { + status: "unresolved", + mutation_allowed: false, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["observe-configured-device-network"], + actions: { + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "durable-policy-k1", + required_connection_mode: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + }; + + assert.equal( + lifecycle.readOnlyConnectionObservationTarget( + state, + "stale-browser-k1", + "quick-connect", + ), + null, + ); +}); + +test("local receiver cleanup never targets a replay session's retained acquisition", () => { + assert.deepEqual(lifecycle.localReceiverStopPlan({ + source_mode: "replay", + acquisition: { + acquisition_id: "terminal-live-acquisition", + state: "completed", + cleanup_pending: false, + }, + }), { kind: "compatibility" }); + + assert.deepEqual(lifecycle.localReceiverStopPlan({ + source_mode: "idle", + acquisition: { + acquisition_id: " exact-cleanup-target ", + state: "failed", + cleanup_pending: true, + }, + }), { + kind: "acquisition", + acquisitionId: "exact-cleanup-target", + }); + + assert.deepEqual(lifecycle.localReceiverStopPlan({ + source_mode: "idle", + acquisition: { + acquisition_id: "released-terminal-acquisition", + state: "failed", + cleanup_pending: false, + }, + }), { kind: "compatibility" }); +}); + +test("local receiver policy denial distinguishes proven idle from active cleanup", () => { + assert.equal(lifecycle.isProvenLocalReceiverInactive({ + source_mode: "idle", + acquisition: null, + }), true); + assert.equal(lifecycle.isProvenLocalReceiverInactive({ + source_mode: "idle", + acquisition: { + state: "failed", + cleanup_pending: false, + }, + }), true); + assert.equal(lifecycle.isProvenLocalReceiverInactive({ + source_mode: "idle", + acquisition: { + state: "failed", + cleanup_pending: true, + }, + }), false); + assert.equal(lifecycle.isProvenLocalReceiverInactive({ + source_mode: "live", + acquisition: { + state: "acquiring", + cleanup_pending: false, + }, + }), false); + assert.equal(lifecycle.isProvenLocalReceiverInactive({ + source_mode: "replay", + acquisition: { + state: "completed", + cleanup_pending: false, + }, + }), false); +}); + +test("trusted K1 recovery uses only one exact backend-owned device and mode", () => { + const state = { + physical_command: { + status: "unresolved", + requires_reconciliation: true, + resolved_active_recovery_required: false, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["observe-configured-device-network"], + actions: { + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "fresh-scan", + required_transport_ref: "physical-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "physical-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + semantic_topology_store: { + record: { + transport_ref: "semantic-k1", + connection_mode: "quick-connect", + }, + }, + current_device_recovery: { + transport_ref: "process-k1", + connection_mode: "direct-connect", + }, + }; + + assert.deepEqual(lifecycle.trustedConnectionBinding(state), { + deviceId: "physical-k1", + connectionMode: "bridge", + }); + + delete state.connection_policy.actions["provision-fresh-device"]; + state.physical_command.requires_reconciliation = false; + assert.deepEqual(lifecycle.trustedConnectionBinding(state), { + deviceId: "semantic-k1", + connectionMode: "quick-connect", + }); + + state.semantic_topology_store.record.connection_mode = null; + assert.deepEqual(lifecycle.trustedConnectionBinding(state), { + deviceId: "process-k1", + connectionMode: "direct-connect", + }); + + state.current_device_recovery.connection_mode = null; + assert.equal(lifecycle.trustedConnectionBinding(state), null); +}); + +test("only an unresolved durable STOP blocks connection controls during backend cleanup", () => { + const unresolvedStop = { + source_mode: "live", + acquisition: { + state: "stopping", + cleanup_pending: true, + }, + application_control_session: { + state: "awaiting-standby-confirmation", + physical_command: { + status: "unresolved", + requires_reconciliation: true, + record: { + action: "stop", + stage: "requested", + resolution: null, + }, + }, + }, + }; + assert.equal(lifecycle.isPhysicalStopRecoverySettling(unresolvedStop), true); + + const boundedTimeout = structuredClone(unresolvedStop); + boundedTimeout.source_mode = "idle"; + boundedTimeout.acquisition.state = "failed"; + boundedTimeout.acquisition.cleanup_pending = false; + boundedTimeout.operations = [{ + action: "acquisition.stop", + status: "timed_out", + }]; + // The durable physical ledger remains intentionally unresolved, but all + // local work is released. The connection screen must leave the STOP loader + // and may start the separate bounded connection recovery path. + assert.equal(lifecycle.isPhysicalStopRecoverySettling(boundedTimeout), false); + + const backendObservedStandby = structuredClone(unresolvedStop); + backendObservedStandby.application_control_session.state = "idle"; + backendObservedStandby.application_control_session.physical_command.status = "resolved"; + backendObservedStandby.application_control_session.physical_command.requires_reconciliation = false; + backendObservedStandby.application_control_session.physical_command.record.stage = "resolved"; + backendObservedStandby.application_control_session.physical_command.record.resolution = + "stop-standby-observed"; + assert.equal( + lifecycle.isPhysicalStopRecoverySettling(backendObservedStandby), + false, + ); + + const unresolvedStart = structuredClone(unresolvedStop); + unresolvedStart.application_control_session.physical_command.record.action = "start"; + assert.equal(lifecycle.isPhysicalStopRecoverySettling(unresolvedStart), false); +}); + +test("physical recovery stays pinned to the original K1 across a multi-device BLE scan", () => { + const state = { + physical_command: { + status: "unresolved", + requires_reconciliation: true, + resolved_active_recovery_required: false, + }, + ble_discovery_generation: 9, + devices: [ + { device_id: "nearby-other-k1", name: "Nearby K1", connectable: true }, + { device_id: "original-k1", name: "Original K1", connectable: true }, + ], + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["scan-ble", "observe-fresh-device-network"], + actions: { + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-device-already-active"], + target_source: "fresh-scan", + required_transport_ref: "original-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "observe-fresh-device-network": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: "original-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + }; + + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(state), true); + assert.deepEqual(lifecycle.readOnlyPhysicalRecoveryBinding(state), { + deviceId: "original-k1", + connectionMode: "bridge", + }); + assert.deepEqual( + lifecycle.readOnlyConnectionObservationTarget( + state, + "nearby-other-k1", + "quick-connect", + ), + { + action: "observe-fresh-device-network", + deviceId: "original-k1", + connectionMode: "bridge", + source: "fresh-scan", + serverBound: true, + expectedDiscoveryGeneration: 9, + }, + ); + + state.devices = [{ device_id: "nearby-other-k1", name: "Nearby K1", connectable: true }]; + assert.equal( + lifecycle.readOnlyConnectionObservationTarget( + state, + "nearby-other-k1", + "quick-connect", + ), + null, + "another nearby K1 must not become a fallback recovery target", + ); +}); + +test("READY-classified STOP exits recovery UI while START follows backend authority", () => { + const readyPhysical = { + status: "resolved", + reason_code: null, + requires_reconciliation: false, + resolved_active_recovery_required: false, + observed_session_state: "ready", + record: { + action: "stop", + stage: "resolved", + resolution: "not-dispatched", + reconciled_physical_state: "standby", + }, + }; + const readySuccessor = { + physical_command: readyPhysical, + application_control_session: { + physical_command: readyPhysical, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + allowed_actions: ["start-acquisition"], + actions: { + "start-acquisition": { + allowed: true, + reason_codes: [], + target_source: "durable-physical-command", + required_transport_ref: "original-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "fresh-scan", + required_transport_ref: "original-k1", + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + connection_lifecycle: { + schema_version: "missioncore.xgrids-k1-connection-lifecycle/v1", + mode_selection: { + allowed: false, + reason_codes: ["connection-mode-selection-physical-state-unsafe"], + automatic_retry: false, + }, + allowed_actions: ["start-acquisition"], + }, + }; + + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(readySuccessor), false); + assert.equal(lifecycle.connectionPolicyAllows(readySuccessor, "start-acquisition"), true); + assert.equal( + lifecycle.connectionPolicyAllows(readySuccessor, "provision-fresh-device"), + false, + "the exact successor binding remains pinned against network mutation", + ); + assert.equal( + lifecycle.canSelectConnectionMode(readySuccessor), + false, + "the exact successor binding remains pinned against mode changes", + ); + + const unresolved = structuredClone(readySuccessor); + unresolved.application_control_session.physical_command.requires_reconciliation = true; + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(unresolved), true); + + const scanOver = structuredClone(readySuccessor); + scanOver.application_control_session.physical_command.requires_reconciliation = true; + scanOver.application_control_session.physical_command.resolved_scan_over_recovery_required = true; + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(scanOver), true); + + const reconciledActive = structuredClone(readySuccessor); + reconciledActive.application_control_session.physical_command.resolved_active_recovery_required = true; + reconciledActive.application_control_session.physical_command.observed_session_state = "scanning"; + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(reconciledActive), true); + + const reopened = structuredClone(readySuccessor); + reopened.application_control_session.physical_command.resolved_active_recovery_required = true; + reopened.application_control_session.physical_command.reopened_physical_state_recovery_required = true; + assert.equal(lifecycle.requiresReadOnlyPhysicalRecovery(reopened), true); +}); + +test("only a newer authoritative reachable lease resolves connection errors", () => { + const error = { + action: "connect", + runtimeId: "runtime-a", + leaseGeneration: 4, + }; + const reachableBridge = supervisedConnectionState({ generation: 5 }); + + assert.equal( + lifecycle.authoritativeReachableLeaseSupersedesError(error, reachableBridge), + true, + ); + assert.equal( + lifecycle.authoritativeReachableLeaseSupersedesError( + { ...error, action: "scan" }, + reachableBridge, + ), + false, + ); + assert.equal( + lifecycle.authoritativeReachableLeaseSupersedesError( + error, + supervisedConnectionState({ generation: 4 }), + ), + false, + ); + assert.equal( + lifecycle.authoritativeReachableLeaseSupersedesError( + error, + { ...reachableBridge, network_write_reconciliation: {} }, + ), + false, + ); + assert.equal( + lifecycle.authoritativeReachableLeaseSupersedesError( + error, + { ...reachableBridge, snapshot_runtime_id: "runtime-b" }, + ), + false, + ); +}); + +test("connection-mode reset is explicit and CAS-fenced before the next flow", async () => { + const pipelineSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx", + import.meta.url, + ), + "utf8", + ); + const connectionSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx", + import.meta.url, + ), + "utf8", + ); + + assert.doesNotMatch(pipelineSource, /nextReachableConnectionModeSynchronization/); + assert.doesNotMatch(pipelineSource, /synchronizedLeaseKeyRef/); + assert.doesNotMatch(pipelineSource, /setConnectionMode\(/); + assert.match(connectionSource, /state\.desired_connection_mode/); + assert.doesNotMatch(connectionSource, /selectConnectionMode\(\{/); + assert.match(connectionSource, /desiredModeLocallyDirty\.current = mode !== state\?\.desired_connection_mode/); + assert.match(connectionSource, /setDesiredConnectionMode\(mode\)/); + const localModeChange = pipelineSource.slice( + pipelineSource.indexOf("const changeDesiredConnectionMode"), + pipelineSource.indexOf("const selectFreshDevice"), + ); + const explicitModeCommit = pipelineSource.slice( + pipelineSource.indexOf("const commitDesiredModeForExplicitAction"), + pipelineSource.indexOf("const currentPreparingReconfigurationRequest"), + ); + assert.match(localModeChange, /await selectConnectionMode\(\{/); + assert.match(localModeChange, /expected_revision: expectedRevision as number/); + assert.match(localModeChange, /reset_scenario: true/); + assert.match(localModeChange, /reset_id: resetId/); + assert.doesNotMatch( + localModeChange, + /scanWithResult\(|connect\(|verifyConnection\(|prepareConnection/, + ); + assert.match(explicitModeCommit, /selectConnectionMode\(\{/); + assert.match(explicitModeCommit, /expected_revision: expectedRevision as number/); + assert.match( + pipelineSource, + /const repeatDeviceScan[\s\S]*?await commitDesiredModeForExplicitAction\(\)/, + ); + assert.match( + pipelineSource, + /const submitConnect[\s\S]*?await commitDesiredModeForExplicitAction\(\)/, + ); +}); + +test("connection-mode selector follows the authoritative backend admission", () => { + const ready = supervisedConnectionState({ connectionReady: true }); + assert.equal(lifecycle.canSelectConnectionMode(ready), true); + + const prepared = structuredClone(ready); + prepared.acquisition = { + acquisition_id: "acq-prepared", + state: "prepared", + project_name: "TEST001", + }; + assert.equal(lifecycle.canSelectConnectionMode(prepared), true); + + for (const reasonCode of [ + "connection-mode-selection-control-state-unsafe", + "connection-mode-selection-acquisition-active", + "connection-mode-selection-physical-state-unsafe", + ]) { + const blocked = structuredClone(ready); + blocked.connection_lifecycle.mode_selection = { + allowed: false, + reason_codes: [reasonCode], + automatic_retry: false, + }; + blocked.connection_lifecycle.allowed_actions = ["start-acquisition"]; + assert.equal(lifecycle.canSelectConnectionMode(blocked), false); + } + + const drifted = structuredClone(ready); + drifted.connection_lifecycle.allowed_actions = ["start-acquisition"]; + assert.equal(lifecycle.canSelectConnectionMode(drifted), false); + assert.equal(lifecycle.canSelectConnectionMode({}), false); +}); + +test("prepared project survives a mode draft and its cancellation", () => { + assert.equal(projectName.shouldHydratePreparedProject({ + acquisitionId: "acq-prepared", + hydratedAcquisitionId: null, + modeSwitchRequired: false, + }), true); + let displayedProject = projectName.projectNameAfterConnectionModeSelection("TEST001"); + assert.equal(displayedProject, "TEST001"); + assert.equal(projectName.validateProjectName(displayedProject).error, null); + + assert.equal(projectName.shouldHydratePreparedProject({ + acquisitionId: "acq-prepared", + hydratedAcquisitionId: "acq-prepared", + modeSwitchRequired: true, + }), false); + displayedProject = projectName.projectNameAfterConnectionModeSelection("TEST001"); + assert.equal(displayedProject, "TEST001"); + assert.equal(projectName.validateProjectName(displayedProject).error, null); + + assert.equal(projectName.shouldHydratePreparedProject({ + acquisitionId: "acq-prepared", + hydratedAcquisitionId: "acq-prepared", + modeSwitchRequired: false, + }), true); + + assert.equal(projectName.projectNameAfterConnectionModeSelection(null), ""); +}); + +test("one in-flight provisioning call keeps its idempotency key internally", () => { let created = 0; const createUuid = () => { created += 1; @@ -711,13 +3200,184 @@ test("provisioning intent keeps one idempotency key and exposes unsafe outcomes" const repeated = lifecycle.provisioningIntentKey(first, createUuid); const failedOperation = { status: "failed", - error: { safe_to_retry: false }, + error: { safe_to_retry: false, side_effect_status: "unknown" }, + }; + const safePreWriteFailure = { + status: "failed", + error: { safe_to_retry: true, side_effect_status: "none" }, }; assert.equal(first, "network-provision:11111111-1111-4111-8111-111111111111"); assert.equal(repeated, first); assert.equal(created, 1); assert.equal(lifecycle.operationNeedsReconciliation(failedOperation), true); + assert.equal( + lifecycle.operationAllowsFreshProvisioningIntent(safePreWriteFailure), + true, + ); + assert.equal( + lifecycle.operationAllowsFreshProvisioningIntent(failedOperation), + false, + ); +}); + +test("read-only verification releases only a durably resolved matching write fence", () => { + const fence = { + operation_id: "op-ambiguous", + transport_ref: "k1-a", + }; + const before = { + snapshot_runtime_id: "runtime-a", + network_write_reconciliation: fence, + }; + const resolvedLedger = { + status: "resolved", + mutation_allowed: true, + operation_id: "op-ambiguous", + stage: "resolved", + resolution: "target-observed", + }; + + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + snapshot_runtime_id: "runtime-restarted", + network_write_reconciliation: null, + network_mutation_ledger: resolvedLedger, + }, + "k1-a", + ), + true, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + network_write_reconciliation: null, + network_mutation_ledger: { + status: "empty", + mutation_allowed: true, + operation_id: null, + stage: null, + resolution: null, + }, + }, + "k1-a", + ), + true, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation(before, {}, "k1-a"), + false, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + network_write_reconciliation: fence, + network_mutation_ledger: { + ...resolvedLedger, + status: "unresolved", + mutation_allowed: false, + stage: "observing", + resolution: null, + }, + }, + "k1-a", + ), + false, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + network_write_reconciliation: null, + network_mutation_ledger: resolvedLedger, + }, + "k1-b", + ), + false, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + network_write_reconciliation: null, + network_mutation_ledger: { + ...resolvedLedger, + operation_id: "op-different", + }, + }, + "k1-a", + ), + false, + ); + assert.equal( + lifecycle.readOnlyVerificationClearedReconciliation( + before, + { + network_write_reconciliation: null, + network_mutation_ledger: { + ...resolvedLedger, + resolution: null, + }, + }, + "k1-a", + ), + false, + ); +}); + +test("each terminal explicit provisioning click starts a fresh operation identity", async () => { + const hookSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), + "utf8", + ); + const pipelineSource = await readFile( + new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx", + import.meta.url, + ), + "utf8", + ); + const connectRecovery = hookSource.slice( + hookSource.indexOf("const connect = useCallback"), + hookSource.indexOf("const verifyConnection = useCallback"), + ); + const submitConnect = pipelineSource.slice( + pipelineSource.indexOf("const submitConnect = async"), + pipelineSource.indexOf("return (", pipelineSource.indexOf("const submitConnect = async")), + ); + + assert.match( + connectRecovery, + /failedOperation\?\.status === "succeeded"[\s\S]*?!exactAppliedNetworkIntentCompleted\([\s\S]*?&& !hasExactConnectionReady/, + ); + assert.match( + connectRecovery, + /const exactNetworkIntentCompleted = exactAppliedNetworkIntentCompleted\([\s\S]*?nextState = acceptSuccessfulConnectState\(nextState\)/, + ); + assert.match(connectRecovery, /acceptedBleSessionKeyAfterConnect\(/); + assert.match( + connectRecovery, + /acceptedSessionKey:\s*networkIntentCompleted \? acceptedSessionKey : null/, + ); + assert.match(submitConnect, /provisioningIntentKey\(null\)/); + assert.doesNotMatch(submitConnect, /provisioningIntentRef/); + assert.match( + submitConnect, + /const freshStartAllowed = result\.intentDisposition === "release"/, + ); + assert.match( + submitConnect, + /setExplicitProvisioningDraft\(null\);[\s\S]*?setPassword\(""\);[\s\S]*?await connect\(/, + ); + assert.match(submitConnect, /setSelectedDeviceSnapshot\(null\)/); + assert.doesNotMatch(pipelineSource, /Проверить K1 без записи/); }); test("device mutations send explicit nested compatibility attestation", async () => { @@ -739,12 +3399,15 @@ test("device mutations send explicit nested compatibility attestation", async () try { await xgridsK1Api.connect({ + expected_snapshot_runtime_id: "snapshot-runtime-test", device_id: "ble-device", ssid: "lab-network", password: syntheticCredential, connection_mode: "bridge", compatibility_attestation: attestation, idempotency_key: "network-provision:test", + expected_mode_revision: 4, + expected_discovery_generation: 9, }); await xgridsK1Api.prepareAcquisition({ project_name: "Mission 01", @@ -761,11 +3424,17 @@ test("device mutations send explicit nested compatibility attestation", async () const prepare = JSON.parse(calls[1].init.body); assert.deepEqual(provisioning.input.compatibility_attestation, attestation); assert.equal(provisioning.input.idempotency_key, "network-provision:test"); + assert.equal(provisioning.input.expected_mode_revision, 4); + assert.equal(provisioning.input.expected_discovery_generation, 9); + assert.equal( + provisioning.input.expected_snapshot_runtime_id, + "snapshot-runtime-test", + ); assert.deepEqual(prepare.input.compatibility_attestation, attestation); assert.equal(prepare.input.project_name, "Mission 01"); }); -test("connection verification supports refresh and explicit read-only adoption", async () => { +test("endpoint probing is separate from BLE refresh and read-only adoption", async () => { const originalFetch = globalThis.fetch; const calls = []; globalThis.fetch = async (path, init) => { @@ -782,30 +3451,451 @@ test("connection verification supports refresh and explicit read-only adoption", }; try { - await xgridsK1Api.verifyConnection(); await xgridsK1Api.verifyConnection({ + expected_snapshot_runtime_id: "snapshot-runtime-test", device_id: "fresh-ble-device", + source: "fresh-scan", compatibility_attestation: attestation, + expected_discovery_generation: 11, + }); + await xgridsK1Api.verifyConnection({ + expected_snapshot_runtime_id: "snapshot-runtime-test", + device_id: "durable-ble-device", + source: "durable-configured-state", + compatibility_attestation: attestation, + }); + await xgridsK1Api.probeConfiguredEndpoint({ + expected_snapshot_runtime_id: "snapshot-runtime-test", }); } finally { globalThis.fetch = originalFetch; } - assert.equal(calls.length, 2); + assert.equal(calls.length, 3); assert.match(String(calls[0].path), /actions\/connection\.verify$/); - assert.match(String(calls[1].path), /actions\/connection\.verify$/); - assert.deepEqual(JSON.parse(calls[0].init.body), { input: {} }); - const adoption = JSON.parse(calls[1].init.body).input; + const adoption = JSON.parse(calls[0].init.body).input; assert.deepEqual(adoption, { + expected_snapshot_runtime_id: "snapshot-runtime-test", device_id: "fresh-ble-device", + source: "fresh-scan", compatibility_attestation: attestation, + expected_discovery_generation: 11, }); assert.equal("ssid" in adoption, false); assert.equal("password" in adoption, false); assert.equal("connection_mode" in adoption, false); + const durableAdoption = JSON.parse(calls[1].init.body).input; + assert.deepEqual(durableAdoption, { + expected_snapshot_runtime_id: "snapshot-runtime-test", + device_id: "durable-ble-device", + source: "durable-configured-state", + compatibility_attestation: attestation, + }); + assert.equal("expected_discovery_generation" in durableAdoption, false); + assert.match(String(calls[2].path), /actions\/connection\.endpoint-probe$/); + assert.deepEqual(JSON.parse(calls[2].init.body), { + input: { expected_snapshot_runtime_id: "snapshot-runtime-test" }, + }); }); -test("rejected network-profile writes use safe operator copy", () => { +test("connection verification accepts only the backend literal status and lease contract", () => { + const statuses = [ + "not-probed", + "device-network-applied", + "device-network-applied-host-failed", + "adopted", + "host-route-mismatch", + "endpoint-unreachable", + "tcp-reachable-device-info-unverified", + "reachable", + "recovered", + "control-transport-lost", + "unreachable", + ]; + const leaseStates = ["disconnected", "configured-unverified", "reachable"]; + assert.deepEqual([...XGRIDS_CONNECTION_VERIFICATION_STATUSES], statuses); + assert.deepEqual([...XGRIDS_CONNECTION_VERIFICATION_LEASE_STATES], leaseStates); + + const exact = { + status: "not-probed", + lease_state: "disconnected", + lease_generation: 0, + supervisor_revision: 1, + endpoint_validation: "not-performed", + network_reachability: "unknown", + observed_at: null, + }; + for (const status of statuses) { + assert.equal(isXgridsConnectionVerification({ ...exact, status }), true, status); + } + for (const lease_state of leaseStates) { + assert.equal( + isXgridsConnectionVerification({ ...exact, lease_state }), + true, + lease_state, + ); + } + + for (const malformed of [ + { ...exact, status: "configured" }, + { ...exact, lease_state: "lost" }, + { ...exact, lease_generation: -1 }, + { ...exact, supervisor_revision: 1.25 }, + { ...exact, network_reachability: "degraded" }, + { ...exact, observed_at: 123 }, + Object.fromEntries(Object.entries(exact).filter(([key]) => key !== "status")), + Object.fromEntries(Object.entries(exact).filter(([key]) => key !== "lease_state")), + ]) { + assert.equal(isXgridsConnectionVerification(malformed), false); + } +}); + +test("connection policy accepts the exact restart-recovery actions, sources and modes", () => { + assert.deepEqual([...XGRIDS_CONNECTION_POLICY_ACTIONS], [ + "scan-ble", + "provision-fresh-device", + "prepare-select-device", + "prepare-change-network", + "cancel-reconfiguration", + "recover-current-device-network", + "observe-fresh-device-network", + "observe-current-device-network", + "observe-configured-device-network", + "inspect-configured-endpoint", + "inspect-host-network", + "probe-endpoint", + "verify-control-device-info", + "start-acquisition", + "stop-acquisition", + "stop-local-receiver", + "retire-unavailable-physical-target", + "acknowledge-data-loss", + ]); + assert.deepEqual([...XGRIDS_CONNECTION_POLICY_TARGET_SOURCES], [ + "none", + "fresh-scan", + "retained-current-process", + "durable-configured-state", + "configured-topology", + "connection-supervisor", + "local-runtime", + "local-prestart-handoff", + "local-reconfiguration-intent", + "durable-physical-command", + ]); + + const retainedDecision = { + allowed: true, + reason_codes: [], + target_source: "retained-current-process", + required_transport_ref: "9AE978F2-37A8-4B4B-9BCD-BD7010BB20D1", + required_connection_mode: "quick-connect", + requires_live_gatt_validation: true, + automatic_retry: false, + }; + const durableDecision = { + ...retainedDecision, + target_source: "durable-configured-state", + required_transport_ref: "0CA8AB68-E37B-46A2-B09F-C34B5C49428C", + required_connection_mode: "bridge", + }; + const policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + supervisor_revision: 17, + network_ledger_revision: 4, + recommended_action: "observe-current-device-network", + allowed_actions: [ + "observe-current-device-network", + "observe-configured-device-network", + ], + actions: { + "observe-current-device-network": retainedDecision, + "observe-configured-device-network": durableDecision, + }, + facts: { retained_context_is_presence: false }, + }; + + assert.equal(isXgridsConnectionPolicyDecision(retainedDecision), true); + assert.equal(isXgridsConnectionPolicy(policy), true); + const retirementDecision = { + allowed: true, + reason_codes: [], + target_source: "durable-physical-command", + required_transport_ref: "0CA8AB68-E37B-46A2-B09F-C34B5C49428C", + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + physical_command_allowed: false, + physical_outcome: "unknown", + device_write_performed: false, + automatic_retry: false, + }; + assert.equal(isXgridsConnectionPolicyDecision(retirementDecision), true); + assert.equal(isXgridsConnectionPolicy({ + ...policy, + recommended_action: "retire-unavailable-physical-target", + allowed_actions: ["retire-unavailable-physical-target"], + actions: { + "retire-unavailable-physical-target": retirementDecision, + }, + }), true); + for (const malformedRetirement of [ + { ...retirementDecision, target_source: "durable-configured-state" }, + { ...retirementDecision, physical_command_allowed: undefined }, + { ...retirementDecision, physical_outcome: undefined }, + { ...retirementDecision, device_write_performed: undefined }, + { ...retirementDecision, requires_live_gatt_validation: true }, + ]) { + assert.equal(isXgridsConnectionPolicy({ + ...policy, + recommended_action: "retire-unavailable-physical-target", + allowed_actions: ["retire-unavailable-physical-target"], + actions: { + "retire-unavailable-physical-target": malformedRetirement, + }, + }), false); + } + assert.equal( + isXgridsConnectionPolicyDecision({ + ...retainedDecision, + required_connection_mode: undefined, + }), + true, + "older decisions may omit the optional mode field", + ); + + for (const malformed of [ + { ...retainedDecision, target_source: "server-cache" }, + { ...retainedDecision, required_connection_mode: "automatic" }, + { ...retainedDecision, automatic_retry: true }, + ]) { + assert.equal(isXgridsConnectionPolicyDecision(malformed), false); + } + assert.equal(isXgridsConnectionPolicy({ + ...policy, + actions: { + ...policy.actions, + "observe-current-device-network": { + ...retainedDecision, + required_connection_mode: undefined, + }, + }, + }), false, "an allowed retained recovery must pin its mode"); + assert.equal(isXgridsConnectionPolicy({ + ...policy, + actions: { + ...policy.actions, + "observe-configured-device-network": { + ...durableDecision, + target_source: "configured-topology", + }, + }, + }), false, "durable recovery cannot drift to an endpoint-only source"); +}); + +test("connection reconfiguration accepts only the exact resumable v1 projection", () => { + const exact = { + schema_version: "missioncore.xgrids-k1-connection-reconfiguration/v1", + revision: 4, + intent_id: "connection-reconfigure-001", + intent: "change-network", + status: "fresh-scan-completed", + required_transport_ref: "BLE-DEVICE-001", + required_connection_mode: "bridge", + minimum_discovery_generation: 10, + fresh_discovery_generation: 10, + required_transport_observed: true, + prepared_at: "2026-08-10T12:00:00Z", + automatic_retry: false, + }; + assert.equal(isXgridsConnectionReconfiguration(exact), true); + assert.equal(isXgridsConnectionReconfiguration({ + ...exact, + status: "idle", + intent: null, + intent_id: null, + required_transport_ref: null, + required_connection_mode: null, + minimum_discovery_generation: null, + fresh_discovery_generation: null, + required_transport_observed: null, + prepared_at: null, + }), true); + for (const malformed of [ + { ...exact, revision: -1 }, + { ...exact, intent: "cancel" }, + { ...exact, status: "scanning" }, + { ...exact, required_connection_mode: "bluetooth" }, + { ...exact, automatic_retry: true }, + { ...exact, intent_id: null }, + ]) { + assert.equal(isXgridsConnectionReconfiguration(malformed), false); + } +}); + +test("connection attempt phase distinguishes no write from an unknown write outcome", async () => { + assert.deepEqual([...XGRIDS_CONNECTION_ATTEMPT_PHASES], [ + "network_applied", + "network_not_applied", + "network_outcome_unknown", + ]); + for (const phase of XGRIDS_CONNECTION_ATTEMPT_PHASES) { + assert.equal(isXgridsConnectionAttemptPhase(phase), true, phase); + } + for (const phase of [ + "network_write_failed", + "network_not_confirmed", + "outcome_unknown", + null, + ]) { + assert.equal(isXgridsConnectionAttemptPhase(phase), false, String(phase)); + } + + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ + state: { + source_mode: "idle", + connection_attempt: { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + phase: "network_outcome_unknown", + automatic_retry: false, + }, + }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + try { + const state = await xgridsK1Api.getState(); + assert.equal(state.connection_attempt.phase, "network_outcome_unknown"); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("state API rejects an unreviewed connection attempt phase", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ + state: { + source_mode: "idle", + connection_attempt: { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + phase: "network_write_failed", + automatic_retry: false, + }, + }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + try { + await assert.rejects( + xgridsK1Api.getState(), + (error) => error instanceof ApiError + && error.message === "Локальный сервер вернул некорректное состояние.", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("state API rejects a drifted connection verification object at runtime", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ + state: { + source_mode: "idle", + connection_verification: { + status: "configured", + lease_state: "lost", + lease_generation: 1, + supervisor_revision: 2, + endpoint_validation: "not-performed", + network_reachability: "degraded", + observed_at: null, + }, + }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + try { + await assert.rejects( + xgridsK1Api.getState(), + (error) => error instanceof ApiError + && error.message === "Локальный сервер вернул некорректное состояние.", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("state API rejects a drifted restart-recovery policy at runtime", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ + state: { + source_mode: "idle", + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + supervisor_revision: 5, + network_ledger_revision: 3, + recommended_action: "observe-configured-device-network", + allowed_actions: ["observe-configured-device-network"], + actions: { + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "exact-k1", + required_connection_mode: "automatic", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + facts: { retained_context_is_presence: false }, + }, + }, + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + + try { + await assert.rejects( + xgridsK1Api.getState(), + (error) => error instanceof ApiError + && error.message === "Локальный сервер вернул некорректное состояние.", + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("host diagnostic guard rejects every unreviewed literal dimension", () => { + const exact = { + schema_version: "missioncore.host-failure-diagnostic/v1", + code: "host.keychain.interaction-required", + domain: "keychain", + impact: "control", + operator_action: "unlock-or-authorize-keychain", + automatic_retry: false, + redacted: true, + }; + assert.equal(isXgridsHostFailureDiagnostic(exact), true); + for (const malformed of [ + { ...exact, code: "PermissionError: private" }, + { ...exact, domain: "python-runtime" }, + { ...exact, impact: "unknown-impact" }, + { ...exact, operator_action: "run-private-shell-command" }, + { ...exact, automatic_retry: true }, + { ...exact, redacted: false }, + ]) { + assert.equal(isXgridsHostFailureDiagnostic(malformed), false); + } +}); + +test("failed network-profile writes close the UI session with a fresh explicit next step", () => { const message = networkProvisionFailureMessage({ status: "failed", error: { code: "BleakGATTProtocolError" }, @@ -813,36 +3903,237 @@ test("rejected network-profile writes use safe operator copy", () => { assert.equal( message, - "Сканер отклонил запись сетевого профиля. Результат изменения сети неизвестен; автоматический повтор запрещён. Проверьте текущее состояние K1 или подхватите существующее подключение без изменения настроек Wi‑Fi.", + "Bluetooth-периферия завершила сетевую операцию ошибкой. Команда могла быть принята K1; итог текущей попытки не подтверждён. Автоматический повтор команды K1 не отправлялся. Сессия подключения в интерфейсе сброшена. Выполните новый поиск Bluetooth, выберите K1 и запустите подключение ещё раз.", ); assert.doesNotMatch(message, /Bleak|GATT|ATT/i); + + const attMessage = networkProvisionFailureMessage({ + action: "network.provision", + status: "failed", + error: { + code: "BleakGATTProtocolError", + ble_att_error_code: 4, + ble_att_error_name: "INVALID_PDU", + }, + }); + assert.match(attMessage, /ATT 4 INVALID_PDU/); + assert.match(attMessage, /Автоматический повтор команды K1 не отправлялся/); + assert.match(attMessage, /Сессия подключения в интерфейсе сброшена/); + assert.match(attMessage, /новый поиск Bluetooth, выберите K1/); + + const preWriteAttMessage = networkProvisionFailureMessage({ + action: "network.provision", + status: "failed", + error: { + code: "BleakGATTProtocolError", + operation_stage: "baseline-read", + device_write_attempted: false, + side_effect_status: "none", + safe_to_retry: true, + ble_att_error_code: 4, + ble_att_error_name: "INVALID_PDU", + }, + }); + assert.match(preWriteAttMessage, /до команды изменения сети/); + assert.match(preWriteAttMessage, /Запись сетевого профиля не выполнялась/); + assert.doesNotMatch(preWriteAttMessage, /результат изменения сети неизвестен/i); }); -test("host Wi-Fi helper failures do not fabricate a missing-password diagnosis", () => { +test("post-dispatch ambiguity resets the UI session without an automatic retry", () => { + const message = networkProvisionFailureMessage({ + action: "network.provision", + status: "failed", + error: { + code: "network-provision-target-not-distinguishable-from-baseline", + device_write_attempted: true, + side_effect_status: "unknown", + safe_to_retry: false, + }, + }); + + assert.equal( + message, + "После BLE-команды K1 вернул сетевой статус, неотличимый от исходного; итог текущей попытки подключения не подтверждён. Автоматический повтор команды K1 не отправлялся. Сессия подключения в интерфейсе сброшена. Выполните новый поиск Bluetooth, выберите K1 и запустите подключение ещё раз.", + ); + assert.doesNotMatch(message, /ручн|read-only|защитный барьер/i); +}); + +test("Bluetooth scan failures explain whether a device command was sent", () => { + const busy = discoveryScanFailureMessage({ + action: "discovery.scan", + status: "failed", + error: { code: "ble-runtime-busy" }, + }); + const cleanupPending = discoveryScanFailureMessage({ + action: "discovery.scan", + status: "failed", + error: { code: "ble-runtime-cleanup-pending" }, + }); + const timedOut = discoveryScanFailureMessage({ + action: "discovery.scan", + status: "failed", + error: { code: "ble-discovery-timeout" }, + }); + + assert.match(busy, /занят другой локальной операцией/); + assert.match(cleanupPending, /подтверждает отключение/); + assert.match(timedOut, /принудительно остановлен/); + assert.match(timedOut, /Команды K1 не отправлялись/); +}); + +test("connection recovery exposes safe operator-facing failure classes", () => { + const addressUnavailable = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "connection-verify-address-unavailable", + side_effect_status: "none", + safe_to_retry: true, + }, + }); + const missingAdvertisement = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { code: "connection-verify-device-not-rediscovered" }, + }); + const statusReadFailed = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "connection-verify-status-read-failed", + operation_stage: "exact-uuid-scan", + side_effect_status: "none", + }, + }); + const exactUuidScanTimedOut = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "connection-verify-exact-uuid-scan-timeout", + operation_stage: "exact-uuid-scan", + side_effect_status: "none", + }, + }); + const indistinguishableFromBaseline = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "connection-verify-target-not-distinguishable-from-baseline", + side_effect_status: "none", + safe_to_retry: false, + }, + }); + const staleEndpoint = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "connection-verify-mqtt-unreachable", + side_effect_status: "none", + safe_to_retry: true, + }, + }); + const bindingChanged = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "application-connection-binding-lost", + side_effect_status: "none", + safe_to_retry: true, + }, + }); + const physicalProofTimedOut = connectionVerificationFailureMessage({ + action: "connection.verify", + status: "failed", + error: { + code: "physical-command-reconciliation-proof-timeout", + side_effect_status: "none", + safe_to_retry: true, + }, + }); + + assert.match(addressUnavailable, /K1 ответил/); + assert.match(addressUnavailable, /настройки устройства не менялись/); + assert.match(missingAdvertisement, /Mission Core не получил объявление/); + assert.match(missingAdvertisement, /Команды K1 не отправлялись/); + assert.doesNotMatch(missingAdvertisement, /питани|перезапуст|не работает|пропал/i); + assert.match(statusReadFailed, /Mission Core не завершил/); + assert.match(statusReadFailed, /команды не отправлялись/); + assert.doesNotMatch(statusReadFailed, /питани|перезапуст|не работает|K1 не ответил/i); + assert.match(exactUuidScanTimedOut, /точного сохранённого CoreBluetooth UUID/); + assert.match(exactUuidScanTimedOut, /не является выводом о состоянии устройства/); + assert.doesNotMatch(exactUuidScanTimedOut, /питани|перезапуст|не работает|K1 не ответил/i); + assert.match(staleEndpoint, /можно заново применить настройки общей сети/); + assert.match(staleEndpoint, /Команда Wi-Fi не отправлялась/); + assert.doesNotMatch(staleEndpoint, /новый поиск|read-only/i); + assert.match(bindingChanged, /нажмите «Подключиться заново»/); + assert.doesNotMatch(bindingChanged, /поиск Bluetooth/i); + assert.match(physicalProofTimedOut, /START и STOP не отправлялись/); + assert.match(physicalProofTimedOut, /нажмите «Подключиться заново»/); + assert.doesNotMatch(physicalProofTimedOut, /поиск Bluetooth/i); + assert.equal( + indistinguishableFromBaseline, + "K1 ответил, но приложение не смогло подтвердить, что прежние настройки сети были применены. Автоматического повтора и новой записи не было.", + ); + assert.doesNotMatch(indistinguishableFromBaseline, /поиск Bluetooth|повторите запись/i); +}); + +test("scan and verify errors correlate only with the requested operation id", () => { + const state = { + operations: [ + { + operation_id: "op-requested", + action: "discovery.scan", + status: "failed", + error: { code: "ble-discovery-timeout" }, + }, + { + operation_id: "op-other-tab", + action: "discovery.scan", + status: "failed", + error: { code: "ble-discovery-already-running" }, + }, + ], + last_operation: { + operation_id: "op-other-tab", + action: "discovery.scan", + status: "failed", + }, + }; + + assert.equal( + operationById(state, "discovery.scan", "op-requested")?.error?.code, + "ble-discovery-timeout", + ); + assert.equal(operationById(state, "discovery.scan", "op-missing"), null); +}); + +test("host Wi-Fi failures close the attempt and require a fresh explicit connection", () => { const operationTimeout = networkProvisionFailureMessage({ status: "failed", error: { code: "host-wifi-operation-timeout" }, }); - const buildTimeout = networkProvisionFailureMessage({ + const missingNetwork = networkProvisionFailureMessage({ status: "failed", error: { - code: "host-wifi-helper-build-timeout", + code: "network-not-found", + side_effect_status: "confirmed", + safe_to_retry: false, + scan_attempt_count: 13, + scan_elapsed_ms: 17524, + }, + }); + const preWriteKeychain = networkProvisionFailureMessage({ + status: "failed", + error: { + code: "keychain-authorization-required", side_effect_status: "none", safe_to_retry: true, }, }); - const buildFailed = networkProvisionFailureMessage({ + const postWriteKeychain = networkProvisionFailureMessage({ status: "failed", error: { - code: "host-wifi-helper-build-failed", - side_effect_status: "none", - safe_to_retry: true, - }, - }); - const postWriteBuildFailed = networkProvisionFailureMessage({ - status: "failed", - error: { - code: "host-wifi-helper-build-failed", + code: "keychain-authorization-denied", side_effect_status: "confirmed", safe_to_retry: false, }, @@ -850,19 +4141,12 @@ test("host Wi-Fi helper failures do not fabricate a missing-password diagnosis", assert.equal( operationTimeout, - "Локальная операция подготовки Wi‑Fi не завершилась вовремя. Это могло произойти до изменения состояния K1; наличие сохранённого пароля этим кодом не подтверждается и не опровергается. Проверьте состояние K1 и повторите подключение отдельным действием.", + "Локальная операция подготовки Wi‑Fi не завершилась вовремя; итог текущей попытки подключения не подтверждён. Автоматический повтор команды K1 не отправлялся. Сессия подключения в интерфейсе сброшена. Выполните новый поиск Bluetooth, выберите K1 и запустите подключение ещё раз.", ); assert.doesNotMatch(operationTimeout, /получите пароль|пароль отсутствует/i); - assert.equal( - buildTimeout, - "Локальный компонент Wi‑Fi не успел собраться за отведённое время. Команда K1 не отправлялась; подготовьте локальный компонент и повторите подключение отдельным действием.", - ); - assert.equal( - buildFailed, - "Локальный компонент Wi‑Fi не удалось собрать. Команда K1 не отправлялась; подготовьте локальный компонент и повторите подключение отдельным действием.", - ); - assert.equal( - postWriteBuildFailed, - "Локальный компонент Wi‑Fi не удалось собрать уже после начала операции с K1. Состояние устройства нельзя выводить из этой локальной ошибки; автоматического повтора команды не было. Выполните read-only проверку K1 перед новым подключением.", - ); + assert.match(missingNetwork, /K1 принял команду Quick Connect/); + assert.match(missingNetwork, /13 проверок за 17\.5 с/); + assert.match(missingNetwork, /автоматического повтора не было/); + assert.match(preWriteKeychain, /Команда устройству не отправлялась/); + assert.match(postWriteKeychain, /Дополнительный пароль не запрашивался/); }); diff --git a/apps/control-station/test/devicePluginFrontendBoundary.test.mjs b/apps/control-station/test/devicePluginFrontendBoundary.test.mjs index 712cb43..a656d02 100644 --- a/apps/control-station/test/devicePluginFrontendBoundary.test.mjs +++ b/apps/control-station/test/devicePluginFrontendBoundary.test.mjs @@ -105,6 +105,35 @@ test("each device plugin contributes its own connection pipeline component", () ); }); +test("selected-model shell leaves the model name to the connection heading", () => { + const workspace = readFileSync( + join(coreSourceRoot, "workspaces/DeviceWorkspace.tsx"), + "utf8", + ); + const app = readFileSync(join(coreSourceRoot, "App.tsx"), "utf8"); + + const selectedSlot = workspace.slice(workspace.indexOf("const ConnectionView =")); + assert.match(workspace, /

\{model\.displayName\}<\/h3>/); + assert.match(selectedSlot, /СЦЕНАРИЙ ПОДКЛЮЧЕНИЯ/); + assert.match(selectedSlot, /Модель выбрана<\/strong>/); + assert.doesNotMatch(selectedSlot, /selection\.model\.displayName/); + assert.doesNotMatch( + selectedSlot, + /selection\.plugin\.manifest\.metadata\.displayName/, + ); + + const localContour = app.slice( + app.indexOf('id: "local-contour"'), + app.indexOf("items={rootWorkspaces", app.indexOf('id: "local-contour"')), + ); + assert.match( + localContour, + /description: selection \? "Подключение" : "Модель не выбрана"/, + ); + assert.doesNotMatch(localContour, /activeDevice\?\.endpointLabel/); + assert.doesNotMatch(localContour, /selection\?\.model\.displayName/); +}); + test("registry exposes an optional model-scoped spatial controls contribution", () => { const connectionView = () => null; const spatialControlsView = () => null; @@ -146,6 +175,7 @@ test("XGRIDS frontend is physically plugin-owned and split by operator pipeline" "components/K1AcquisitionPipeline.tsx", "components/K1SpatialControls.tsx", "components/K1Diagnostics.tsx", + "physicalCommandConfirmation.ts", "projectName.ts", ]) { assert.equal(existsSync(join(pluginFrontendRoot, relativePath)), true, relativePath); @@ -159,24 +189,523 @@ test("XGRIDS frontend is physically plugin-owned and split by operator pipeline" assert.match(spatialControls, /cleanup_pending/); assert.match(spatialControls, /spatialActionFailure/); assert.match(spatialControls, /role="alert"/); - assert.match(spatialControls, /Повторить остановку/); + assert.doesNotMatch(spatialControls, /Повторить остановку/); + assert.match(spatialControls, /stopLocalReceiver/); + assert.match(spatialControls, / { +test("K1 connection surface enforces one scan, local selection, and one Apply", () => { const provisioning = readFileSync( join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), "utf8", ); - assert.match(provisioning, /connectionMode === "bridge"/); - assert.match(provisioning, /Подхватить существующее подключение/); - assert.match(provisioning, /pendingAction === "verify"/); + const lifecycle = readFileSync(join(pluginFrontendRoot, "lifecycle.ts"), "utf8"); + const submitPrerequisites = lifecycle.slice( + lifecycle.indexOf("export function canSubmitProvisioningMutation"), + lifecycle.indexOf("export function canAdmitProvisioningConnection"), + ); + assert.match(provisioning, /bridge:\s*\{/); assert.match( provisioning, - /compatibility_attestation: profileSelectionForConnectionMode\("bridge"\)/, + /compatibility_attestation: profileSelectionForConnectionMode\([\s\S]*attemptedConnectionMode/, ); - assert.match(provisioning, /без изменения настроек Wi‑Fi/); - assert.match(provisioning, /deviceSummary !== null/); - assert.match(provisioning, /device_id: deviceSummary\.device_id/); + assert.match(provisioning, /scanSecondsRemaining/); + assert.match(provisioning, /setInterval\(updateCountdown, 250\)/); + assert.match(provisioning, /Поиск Bluetooth · \{scanSecondsRemaining \?\? 6\} с/); + assert.match(provisioning, /scanWithResult\(\{[^}]*durationSeconds:\s*6/); + assert.doesNotMatch(provisioning, /K1 уже доступен|Сетевой адрес K1 доступен/); + assert.doesNotMatch( + provisioning, + /automaticRecovery|automaticDiscovery|reconnectFallbackAction/, + ); + assert.match(provisioning, /Переподключиться/); + assert.match(provisioning, /Подключить новый K1/); + assert.doesNotMatch(provisioning, /Вернуть прежний K1 и проверить/); + assert.match(provisioning, /explicitProvisioningDraftMatches/); + assert.doesNotMatch( + provisioning, + /powerConfirmed|powerConfirmationEpoch|resetPowerConfirmation|Питание включено|title="Питание"/, + ); + assert.doesNotMatch(submitPrerequisites, /power|питани/i); + assert.match(provisioning, /ПОДКЛЮЧЕНИЕ · ШАГИ 01–02/); + assert.match( + provisioning, + /number="01"[\s\S]*?title="Подключение"/, + ); + assert.match(provisioning, /number="02"[\s\S]*?title="Сеть"/); + assert.doesNotMatch(provisioning, /number="03"|showDeviceStep/); + assert.match( + provisioning, + /const backendScanAllowed = scanAllowedByPolicy\s*&& !isBusy\s*&& !networkRecoveryRequired/, + ); + assert.match(provisioning, /const showNetworkStep = Boolean\([\s\S]*explicitProvisioningDraftRetained/); + assert.match( + provisioning, + /if \(result\.networkIntentCompleted\)[\s\S]*setExplicitProvisioningDraft\(null\)/, + ); + assert.equal( + (provisioning.match(/buttonLabel:\s*"Применить"/g) ?? []).length, + 3, + ); + assert.doesNotMatch(provisioning, /<(?:button|input|select|textarea)\b/); + assert.doesNotMatch(provisioning, /allow_host_wifi_switch/); + assert.doesNotMatch(provisioning, /(?:color|background(?:-color)?):\s*(?:#[0-9a-f]{3,8}|rgba?\()/i); + for (const sharedControl of ["Button", "IconButton", "TextField", "ActivityIndicator", "StatusBadge"]) { + assert.match(provisioning, new RegExp(`<${sharedControl}\\b`), sharedControl); + } + assert.doesNotMatch( + provisioning, + /Подключиться к сохранённому|Исходный K1|Проверить связь с K1|Проверить прежнее подключение/, + ); +}); + +test("K1 click-owned actions are fenced without hidden frontend continuations", () => { + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const search = provisioning.slice( + provisioning.indexOf("const repeatDeviceScan"), + provisioning.indexOf("const submitConnect"), + ); + const apply = provisioning.slice( + provisioning.indexOf("const submitConnect"), + provisioning.indexOf("const verifyAppliedNetwork"), + ); + assert.equal((search.match(/scanWithResult\(/g) ?? []).length, 1); + assert.match(search, /durationSeconds:\s*6/); + assert.equal((apply.match(/await connect\(/g) ?? []).length, 1); + assert.doesNotMatch( + apply, + /scanWithResult\(|verifyConnection\(|candidateRefresh|void submitConnect/, + ); + assert.match(runtime, /class SnapshotRuntimeActionArbiter/); + assert.match(runtime, /runtimeActionArbiter\.current\.isCurrent\(actionToken\)/); + assert.match(runtime, /runtimeActionArbiter\.current\.settle\(actionToken\)/); +}); + +test("connected presentation uses canonical process copy", () => { + const connection = readFileSync( + join(pluginFrontendRoot, "XgridsK1Connection.tsx"), + "utf8", + ); + assert.match( + connection, + /connectionTopology\?\.status === "active"\s*\? "Подключение установлено"/, + ); + assert.match(connection, /"Готово к новой сессии\."/); + assert.match(connection, /

Подключение \{model\.displayName\}<\/h2>/); + assert.match( + connection, + /const operationalPanelsVisible = shouldRenderK1OperationalPanels\(state\)/, + ); + assert.match( + connection, + /operationalPanelsVisible \? : null/, + ); + assert.match( + connection, + / { + const app = readFileSync(join(coreSourceRoot, "App.tsx"), "utf8"); + const shellPresentation = readFileSync( + join(coreSourceRoot, "presentation.ts"), + "utf8", + ); + const connection = readFileSync( + join(pluginFrontendRoot, "XgridsK1Connection.tsx"), + "utf8", + ); + const operatorError = readFileSync( + join(pluginFrontendRoot, "components/K1OperatorError.tsx"), + "utf8", + ); + const acquisition = readFileSync( + join(pluginFrontendRoot, "components/K1AcquisitionPipeline.tsx"), + "utf8", + ); + const metrics = readFileSync( + join(pluginFrontendRoot, "components/K1Metrics.tsx"), + "utf8", + ); + + assert.match(connection, /

Подключение \{model\.displayName\}<\/h2>/); + assert.doesNotMatch(connection, /:\s*message\}/); + assert.match( + operatorError, + /Подключение не завершено\. Автоматического повтора не было/, + ); + + const deviceHeader = app.slice( + app.indexOf('activeDefinition.kind === "device" ? ('), + app.indexOf('activeDefinition.kind === "spatial"', app.indexOf('activeDefinition.kind === "device" ? (')), + ); + assert.match(deviceHeader, /localConnectionPhaseLabel\(runtime\.state\?\.phase\)/); + assert.doesNotMatch(deviceHeader, /phaseLabel\(runtime\.state\?\.phase\)/); + assert.match( + shellPresentation, + /phase === "configuring"\) return "Подключение"/, + ); + assert.match( + shellPresentation, + /phase === "connected"\) return "Подключение установлено"/, + ); + assert.match(shellPresentation, /configuring: "Настройка устройства"/); + assert.match(shellPresentation, /connected: "Устройство подключено"/); + + assert.match(acquisition, /hint="Локальный файл записи"/); + assert.match(acquisition, /Состояние сканирования остаётся неизвестным/); + assert.doesNotMatch( + acquisition, + /Локальный файл исходных данных|Физическое состояние сканера/, + ); + assert.match(metrics, /Данные потока при этом сохраняются/); + assert.doesNotMatch(metrics, /Исходные данные при этом сохраняются/); +}); + +test("K1 START renders an in-button spinner for the complete live orchestration", () => { + const acquisition = readFileSync( + join(pluginFrontendRoot, "components/K1AcquisitionPipeline.tsx"), + "utf8", + ); + + assert.match(acquisition, /pendingAction === "live"[\s\S]* { + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + const exactReady = runtime.slice( + runtime.indexOf("function hasExactConnectionReady"), + runtime.indexOf("async function waitForPhysicalReconciliationProof"), + ); + const connectFlow = runtime.slice( + runtime.indexOf("const connect = useCallback"), + runtime.indexOf("const verifyConnection = useCallback"), + ); + const appliedProof = runtime.slice( + runtime.indexOf("export function exactAppliedNetworkIntentCompleted"), + runtime.indexOf("function requireExactReadOnlyVerificationOutcome"), + ); + + assert.match(exactReady, /state\.desired_connection_mode === connectionMode/); + assert.match(exactReady, /state\.active_connection_mode === connectionMode/); + assert.match(exactReady, /application_control_session\?\.state === "connection-ready"/); + assert.match(exactReady, /currentAppliedConnectionTopology\(state, connectionMode\)\?\.status === "active"/); + assert.doesNotMatch(connectFlow, /openApplicationControlSession|waitForControlPhase/); + assert.doesNotMatch(connectFlow, /verifyConnection\(|scanWithResult\(/); + assert.doesNotMatch(connectFlow, /startAcquisition|startPreparedAcquisition|START acquisition/); + assert.match(connectFlow, /networkIntentCompleted/); + assert.match(appliedProof, /attempt\.phase === "network_applied"/); + assert.match(appliedProof, /operation\.status === "succeeded"/); + assert.match(appliedProof, /operationPhase === "network_applied"/); + assert.match(appliedProof, /ledger\.resolution === "target-observed"/); + assert.match(appliedProof, /deviceNetwork\?\.state === "applied"/); + assert.doesNotMatch(appliedProof, /control_state/); + assert.match( + connectFlow, + /const exactNetworkIntentCompleted = exactAppliedNetworkIntentCompleted\([\s\S]*?if \(\s*!exactNetworkIntentCompleted\s*&& !hasExactConnectionReady\([\s\S]*?return requireExactConnectionReady\([\s\S]*?nextState = acceptSuccessfulConnectState\(nextState\)/, + ); + assert.match(provisioning, /Подключение установлено/); + assert.doesNotMatch( + provisioning, + /Подключиться к сохранённому|Проверить связь с K1|Проверить прежнее подключение/, + ); + assert.match(provisioning, /Переподключиться/); + assert.match(provisioning, /Подключить новый K1/); + assert.match(provisioning, /expected_mode_revision: modeAuthority\.desiredModeRevision/); + assert.match( + provisioning, + /expected_discovery_generation: modeAuthority\.discoveryGeneration/, + ); + assert.doesNotMatch(provisioning, /Настройки сети применены/); +}); + +test("K1 mode reset is explicit while Scan and Apply keep exact backend CAS", () => { + const api = readFileSync(join(pluginFrontendRoot, "api.ts"), "utf8"); + const manifest = readFileSync(join(pluginFrontendRoot, "manifest.ts"), "utf8"); + const connection = readFileSync(join(pluginFrontendRoot, "XgridsK1Connection.tsx"), "utf8"); + const runtime = readFileSync(join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), "utf8"); + const acquisition = readFileSync( + join(pluginFrontendRoot, "components/K1AcquisitionPipeline.tsx"), + "utf8", + ); + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + + assert.match(manifest, /connectionModeSelect:[\s\S]*"connection\.mode\.select"/); + assert.match(api, /interface SelectConnectionModeRequest/); + assert.match(api, /expected_revision: number/); + assert.match(api, /reset_scenario\?: true/); + assert.match(api, /reset_id\?: string/); + assert.match(api, /expected_mode_revision: number/); + assert.match(api, /expected_discovery_generation: number/); + const localModeHandler = connection.slice( + connection.indexOf("const updateDesiredConnectionMode"), + connection.indexOf("const sourceTone"), + ); + assert.match(localModeHandler, /setDesiredConnectionMode\(mode\)/); + assert.doesNotMatch(localModeHandler, /await|selectConnectionMode\(|refresh\(|connect\(/); + assert.match(provisioning, /const commitDesiredModeForExplicitAction = useCallback\(async/); + assert.match(provisioning, /await selectConnectionMode\(\{/); + assert.match(provisioning, /expected_revision: expectedRevision as number/); + assert.match(provisioning, /reset_scenario: true/); + assert.match(provisioning, /const resetId = newOperationId\(\)/); + assert.match(provisioning, /reset_id: resetId/); + assert.match(provisioning, /Подключить новый K1/); + const configurationAnchor = provisioning.slice( + provisioning.indexOf('
'), + provisioning.indexOf('
'), + ); + assert.doesNotMatch(configurationAnchor, /Подключить новый K1/); + assert.match(provisioning, /value=\{connectionMode\}/); + assert.doesNotMatch( + provisioning, + /disabled=\{\s*isBusy\s*\|\|\s*networkRecoveryRequired\s*\|\|\s*physicalRecoveryRequired\s*\|\|\s*connectionRecoveryRequired/, + ); + assert.match(runtime, /catch \(selectionError\)[\s\S]*xgridsK1Api\.getState\(\)/); + assert.match(acquisition, /state\?\.active_connection_mode/); + assert.match(acquisition, /desiredSelectionCommitted/); + assert.match(acquisition, /configuredConnectionMode !== desiredConnectionMode/); + assert.match(acquisition, /Выбран другой способ связи/); +}); + +test("top-right device utility is an explicit pending-aware K1 scenario reset", async () => { + const { deviceRuntimeUtilityAction } = await server.ssrLoadModule( + "/src/components/useApplicationPanelActions.ts", + ); + let resetCalls = 0; + let refreshCalls = 0; + const reset = deviceRuntimeUtilityAction({ + refreshRuntime: () => { + refreshCalls += 1; + }, + resetConnectionScenario: async () => { + resetCalls += 1; + return true; + }, + connectionScenarioResetting: false, + }); + + assert.equal(reset.label, "Сбросить подключение"); + assert.equal(reset.icon, "refresh"); + assert.equal(reset.disabled, undefined); + reset.onClick(); + await Promise.resolve(); + assert.equal(resetCalls, 1); + assert.equal(refreshCalls, 0); + + const pending = deviceRuntimeUtilityAction({ + refreshRuntime: () => { + refreshCalls += 1; + }, + resetConnectionScenario: async () => true, + connectionScenarioResetting: true, + }); + assert.equal(pending.label, "Сбрасываем подключение"); + assert.equal(pending.icon, "activity"); + assert.equal(pending.disabled, true); + pending.onClick(); + await Promise.resolve(); + assert.equal(resetCalls, 1); + assert.equal(refreshCalls, 0); + + const app = readFileSync(join(coreSourceRoot, "App.tsx"), "utf8"); + const contracts = readFileSync( + join(coreSourceRoot, "core/runtime/contracts.ts"), + "utf8", + ); + const runtimeContext = readFileSync( + join(pluginFrontendRoot, "runtimeContext.tsx"), + "utf8", + ); + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + const connection = readFileSync( + join(pluginFrontendRoot, "XgridsK1Connection.tsx"), + "utf8", + ); + + assert.match(contracts, /resetConnectionScenario\?: \(\) => Promise/); + assert.match(app, /resetConnectionScenario: runtime\.resetConnectionScenario/); + assert.match(app, /connectionScenarioResetting: runtime\.pendingAction === "mode"/); + assert.match( + runtimeContext, + /resetConnectionScenario: controller\.resetConnectionScenario/, + ); + + const resetStart = runtime.indexOf("const resetConnectionScenario"); + const resetEnd = runtime.indexOf( + "const prepareConnectionReconfigurationWithResult", + resetStart, + ); + assert.notEqual(resetStart, -1); + assert.notEqual(resetEnd, -1); + const resetFlow = runtime.slice(resetStart, resetEnd); + assert.equal((resetFlow.match(/selectConnectionMode\(/g) ?? []).length, 1); + assert.match(resetFlow, /connection_mode: DEFAULT_CONNECTION_MODE/); + assert.match(resetFlow, /expected_revision: expectedRevision as number/); + assert.match(resetFlow, /reset_scenario: true/); + assert.match(resetFlow, /reset_id: newOperationId\(\)/); + assert.doesNotMatch( + resetFlow, + /refresh\(|getState\(|scan|verify|connect\(|prepare|start|stop|camera/i, + ); + + const selectStart = runtime.indexOf("const selectConnectionMode"); + const selectEnd = runtime.indexOf("const resetConnectionScenario", selectStart); + const selectFlow = runtime.slice(selectStart, selectEnd); + assert.match(selectFlow, /expected_snapshot_runtime_id: expectedSnapshotRuntimeId\(\)/); + assert.match(selectFlow, /supersedePending: request\.reset_scenario === true/); + + const runStart = runtime.indexOf("const run = useCallback"); + const runEnd = runtime.indexOf("const scanWithResult", runStart); + const runFlow = runtime.slice(runStart, runEnd); + assert.match(runFlow, /setPendingAction\(action\)/); + assert.match(runFlow, /setPresentedErrorCorrelation\(null\)/); + assert.match(runFlow, /setError\(null\)/); + assert.match(runFlow, /setErrorDiagnostic\(null\)/); + assert.match(runFlow, /setPendingAction\(null\)/); + + const draftFenceStart = provisioning.indexOf("const nextFence = localProvisioningDraftFenceKey"); + assert.notEqual(draftFenceStart, -1); + const draftFence = provisioning.slice(draftFenceStart, draftFenceStart + 2_700); + assert.match(draftFence, /reconfigurationRevision/); + assert.match(draftFence, /setSelectedDeviceId\(""\)/); + assert.match(draftFence, /setSelectedDeviceSnapshot\(null\)/); + assert.match(draftFence, /setExplicitProvisioningDraft\(null\)/); + assert.match(draftFence, /setSsid\(""\)/); + assert.match(draftFence, /setPassword\(""\)/); + assert.match(draftFence, /setCandidateUnavailableMessage\(null\)/); + assert.match(draftFence, /resetSearchPresentation\(\)/); + assert.match( + provisioning, + /const hydratedScenarioResetPresentationKey = useRef\(null\)/, + ); + assert.match( + provisioning, + /hydratedScenarioResetPresentationKey\.current = scenarioResetPresentationKey/, + ); + assert.match(provisioning, /setConnectionAttemptPresentation\(null\)/); + assert.match( + provisioning, + /const modeResetInFlight = pendingAction === "mode" \|\| modeResetPending !== null/, + ); + assert.match(provisioning, /if \(modeResetInFlight\) return/); + assert.equal((provisioning.match(/disabled=\{modeResetInFlight\}/g) ?? []).length, 2); + assert.match(provisioning, /disabled=\{isBusy \|\| modeResetInFlight\}/); + + assert.match(connection, /const hydratedScenarioResetKey = useRef\(null\)/); + assert.match( + connection, + /scenarioReset\.revision === state\?\.desired_connection_mode_revision/, + ); + assert.match(connection, /scenarioReset\.desired_mode === backendDesiredMode/); + assert.match(connection, /desiredModeLocallyDirty\.current = false/); + assert.match(connection, /setDesiredConnectionMode\(backendDesiredMode\)/); +}); + +test("background polling stays read-only while backend state reconciliation retires terminal control", () => { + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + + const refreshFlow = runtime.slice( + runtime.indexOf("const refresh = useCallback"), + runtime.indexOf("const run = useCallback"), + ); + assert.match(runtime, /refresh\(false\)/); + assert.match(runtime, /refresh\(true\)/); + assert.doesNotMatch(runtime, /terminalControlCleanupInFlight/); + assert.doesNotMatch( + refreshFlow, + /closeApplicationControlSession|startAcquisition|stopAcquisition|networkProvision/, + ); + assert.match(refreshFlow, /xgridsK1Api\.getState\(\)/); +}); + +test("automatic K1 live start opens the selected delivered camera despite an older saved layout", () => { + const app = readFileSync(join(coreSourceRoot, "App.tsx"), "utf8"); + const layout = readFileSync( + join(coreSourceRoot, "core/observation/useObservationLayout.ts"), + "utf8", + ); + + assert.match(app, /observationLayout\.activateAutomaticDefaults\(\)/); + assert.match(layout, /const activateAutomaticDefaults = useCallback/); + assert.match(layout, /restoredLayoutAuthorityRef\.current = false/); + assert.match(layout, /sources\.filter\(canOpenByDefault\)/); + assert.match(layout, /source\.capabilities\.overlay/); +}); + +test("K1 connect errors reset the UI session without exposing reconciliation ceremony", () => { + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const connectStart = runtime.indexOf("const connect = useCallback("); + const connectEnd = runtime.indexOf("const verifyConnection = useCallback(", connectStart); + assert.notEqual(connectStart, -1); + assert.notEqual(connectEnd, -1); + const connectFlow = runtime.slice(connectStart, connectEnd); + + assert.match(connectFlow, /operationByIdempotencyKey\(/); + assert.match(connectFlow, /failedOperation\?\.status === "succeeded"/); + assert.match(connectFlow, /resetConnectSessionMessage\(/); + assert.doesNotMatch( + connectFlow, + /требует ручной проверки|измените параметры только после проверки устройства|Сохранён тот же ключ/, + ); + assert.match(runtime, /Сессия подключения в интерфейсе сброшена/); + assert.match(runtime, /новый поиск Bluetooth, выберите K1 и запустите подключение ещё раз/); + const verifyStart = runtime.indexOf("const verifyConnection = useCallback("); + const verifyEnd = runtime.indexOf("const probeConfiguredEndpoint = useCallback(", verifyStart); + const verifyFlow = runtime.slice(verifyStart, verifyEnd); + assert.match(verifyFlow, /failedOperation\?\.status === "succeeded"/); + assert.doesNotMatch(verifyFlow, /xgridsK1Api\.verifyConnection\([^)]*\)[\s\S]*xgridsK1Api\.verifyConnection/); + assert.doesNotMatch(runtime, /automatic.?retry\s*:\s*true/); }); test("K1 provisioning keeps the operator draft separate from the backend lease", () => { @@ -184,14 +713,432 @@ test("K1 provisioning keeps the operator draft separate from the backend lease", join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), "utf8", ); + const selection = provisioning.slice( + provisioning.indexOf("const selectFreshDevice"), + provisioning.indexOf("const chooseAnother"), + ); + const apply = provisioning.slice( + provisioning.indexOf("const submitConnect"), + provisioning.indexOf("const verifyAppliedNetwork"), + ); + assert.match(provisioning, /explicitProvisioningDraftRetained/); + assert.match(provisioning, /localProvisioningDraftFenceKey/); + assert.match(provisioning, /provisioningIntentKey\(null\)/); + assert.match(provisioning, /const \[selectedDeviceSnapshot, setSelectedDeviceSnapshot\]/); + assert.doesNotMatch(selection, /await|scanWithResult\(|verifyConnection\(|connect\(/); + assert.match(selection, /requestExplicitProvisioning\(device\.device_id/); + assert.equal((apply.match(/await connect\(/g) ?? []).length, 1); + assert.doesNotMatch(apply, /scanWithResult\(|verifyConnection\(|candidateRefresh/); + assert.match(provisioning, /scanWithResult\(\{ durationSeconds: 6 \}\)/); + assert.match( + provisioning, + /onChange=\{\(event\) => setPassword\(event\.target\.value\)\}/, + ); +}); - assert.doesNotMatch(provisioning, /setSelectedDeviceId\(state\.selected_device_id\)/); - assert.doesNotMatch(provisioning, /setConnectionMode\(state\.connection_mode\)/); - assert.match(provisioning, /canSubmitProvisioningMutation\(\{/); - assert.match(provisioning, /isReachableConnectionLease\(state, connectionMode\)/); - assert.doesNotMatch(provisioning, /else if \(connectionMode === "quick-connect"\)/); - assert.match(provisioning, /const succeeded = await verifyConnection\(\{/); - assert.match(provisioning, /if \(succeeded\) \{\s*provisioningIntentRef\.current = null;/); +test("K1 plugin layout follows its contribution width and contains long topology text", () => { + const styles = readFileSync(join(pluginFrontendRoot, "styles.css"), "utf8"); + const baseGrid = styles.slice( + styles.indexOf(".device-workspace__grid"), + styles.indexOf(".device-workspace__side"), + ); + const splitThreshold = styles.match( + /@container xgrids-k1 \(min-width:\s*([0-9.]+)rem\)/, + ); + const splitColumns = styles.match( + /grid-template-columns:\s*minmax\(([0-9.]+)rem,\s*0\.8fr\)\s*minmax\(([0-9.]+)rem,\s*1\.2fr\)/, + ); + + assert.match(styles, /container:\s*xgrids-k1\s*\/\s*inline-size/); + assert.match(styles, /container:\s*k1-connection-panel\s*\/\s*inline-size/); + assert.match(styles, /container:\s*k1-session-panel\s*\/\s*inline-size/); + assert.match(styles, /@container xgrids-k1 \(min-width:\s*78rem\)/); + assert.match( + styles, + /grid-template-columns:\s*minmax\(32rem,\s*0\.8fr\)\s*minmax\(38rem,\s*1\.2fr\)/, + ); + assert.match(styles, /@container k1-connection-panel \(max-width:\s*48rem\)/); + assert.match(styles, /@container k1-session-panel \(max-width:\s*48rem\)/); + assert.match(styles, /@container xgrids-k1 \(max-width:\s*48rem\)/); + assert.match(styles, /overflow-wrap:\s*anywhere/); + assert.match(baseGrid, /grid-template-columns:\s*minmax\(0,\s*1fr\)/); + assert.ok(splitThreshold); + assert.ok(splitColumns); + const splitThresholdPixels = Number(splitThreshold[1]) * 16; + const minimumSplitPixels = (Number(splitColumns[1]) + Number(splitColumns[2]) + 0.85) * 16; + assert.equal(splitThresholdPixels, 1248); + assert.ok(minimumSplitPixels < splitThresholdPixels); + assert.ok(390 < splitThresholdPixels); + assert.ok(760 < splitThresholdPixels); + assert.ok(1280 > splitThresholdPixels); + assert.match( + styles, + /\.wizard-list,[\s\S]*?\.wizard-step,[\s\S]*?\.session-form,[\s\S]*?\.device-row,[\s\S]*?\.detail-list\s*\{[^}]*min-width:\s*0[^}]*max-width:\s*100%/, + ); + assert.match( + styles, + /> \*\s*\{[^}]*min-width:\s*0[^}]*max-width:\s*100%/, + ); + assert.match( + styles, + /\.metrics-grid > \*,[\s\S]*?\.device-workspace__grid > \*,[\s\S]*?\.diagnostics-grid > \*\s*\{[^}]*min-width:\s*0[^}]*max-width:\s*100%/, + ); + assert.match( + styles, + /\.device-row code,[\s\S]*?\.detail-row code\s*\{[^}]*overflow-wrap:\s*anywhere[^}]*white-space:\s*normal/, + ); + assert.match( + styles, + /\.detail-row dd\s*\{[^}]*overflow-wrap:\s*anywhere[^}]*text-overflow:\s*clip[^}]*white-space:\s*normal/, + ); + assert.match( + styles, + /\.error-banner__actions > \.nodedc-button\s*\{[^}]*min-width:\s*0[^}]*max-width:\s*100%[^}]*overflow-wrap:\s*anywhere/, + ); + assert.doesNotMatch( + styles, + /\.connection-panel,[\s\S]*?\.session-panel\s*\{[^}]*overflow:\s*(?:clip|hidden)/, + ); + assert.match( + styles, + /@container k1-connection-panel \(max-width:\s*48rem\)[\s\S]*?\.wizard-step__content > header[^}]*flex-wrap:\s*wrap/, + ); + assert.match( + styles, + /@container k1-connection-panel \(max-width:\s*48rem\)[\s\S]*?\.retained-recovery-target > div[^}]*flex-direction:\s*column/, + ); + assert.match( + styles, + /@container k1-session-panel \(max-width:\s*48rem\)[\s\S]*?\.panel-heading[^}]*flex-wrap:\s*wrap/, + ); + assert.match( + styles, + /\.connection-summary__value > \.nodedc-status,[\s\S]*?white-space:\s*normal/, + ); + assert.doesNotMatch(styles, /\.nodedc-checker(?:__copy|__label)?\s*\{/); + assert.match( + styles, + /\.workspace-lead__status > span,[\s\S]*?\.nodedc-field__description,[\s\S]*?\.retained-recovery-target small,[\s\S]*?\.session-footer p\s*\{[^}]*overflow-wrap:\s*anywhere/, + ); + assert.match( + styles, + /@container xgrids-k1 \(max-width:\s*32rem\)[\s\S]*?\.error-banner__actions\s*\{[^}]*align-items:\s*stretch[^}]*flex-direction:\s*column/, + ); + assert.doesNotMatch(styles, /@media \(max-width:\s*(?:1280|1480)px\)/); + assert.doesNotMatch(styles, /device-recovery-choice/); +}); + +test("Mission Core shell protects the device workspace before the shared mobile breakpoint", () => { + const responsive = readFileSync(join(coreSourceRoot, "styles/responsive.css"), "utf8"); + + assert.match( + responsive, + /@media \(min-width:\s*761px\) and \(max-width:\s*929px\)/, + ); + assert.match( + responsive, + /\.nodedc-app-shell__navigation,\s*\.nodedc-app-shell__content\s*\{[^}]*left:\s*var\(--nodedc-app-page-pad\)[^}]*width:\s*auto/s, + ); + assert.match( + responsive, + /\[data-content-open="true"\] \.nodedc-app-shell__navigation\s*\{[^}]*opacity:\s*0[^}]*pointer-events:\s*none/s, + ); + assert.match( + responsive, + /@media \(max-width:\s*760px\)[\s\S]*?\.nodedc-application-panel__head,\s*\.nodedc-application-panel__body\s*\{[^}]*width:\s*auto[^}]*min-width:\s*0[^}]*max-width:\s*100%/, + ); + assert.match( + responsive, + /\.nodedc-application-panel__head\s*\{[^}]*grid-template-columns:\s*minmax\(0,\s*1fr\)\s*auto/, + ); +}); + +test("K1 frontend state models durable mutation and connection supervision facts", () => { + const api = readFileSync(join(pluginFrontendRoot, "api.ts"), "utf8"); + const lifecycle = readFileSync(join(pluginFrontendRoot, "lifecycle.ts"), "utf8"); + + assert.match(api, /interface XgridsNetworkMutationLedger/); + assert.match(api, /scope:\s*"durable-ledger"/); + assert.match(api, /network_mutation_ledger\?: XgridsNetworkMutationLedger \| null/); + assert.match(api, /handle_retained\?: boolean/); + assert.match(api, /gatt_validated_recently\?: boolean/); + assert.match(api, /interface XgridsConnectionSupervisor/); + assert.match(api, /interface XgridsConnectionSupervisorDeviceNetwork/); + assert.match(api, /device_network: XgridsConnectionSupervisorDeviceNetwork/); + assert.match(api, /missioncore\.k1-connection-supervisor\/v1/); + assert.match(api, /connection_supervisor\?: XgridsConnectionSupervisor \| null/); + assert.match(api, /interface XgridsConnectionPolicy/); + assert.match(api, /missioncore\.xgrids-k1-connection-policy\/v1/); + assert.match(api, /connection_policy\?: XgridsConnectionPolicy \| null/); + assert.match(api, /interface XgridsSemanticTopologyStore/); + assert.match(api, /configured_offline_evidence: boolean/); + assert.match(api, /live_connection_authority: false/); + assert.match(api, /semantic_topology_store\?: XgridsSemanticTopologyStore \| null/); + + const reconciliation = lifecycle.slice( + lifecycle.indexOf("export function readOnlyVerificationClearedReconciliation"), + lifecycle.indexOf("export function provisioningCandidateById"), + ); + assert.match(reconciliation, /nextState\?\.network_mutation_ledger/); + assert.match(reconciliation, /nextLedger\.status === "resolved"/); + assert.match(reconciliation, /nextLedger\.operation_id === previousOperationId/); + assert.doesNotMatch(reconciliation, /snapshot_runtime_id/); +}); + +test("Bridge device and network reconfiguration stays backend-owned and CAS-fenced", () => { + const api = readFileSync(join(pluginFrontendRoot, "api.ts"), "utf8"); + const manifest = readFileSync(join(pluginFrontendRoot, "manifest.ts"), "utf8"); + const lifecycle = readFileSync(join(pluginFrontendRoot, "lifecycle.ts"), "utf8"); + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + const prepare = provisioning.slice( + provisioning.indexOf("const prepareReconfiguration"), + provisioning.indexOf("const changeDesiredConnectionMode"), + ); + const selection = provisioning.slice( + provisioning.indexOf("const selectFreshDevice"), + provisioning.indexOf("const chooseAnother"), + ); + assert.match(manifest, /connectionReconfigurePrepare:[\s\S]*"connection\.reconfigure\.prepare"/); + assert.match(api, /expected_reconfiguration_revision: number/); + assert.match(api, /expected_reconfiguration_intent_id/); + assert.match(prepare, /prepareConnectionReconfigurationWithResult\(\{/); + assert.doesNotMatch(prepare, /scanWithResult\(|verifyConnection\(|connect\(/); + assert.doesNotMatch(selection, /await|scanWithResult\(|verifyConnection\(|connect\(/); + assert.match(provisioning, /showNetworkStep = Boolean\([\s\S]*changeNetworkDialogue/); + assert.match(provisioning, /localProvisioningDraftFenceKey/); + assert.match(lifecycle, /reconfigurationAllowsFreshDevice/); + assert.match(runtime, /prepareConnectionReconfiguration/); +}); + +test("shared runtime exposes only a reachable K1 endpoint as active", () => { + const runtimeContext = readFileSync( + join(pluginFrontendRoot, "runtimeContext.tsx"), + "utf8", + ); + + assert.match( + runtimeContext, + /endpointLabel: activeConnectionEndpointLabel\(state\)/, + ); + assert.doesNotMatch(runtimeContext, /endpointLabel: state\.k1_ip/); +}); + +test("every K1 connection and acquisition action is fenced to the accepted backend runtime", () => { + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + + assert.match(runtime, /latestState\.current\?\.snapshot_runtime_id/); + assert.equal( + [...runtime.matchAll( + /expected_snapshot_runtime_id: expectedSnapshotRuntimeId\(\)/g, + )].length, + 14, + ); + assert.match( + runtime, + /mode:\s*"graceful",[\s\S]*?expected_snapshot_runtime_id: checkpoint\.snapshotRuntimeId/, + ); + assert.match( + runtime, + /xgridsK1Api\.scanBle\([\s\S]*?expected_snapshot_runtime_id/, + ); + assert.match( + runtime, + /xgridsK1Api\.verifyConnection\([\s\S]*?expected_snapshot_runtime_id/, + ); + assert.match( + runtime, + /xgridsK1Api\.connect\([\s\S]*?expected_snapshot_runtime_id/, + ); + assert.match( + runtime, + /xgridsK1Api\.retireUnavailablePhysicalCommand\([\s\S]*?expected_snapshot_runtime_id:\s*exactSnapshotRuntimeId/, + ); + assert.match( + runtime, + /run\("retire",[\s\S]*?surfaceErrors: false[\s\S]*?await refresh\(false\)/, + ); + assert.match( + runtime, + /const exactSnapshotRuntimeId = actionSnapshotRuntimeId\.trim\(\)[\s\S]*?isSnapshotRuntimeCurrent\(exactSnapshotRuntimeId\)[\s\S]*?xgridsK1Api\.reopenRetiredPhysicalReconciliation\([\s\S]*?expected_snapshot_runtime_id: exactSnapshotRuntimeId/, + ); + for (const action of [ + "openApplicationControlSession", + "enterApplicationWorkspace", + "closeApplicationControlSession", + "prepareAcquisition", + "startAcquisition", + "abortAcquisition", + "reconcilePhysicalCommand", + ]) { + assert.match( + runtime, + new RegExp(`xgridsK1Api\\.${action}\\([\\s\\S]*?expected_snapshot_runtime_id`), + action, + ); + } + assert.equal( + [...runtime.matchAll( + /expected_snapshot_runtime_id: actionSnapshotRuntimeId/g, + )].length, + 3, + ); + assert.match( + runtime, + /stopSessionCompatibility\(\{[\s\S]*?expected_snapshot_runtime_id/, + ); + assert.match( + runtime, + /run\([\s\S]*?"reopen"[\s\S]*?surfaceErrors: false, supersedePending: true[\s\S]*?await refresh\(false\)/, + ); + assert.match( + runtime, + /const actionSnapshotRuntimeId =\s*options\.expectedSnapshotRuntimeId\?\.trim\(\) \|\| null[\s\S]*?expected_snapshot_runtime_id:\s*actionSnapshotRuntimeId \?\? expectedSnapshotRuntimeId\(\)/, + ); +}); + +test("fresh Scan treats an exact prior K1 as one local Select action", () => { + const provisioning = readFileSync( + join(pluginFrontendRoot, "components/K1ProvisioningPipeline.tsx"), + "utf8", + ); + const lifecycle = readFileSync(join(pluginFrontendRoot, "lifecycle.ts"), "utf8"); + const runtime = readFileSync( + join(pluginFrontendRoot, "useXgridsK1Runtime.ts"), + "utf8", + ); + const selection = provisioning.slice( + provisioning.indexOf("const selectFreshDevice"), + provisioning.indexOf("const chooseAnother"), + ); + const apply = provisioning.slice( + provisioning.indexOf("const submitConnect"), + provisioning.indexOf("const verifyAppliedNetwork"), + ); + assert.match(selection, /requestExplicitProvisioning\(device\.device_id/); + assert.match(selection, /const actionableDevices = devices\.filter\(candidateSelectionAllowed\)/); + assert.doesNotMatch( + selection, + /physicallyRetired|retiredPhysical|await|verifyConnection\(|retireUnavailable|reopenRetired/, + ); + const resultRows = provisioning.slice( + provisioning.indexOf('
'), + provisioning.indexOf('
'), + ); + assert.match(resultRows, /actionLabel="Выбрать"/); + assert.match(resultRows, /onSelect=\{\(\) => selectCandidate\(device\)\}/); + assert.doesNotMatch(resultRows, /Переподключиться|reopen|verifyConnection/); + assert.doesNotMatch(provisioning, /recoverRetiredPhysicalCandidate/); + assert.doesNotMatch(apply, /verifyConnection\(|retireUnavailable|reopenRetired/); + assert.match(provisioning, /Переподключиться/); + assert.match(provisioning, /Подключить новый K1/); + assert.doesNotMatch(provisioning, /Вернуть прежний K1 и проверить/); + assert.match(runtime, /reopenRetiredPhysicalReconciliation/); + assert.match(runtime, /retireUnavailablePhysicalCommand/); + assert.match(lifecycle, /retiredPhysicalReopenAuthority/); +}); + +test("K1 orchestration accepts backend recovery but still requires exact topology before physical START", () => { + const acquisition = readFileSync( + join(pluginFrontendRoot, "components/K1AcquisitionPipeline.tsx"), + "utf8", + ); + const diagnostics = readFileSync( + join(pluginFrontendRoot, "components/K1Diagnostics.tsx"), + "utf8", + ); + + assert.match(acquisition, /currentAppliedConnectionTopology\(state\)/); + assert.ok( + [...acquisition.matchAll(/!connectionConfigured/g)].length >= 2, + "handler and button must both reject absent or configured-offline topology", + ); + assert.match(acquisition, /desiredModeMatchesActive/); + assert.match(acquisition, /modeSwitchRequired/); + assert.match(acquisition, /Выбран другой способ связи/); + assert.match(acquisition, /prepareCanonicalAcquisition/); + assert.match(acquisition, /startPreparedAcquisition/); + assert.match(acquisition, /operatorActionPhysicalAcceptance\(\)/); + assert.doesNotMatch(acquisition, /K1PhysicalCommandConfirmation/); + assert.match(acquisition, /connectionPolicyAllows\(state, "start-acquisition"\)/); + assert.match(acquisition, /canIssueCanonicalStop\(state, physicalStopIntentSpent\)/); + assert.match(acquisition, /physicalStopInFlight \|\| physicalStopExecutable/); + assert.match(acquisition, /connectionPolicyAllows\(state, "stop-local-receiver"\)/); + assert.match(acquisition, /if \(finalStartTarget\)/); + assert.match(acquisition, /if \(physicalStopExecutable\)/); + assert.match(acquisition, /if \(!physicalStartAllowed\) return;/); + assert.doesNotMatch(acquisition, /physicalStopAllowed/); + const replaySubmit = acquisition.slice( + acquisition.indexOf("const submitReplay ="), + acquisition.indexOf("return (", acquisition.indexOf("const submitReplay =")), + ); + assert.doesNotMatch( + replaySubmit, + /connectionPolicyAllows|physicalStartAllowed|physicalStopExecutable/, + ); + assert.match(acquisition, /void stopLocalReceiver\(\);/); + assert.match(acquisition, /onClick=\{\(\) => void abort\(\)\}/); + assert.doesNotMatch(acquisition, /acknowledge-data-loss/); + assert.doesNotMatch(acquisition, /PHYSICAL_ACCEPTANCE/); + assert.doesNotMatch(acquisition, /!state\?\.k1_ip/); + assert.match(diagnostics, /activeConnectionEndpointLabel\(state\)/); + assert.match(diagnostics, /Адрес конфигурации/); + assert.match(diagnostics, /связь не подтверждена/); + assert.doesNotMatch(diagnostics, /state\?\.k1_ip/); +}); + +test("one explicit K1 action performs START or STOP without a redundant checklist modal", () => { + const acquisition = readFileSync( + join(pluginFrontendRoot, "components/K1AcquisitionPipeline.tsx"), + "utf8", + ); + const spatial = readFileSync( + join(pluginFrontendRoot, "components/K1SpatialControls.tsx"), + "utf8", + ); + const confirmation = readFileSync( + join(pluginFrontendRoot, "physicalCommandConfirmation.ts"), + "utf8", + ); + const styles = readFileSync(join(pluginFrontendRoot, "styles.css"), "utf8"); + + assert.equal( + existsSync(join(pluginFrontendRoot, "components/K1PhysicalCommandConfirmation.tsx")), + false, + ); + assert.doesNotMatch(acquisition, /K1PhysicalCommandConfirmation|ConfirmationModal/); + assert.doesNotMatch(spatial, /K1PhysicalCommandConfirmation|ConfirmationModal/); + assert.match(acquisition, /operatorActionPhysicalAcceptance\(\)/); + assert.match(acquisition, /await submitFinalStart\(\)/); + assert.match(spatial, /stop\(operatorActionPhysicalAcceptance\(\)\)/); + assert.match(spatial, /physicalStopExecutable/); + assert.match(spatial, /canIssueCanonicalStop\(state, physicalStopIntentSpent\)/); + assert.match(spatial, /physicalStopInFlight \|\| physicalStopExecutable/); + assert.match(spatial, /connectionPolicyAllows\(state, "stop-local-receiver"\)/); + assert.doesNotMatch(spatial, /Физическая остановка K1 недоступна/); + assert.match(spatial, /onClick=\{\(\) => void stopLocalReceiver\(\)\}/); + assert.match(spatial, /Завершить локальный приём/); + assert.doesNotMatch(spatial, /Повторить остановку/); + assert.match(acquisition, /canIssueCanonicalStop\(state, physicalStopIntentSpent\)/); + assert.match(acquisition, /stopLocalReceiver/); + assert.match(acquisition, /Повторная команда устройству не отправляется/); + assert.doesNotMatch(spatial, /acknowledge-data-loss/); + assert.match(confirmation, /operatorActionPhysicalAcceptance/); + assert.match(confirmation, /operator_present:\s*true/); + assert.match(confirmation, /acquisition\.state !== "prepared"/); + assert.match(confirmation, /control\.state !== "project-ready"/); + assert.match(confirmation, /latest_device_session_state/); + assert.match(confirmation, /acquisition_start_allowed !== true/); + assert.doesNotMatch(styles, /xgrids-k1-physical-confirmation/); }); test("generic Control Station has one composition import and no K1 implementation knowledge", () => { diff --git a/apps/control-station/test/devicePluginHostPersistence.test.mjs b/apps/control-station/test/devicePluginHostPersistence.test.mjs new file mode 100644 index 0000000..7b7b54d --- /dev/null +++ b/apps/control-station/test/devicePluginHostPersistence.test.mjs @@ -0,0 +1,219 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { after, before, test } from "node:test"; + +import React, { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { createServer } from "vite"; + +let server; +let DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY; +let DevicePluginHostProvider; +let commitPersistedDeviceModelId; +let restorePersistedDeviceModelId; + +const hostSourceUrl = new URL( + "../src/core/device-plugins/DevicePluginHost.tsx", + import.meta.url, +); + +before(async () => { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ + DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY, + DevicePluginHostProvider, + commitPersistedDeviceModelId, + restorePersistedDeviceModelId, + } = await server.ssrLoadModule("/src/core/device-plugins/DevicePluginHost.tsx")); +}); + +after(async () => { + await server?.close(); +}); + +function memoryStorage(seed = {}) { + const values = new Map(Object.entries(seed)); + const calls = []; + return { + calls, + getItem(key) { + calls.push(["get", key]); + return values.get(key) ?? null; + }, + setItem(key, value) { + calls.push(["set", key, value]); + values.set(key, value); + }, + removeItem(key) { + calls.push(["remove", key]); + values.delete(key); + }, + value(key) { + return values.get(key) ?? null; + }, + }; +} + +function registryWith(...modelIds) { + const registered = new Set(modelIds); + return { + resolveModel(modelId) { + return registered.has(modelId) ? { model: { id: modelId } } : null; + }, + }; +} + +function fakePlugin(modelId) { + function RuntimeProvider({ activeModel, children }) { + return createElement( + "div", + { "data-active-model": activeModel?.id ?? "" }, + children, + ); + } + function ConnectionView() { + return null; + } + return { + manifest: { + apiVersion: "missioncore.nodedc/v1alpha1", + kind: "DevicePlugin", + metadata: { + id: "test.device.plugin", + version: "1.0.0", + displayName: "Test device", + }, + spec: { + hostApiRange: "v1alpha1", + runtime: { + backendEntrypoint: "test.device:plugin", + isolation: "transitional-in-process", + }, + permissions: [], + actions: [{ id: "state.read", mutating: false, secretFields: [] }], + models: [{ + id: modelId, + vendor: "Test", + displayName: "Test model", + category: "test", + description: "test", + verified: true, + capabilities: [], + ui: { + slot: "device.connection", + componentKey: "test.connection", + }, + }], + }, + }, + RuntimeProvider, + connectionViews: { "test.connection": ConnectionView }, + }; +} + +test("persisted model restore admits only an id in the current plugin registry", () => { + const key = DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY; + const registry = registryWith("xgrids.lixelkity-k1"); + const valid = memoryStorage({ [key]: " xgrids.lixelkity-k1 " }); + + assert.equal( + restorePersistedDeviceModelId(registry, valid), + "xgrids.lixelkity-k1", + ); + assert.equal(valid.calls.some(([operation]) => operation === "remove"), false); + + for (const staleValue of ["removed.model", " "]) { + const stale = memoryStorage({ [key]: staleValue }); + assert.equal(restorePersistedDeviceModelId(registry, stale), null); + assert.equal(stale.value(key), null, "a stale model id must be cleared"); + } + assert.equal(restorePersistedDeviceModelId(registry, null), null); +}); + +test("fresh provider mount immediately activates the registry-validated persisted model", () => { + const modelId = "test.model.one"; + const storage = memoryStorage({ + [DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY]: modelId, + }); + const previousWindow = globalThis.window; + globalThis.window = { localStorage: storage }; + try { + const markup = renderToStaticMarkup(createElement( + DevicePluginHostProvider, + { plugins: [fakePlugin(modelId)] }, + createElement("span", null, "runtime child"), + )); + assert.match(markup, /data-active-model="test\.model\.one"/); + } finally { + if (previousWindow === undefined) delete globalThis.window; + else globalThis.window = previousWindow; + } +}); + +test("successful selection commits and explicit clear removes the same durable key", () => { + const storage = memoryStorage(); + commitPersistedDeviceModelId("xgrids.lixelkity-k1", storage); + assert.equal( + storage.value(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY), + "xgrids.lixelkity-k1", + ); + + commitPersistedDeviceModelId(null, storage); + assert.equal(storage.value(DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY), null); + assert.deepEqual( + storage.calls.slice(-1)[0], + ["remove", DEVICE_PLUGIN_SELECTED_MODEL_STORAGE_KEY], + ); +}); + +test("unavailable browser storage fails closed without blocking host state", () => { + const denied = { + getItem() { + throw new Error("storage denied"); + }, + setItem() { + throw new Error("storage denied"); + }, + removeItem() { + throw new Error("storage denied"); + }, + }; + assert.equal( + restorePersistedDeviceModelId(registryWith("xgrids.lixelkity-k1"), denied), + null, + ); + assert.doesNotThrow(() => + commitPersistedDeviceModelId("xgrids.lixelkity-k1", denied) + ); + assert.doesNotThrow(() => commitPersistedDeviceModelId(null, denied)); +}); + +test("host writes persistence only after plugin deactivation succeeds", () => { + const source = readFileSync(hostSourceUrl, "utf8"); + const failedDeactivation = source.indexOf("if (!(await deactivate()))"); + const admittedState = source.indexOf("setSelectedModelId(nextModelId);"); + const durableCommit = source.indexOf( + "commitPersistedDeviceModelId(nextModelId, selectionStorage);", + admittedState, + ); + + assert.ok(failedDeactivation >= 0); + assert.ok( + failedDeactivation < admittedState && admittedState < durableCommit, + "failed deactivation branches must return before in-memory and durable commit", + ); + assert.match( + source, + /if \(nextModelId === selectedModelId\) \{[\s\S]*?commitPersistedDeviceModelId\(nextModelId, selectionStorage\);[\s\S]*?return true;/, + "explicit clear must remove stale persistence even from an already-empty host", + ); + assert.match( + source, + /useState\(\(\) =>\s*restorePersistedDeviceModelId\(registry, selectionStorage\)/, + "a fresh provider mount must restore before runtime providers receive activeModel", + ); +}); diff --git a/apps/control-station/test/k1ActiveStreamRecovery.test.mjs b/apps/control-station/test/k1ActiveStreamRecovery.test.mjs new file mode 100644 index 0000000..2a08c54 --- /dev/null +++ b/apps/control-station/test/k1ActiveStreamRecovery.test.mjs @@ -0,0 +1,977 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { after, before, test } from "node:test"; + +import React, { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { createServer } from "vite"; + +let server; +let activeStreamForceFinishAuthority; +let activeStreamForceFinishAuthorityMatches; +let activeStreamRecoveredBrowserAuthority; +let activeStreamRecoveryPresentation; +let activeStreamRecoveryPresentationAuthority; +let activeStreamRecoveryOwnsPresentationDecision; +let exactActiveStreamRecoveryLineage; +let formatActiveStreamRecoveryElapsed; +let suppressGenericErrorDuringActiveStreamRecovery; +let isXgridsActiveStreamRecovery; +let K1AcquisitionPipeline; +let K1SpatialControlsView; +let runSpatialActiveStreamForceFinish; +let shouldRenderK1GenericRuntimeError; +let shouldRenderK1OperationalPanels; +let xgridsK1Actions; +let xgridsK1Api; + +const hookSourceUrl = new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, +); +const acquisitionSourceUrl = new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx", + import.meta.url, +); +const recoverySurfaceSourceUrl = new URL( + "../../../plugins/xgrids-k1/frontend/src/components/ActiveStreamRecoverySurface.tsx", + import.meta.url, +); +const spatialControlsSourceUrl = new URL( + "../../../plugins/xgrids-k1/frontend/src/components/K1SpatialControls.tsx", + import.meta.url, +); +const connectionSourceUrl = new URL( + "../../../plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx", + import.meta.url, +); + +before(async () => { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ + activeStreamForceFinishAuthority, + activeStreamForceFinishAuthorityMatches, + activeStreamRecoveredBrowserAuthority, + activeStreamRecoveryPresentation, + activeStreamRecoveryPresentationAuthority, + activeStreamRecoveryOwnsPresentationDecision, + exactActiveStreamRecoveryLineage, + formatActiveStreamRecoveryElapsed, + suppressGenericErrorDuringActiveStreamRecovery, + } = await server.ssrLoadModule("@xgrids-k1/frontend/activeStreamRecovery.ts")); + ({ isXgridsActiveStreamRecovery, xgridsK1Api } = await server.ssrLoadModule( + "@xgrids-k1/frontend/api.ts", + )); + ({ xgridsK1Actions } = await server.ssrLoadModule( + "@xgrids-k1/frontend/manifest.ts", + )); + ({ K1AcquisitionPipeline } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1AcquisitionPipeline.tsx", + )); + ({ + K1SpatialControlsView, + runSpatialActiveStreamForceFinish, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1SpatialControls.tsx", + )); + ({ + shouldRenderK1GenericRuntimeError, + shouldRenderK1OperationalPanels, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/XgridsK1Connection.tsx", + )); +}); + +after(async () => { + await server?.close(); +}); + +function recoveryContract(overrides = {}) { + return { + schema_version: "missioncore.xgrids-k1-active-stream-recovery/v1", + state: "reconnecting", + generation: 7, + acquisition_id: "acquisition-recovery-001", + attempt: 3, + started_at_utc: "2026-08-11T19:31:00Z", + elapsed_ms: 12_400, + reason_code: "read-only-rebind-in-progress", + force_finish_allowed: true, + automatic_read_only_rebind: true, + automatic_command_retry: false, + start_performed: false, + stop_performed: false, + ble_operation_performed: false, + network_mutation_performed: false, + runtime_producer_generation: 11, + camera_recovery: "owned", + camera_media_state: "pending-first-media", + camera_media_ready: false, + camera_epoch: { + generation: 7, + init_committed: true, + init_committed_age_ms: 250, + first_media_committed: false, + first_media_committed_age_ms: null, + committed_media_segment_count: 0, + last_media_segment_age_ms: null, + }, + ...overrides, + }; +} + +function recoveryState(recoveryOverrides = {}, stateOverrides = {}) { + return { + snapshot_runtime_id: "snapshot-runtime-recovery-001", + snapshot_revision: 43, + producer_generation: 11, + phase: "reconnecting", + source_mode: "live", + acquisition: { + acquisition_id: "acquisition-recovery-001", + device_id: "device-k1-001", + device_session_id: "device-session-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_mode: "plugin-commanded", + requested_streams: ["spatial.point-cloud.live"], + target_host: "127.0.0.1", + duration_seconds: 0, + evidence_policy: "required", + state: "acquiring", + state_revision: 9, + cleanup_pending: false, + }, + connection_recovery: recoveryContract(recoveryOverrides), + ...stateOverrides, + }; +} + +function coldRestartPrePclState() { + const state = recoveryState({ + generation: 1, + acquisition_id: "acquisition-before-backend-restart-001", + attempt: 0, + started_at_utc: "2026-08-14T00:31:00Z", + elapsed_ms: 450, + reason_code: "restart-receiver-awaiting-first-pcl", + runtime_producer_generation: 1, + camera_recovery: "inactive", + camera_media_state: "inactive", + camera_media_ready: false, + camera_epoch: null, + }, { + snapshot_runtime_id: "snapshot-runtime-after-cold-restart-001", + snapshot_revision: 2, + producer_generation: 1, + acquisition: { + ...recoveryState().acquisition, + acquisition_id: "acquisition-before-backend-restart-001", + state: "awaiting_external_start", + state_revision: 4, + }, + camera_preview: { + activation_admission: { + state: "waiting-for-first-authoritative-pcl", + basis: "post-rerun-publish-pcl-frame", + runtime_producer_generation: 1, + device_command_sent: false, + }, + }, + }); + return state; +} + +function coldRestartRecoveredState() { + const state = coldRestartPrePclState(); + return { + ...state, + snapshot_revision: state.snapshot_revision + 1, + phase: "live", + acquisition: { + ...state.acquisition, + state: "acquiring", + state_revision: state.acquisition.state_revision + 1, + }, + connection_recovery: { + ...state.connection_recovery, + state: "recovered", + elapsed_ms: null, + reason_code: null, + force_finish_allowed: false, + camera_recovery: "owned", + camera_media_state: "pending-epoch", + camera_media_ready: false, + camera_epoch: null, + }, + camera_preview: { + activation_admission: { + state: "activating", + basis: "post-rerun-publish-pcl-frame", + runtime_producer_generation: 1, + device_command_sent: false, + }, + }, + }; +} + +function controller(state, overrides = {}) { + return { + state, + pendingAction: null, + error: null, + physicalStopIntentSpent: false, + physicalStopInFlight: false, + closeApplicationControlSession: async () => false, + prepareCanonicalAcquisition: async () => false, + startPreparedAcquisition: async () => false, + startReplay: async () => false, + stop: async () => false, + stopLocalReceiver: async () => false, + forceFinishActiveStreamLocally: async () => false, + abort: async () => false, + ...overrides, + }; +} + +function renderPipeline(state, overrides = {}) { + return renderToStaticMarkup(createElement(K1AcquisitionPipeline, { + controller: controller(state, overrides), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + })); +} + +function renderSpatialControls(state, overrides = {}) { + return renderToStaticMarkup(createElement(K1SpatialControlsView, { + controller: controller(state, overrides), + })); +} + +function buttonsWithText(markup, text) { + return (markup.match(//g) ?? []) + .filter((button) => button.includes(text)); +} + +function sourceSlice(source, startMarker, endMarker) { + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + assert.notEqual(start, -1, `missing source marker: ${startMarker}`); + assert.notEqual(end, -1, `missing source marker: ${endMarker}`); + return source.slice(start, end); +} + +test("active recovery contract is strict about all no-write invariants", () => { + assert.equal(isXgridsActiveStreamRecovery(recoveryContract()), true); + for (const field of [ + "automatic_command_retry", + "start_performed", + "stop_performed", + "ble_operation_performed", + "network_mutation_performed", + ]) { + assert.equal( + isXgridsActiveStreamRecovery(recoveryContract({ [field]: true })), + false, + field, + ); + } + assert.equal( + isXgridsActiveStreamRecovery(recoveryContract({ state: "retrying-command" })), + false, + ); + assert.equal( + isXgridsActiveStreamRecovery(recoveryContract({ elapsed_ms: -1 })), + false, + ); + const missingMediaState = recoveryContract(); + delete missingMediaState.camera_media_state; + assert.equal(isXgridsActiveStreamRecovery(missingMediaState), false); + assert.equal( + isXgridsActiveStreamRecovery(recoveryContract({ camera_media_ready: true })), + false, + ); + assert.equal( + isXgridsActiveStreamRecovery(recoveryContract({ + camera_media_state: "ready", + camera_media_ready: true, + camera_epoch: { + ...recoveryContract().camera_epoch, + generation: 0, + first_media_committed: true, + first_media_committed_age_ms: 1, + committed_media_segment_count: 1, + last_media_segment_age_ms: 1, + }, + })), + false, + ); +}); + +test("cold restart before first PCL owns exact reconnect UI without camera or physical actions", () => { + const state = coldRestartPrePclState(); + assert.equal(isXgridsActiveStreamRecovery(state.connection_recovery), true); + + const lineage = exactActiveStreamRecoveryLineage(state); + assert.deepEqual(lineage && { + runtime: lineage.snapshotRuntimeId, + acquisition: lineage.acquisitionId, + revision: lineage.acquisitionStateRevision, + recovery: lineage.recoveryGeneration, + producer: lineage.runtimeProducerGeneration, + }, { + runtime: "snapshot-runtime-after-cold-restart-001", + acquisition: "acquisition-before-backend-restart-001", + revision: 4, + recovery: 1, + producer: 1, + }); + assert.notEqual(activeStreamRecoveryPresentationAuthority(state), null); + assert.notEqual(activeStreamForceFinishAuthority(state), null); + assert.equal(activeStreamRecoveredBrowserAuthority(state), null); + assert.equal(state.connection_recovery.camera_recovery, "inactive"); + assert.equal(state.connection_recovery.camera_media_state, "inactive"); + assert.equal(state.connection_recovery.camera_media_ready, false); + assert.equal(state.connection_recovery.camera_epoch, null); + assert.deepEqual(state.camera_preview.activation_admission, { + state: "waiting-for-first-authoritative-pcl", + basis: "post-rerun-publish-pcl-frame", + runtime_producer_generation: 1, + device_command_sent: false, + }); + + const pipeline = renderPipeline(state); + assert.match(pipeline, /Восстанавливаем соединение/); + assert.match(pipeline, /START, STOP, Bluetooth и настройки устройства не отправляются/); + assert.equal(buttonsWithText(pipeline, "Прервать соединение").length, 1); + assert.equal(buttonsWithText(pipeline, "Запустить приём").length, 0); + assert.equal(buttonsWithText(pipeline, "Остановить устройство и запись").length, 0); + assert.equal(buttonsWithText(pipeline, "Остановить сканирование").length, 0); + assert.doesNotMatch( + pipeline, + /СВЯЗЬ ВОССТАНОВЛЕНА|Связь восстановлена · приём продолжается|Продолжаем тот же приём/, + ); + + const spatial = renderSpatialControls(state); + assert.match(spatial, /Восстанавливаем соединение/); + assert.equal(buttonsWithText(spatial, "Прервать соединение").length, 1); + assert.doesNotMatch( + spatial, + /Остановить устройство|Остановить K1|Завершить локальный приём/, + ); +}); + +test("cold restart first PCL preserves lineage and renders recovered continuation copy", () => { + const beforePcl = coldRestartPrePclState(); + const state = coldRestartRecoveredState(); + assert.equal(isXgridsActiveStreamRecovery(state.connection_recovery), true); + assert.equal( + state.connection_recovery.acquisition_id, + beforePcl.connection_recovery.acquisition_id, + ); + assert.equal( + state.connection_recovery.generation, + beforePcl.connection_recovery.generation, + ); + assert.equal( + state.connection_recovery.runtime_producer_generation, + beforePcl.connection_recovery.runtime_producer_generation, + ); + + const lineage = exactActiveStreamRecoveryLineage(state); + const browserAuthority = activeStreamRecoveredBrowserAuthority(state); + assert.deepEqual(lineage && { + runtime: lineage.snapshotRuntimeId, + acquisition: lineage.acquisitionId, + revision: lineage.acquisitionStateRevision, + recovery: lineage.recoveryGeneration, + producer: lineage.runtimeProducerGeneration, + }, { + runtime: "snapshot-runtime-after-cold-restart-001", + acquisition: "acquisition-before-backend-restart-001", + revision: 5, + recovery: 1, + producer: 1, + }); + assert.deepEqual(browserAuthority, lineage); + assert.equal(activeStreamRecoveryPresentationAuthority(state), null); + assert.equal(activeStreamForceFinishAuthority(state), null); + assert.equal(state.connection_recovery.camera_recovery, "owned"); + assert.equal(state.camera_preview.activation_admission.device_command_sent, false); + + const pipeline = renderPipeline(state); + assert.doesNotMatch(pipeline, /Восстанавливаем соединение|Прервать соединение/); + assert.match(pipeline, /СВЯЗЬ ВОССТАНОВЛЕНА · АКТИВНЫЙ ПРИЁМ/); + assert.match(pipeline, /Связь восстановлена · приём продолжается/); + assert.match(pipeline, /Продолжаем тот же приём без нового START/); + assert.doesNotMatch(pipeline, /Назовите проект и запустите приём|Запустить приём/); +}); + +test("force-finish authority requires exact runtime, acquisition, revision and generations", () => { + const state = recoveryState(); + const authority = activeStreamForceFinishAuthority(state); + assert.deepEqual(authority && { + snapshotRuntimeId: authority.snapshotRuntimeId, + acquisitionId: authority.acquisitionId, + acquisitionStateRevision: authority.acquisitionStateRevision, + recoveryGeneration: authority.recoveryGeneration, + runtimeProducerGeneration: authority.runtimeProducerGeneration, + }, { + snapshotRuntimeId: "snapshot-runtime-recovery-001", + acquisitionId: "acquisition-recovery-001", + acquisitionStateRevision: 9, + recoveryGeneration: 7, + runtimeProducerGeneration: 11, + }); + assert.equal(activeStreamForceFinishAuthorityMatches(authority, state), true); + + const staleCases = [ + (() => { + const value = structuredClone(state); + value.snapshot_runtime_id = "snapshot-runtime-recovery-002"; + return value; + })(), + (() => { + const value = structuredClone(state); + value.acquisition.acquisition_id = "acquisition-recovery-002"; + return value; + })(), + (() => { + const value = structuredClone(state); + value.acquisition.state_revision += 1; + return value; + })(), + (() => { + const value = structuredClone(state); + value.connection_recovery.generation += 1; + return value; + })(), + (() => { + const value = structuredClone(state); + value.producer_generation += 1; + return value; + })(), + ]; + for (const stale of staleCases) { + assert.equal(activeStreamForceFinishAuthorityMatches(authority, stale), false); + } + + assert.equal( + activeStreamForceFinishAuthority(recoveryState({ state: "recovered", force_finish_allowed: false })), + null, + ); + assert.equal( + activeStreamForceFinishAuthority(recoveryState({ force_finish_allowed: false })), + null, + ); +}); + +test("reconnecting presentation is exact and owns stale supervisor projection", () => { + const state = recoveryState({ force_finish_allowed: false }); + const authority = activeStreamRecoveryPresentationAuthority(state); + assert.deepEqual(authority && { + runtime: authority.snapshotRuntimeId, + acquisition: authority.acquisitionId, + producer: authority.runtimeProducerGeneration, + recovery: authority.recoveryGeneration, + }, { + runtime: "snapshot-runtime-recovery-001", + acquisition: "acquisition-recovery-001", + producer: 11, + recovery: 7, + }); + assert.equal(activeStreamRecoveryOwnsPresentationDecision(state), true); + + const wrongProducer = structuredClone(state); + wrongProducer.producer_generation += 1; + assert.equal(activeStreamRecoveryPresentationAuthority(wrongProducer), null); + assert.equal( + activeStreamRecoveryOwnsPresentationDecision(wrongProducer), + true, + "a valid reconnect contract must block stale ordinary data projection", + ); + + for (const recoveryStateName of [ + "blocked", + "standby", + "fault", + "force-finishing", + "force-finished", + ]) { + const terminal = recoveryState({ state: recoveryStateName }); + assert.equal(activeStreamRecoveryPresentationAuthority(terminal), null); + assert.equal(activeStreamRecoveryOwnsPresentationDecision(terminal), true); + } + assert.equal( + activeStreamRecoveryOwnsPresentationDecision(recoveryState({ state: "recovered" })), + false, + ); +}); + +test("recovered browser carryover keeps exact lineage without restoring recovery controls", () => { + const state = recoveryState({ + state: "recovered", + force_finish_allowed: false, + elapsed_ms: null, + reason_code: null, + }, { + phase: "live", + }); + const authority = activeStreamRecoveredBrowserAuthority(state); + assert.deepEqual(authority && { + runtime: authority.snapshotRuntimeId, + acquisition: authority.acquisitionId, + revision: authority.acquisitionStateRevision, + producer: authority.runtimeProducerGeneration, + recovery: authority.recoveryGeneration, + }, { + runtime: "snapshot-runtime-recovery-001", + acquisition: "acquisition-recovery-001", + revision: 9, + producer: 11, + recovery: 7, + }); + assert.equal(activeStreamRecoveryPresentation(state), null); + assert.equal(activeStreamForceFinishAuthority(state), null); + + const staleProducer = structuredClone(state); + staleProducer.producer_generation += 1; + assert.equal(activeStreamRecoveredBrowserAuthority(staleProducer), null); + assert.equal( + activeStreamRecoveredBrowserAuthority(recoveryState({ + state: "recovered", + camera_recovery: "blocked", + force_finish_allowed: false, + }, { phase: "live" })), + null, + ); +}); + +test("only an exact reconnecting lineage suppresses the generic red error", () => { + const state = recoveryState(); + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(state, null), true); + assert.equal( + shouldRenderK1GenericRuntimeError("Ошибка локального приёмника", false, state, null), + false, + ); + assert.equal( + suppressGenericErrorDuringActiveStreamRecovery(state, "force-finish"), + false, + ); + assert.equal( + shouldRenderK1GenericRuntimeError( + "Локальное завершение не выполнено", + false, + state, + "force-finish", + ), + true, + "a failed explicit local finish must keep the generic error banner visible", + ); + + const wrongProducer = structuredClone(state); + wrongProducer.producer_generation += 1; + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(wrongProducer), false); + + const wrongAcquisition = structuredClone(state); + wrongAcquisition.connection_recovery.acquisition_id = "acquisition-stale"; + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(wrongAcquisition), false); + + const missingRuntime = structuredClone(state); + delete missingRuntime.snapshot_runtime_id; + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(missingRuntime), false); + + const blocked = recoveryState({ state: "blocked" }); + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(blocked), false); + + const nonOwned = recoveryState({ automatic_read_only_rebind: false }); + assert.equal(suppressGenericErrorDuringActiveStreamRecovery(nonOwned), false); + + const inactiveCases = [ + recoveryState({}, { phase: "error" }), + recoveryState({}, { source_mode: "idle" }), + recoveryState({}, { + acquisition: { + ...state.acquisition, + state: "failed", + }, + }), + ]; + for (const inactive of inactiveCases) { + assert.equal( + suppressGenericErrorDuringActiveStreamRecovery(inactive, null), + false, + "stale recovery projection must fail open to the error banner", + ); + } +}); + +test("reconnecting presentation is neutral, timed and exposes explicit local finish", () => { + const state = recoveryState(); + const presentation = activeStreamRecoveryPresentation(state); + assert.equal(presentation?.state, "reconnecting"); + assert.equal(presentation?.tone, "neutral"); + assert.equal(presentation?.progressLabel, "Попытка 3 · 12 с"); + assert.equal(presentation?.forceFinishAvailable, true); + + const markup = renderPipeline(state); + assert.match(markup, /Восстанавливаем соединение/); + assert.match(markup, /Попытка 3 · 12 с/); + assert.match(markup, /class="nodedc-activity-indicator/); + assert.equal(buttonsWithText(markup, "Прервать соединение").length, 1); + assert.match(markup, /START, STOP, Bluetooth и настройки устройства не отправляются/); + assert.doesNotMatch(markup, /Ошибка локального приёмника/); + assert.equal(shouldRenderK1OperationalPanels(state), true); +}); + +test("spatial scene owns the same recovery spinner and explicit local finish", () => { + const markup = renderSpatialControls(recoveryState()); + assert.match(markup, /Восстанавливаем соединение/); + assert.match(markup, /Попытка 3 · 12 с/); + assert.match(markup, /class="nodedc-activity-indicator/); + assert.match(markup, /data-recovery-state="reconnecting"/); + assert.equal(buttonsWithText(markup, "Прервать соединение").length, 1); + assert.match(markup, /локальный front\/back-приём/); + assert.doesNotMatch( + markup, + /Остановить устройство|Остановить K1|Завершить локальный приём/, + "recovery must not expose canonical STOP or the generic receiver stop", + ); +}); + +test("spatial blocked and fault recovery copy is terminal and truthful", () => { + const blocked = renderSpatialControls(recoveryState({ + state: "blocked", + reason_code: "exact-binding-changed", + })); + assert.match(blocked, /Связь не восстановлена/); + assert.match(blocked, /Восстановление остановлено/); + assert.doesNotMatch(blocked, /class="nodedc-activity-indicator/); + assert.equal(buttonsWithText(blocked, "Прервать соединение").length, 1); + + const fault = renderSpatialControls(recoveryState({ + state: "fault", + force_finish_allowed: false, + reason_code: "active-stream-recovery-system-error", + })); + assert.match(fault, /K1 сообщил об ошибке/); + assert.match(fault, /Автоматических команд и повторов нет/); + assert.equal(buttonsWithText(fault, "Прервать соединение").length, 0); +}); + +test("spatial recovery interaction routes only to exact local force-finish", async () => { + let forceFinishCalls = 0; + const current = recoveryState(); + const invoked = await runSpatialActiveStreamForceFinish({ + state: current, + forceFinishActiveStreamLocally: async () => { + forceFinishCalls += 1; + return true; + }, + }); + assert.equal(invoked, true); + assert.equal(forceFinishCalls, 1); + + const stale = structuredClone(current); + stale.connection_recovery.runtime_producer_generation += 1; + const rejected = await runSpatialActiveStreamForceFinish({ + state: stale, + forceFinishActiveStreamLocally: async () => { + forceFinishCalls += 1; + return true; + }, + }); + assert.equal(rejected, false); + assert.equal(forceFinishCalls, 1, "stale lineage must not dispatch any action"); +}); + +test("spatial force-finish pending owns the surface without a second action", () => { + const markup = renderSpatialControls( + recoveryState({ + state: "force-finishing", + acquisition_id: null, + force_finish_allowed: false, + automatic_read_only_rebind: false, + camera_recovery: "inactive", + }), + { pendingAction: "force-finish" }, + ); + assert.match(markup, /Завершаем локальный приём/); + assert.match(markup, /data-recovery-state="force-finishing"/); + assert.match(markup, /Команда STOP устройству не отправляется/); + assert.equal(buttonsWithText(markup, "Прервать соединение").length, 0); +}); + +test("blocked, camera-blocked, standby and fault copy stay truthful", () => { + const blocked = recoveryState({ state: "blocked", reason_code: "exact-binding-changed" }); + const blockedMarkup = renderPipeline(blocked); + assert.match(blockedMarkup, /Связь не восстановлена/); + assert.match(blockedMarkup, /Восстановление остановлено/); + assert.doesNotMatch(blockedMarkup, /class="nodedc-activity-indicator/); + assert.equal(buttonsWithText(blockedMarkup, "Прервать соединение").length, 1); + + const cameraBlocked = recoveryState({ + state: "blocked", + camera_recovery: "blocked", + reason_code: "camera-recovery-failed", + }); + assert.match(renderPipeline(cameraBlocked), /Видеопоток не восстановлен/); + + const standby = recoveryState({ + state: "standby", + force_finish_allowed: false, + reason_code: "device-reported-standby", + }); + const standbyMarkup = renderPipeline(standby); + assert.match(standbyMarkup, /Устройство перешло в ожидание/); + assert.match(standbyMarkup, /без команды STOP/); + assert.equal(buttonsWithText(standbyMarkup, "Прервать соединение").length, 0); + assert.equal(shouldRenderK1OperationalPanels(standby), true); + + const fault = recoveryState({ + state: "fault", + force_finish_allowed: false, + reason_code: "active-stream-recovery-system-error", + }); + const faultMarkup = renderPipeline(fault); + assert.match(faultMarkup, /K1 сообщил об ошибке/); + assert.match(faultMarkup, /Автоматических команд и повторов нет/); + assert.equal(buttonsWithText(faultMarkup, "Прервать соединение").length, 0); + assert.equal(shouldRenderK1OperationalPanels(fault), true); +}); + +test("recovered active lineage renders the continued session and one exact STOP", () => { + const state = recoveryState({ + state: "recovered", + force_finish_allowed: false, + elapsed_ms: null, + reason_code: null, + }, { + phase: "live", + compatibility: { + vendor_writes_enabled: true, + permitted_mode: "active-control", + }, + application_control_session: { + session_generation: 5, + state_revision: 8, + state: "scanning", + can_stop: true, + control_socket_open: true, + }, + connection_policy: { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["stop-acquisition"], + actions: { + "stop-acquisition": { + allowed: true, + reason_codes: [], + target_source: "connection-supervisor", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }, + }); + assert.equal(activeStreamRecoveryPresentation(state), null); + const markup = renderPipeline(state); + assert.doesNotMatch(markup, /Восстанавливаем соединение|Прервать соединение/); + assert.match(markup, /СВЯЗЬ ВОССТАНОВЛЕНА · АКТИВНЫЙ ПРИЁМ/); + assert.match(markup, /Связь восстановлена · приём продолжается/); + assert.match(markup, /Продолжаем тот же приём без нового START/); + assert.doesNotMatch(markup, /Назовите проект и запустите приём|Запустить приём/); + + const stopButtons = buttonsWithText(markup, "Остановить устройство и запись"); + assert.equal(stopButtons.length, 1); + assert.doesNotMatch(stopButtons[0], /\bdisabled(?:=|\s|>)/); +}); + +test("stale recovered marker on an idle released runtime fails closed to idle UI", () => { + const state = recoveryState({ + state: "recovered", + force_finish_allowed: false, + elapsed_ms: null, + reason_code: null, + }, { + phase: "idle", + source_mode: "idle", + acquisition: { + ...recoveryState().acquisition, + state: "completed", + cleanup_pending: false, + }, + application_control_session: { + session_generation: 5, + state_revision: 9, + state: "completed", + can_stop: false, + control_socket_open: false, + }, + }); + assert.equal(activeStreamRecoveryPresentation(state), null); + const markup = renderPipeline(state); + assert.doesNotMatch( + markup, + /СВЯЗЬ ВОССТАНОВЛЕНА|Связь восстановлена · приём продолжается|Продолжаем тот же приём/, + ); + assert.match(markup, /Назовите проект и запустите приём/); + assert.equal(buttonsWithText(markup, "Остановить устройство и запись").length, 0); + assert.equal(buttonsWithText(markup, "Остановить сканирование").length, 0); +}); + +test("force-finishing shows one local-only pending owner and no second action", () => { + const state = recoveryState({ + state: "force-finishing", + acquisition_id: null, + force_finish_allowed: false, + automatic_read_only_rebind: false, + runtime_producer_generation: 11, + camera_recovery: "inactive", + }); + const markup = renderPipeline(state, { pendingAction: "force-finish" }); + assert.match(markup, /Завершаем локальный приём/); + assert.match(markup, /Команда STOP устройству не отправляется/); + assert.match(markup, /class="nodedc-activity-indicator/); + assert.equal(buttonsWithText(markup, "Прервать соединение").length, 0); +}); + +test("elapsed presentation is deterministic", () => { + assert.equal(formatActiveStreamRecoveryElapsed(null), null); + assert.equal(formatActiveStreamRecoveryElapsed(-1), null); + assert.equal(formatActiveStreamRecoveryElapsed(999), "0 с"); + assert.equal(formatActiveStreamRecoveryElapsed(59_999), "59 с"); + assert.equal(formatActiveStreamRecoveryElapsed(60_000), "1 мин"); + assert.equal(formatActiveStreamRecoveryElapsed(125_900), "2 мин 5 с"); +}); + +test("force-finish manifest/API sends the exact fenced local-only request", async () => { + assert.equal( + xgridsK1Actions.acquisitionForceFinishLocal, + "acquisition.force-finish-local", + ); + const request = { + expected_snapshot_runtime_id: "snapshot-runtime-recovery-001", + acquisition_id: "acquisition-recovery-001", + expected_state_revision: 9, + expected_recovery_generation: 7, + operator_confirmed: true, + operation_id: "op-00000000-0000-4000-8000-000000000321", + idempotency_key: + "acquisition.force-finish-local:op-00000000-0000-4000-8000-000000000321", + deadline_seconds: 30, + }; + let capturedUrl = null; + let capturedInit = null; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (input, init) => { + capturedUrl = String(input); + capturedInit = init; + return new Response(JSON.stringify({ state: recoveryState() }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }; + try { + await xgridsK1Api.forceFinishAcquisitionLocally(request); + } finally { + globalThis.fetch = originalFetch; + } + assert.match( + capturedUrl, + /\/actions\/acquisition\.force-finish-local$/, + ); + assert.equal(capturedInit.method, "POST"); + assert.deepEqual(JSON.parse(capturedInit.body), { input: request }); +}); + +test("state API rejects a drifted active recovery contract", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => new Response(JSON.stringify({ + state: recoveryState({ stop_performed: true }), + }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + try { + await assert.rejects( + () => xgridsK1Api.getState(), + /некорректное состояние/, + ); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("frontend boundary keeps recovery automatic work read-only and local finish explicit", () => { + const hookSource = readFileSync(hookSourceUrl, "utf8"); + const forceFinish = sourceSlice( + hookSource, + "const forceFinishActiveStreamLocally", + "const abort", + ); + assert.match(forceFinish, /run\("force-finish"/); + assert.match(forceFinish, /activeStreamForceFinishAuthority\(latestState\.current\)/); + assert.match(forceFinish, /expected_snapshot_runtime_id:\s*authority\.snapshotRuntimeId/); + assert.match(forceFinish, /acquisition_id:\s*authority\.acquisitionId/); + assert.match(forceFinish, /expected_state_revision:\s*authority\.acquisitionStateRevision/); + assert.match(forceFinish, /expected_recovery_generation:\s*authority\.recoveryGeneration/); + assert.match(forceFinish, /operator_confirmed:\s*true/); + assert.match( + forceFinish, + /newMutationContext\("acquisition\.force-finish-local"\)/, + ); + assert.equal( + (forceFinish.match(/forceFinishAcquisitionLocally\(/g) ?? []).length, + 1, + ); + assert.doesNotMatch( + forceFinish, + /startAcquisition|stopAcquisition|scanBle|selectCameraPreview|connect\(/, + ); + + const acquisitionSource = readFileSync(acquisitionSourceUrl, "utf8"); + assert.equal( + (acquisitionSource.match(/forceFinishActiveStreamLocally\(\)/g) ?? []).length, + 1, + "the explicit recovery button is the only frontend caller", + ); + const recoverySurface = readFileSync(recoverySurfaceSourceUrl, "utf8"); + assert.match(recoverySurface, / { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ normalizeXgridsK1MissionState } = await server.ssrLoadModule( + "@xgrids-k1/frontend/runtimeContext.tsx", + )); + ({ + SnapshotRuntimeActionArbiter, + connectionActionAuthoritySnapshot, + exactAppliedNetworkIntentCompleted, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/useXgridsK1Runtime.ts", + )); + ({ selectMonotonicXgridsState } = await server.ssrLoadModule( + "@xgrids-k1/frontend/stateOrdering.ts", + )); + ({ xgridsK1Manifest } = await server.ssrLoadModule( + "@xgrids-k1/frontend/manifest.ts", + )); + ({ workspaces } = await server.ssrLoadModule("/src/productModel.ts")); + ({ connectionModeOptions } = await server.ssrLoadModule( + "@xgrids-k1/frontend/configuration.ts", + )); + ({ K1Metrics } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1Metrics.tsx", + )); + ({ + K1ProvisioningPipeline, + RuntimeActionFenceTestContext, + emptySearchPresentation, + emptyProvisioningAttemptPresentation, + provisioningAttemptViewState, + unavailablePhysicalRetirementAuthority, + connectionAttemptOwnsAppliedNetworkRecovery, + connectionRecoveryObservationTargetMatches, + connectionRecoveryEscapeKey, + connectionRecoveryEscapeAfterScan, + connectionRecoveryIsRequired, + clearConnectionFailureAfterSuccessfulRefresh, + dispatchConnectionRecoveryObservationForCurrentRuntime, + dispatchUnavailablePhysicalRetirementForCurrentRuntime, + dispatchRetiredPhysicalReconciliationForCurrentRuntime, + retiredPhysicalReopenVerificationContext, + admitPhysicalReopenPresentation, + physicalReopenClickAuthority, + physicalReopenPresentationIsCurrent, + physicalReopenSettlementIsCurrent, + scenarioResetPresentationBoundary, + localScenarioActionEpochIsCurrent, + explicitProvisioningDraftMatches, + localProvisioningDraftFenceKey, + exactChangeNetworkCandidate, + connectionActionAuthorityMatches, + reconfigurationContinuationAuthority, + observedConnectionAuthorityAllowsTarget, + connectContinuationAuthority, + runtimeActionFenceMatches, + currentRuntimeActionRequest, + connectedReconfigurationActionApplicable, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1ProvisioningPipeline.tsx", + )); + ({ K1AcquisitionPipeline } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1AcquisitionPipeline.tsx", + )); + ({ K1Diagnostics } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1Diagnostics.tsx", + )); + ({ + K1SpatialControlsView, + k1SpatialAuthorityState, + k1SpatialPhasePresentation, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1SpatialControls.tsx", + )); + ({ + connectionModeSelectionGuidance, + connectionPolicyOperatorGuidance, + physicalRetirementGuidance, + physicalReopenGuidance, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/presentation.ts", + )); + ({ + reconfigurationAllowsFreshDevice, + readOnlyObservationShowsNetworkUnavailable, + readOnlyFailureShowsNetworkUnavailable, + provisioningFailureRequiresFreshCandidate, + trustedConnectionBinding, + transportRefEquivalenceKey, + shouldRevealProvisioningNetworkStep, + retiredPhysicalReopenAuthority, + canIssueCanonicalStop, + physicalStopIntentCheckpoint, + authoritativeStateSupersedesPhysicalStopIntent, + connectionAttemptForRuntimeError, + recommendedConnectionRecoveryObservationTarget, + readOnlyConnectionObservationTarget, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/lifecycle.ts", + )); + ({ + K1ConnectionPipelines, + physicalRecoveryConnectionDetail, + shouldRenderK1OperationalPanels, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/XgridsK1Connection.tsx", + )); + ({ + K1OperatorError, + attemptNetworkPhaseLabel, + } = await server.ssrLoadModule( + "@xgrids-k1/frontend/components/K1OperatorError.tsx", + )); + ({ hostFailureDiagnosticPresentation } = await server.ssrLoadModule( + "@xgrids-k1/frontend/hostDiagnosticPresentation.ts", + )); + shellPresentation = await server.ssrLoadModule("/src/presentation.ts"); + ({ contourRuntimeAuthorityPresentation } = await server.ssrLoadModule( + "/src/workspaces/ContourHealthWorkspace.tsx", + )); +}); + +after(async () => { + await server?.close(); +}); + +function sourceSlice(source, startMarker, endMarker) { + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + assert.notEqual(start, -1, `missing source marker: ${startMarker}`); + assert.notEqual(end, -1, `missing source marker: ${endMarker}`); + return source.slice(start, end); +} + +test("K1 one-intent source contract makes mode reset explicit and keeps device I/O separate", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const search = sourceSlice(source, "const repeatDeviceScan", "const submitConnect"); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + const mode = sourceSlice(source, "const changeDesiredConnectionMode", "const selectFreshDevice"); + const selection = sourceSlice( + source, + "const selectFreshDevice", + "const chooseAnother", + ); + + assert.match(search, /scanWithResult\(\{[^}]*durationSeconds:\s*6/); + assert.equal((search.match(/scanWithResult\(/g) ?? []).length, 1); + assert.doesNotMatch(search, /\b(?:connect|verifyConnection|submitConnect)\s*\(/); + + assert.match(mode, /await selectConnectionMode\(\{/); + assert.match(mode, /reset_scenario: true/); + assert.match(mode, /reset_id: resetId/); + assert.doesNotMatch( + mode, + /scanWithResult\(|verifyConnection\(|connect\(|prepareConnection/, + ); + assert.doesNotMatch( + selection, + /await|scanWithResult\(|verifyConnection\(|connect\(|retireUnavailablePhysicalCommand|reopenRetiredPhysicalReconciliation/, + ); + assert.match(selection, /requestExplicitProvisioning\(device\.device_id/); + assert.match(selection, /const actionableDevices = devices\.filter\(candidateSelectionAllowed\)/); + assert.doesNotMatch(selection, /physicallyRetired|reopenAuthority/); + const resultRows = sourceSlice( + source, + '
', + '
', + ); + assert.match(resultRows, /actionLabel="Выбрать"/); + assert.match(resultRows, /onSelect=\{\(\) => selectCandidate\(device\)\}/); + assert.doesNotMatch(resultRows, /Переподключиться|reopen|verifyConnection/); + assert.match(source, /onChange=\{\(event\) => setSsid\(event\.target\.value\)\}/); + assert.match(source, /onChange=\{\(event\) => setPassword\(event\.target\.value\)\}/); + + assert.equal((apply.match(/await connect\(/g) ?? []).length, 1); + assert.doesNotMatch( + apply, + /scanWithResult\(|verifyConnection\(|retireUnavailablePhysicalCommand|reopenRetiredPhysicalReconciliation|candidateRefreshRequest|void submitConnect/, + ); + assert.doesNotMatch(source, /allow_host_wifi_switch/); + assert.doesNotMatch(source, /<(?:button|input|select|textarea)\b/); + assert.doesNotMatch( + source, + /(?:color|background(?:-color)?):\s*(?:#[0-9a-f]{3,8}|rgba?\()/i, + ); + for (const sharedControl of ["Button", "IconButton", "TextField", "ActivityIndicator", "StatusBadge"]) { + assert.match(source, new RegExp(`<${sharedControl}\\b`), sharedControl); + } + assert.equal((source.match(/buttonLabel:\s*"Применить"/g) ?? []).length, 3); + assert.match(source, /устарел|stale/i); + assert.match(source, /неизвест|outcome-unknown|safe_to_retry/i); +}); + +test("exact network-applied REST proof completes frontend Apply before control is ready", () => { + const request = { + device_id: "BLE-K1-ONE", + connection_mode: "bridge", + idempotency_key: "apply-intent-one", + }; + const operation = { + operation_id: "network-operation-one", + action: "network.provision", + status: "succeeded", + idempotency_key: request.idempotency_key, + context: { connection_mode: request.connection_mode }, + result: { phase: "network_applied" }, + }; + const exactState = (controlState) => ({ + connection_attempt: { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: operation.operation_id, + phase: "network_applied", + connection_mode: request.connection_mode, + control_state: controlState, + }, + network_mutation_ledger: { + status: "resolved", + mutation_allowed: true, + operation_id: operation.operation_id, + intended_mode: request.connection_mode, + resolution: "target-observed", + transport_ref: request.device_id, + }, + connection_supervisor: { + observed: { + device_network: { + state: "applied", + connection_mode: request.connection_mode, + transport_ref: request.device_id.toLowerCase(), + }, + }, + }, + }); + + assert.equal( + exactAppliedNetworkIntentCompleted( + exactState("control_not_ready"), + request, + operation, + ), + true, + ); + assert.equal( + exactAppliedNetworkIntentCompleted(exactState("unknown"), request, operation), + true, + ); + assert.equal( + exactAppliedNetworkIntentCompleted({ + ...exactState("control_not_ready"), + connection_attempt: { + ...exactState("control_not_ready").connection_attempt, + attempt_id: "another-operation", + }, + }, request, operation), + false, + ); + + const runtime = readFileSync(new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), "utf8"); + const connectFlow = sourceSlice( + runtime, + "const connect = useCallback", + "const verifyConnection = useCallback", + ); + assert.match( + connectFlow, + /const exactNetworkIntentCompleted = exactAppliedNetworkIntentCompleted\([\s\S]*?if \(\s*!exactNetworkIntentCompleted\s*&& !hasExactConnectionReady\([\s\S]*?return requireExactConnectionReady\([\s\S]*?nextState = acceptSuccessfulConnectState\(nextState\)/, + ); + assert.match( + connectFlow, + /failedOperation\?\.status === "succeeded"[\s\S]*?!exactAppliedNetworkIntentCompleted\([\s\S]*?&& !hasExactConnectionReady/, + ); + assert.doesNotMatch( + connectFlow, + /openApplicationControlSession|waitForControlPhase|verifyConnection\(|scanWithResult\(/, + ); +}); + +function model() { + const activeModel = xgridsK1Manifest.spec.models[0]; + assert.ok(activeModel); + return activeModel; +} + +function supervisor({ + control = true, + data = false, + dataPlaneState = "idle", +} = {}) { + const target = { ipv4: "192.168.68.52", port: 1883 }; + return { + schema_version: "missioncore.k1-connection-supervisor/v1", + revision: 9, + closed: false, + intent: { + intent_id: "intent-001", + requested_mode: "bridge", + expected_device_id: "device-k1-001", + requested_at: OBSERVED_AT, + }, + observed: { + device_network: { + state: "applied", + intent_id: "intent-001", + transport_ref: "ble-k1-001", + connection_mode: "bridge", + target, + source: "ble-read-only-status", + observed_at: OBSERVED_AT, + }, + host_path: { + epoch: 5, + available: true, + fingerprint: "en0:192.168.68.10", + interface: "en0", + source_ipv4: "192.168.68.10", + route_class: "direct", + reason_code: null, + observed_at: OBSERVED_AT, + }, + endpoint: { + target, + tcp_state: "reachable", + intent_id: "intent-001", + host_path_epoch: 5, + reason_code: null, + observed_at: OBSERVED_AT, + }, + device_identity: { + state: control ? "verified" : "stale", + intent_id: "intent-001", + logical_device_id: "device-k1-001", + compatibility_profile_id: PROFILE_ID, + connection_mode: "bridge", + source: "mqtt-device-info", + host_path_epoch: 5, + observed_at: OBSERVED_AT, + }, + control_plane: { + state: control ? "healthy" : "lost", + session_id: control ? "control-session-001" : null, + host_path_epoch: control ? 5 : null, + reason_code: control ? null : "control-heartbeat-lost", + observed_at: OBSERVED_AT, + }, + data_plane: { + state: data ? "healthy" : dataPlaneState, + session_id: data || dataPlaneState !== "idle" ? "data-session-001" : null, + host_path_epoch: data || dataPlaneState !== "idle" ? 5 : null, + reason_code: dataPlaneState === "lost" ? "data-heartbeat-lost" : null, + observed_at: data || dataPlaneState !== "idle" ? OBSERVED_AT : null, + }, + }, + lease: { + state: "reachable", + generation: 6, + intent_id: "intent-001", + host_path_epoch: 5, + connection_mode: "bridge", + target, + logical_device_id: "device-k1-001", + reason_code: null, + observed_at: OBSERVED_AT, + }, + authority: { + network_mutation_allowed: false, + control_allowed: control, + acquisition_start_allowed: control, + data_ingest_authoritative: control && data, + physical_motion_allowed: false, + reason_codes: control ? [] : ["control-heartbeat-lost"], + }, + last_known: null, + allowed_actions: control ? ["stop-acquisition"] : ["probe-endpoint"], + }; +} + +function connectionLifecycle({ + control = true, + mode = "bridge", + revision = 9, +} = {}) { + return { + schema_version: "missioncore.xgrids-k1-connection-lifecycle/v1", + revision, + desired_mode: mode, + configured_mode: mode, + active_mode: control ? mode : null, + mode_change: { + state: control ? "ready" : "awaiting-control", + from: mode, + to: mode, + }, + mode_selection: { + allowed: true, + reason_codes: [], + automatic_retry: false, + }, + active_binding_key: control ? `binding-intent-001-${mode}` : null, + active_binding: control + ? { + binding_key: `binding-intent-001-${mode}`, + intent_id: "intent-001", + transport_ref: "ble-k1-001", + connection_mode: mode, + target_ipv4: "192.168.68.52", + target_port: 1883, + host_path_epoch: 5, + control_session_id: "control-session-001", + logical_device_id: "device-k1-001", + compatibility_profile_id: PROFILE_ID, + } + : null, + connection_ready: control, + ready_to_start: control, + operation: null, + allowed_actions: control + ? ["start-acquisition", "select-connection-mode"] + : ["select-connection-mode", "verify-control-read-only"], + automatic_retry: false, + }; +} + +function runtimeState() { + return { + phase: "live", + source_mode: "live", + ble_discovery_generation: 4, + connection_mode: "bridge", + configured_connection_mode: "bridge", + active_connection_mode: "bridge", + desired_connection_mode: "bridge", + desired_connection_mode_revision: 0, + k1_ip: "192.168.68.52", + rerun_grpc_url: "rerun+http://127.0.0.1:9877/proxy", + compatibility: { + profile_id: PROFILE_ID, + vendor_writes_enabled: true, + permitted_mode: "active-control", + }, + device_ref: { + device_id: "device-k1-001", + model_id: model().id, + identity_stability: "stable", + identity_basis: "hardware-identifier", + }, + device_session: { + device_session_id: "device-session-001", + device_id: "device-k1-001", + compatibility_profile_id: PROFILE_ID, + connectivity: "connected", + }, + acquisition: { + acquisition_id: "acquisition-001", + device_id: "device-k1-001", + device_session_id: "device-session-001", + compatibility_profile_id: PROFILE_ID, + control_mode: "plugin-commanded", + requested_streams: ["spatial.point-cloud.live"], + target_host: "127.0.0.1", + duration_seconds: 0, + evidence_policy: "required", + state: "acquiring", + state_revision: 4, + }, + sensor_catalog: { + schema_version: "missioncore.sensor-catalog/v1alpha2", + revision: "runtime-test", + streams: [{ + stream_id: "spatial.point-cloud.live", + modality: "point-cloud", + availability: "streaming", + }], + }, + metrics: { + pcl_frames: 10, + pose_frames: 5, + pipeline_ms: 12.5, + frame_rate_hz: 8, + point_count: 42_000, + ai_frame_rate_hz: 5, + device_elapsed_seconds: 30, + }, + connection_supervisor: supervisor({ control: true, data: true }), + connection_lifecycle: connectionLifecycle(), + }; +} + +function activeRecoveryRuntimeState(recoveryOverrides = {}, stateOverrides = {}) { + const state = runtimeState(); + return { + ...state, + snapshot_runtime_id: "runtime-active-recovery-001", + snapshot_revision: 51, + producer_generation: 13, + phase: "reconnecting", + connection_supervisor: supervisor({ + control: false, + data: false, + dataPlaneState: "lost", + }), + connection_lifecycle: connectionLifecycle({ control: false }), + connection_recovery: { + schema_version: "missioncore.xgrids-k1-active-stream-recovery/v1", + state: "reconnecting", + generation: 5, + acquisition_id: state.acquisition.acquisition_id, + attempt: 1, + started_at_utc: "2026-08-11T21:19:59Z", + elapsed_ms: 16_869, + reason_code: "host-route-unavailable", + force_finish_allowed: true, + automatic_read_only_rebind: true, + automatic_command_retry: false, + start_performed: false, + stop_performed: false, + ble_operation_performed: false, + network_mutation_performed: false, + runtime_producer_generation: 13, + camera_recovery: "owned", + camera_media_state: "pending-epoch", + camera_media_ready: false, + camera_epoch: null, + ...recoveryOverrides, + }, + ...stateOverrides, + }; +} + +function normalize(state) { + return normalizeXgridsK1MissionState({ state }, model()); +} + +function durableTopologyState() { + return { + phase: "idle", + source_mode: "idle", + ble_discovery_generation: 0, + connection_mode: "bridge", + configured_connection_mode: "bridge", + active_connection_mode: null, + desired_connection_mode: "bridge", + desired_connection_mode_revision: 0, + devices: [], + operations: [], + network_write_reconciliation: null, + network_mutation_ledger: { + status: "empty", + mutation_allowed: true, + reason_code: null, + operation_id: null, + transport_ref: null, + intended_mode: null, + stage: null, + revision: null, + resolution: null, + updated_at_utc: null, + }, + semantic_topology_store: { + status: "available", + configured_offline_evidence: true, + live_connection_authority: false, + reason_code: null, + record: { + schema_version: "missioncore.xgrids-k1-semantic-topology/v1", + revision: 3, + transport_ref: "ble-k1-001", + connection_mode: "bridge", + ipv4: "192.168.68.52", + compatibility_profile_id: PROFILE_ID, + firmware_version: "3.0.2", + source: "ble-read-only-status", + observed_at_utc: OBSERVED_AT, + }, + }, + }; +} + +function provisioningController(state) { + return { + state, + pendingAction: null, + scan: async () => undefined, + connect: async () => ({ + succeeded: false, + networkIntentCompleted: false, + intentDisposition: "retain", + acceptedSessionKey: null, + }), + verifyConnection: async () => ({ + succeeded: false, + reconciliationCompleted: false, + observedState: state, + }), + retireUnavailablePhysicalCommand: async () => false, + retireUnavailablePhysicalCommandWithResult: async () => ({ + succeeded: false, + observedState: state, + }), + reopenRetiredPhysicalReconciliation: async () => ({ + succeeded: false, + observedState: state, + }), + prepareConnectionReconfiguration: async () => false, + probeConfiguredEndpoint: async () => undefined, + }; +} + +function provisioningAttemptPresentation(overrides = {}) { + return { + snapshotRuntimeId: "runtime-network-action", + connectionMode: "bridge", + idempotencyKey: "apply-intent-one", + attemptId: null, + deviceId: "f89438fa-55ed-85ad-eed7-734ac84746d8", + label: "XGR-A46BE7", + rssi: -51, + ssid: "FIELD-NET", + localPhase: "submitting", + failureMessage: null, + freshStartAllowed: false, + ...overrides, + }; +} + +function terminalConnectionRecoveryState({ + phase = "network_outcome_unknown", + safeNextAction = "verify-control-read-only", + includeConfiguredTarget = true, + includeScan = true, +} = {}) { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-terminal-connection-recovery"; + state.snapshot_runtime_started_at_utc = "2026-08-11T18:00:00Z"; + state.snapshot_revision = 27; + state.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "op-b7ea404d-b0a2-460a-80bd-6922e85a2df5", + connection_mode: "bridge", + status: "failed", + phase, + control_state: "unknown", + stage: "network-operation-failed", + public_error_code: null, + side_effect_status: phase === "network_outcome_unknown" ? "unknown" : "none", + safe_next_action: safeNextAction, + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + diagnostic_bundle: { + schema_version: "missioncore.xgrids-k1-connection-diagnostic/v1", + redacted: true, + generated_at_utc: OBSERVED_AT, + snapshot_runtime_id: "runtime-terminal-connection-recovery", + attempt: {}, + network_mutation_ledger: {}, + connection_supervisor: {}, + automatic_retry: false, + }, + }; + const allowedActions = []; + const actions = {}; + if (includeConfiguredTarget) { + allowedActions.push("observe-configured-device-network"); + actions["observe-configured-device-network"] = { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + automatic_retry: false, + }; + } + if (includeScan) { + allowedActions.push("scan-ble"); + actions["scan-ble"] = { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }; + } + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + recommended_action: includeConfiguredTarget + ? "observe-configured-device-network" + : includeScan + ? "scan-ble" + : "manual-recovery-required", + allowed_actions: allowedActions, + actions, + }; + return state; +} + +function reopenedPhysicalState({ + stage = "observing", + advertisedRef = "f89438fa-55ed-85ad-eed7-734ac84746d8", + backendRef = "F89438FA-55ED-85AD-EED7-734AC84746D8", + reopeningId = "reopening-returned-k1", + discoveryGeneration = 13, + snapshotRevision = 20, +} = {}) { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-reopened-k1"; + state.snapshot_runtime_started_at_utc = "2026-08-10T12:00:00Z"; + state.snapshot_revision = snapshotRevision; + state.desired_connection_mode = "bridge"; + state.desired_connection_mode_revision = 7; + state.ble_discovery_generation = discoveryGeneration; + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.devices = [{ + device_id: advertisedRef, + name: "XGR-A46BE7", + rssi: -61, + connectable: true, + likely_k1: true, + }]; + state.physical_command = { + status: "unresolved", + reason_code: "physical-command-reconciliation-required", + requires_reconciliation: true, + automatic_replay_allowed: false, + normal_session_recovery_supported: false, + runtime_bound: false, + reconciliation_ready: true, + record: { + revision: 282, + operation_id: "old-stop-operation", + action: "stop", + stage, + resolution: null, + connection: { transport_ref: backendRef }, + operator_retirements: [{ + retirement_id: "retirement-old-stop", + retired_transport_ref: backendRef, + }], + operator_reconciliation_reopens: [{ + reopening_id: reopeningId, + retirement_id: "retirement-old-stop", + retired_record_revision: 281, + reopened_transport_ref: advertisedRef, + discovery_generation: discoveryGeneration, + reason: "device-returned-for-explicit-reconciliation", + }], + }, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { + retained_context_is_presence: false, + retired_transport_refs: [], + }, + allowed_actions: ["observe-fresh-device-network", "scan-ble"], + actions: { + "observe-fresh-device-network": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: backendRef, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "fresh-scan", + required_transport_ref: backendRef, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + }; + return state; +} + +function retiredPhysicalReopenReadyState({ + advertisedRef = "f89438fa-55ed-85ad-eed7-734ac84746d8", + backendRef = "F89438FA-55ED-85AD-EED7-734AC84746D8", + discoveryGeneration = 13, + snapshotRuntimeId = "runtime-reopened-k1", +} = {}) { + const state = reopenedPhysicalState({ + advertisedRef, + backendRef, + discoveryGeneration, + }); + state.snapshot_runtime_id = snapshotRuntimeId; + state.physical_command = { + status: "resolved", + reason_code: null, + requires_reconciliation: false, + automatic_replay_allowed: false, + normal_session_recovery_supported: false, + runtime_bound: false, + reconciliation_ready: false, + operator_reconciliation_reopen: { + allowed: true, + reason_codes: [], + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: backendRef, + expected_discovery_generation: discoveryGeneration, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + device_io_performed: false, + automatic_retry: false, + }, + record: { + revision: 281, + operation_id: "old-stop-operation", + action: "stop", + stage: "resolved", + resolution: "operator-retired-outcome-unknown", + connection: { + transport_ref: backendRef, + connection_mode: "bridge", + }, + operator_retirements: [{ + retirement_id: "retirement-old-stop", + retired_transport_ref: backendRef, + }], + operator_reconciliation_reopens: [], + }, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { + retained_context_is_presence: false, + retired_transport_refs: [backendRef], + }, + allowed_actions: ["scan-ble"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-target-retired"], + target_source: "fresh-scan", + required_transport_ref: backendRef, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + }; + return state; +} + + +function reopenedPhysicalAuthority({ discoveryGeneration = 13 } = {}) { + return { + snapshotRuntimeId: "runtime-reopened-k1", + connectionMode: "bridge", + desiredModeRevision: 7, + reconfigurationRevision: 0, + reconfigurationIntentId: null, + activeBindingKey: null, + discoveryGeneration, + }; +} + +function connectionReconfiguration({ + intent = null, + status = intent ? "awaiting-fresh-scan" : "idle", + revision = 0, + intentId = intent ? `reconfigure-${revision}` : null, + requiredTransportRef = null, + requiredConnectionMode = null, + discoveryGeneration = null, + requiredTransportObserved = null, +} = {}) { + return { + schema_version: "missioncore.xgrids-k1-connection-reconfiguration/v1", + revision, + intent_id: intentId, + intent, + status, + required_transport_ref: requiredTransportRef, + required_connection_mode: requiredConnectionMode, + minimum_discovery_generation: intent ? (discoveryGeneration ?? 1) : null, + fresh_discovery_generation: status === "fresh-scan-completed" + ? discoveryGeneration + : null, + required_transport_observed: requiredTransportObserved, + prepared_at: intent ? OBSERVED_AT : null, + automatic_retry: false, + }; +} + +function acquisitionController(state) { + return { + state, + pendingAction: null, + error: null, + physicalStopIntentSpent: false, + physicalStopInFlight: false, + closeApplicationControlSession: async () => false, + prepareCanonicalAcquisition: async () => false, + startPreparedAcquisition: async () => false, + startReplay: async () => false, + stop: async () => false, + stopLocalReceiver: async () => false, + abort: async () => false, + }; +} + +function connectionPipelinesController(state) { + return { + ...provisioningController(state), + ...acquisitionController(state), + backendStatus: "offline", + eventStatus: "disconnected", + latencyHistory: [], + scanWithResult: async () => ({ + observedState: state, + discoveryGeneration: state?.ble_discovery_generation ?? 0, + }), + prepareConnectionReconfigurationWithResult: async () => ({ + succeeded: false, + observedState: state, + }), + getConnectionActionAuthority: () => null, + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => true, + }; +} + +function renderConnectionPipelines(state) { + return renderToStaticMarkup(createElement(K1ConnectionPipelines, { + controller: connectionPipelinesController(state), + desiredConnectionMode: "bridge", + onDesiredConnectionModeChange() {}, + operationalPanelsVisible: shouldRenderK1OperationalPanels(state), + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + sourceLabel: "Ожидание", + })); +} + +function buttonMarkupWithText(markup, text) { + return (markup.match(//g) ?? []) + .filter((button) => button.includes(text)); +} + +function assertCanonicalConnectionCopy(markup) { + assert.doesNotMatch( + markup, + /[Ии]сходн|[Сс]охранён|[Пп]редыдущ|[Фф]изическ|[Пп]одключаемся/u, + ); +} + +function deviceRowOpeningTag(markup, deviceId) { + const deviceOffset = markup.indexOf(`${deviceId}`); + assert.notEqual(deviceOffset, -1, `${deviceId} must be rendered`); + const rowOffset = markup.lastIndexOf('
", rowOffset) + 1); +} + +function deviceRowActionButton(markup, deviceId) { + const deviceOffset = markup.indexOf(`${deviceId}`); + assert.notEqual(deviceOffset, -1, `${deviceId} must be rendered`); + const buttonOffset = markup.indexOf("", buttonOffset); + assert.notEqual(buttonEnd, -1, `${deviceId} action must be complete`); + return markup.slice(buttonOffset, buttonEnd + "".length); +} + +function renderProvisioning(props) { + return renderToStaticMarkup(createElement(K1ProvisioningPipeline, props)); +} + +function captureProvisioningTree(props) { + let capturedTree = null; + + function CaptureHarness() { + capturedTree = K1ProvisioningPipeline(props); + return capturedTree; + } + + renderToStaticMarkup(createElement(CaptureHarness)); + assert.ok(capturedTree); + return capturedTree; +} + +function captureProvisioningTreeAfterSearch( + props, + { completedDiscoveryGeneration } = {}, +) { + let capturedTree = null; + + function SearchCompletedCaptureHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + dispatcher.useState = (initialState) => { + if (initialState === emptySearchPresentation) { + const state = props.controller.state; + return [{ + sequence: 1, + snapshotRuntimeId: state.snapshot_runtime_id ?? null, + connectionMode: props.desiredMode, + desiredModeRevision: state.desired_connection_mode_revision ?? null, + active: false, + completedDiscoveryGeneration: completedDiscoveryGeneration + ?? state.ble_discovery_generation + ?? null, + }, () => undefined]; + } + return originalUseState(initialState); + }; + try { + capturedTree = K1ProvisioningPipeline(props); + return capturedTree; + } finally { + dispatcher.useState = originalUseState; + } + } + + renderToStaticMarkup(createElement(SearchCompletedCaptureHarness)); + assert.ok(capturedTree); + return capturedTree; +} + +function elementByProp(node, propName, expectedValue) { + if (Array.isArray(node)) { + for (const child of node) { + const found = elementByProp(child, propName, expectedValue); + if (found) return found; + } + return null; + } + if (!React.isValidElement(node)) return null; + if (node.props[propName] === expectedValue) return node; + return elementByProp(node.props.children, propName, expectedValue); +} + +function createStatefulProvisioningHarness(initialProps) { + const hookSlots = []; + let currentProps = initialProps; + let capturedTree = null; + let pendingEffects = []; + + const dependenciesMatch = (left, right) => Boolean( + left + && right + && left.length === right.length + && left.every((value, index) => Object.is(value, right[index])), + ); + + const render = (nextProps = currentProps) => { + currentProps = nextProps; + pendingEffects = []; + + function StatefulCaptureHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originals = { + useState: dispatcher.useState, + useRef: dispatcher.useRef, + useMemo: dispatcher.useMemo, + useCallback: dispatcher.useCallback, + useEffect: dispatcher.useEffect, + }; + let hookIndex = 0; + + dispatcher.useState = (initialState) => { + const index = hookIndex; + hookIndex += 1; + if (!hookSlots[index]) { + hookSlots[index] = { + kind: "state", + initializer: initialState, + value: typeof initialState === "function" + ? initialState() + : initialState, + }; + } + const slot = hookSlots[index]; + assert.equal(slot.kind, "state"); + const setValue = (nextValue) => { + slot.value = typeof nextValue === "function" + ? nextValue(slot.value) + : nextValue; + }; + return [slot.value, setValue]; + }; + dispatcher.useRef = (initialValue) => { + const index = hookIndex; + hookIndex += 1; + if (!hookSlots[index]) { + hookSlots[index] = { + kind: "ref", + value: { current: initialValue }, + }; + } + const slot = hookSlots[index]; + assert.equal(slot.kind, "ref"); + return slot.value; + }; + dispatcher.useMemo = (factory, dependencies) => { + const index = hookIndex; + hookIndex += 1; + const previous = hookSlots[index]; + if ( + !previous + || previous.kind !== "memo" + || !dependenciesMatch(previous.dependencies, dependencies) + ) { + hookSlots[index] = { + kind: "memo", + dependencies, + value: factory(), + }; + } + return hookSlots[index].value; + }; + dispatcher.useCallback = (callback, dependencies) => { + const index = hookIndex; + hookIndex += 1; + const previous = hookSlots[index]; + if ( + !previous + || previous.kind !== "callback" + || !dependenciesMatch(previous.dependencies, dependencies) + ) { + hookSlots[index] = { + kind: "callback", + dependencies, + value: callback, + }; + } + return hookSlots[index].value; + }; + dispatcher.useEffect = (effect, dependencies) => { + const index = hookIndex; + hookIndex += 1; + const previous = hookSlots[index]; + const changed = !previous + || previous.kind !== "effect" + || !dependenciesMatch(previous.dependencies, dependencies); + hookSlots[index] = { + kind: "effect", + dependencies, + cleanup: previous?.kind === "effect" ? previous.cleanup : undefined, + }; + if (changed) pendingEffects.push({ index, effect }); + }; + + try { + capturedTree = K1ProvisioningPipeline(currentProps); + } finally { + dispatcher.useState = originals.useState; + dispatcher.useRef = originals.useRef; + dispatcher.useMemo = originals.useMemo; + dispatcher.useCallback = originals.useCallback; + dispatcher.useEffect = originals.useEffect; + } + return null; + } + + renderToStaticMarkup(createElement(StatefulCaptureHarness)); + assert.ok(capturedTree); + return capturedTree; + }; + + const flushEffects = () => { + const effects = pendingEffects; + pendingEffects = []; + for (const { index, effect } of effects) { + const slot = hookSlots[index]; + slot.cleanup?.(); + const cleanup = effect(); + slot.cleanup = typeof cleanup === "function" ? cleanup : undefined; + } + }; + + const stateValue = (initializer) => hookSlots.find( + (slot) => slot?.kind === "state" && slot.initializer === initializer, + )?.value; + + const dispose = () => { + for (const slot of hookSlots) { + if (slot?.kind === "effect") slot.cleanup?.(); + } + }; + + return { dispose, flushEffects, render, stateValue }; +} + +function reactNodeText(node) { + if (typeof node === "string" || typeof node === "number") return String(node); + if (Array.isArray(node)) return node.map(reactNodeText).join(""); + if (!React.isValidElement(node)) return ""; + return reactNodeText(node.props.children); +} + +function actionByLabel(node, label) { + if (Array.isArray(node)) { + for (const child of node) { + const found = actionByLabel(child, label); + if (found) return found; + } + return null; + } + if (!React.isValidElement(node)) return null; + if ( + typeof node.props.onClick === "function" + && reactNodeText(node.props.children).includes(label) + ) return node; + return actionByLabel(node.props.children, label); +} + +function renderProvisioningWithAttempt(props, presentation) { + function AttemptHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + dispatcher.useState = (initialState) => { + if (initialState === emptyProvisioningAttemptPresentation) { + return [presentation, () => undefined]; + } + return originalUseState(initialState); + }; + try { + return K1ProvisioningPipeline(props); + } finally { + dispatcher.useState = originalUseState; + } + } + + return renderToStaticMarkup(createElement(AttemptHarness)); +} + +function captureProvisioningTreeWithAttempt(props, presentation) { + let capturedTree = null; + + function AttemptCaptureHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + dispatcher.useState = (initialState) => { + if (initialState === emptyProvisioningAttemptPresentation) { + return [presentation, () => undefined]; + } + return originalUseState(initialState); + }; + try { + capturedTree = K1ProvisioningPipeline(props); + return capturedTree; + } finally { + dispatcher.useState = originalUseState; + } + } + + renderToStaticMarkup(createElement(AttemptCaptureHarness)); + assert.ok(capturedTree); + return capturedTree; +} + +function renderProvisioningAfterSearch( + props, + { completedDiscoveryGeneration } = {}, +) { + function SearchCompletedHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + dispatcher.useState = (initialState) => { + if (initialState === emptySearchPresentation) { + const state = props.controller.state; + return [{ + sequence: 1, + snapshotRuntimeId: state.snapshot_runtime_id ?? null, + connectionMode: props.desiredMode, + desiredModeRevision: state.desired_connection_mode_revision ?? null, + active: false, + completedDiscoveryGeneration: completedDiscoveryGeneration + ?? state.ble_discovery_generation + ?? null, + }, () => undefined]; + } + return originalUseState(initialState); + }; + try { + return K1ProvisioningPipeline(props); + } finally { + dispatcher.useState = originalUseState; + } + } + + return renderToStaticMarkup(createElement(SearchCompletedHarness)); +} + + +function renderProvisioningWithCurrentPendingAction( + props, + { publicSearch = true } = {}, +) { + function CurrentPendingActionHarness() { + const state = props.controller.state; + const fence = { + snapshotRuntimeId: state.snapshot_runtime_id, + connectionMode: props.desiredMode, + desiredModeRevision: state.desired_connection_mode_revision, + reconfigurationRevision: state.connection_reconfiguration?.revision ?? 0, + reconfigurationIntentId: state.connection_reconfiguration?.intent_id ?? null, + activeBindingKey: state.connection_lifecycle?.active_binding_key ?? null, + discoveryGeneration: state.ble_discovery_generation, + clickToken: 1, + }; + return createElement( + RuntimeActionFenceTestContext.Provider, + { value: fence }, + publicSearch + ? createElement(SearchCompletedPendingHarness, { props }) + : createElement(K1ProvisioningPipeline, props), + ); + } + + function SearchCompletedPendingHarness({ props: childProps }) { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + dispatcher.useState = (initialState) => { + if (initialState === emptySearchPresentation) { + const state = childProps.controller.state; + return [{ + sequence: 1, + snapshotRuntimeId: state.snapshot_runtime_id ?? null, + connectionMode: childProps.desiredMode, + desiredModeRevision: state.desired_connection_mode_revision ?? null, + active: true, + completedDiscoveryGeneration: null, + }, () => undefined]; + } + return originalUseState(initialState); + }; + try { + return K1ProvisioningPipeline(childProps); + } finally { + dispatcher.useState = originalUseState; + } + } + + return renderToStaticMarkup(createElement(CurrentPendingActionHarness)); +} + + +function renderProvisioningWithRetainedDraft( + props, + { device, draft, ssid, password }, +) { + function RetainedDraftHarness() { + const internals = + React.__CLIENT_INTERNALS_DO_NOT_USE_OR_WARN_USERS_THEY_CANNOT_UPGRADE; + const dispatcher = internals?.H; + assert.equal(typeof dispatcher?.useState, "function"); + const originalUseState = dispatcher.useState; + let emptyStringStateIndex = 0; + let nullStateIndex = 0; + dispatcher.useState = (initialState) => { + if (initialState === "") { + emptyStringStateIndex += 1; + const seededValue = emptyStringStateIndex === 1 + ? device.device_id + : emptyStringStateIndex === 2 + ? ssid + : emptyStringStateIndex === 3 + ? password + : initialState; + return [seededValue, () => undefined]; + } + if (initialState === null) { + nullStateIndex += 1; + if (nullStateIndex === 1) return [device, () => undefined]; + if (nullStateIndex === 8) return [draft, () => undefined]; + } + return originalUseState(initialState); + }; + try { + return K1ProvisioningPipeline(props); + } finally { + dispatcher.useState = originalUseState; + } + } + + return renderToStaticMarkup(createElement(RetainedDraftHarness)); +} + + +test("K1 provisioning password authority is scoped to one exact operator draft", () => { + const draft = { + snapshotRuntimeId: "runtime-a", + deviceId: "ble-k1-001", + connectionMode: "bridge", + desiredModeRevision: 7, + discoveryGeneration: 13, + reconfigurationRevision: 5, + reconfigurationIntentId: "reconfigure-001", + activeBindingKey: null, + requiredTransportRef: "ble-k1-001", + requiredConnectionMode: "bridge", + }; + const current = { + snapshotRuntimeId: "runtime-a", + deviceId: "ble-k1-001", + connectionMode: "bridge", + desiredModeRevision: 7, + discoveryGeneration: 13, + reconfigurationRevision: 5, + reconfigurationIntentId: "reconfigure-001", + activeBindingKey: null, + requiredTransportRef: "ble-k1-001", + requiredConnectionMode: "bridge", + }; + + assert.equal(explicitProvisioningDraftMatches(draft, current), true); + for (const drifted of [ + { ...current, snapshotRuntimeId: "runtime-b" }, + { ...current, deviceId: "another-k1" }, + { ...current, connectionMode: "direct-connect" }, + { ...current, desiredModeRevision: 8 }, + { ...current, discoveryGeneration: 14 }, + { ...current, reconfigurationRevision: 6 }, + { ...current, reconfigurationIntentId: "reconfigure-002" }, + { ...current, activeBindingKey: "binding-new" }, + { ...current, requiredTransportRef: "ble-k1-002" }, + { ...current, requiredConnectionMode: "direct-connect" }, + { ...current, deviceId: null }, + ]) { + assert.equal(explicitProvisioningDraftMatches(draft, drifted), false); + } +}); + +test("K1 provisioning local credential fence changes with backend runtime authority", () => { + const fence = { + snapshotRuntimeId: "runtime-a", + reconfigurationRevision: 5, + reconfigurationIntentId: "reconfigure-001", + activeBindingKey: null, + requiredTransportRef: "ble-k1-001", + requiredConnectionMode: "bridge", + }; + assert.notEqual( + localProvisioningDraftFenceKey(fence), + localProvisioningDraftFenceKey({ + ...fence, + snapshotRuntimeId: "runtime-b", + }), + ); +}); + +test("K1 click-owned action requests disappear across runtime and click interleavings", () => { + const authorityA = { + snapshotRuntimeId: "runtime-a", + connectionMode: "bridge", + desiredModeRevision: 7, + reconfigurationRevision: 10, + reconfigurationIntentId: "reconfigure-a", + activeBindingKey: "binding-a", + discoveryGeneration: 5, + }; + const firstClick = { + ...authorityA, + clickToken: 17, + }; + const awaitingScan = { + ...firstClick, + deviceId: "ble-k1-001", + previousDiscoveryGeneration: 5, + expectedDiscoveryGeneration: null, + scanTransportRefs: [], + stage: "awaiting-scan", + }; + let latestAuthority = authorityA; + const latestAuthorityMatches = (fence) => + connectionActionAuthorityMatches(fence, latestAuthority); + + assert.equal( + runtimeActionFenceMatches( + firstClick, + "runtime-a", + firstClick, + latestAuthorityMatches, + ), + true, + ); + assert.equal( + currentRuntimeActionRequest( + awaitingScan, + "runtime-a", + firstClick, + latestAuthorityMatches, + ), + awaitingScan, + ); + + // Same-runtime authority drift must invalidate the old click before the new + // React render commits. A late Quick candidate cannot re-arm or auto-submit. + latestAuthority = { ...authorityA, discoveryGeneration: 6 }; + assert.equal( + runtimeActionFenceMatches( + firstClick, + "runtime-a", + firstClick, + latestAuthorityMatches, + ), + false, + ); + assert.equal( + currentRuntimeActionRequest( + awaitingScan, + "runtime-a", + firstClick, + latestAuthorityMatches, + ), + null, + ); + + // Simulate acceptState(runtime-b) updating latestState.current before React + // commits the runtime-b render. + latestAuthority = { ...authorityA, snapshotRuntimeId: "runtime-b" }; + + const secondClick = { + ...latestAuthority, + clickToken: 18, + }; + assert.equal( + runtimeActionFenceMatches( + firstClick, + "runtime-b", + secondClick, + latestAuthorityMatches, + ), + false, + ); + assert.equal( + runtimeActionFenceMatches( + secondClick, + "runtime-b", + secondClick, + latestAuthorityMatches, + ), + true, + ); +}); + + +test("K1 reconfiguration continuation requires exact correlated +1 revisions", () => { + const fence = { + snapshotRuntimeId: "runtime-a", + connectionMode: "bridge", + desiredModeRevision: 7, + reconfigurationRevision: 10, + reconfigurationIntentId: null, + activeBindingKey: "binding-a", + discoveryGeneration: 5, + }; + const preparedState = { + snapshot_runtime_id: "runtime-a", + desired_connection_mode: "bridge", + desired_connection_mode_revision: 7, + ble_discovery_generation: 6, + connection_reconfiguration: connectionReconfiguration({ + intent: "select-device", + status: "awaiting-fresh-scan", + revision: 11, + intentId: "reconfigure-b", + discoveryGeneration: 6, + }), + connection_lifecycle: { + active_binding_key: null, + active_binding: null, + }, + }; + const preparedAuthority = { + ...fence, + reconfigurationRevision: 11, + reconfigurationIntentId: "reconfigure-b", + activeBindingKey: null, + discoveryGeneration: 6, + }; + assert.deepEqual( + reconfigurationContinuationAuthority( + fence, + preparedState, + "select-device", + preparedAuthority, + ), + preparedAuthority, + ); + + for (const drifted of [ + { ...preparedAuthority, discoveryGeneration: 7 }, + { ...preparedAuthority, reconfigurationRevision: 12 }, + { ...preparedAuthority, desiredModeRevision: 8 }, + { ...preparedAuthority, activeBindingKey: "foreign-binding" }, + ]) { + assert.equal( + reconfigurationContinuationAuthority( + fence, + preparedState, + "select-device", + drifted, + ), + null, + ); + } + assert.equal( + reconfigurationContinuationAuthority( + fence, + preparedState, + "change-network", + preparedAuthority, + ), + null, + ); + const contaminatedPreparedState = { + ...preparedState, + connection_lifecycle: { + active_binding_key: "binding-foreign", + active_binding: { + binding_key: "binding-foreign", + transport_ref: "ble-k1-foreign", + connection_mode: "bridge", + }, + }, + }; + assert.equal( + reconfigurationContinuationAuthority( + fence, + contaminatedPreparedState, + "select-device", + { ...preparedAuthority, activeBindingKey: "binding-foreign" }, + ), + null, + ); + + const cancelFence = preparedAuthority; + const cancelledState = { + ...preparedState, + ble_discovery_generation: 7, + connection_reconfiguration: connectionReconfiguration({ revision: 12 }), + }; + const cancelledAuthority = { + ...cancelFence, + reconfigurationRevision: 12, + reconfigurationIntentId: null, + discoveryGeneration: 7, + }; + assert.deepEqual( + reconfigurationContinuationAuthority( + cancelFence, + cancelledState, + "cancel", + cancelledAuthority, + ), + cancelledAuthority, + ); +}); + +test("separately explicit Verify rejects a foreign same-runtime binding", () => { + const exact = { + selected_device_id: "ble-k1-001", + connection_mode: "bridge", + connection_lifecycle: { + active_binding_key: "binding-a", + active_binding: { + binding_key: "binding-a", + transport_ref: "ble-k1-001", + connection_mode: "bridge", + }, + }, + }; + assert.equal( + observedConnectionAuthorityAllowsTarget( + exact, + "ble-k1-001", + "bridge", + ), + true, + ); + assert.equal( + observedConnectionAuthorityAllowsTarget( + { + ...exact, + selected_device_id: "ble-k1-foreign", + connection_lifecycle: { + active_binding_key: "binding-b", + active_binding: { + binding_key: "binding-b", + transport_ref: "ble-k1-foreign", + connection_mode: "bridge", + }, + }, + }, + "ble-k1-001", + "bridge", + { allowUnbound: true }, + ), + false, + ); + assert.equal( + observedConnectionAuthorityAllowsTarget( + { + selected_device_id: null, + connection_mode: null, + connection_lifecycle: { + active_binding_key: null, + active_binding: null, + }, + }, + "ble-k1-001", + "bridge", + { allowUnbound: true }, + ), + true, + ); +}); + +test("K1 Connect continuation accepts only unchanged or exactly consumed authority", () => { + const ordinaryFence = { + snapshotRuntimeId: "runtime-a", + connectionMode: "bridge", + desiredModeRevision: 7, + reconfigurationRevision: 10, + reconfigurationIntentId: null, + activeBindingKey: null, + discoveryGeneration: 5, + }; + const connectedState = ({ + deviceId = "ble-k1-001", + reconfigurationRevision = 10, + discoveryGeneration = 5, + } = {}) => ({ + snapshot_runtime_id: "runtime-a", + desired_connection_mode: "bridge", + desired_connection_mode_revision: 7, + ble_discovery_generation: discoveryGeneration, + selected_device_id: deviceId, + connection_mode: "bridge", + connection_reconfiguration: connectionReconfiguration({ + revision: reconfigurationRevision, + }), + connection_lifecycle: { + active_binding_key: `binding-${deviceId}`, + active_binding: { + binding_key: `binding-${deviceId}`, + transport_ref: deviceId, + connection_mode: "bridge", + }, + }, + }); + const ordinaryState = connectedState(); + const ordinaryCurrent = { + ...ordinaryFence, + activeBindingKey: "binding-ble-k1-001", + }; + assert.deepEqual( + connectContinuationAuthority( + ordinaryFence, + ordinaryState, + ordinaryCurrent, + "ble-k1-001", + "bridge", + ), + ordinaryCurrent, + ); + + const reconfigurationFence = { + ...ordinaryFence, + reconfigurationIntentId: "reconfigure-a", + }; + const consumedState = connectedState({ + reconfigurationRevision: 11, + discoveryGeneration: 6, + }); + const consumedCurrent = { + ...reconfigurationFence, + reconfigurationRevision: 11, + reconfigurationIntentId: null, + activeBindingKey: "binding-ble-k1-001", + discoveryGeneration: 6, + }; + assert.deepEqual( + connectContinuationAuthority( + reconfigurationFence, + consumedState, + consumedCurrent, + "ble-k1-001", + "bridge", + ), + consumedCurrent, + ); + + const skippedState = connectedState({ + reconfigurationRevision: 12, + discoveryGeneration: 7, + }); + const skippedCurrent = { + ...consumedCurrent, + reconfigurationRevision: 12, + discoveryGeneration: 7, + }; + assert.equal( + connectContinuationAuthority( + reconfigurationFence, + skippedState, + skippedCurrent, + "ble-k1-001", + "bridge", + ), + null, + ); + const foreignState = connectedState({ deviceId: "ble-k1-foreign" }); + assert.equal( + connectContinuationAuthority( + ordinaryFence, + foreignState, + { + ...ordinaryCurrent, + activeBindingKey: "binding-ble-k1-foreign", + }, + "ble-k1-001", + "bridge", + ), + null, + ); +}); + +test("K1 runtime replacement retires A and isolates a new B action from late settlement", () => { + const arbiter = new SnapshotRuntimeActionArbiter(); + const actionA = arbiter.begin(4); + assert.ok(actionA); + let pendingAction = "verify"; + let acceptedState = "runtime-a"; + let surfacedError = null; + + assert.equal( + arbiter.retireForSnapshotChange("runtime-a", "runtime-b"), + true, + ); + pendingAction = null; + acceptedState = "runtime-b"; + assert.equal(pendingAction, null, "runtime-b controls are enabled without A loader"); + + const actionB = arbiter.begin(4); + assert.ok(actionB, "runtime-b click is admitted before action A settles"); + pendingAction = "scan"; + + const acceptLateResult = (token, nextState) => { + if (!arbiter.isCurrent(token)) return false; + acceptedState = nextState; + return true; + }; + const surfaceLateFailure = (token, message) => { + if (!arbiter.isCurrent(token)) return false; + surfacedError = message; + return true; + }; + const finish = (token) => { + if (!arbiter.settle(token)) return false; + pendingAction = null; + return true; + }; + + assert.equal(acceptLateResult(actionA, "runtime-a-late"), false); + assert.equal(surfaceLateFailure(actionA, "late A error"), false); + assert.equal(finish(actionA), false); + assert.equal(acceptedState, "runtime-b"); + assert.equal(surfacedError, null); + assert.equal(pendingAction, "scan", "late A finally cannot clear B pending state"); + + assert.equal(finish(actionB), true); + assert.equal(pendingAction, null); +}); + +test("explicit scenario reset B supersedes pending callback A in one runtime", () => { + const arbiter = new SnapshotRuntimeActionArbiter(); + const actionA = arbiter.begin(12); + assert.ok(actionA); + + const actionB = arbiter.begin(12, true); + assert.ok(actionB, "reset B is admitted while A is still pending"); + assert.equal(arbiter.isCurrent(actionA), false); + assert.equal(arbiter.settle(actionA), false, "late A cannot clear B loader"); + assert.equal(arbiter.isCurrent(actionB), true); + assert.equal(arbiter.settle(actionB), true); +}); + +test("acquisition and spatial STOP surfaces share one runtime action slot", () => { + const arbiter = new SnapshotRuntimeActionArbiter(); + let physicalStopCallCount = 0; + const clickPhysicalStop = () => { + const token = arbiter.begin(7); + if (!token) return null; + physicalStopCallCount += 1; + return token; + }; + + const acquisitionSurface = clickPhysicalStop(); + const spatialSurface = clickPhysicalStop(); + assert.ok(acquisitionSurface); + assert.equal(spatialSurface, null); + assert.equal(physicalStopCallCount, 1); + assert.equal(arbiter.settle(acquisitionSurface), true); +}); + +test("change-network exact lookup never substitutes a neighboring UUID", () => { + const reconfiguration = connectionReconfiguration({ + intent: "change-network", + status: "fresh-scan-completed", + revision: 8, + requiredTransportRef: "ble-k1-original", + requiredConnectionMode: "bridge", + discoveryGeneration: 11, + requiredTransportObserved: true, + }); + const devices = [ + { device_id: "ble-k1-foreign", name: "Nearby", connectable: true }, + { device_id: "ble-k1-original", name: "Original", connectable: true }, + ]; + assert.equal( + exactChangeNetworkCandidate(reconfiguration, devices, 11)?.device_id, + "ble-k1-original", + ); + assert.equal(exactChangeNetworkCandidate(reconfiguration, devices, 12), null); + assert.equal( + exactChangeNetworkCandidate( + { ...reconfiguration, required_transport_observed: false }, + devices, + 11, + ), + null, + ); +}); + +test("cold connection workflow exposes the mode selector and explicit Scan immediately", () => { + const state = durableTopologyState(); + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.devices = [{ + device_id: "ble-k1-001", + name: "XGR-A46BE7", + rssi: -45, + connectable: true, + likely_k1: true, + }]; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble", "provision-fresh-device"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "prepare-select-device": { + allowed: false, + reason_codes: ["acquisition-active"], + target_source: "local-prestart-handoff", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + device_write_performed: false, + automatic_retry: false, + }, + "prepare-change-network": { + allowed: false, + reason_codes: ["acquisition-active"], + target_source: "local-prestart-handoff", + required_transport_ref: "ble-k1-001", + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + device_write_performed: false, + automatic_retry: false, + }, + }, + }; + + const markup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(state), + desiredMode: "bridge", + })); + assert.equal(physicalRecoveryConnectionDetail(state), null); + assert.match(markup, /ПОДКЛЮЧЕНИЕ · ШАГИ 01–02/); + assert.match(markup, /01<\/span>/); + assert.match(markup, /

Подключение<\/h3>/); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /Питание|индикатор горит/); + assert.doesNotMatch(markup, /Название общей сети Wi‑Fi|Пароль Wi‑Fi/); + assert.doesNotMatch(markup, /class="nodedc-activity-indicator"/); + const scanButton = buttonMarkupWithText(markup, "Найти по Bluetooth")[0]; + assert.ok(scanButton); + assert.doesNotMatch(scanButton, /\bdisabled(?:=|\s|>)/); + assert.equal( + buttonMarkupWithText(markup, "Подключить новый K1").length, + 0, + ); + const modeToggle = markup.match( + /]*aria-label="Способ подключения"[^>]*>/, + )?.[0]; + assert.ok(modeToggle); + assert.doesNotMatch(modeToggle, /\bdisabled(?:=|\s|>)/); + assertCanonicalConnectionCopy(markup); +}); + +test("cold Scan click dispatches from the live runtime fence and releases it for retry", async () => { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-live-scan-click"; + state.snapshot_revision = 1; + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.connection_reconfiguration = connectionReconfiguration(); + state.connection_lifecycle = connectionLifecycle({ control: false }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + const authority = connectionActionAuthoritySnapshot(state, "bridge"); + assert.ok(authority); + let scanCalls = 0; + let scanDispatched = null; + const controller = { + ...provisioningController(state), + scanWithResult: async () => { + scanCalls += 1; + scanDispatched?.(); + return { + succeeded: true, + snapshotRuntimeId: state.snapshot_runtime_id, + discoveryGeneration: state.ble_discovery_generation + scanCalls, + transportRefs: [], + }; + }, + selectConnectionMode: async () => true, + getConnectionActionAuthority: () => authority, + isSnapshotRuntimeCurrent: (runtimeId) => runtimeId === state.snapshot_runtime_id, + isConnectionActionAuthorityCurrent: (candidate) => + connectionActionAuthorityMatches(candidate, authority), + isConnectionPolicyActionAllowedCurrent: (action) => action === "scan-ble", + }; + const tree = captureProvisioningTree({ + controller, + desiredMode: "bridge", + }); + const scanButton = actionByLabel(tree, "Найти по Bluetooth"); + assert.ok(scanButton); + + const clickAndWaitForDispatch = async () => { + const dispatched = new Promise((resolve) => { + scanDispatched = resolve; + }); + scanButton.props.onClick(); + await Promise.race([ + dispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("Scan click did not reach scanWithResult")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + }; + + await clickAndWaitForDispatch(); + await clickAndWaitForDispatch(); + assert.equal(scanCalls, 2); +}); + +test("scenario reset epoch rejects every late settlement from the old local scenario", () => { + assert.equal(localScenarioActionEpochIsCurrent(4, 4), true); + assert.equal(localScenarioActionEpochIsCurrent(4, 5), false); + + const source = readFileSync(provisioningSourceUrl, "utf8"); + const search = sourceSlice(source, "const repeatDeviceScan", "const submitConnect"); + const afterScan = search.indexOf("scanResult = await scanWithResult"); + const epochGuard = search.indexOf( + "localScenarioActionEpochIsCurrent", + afterScan, + ); + const firstSettlement = search.indexOf("setEscapedAppliedAttemptKey", afterScan); + assert.ok(afterScan >= 0); + assert.ok(epochGuard > afterScan); + assert.ok(firstSettlement > epochGuard); + + const toolbarResetBoundary = sourceSlice( + source, + "if (\n scenarioResetPresentationKey", + "const currentRuntimeActionFence", + ); + assert.match(toolbarResetBoundary, /localScenarioActionEpoch\.current \+= 1/); + assert.match(toolbarResetBoundary, /activeRuntimeActionFence\.current = null/); + assert.match(toolbarResetBoundary, /activeConnectIntent\.current = null/); + + const resetEffect = sourceSlice( + source, + "useEffect(() => {\n if (\n !scenarioResetPresentationKey", + "const searchPresentationIsCurrent", + ); + assert.match(resetEffect, /setConnectionAttemptPresentation\(null\)/); + assert.match(resetEffect, /setPreparingReconfigurationRequest\(null\)/); + assert.match(resetEffect, /setCandidateUnavailableMessage\(null\)/); + assert.doesNotMatch(source, /physicalReopenInFlight|setPhysicalReopenPresentation/); +}); + +test("toolbar reset in one runtime rejects late Scan A and admits exactly one Scan B", async () => { + const initialState = durableTopologyState(); + initialState.snapshot_runtime_id = "runtime-toolbar-reset-during-scan"; + initialState.snapshot_revision = 1; + initialState.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + initialState.connection_reconfiguration = connectionReconfiguration(); + initialState.connection_lifecycle = connectionLifecycle({ control: false }); + initialState.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + + let currentState = initialState; + let resolveScanA; + let markScanADispatched; + let markScanBDispatched; + const scanAResult = new Promise((resolve) => { + resolveScanA = resolve; + }); + const scanADispatched = new Promise((resolve) => { + markScanADispatched = resolve; + }); + const scanBDispatched = new Promise((resolve) => { + markScanBDispatched = resolve; + }); + let scanCalls = 0; + let scanBCalls = 0; + + const controller = { + ...provisioningController(initialState), + selectConnectionMode: async () => true, + scanWithResult: async () => { + scanCalls += 1; + if (scanCalls === 1) { + markScanADispatched(); + return scanAResult; + } + scanBCalls += 1; + markScanBDispatched(); + return { + succeeded: true, + snapshotRuntimeId: currentState.snapshot_runtime_id, + discoveryGeneration: currentState.ble_discovery_generation + 1, + transportRefs: [], + }; + }, + getCurrentState: () => currentState, + getConnectionActionAuthority: (mode) => + connectionActionAuthoritySnapshot(currentState, mode), + isSnapshotRuntimeCurrent: (runtimeId) => + runtimeId === currentState.snapshot_runtime_id, + isConnectionActionAuthorityCurrent: (candidate) => + connectionActionAuthorityMatches( + candidate, + connectionActionAuthoritySnapshot( + currentState, + candidate.connectionMode, + ), + ), + isConnectionPolicyActionAllowedCurrent: (action) => + currentState.connection_policy.allowed_actions.includes(action), + }; + const props = () => ({ + controller: { ...controller, state: currentState }, + desiredMode: "bridge", + }); + const harness = createStatefulProvisioningHarness(props()); + + try { + let tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + harness.flushEffects(); + const scanAButton = actionByLabel(tree, "Найти по Bluetooth"); + assert.ok(scanAButton); + scanAButton.props.onClick(); + await Promise.race([ + scanADispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("Scan A did not dispatch")), + 100, + )), + ]); + + const resetState = structuredClone(initialState); + resetState.snapshot_revision = 2; + resetState.desired_connection_mode_revision = 1; + resetState.connection_scenario_reset = { + reset_id: "toolbar-reset-same-runtime", + request_revision: 0, + revision: 1, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: 0, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: null, + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + operation_sequence: 2, + }; + currentState = resetState; + + tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + harness.flushEffects(); + assert.equal(harness.stateValue(emptySearchPresentation), null); + assert.ok(actionByLabel(tree, "Найти по Bluetooth")); + assert.equal(actionByLabel(tree, "Повторить поиск Bluetooth"), null); + assert.equal(actionByLabel(tree, "Применить"), null); + + resolveScanA({ + succeeded: true, + snapshotRuntimeId: initialState.snapshot_runtime_id, + discoveryGeneration: 1, + transportRefs: ["stale-device-from-scan-a"], + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + harness.flushEffects(); + assert.equal( + harness.stateValue(emptySearchPresentation), + null, + "late Scan A must not repopulate the reset search latch", + ); + assert.ok(actionByLabel(tree, "Найти по Bluetooth")); + assert.equal(actionByLabel(tree, "Повторить поиск Bluetooth"), null); + assert.equal(actionByLabel(tree, "Применить"), null); + + const scanBButton = actionByLabel(tree, "Найти по Bluetooth"); + scanBButton.props.onClick(); + await Promise.race([ + scanBDispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("Scan B did not dispatch")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(scanCalls, 2); + assert.equal(scanBCalls, 1); + } finally { + harness.dispose(); + } +}); + +test("local connection navigation and selector copy use process nouns", () => { + const localConnection = workspaces.find((workspace) => workspace.id === "local-device"); + assert.equal(localConnection?.label, "Подключение"); + assert.equal(localConnection?.title, "Подключение"); + assert.equal(model().displayName, "XGRIDS LixelKity K1"); + assert.deepEqual( + connectionModeOptions.map(({ label }) => label), + [ + "Общая сеть · Bridge", + "Локальная сеть · Quick Connect", + "Хотспот контроллера · Direct Connect", + ], + ); + assertCanonicalConnectionCopy( + connectionModeOptions + .map(({ label, description }) => `${label} ${description ?? ""}`) + .join(" "), + ); +}); + +test("cold adjacent connection panels do not repeat the model or device noun", () => { + const state = durableTopologyState(); + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble", "provision-fresh-device"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + const provisioningMarkup = renderToStaticMarkup(createElement( + K1ProvisioningPipeline, + { controller: provisioningController(state), desiredMode: "bridge" }, + )); + const acquisitionMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: acquisitionController(state), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + const diagnosticsMarkup = renderToStaticMarkup(createElement( + K1Diagnostics, + { + controller: { + state, + backendStatus: "offline", + eventStatus: "disconnected", + latencyHistory: [], + }, + sourceLabel: "Ожидание", + }, + )); + assertCanonicalConnectionCopy( + `${provisioningMarkup}${acquisitionMarkup}${diagnosticsMarkup}`, + ); +}); + +test("legacy live fields do not create device, stream or metrics authority", () => { + const state = runtimeState(); + delete state.connection_supervisor; + + const normalized = normalize(state); + assert.ok(normalized); + assert.equal(normalized.activeDevice, null); + assert.equal(normalized.deviceSession.connectivity, "unknown"); + assert.equal(normalized.sourceMode, "idle"); + assert.equal(normalized.phase, "starting"); + assert.equal(normalized.spatialSource, null); + assert.equal(normalized.metrics.latencyMs, null); + assert.equal(normalized.metrics.frameRateHz, null); + assert.equal(normalized.metrics.aiFrameRateHz, null); + assert.equal(normalized.observationSources[0].availability, "unverified"); + assert.equal(normalized.observationSources[0].binding.deviceId, null); +}); + +test("control authority exposes the session but not unconfirmed data", () => { + const state = runtimeState(); + state.connection_supervisor = supervisor({ control: true, data: false }); + + const normalized = normalize(state); + assert.ok(normalized?.activeDevice); + assert.equal(normalized.activeDevice.endpointLabel, "192.168.68.52"); + assert.equal(normalized.deviceSession.connectivity, "connected"); + assert.equal(normalized.sourceMode, "idle"); + assert.equal(normalized.phase, "starting"); + assert.equal(normalized.spatialSource, null); + assert.equal(normalized.metrics.pointCount, null); + assert.equal(normalized.observationSources[0].availability, "available"); + assert.equal(normalized.observationSources[0].previewUrl, null); +}); + +test("control plus data authority is the only live streaming presentation", () => { + const normalized = normalize(runtimeState()); + assert.ok(normalized?.activeDevice); + assert.equal(normalized.sourceMode, "live"); + assert.equal(normalized.phase, "streaming"); + assert.equal(normalized.deviceSession.connectivity, "connected"); + assert.equal(normalized.spatialSource.url, "rerun+http://127.0.0.1:9877/proxy"); + assert.equal(normalized.metrics.publishedFrameCount, 15); + assert.equal(normalized.metrics.latencyMs, 12.5); + assert.equal(normalized.metrics.aiFrameRateHz, 5); + assert.equal(normalized.observationSources[0].availability, "streaming"); +}); + +test("successful K1 scanning state stays a success message", () => { + const state = runtimeState(); + state.message = "K1 подтвердил режим сканирования; приём и запись активны."; + + const normalized = normalize(state); + assert.equal(normalized.message, state.message); + assert.notEqual( + normalized.message, + "Операция не выполнена. Технические подробности сохранены в журнале сервера.", + ); +}); + +test("data loss degrades the controlled device and withdraws stale stream data", () => { + const state = runtimeState(); + state.connection_supervisor = supervisor({ + control: true, + data: false, + dataPlaneState: "lost", + }); + + const normalized = normalize(state); + assert.ok(normalized?.activeDevice); + assert.equal(normalized.deviceSession.connectivity, "degraded"); + assert.equal(normalized.sourceMode, "idle"); + assert.equal(normalized.phase, "starting"); + assert.equal(normalized.spatialSource, null); + assert.equal(normalized.metrics.frameRateHz, null); + assert.equal(normalized.observationSources[0].availability, "degraded"); + assert.equal(normalized.observationSources[0].previewUrl, null); +}); + +test("exact reconnecting lineage retains only the current spatial presentation lease", () => { + const healthy = normalize(runtimeState()); + const recoveryState = activeRecoveryRuntimeState(); + const recovering = normalize(recoveryState); + assert.ok(healthy?.spatialSource && recovering?.spatialSource); + assert.equal(recovering.sourceMode, "live"); + assert.equal(recovering.phase, "starting"); + assert.equal(recovering.spatialSource.id, healthy.spatialSource.id); + assert.equal(recovering.spatialSource.url, healthy.spatialSource.url); + assert.equal(recovering.activeDevice, null); + assert.equal(recovering.metrics.publishedFrameCount, null); + assert.equal(recovering.metrics.latencyMs, null); + assert.equal(recovering.observationSources[0].availability, "connecting"); + assert.equal(recovering.observationSources[0].previewUrl, healthy.spatialSource.url); + assert.deepEqual(recovering.observationSources[0].presentationLease, { + kind: "active-stream-recovery", + runtimeId: "runtime-active-recovery-001", + acquisitionId: "acquisition-001", + acquisitionStateRevision: 4, + producerGeneration: 13, + recoveryGeneration: 5, + }); + + const noCurrentUrl = structuredClone(recoveryState); + delete noCurrentUrl.rerun_grpc_url; + assert.equal(normalize(noCurrentUrl).spatialSource, null); + assert.equal(normalize(noCurrentUrl).observationSources[0].previewUrl, null); +}); + +test("stale or terminal recovery lineage cannot retain a spatial source", () => { + const staleProducer = activeRecoveryRuntimeState(); + staleProducer.producer_generation += 1; + + const differentAcquisition = activeRecoveryRuntimeState({ + acquisition_id: "acquisition-from-old-runtime", + }); + + for (const state of [ + staleProducer, + differentAcquisition, + activeRecoveryRuntimeState({ state: "blocked" }), + activeRecoveryRuntimeState({ state: "standby" }), + activeRecoveryRuntimeState({ state: "fault" }), + ]) { + const normalized = normalize(state); + assert.equal(normalized.sourceMode, "idle"); + assert.equal(normalized.spatialSource, null); + assert.equal(normalized.metrics.publishedFrameCount, null); + assert.equal(normalized.observationSources[0].previewUrl, null); + assert.equal(normalized.observationSources[0].presentationLease, null); + } +}); + +test("control loss turns a selected device back into an unverified snapshot", () => { + const state = runtimeState(); + state.connection_supervisor = supervisor({ control: false, data: false }); + + const normalized = normalize(state); + assert.ok(normalized); + assert.equal(normalized.activeDevice, null); + assert.equal(normalized.deviceSession.connectivity, "connecting"); + assert.equal(normalized.sourceMode, "idle"); + assert.equal(normalized.spatialSource, null); + assert.equal(normalized.observationSources[0].availability, "unverified"); + assert.equal(normalized.observationSources[0].binding.deviceId, null); +}); + +test("operator banner renders all safe host diagnostics and never the raw exception", () => { + const diagnostic = { + schema_version: "missioncore.host-failure-diagnostic/v1", + code: "host.keychain.interaction-required", + domain: "keychain", + impact: "control", + operator_action: "unlock-or-authorize-keychain", + automatic_retry: false, + redacted: true, + }; + const presentation = hostFailureDiagnosticPresentation(diagnostic); + assert.ok(presentation); + assert.equal( + presentation.operatorActionLabel, + "Разблокируйте связку ключей macOS и подтвердите доступ Mission Core к профилю подключения.", + ); + + const privateException = "SecurityError: account=private-user password=never-render-this"; + const attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "op-00000000-0000-4000-8000-000000000123", + connection_mode: "bridge", + status: "failed", + stage: "status-observing-failed", + public_error_code: "network-session-interrupted", + side_effect_status: "unknown", + safe_next_action: "scan-select-connect", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + diagnostic_bundle: { + schema_version: "missioncore.xgrids-k1-connection-diagnostic/v1", + redacted: true, + generated_at_utc: OBSERVED_AT, + snapshot_runtime_id: "runtime-test", + attempt: {}, + network_mutation_ledger: {}, + connection_supervisor: {}, + automatic_retry: false, + }, + }; + const markup = renderToStaticMarkup(createElement(K1OperatorError, { + message: privateException, + diagnostic, + attempt, + onRefresh() {}, + onClear() {}, + })); + + assert.match(markup, /Причина/); + assert.match(markup, /Связка ключей требует явного подтверждения оператора/); + assert.match(markup, /Системный контур/); + assert.match(markup, /Связка ключей macOS/); + assert.match(markup, /Управляющая связь не установлена/); + assert.match(markup, /Разблокируйте связку ключей macOS/); + assert.match(markup, /Автоматического повтора не было/); + assert.match(markup, /op-00000000-0000-4000-8000-000000000123/); + assert.match(markup, /Ожидание ответа/); + assert.match(markup, /Результат команды не подтверждён/); + assert.match(markup, /новый поиск и выбрать результат/); + assert.match(markup, /Скопировать диагностику/); + assert.match(markup, /
/); + assert.doesNotMatch(markup, /]*\sopen/); + assert.match(markup, /Проверить состояние/); + assert.doesNotMatch(markup, /SecurityError|private-user|never-render-this/); + assert.doesNotMatch(markup, /host\.keychain\.interaction-required/); +}); + +test("operator banner presents every canonical connection next action without fallback drift", () => { + const labels = new Map([ + ["wait-for-current-attempt", "Дождаться завершения текущей попытки"], + ["continue-with-control-verification", "Продолжить текущее подключение"], + ["verify-control-read-only", "Проверить управление без изменения сети"], + ["start-acquisition", "Готово к запуску приёма"], + ["stop-local-receiver", "Завершить только локальный приём"], + [ + "retire-unavailable-physical-target", + "Исключить недоступный прежний K1 и выбрать другой", + ], + ["scan-select-connect", "Выполнить новый поиск и выбрать результат"], + ["manual-recovery-required", "Требуется ручное восстановление"], + ]); + for (const [safeNextAction, expectedLabel] of labels) { + const attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: `attempt-${safeNextAction}`, + connection_mode: "bridge", + status: "failed", + stage: "device-info-failed", + public_error_code: "connection-not-ready", + side_effect_status: "none", + safe_next_action: safeNextAction, + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + const markup = renderToStaticMarkup(createElement(K1OperatorError, { + message: "Связь не подтверждена", + attempt, + onRefresh() {}, + onClear() {}, + })); + assert.match(markup, new RegExp(expectedLabel)); + } +}); + +test("operator banner never renders an unstructured legacy connection error", () => { + const markup = renderToStaticMarkup(createElement(K1OperatorError, { + message: + "K1: исходное устройство не найдено; подключаемся к сохранённому устройству", + onRefresh() {}, + onClear() {}, + })); + + assert.match( + markup, + /Подключение не завершено\. Автоматического повтора не было/, + ); + assertCanonicalConnectionCopy(markup); +}); + +test("compact connection error can expose only its contextual recovery path", () => { + const markup = renderToStaticMarkup(createElement(K1OperatorError, { + message: "hidden transport error", + compact: true, + showDefaultActions: false, + recoveryActions: createElement("button", null, "Подключить новый K1"), + onRefresh() {}, + onClear() {}, + })); + + assert.match(markup, /Подключение не завершено/); + assert.match(markup, />Подключить новый K1<\/button>/); + assert.doesNotMatch(markup, /Проверить состояние|>Закрыть<\/button>/); +}); + +test("connection attempt network phase copy distinguishes absent, applied and unknown outcomes", () => { + assert.equal( + attemptNetworkPhaseLabel("network_not_applied"), + "Настройки сети не применены", + ); + assert.equal( + attemptNetworkPhaseLabel("network_applied"), + "Настройки сети применены", + ); + assert.equal( + attemptNetworkPhaseLabel("network_outcome_unknown"), + "Результат применения настроек сети не подтверждён", + ); +}); + +test("runtime errors borrow diagnostics only from their exact Connect attempt", () => { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-correlated-error"; + state.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "op-exact-connect", + connection_mode: "bridge", + status: "failed", + phase: "network_outcome_unknown", + control_state: "unknown", + stage: "ble-write-dispatched", + public_error_code: null, + side_effect_status: "unknown", + safe_next_action: "verify-control-read-only", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + const exact = { + action: "connect", + runtimeId: state.snapshot_runtime_id, + leaseGeneration: 0, + connectionAttemptId: state.connection_attempt.attempt_id, + }; + + assert.equal(connectionAttemptForRuntimeError(exact, state), state.connection_attempt); + for (const mismatch of [ + { ...exact, action: "scan" }, + { ...exact, runtimeId: "runtime-replaced" }, + { ...exact, connectionAttemptId: "op-another-connect" }, + { ...exact, connectionAttemptId: null }, + ]) { + assert.equal(connectionAttemptForRuntimeError(mismatch, state), null); + } + assert.equal( + connectionAttemptForRuntimeError(exact, { + ...state, + connection_attempt: { ...state.connection_attempt, status: "running" }, + }), + null, + ); +}); + +test("connection recovery observation follows recommended exact policy then safe priority", () => { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-recovery-priority"; + state.devices = [{ + device_id: "fresh-k1", + name: "Fresh K1", + connectable: true, + likely_k1: true, + }]; + const decision = (target_source, required_transport_ref) => ({ + allowed: true, + reason_codes: [], + target_source, + required_transport_ref, + required_connection_mode: "bridge", + requires_live_gatt_validation: target_source === "fresh-scan", + automatic_retry: false, + }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + recommended_action: "observe-configured-device-network", + allowed_actions: [ + "observe-current-device-network", + "observe-configured-device-network", + "observe-fresh-device-network", + ], + actions: { + "observe-current-device-network": decision( + "retained-current-process", + "current-k1", + ), + "observe-configured-device-network": decision( + "durable-configured-state", + "configured-k1", + ), + "observe-fresh-device-network": decision("fresh-scan", "fresh-k1"), + }, + }; + + const recommended = recommendedConnectionRecoveryObservationTarget(state); + assert.equal(recommended?.action, "observe-configured-device-network"); + assert.equal(recommended?.deviceId, "configured-k1"); + + state.connection_policy.recommended_action = "scan-ble"; + const fallback = recommendedConnectionRecoveryObservationTarget(state); + assert.equal(fallback?.action, "observe-current-device-network"); + assert.equal(fallback?.deviceId, "current-k1"); + assert.equal(connectionRecoveryObservationTargetMatches(fallback, fallback), true); + assert.equal( + connectionRecoveryObservationTargetMatches(fallback, { + ...fallback, + deviceId: "foreign-k1", + }), + false, + ); +}); + +test("cold Quick recovery dispatches exact Verify without borrowing the mode draft", async () => { + const target = { + action: "observe-configured-device-network", + deviceId: "configured-quick-k1", + connectionMode: "quick-connect", + source: "durable-configured-state", + serverBound: true, + expectedDiscoveryGeneration: null, + }; + let calls = 0; + const dispatched = await dispatchConnectionRecoveryObservationForCurrentRuntime( + "runtime-cold-quick", + target, + (runtimeId) => runtimeId === "runtime-cold-quick", + () => ({ ...target }), + async (exactTarget) => { + calls += 1; + assert.equal(exactTarget.connectionMode, "quick-connect"); + return "verified"; + }, + ); + assert.deepEqual(dispatched, { dispatched: true, result: "verified" }); + assert.equal(calls, 1); + + for (const [runtimeCurrent, currentTarget] of [ + [false, target], + [true, { ...target, deviceId: "replacement-k1" }], + ]) { + const blocked = await dispatchConnectionRecoveryObservationForCurrentRuntime( + "runtime-cold-quick", + target, + () => runtimeCurrent, + () => currentTarget, + async () => { + calls += 1; + return "must-not-dispatch"; + }, + ); + assert.deepEqual(blocked, { dispatched: false, result: null }); + } + assert.equal(calls, 1); +}); + +test("successful recovery Refresh clears only the compact connection failure", async () => { + let clearCalls = 0; + const refreshed = await clearConnectionFailureAfterSuccessfulRefresh( + async () => ({ snapshot_runtime_id: "runtime-refreshed" }), + () => { + clearCalls += 1; + }, + ); + assert.equal(refreshed, true); + assert.equal(clearCalls, 1); + + const unavailable = await clearConnectionFailureAfterSuccessfulRefresh( + async () => null, + () => { + clearCalls += 1; + }, + ); + assert.equal(unavailable, false); + assert.equal(clearCalls, 1); +}); + +test("successful recovery Scan survives configured-to-fresh promotion for the same K1", () => { + const attempt = terminalConnectionRecoveryState().connection_attempt; + const configuredTarget = { + action: "observe-configured-device-network", + deviceId: "F89438FA-55ED-85AD-EED7-734AC84746D8", + connectionMode: "bridge", + source: "durable-configured-state", + serverBound: true, + expectedDiscoveryGeneration: null, + }; + const freshTarget = { + ...configuredTarget, + action: "observe-fresh-device-network", + deviceId: "f89438fa-55ed-85ad-eed7-734ac84746d8", + source: "fresh-scan", + expectedDiscoveryGeneration: 28, + }; + const beforeScan = connectionRecoveryEscapeKey({ + snapshotRuntimeId: "runtime-recovery-scan", + attempt, + target: configuredTarget, + }); + const afterSuccessfulScan = connectionRecoveryEscapeKey({ + snapshotRuntimeId: "runtime-recovery-scan", + attempt, + target: freshTarget, + }); + + assert.equal(afterSuccessfulScan, beforeScan); + const successfulEscape = connectionRecoveryEscapeAfterScan(true, beforeScan); + // Success stores the pre-Scan key, so the promoted fresh results own Step 01. + assert.equal( + connectionRecoveryIsRequired(afterSuccessfulScan, successfulEscape), + false, + ); + // A failed Scan stores no escape and leaves the recovery card authoritative. + const failedEscape = connectionRecoveryEscapeAfterScan(false, beforeScan); + assert.equal(failedEscape, null); + assert.equal(connectionRecoveryIsRequired(afterSuccessfulScan, failedEscape), true); + + for (const changed of [ + { + snapshotRuntimeId: "runtime-replaced", + attempt, + target: freshTarget, + }, + { + snapshotRuntimeId: "runtime-recovery-scan", + attempt: { ...attempt, attempt_id: "op-new-attempt" }, + target: freshTarget, + }, + { + snapshotRuntimeId: "runtime-recovery-scan", + attempt, + target: { ...freshTarget, deviceId: "replacement-k1" }, + }, + { + snapshotRuntimeId: "runtime-recovery-scan", + attempt, + target: { ...freshTarget, connectionMode: "quick-connect" }, + }, + ]) { + assert.equal( + connectionRecoveryIsRequired( + connectionRecoveryEscapeKey(changed), + beforeScan, + ), + true, + ); + } +}); + +test("metric cards render values only for replay or supervisor-authoritative live data", () => { + const legacy = runtimeState(); + delete legacy.connection_supervisor; + const legacyMarkup = renderToStaticMarkup(createElement(K1Metrics, { + controller: { state: legacy }, + })); + assert.doesNotMatch(legacyMarkup, /12,5/); + assert.match(legacyMarkup, /ДО ПУБЛИКАЦИИ/); + + const authoritativeMarkup = renderToStaticMarkup(createElement(K1Metrics, { + controller: { state: runtimeState() }, + })); + assert.match(authoritativeMarkup, /12,5/); + assert.match(authoritativeMarkup, /42[\s ]000/); + assert.match(authoritativeMarkup, /Данные потока при этом сохраняются/); + assert.doesNotMatch(authoritativeMarkup, /Исходные данные/); + assertCanonicalConnectionCopy(authoritativeMarkup); +}); + +test("cold disconnected connection SSR hides every operational panel", () => { + const state = durableTopologyState(); + const markup = renderConnectionPipelines(state); + const unresolvedHistory = reopenedPhysicalState(); + const unresolvedHistoryMarkup = renderConnectionPipelines(unresolvedHistory); + + assert.equal(shouldRenderK1OperationalPanels(state), false); + assert.equal( + shouldRenderK1OperationalPanels(unresolvedHistory), + false, + "unresolved durable physical history is handled inside the connection pipeline", + ); + assert.match(markup, /class="[^"]*\bconnection-panel\b/); + assert.match(markup, /ПОДКЛЮЧЕНИЕ · ШАГИ 01–02/); + assert.doesNotMatch(markup, /class="metrics-grid/); + assert.doesNotMatch(markup, /class="[^"]*\bsession-panel\b/); + assert.doesNotMatch(markup, /class="diagnostics-grid/); + assert.doesNotMatch(markup, /class="device-workspace__side/); + assert.doesNotMatch(unresolvedHistoryMarkup, /class="metrics-grid/); + assert.doesNotMatch(unresolvedHistoryMarkup, /class="[^"]*\bsession-panel\b/); + assert.doesNotMatch(unresolvedHistoryMarkup, /class="diagnostics-grid/); + assert.doesNotMatch(unresolvedHistoryMarkup, /class="device-workspace__side/); +}); + +test("connected, active, replay and recovery SSR retain operational panels", () => { + const connected = runtimeState(); + connected.phase = "connected"; + connected.source_mode = "idle"; + connected.acquisition = null; + + const active = runtimeState(); + delete active.connection_supervisor; + + const replay = runtimeState(); + replay.phase = "replay"; + replay.source_mode = "replay"; + replay.acquisition = null; + delete replay.connection_supervisor; + + const recovery = durableTopologyState(); + recovery.acquisition = { + acquisition_id: "acquisition-recovery", + device_id: "device-recovery", + device_session_id: "session-recovery", + compatibility_profile_id: PROFILE_ID, + control_mode: "plugin-commanded", + requested_streams: ["spatial.point-cloud.live"], + target_host: "127.0.0.1", + duration_seconds: 0, + evidence_policy: "required", + state: "failed", + state_revision: 8, + cleanup_pending: true, + }; + + for (const [name, state] of [ + ["connected", connected], + ["active", active], + ["replay", replay], + ["recovery", recovery], + ]) { + assert.equal( + shouldRenderK1OperationalPanels(state), + true, + `${name} must keep operational controls visible`, + ); + const markup = renderConnectionPipelines(state); + assert.match(markup, /class="metrics-grid/, name); + assert.match(markup, /class="[^"]*\bsession-panel\b/, name); + assert.match(markup, /class="diagnostics-grid/, name); + assert.match(markup, /class="device-workspace__side/, name); + } +}); + +test("local connection shell scopes process-only configuring and connected labels", () => { + assert.equal(shellPresentation.localConnectionPhaseLabel("configuring"), "Подключение"); + assert.equal( + shellPresentation.localConnectionPhaseLabel("connected"), + "Подключение установлено", + ); + assert.equal( + shellPresentation.phaseLabel("configuring"), + "Настройка устройства", + "other workspaces keep the global runtime label", + ); + assert.equal( + shellPresentation.phaseLabel("connected"), + "Устройство подключено", + "other workspaces keep the global runtime label", + ); +}); + +test("replay and local-only stop branches use bounded process copy", () => { + const replayState = runtimeState(); + replayState.phase = "replay"; + replayState.source_mode = "replay"; + const replayMarkup = renderToStaticMarkup(createElement(K1AcquisitionPipeline, { + controller: acquisitionController(replayState), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + })); + assert.match(replayMarkup, /Локальный файл записи/); + assert.doesNotMatch(replayMarkup, /Локальный файл исходных данных/); + assertCanonicalConnectionCopy(replayMarkup); + + const stopState = runtimeState(); + stopState.compatibility.vendor_writes_enabled = false; + stopState.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["stop-local-receiver"], + actions: { + "stop-local-receiver": { + allowed: true, + reason_codes: [], + target_source: "local-runtime", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + const stopMarkup = renderToStaticMarkup(createElement(K1AcquisitionPipeline, { + controller: acquisitionController(stopState), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + })); + assert.match(stopMarkup, /Состояние сканирования остаётся неизвестным/); + assert.doesNotMatch(stopMarkup, /Физическое состояние сканера/); + assertCanonicalConnectionCopy(stopMarkup); +}); + +test("physical STOP is fail-closed and both acquisition surfaces fall back to local cleanup", () => { + const state = runtimeState(); + state.snapshot_runtime_id = "runtime-stop-001"; + state.snapshot_revision = 40; + state.application_control_session = { + session_generation: 5, + state_revision: 8, + state: "scanning", + can_stop: true, + control_socket_open: true, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["stop-acquisition", "stop-local-receiver"], + actions: { + "stop-acquisition": { + allowed: true, + reason_codes: [], + target_source: "connection-supervisor", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "stop-local-receiver": { + allowed: true, + reason_codes: [], + target_source: "local-runtime", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + + assert.equal(canIssueCanonicalStop(state, false), true); + assert.equal(canIssueCanonicalStop(state, true), false); + const missingRuntimeIdentity = structuredClone(state); + delete missingRuntimeIdentity.snapshot_runtime_id; + assert.equal(physicalStopIntentCheckpoint(missingRuntimeIdentity), null); + assert.equal(canIssueCanonicalStop(missingRuntimeIdentity, false), false); + const zeroGeneration = structuredClone(state); + zeroGeneration.application_control_session.session_generation = 0; + assert.equal(physicalStopIntentCheckpoint(zeroGeneration), null); + const zeroRevision = structuredClone(state); + zeroRevision.application_control_session.state_revision = 0; + assert.equal(physicalStopIntentCheckpoint(zeroRevision), null); + const spentCheckpoint = physicalStopIntentCheckpoint(state); + assert.ok(spentCheckpoint); + const sameCasPoll = structuredClone(state); + sameCasPoll.snapshot_revision += 1; + assert.equal( + authoritativeStateSupersedesPhysicalStopIntent(spentCheckpoint, sameCasPoll), + false, + "a newer polling snapshot with the same control CAS must stay spent", + ); + const nextControlTransition = structuredClone(sameCasPoll); + nextControlTransition.application_control_session.state_revision += 1; + assert.equal( + authoritativeStateSupersedesPhysicalStopIntent( + spentCheckpoint, + nextControlTransition, + ), + true, + "a newer atomic snapshot plus a newer exact control CAS is fresh authority", + ); + const sameTargetReplacement = structuredClone(state); + sameTargetReplacement.snapshot_runtime_id = "runtime-stop-002"; + sameTargetReplacement.snapshot_revision = 1; + assert.equal( + authoritativeStateSupersedesPhysicalStopIntent( + spentCheckpoint, + sameTargetReplacement, + ), + true, + "a newer accepted runtime with an exact STOP gate is fresh authority", + ); + const nonAuthoritativeReplacement = structuredClone(sameTargetReplacement); + nonAuthoritativeReplacement.application_control_session.can_stop = false; + assert.equal( + authoritativeStateSupersedesPhysicalStopIntent( + spentCheckpoint, + nonAuthoritativeReplacement, + ), + false, + "runtime replacement without an exact current STOP gate stays spent", + ); + const distinctAcquisition = structuredClone(sameCasPoll); + distinctAcquisition.acquisition.acquisition_id = "acquisition-002"; + assert.equal( + authoritativeStateSupersedesPhysicalStopIntent( + spentCheckpoint, + distinctAcquisition, + ), + true, + "a newer accepted snapshot may admit a distinct acquisition target", + ); + + state.application_control_session.state = "failed"; + assert.equal(canIssueCanonicalStop(state, false), false); + state.application_control_session.state = "scanning"; + state.application_control_session.can_stop = false; + assert.equal(canIssueCanonicalStop(state, false), false); + state.application_control_session.can_stop = true; + state.connection_policy.actions["stop-acquisition"].allowed = false; + state.connection_policy.actions["stop-acquisition"].reason_codes = [ + "physical-control-authority-unavailable", + ]; + state.connection_policy.allowed_actions = ["stop-local-receiver"]; + assert.equal(canIssueCanonicalStop(state, false), false); + + state.connection_policy.actions["stop-acquisition"].allowed = true; + state.connection_policy.actions["stop-acquisition"].reason_codes = []; + state.connection_policy.allowed_actions = [ + "stop-acquisition", + "stop-local-receiver", + ]; + const physicalMarkup = renderToStaticMarkup(createElement(K1AcquisitionPipeline, { + controller: acquisitionController(state), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + })); + const physicalButtons = buttonMarkupWithText( + physicalMarkup, + "Остановить устройство и запись", + ); + assert.equal(physicalButtons.length, 1); + assert.doesNotMatch(physicalButtons[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch(physicalMarkup, /Завершить локальный приём/); + + const unrelatedErrorMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(state), + error: "unrelated local viewer error", + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.equal( + buttonMarkupWithText( + unrelatedErrorMarkup, + "Остановить устройство и запись", + ).length, + 1, + "a presentation error that did not spend physical STOP must not hide it", + ); + + const dismissedFailureMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(state), + // Closing a presentation banner clears `error`; the independent spent + // checkpoint must continue to suppress the same physical mutation. + error: null, + physicalStopIntentSpent: true, + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.doesNotMatch(dismissedFailureMarkup, /Остановить устройство и запись/); + assert.match(dismissedFailureMarkup, /Повторная команда устройству не отправляется/); + const localButtons = buttonMarkupWithText( + dismissedFailureMarkup, + "Завершить локальный приём", + ); + assert.equal(localButtons.length, 1); + assert.doesNotMatch(localButtons[0], /\bdisabled(?:=|\s|>)/); + assertCanonicalConnectionCopy(dismissedFailureMarkup); + + const classifiedStopState = structuredClone(state); + classifiedStopState.acquisition.state = "awaiting_external_stop"; + classifiedStopState.connection_policy.actions["stop-acquisition"].allowed = false; + classifiedStopState.connection_policy.allowed_actions = ["stop-local-receiver"]; + const classifiedStopMarkup = renderToStaticMarkup(createElement( + K1SpatialControlsView, + { + controller: { + ...acquisitionController(classifiedStopState), + error: "STOP не был отправлен; требуется read-only восстановление", + physicalStopIntentSpent: true, + }, + }, + )); + assert.equal( + buttonMarkupWithText(classifiedStopMarkup, "Завершить локальный приём").length, + 0, + "a classified STOP must not leave a permanently disabled local action in the scene", + ); + + const localCleanupPendingMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(state), + pendingAction: "stop", + error: null, + physicalStopIntentSpent: true, + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(localCleanupPendingMarkup, /Завершаем локальный приём…/); + assert.doesNotMatch(localCleanupPendingMarkup, /Останавливаем устройство…/); + + const physicalStopPendingMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(state), + pendingAction: "stop", + physicalStopIntentSpent: true, + physicalStopInFlight: true, + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(physicalStopPendingMarkup, /Останавливаем устройство…/); + assert.doesNotMatch(physicalStopPendingMarkup, /Завершаем локальный приём…/); + + const terminalDeniedState = structuredClone(state); + terminalDeniedState.phase = "error"; + terminalDeniedState.source_mode = "idle"; + terminalDeniedState.acquisition.state = "failed"; + terminalDeniedState.acquisition.cleanup_pending = true; + terminalDeniedState.connection_policy.actions["stop-acquisition"].allowed = false; + terminalDeniedState.connection_policy.actions["stop-acquisition"].reason_codes = [ + "physical-control-authority-unavailable", + ]; + terminalDeniedState.connection_policy.allowed_actions = ["stop-local-receiver"]; + const terminalDeniedMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: acquisitionController(terminalDeniedState), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(terminalDeniedMarkup, /ВОССТАНОВЛЕНИЕ · ЛОКАЛЬНЫЙ КОНТУР/); + assert.match(terminalDeniedMarkup, /Доступно локальное завершение приёма/); + assert.doesNotMatch(terminalDeniedMarkup, /Локальная запись завершена/); + assert.doesNotMatch( + terminalDeniedMarkup, + /явный STOP|Остановить сканирование|Сканирование продолжается|Требуется остановка/, + ); + + const terminalSpentState = structuredClone(terminalDeniedState); + terminalSpentState.connection_policy.actions["stop-acquisition"].allowed = true; + terminalSpentState.connection_policy.actions["stop-acquisition"].reason_codes = []; + terminalSpentState.connection_policy.allowed_actions = [ + "stop-acquisition", + "stop-local-receiver", + ]; + const terminalReadyMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: acquisitionController(terminalSpentState), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(terminalReadyMarkup, /Локальный приём ещё требует завершения/); + assert.doesNotMatch(terminalReadyMarkup, /Локальная запись (?:уже )?завершена/); + const terminalSpentMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(terminalSpentState), + error: null, + physicalStopIntentSpent: true, + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(terminalSpentMarkup, /ВОССТАНОВЛЕНИЕ · ЛОКАЛЬНЫЙ КОНТУР/); + assert.match(terminalSpentMarkup, /Повторная команда K1 не отправляется/); + assert.doesNotMatch( + terminalSpentMarkup, + /явный STOP|Остановить сканирование|Сканирование продолжается|Требуется остановка/, + ); + + const terminalInFlightMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: { + ...acquisitionController(terminalSpentState), + pendingAction: "stop", + physicalStopIntentSpent: true, + physicalStopInFlight: true, + }, + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.match(terminalInFlightMarkup, /ВОССТАНОВЛЕНИЕ · КОМАНДА ОТПРАВЛЕНА/); + assert.match(terminalInFlightMarkup, /Команда остановки устройства уже отправлена/); + assert.match(terminalInFlightMarkup, /Останавливаем устройство…/); + assert.doesNotMatch( + terminalInFlightMarkup, + /требуется явный STOP|Нажмите «Остановить сканирование»/, + ); + + const noStopActionState = structuredClone(state); + noStopActionState.connection_policy.actions["stop-acquisition"].allowed = false; + noStopActionState.connection_policy.actions["stop-acquisition"].reason_codes = [ + "physical-control-authority-unavailable", + ]; + noStopActionState.connection_policy.actions["stop-local-receiver"].allowed = false; + noStopActionState.connection_policy.actions["stop-local-receiver"].reason_codes = [ + "local-acquisition-receiver-not-active", + ]; + noStopActionState.connection_policy.allowed_actions = []; + noStopActionState.connection_policy.recommended_action = "wait-for-operation"; + noStopActionState.phase = "error"; + noStopActionState.source_mode = "idle"; + noStopActionState.acquisition.state = "failed"; + noStopActionState.acquisition.cleanup_pending = true; + const noStopActionMarkup = renderToStaticMarkup(createElement( + K1AcquisitionPipeline, + { + controller: acquisitionController(noStopActionState), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + }, + )); + assert.equal( + buttonMarkupWithText(noStopActionMarkup, "Завершить локальный приём").length, + 0, + ); + assert.equal( + buttonMarkupWithText(noStopActionMarkup, "Остановить устройство и запись").length, + 0, + ); + assert.match( + noStopActionMarkup, + /ВОССТАНОВЛЕНИЕ · ТОЛЬКО ЧТЕНИЕ/, + ); + assert.match(noStopActionMarkup, /Доступно только read-only восстановление/); + assert.doesNotMatch( + noStopActionMarkup, + /Доступно локальное завершение|Завершите локальный приём|явный STOP|Остановить сканирование/, + ); +}); + +test("recovered physical SCANNING overrides replay with one explicit K1 STOP", () => { + const state = runtimeState(); + state.snapshot_runtime_id = "runtime-recovered-stop"; + state.snapshot_revision = 12; + state.phase = "replay"; + state.source_mode = "replay"; + state.acquisition = { + ...state.acquisition, + state: "failed", + state_revision: 9, + cleanup_pending: false, + }; + state.application_control_session = { + session_generation: 4, + state_revision: 11, + state: "scanning", + can_stop: true, + control_socket_open: true, + physical_command: { + requires_reconciliation: false, + resolved_active_recovery_required: true, + observed_session_state: "scanning", + }, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["stop-acquisition"], + actions: { + "stop-acquisition": { + allowed: true, + reason_codes: [], + target_source: "connection-supervisor", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + + const markup = renderToStaticMarkup(createElement(K1AcquisitionPipeline, { + controller: acquisitionController(state), + desiredConnectionMode: "bridge", + openSpatialScene() {}, + activateAutomaticSpatialSource() {}, + })); + + assert.match(markup, /ВОССТАНОВЛЕНИЕ · ОСТАНОВКА/); + assert.match(markup, /Сканирование продолжается/); + assert.match(markup, /Требуется остановка/); + assert.match(markup, /Сканирование подтверждено; требуется явный STOP/); + assert.doesNotMatch(markup, /Назовите проект и запустите приём/); + assert.doesNotMatch(markup, /Путь к записи|Запустить повтор записи/); + assert.doesNotMatch(markup, /Подготовить проект|Запустить сканирование/); + + const stopButtons = buttonMarkupWithText(markup, "Остановить сканирование"); + assert.equal(stopButtons.length, 1); + assert.doesNotMatch(stopButtons[0], /\bdisabled(?:=|\s|>)/); + assertCanonicalConnectionCopy(markup); +}); + + + + + + +test("authoritative Bridge renders all completed steps with canonical process copy", () => { + const state = runtimeState(); + state.phase = "connected"; + state.source_mode = "idle"; + state.acquisition = null; + state.devices = []; + state.message = "Готово. Включите устройство и начните с поиска по Bluetooth."; + state.connection_supervisor = supervisor({ control: true, data: false }); + state.connection_lifecycle = connectionLifecycle({ control: true }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["start-acquisition"], + actions: { + "scan-ble": { + allowed: false, + reason_codes: ["acquisition-active"], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + recommended_action: "start-acquisition", + }; + + const markup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(state), + desiredMode: "bridge", + })); + + assert.doesNotMatch(markup, /Питание|aria-checked=/); + assert.match(markup, /01<\/span>/); + assert.match(markup, /02<\/span>/); + assert.match(markup, /

Подключение<\/h3>/); + assert.match(markup, /

Сеть<\/h3>/); + assert.match(markup, /Подключение установлено/); + assert.match(markup, /ble-k1-001<\/strong>/); + assert.match(markup, /Общая сеть · Bridge<\/strong>/); + assert.match(markup, /192\.168\.68\.52<\/small>/); + assert.equal(buttonMarkupWithText(markup, "Выбрать другое").length, 0); + assert.equal(buttonMarkupWithText(markup, "Изменить сеть").length, 0); + assert.doesNotMatch(markup, /class="nodedc-activity-indicator"/); + assert.doesNotMatch( + markup, + /Найти устройство Bluetooth|Повторить поиск Bluetooth|Устройства пока не найдены/, + ); + assertCanonicalConnectionCopy(markup); +}); + +test("connected Bridge action slots survive transient lifecycle contention A to B to A", () => { + const actionDecision = (reasonCodes) => ({ + allowed: reasonCodes.length === 0, + reason_codes: reasonCodes, + target_source: "local-prestart-handoff", + required_transport_ref: "ble-k1-001", + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + device_write_performed: false, + automatic_retry: false, + }); + const connectedState = (snapshotRevision, reasonCodes = []) => { + const state = runtimeState(); + state.snapshot_runtime_id = "runtime-connected-actions"; + state.snapshot_runtime_started_at_utc = "2026-08-11T06:00:00Z"; + state.snapshot_revision = snapshotRevision; + state.phase = "connected"; + state.source_mode = "idle"; + state.snapshot_runtime_id = "runtime-ready-pending"; + state.acquisition = null; + state.devices = []; + state.connection_supervisor = supervisor({ control: true, data: false }); + state.connection_lifecycle = connectionLifecycle({ control: true }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: reasonCodes.length === 0 + ? ["prepare-select-device", "prepare-change-network"] + : [], + actions: { + "prepare-select-device": actionDecision(reasonCodes), + "prepare-change-network": actionDecision(reasonCodes), + }, + }; + return state; + }; + const frames = [ + connectedState(40), + connectedState(41, [ + "connection-reconfiguration-lifecycle-busy", + "k1-lifecycle-process-lease-network-owned", + ]), + connectedState(42), + ]; + + for (const reason of [ + "connection-reconfiguration-lifecycle-busy", + "k1-lifecycle-process-lease-network-owned", + ]) { + const singlyContended = connectedState(39, [reason]); + assert.equal( + connectedReconfigurationActionApplicable( + singlyContended, + "prepare-select-device", + ), + true, + ); + assert.equal( + connectedReconfigurationActionApplicable( + singlyContended, + "prepare-change-network", + ), + true, + ); + } + + for (const [index, state] of frames.entries()) { + assert.equal( + connectedReconfigurationActionApplicable(state, "prepare-select-device"), + true, + ); + assert.equal( + connectedReconfigurationActionApplicable(state, "prepare-change-network"), + true, + ); + const markup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(state), + desiredMode: "bridge", + })); + assert.match(markup, /Подключение установлено/); + const selectAction = buttonMarkupWithText(markup, "Выбрать другое"); + const networkAction = buttonMarkupWithText(markup, "Изменить сеть"); + assert.equal(selectAction.length, 1, `frame ${index} keeps select action slot`); + assert.equal(networkAction.length, 1, `frame ${index} keeps network action slot`); + if (index === 1) { + assert.match(selectAction[0], /\bdisabled(?:=|\s|>)/); + assert.match(networkAction[0], /\bdisabled(?:=|\s|>)/); + } else { + assert.doesNotMatch(selectAction[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch(networkAction[0], /\bdisabled(?:=|\s|>)/); + } + } + + const durableBlocker = connectedState(43, ["acquisition-active"]); + assert.equal( + connectedReconfigurationActionApplicable( + durableBlocker, + "prepare-select-device", + ), + false, + ); + assert.equal( + connectedReconfigurationActionApplicable( + durableBlocker, + "prepare-change-network", + ), + false, + ); + const durableMarkup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(durableBlocker), + desiredMode: "bridge", + })); + assert.equal(buttonMarkupWithText(durableMarkup, "Выбрать другое").length, 0); + assert.equal(buttonMarkupWithText(durableMarkup, "Изменить сеть").length, 0); +}); + +for (const pendingAction of ["connect", "verify"]) { + test(`authoritative Ready with pending ${pendingAction} keeps one canonical progress surface`, () => { + const state = runtimeState(); + state.phase = "connected"; + state.source_mode = "idle"; + state.snapshot_runtime_id = "runtime-ready-pending"; + state.acquisition = null; + state.devices = []; + state.connection_supervisor = supervisor({ control: true, data: false }); + state.connection_lifecycle = connectionLifecycle({ control: true }); + + const markup = renderProvisioningWithCurrentPendingAction({ + controller: { + ...provisioningController(state), + pendingAction, + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => true, + }, + desiredMode: "bridge", + }, { publicSearch: false }); + + assert.equal( + (markup.match(/class="nodedc-activity-indicator"/g) ?? []).length, + 1, + ); + assert.match(markup, /ble-k1-001<\/strong>/); + if (pendingAction === "connect") { + assert.match(markup, /

Сеть<\/h3>/); + assert.match(markup, /Настройка сети…/); + } else { + assert.match(markup, /

Сеть<\/h3>/); + assert.match(markup, /Проверяем подключение без изменения сети…/); + assert.match(markup, /02<\/span>/); + } + assertCanonicalConnectionCopy(markup); + }); +} + + + + + + + + +test("explicit read-only recovery keeps its network-unavailable classification", () => { + const base = { + connection_verification: { + status: "endpoint-unreachable", + lease_state: "configured-unverified", + network_reachability: "unreachable", + }, + }; + assert.equal(readOnlyObservationShowsNetworkUnavailable(base), true); + assert.equal(readOnlyObservationShowsNetworkUnavailable({ + connection_verification: { + ...base.connection_verification, + status: "tcp-reachable-device-info-unverified", + network_reachability: "reachable", + }, + }), false); + assert.equal(readOnlyObservationShowsNetworkUnavailable(null), false); + assert.equal(readOnlyObservationShowsNetworkUnavailable({ + connection_verification: { + status: "device-network-applied", + lease_state: "configured-unverified", + network_reachability: "unknown", + reason_code: "endpoint-target-unconfigured", + }, + }), true); + for (const reasonCode of [ + "connection-verify-address-unavailable", + "connection-verify-connection-missing", + "connection-verify-route-mismatch", + "connection-verify-mqtt-unreachable", + "configured-endpoint-unavailable", + "endpoint-target-unconfigured", + ]) { + assert.equal(readOnlyFailureShowsNetworkUnavailable(reasonCode), true); + } + for (const reasonCode of [ + "connection-verify-device-not-rediscovered", + "connection-verify-status-read-invalid", + "device-identity-pin-mismatch", + "connection-verify-discovery-generation-conflict", + "connection-verify-lease-changed", + "connection-reconfiguration-revision-conflict", + "connection-verify-busy", + null, + ]) { + assert.equal(readOnlyFailureShowsNetworkUnavailable(reasonCode), false); + } +}); + +test("pre-write candidate loss is classified stale without granting UI continuation", () => { + for (const reasonCode of [ + "BleakDeviceNotFoundError", + "network-provision-candidate-not-fresh", + "network-provision-candidate-changed", + "network-provision-discovery-generation-conflict", + ]) { + assert.equal(provisioningFailureRequiresFreshCandidate(reasonCode), true); + } + for (const reasonCode of [ + "BleakGATTProtocolError", + "network-provision-target-not-distinguishable-from-baseline", + "network-provision-lifecycle-busy", + "host-wifi-operation-timeout", + null, + ]) { + assert.equal(provisioningFailureRequiresFreshCandidate(reasonCode), false); + } +}); + +test("stale old-device reachability cannot own a fresh selection outcome", () => { + const staleOldDeviceProjection = { + connection_verification: { + status: "endpoint-unreachable", + lease_state: "configured-unverified", + network_reachability: "unreachable", + transport_ref: "ble-k1-old", + }, + }; + + // The global projection is deliberately stale and unavailable, but the + // correlated fresh-device operation failed identity validation. The normal + // one-intent UI renders this only as a stale/safety outcome; it never scans, + // verifies or submits automatically. + assert.equal( + readOnlyObservationShowsNetworkUnavailable(staleOldDeviceProjection), + true, + ); + assert.equal( + readOnlyFailureShowsNetworkUnavailable("connection-verify-status-read-invalid"), + false, + ); + assert.equal( + readOnlyFailureShowsNetworkUnavailable("device-identity-pin-mismatch"), + false, + ); + assert.equal( + readOnlyFailureShowsNetworkUnavailable("connection-reconfiguration-revision-conflict"), + false, + ); + assert.equal( + readOnlyFailureShowsNetworkUnavailable("BleakGATTProtocolError"), + false, + ); +}); + +test("Quick Connect does not expose Bridge reconfiguration actions", () => { + const state = runtimeState(); + state.phase = "connected"; + state.source_mode = "idle"; + state.acquisition = null; + state.connection_mode = "quick-connect"; + state.configured_connection_mode = "quick-connect"; + state.active_connection_mode = "quick-connect"; + state.desired_connection_mode = "quick-connect"; + state.connection_supervisor = supervisor({ control: true, data: false }); + state.connection_supervisor.intent.requested_mode = "quick-connect"; + state.connection_supervisor.observed.device_network.connection_mode = "quick-connect"; + state.connection_supervisor.observed.device_identity.connection_mode = "quick-connect"; + state.connection_supervisor.lease.connection_mode = "quick-connect"; + state.connection_lifecycle = connectionLifecycle({ mode: "quick-connect" }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: [], + actions: {}, + }; + + const markup = renderProvisioning({ + controller: provisioningController(state), + desiredMode: "quick-connect", + }); + assert.doesNotMatch(markup, /Выбрать другое устройство|Изменить сеть/); +}); + +test("a verify without current click authority does not invent network step 02", () => { + const state = durableTopologyState(); + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.connection_supervisor = supervisor({ control: false, data: false }); + state.connection_lifecycle = connectionLifecycle({ control: false }); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: [], + actions: {}, + }; + + const markup = renderProvisioning({ + controller: { + ...provisioningController(state), + pendingAction: "verify", + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => false, + }, + desiredMode: "bridge", + }); + + assert.match(markup, /01<\/span>/); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /aria-busy="true"/); + assert.equal((markup.match(/class="nodedc-activity-indicator"/g) ?? []).length, 0); + assertCanonicalConnectionCopy(markup); +}); + + + + +test("a blocked local-only device escape explains backend safety reasons", () => { + const state = durableTopologyState(); + state.physical_command = { + operator_retirement: { + allowed: false, + reason_codes: ["physical-command-retirement-operation-conflict"], + expected_operation_id: "old-stop-operation", + expected_revision: 280, + expected_transport_ref: "ble-k1-001", + physical_outcome: "unknown", + device_io_performed: false, + automatic_retry: false, + }, + }; + assert.match( + physicalRetirementGuidance(state), + /завершается другая операция с физическим состоянием устройства/, + ); + + for (const [reasonCode, expectedCopy] of [ + ["physical-command-retirement-state-unsafe", /Состояние предыдущей физической команды изменилось/], + ["physical-command-retirement-not-required", /больше не удерживает выбор устройства/], + ["physical-command-target-retired", /уже выведено из текущего контура/], + ["acquisition-active", /Сканирование ещё активно/], + ["control-session-state-unsafe", /Управляющая сессия устройства ещё не завершена/], + ["ble-runtime-busy", /Bluetooth занят другой операцией устройства/], + ["ble-runtime-cleanup-pending", /Bluetooth ещё завершает предыдущую операцию/], + ]) { + state.physical_command.operator_retirement.reason_codes = [reasonCode]; + assert.match(physicalRetirementGuidance(state), expectedCopy); + } +}); + +test("a blocked exact retired-device check explains safe next actions", () => { + const state = durableTopologyState(); + state.physical_command = { + operator_reconciliation_reopen: { + allowed: false, + reason_codes: ["physical-command-reconciliation-reopen-target-not-observed"], + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "ble-k1-001", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 0, + device_io_performed: false, + automatic_retry: false, + }, + }; + assert.match(physicalReopenGuidance(state), /не найдено в последнем Bluetooth-поиске/); + state.physical_command.operator_reconciliation_reopen.reason_codes = [ + "physical-command-reconciliation-reopen-target-not-connectable", + ]; + assert.match(physicalReopenGuidance(state), /не принимает Bluetooth-подключение/); + state.physical_command.operator_reconciliation_reopen.reason_codes = [ + "physical-command-reconciliation-reopen-candidate-ambiguous", + ]; + assert.match(physicalReopenGuidance(state), /не подтвердил один точный экземпляр/); + state.physical_command.operator_reconciliation_reopen.reason_codes = [ + "device-calibration-read-active", + ]; + assert.match(physicalReopenGuidance(state), /читается калибровка/); +}); + +test("fresh Scan keeps retired audit rows selectable without recovery I/O", () => { + const state = durableTopologyState(); + state.current_device_recovery = { + transport_ref: "ble-k1-001", + connection_mode: "bridge", + handle_retained: false, + advertised_now: false, + }; + state.physical_command = { + status: "resolved", + reason_code: null, + requires_reconciliation: false, + automatic_replay_allowed: false, + normal_session_recovery_supported: false, + runtime_bound: false, + reconciliation_ready: false, + record: { + revision: 281, + operation_id: "old-stop-operation", + action: "stop", + stage: "resolved", + resolution: "operator-retired-outcome-unknown", + connection: { transport_ref: "ble-k1-001" }, + operator_retirements: [{ retired_transport_ref: "ble-k1-001" }], + }, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { + retained_context_is_presence: false, + retired_transport_refs: ["ble-k1-001"], + }, + allowed_actions: ["scan-ble", "provision-fresh-device"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + state.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "historical-network-attempt-before-retirement", + connection_mode: "bridge", + status: "failed", + phase: "network_applied", + control_state: "unknown", + stage: "host-route-and-control-endpoint", + public_error_code: "control-bootstrap-failed", + side_effect_status: "network-applied", + safe_next_action: "scan-select-connect", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + state.devices = [ + { + device_id: "ble-k1-001", + name: "Retired old device", + rssi: -39, + connectable: true, + likely_k1: true, + }, + { + device_id: "ble-k1-new", + name: "Replacement device", + rssi: -52, + connectable: true, + likely_k1: true, + }, + ]; + + assert.equal(trustedConnectionBinding(state), null); + assert.equal( + connectionAttemptOwnsAppliedNetworkRecovery(state.connection_attempt), + false, + ); + const markup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + assert.match(markup, /Результатов: 2/); + assert.match(markup, /Повторить поиск Bluetooth/); + assert.doesNotMatch(markup, /old-stop-operation/); + const priorAction = deviceRowActionButton(markup, "ble-k1-001"); + assert.match(priorAction, />Выбрать<\/button>/); + assert.doesNotMatch(priorAction, /\bdisabled(?:=|\s|>)/); + const replacementAction = deviceRowActionButton(markup, "ble-k1-new"); + assert.match(replacementAction, />Выбрать<\/button>/); + assert.doesNotMatch(replacementAction, /\bdisabled(?:=|\s|>)/); + assert.equal( + buttonMarkupWithText(markup, "Подключить новый K1").length, + 0, + ); + assert.doesNotMatch(markup, /]*disabled[^>]*>Недоступно<\/button>/); + assert.doesNotMatch( + markup, + /Попытка настройки завершена|Проверить подключение без изменения сети|Управление не подтверждено/, + ); + assert.doesNotMatch(markup, /Название сети Wi‑Fi|Пароль Wi‑Fi|>ПрименитьВыбрать<\/button>/); + assert.doesNotMatch(onlyPriorAction, /\bdisabled(?:=|\s|>)/); + assert.match(onlyPriorMarkup, /Результатов: 1/); + assert.doesNotMatch(onlyPriorMarkup, />Недоступно|Переподключиться/); + + // Historical audit data is not an active deny once the authoritative policy + // projection removes the reference. The same advertised row then behaves as + // an ordinary fresh result, without a saved/reopen branch in the UI. + state.connection_policy.facts.retired_transport_refs = []; + state.connection_lifecycle = connectionLifecycle({ control: false }); + state.connection_supervisor = supervisor({ control: false, data: false }); + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.current_device_recovery = null; + const historicalAuditMarkup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + const historicallyRetiredAction = deviceRowActionButton( + historicalAuditMarkup, + "ble-k1-001", + ); + assert.match(historicallyRetiredAction, />Выбрать<\/button>/); + assert.doesNotMatch( + historicallyRetiredAction, + /\bdisabled(?:=|\s|>)/, + ); + assertCanonicalConnectionCopy(historicalAuditMarkup); +}); + +test("only current network recovery safe-next states retain a historical applied attempt", () => { + const attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "attempt-network-recovery-ownership", + connection_mode: "bridge", + status: "failed", + phase: "network_applied", + control_state: "control_not_ready", + stage: "host-route-and-control-endpoint", + public_error_code: "control-bootstrap-failed", + side_effect_status: "network-applied", + safe_next_action: "verify-control-read-only", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + for (const safeNextAction of [ + "continue-with-control-verification", + "verify-control-read-only", + "manual-recovery-required", + ]) { + assert.equal( + connectionAttemptOwnsAppliedNetworkRecovery({ + ...attempt, + safe_next_action: safeNextAction, + }), + true, + safeNextAction, + ); + } + for (const safeNextAction of [ + "scan-select-connect", + "stop-local-receiver", + "retire-unavailable-physical-target", + "start-acquisition", + ]) { + assert.equal( + connectionAttemptOwnsAppliedNetworkRecovery({ + ...attempt, + safe_next_action: safeNextAction, + }), + false, + safeNextAction, + ); + } + assert.equal( + connectionAttemptOwnsAppliedNetworkRecovery({ + ...attempt, + status: "running", + safe_next_action: "wait-for-current-attempt", + }), + true, + ); + assert.equal( + connectionAttemptOwnsAppliedNetworkRecovery({ + ...attempt, + control_state: "ready", + }), + false, + ); +}); + +test("an exact prior CoreBluetooth UUID remains a normal fresh selection across casing", () => { + const state = retiredPhysicalReopenReadyState({ + advertisedRef: "f89438fa-55ed-85ad-eed7-734ac84746d8", + backendRef: "F89438FA-55ED-85AD-EED7-734AC84746D8", + }); + + assert.equal( + transportRefEquivalenceKey(" F89438FA-55ED-85AD-EED7-734AC84746D8 "), + "f89438fa-55ed-85ad-eed7-734ac84746d8", + ); + assert.equal(trustedConnectionBinding(state), null); + const markup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + const priorDeviceAction = deviceRowActionButton( + markup, + "f89438fa-55ed-85ad-eed7-734ac84746d8", + ); + assert.match(priorDeviceAction, />Выбрать<\/button>/); + assert.doesNotMatch(priorDeviceAction, /\bdisabled(?:=|\s|>)/); + assert.equal(buttonMarkupWithText(markup, "Переподключиться").length, 0); + assertCanonicalConnectionCopy(markup); +}); + +test("an exact fresh prior row exposes one enabled Select without reconnect", () => { + const state = retiredPhysicalReopenReadyState(); + assert.ok(retiredPhysicalReopenAuthority( + state, + state.devices[0].device_id, + "bridge", + )); + const markup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + const selectActions = buttonMarkupWithText(markup, "Выбрать"); + assert.equal(selectActions.length, 1); + assert.doesNotMatch(selectActions[0], /\bdisabled(?:=|\s|>)/); + assert.equal(buttonMarkupWithText(markup, "Переподключиться").length, 0); + assert.doesNotMatch(markup, /Вернуть прежний K1 и проверить/); + assert.doesNotMatch(markup, /Название общей сети Wi‑Fi|Пароль Wi‑Fi|>Применить { + const initialState = retiredPhysicalReopenReadyState(); + initialState.devices.push({ + device_id: "A1619D95-C352-1069-D430-5FB0BC13F7F9", + name: "dcCONSTRUCTIONS", + rssi: -55, + connectable: true, + likely_k1: false, + }); + initialState.connection_scenario_reset = { + reset_id: "reset-before-retired-reconnect", + request_revision: 6, + revision: 7, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: initialState.ble_discovery_generation, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: "retired", + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + operation_sequence: 21, + }; + let currentState = initialState; + let scanCalls = 0; + let reopenCalls = 0; + let verifyCalls = 0; + let connectCalls = 0; + let resetCalls = 0; + let prepareCalls = 0; + let markScanDispatched; + const scanDispatched = new Promise((resolve) => { + markScanDispatched = resolve; + }); + const controller = { + ...provisioningController(initialState), + scanWithResult: async () => { + scanCalls += 1; + markScanDispatched(); + return { + succeeded: true, + snapshotRuntimeId: currentState.snapshot_runtime_id, + discoveryGeneration: currentState.ble_discovery_generation, + transportRefs: currentState.devices.map((device) => device.device_id), + }; + }, + reopenRetiredPhysicalReconciliation: async () => { + reopenCalls += 1; + return { succeeded: false, observedState: currentState }; + }, + verifyConnection: async () => { + verifyCalls += 1; + return { + succeeded: false, + reconciliationCompleted: false, + observedState: currentState, + }; + }, + connect: async () => { + connectCalls += 1; + return { + succeeded: false, + networkIntentCompleted: false, + intentDisposition: "retain", + acceptedSessionKey: null, + }; + }, + selectConnectionMode: async () => { + resetCalls += 1; + return true; + }, + prepareConnectionReconfigurationWithResult: async () => { + prepareCalls += 1; + return { succeeded: false, observedState: currentState }; + }, + getCurrentState: () => currentState, + getConnectionRecoveryObservationTarget: () => + recommendedConnectionRecoveryObservationTarget(currentState), + getConnectionActionAuthority: (mode) => + connectionActionAuthoritySnapshot(currentState, mode), + isSnapshotRuntimeCurrent: (runtimeId) => + runtimeId === currentState.snapshot_runtime_id, + isConnectionActionAuthorityCurrent: (authority) => + connectionActionAuthorityMatches( + authority, + connectionActionAuthoritySnapshot( + currentState, + authority.connectionMode, + ), + ), + isConnectionPolicyActionAllowedCurrent: (action) => + currentState.connection_policy.allowed_actions.includes(action), + }; + const props = () => ({ + controller: { ...controller, state: currentState }, + desiredMode: "bridge", + }); + const harness = createStatefulProvisioningHarness(props()); + + try { + let tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + harness.flushEffects(); + const scan = actionByLabel(tree, "Найти по Bluetooth"); + assert.ok(scan); + scan.props.onClick(); + await Promise.race([ + scanDispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("post-reset Scan did not dispatch")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + + tree = harness.render(props()); + harness.flushEffects(); + const priorRow = elementByProp(tree, "actionLabel", "Выбрать"); + assert.ok(priorRow); + assert.equal(priorRow.props.device.device_id, initialState.devices[0].device_id); + assert.equal(priorRow.props.selectionDisabled, false); + assert.equal(actionByLabel(tree, "Переподключиться"), null); + priorRow.props.onSelect(); + + tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + const selectedMarkup = renderToStaticMarkup(tree); + assert.equal(scanCalls, 1); + assert.equal(reopenCalls, 0); + assert.equal(verifyCalls, 0); + assert.equal(connectCalls, 0); + assert.equal(resetCalls, 0); + assert.equal(prepareCalls, 0); + assert.match(selectedMarkup, /02<\/span>/); + assert.match(selectedMarkup, /Название общей сети Wi‑Fi/); + assert.match(selectedMarkup, /Пароль Wi‑Fi/); + assert.doesNotMatch(selectedMarkup, /Переподключиться/); + assert.equal( + harness.stateValue(emptyProvisioningAttemptPresentation), + null, + "local selection must not manufacture a backend Apply attempt", + ); + + const passwordField = elementByProp(tree, "label", "Пароль Wi‑Fi"); + assert.ok(passwordField); + assert.equal(passwordField.props.type, "password"); + passwordField.props.onChange({ target: { value: "test-only-password" } }); + tree = harness.render(props()); + harness.flushEffects(); + const showPassword = elementByProp(tree, "label", "Показать пароль"); + assert.ok(showPassword); + assert.equal(showPassword.props.disabled, false); + showPassword.props.onClick(); + tree = harness.render(props()); + harness.flushEffects(); + assert.equal( + elementByProp(tree, "label", "Пароль Wi‑Fi").props.type, + "text", + ); + assert.ok(elementByProp(tree, "label", "Скрыть пароль")); + + const chooseAnother = actionByLabel(tree, "Выбрать другое"); + assert.ok(chooseAnother); + chooseAnother.props.onClick(); + + tree = harness.render(props()); + harness.flushEffects(); + tree = harness.render(props()); + const returnedMarkup = renderToStaticMarkup(tree); + const returnedRows = buttonMarkupWithText(returnedMarkup, "Выбрать"); + assert.equal(returnedRows.length, initialState.devices.length); + assert.ok(returnedRows.every((row) => !/\bdisabled(?:=|\s|>)/.test(row))); + assert.match(returnedMarkup, /Повторить поиск Bluetooth/); + assert.doesNotMatch(returnedMarkup, />Найти по Bluetooth02<\/span>/, + ); + assert.equal(scanCalls, 1); + assert.equal(reopenCalls, 0); + assert.equal(verifyCalls, 0); + assert.equal(connectCalls, 0); + assert.equal(resetCalls, 0); + assert.equal(prepareCalls, 0); + } finally { + harness.dispose(); + } +}); + +test("a rendered prior-device selection never dispatches reopen or Verify", async () => { + const initialState = retiredPhysicalReopenReadyState(); + let currentState = initialState; + let reopenCalls = 0; + let verifyCalls = 0; + let reopenedRequest = null; + let verifiedRequest = null; + let verifiedOptions = null; + let markVerifyDispatched; + const verifyDispatched = new Promise((resolve) => { + markVerifyDispatched = resolve; + }); + + const controller = { + ...provisioningController(initialState), + getCurrentState: () => currentState, + getConnectionRecoveryObservationTarget: () => + recommendedConnectionRecoveryObservationTarget(currentState), + getConnectionActionAuthority: (mode) => + connectionActionAuthoritySnapshot(currentState, mode), + isSnapshotRuntimeCurrent: (runtimeId) => + runtimeId === currentState.snapshot_runtime_id, + reopenRetiredPhysicalReconciliation: async (request, runtimeId) => { + reopenCalls += 1; + reopenedRequest = request; + assert.equal(runtimeId, initialState.snapshot_runtime_id); + const committed = reopenedPhysicalState({ + reopeningId: request.reopening_id, + discoveryGeneration: request.expected_discovery_generation, + }); + committed.connection_policy.recommended_action = "select-connection-intent"; + committed.connection_policy.allowed_actions = [ + "observe-fresh-device-network", + "observe-configured-device-network", + "scan-ble", + ]; + committed.connection_policy.actions["observe-configured-device-network"] = { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: request.expected_transport_ref, + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + automatic_retry: false, + }; + currentState = committed; + assert.equal( + recommendedConnectionRecoveryObservationTarget(currentState).source, + "durable-configured-state", + "the generic selector must reproduce the competing configured target", + ); + assert.equal( + readOnlyConnectionObservationTarget(currentState).source, + "fresh-scan", + ); + return { succeeded: true, observedState: currentState }; + }, + verifyConnection: async (request, options) => { + verifyCalls += 1; + verifiedRequest = request; + verifiedOptions = options; + markVerifyDispatched(); + return { + succeeded: true, + reconciliationCompleted: true, + observedState: currentState, + }; + }, + }; + const tree = captureProvisioningTreeAfterSearch({ + controller, + desiredMode: "bridge", + }); + const priorDeviceRow = elementByProp( + tree, + "actionLabel", + "Выбрать", + ); + assert.ok(priorDeviceRow); + assert.equal(typeof priorDeviceRow.props.onSelect, "function"); + assert.equal(actionByLabel(tree, "Переподключиться"), null); + + priorDeviceRow.props.onSelect(); + await new Promise((resolve) => setTimeout(resolve, 0)); + + assert.equal(reopenCalls, 0); + assert.equal(verifyCalls, 0); + assert.equal(reopenedRequest, null); + assert.equal(verifiedRequest, null); + assert.equal(verifiedOptions, null); +}); + +test("retired physical reopen proof uses the backend target, not the visible row", () => { + const request = { + reopening_id: "reopening-returned-k1", + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const state = reopenedPhysicalState(); + const target = recommendedConnectionRecoveryObservationTarget(state); + const authority = connectionActionAuthoritySnapshot(state, "bridge"); + const context = retiredPhysicalReopenVerificationContext( + state, + request, + "runtime-reopened-k1", + target, + authority, + ); + assert.ok(context); + assert.equal(context.target.deviceId, request.expected_transport_ref); + assert.equal(context.target.serverBound, true); + assert.equal( + context.target.deviceId, + state.connection_policy.actions["observe-fresh-device-network"] + .required_transport_ref, + ); + assert.notEqual(context.target.deviceId, state.devices[0].device_id); + assert.equal( + transportRefEquivalenceKey(context.target.deviceId), + transportRefEquivalenceKey(state.devices[0].device_id), + ); + + const wrongTarget = { + ...target, + deviceId: "visible-row-but-not-server-target", + }; + assert.equal(retiredPhysicalReopenVerificationContext( + state, + request, + "runtime-reopened-k1", + wrongTarget, + authority, + ), null); +}); + +test("exact lost reopen response continues once to read-only Verify", async () => { + const request = { + reopening_id: "reopening-returned-k1", + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const committed = reopenedPhysicalState(); + let verifyCalls = 0; + let verifiedTarget = null; + const result = await dispatchRetiredPhysicalReconciliationForCurrentRuntime({ + snapshotRuntimeId: "runtime-reopened-k1", + request, + isSnapshotRuntimeCurrent: (runtimeId) => runtimeId === "runtime-reopened-k1", + reopen: async () => ({ succeeded: false, observedState: committed }), + getCurrentState: () => committed, + getCurrentTarget: () => recommendedConnectionRecoveryObservationTarget(committed), + getCurrentAuthority: (mode) => connectionActionAuthoritySnapshot(committed, mode), + expectedAuthority: connectionActionAuthoritySnapshot(committed, "bridge"), + verify: async (context, runtimeId) => { + verifyCalls += 1; + verifiedTarget = context.target; + assert.equal(runtimeId, "runtime-reopened-k1"); + return { succeeded: true }; + }, + }); + assert.equal(result.reopenDispatched, true); + assert.equal(result.reopenResult.succeeded, false); + assert.equal(result.verifyDispatched, true); + assert.equal(verifyCalls, 1); + assert.equal(verifiedTarget.deviceId, request.expected_transport_ref); +}); + +test("retired combined recovery keeps the exact fresh target when configured observation is also allowed", async () => { + const request = { + reopening_id: "reopening-combined-live", + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const committed = reopenedPhysicalState({ + reopeningId: request.reopening_id, + }); + committed.connection_policy.recommended_action = "select-connection-intent"; + committed.connection_policy.allowed_actions = [ + "observe-fresh-device-network", + "observe-configured-device-network", + "scan-ble", + ]; + committed.connection_policy.actions["observe-configured-device-network"] = { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: request.expected_transport_ref, + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + automatic_retry: false, + }; + + const genericTarget = recommendedConnectionRecoveryObservationTarget(committed); + const exactFreshTarget = readOnlyConnectionObservationTarget(committed); + assert.equal(genericTarget.action, "observe-configured-device-network"); + assert.equal(exactFreshTarget.action, "observe-fresh-device-network"); + assert.equal(exactFreshTarget.source, "fresh-scan"); + assert.equal(exactFreshTarget.expectedDiscoveryGeneration, 13); + + let verifyCalls = 0; + const result = await dispatchRetiredPhysicalReconciliationForCurrentRuntime({ + snapshotRuntimeId: committed.snapshot_runtime_id, + request, + isSnapshotRuntimeCurrent: () => true, + reopen: async () => ({ succeeded: true, observedState: committed }), + getCurrentState: () => committed, + getCurrentTarget: () => readOnlyConnectionObservationTarget(committed), + getCurrentAuthority: (mode) => connectionActionAuthoritySnapshot(committed, mode), + expectedAuthority: connectionActionAuthoritySnapshot(committed, "bridge"), + verify: async ({ target }) => { + verifyCalls += 1; + assert.equal(target.action, "observe-fresh-device-network"); + assert.equal(target.source, "fresh-scan"); + assert.equal(target.deviceId, request.expected_transport_ref); + return { succeeded: true }; + }, + }); + assert.equal(result.verifyDispatched, true); + assert.equal(verifyCalls, 1); +}); + +test("runtime replacement and same-runtime re-retirement suppress post-reopen Verify", async () => { + const request = { + reopening_id: "reopening-returned-k1", + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const committed = reopenedPhysicalState(); + let runtimeId = "runtime-B"; + let reopenCalls = 0; + let verifyCalls = 0; + const staleBeforeDispatch = await dispatchRetiredPhysicalReconciliationForCurrentRuntime({ + snapshotRuntimeId: "runtime-A", + request, + isSnapshotRuntimeCurrent: (expected) => expected === runtimeId, + reopen: async () => { + reopenCalls += 1; + return { succeeded: true, observedState: committed }; + }, + getCurrentState: () => committed, + getCurrentTarget: () => recommendedConnectionRecoveryObservationTarget(committed), + getCurrentAuthority: (mode) => connectionActionAuthoritySnapshot(committed, mode), + expectedAuthority: connectionActionAuthoritySnapshot(committed, "bridge"), + verify: async () => { + verifyCalls += 1; + return { succeeded: true }; + }, + }); + assert.equal(staleBeforeDispatch.reopenDispatched, false); + assert.equal(reopenCalls, 0); + assert.equal(verifyCalls, 0); + + runtimeId = "runtime-reopened-k1"; + const reretired = retiredPhysicalReopenReadyState(); + let currentState = committed; + const changedAfterSuccess = await dispatchRetiredPhysicalReconciliationForCurrentRuntime({ + snapshotRuntimeId: runtimeId, + request, + isSnapshotRuntimeCurrent: (expected) => expected === runtimeId, + reopen: async () => { + currentState = reretired; + return { succeeded: true, observedState: committed }; + }, + getCurrentState: () => currentState, + getCurrentTarget: () => recommendedConnectionRecoveryObservationTarget(currentState), + getCurrentAuthority: (mode) => connectionActionAuthoritySnapshot(currentState, mode), + expectedAuthority: connectionActionAuthoritySnapshot(committed, "bridge"), + verify: async () => { + verifyCalls += 1; + return { succeeded: true }; + }, + }); + assert.equal(changedAfterSuccess.reopenDispatched, true); + assert.equal(changedAfterSuccess.verifyDispatched, false); + assert.equal(verifyCalls, 0); +}); + +test("retired reopen click and settlement retain the complete authority fence", () => { + const rendered = retiredPhysicalReopenReadyState(); + rendered.connection_reconfiguration = connectionReconfiguration(); + rendered.connection_lifecycle = { active_binding_key: "binding-A" }; + const reopenAuthority = retiredPhysicalReopenAuthority( + rendered, + rendered.devices[0].device_id, + "bridge", + ); + const authorityA = connectionActionAuthoritySnapshot(rendered, "bridge"); + assert.ok(authorityA); + assert.deepEqual( + physicalReopenClickAuthority(rendered, "bridge", authorityA, reopenAuthority), + authorityA, + ); + + const reconfigured = structuredClone(rendered); + reconfigured.connection_reconfiguration = connectionReconfiguration({ + intent: "select-device", + revision: 1, + intentId: "reconfigure-B", + }); + const authorityR1 = connectionActionAuthoritySnapshot(reconfigured, "bridge"); + assert.equal( + physicalReopenClickAuthority(rendered, "bridge", authorityR1, reopenAuthority), + null, + "a rendered R0 CTA cannot borrow latest R1 before POST", + ); + + const rebound = structuredClone(rendered); + rebound.connection_lifecycle.active_binding_key = "binding-B"; + const authorityB = connectionActionAuthoritySnapshot(rebound, "bridge"); + assert.equal( + physicalReopenClickAuthority(rendered, "bridge", authorityB, reopenAuthority), + null, + "a rendered binding A CTA cannot borrow latest binding B before POST", + ); + + const requestA = { + reopening_id: "reopening-A", + expected_revision: reopenAuthority.expectedRevision, + expected_retirement_id: reopenAuthority.expectedRetirementId, + expected_transport_ref: reopenAuthority.expectedTransportRef, + expected_discovery_generation: reopenAuthority.expectedDiscoveryGeneration, + expected_desired_mode: reopenAuthority.expectedDesiredMode, + expected_desired_mode_revision: reopenAuthority.expectedDesiredModeRevision, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const presentationA = { + key: "runtime-A:reopening-A:device", + snapshotRuntimeId: authorityA.snapshotRuntimeId, + request: requestA, + authority: authorityA, + }; + assert.equal( + admitPhysicalReopenPresentation(null, presentationA, authorityA), + presentationA, + ); + assert.equal( + admitPhysicalReopenPresentation(presentationA, presentationA, authorityA), + null, + "a double click is coalesced while A owns the action", + ); + + const authorityRuntimeB = { + ...authorityA, + snapshotRuntimeId: "runtime-B", + }; + const presentationB = { + ...presentationA, + key: "runtime-B:reopening-B:device", + snapshotRuntimeId: "runtime-B", + request: { ...requestA, reopening_id: "reopening-B" }, + authority: authorityRuntimeB, + }; + assert.equal(physicalReopenPresentationIsCurrent( + presentationA, + authorityRuntimeB, + ), false); + assert.equal( + admitPhysicalReopenPresentation( + presentationA, + presentationB, + authorityRuntimeB, + ), + presentationB, + "runtime B synchronously replaces stale A without waiting for an effect", + ); + assert.equal(physicalReopenSettlementIsCurrent( + presentationB, + presentationA.key, + authorityRuntimeB, + ), false, "late A cannot message or clear B"); + + for (const [label, driftedAuthority] of [ + ["new reset revision", { + ...authorityA, + desiredModeRevision: authorityA.desiredModeRevision + 1, + }], + ["new discovery generation", { + ...authorityA, + discoveryGeneration: authorityA.discoveryGeneration + 1, + }], + ["new reconfiguration", authorityR1], + ["new binding", authorityB], + ]) { + assert.equal(physicalReopenPresentationIsCurrent( + presentationA, + driftedAuthority, + ), false); + const replacement = { + ...presentationA, + key: `runtime-A:reopening-B:${label}`, + request: { + ...requestA, + reopening_id: `reopening-B-${label}`, + expected_desired_mode_revision: driftedAuthority.desiredModeRevision, + expected_discovery_generation: driftedAuthority.discoveryGeneration, + }, + authority: driftedAuthority, + }; + assert.equal( + admitPhysicalReopenPresentation( + presentationA, + replacement, + driftedAuthority, + ), + replacement, + `${label} releases A synchronously so a current B click can start`, + ); + } +}); + +test("same-runtime reconfiguration or binding drift after reopen suppresses Verify", async () => { + const request = { + reopening_id: "reopening-returned-k1", + expected_revision: 281, + expected_retirement_id: "retirement-old-stop", + expected_transport_ref: "F89438FA-55ED-85AD-EED7-734AC84746D8", + expected_discovery_generation: 13, + expected_desired_mode: "bridge", + expected_desired_mode_revision: 7, + operator_confirmed: true, + reason: "device-returned-for-explicit-reconciliation", + }; + const committed = reopenedPhysicalState(); + committed.connection_reconfiguration = connectionReconfiguration(); + committed.connection_lifecycle = { active_binding_key: "binding-A" }; + const expectedAuthority = connectionActionAuthoritySnapshot(committed, "bridge"); + for (const drift of ["reconfiguration", "binding"]) { + const current = structuredClone(committed); + if (drift === "reconfiguration") { + current.connection_reconfiguration = connectionReconfiguration({ + intent: "select-device", + revision: 1, + intentId: "reconfiguration-B", + }); + } else { + current.connection_lifecycle.active_binding_key = "binding-B"; + } + let verifyCalls = 0; + const result = await dispatchRetiredPhysicalReconciliationForCurrentRuntime({ + snapshotRuntimeId: committed.snapshot_runtime_id, + request, + isSnapshotRuntimeCurrent: () => true, + reopen: async () => ({ succeeded: true, observedState: committed }), + getCurrentState: () => current, + getCurrentTarget: () => recommendedConnectionRecoveryObservationTarget(current), + getCurrentAuthority: (mode) => connectionActionAuthoritySnapshot(current, mode), + expectedAuthority, + verify: async () => { + verifyCalls += 1; + return { succeeded: true }; + }, + }); + assert.equal(result.verifyDispatched, false, drift); + assert.equal(verifyCalls, 0, drift); + } +}); + +test("a settled reset marker keeps the exact prior UUID as an ordinary Select row", () => { + const state = retiredPhysicalReopenReadyState(); + state.connection_scenario_reset = { + reset_id: "reset-settled", + request_revision: 6, + revision: 7, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: 13, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: "retired", + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + operation_sequence: 21, + }; + assert.deepEqual(scenarioResetPresentationBoundary(state, "bridge"), { + current: true, + active: false, + key: "runtime-reopened-k1:7", + }); + const markup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + assert.doesNotMatch(markup, /Прежний локальный сеанс закрыт/); + const select = buttonMarkupWithText(markup, "Выбрать"); + assert.equal(select.length, 1); + assert.doesNotMatch(select[0], /\bdisabled(?:=|\s|>)/); + assert.equal(buttonMarkupWithText(markup, "Переподключиться").length, 0); + assert.doesNotMatch(markup, /Вернуть прежний K1 и проверить/); +}); + +test("an unresolved physical command has one explicit server-bound read-only recovery surface", () => { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-physical-read-only"; + const recoveryTransport = "F89438FA-55ED-85AD-EED7-734AC84746D8"; + state.physical_command = { + status: "unresolved", + reason_code: "physical-command-reconciliation-required", + requires_reconciliation: true, + resolved_active_recovery_required: false, + automatic_replay_allowed: false, + normal_session_recovery_supported: false, + runtime_bound: true, + reconciliation_ready: true, + observed_session_state: "scan_stopping", + active_operation_id: "old-stop-operation", + record: { + revision: 306, + operation_id: "old-stop-operation", + action: "stop", + stage: "observing", + resolution: null, + connection: { + transport_ref: recoveryTransport, + connection_mode: "bridge", + target_ipv4: "192.168.68.51", + }, + }, + }; + state.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "historical-network-attempt-after-powered-off-start", + connection_mode: "bridge", + status: "failed", + phase: "network_applied", + control_state: "control_not_ready", + stage: "host-route-and-control-endpoint", + public_error_code: "control-bootstrap-failed", + side_effect_status: "network-applied", + safe_next_action: "manual-recovery-required", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false, retired_transport_refs: [] }, + allowed_actions: ["scan-ble", "observe-configured-device-network"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "fresh-scan", + required_transport_ref: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: recoveryTransport, + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + state.devices = [ + { + device_id: recoveryTransport.toLowerCase(), + name: "XGR-A46BE7", + rssi: -42, + connectable: true, + likely_k1: true, + }, + { + device_id: "nearby-unrelated-device", + name: "Nearby unrelated device", + rssi: -48, + connectable: true, + likely_k1: false, + }, + ]; + + const markup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(state), + desiredMode: "bridge", + })); + const phaseDetail = physicalRecoveryConnectionDetail(state); + assert.match(phaseDetail, /Проверьте состояние прежнего K1/); + assert.match(phaseDetail, /сохранённое системой подключение/); + assert.match(phaseDetail, /не отправляет START, STOP или настройки сети/); + assert.doesNotMatch(phaseDetail, /read-only/); + assert.doesNotMatch(phaseDetail, /серверн.*цел/i); + assert.doesNotMatch(markup, /read-only|серверн.*цел/i); + assert.doesNotMatch(phaseDetail, /[Пп]одтвердите питание|локально исключить/); + const reconnect = buttonMarkupWithText(markup, "Переподключиться"); + const connectNew = buttonMarkupWithText(markup, "Подключить новый K1"); + assert.equal(reconnect.length, 1); + assert.equal(connectNew.length, 1); + assert.doesNotMatch(reconnect[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch(connectNew[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch( + markup, + /Предыдущая команда завершилась|Проверить прежний K1 без изменений|Выберите способ восстановления|Подтверждаю:|Питание|индикатор горит/, + ); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /Попытка настройки завершена|Проверить подключение без изменения сети/); + assert.doesNotMatch(markup, /nearby-unrelated-device/); + assert.doesNotMatch(markup, />Применить<|>Выбрать<|Обновить Bluetooth-поиск/); + assert.doesNotMatch(markup, /old-stop-operation/); + + const searchedMarkup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + assert.equal(buttonMarkupWithText(searchedMarkup, "Переподключиться").length, 1); + assert.equal(buttonMarkupWithText(searchedMarkup, "Подключить новый K1").length, 1); + assert.doesNotMatch( + searchedMarkup, + /Результаты последнего Bluetooth-поиска|nearby-unrelated-device|>Выбрать<|>Применить)/); + assert.doesNotMatch( + policyDeniedMarkup, + /nearby-unrelated-device|Старый K1 недоступен|Подтверждаю:|>Применить true, + isConnectionActionAuthorityCurrent: () => true, + }, + desiredMode: "bridge", + }, { publicSearch: false }); + assert.equal( + (pendingMarkup.match(/class="connection-action-progress"/g) ?? []).length, + 1, + ); + assert.doesNotMatch(pendingMarkup, /Переподключиться|Подключить новый K1|Подтверждаю:/); + + const source = readFileSync(provisioningSourceUrl, "utf8"); + const physicalRecovery = sourceSlice( + source, + "const verifyPhysicalRecovery", + "const prepareReconfiguration", + ); + assert.equal((physicalRecovery.match(/await verifyConnection\(/g) ?? []).length, 1); + assert.doesNotMatch( + physicalRecovery, + /scanWithResult\(|connect\(|retireUnavailable|reopenRetired|reconcilePhysicalCommand/, + ); + assert.match(physicalRecovery, /physicalRecoveryTarget\?\.serverBound/); +}); + +test("unavailable physical replacement stays CAS-protected behind the simple new-device path", () => { + const state = durableTopologyState(); + const transportRef = "F89438FA-55ED-85AD-EED7-734AC84746D8"; + state.snapshot_runtime_id = "runtime-retire-old-k1"; + state.physical_command = { + status: "unresolved", + reason_code: "physical-command-reconciliation-required", + requires_reconciliation: true, + resolved_active_recovery_required: false, + automatic_replay_allowed: false, + normal_session_recovery_supported: false, + runtime_bound: false, + reconciliation_ready: true, + observed_session_state: "unknown", + active_operation_id: "old-start-operation", + operator_retirement: { + allowed: true, + reason_codes: [], + expected_operation_id: "old-start-operation", + expected_revision: 41, + expected_transport_ref: transportRef, + physical_outcome: "unknown", + device_io_performed: false, + automatic_retry: false, + }, + record: { + revision: 41, + operation_id: "old-start-operation", + action: "start", + stage: "observing", + resolution: null, + connection: { + transport_ref: transportRef, + connection_mode: "bridge", + }, + }, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false, retired_transport_refs: [] }, + allowed_actions: ["retire-unavailable-physical-target"], + actions: { + "provision-fresh-device": { + allowed: false, + reason_codes: ["physical-command-reconciliation-required"], + target_source: "fresh-scan", + required_transport_ref: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + "retire-unavailable-physical-target": { + allowed: true, + reason_codes: [], + target_source: "durable-physical-command", + required_transport_ref: transportRef, + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + physical_command_allowed: false, + physical_outcome: "unknown", + device_write_performed: false, + automatic_retry: false, + }, + }, + }; + + assert.deepEqual(unavailablePhysicalRetirementAuthority(state), { + expectedOperationId: "old-start-operation", + expectedRevision: 41, + expectedTransportRef: transportRef, + }); + const markup = renderToStaticMarkup(createElement(K1ProvisioningPipeline, { + controller: provisioningController(state), + desiredMode: "bridge", + })); + const connectNew = buttonMarkupWithText(markup, "Подключить новый K1"); + assert.equal(connectNew.length, 1); + assert.doesNotMatch(connectNew[0], /\bdisabled(?:=|\s|>)/); + assert.equal(buttonMarkupWithText(markup, "Переподключиться").length, 0); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch( + markup, + /Старый K1 недоступен|Подтверждаю:|Выберите способ восстановления|Проверить прежний K1 без изменений|[Пп]итани/, + ); + const phaseDetail = physicalRecoveryConnectionDetail(state); + assert.match(phaseDetail, /можно локально исключить без связи с устройством/); + assert.match(phaseDetail, /не отправляет START, STOP или настройки сети/); + assert.doesNotMatch(phaseDetail, /[Пп]итани|read-only/); + + const denied = structuredClone(state); + denied.connection_policy.allowed_actions = []; + denied.connection_policy.actions["retire-unavailable-physical-target"].allowed = false; + denied.connection_policy.actions["retire-unavailable-physical-target"].reason_codes = [ + "physical-command-retirement-state-unsafe", + ]; + assert.equal(unavailablePhysicalRetirementAuthority(denied), null); + const deniedMarkup = renderProvisioning({ + controller: provisioningController(denied), + desiredMode: "bridge", + }); + const deniedNew = buttonMarkupWithText(deniedMarkup, "Подключить новый K1"); + assert.equal(deniedNew.length, 1); + assert.doesNotMatch(deniedNew[0], /\bdisabled(?:=|\s|>)/); + assert.equal(buttonMarkupWithText(deniedMarkup, "Переподключиться").length, 0); + assert.doesNotMatch(deniedMarkup, /Старый K1 недоступен|Подтверждаю:/); + assert.doesNotMatch( + physicalRecoveryConnectionDetail(denied), + /можно локально исключить/, + ); + + const resolvedActive = structuredClone(state); + resolvedActive.physical_command.status = "resolved"; + resolvedActive.physical_command.requires_reconciliation = false; + resolvedActive.physical_command.resolved_active_recovery_required = true; + resolvedActive.physical_command.observed_session_state = "scanning"; + resolvedActive.physical_command.record.stage = "resolved"; + resolvedActive.physical_command.record.resolution = "start-active-observed"; + resolvedActive.connection_policy.actions["provision-fresh-device"].reason_codes = [ + "physical-device-already-active", + ]; + const resolvedActiveMarkup = renderToStaticMarkup(createElement( + K1ProvisioningPipeline, + { + controller: provisioningController(resolvedActive), + desiredMode: "bridge", + }, + )); + const resolvedNew = buttonMarkupWithText( + resolvedActiveMarkup, + "Подключить новый K1", + ); + assert.equal(resolvedNew.length, 1); + assert.doesNotMatch(resolvedNew[0], /\bdisabled(?:=|\s|>)/); + assert.equal( + buttonMarkupWithText(resolvedActiveMarkup, "Переподключиться").length, + 0, + ); + assert.doesNotMatch(resolvedActiveMarkup, /[Пп]итани/); + assert.doesNotMatch(resolvedActiveMarkup, /02<\/span>|03<\/span>/); + + const bothAllowed = structuredClone(state); + bothAllowed.semantic_topology_store.record.transport_ref = transportRef; + bothAllowed.connection_policy.allowed_actions = [ + "observe-configured-device-network", + "retire-unavailable-physical-target", + ]; + bothAllowed.connection_policy.actions["observe-configured-device-network"] = { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: transportRef, + required_connection_mode: "bridge", + requires_live_gatt_validation: false, + automatic_retry: false, + }; + const bothMarkup = renderToStaticMarkup(createElement( + K1ProvisioningPipeline, + { + controller: provisioningController(bothAllowed), + desiredMode: "bridge", + }, + )); + const bothReconnect = buttonMarkupWithText(bothMarkup, "Переподключиться"); + const bothNew = buttonMarkupWithText(bothMarkup, "Подключить новый K1"); + assert.equal(bothReconnect.length, 1); + assert.equal(bothNew.length, 1); + assert.doesNotMatch(bothReconnect[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch(bothNew[0], /\bdisabled(?:=|\s|>)/); + assert.ok( + bothMarkup.indexOf("Переподключиться") + < bothMarkup.indexOf("Подключить новый K1"), + "read-only reconnect must be first", + ); + assert.doesNotMatch( + bothMarkup, + /Выберите способ восстановления|Если прежний K1 снова доступен|Если прежний K1 недоступен постоянно|Подтверждаю:/, + ); + assert.doesNotMatch(bothMarkup, /[Пп]итани|индикатор горит/); + assert.doesNotMatch(bothMarkup, /02<\/span>|03<\/span>/); + const bothPhaseDetail = physicalRecoveryConnectionDetail(bothAllowed); + assert.match(bothPhaseDetail, /Если прежний K1 снова доступен/); + assert.match(bothPhaseDetail, /недоступен постоянно или заменён/); + assert.match(bothPhaseDetail, /не отправляет START, STOP или настройки сети/); + assert.match(bothPhaseDetail, /без связи с устройством/); + assert.doesNotMatch(bothPhaseDetail, /read-only/); + assert.doesNotMatch(bothMarkup, /read-only/); + + const bothPendingMarkup = renderProvisioningWithCurrentPendingAction({ + controller: { + ...provisioningController(bothAllowed), + pendingAction: "verify", + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => true, + }, + desiredMode: "bridge", + }, { publicSearch: false }); + assert.match( + bothPendingMarkup, + /Проверяем прежний K1 без повторения START, STOP или настроек сети/, + ); + assert.equal( + (bothPendingMarkup.match(/class="connection-action-progress"/g) ?? []).length, + 1, + ); + assert.doesNotMatch( + bothPendingMarkup, + /Переподключиться|Подключить новый K1|Подтверждаю:/, + ); + + const bothResolvedActive = structuredClone(bothAllowed); + bothResolvedActive.physical_command.status = "resolved"; + bothResolvedActive.physical_command.requires_reconciliation = false; + bothResolvedActive.physical_command.resolved_active_recovery_required = true; + bothResolvedActive.physical_command.observed_session_state = "scanning"; + bothResolvedActive.physical_command.record.stage = "resolved"; + bothResolvedActive.physical_command.record.resolution = "start-active-observed"; + bothResolvedActive.connection_policy.actions[ + "provision-fresh-device" + ].reason_codes = ["physical-device-already-active"]; + const bothResolvedActiveMarkup = renderToStaticMarkup(createElement( + K1ProvisioningPipeline, + { + controller: provisioningController(bothResolvedActive), + desiredMode: "bridge", + }, + )); + assert.equal( + buttonMarkupWithText(bothResolvedActiveMarkup, "Переподключиться").length, + 1, + ); + assert.equal( + buttonMarkupWithText(bothResolvedActiveMarkup, "Подключить новый K1").length, + 1, + ); + assert.doesNotMatch( + bothResolvedActiveMarkup, + /02<\/span>|03<\/span>/, + ); +}); + +test("physical recovery new-device action performs only an explicit scenario reset", async () => { + const state = reopenedPhysicalState(); + const resetRequests = []; + let desiredModeChanges = 0; + let scanCalls = 0; + let connectCalls = 0; + let verifyCalls = 0; + let reopenCalls = 0; + let markResetDispatched; + const resetDispatched = new Promise((resolve) => { + markResetDispatched = resolve; + }); + const controller = { + ...provisioningController(state), + selectConnectionMode: async (request) => { + resetRequests.push(request); + markResetDispatched(); + return true; + }, + scanWithResult: async () => { + scanCalls += 1; + return { succeeded: false }; + }, + connect: async () => { + connectCalls += 1; + return { succeeded: false }; + }, + verifyConnection: async () => { + verifyCalls += 1; + return { succeeded: false }; + }, + reopenRetiredPhysicalReconciliation: async () => { + reopenCalls += 1; + return { succeeded: false }; + }, + }; + const props = { + controller, + desiredMode: "bridge", + onDesiredModeChange: async (mode) => { + assert.equal(mode, "bridge"); + desiredModeChanges += 1; + }, + }; + const markup = renderProvisioning(props); + const resetButtons = buttonMarkupWithText(markup, "Подключить новый K1"); + assert.equal(resetButtons.length, 1); + assert.doesNotMatch(resetButtons[0], /\bdisabled(?:=|\s|>)/); + + const tree = captureProvisioningTree(props); + const connectNew = actionByLabel(tree, "Подключить новый K1"); + assert.ok(connectNew); + connectNew.props.onClick(); + await Promise.race([ + resetDispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("new-device action did not dispatch scenario reset")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + + assert.equal(resetRequests.length, 1); + assert.equal(resetRequests[0].connection_mode, "bridge"); + assert.equal( + resetRequests[0].expected_revision, + state.desired_connection_mode_revision, + ); + assert.equal(resetRequests[0].reset_scenario, true); + assert.equal(typeof resetRequests[0].reset_id, "string"); + assert.ok(resetRequests[0].reset_id.length > 0); + assert.equal(desiredModeChanges, 1); + assert.equal(scanCalls, 0); + assert.equal(connectCalls, 0); + assert.equal(verifyCalls, 0); + assert.equal(reopenCalls, 0); +}); + +test("physical retirement dispatch refuses a rendered runtime A after runtime B is current", async () => { + let currentRuntimeId = "runtime-B"; + let mutationRequests = 0; + const stale = await dispatchUnavailablePhysicalRetirementForCurrentRuntime( + "runtime-A", + (expectedSnapshotRuntimeId) => expectedSnapshotRuntimeId === currentRuntimeId, + async () => { + mutationRequests += 1; + return { succeeded: true }; + }, + ); + assert.deepEqual(stale, { dispatched: false, result: null }); + assert.equal(mutationRequests, 0); + + currentRuntimeId = "runtime-A"; + const current = await dispatchUnavailablePhysicalRetirementForCurrentRuntime( + "runtime-A", + (expectedSnapshotRuntimeId) => expectedSnapshotRuntimeId === currentRuntimeId, + async (expectedSnapshotRuntimeId) => { + mutationRequests += 1; + return { expectedSnapshotRuntimeId }; + }, + ); + assert.deepEqual(current, { + dispatched: true, + result: { expectedSnapshotRuntimeId: "runtime-A" }, + }); + assert.equal(mutationRequests, 1); + + const runtimeSource = readFileSync(new URL( + "../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts", + import.meta.url, + ), "utf8"); + const retirementHook = sourceSlice( + runtimeSource, + "const retireUnavailablePhysicalCommandWithResult", + "const retireUnavailablePhysicalCommand = useCallback", + ); + assert.match( + retirementHook, + /!isSnapshotRuntimeCurrent\(exactSnapshotRuntimeId\)[\s\S]*?expected_snapshot_runtime_id:\s*exactSnapshotRuntimeId/, + ); + assert.doesNotMatch( + retirementHook, + /expected_snapshot_runtime_id:\s*expectedSnapshotRuntimeId\(\)/, + ); +}); + + + +test("an explicit network action keeps one disabled Step-2 form with an in-place loader", () => { + const state = durableTopologyState(); + state.snapshot_runtime_id = "runtime-network-action"; + state.desired_connection_mode_revision = 7; + state.ble_discovery_generation = 13; + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + const device = { + device_id: "f89438fa-55ed-85ad-eed7-734ac84746d8", + name: "XGR-A46BE7", + rssi: -51, + connectable: true, + likely_k1: true, + }; + state.devices = [device]; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["provision-fresh-device"], + actions: { + "provision-fresh-device": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: null, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + }; + const presentation = provisioningAttemptPresentation({ + snapshotRuntimeId: state.snapshot_runtime_id, + deviceId: device.device_id, + }); + const markup = renderProvisioningWithAttempt({ + controller: provisioningController(state), + desiredMode: "bridge", + }, presentation); + + assert.doesNotMatch(markup, /class="connection-action-progress"/); + assert.match(markup, /aria-busy="true"/); + assert.equal((markup.match(/class="nodedc-activity-indicator"/g) ?? []).length, 1); + assert.match(markup, /Устройство выбрано/); + assert.match(markup, /XGR-A46BE7/); + assert.match(markup, /02<\/span>/); + assert.match(markup, /Применяем настройки…/); + assert.match(markup, /FIELD-NET/); + assert.match(markup, /Пароль передан/); + assert.doesNotMatch(markup, /secret/); + assert.equal((markup.match(/]*disabled=""/g) ?? []).length, 2); + assert.doesNotMatch(markup, />Применить<\/button>/); + assertCanonicalConnectionCopy(markup); +}); + +test("Apply presentation stays correlated through ACK, bootstrap, terminal and connected projections", () => { + const presentation = provisioningAttemptPresentation({ + deviceId: "ble-k1-001", + }); + const initialState = { + ...durableTopologyState(), + snapshot_runtime_id: presentation.snapshotRuntimeId, + }; + assert.equal( + provisioningAttemptViewState(presentation, initialState), + "submitting", + ); + + const operation = { + operation_id: "network-operation-one", + action: "network.provision", + status: "succeeded", + idempotency_key: presentation.idempotencyKey, + }; + assert.equal( + provisioningAttemptViewState({ + ...presentation, + localPhase: "failed", + }, { + ...initialState, + operations: [{ ...operation, status: "running" }], + }), + "settling", + "an authoritative running operation supersedes a presentation error", + ); + assert.equal( + provisioningAttemptViewState(presentation, { + ...initialState, + operations: [operation], + }), + "settling", + "a fast network ACK is not yet a connected result", + ); + + const correlated = { + ...presentation, + attemptId: operation.operation_id, + localPhase: "settling", + }; + const attempt = { + attempt_id: operation.operation_id, + connection_mode: "bridge", + status: "running", + phase: "network_applied", + control_state: "unknown", + safe_next_action: "wait-for-current-attempt", + }; + assert.equal( + provisioningAttemptViewState(correlated, { + ...initialState, + operations: [operation], + connection_attempt: attempt, + }), + "settling", + ); + assert.equal( + provisioningAttemptViewState(correlated, { + ...initialState, + operations: [operation], + connection_attempt: { + ...attempt, + status: "succeeded", + control_state: "ready", + safe_next_action: "start-acquisition", + }, + }), + "settling", + "ready child success waits for the exact reachable lease projection", + ); + assert.equal( + provisioningAttemptViewState(correlated, { + ...initialState, + operations: [operation], + connection_attempt: { + ...attempt, + status: "failed", + control_state: "control_not_ready", + safe_next_action: "manual-recovery-required", + }, + }), + "failed", + ); + + const connectedState = runtimeState(); + connectedState.snapshot_runtime_id = presentation.snapshotRuntimeId; + assert.equal( + provisioningAttemptViewState(correlated, connectedState), + "connected", + ); + assert.equal( + provisioningAttemptViewState(correlated, { + ...initialState, + snapshot_runtime_id: "replacement-runtime", + }), + "retired", + ); +}); + +test("Apply spends secret state before I/O and never stores a password in its presentation latch", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const presentationType = sourceSlice( + source, + "export interface ProvisioningAttemptPresentation", + "export function emptyProvisioningAttemptPresentation", + ); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + assert.doesNotMatch(presentationType, /password\s*:/i); + assert.ok(apply.indexOf('setPassword("")') < apply.indexOf("await connect(")); + assert.ok(apply.indexOf("setExplicitProvisioningDraft(null)") < apply.indexOf("await connect(")); + assert.match(apply, /const attemptedPassword = password/); + assert.match(apply, /value|password/); + assert.match(source, /\|\| connectionAttemptSettling[\s\S]*?\|\| connectionAttemptFailed/); + assert.match(source, /icon=\{\}[\s\S]*?disabled/); +}); + +test("terminal Apply recovery is bounded and never claims that the network is ready", () => { + const presentation = provisioningAttemptPresentation({ + localPhase: "settling", + attemptId: "network-operation-one", + }); + const state = durableTopologyState(); + state.snapshot_runtime_id = presentation.snapshotRuntimeId; + state.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: presentation.attemptId, + connection_mode: "bridge", + status: "failed", + phase: "network_applied", + control_state: "control_not_ready", + stage: "host-route-and-control-endpoint", + public_error_code: "control-bootstrap-failed", + side_effect_status: "network-applied", + safe_next_action: "manual-recovery-required", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: OBSERVED_AT, + timeline: [], + }; + state.operations = [{ + operation_id: presentation.attemptId, + action: "network.provision", + status: "failed", + idempotency_key: presentation.idempotencyKey, + }]; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: [], + actions: {}, + }; + + const markup = renderProvisioningWithAttempt({ + controller: provisioningController(state), + desiredMode: "bridge", + }, presentation); + assert.match(markup, /Управление не подтверждено/); + assert.match(markup, /Попытка настройки завершена/); + assert.doesNotMatch(markup, /Сеть (?:готова|настроена)|Сетевые настройки уже применены/); + assert.doesNotMatch(markup, /Пароль передан|FIELD-NET|>Применить { + const state = terminalConnectionRecoveryState({ + phase: "network_not_applied", + safeNextAction: "scan-select-connect", + includeConfiguredTarget: true, + }); + // A proven pre-dispatch failure must follow its exact Scan recovery instead + // of borrowing an older durable K1 observation that is still policy-visible. + state.connection_policy.recommended_action = "scan-ble"; + const presentation = provisioningAttemptPresentation({ + snapshotRuntimeId: state.snapshot_runtime_id, + attemptId: state.connection_attempt.attempt_id, + localPhase: "failed", + failureMessage: "Результат отправки неизвестен.", + freshStartAllowed: false, + }); + const exactCorrelation = { + action: "connect", + runtimeId: state.snapshot_runtime_id, + leaseGeneration: 0, + connectionAttemptId: state.connection_attempt.attempt_id, + }; + let resetCalls = 0; + let scanCalls = 0; + let resetRequest = null; + let markResetDispatched; + const resetDispatched = new Promise((resolve) => { + markResetDispatched = resolve; + }); + const controller = { + ...provisioningController(state), + error: "private transport exception must stay hidden", + errorDiagnostic: null, + errorCorrelation: exactCorrelation, + refresh: async () => state, + clearError() {}, + selectConnectionMode: async (request) => { + resetCalls += 1; + resetRequest = request; + markResetDispatched(); + return true; + }, + scanWithResult: async () => { + scanCalls += 1; + return { succeeded: false }; + }, + }; + const markup = renderProvisioningWithAttempt({ + controller, + desiredMode: "bridge", + }, presentation); + + assert.match(markup, /Подключение не завершено/); + assert.match(markup, /
/); + assert.doesNotMatch(markup, /]*\sopen/); + assert.equal( + buttonMarkupWithText(markup, "Подключить новый K1").length, + 1, + ); + assert.equal( + buttonMarkupWithText(markup, "Проверить прежнее подключение").length, + 0, + ); + assert.doesNotMatch(markup, /Проверить состояние|>Закрыть<\/button>/); + assert.doesNotMatch(markup, /Выбрать другое|Пароль Wi‑Fi|Пароль передан|>Применить02<\/span>|private transport exception/); + + const busyMarkup = renderProvisioningWithAttempt({ + controller: { + ...controller, + pendingAction: "mode", + }, + desiredMode: "bridge", + }, presentation); + const busyReset = buttonMarkupWithText( + busyMarkup, + "Подключить новый K1", + )[0]; + assert.ok(busyReset); + assert.match(busyReset, /\bdisabled(?:=|\s|>)/); + + const tree = captureProvisioningTreeWithAttempt({ + controller, + desiredMode: "bridge", + }, presentation); + const errorSurface = elementByProp( + tree, + "title", + "Подключение не завершено", + ); + assert.ok(errorSurface); + const resetAction = actionByLabel( + errorSurface.props.recoveryActions, + "Подключить новый K1", + ); + assert.ok(resetAction); + resetAction.props.onClick(); + await Promise.race([ + resetDispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("terminal recovery did not dispatch scenario reset")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal(resetCalls, 1); + assert.deepEqual( + { + connection_mode: resetRequest.connection_mode, + expected_revision: resetRequest.expected_revision, + reset_scenario: resetRequest.reset_scenario, + }, + { + connection_mode: "bridge", + expected_revision: state.desired_connection_mode_revision, + reset_scenario: true, + }, + ); + assert.equal(typeof resetRequest.reset_id, "string"); + assert.ok(resetRequest.reset_id.length > 0); + assert.equal(scanCalls, 0); + + const cleanState = structuredClone(state); + cleanState.connection_attempt = null; + cleanState.operations = []; + cleanState.desired_connection_mode_revision += 1; + cleanState.connection_scenario_reset = { + reset_id: resetRequest.reset_id, + request_revision: state.desired_connection_mode_revision, + revision: cleanState.desired_connection_mode_revision, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: cleanState.ble_discovery_generation, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: null, + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + operation_sequence: 9, + }; + const cleanMarkup = renderProvisioning({ + controller: provisioningController(cleanState), + desiredMode: "bridge", + }); + const cleanScan = buttonMarkupWithText(cleanMarkup, "Найти по Bluetooth"); + assert.equal(cleanScan.length, 1); + assert.doesNotMatch(cleanScan[0], /\bdisabled(?:=|\s|>)/); + assert.equal( + buttonMarkupWithText(cleanMarkup, "Подключить новый K1").length, + 0, + ); + assert.equal(scanCalls, 0); + + const deniedState = terminalConnectionRecoveryState({ + phase: "network_not_applied", + safeNextAction: "scan-select-connect", + includeConfiguredTarget: false, + includeScan: false, + }); + const deniedMarkup = renderProvisioningWithAttempt({ + controller: { + ...provisioningController(deniedState), + error: "denied", + errorCorrelation: { + ...exactCorrelation, + runtimeId: deniedState.snapshot_runtime_id, + connectionAttemptId: deniedState.connection_attempt.attempt_id, + }, + refresh: async () => deniedState, + clearError() {}, + }, + desiredMode: "bridge", + }, { + ...presentation, + snapshotRuntimeId: deniedState.snapshot_runtime_id, + attemptId: deniedState.connection_attempt.attempt_id, + }); + assert.equal( + buttonMarkupWithText(deniedMarkup, "Подключить новый K1").length, + 1, + ); + + const source = readFileSync(provisioningSourceUrl, "utf8"); + const search = sourceSlice(source, "const repeatDeviceScan", "const submitConnect"); + const actions = sourceSlice( + source, + "const connectionRecoveryActions", + "return (", + ); + assert.equal((search.match(/scanWithResult\(/g) ?? []).length, 1); + assert.match( + search, + /isConnectionPolicyActionAllowedCurrent\("scan-ble"\)/, + ); + assert.match( + actions, + /changeDesiredConnectionMode\(connectionMode\)/, + ); + assert.doesNotMatch( + actions, + /repeatDeviceScan\(|scanWithResult\(|xgridsK1Api|\.scanBle\(/, + ); +}); + +test("unknown and cold durable recovery offer one reconnect and one new-device path", () => { + const state = terminalConnectionRecoveryState(); + const target = recommendedConnectionRecoveryObservationTarget(state); + assert.equal(target?.action, "observe-configured-device-network"); + const presentation = provisioningAttemptPresentation({ + snapshotRuntimeId: state.snapshot_runtime_id, + attemptId: state.connection_attempt.attempt_id, + localPhase: "failed", + failureMessage: "Результат отправки неизвестен.", + freshStartAllowed: false, + }); + const controller = { + ...provisioningController(state), + error: "hidden", + errorDiagnostic: null, + errorCorrelation: { + action: "connect", + runtimeId: state.snapshot_runtime_id, + leaseGeneration: 0, + connectionAttemptId: state.connection_attempt.attempt_id, + }, + refresh: async () => state, + clearError() {}, + getConnectionRecoveryObservationTarget: () => target, + }; + const markup = renderProvisioningWithAttempt({ + controller, + desiredMode: "bridge", + }, presentation); + const reconnectOffset = markup.indexOf("Переподключиться"); + const newDeviceOffset = markup.indexOf("Подключить новый K1"); + assert.ok(reconnectOffset > 0); + assert.ok(newDeviceOffset > reconnectOffset); + assert.match(markup, /Результат применения настроек сети не подтверждён/); + assert.doesNotMatch(markup, /Настройки сети не применены/); + assert.doesNotMatch(markup, /Сеть (?:готова|настроена)|>Применить02<\/span>/); + + const coldState = structuredClone(state); + coldState.connection_attempt = null; + const coldTarget = recommendedConnectionRecoveryObservationTarget(coldState); + const coldMarkup = renderProvisioning({ + controller: { + ...provisioningController(coldState), + getConnectionRecoveryObservationTarget: () => coldTarget, + }, + desiredMode: "bridge", + }); + const coldReconnectOffset = coldMarkup.indexOf("Переподключиться"); + const coldNewDeviceOffset = coldMarkup.indexOf("Подключить новый K1"); + assert.ok(coldReconnectOffset > 0); + assert.ok(coldNewDeviceOffset > coldReconnectOffset); + assert.doesNotMatch(coldMarkup, /Локальная операция завершилась ошибкой/); + assert.doesNotMatch(coldMarkup, /02<\/span>|>Применить coldQuickTarget, + }, + desiredMode: "bridge", + }); + const coldQuickReconnect = buttonMarkupWithText( + coldQuickMarkup, + "Переподключиться", + )[0]; + assert.ok(coldQuickReconnect); + assert.doesNotMatch(coldQuickReconnect, /\bdisabled(?:=|\s|>)/); + assert.equal( + buttonMarkupWithText(coldQuickMarkup, "Подключить новый K1").length, + 1, + ); + assert.match(coldQuickMarkup, /Quick Connect/); + + const unrelatedMarkup = renderProvisioningWithAttempt({ + controller: { + ...controller, + errorCorrelation: { ...controller.errorCorrelation, action: "scan" }, + }, + desiredMode: "bridge", + }, presentation); + assert.doesNotMatch(unrelatedMarkup, /op-b7ea404d/); + assert.match(unrelatedMarkup, /Результат применения сети не подтверждён/); +}); + +test("cold saved new-device path resets once, performs zero Scan, and survives reload", async () => { + const state = terminalConnectionRecoveryState(); + state.connection_attempt = null; + const target = recommendedConnectionRecoveryObservationTarget(state); + assert.ok(target); + const resetRequests = []; + let scanCalls = 0; + let verifyCalls = 0; + let connectCalls = 0; + let markResetDispatched; + const resetDispatched = new Promise((resolve) => { + markResetDispatched = resolve; + }); + const controller = { + ...provisioningController(state), + getConnectionRecoveryObservationTarget: () => target, + selectConnectionMode: async (request) => { + resetRequests.push(request); + markResetDispatched(); + return true; + }, + scanWithResult: async () => { + scanCalls += 1; + return { succeeded: false }; + }, + verifyConnection: async () => { + verifyCalls += 1; + return { succeeded: false }; + }, + connect: async () => { + connectCalls += 1; + return { succeeded: false }; + }, + }; + const props = { + controller, + desiredMode: "bridge", + }; + const markup = renderProvisioning(props); + const reconnect = buttonMarkupWithText(markup, "Переподключиться"); + const connectNew = buttonMarkupWithText(markup, "Подключить новый K1"); + assert.equal(reconnect.length, 1); + assert.equal(connectNew.length, 1); + assert.doesNotMatch(reconnect[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch(connectNew[0], /\bdisabled(?:=|\s|>)/); + + const tree = captureProvisioningTree(props); + const resetAction = actionByLabel(tree, "Подключить новый K1"); + assert.ok(resetAction); + resetAction.props.onClick(); + await Promise.race([ + resetDispatched, + new Promise((_, reject) => setTimeout( + () => reject(new Error("cold new-device path did not dispatch reset")), + 100, + )), + ]); + await new Promise((resolve) => setTimeout(resolve, 0)); + + assert.equal(resetRequests.length, 1); + assert.deepEqual( + { + connection_mode: resetRequests[0].connection_mode, + expected_revision: resetRequests[0].expected_revision, + reset_scenario: resetRequests[0].reset_scenario, + }, + { + connection_mode: "bridge", + expected_revision: state.desired_connection_mode_revision, + reset_scenario: true, + }, + ); + assert.equal(typeof resetRequests[0].reset_id, "string"); + assert.ok(resetRequests[0].reset_id.length > 0); + assert.equal(scanCalls, 0); + assert.equal(verifyCalls, 0); + assert.equal(connectCalls, 0); + + const reloaded = structuredClone(state); + reloaded.desired_connection_mode_revision += 1; + reloaded.connection_scenario_reset = { + reset_id: resetRequests[0].reset_id, + request_revision: state.desired_connection_mode_revision, + revision: reloaded.desired_connection_mode_revision, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: reloaded.ble_discovery_generation, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: null, + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + operation_sequence: 31, + }; + const reloadedMarkup = renderProvisioning({ + controller: { + ...provisioningController(reloaded), + getConnectionRecoveryObservationTarget: () => + recommendedConnectionRecoveryObservationTarget(reloaded), + }, + desiredMode: "bridge", + }); + assert.equal(buttonMarkupWithText(reloadedMarkup, "Переподключиться").length, 0); + const cleanScan = buttonMarkupWithText(reloadedMarkup, "Найти по Bluetooth"); + assert.equal(cleanScan.length, 1); + assert.doesNotMatch(cleanScan[0], /\bdisabled(?:=|\s|>)/); + assert.doesNotMatch( + reloadedMarkup, + /Нужна проверка|Прежнее подключение не подтверждено|Проверяем прежний K1/, + ); +}); + +test("a current post-reset failed or unknown attempt outranks the durable reset marker after reload", () => { + const withSettledReset = (state) => { + state.desired_connection_mode_revision = 1; + state.connection_scenario_reset = { + reset_id: "reset-before-new-attempt", + request_revision: 0, + revision: 1, + desired_mode: "bridge", + active: false, + settled_by_discovery_generation: state.ble_discovery_generation, + local_session_closed: true, + previous_device_may_continue_scanning: false, + physical_disposition: null, + network_disposition: null, + device_command_performed: false, + network_write_performed: false, + automatic_scan: false, + // Operation sequence is per operation, not a global chronology. The new + // post-reset operation below can therefore legitimately have 3 < 6. + operation_sequence: 6, + }; + state.operations = [{ + operation_id: state.connection_attempt.attempt_id, + action: "network.provision", + status: "failed", + sequence: 3, + idempotency_key: "post-reset-new-network-attempt", + }]; + return state; + }; + + const unknown = withSettledReset(terminalConnectionRecoveryState({ + phase: "network_outcome_unknown", + safeNextAction: "verify-control-read-only", + })); + const unknownTarget = recommendedConnectionRecoveryObservationTarget(unknown); + assert.ok(unknownTarget); + const unknownReloadMarkup = renderProvisioning({ + controller: { + ...provisioningController(unknown), + getConnectionRecoveryObservationTarget: () => unknownTarget, + }, + desiredMode: "bridge", + }); + assert.match(unknownReloadMarkup, /Результат применения сети не подтверждён/); + assert.equal( + buttonMarkupWithText(unknownReloadMarkup, "Переподключиться").length, + 1, + ); + assert.equal( + buttonMarkupWithText(unknownReloadMarkup, "Подключить новый K1").length, + 1, + ); + assert.equal( + buttonMarkupWithText(unknownReloadMarkup, "Найти по Bluetooth").length, + 0, + ); + + const failed = withSettledReset(terminalConnectionRecoveryState({ + phase: "network_applied", + safeNextAction: "manual-recovery-required", + })); + failed.connection_attempt.control_state = "control_not_ready"; + const failedReloadMarkup = renderProvisioning({ + controller: provisioningController(failed), + desiredMode: "bridge", + }); + assert.match(failedReloadMarkup, /Управление не подтверждено/); + assert.match(failedReloadMarkup, /Новый выбор временно заблокирован/); + assert.doesNotMatch(failedReloadMarkup, /Совпадений нет|Ожидает/); +}); + +test("Apply never owns hidden refresh or an automatic continuation", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + assert.doesNotMatch( + apply, + /scanWithResult\(|candidateRefresh|verifyConnection\(|void submitConnect/, + ); +}); + +test("stale authority is explicit and never settles a hidden refresh", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + assert.doesNotMatch(apply, /candidateRefresh|scanWithResult\(/); + assert.match(source, /устарел|stale/i); +}); + +test("a same-runtime authority drift keeps one neutral settling surface", () => { + const state = durableTopologyState(); + const markup = renderProvisioning({ + controller: { + ...provisioningController(state), + pendingAction: "verify", + // A synchronous authority read has already moved beyond the click-owned + // fence while the controller is still settling its old I/O. + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => false, + }, + desiredMode: "bridge", + }); + + assert.match(markup, /01<\/span>/); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /Завершаем ранее начатое действие…/); + assert.equal((markup.match(/class="nodedc-activity-indicator"/g) ?? []).length, 0); + assert.doesNotMatch(markup, /Проверяем сохранённое подключение…/); + assert.doesNotMatch(markup, /Подключаемся к сохранённому K1…/); +}); + +test("a stale reopen settlement cannot own the current wizard surface", () => { + const state = durableTopologyState(); + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + const markup = renderProvisioning({ + controller: { + ...provisioningController(state), + pendingAction: "reopen", + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => false, + }, + desiredMode: "bridge", + }); + + assert.match(markup, /01<\/span>/); + assert.doesNotMatch(markup, /Подключение…|aria-busy="true"/); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.match(markup, />Найти по Bluetooth<\/button>/); + const modeToggle = markup.match( + /]*aria-label="Способ подключения"[^>]*>/, + )?.[0]; + assert.ok(modeToggle); + assert.doesNotMatch(modeToggle, /\bdisabled(?:=|\s|>)/); + assertCanonicalConnectionCopy(markup); +}); + +test("network step 02 is disclosed only by a selected device or a real network intent", () => { + assert.equal(shouldRevealProvisioningNetworkStep({ + deviceExplicitlySelectedOrAdmitted: false, + networkIntentStarted: false, + }), false); + assert.equal(shouldRevealProvisioningNetworkStep({ + deviceExplicitlySelectedOrAdmitted: true, + networkIntentStarted: false, + }), true); + assert.equal(shouldRevealProvisioningNetworkStep({ + deviceExplicitlySelectedOrAdmitted: false, + networkIntentStarted: true, + }), true); +}); + +test("a current-authority Bluetooth scan owns Step 01 and no network step", () => { + const state = reopenedPhysicalState(); + state.physical_command = null; + state.connection_policy.actions["provision-fresh-device"] = { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: null, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }; + state.connection_policy.allowed_actions.push("provision-fresh-device"); + const authority = reopenedPhysicalAuthority(); + const markup = renderProvisioningWithCurrentPendingAction({ + controller: { + ...provisioningController(state), + pendingAction: "scan", + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: (candidate) => + connectionActionAuthorityMatches(candidate, authority), + }, + desiredMode: "bridge", + }); + + assert.match(markup, /01<\/span>/); + assert.match(markup, /Поиск Bluetooth · 6 с/); + assert.equal( + (markup.match(/class="nodedc-activity-indicator"/g) ?? []).length, + 1, + ); + assert.doesNotMatch(markup, /Найдено: 0|Совпадений нет|Результатов:/); + assert.equal( + buttonMarkupWithText(markup, "Подключить новый K1").length, + 0, + ); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /Настройка сети|Название общей сети Wi‑Fi/); + assertCanonicalConnectionCopy(markup); +}); + +test("stale raw Bluetooth results stay hidden until an explicit search", () => { + const state = durableTopologyState(); + state.semantic_topology_store = { + ...state.semantic_topology_store, + status: "empty", + configured_offline_evidence: false, + record: null, + }; + state.devices = [{ + device_id: "replacement-k1", + name: "Replacement K1", + rssi: -44, + connectable: true, + likely_k1: true, + }]; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble", "provision-fresh-device"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + "provision-fresh-device": { + allowed: true, + reason_codes: [], + target_source: "fresh-scan", + required_transport_ref: null, + required_connection_mode: "bridge", + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + }; + + const markup = renderProvisioning({ + controller: { + ...provisioningController(state), + // The raw promise is still settling, but its click authority has already + // been retired by the completed discovery-generation transition. + pendingAction: "scan", + isSnapshotRuntimeCurrent: () => true, + isConnectionActionAuthorityCurrent: () => false, + }, + desiredMode: "bridge", + }); + + assert.match(markup, /01<\/span>/); + assert.doesNotMatch(markup, /Replacement K1/); + assert.match(markup, />Найти по Bluetooth<\/button>/); + assert.doesNotMatch(markup, /aria-busy="true"/); + assert.doesNotMatch(markup, /02<\/span>|03<\/span>/); + assert.doesNotMatch(markup, /Завершаем предыдущее действие|Завершаем ранее начатое действие/); + assert.doesNotMatch(markup, /Настройка сети|Сначала найдите и выберите K1/); + assertCanonicalConnectionCopy(markup); +}); + +test("a completed search never adopts another tab's discovery generation", () => { + const state = durableTopologyState(); + state.ble_discovery_generation = 8; + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.devices = [{ + device_id: "generation-eight-result", + name: "Generation eight", + rssi: -46, + connectable: true, + likely_k1: true, + }]; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["scan-ble"], + actions: { + "scan-ble": { + allowed: true, + reason_codes: [], + target_source: "none", + required_transport_ref: null, + required_connection_mode: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + }; + + const markup = renderProvisioningAfterSearch({ + controller: provisioningController(state), + desiredMode: "bridge", + }, { completedDiscoveryGeneration: 7 }); + + assert.doesNotMatch(markup, /generation-eight-result|Generation eight/); + assert.match(markup, /Найти по Bluetooth/); + assert.doesNotMatch(markup, /Результатов:|Совпадений нет/); + assertCanonicalConnectionCopy(markup); +}); + +test("change-network opens credentials immediately and Apply does not refresh", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + assert.doesNotMatch(apply, /scanWithResult\(|verifyConnection\(|candidateRefresh/); + assert.equal((source.match(/buttonLabel:\s*"Применить"/g) ?? []).length, 3); +}); + +test("Step-2 credentials and Apply share one canonical field stack", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const styles = readFileSync(provisioningStylesUrl, "utf8"); + const form = sourceSlice( + source, + "const provisioningDraftContent = (", + "return (", + ); + + assert.match( + form, + /:\s*\(\s*
/, + ); + assert.match(form, /disabled=\{provisioningFieldsDisabled\}/); + assert.match(form, /value=\{connectionAttemptOwnsDraft \? "" : password\}/); + assert.match( + styles, + /\.field-stack,\s*\.session-form\s*\{\s*display:\s*grid;\s*gap:\s*0\.85rem;/, + ); +}); + +test("applied-network recovery spends the old intent and admits only explicit server-authorized paths", () => { + const source = readFileSync(provisioningSourceUrl, "utf8"); + const apply = sourceSlice(source, "const submitConnect", "const verifyAppliedNetwork"); + const recoveryState = sourceSlice( + source, + "const appliedNetworkAttempt", + "const localProvisioningPrerequisitesReady", + ); + const appliedRecoveryTarget = sourceSlice( + source, + "const appliedNetworkRecoveryTarget", + "const networkRecoveryModeLabel", + ); + const connectAuthority = sourceSlice( + source, + "const canConnect", + "const scanAllowedByPolicy", + ); + const scanAuthority = sourceSlice( + source, + "const backendScanAllowed", + "const physicalStopRecoverySettling", + ); + const search = sourceSlice(source, "const repeatDeviceScan", "const submitConnect"); + const recovery = sourceSlice( + source, + "const verifyAppliedNetwork", + "const verifyConnectionRecoveryTarget", + ); + const prepare = sourceSlice( + source, + "const prepareReconfiguration", + "const changeDesiredConnectionMode", + ); + const mode = sourceSlice( + source, + "const changeDesiredConnectionMode", + "const selectFreshDevice", + ); + const selection = sourceSlice( + source, + "const selectFreshDevice", + "// Only an explicit public search", + ); + const recoveryEscape = sourceSlice( + source, + "const recoverBySelectingAnotherDevice", + "const attemptButtonLabel", + ); + + assert.doesNotMatch(apply, /verifyConnection\(/); + assert.match(recoveryState, /state\?\.connection_attempt\?\.phase === "network_applied"/); + assert.match( + recoveryState, + /connectionAttemptOwnsAppliedNetworkRecovery\(appliedNetworkAttempt\)/, + ); + assert.match( + recoveryState, + /`\$\{snapshotRuntimeId\}:\$\{appliedNetworkAttempt\.attempt_id\}`/, + ); + assert.match( + recoveryState, + /escapedAppliedAttemptKey === appliedAttemptRecoveryKey[\s\S]*?\|\| appliedRecoveryReconfigurationPrepared/, + ); + assert.match( + recoveryState, + /connectionAttemptOwnsAppliedNetworkRecovery\(appliedNetworkAttempt\)[\s\S]*?&& !appliedRecoveryExplicitlyEscaped/, + ); + assert.match(recoveryState, /const networkRecoveryRequired = unresolvedAppliedAttempt !== null/); + assert.match( + recoveryState, + /const appliedControlSettlementPending = Boolean\([\s\S]*?\["accepted", "running"\]\.includes\(unresolvedAppliedAttempt\.status\)[\s\S]*?safe_next_action === "wait-for-current-attempt"/, + ); + assert.doesNotMatch( + appliedRecoveryTarget, + /\bconnectionMode\b|selectedDeviceId|selectedTarget/, + ); + assert.match( + appliedRecoveryTarget, + /serverBoundAppliedNetworkObservationTarget\(\s*state,\s*unresolvedAppliedAttempt\.connection_mode/, + ); + + assert.match(connectAuthority, /!unresolvedAppliedAttempt/); + assert.match(scanAuthority, /!networkRecoveryRequired/); + assert.match(apply, /\|\| unresolvedAppliedAttempt/); + assert.match(apply, /provisioningIntentKey\(null\)/); + assert.match(mode, /if \(modeResetInFlight\) return/); + assert.match(mode, /reset_scenario: true/); + assert.match(selection, /networkRecoveryRequired\s*\|\|/); + assert.doesNotMatch( + source, + /powerConfirmed|powerConfirmationEpoch|resetPowerConfirmation|Питание включено|title="Питание"/, + ); + assert.match(source, /value=\{connectionMode\}[\s\S]*?onChange=\{\(value\) => void changeDesiredConnectionMode\(value\)\}/); + assert.match( + sourceSlice(source, " void repeatDeviceScan\(\{\s*appliedRecoveryEscape: true/, + ); + assert.match(source, /Начать новый поиск Bluetooth/); + assert.match(source, /Подтверждаем управляющее подключение/); + assert.match(source, /BLE-команда не повторяется/); + assert.match( + source, + /status=\{\s*searchActive[\s\S]*?: physicalRecoveryRequired[\s\S]*?: networkRecoveryRequired[\s\S]*?>\s*\{searchActive \? \([\s\S]*?Поиск Bluetooth · \{scanSecondsRemaining \?\? 6\} с[\s\S]*?: networkRecoveryRequired && !connectionAttemptSettling \? \(/, + ); + + assert.match( + source, + /if \(!appliedNetworkAttempt\) return;[\s\S]*?setExplicitProvisioningDraft\(null\);[\s\S]*?setSsid\(""\);[\s\S]*?setPassword\(""\)/, + ); + const appliedAttemptEffect = sourceSlice( + source, + "if (!appliedNetworkAttempt) return;", + "if (connectionAttemptView !== \"retired\") return;", + ); + assert.doesNotMatch(appliedAttemptEffect, /setConnectionAttemptPresentation\(null\)/); + assert.doesNotMatch(apply, /void submitConnect|repeatDeviceScan\(/); + + const settlingState = durableTopologyState(); + settlingState.connection_attempt = { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1", + attempt_id: "network-apply-settling", + connection_mode: "bridge", + status: "running", + phase: "network_applied", + control_state: "unknown", + stage: "host-route-and-control-endpoint", + public_error_code: null, + side_effect_status: "none", + safe_next_action: "wait-for-current-attempt", + automatic_retry: false, + accepted_at: OBSERVED_AT, + completed_at: null, + recovery_operation_id: null, + timeline: [], + }; + const settlingMarkup = renderProvisioning({ + controller: provisioningController(settlingState), + desiredMode: "bridge", + }); + assert.match(settlingMarkup, /Подтверждаем управляющее подключение/); + assert.match(settlingMarkup, /BLE-команда не повторяется/); + assert.doesNotMatch(settlingMarkup, /Сеть (?:готова|настроена)|Сетевые настройки уже применены/); + assert.equal( + (settlingMarkup.match(/class="connection-action-progress"/g) ?? []).length, + 1, + ); + assert.doesNotMatch( + settlingMarkup, + /Проверить подключение без изменения сети|Начать выбор другого K1|Начать новый поиск Bluetooth/, + ); +}); + + +test("incomplete historical recovery policy stays outside the normal connection form", () => { + const state = durableTopologyState(); + state.network_mutation_ledger = { + ...state.network_mutation_ledger, + status: "unresolved", + mutation_allowed: false, + operation_id: "operation-002", + transport_ref: "ble-k1-001", + intended_mode: "bridge", + stage: "observing", + revision: 6, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: ["observe-configured-device-network"], + actions: { + "observe-configured-device-network": { + allowed: true, + reason_codes: [], + target_source: "durable-configured-state", + required_transport_ref: "ble-k1-001", + required_connection_mode: null, + requires_live_gatt_validation: true, + automatic_retry: false, + }, + }, + recommended_action: "observe-configured-device-network", + }; + + const markup = renderProvisioning({ + controller: provisioningController(state), + desiredMode: "bridge", + }); + + assert.doesNotMatch(markup, /Сервер пока не выдал одну точную цель/); + assert.doesNotMatch(markup, /не подменяют UUID и режим незавершённой операции/); + assert.doesNotMatch(markup, /Проверить .*без записи/); + assert.equal( + buttonMarkupWithText(markup, "Подключиться к сохранённому устройству").length, + 0, + ); + assert.doesNotMatch(markup, /class="connection-action-progress"/); + assert.doesNotMatch(markup, /Проверить связь/); + assert.doesNotMatch(markup, /Подключить K1 к общей сети/); +}); + + +test("configured endpoint policy explains missing durable evidence without implying BLE", () => { + const state = durableTopologyState(); + state.semantic_topology_store = { + status: "empty", + configured_offline_evidence: false, + live_connection_authority: false, + reason_code: null, + record: null, + }; + state.connection_policy = { + schema_version: "missioncore.xgrids-k1-connection-policy/v1", + facts: { retained_context_is_presence: false }, + allowed_actions: [], + actions: { + "inspect-configured-endpoint": { + allowed: false, + reason_codes: ["configured-endpoint-unavailable"], + target_source: "configured-topology", + required_transport_ref: null, + requires_live_gatt_validation: false, + automatic_retry: false, + }, + }, + recommended_action: "inspect-host-network", + }; + + const guidance = connectionPolicyOperatorGuidance( + state, + "inspect-configured-endpoint", + ); + assert.match(guidance.reason, /Нет подтверждённого сохранённого адреса K1/); + assert.match(guidance.nextAction, /Проверьте активную локальную сеть/); + assert.doesNotMatch(`${guidance.reason} ${guidance.nextAction}`, /Bluetooth|BLE/); +}); + +test("spatial controls distinguish control authority from data authority", () => { + const live = runtimeState(); + assert.deepEqual(k1SpatialAuthorityState(live), { + controlAuthoritative: true, + dataAuthoritative: true, + softwareCommanded: true, + authorityFailure: null, + }); + + live.connection_supervisor = supervisor({ control: true, data: false }); + const waiting = k1SpatialAuthorityState(live); + assert.equal(waiting.controlAuthoritative, true); + assert.equal(waiting.dataAuthoritative, false); + assert.equal(waiting.softwareCommanded, true); + assert.match(waiting.authorityFailure, /Телеметрия скрыта/); + + live.connection_supervisor = supervisor({ control: false, data: false }); + const lost = k1SpatialAuthorityState(live); + assert.equal(lost.controlAuthoritative, false); + assert.equal(lost.dataAuthoritative, false); + assert.equal(lost.softwareCommanded, false); + assert.match(lost.authorityFailure, /команды устройству запрещены/); +}); + +test("spatial controls explain the bounded K1 calibration wait before point data", () => { + const acquisition = runtimeState().acquisition; + assert.ok(acquisition); + for (const state of ["awaiting_external_start", "starting"]) { + const phase = k1SpatialPhasePresentation({ ...acquisition, state }, true); + assert.equal(phase.busy, true); + assert.match(phase.label, /калибруется.*облако точек/i); + assert.match(phase.detail, /десятки секунд/i); + assert.match(phase.detail, /не перемещайте/i); + assert.doesNotMatch(phase.detail, /30\s*с/i); + } +}); + +test("contour health never promotes selection or replay metrics to live authority", () => { + const selectedOnly = { + phase: "starting", + sourceMode: "idle", + activeDevice: { + pluginId: "xgrids-k1", + modelId: "lixelkity-k1", + displayName: "XGRIDS LixelKity K1", + instanceId: "device-k1-001", + endpointLabel: "192.168.68.52", + }, + deviceSession: { + sessionId: "device-session-001", + deviceId: "device-k1-001", + compatibilityProfileId: PROFILE_ID, + connectivity: "unknown", + }, + metrics: { aiFrameRateHz: 8 }, + }; + const unverified = contourRuntimeAuthorityPresentation(selectedOnly); + assert.equal(unverified.controlledDevice, null); + assert.equal(unverified.aiActive, false); + + const live = contourRuntimeAuthorityPresentation({ + ...selectedOnly, + phase: "streaming", + sourceMode: "live", + deviceSession: { ...selectedOnly.deviceSession, connectivity: "connected" }, + }); + assert.equal(live.controlledDevice.instanceId, "device-k1-001"); + assert.equal(live.aiActive, true); + + const replay = contourRuntimeAuthorityPresentation({ + ...selectedOnly, + phase: "replaying", + sourceMode: "replay", + deviceSession: { ...selectedOnly.deviceSession, connectivity: "connected" }, + }); + assert.equal(replay.aiActive, false); +}); diff --git a/apps/control-station/test/liveReceiverWatchdog.test.mjs b/apps/control-station/test/liveReceiverWatchdog.test.mjs index e0cf274..d13a14f 100644 --- a/apps/control-station/test/liveReceiverWatchdog.test.mjs +++ b/apps/control-station/test/liveReceiverWatchdog.test.mjs @@ -4,9 +4,14 @@ import { after, before, test } from "node:test"; import { createServer } from "vite"; let server; +let advanceLiveReceiverOpenWatchdog; let advanceLiveReceiverWatchdog; +let initialLiveReceiverOpenWatchdogState; let initialLiveReceiverWatchdogState; let initialLiveReceiverRecoveryState; +let liveReceiverRecoveryAuthorityIsCurrent; +let liveReceiverRecoveryRetryDelay; +let liveRerunRecoveryAuthorityIdentity; let requestLiveReceiverRecovery; before(async () => { @@ -16,9 +21,14 @@ before(async () => { server: { middlewareMode: true }, }); ({ + advanceLiveReceiverOpenWatchdog, advanceLiveReceiverWatchdog, + initialLiveReceiverOpenWatchdogState, initialLiveReceiverRecoveryState, initialLiveReceiverWatchdogState, + liveReceiverRecoveryAuthorityIsCurrent, + liveReceiverRecoveryRetryDelay, + liveRerunRecoveryAuthorityIdentity, requestLiveReceiverRecovery, } = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts")); }); @@ -92,3 +102,217 @@ test("startup failures request only three bounded viewer restarts", () => { assert.equal(exhausted.attempt, 3); assert.equal(exhausted.state.awaitingRecovery, false); }); + +function livePointCloudDescriptor(overrides = {}) { + return { + id: "xgrids-k1:lixelkity-k1:sensor.lidar.primary", + sourceId: "sensor.lidar.primary", + semanticChannelId: "spatial.point-cloud.live", + label: "K1 point cloud", + description: "live", + modality: "point-cloud", + role: "primary", + availability: "streaming", + transport: "rerun-grpc", + endpointLabel: "Rerun gRPC", + previewUrl: "rerun+http://127.0.0.1:9877/proxy", + delivery: null, + activation: null, + presentationLease: null, + provider: { + pluginId: "xgrids-k1", + pluginVersion: "0.1.0", + modelId: "lixelkity-k1", + compatibilityProfileId: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + }, + binding: { + deviceId: "device-k1-001", + deviceSessionId: "device-session-001", + acquisitionId: "acquisition-001", + }, + capabilities: { + overlay: false, + fullscreen: true, + resizable: false, + defaultVisible: true, + timelineMode: "live-only", + seekable: false, + sessionRecording: false, + clockId: "acquisition-001", + spatialRegistration: "native", + }, + ...overrides, + }; +} + +const liveSpatialSource = { + id: "acquisition-001", + url: "rerun+http://127.0.0.1:9877/proxy", + label: "Live", + kind: "rerun-grpc", +}; + +test("exact live Rerun authority gets durable retries with capped delay", () => { + const authority = liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor(), + liveSpatialSource, + ); + assert.ok(authority); + assert.equal(liveReceiverRecoveryAuthorityIsCurrent(authority, authority), true); + + let state = initialLiveReceiverRecoveryState(); + const delays = []; + for (let attempt = 1; attempt <= 8; attempt += 1) { + const recovery = requestLiveReceiverRecovery(state, { + activeAuthorityIdentity: authority, + expectedAuthorityIdentity: authority, + }); + assert.equal(recovery.signal, "retry"); + assert.equal(recovery.attempt, attempt); + delays.push(recovery.delayMs); + state = recovery.state; + } + assert.deepEqual(delays, [400, 1_000, 2_000, 5_000, 5_000, 5_000, 5_000, 5_000]); + assert.equal(state.attempts, 8); + assert.equal(liveReceiverRecoveryRetryDelay(100), 5_000); +}); + +test("Rerun durable retry fails closed when exact authority is replaced", () => { + const authority = liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor(), + liveSpatialSource, + ); + assert.ok(authority); + const replacement = liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor({ + binding: { + deviceId: "device-k1-001", + deviceSessionId: "device-session-002", + acquisitionId: "acquisition-002", + }, + capabilities: { + ...livePointCloudDescriptor().capabilities, + clockId: "acquisition-002", + }, + }), + { ...liveSpatialSource, id: "acquisition-002" }, + ); + assert.ok(replacement); + + const stale = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState(), { + activeAuthorityIdentity: replacement, + expectedAuthorityIdentity: authority, + }); + assert.equal(stale.signal, "stale"); + assert.equal(stale.delayMs, null); + assert.deepEqual(stale.state, initialLiveReceiverRecoveryState()); +}); + +test("connecting Rerun authority requires an exact recovery generation lease", () => { + const recoveryLease = { + kind: "active-stream-recovery", + runtimeId: "runtime-recovery-001", + acquisitionId: "acquisition-001", + acquisitionStateRevision: 4, + producerGeneration: 17, + recoveryGeneration: 6, + }; + const recoveredAuthority = liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor({ + availability: "connecting", + presentationLease: recoveryLease, + }), + liveSpatialSource, + ); + assert.ok(recoveredAuthority); + assert.equal( + liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor({ availability: "connecting" }), + liveSpatialSource, + ), + null, + ); + assert.equal( + liveRerunRecoveryAuthorityIdentity( + livePointCloudDescriptor({ + availability: "connecting", + presentationLease: { ...recoveryLease, producerGeneration: 0 }, + }), + liveSpatialSource, + ), + null, + ); +}); + +test("opening receiver gets bounded rolling patience while backend publication advances", () => { + let openState = initialLiveReceiverOpenWatchdogState(0, 0); + let recoveryState = initialLiveReceiverRecoveryState(); + + const samples = [ + [134, 3_999, "wait-for-store"], + [266, 4_000, "refresh-receiver"], + ]; + for (const [backendActivitySequence, nowMs, expectedSignal] of samples) { + const observed = advanceLiveReceiverOpenWatchdog( + openState, + recoveryState, + backendActivitySequence, + nowMs, + ); + assert.equal(observed.signal, expectedSignal); + assert.equal(observed.state.lastBackendActivitySequence, backendActivitySequence); + assert.deepEqual(observed.recoveryState, { + attempts: 0, + awaitingRecovery: false, + }); + openState = observed.state; + recoveryState = observed.recoveryState; + } +}); + +test("unchanged opening sequence delegates to bounded receiver restart", () => { + const openState = initialLiveReceiverOpenWatchdogState(486); + const recoveryState = initialLiveReceiverRecoveryState(); + const unchanged = advanceLiveReceiverOpenWatchdog( + openState, + recoveryState, + 486, + ); + + assert.equal(unchanged.signal, "restart-receiver"); + const restart = requestLiveReceiverRecovery(unchanged.recoveryState); + assert.equal(restart.signal, "retry"); + assert.equal(restart.attempt, 1); +}); + +test("fresh backend progress preserves earlier restart debt until viewer admission", () => { + const openState = initialLiveReceiverOpenWatchdogState(486, 0); + const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState()); + const observed = advanceLiveReceiverOpenWatchdog( + openState, + consumedRestart.state, + 600, + 3_999, + ); + + assert.equal(observed.signal, "wait-for-store"); + assert.deepEqual(observed.recoveryState, { + attempts: 1, + awaitingRecovery: true, + }); +}); + +test("aged active receiver refresh does not spend or erase restart debt", () => { + const openState = initialLiveReceiverOpenWatchdogState(486, 0); + const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState()); + const observed = advanceLiveReceiverOpenWatchdog( + openState, + consumedRestart.state, + 900, + 4_000, + ); + + assert.equal(observed.signal, "refresh-receiver"); + assert.deepEqual(observed.recoveryState, consumedRestart.state); + assert.equal(observed.openForMs, 4_000); +}); diff --git a/apps/control-station/test/liveViewerDiagnostics.test.mjs b/apps/control-station/test/liveViewerDiagnostics.test.mjs new file mode 100644 index 0000000..0447951 --- /dev/null +++ b/apps/control-station/test/liveViewerDiagnostics.test.mjs @@ -0,0 +1,216 @@ +import assert from "node:assert/strict"; +import { after, before, test } from "node:test"; + +import { createServer } from "vite"; + +let createLiveViewerDiagnosticLifecycle; +let createAbortFencedBuildVerifier; +let createUiBuildStaleCoordinator; +let liveViewerDiagnosticBody; +let server; +let uiBuildIdFromModuleScripts; + +before(async () => { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ + createAbortFencedBuildVerifier, + createLiveViewerDiagnosticLifecycle, + createUiBuildStaleCoordinator, + liveViewerDiagnosticBody, + uiBuildIdFromModuleScripts, + } = await server.ssrLoadModule("/src/core/observation/liveViewerDiagnostics.ts")); +}); + +after(async () => { + await server?.close(); +}); + +function createFakeScheduler() { + let now = 0; + let nextHandle = 1; + const jobs = new Map(); + const schedule = (callback, delay, interval) => { + const handle = nextHandle; + nextHandle += 1; + jobs.set(handle, { callback, due: now + delay, interval }); + return handle; + }; + const clear = (handle) => jobs.delete(handle); + return { + scheduler: { + setTimeout: (callback, delay) => schedule(callback, delay, null), + clearTimeout: clear, + setInterval: (callback, delay) => schedule(callback, delay, delay), + clearInterval: clear, + }, + advance(milliseconds) { + const target = now + milliseconds; + while (true) { + const next = [...jobs.entries()] + .filter(([, job]) => job.due <= target) + .sort((left, right) => left[1].due - right[1].due)[0]; + if (!next) break; + const [handle, job] = next; + now = job.due; + if (job.interval === null) jobs.delete(handle); + else job.due += job.interval; + job.callback(); + } + now = target; + }, + pending: () => jobs.size, + }; +} + +const lineage = (viewerInstanceId, lifecycleGeneration = 1) => ({ + uiBuildId: "/assets/index-abcdefgh.js", + documentInstanceId: "00000000-0000-4000-8000-000000000001", + viewerInstanceId, + lifecycleGeneration, +}); + +test("mounted viewer admission terminally fences 60 seconds of stale timers", () => { + const clock = createFakeScheduler(); + const callbacks = []; + const posts = []; + const lifecycle = createLiveViewerDiagnosticLifecycle({ + lineage: lineage("00000000-0000-4000-8000-000000000011"), + scheduler: clock.scheduler, + diagnosticPoster: (event, eventLineage) => posts.push({ event, eventLineage }), + buildVerifier: () => undefined, + }); + + lifecycle.armAdmissionTimeout(() => callbacks.push("timeout"), 12_000); + lifecycle.armAdmissionInterval(() => callbacks.push("discovery"), 100); + lifecycle.markAdmitted(); + lifecycle.post({ eventCode: "live_receiver_active_store_admitted" }); + clock.advance(60_000); + + assert.deepEqual(callbacks, []); + assert.equal(clock.pending(), 0); + assert.equal(posts.length, 1); + assert.equal(posts[0].eventLineage.lifecycleGeneration, 1); +}); + +test("two mounted viewers keep timer and diagnostic lineage isolated", () => { + const clock = createFakeScheduler(); + const posts = []; + const first = createLiveViewerDiagnosticLifecycle({ + lineage: lineage("00000000-0000-4000-8000-000000000021"), + scheduler: clock.scheduler, + diagnosticPoster: (event, eventLineage) => posts.push({ event, eventLineage }), + buildVerifier: () => undefined, + }); + const second = createLiveViewerDiagnosticLifecycle({ + lineage: lineage("00000000-0000-4000-8000-000000000022", 7), + scheduler: clock.scheduler, + diagnosticPoster: (event, eventLineage) => posts.push({ event, eventLineage }), + buildVerifier: () => undefined, + }); + + first.armAdmissionTimeout(() => { + first.post({ eventCode: "live_receiver_error" }); + }, 12_000); + second.armAdmissionTimeout(() => { + second.post({ eventCode: "live_receiver_error" }); + }, 12_000); + first.markAdmitted(); + clock.advance(12_000); + + assert.equal(posts.length, 1); + assert.equal( + posts[0].eventLineage.viewerInstanceId, + "00000000-0000-4000-8000-000000000022", + ); + assert.equal(posts[0].eventLineage.lifecycleGeneration, 7); +}); + +test("stale-build and unmount fence callbacks before one reload", () => { + const clock = createFakeScheduler(); + const order = []; + const posts = []; + const lifecycle = createLiveViewerDiagnosticLifecycle({ + lineage: lineage("00000000-0000-4000-8000-000000000031"), + scheduler: clock.scheduler, + diagnosticPoster: (event) => posts.push(event), + buildVerifier: () => undefined, + }); + lifecycle.armAdmissionTimeout(() => { + lifecycle.post({ eventCode: "live_receiver_error" }); + }, 12_000); + const coordinator = createUiBuildStaleCoordinator({ + scheduleReload: (callback, delay) => { + order.push(`scheduled:${delay}`); + clock.scheduler.setTimeout(callback, delay); + }, + reload: () => order.push("reload"), + }); + coordinator.subscribe(() => { + order.push("local-transports-closed"); + lifecycle.dispose(); + }); + + coordinator.report({ + loadedUiBuildId: "/assets/index-abcdefgh.js", + expectedUiBuildId: "/assets/index-ijklmnop.js", + }); + coordinator.report({ + loadedUiBuildId: "/assets/index-abcdefgh.js", + expectedUiBuildId: "/assets/index-qrstuvwx.js", + }); + clock.advance(60_000); + lifecycle.post({ eventCode: "live_receiver_error" }); + + assert.deepEqual(order, ["local-transports-closed", "scheduled:50", "reload"]); + assert.deepEqual(posts, []); + assert.equal(lifecycle.active(), false); +}); + +test("last unsubscribe fences an already queued build verification callback", () => { + const controller = new AbortController(); + const observedSignals = []; + const queuedVerify = createAbortFencedBuildVerifier( + controller.signal, + (signal) => observedSignals.push(signal), + ); + + queuedVerify(); + // stopBuildMonitor aborts the locally captured controller when the last + // mounted viewer unsubscribes. A browser callback already queued before the + // interval/listener removal can still run once, but cannot start a fetch. + controller.abort(); + queuedVerify(); + + assert.deepEqual(observedSignals, [controller.signal]); + assert.equal(observedSignals[0].aborted, true); +}); + +test("diagnostic body and build id retain exact document/viewer/build lineage", () => { + const eventLineage = lineage("00000000-0000-4000-8000-000000000041", 9); + assert.deepEqual( + liveViewerDiagnosticBody( + { eventCode: "live_receiver_recovered", streamId: "acquisition-42" }, + eventLineage, + ), + { + schema_version: "missioncore.live-viewer-diagnostic/v2", + event_code: "live_receiver_recovered", + ui_build_id: "/assets/index-abcdefgh.js", + document_instance_id: "00000000-0000-4000-8000-000000000001", + viewer_instance_id: "00000000-0000-4000-8000-000000000041", + lifecycle_generation: 9, + stream_id: "acquisition-42", + }, + ); + assert.equal( + uiBuildIdFromModuleScripts( + ["https://mission.local/assets/index-dT7dN-y4.js"], + "https://mission.local/park", + ), + "/assets/index-dT7dN-y4.js", + ); +}); diff --git a/apps/control-station/test/observationSources.test.mjs b/apps/control-station/test/observationSources.test.mjs index f9f960b..c0f39d2 100644 --- a/apps/control-station/test/observationSources.test.mjs +++ b/apps/control-station/test/observationSources.test.mjs @@ -12,6 +12,20 @@ let openObservationSource; let shouldRestartObservationSource; let consumeCameraLeaseRetry; let resetCameraLeaseRetryBudget; +let cameraTransportRecoveryIsCurrent; +let cameraTransportCanOpen; +let cameraPendingQueueCanAccept; +let cameraTransportCloseRecoveryMessage; +let createCameraStartupWatchdog; +let cameraStartupWatchdogRecoveryMessage; +let CAMERA_FIRST_MEDIA_TIMEOUT_MS; +let CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS; +let liveCameraPlaybackAuthorityIdentity; +let initialCameraPlaybackRecoveryState; +let reduceCameraPlaybackRecovery; +let cameraTransportCallbackIsCurrent; +let cameraBrowserTransportIdentity; +let liveRerunRecoveryAuthorityIdentity; let initialObservationWindowRect; let ObservationTimeline; let shouldCaptureWorkspacePointer; @@ -48,12 +62,33 @@ before(async () => { ({ openObservationSource, shouldRestartObservationSource } = await server.ssrLoadModule( "/src/core/observation/layoutPolicy.ts", )); - ({ consumeCameraLeaseRetry, resetCameraLeaseRetryBudget } = await server.ssrLoadModule( - "/src/components/MseFmp4WebSocketPlayer.tsx", + ({ + consumeCameraLeaseRetry, + resetCameraLeaseRetryBudget, + cameraTransportRecoveryIsCurrent, + cameraTransportCanOpen, + cameraPendingQueueCanAccept, + cameraTransportCloseRecoveryMessage, + createCameraStartupWatchdog, + cameraStartupWatchdogRecoveryMessage, + CAMERA_FIRST_MEDIA_TIMEOUT_MS, + CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS, + } = await server.ssrLoadModule("/src/components/MseFmp4WebSocketPlayer.tsx")); + ({ + liveCameraPlaybackAuthorityIdentity, + initialCameraPlaybackRecoveryState, + reduceCameraPlaybackRecovery, + cameraTransportCallbackIsCurrent, + cameraBrowserTransportIdentity, + } = await server.ssrLoadModule( + "/src/core/observation/liveCameraRecovery.ts", )); ({ initialObservationWindowRect, shouldCaptureWorkspacePointer } = await server.ssrLoadModule( "/src/components/FloatingObservationWindow.tsx", )); + ({ liveRerunRecoveryAuthorityIdentity } = await server.ssrLoadModule( + "/src/core/observation/liveReceiverWatchdog.ts", + )); ({ ObservationTimeline, normalizeTimelineRange, @@ -574,6 +609,134 @@ function cameraRow(sourceId, label) { }; } +function connectionSupervisor({ dataAuthoritative = false, dataPlaneState = "idle" } = {}) { + const observedAt = "2026-08-06T12:00:00Z"; + const target = { ipv4: "192.168.68.52", port: 1883 }; + return { + schema_version: "missioncore.k1-connection-supervisor/v1", + revision: 7, + closed: false, + intent: { + intent_id: "intent-001", + requested_mode: "bridge", + expected_device_id: "device-k1-001", + requested_at: observedAt, + }, + observed: { + device_network: { + state: "applied", + intent_id: "intent-001", + transport_ref: "ble-k1-001", + connection_mode: "bridge", + target, + source: "ble-read-only-status", + observed_at: observedAt, + }, + host_path: { + epoch: 3, + available: true, + fingerprint: "en0:192.168.68.10", + interface: "en0", + source_ipv4: "192.168.68.10", + route_class: "direct", + reason_code: null, + observed_at: observedAt, + }, + endpoint: { + target, + tcp_state: "reachable", + intent_id: "intent-001", + host_path_epoch: 3, + reason_code: null, + observed_at: observedAt, + }, + device_identity: { + state: "verified", + intent_id: "intent-001", + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + connection_mode: "bridge", + source: "mqtt-device-info", + host_path_epoch: 3, + observed_at: observedAt, + }, + control_plane: { + state: "healthy", + session_id: "control-session-001", + host_path_epoch: 3, + reason_code: null, + observed_at: observedAt, + }, + data_plane: { + state: dataAuthoritative ? "healthy" : dataPlaneState, + session_id: dataAuthoritative || dataPlaneState !== "idle" ? "data-session-001" : null, + host_path_epoch: dataAuthoritative || dataPlaneState !== "idle" ? 3 : null, + reason_code: null, + observed_at: dataAuthoritative || dataPlaneState !== "idle" ? observedAt : null, + }, + }, + lease: { + state: "reachable", + generation: 4, + intent_id: "intent-001", + host_path_epoch: 3, + connection_mode: "bridge", + target, + logical_device_id: "device-k1-001", + reason_code: null, + observed_at: observedAt, + }, + authority: { + network_mutation_allowed: false, + control_allowed: true, + acquisition_start_allowed: true, + data_ingest_authoritative: dataAuthoritative, + physical_motion_allowed: false, + reason_codes: [], + }, + last_known: null, + allowed_actions: ["stop-acquisition"], + }; +} + +function connectionLifecycle() { + return { + schema_version: "missioncore.xgrids-k1-connection-lifecycle/v1", + revision: 8, + desired_mode: "bridge", + configured_mode: "bridge", + active_mode: "bridge", + mode_change: { + state: "ready", + from: "bridge", + to: "bridge", + }, + mode_selection: { + allowed: false, + reason_codes: ["connection-mode-selection-acquisition-active"], + automatic_retry: false, + }, + active_binding_key: "binding-intent-001-bridge", + active_binding: { + binding_key: "binding-intent-001-bridge", + intent_id: "intent-001", + transport_ref: "ble-k1-001", + connection_mode: "bridge", + target_ipv4: "192.168.68.52", + target_port: 1883, + host_path_epoch: 3, + control_session_id: "control-session-001", + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + }, + connection_ready: true, + ready_to_start: true, + operation: null, + allowed_actions: ["stop-acquisition"], + automatic_retry: false, + }; +} + function declaredState(cameraRows = [ cameraRow("sensor.camera.left", "K1 · камера слева"), cameraRow("sensor.camera.right", "K1 · камера справа"), @@ -582,6 +745,7 @@ function declaredState(cameraRows = [ return { phase: "connected", source_mode: "idle", + connection_mode: "bridge", k1_ip: "192.168.7.10", foxglove_ws_url: "ws://192.168.7.10:8765", foxglove_viewer_url: "http://192.168.7.10:8765/vendor-viewer", @@ -601,6 +765,16 @@ function declaredState(cameraRows = [ compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", connectivity: "connected", }, + connection_supervisor: connectionSupervisor(), + connection_lifecycle: connectionLifecycle(), + application_control_session: { + verified_control: { + logical_device_id: "device-k1-001", + compatibility_profile_id: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + control_session_id: "control-session-001", + source: "mqtt-device-info", + }, + }, sensor_catalog: { schema_version: "missioncore.sensor-catalog/v1alpha2", revision: "test-profile", @@ -620,10 +794,12 @@ function declaredState(cameraRows = [ } function pointCloudStreamingState() { + const state = declaredState(); return { - ...declaredState(), + ...state, phase: "streaming", source_mode: "live", + connection_supervisor: connectionSupervisor({ dataAuthoritative: true }), rerun_grpc_url: "rerun+http://127.0.0.1:9877/proxy", acquisition: { acquisition_id: "acquisition-001", @@ -668,9 +844,83 @@ function cameraStreamingState(sourceId) { active_source_id: sourceId, delivery, }; + state.connection_recovery = { + schema_version: "missioncore.xgrids-k1-active-stream-recovery/v1", + state: "inactive", + generation: 0, + acquisition_id: null, + attempt: 0, + started_at_utc: null, + elapsed_ms: null, + reason_code: null, + force_finish_allowed: false, + automatic_read_only_rebind: false, + automatic_command_retry: false, + start_performed: false, + stop_performed: false, + ble_operation_performed: false, + network_mutation_performed: false, + runtime_producer_generation: null, + camera_recovery: "inactive", + camera_media_state: "ready", + camera_media_ready: true, + camera_epoch: { + generation: 7, + init_committed: true, + init_committed_age_ms: 100, + first_media_committed: true, + first_media_committed_age_ms: 80, + committed_media_segment_count: 2, + last_media_segment_age_ms: 20, + }, + }; return state; } +function cameraRecoveringState(sourceId, recoveryOverrides = {}, stateOverrides = {}) { + const state = cameraStreamingState(sourceId); + return { + ...state, + snapshot_runtime_id: "runtime-camera-recovery-001", + snapshot_revision: 29, + producer_generation: 17, + phase: "reconnecting", + connection_supervisor: connectionSupervisor({ dataPlaneState: "lost" }), + connection_recovery: { + schema_version: "missioncore.xgrids-k1-active-stream-recovery/v1", + state: "reconnecting", + generation: 6, + acquisition_id: state.acquisition.acquisition_id, + attempt: 2, + started_at_utc: "2026-08-11T21:19:59Z", + elapsed_ms: 16_869, + reason_code: "host-route-unavailable", + force_finish_allowed: true, + automatic_read_only_rebind: true, + automatic_command_retry: false, + start_performed: false, + stop_performed: false, + ble_operation_performed: false, + network_mutation_performed: false, + runtime_producer_generation: 17, + camera_recovery: "owned", + camera_media_state: "pending-first-media", + camera_media_ready: false, + camera_epoch: { + generation: 7, + init_committed: true, + init_committed_age_ms: 250, + first_media_committed: false, + first_media_committed_age_ms: null, + committed_media_segment_count: 0, + last_media_segment_age_ms: null, + }, + ...recoveryOverrides, + }, + ...stateOverrides, + }; +} + function collectUrlLikeStrings(value, found = []) { if (typeof value === "string") { if (value.includes("://")) found.push(value); @@ -719,6 +969,41 @@ test("descriptor ids remain stable while point cloud and selected camera start s assert.equal(streaming[0].previewUrl, "rerun+http://127.0.0.1:9877/proxy"); }); +test("legacy selected-device fields cannot attest or stream observation sources", () => { + const state = cameraStreamingState("sensor.camera.left"); + delete state.connection_supervisor; + delete state.application_control_session; + + const sources = xgridsK1ObservationSources(state, model()); + const pointCloud = sources.find(({ modality }) => modality === "point-cloud"); + const left = sources.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(pointCloud && left); + assert.equal(pointCloud.availability, "unverified"); + assert.equal(pointCloud.previewUrl, null); + assert.equal(pointCloud.binding.deviceId, null); + assert.equal(pointCloud.binding.deviceSessionId, null); + assert.equal(left.availability, "unverified"); + assert.equal(left.activation.selected, false); + assert.equal(left.activation.controllable, false); + assert.equal(left.delivery, null); +}); + +test("data-plane loss keeps control identity but withdraws all live deliveries", () => { + const state = cameraStreamingState("sensor.camera.left"); + state.connection_supervisor = connectionSupervisor({ dataPlaneState: "lost" }); + + const sources = xgridsK1ObservationSources(state, model()); + const pointCloud = sources.find(({ modality }) => modality === "point-cloud"); + const left = sources.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(pointCloud && left); + assert.equal(pointCloud.availability, "degraded"); + assert.equal(pointCloud.previewUrl, null); + assert.equal(pointCloud.binding.deviceId, "device-k1-001"); + assert.equal(left.activation.selected, true); + assert.equal(left.availability, "degraded"); + assert.equal(left.delivery, null); +}); + test("only the authoritative selected camera receives browser delivery", () => { const sources = xgridsK1ObservationSources( cameraStreamingState("sensor.camera.left"), @@ -740,6 +1025,43 @@ test("only the authoritative selected camera receives browser delivery", () => { assert.match(left.activation.groupId, /device-session-001/); }); +test("ordinary camera delivery requires ready media facts from the exact current epoch", () => { + const pending = cameraStreamingState("sensor.camera.left"); + pending.connection_recovery = { + ...pending.connection_recovery, + camera_media_state: "pending-first-media", + camera_media_ready: false, + camera_epoch: { + generation: 7, + init_committed: true, + init_committed_age_ms: 20, + first_media_committed: false, + first_media_committed_age_ms: null, + committed_media_segment_count: 0, + last_media_segment_age_ms: null, + }, + }; + const staleEpoch = cameraStreamingState("sensor.camera.left"); + staleEpoch.connection_recovery = { + ...staleEpoch.connection_recovery, + camera_epoch: { + ...staleEpoch.connection_recovery.camera_epoch, + generation: 6, + }, + }; + + for (const state of [pending, staleEpoch]) { + const camera = xgridsK1ObservationSources(state, model()).find( + ({ sourceId }) => sourceId === "sensor.camera.left", + ); + assert.ok(camera); + assert.equal(camera.activation.selected, false); + assert.equal(camera.delivery, null); + assert.equal(camera.availability, "available"); + assert.equal(liveCameraPlaybackAuthorityIdentity(camera), null); + } +}); + test("switching left to right keeps ids stable and never exposes both deliveries", () => { const left = xgridsK1ObservationSources( cameraStreamingState("sensor.camera.left"), @@ -781,6 +1103,8 @@ test("unattested, duplicate and unsafe camera entries fail closed", () => { const state = cameraStreamingState("sensor.camera.left"); state.compatibility.profile_id = null; state.device_session.compatibility_profile_id = null; + state.connection_supervisor.observed.device_identity.state = "mismatch"; + state.application_control_session.verified_control = null; state.sensor_catalog.streams.push(duplicate); state.camera_preview.delivery = { ...state.camera_preview.delivery, @@ -825,14 +1149,14 @@ test("camera delivery rejects literal and encoded endpoint or credential leaks", } }); -test("manual camera reconnect restores an exhausted lease retry budget", () => { +test("authoritative camera transport retries indefinitely with a capped backoff", () => { let budget = resetCameraLeaseRetryBudget("delivery-7"); - for (const expectedDelay of [400, 1_000, 2_000]) { + for (const expectedDelay of [400, 1_000, 2_000, 5_000, 5_000, 5_000]) { const retry = consumeCameraLeaseRetry(budget, "delivery-7"); assert.equal(retry.delay, expectedDelay); budget = retry.budget; } - assert.equal(consumeCameraLeaseRetry(budget, "delivery-7").delay, null); + assert.equal(budget.count, 4); budget = resetCameraLeaseRetryBudget("delivery-7"); const retryAfterManualReset = consumeCameraLeaseRetry(budget, "delivery-7"); @@ -840,6 +1164,592 @@ test("manual camera reconnect restores an exhausted lease retry budget", () => { assert.equal(retryAfterManualReset.budget.count, 1); }); +test("camera startup watchdog replaces an open-but-silent MSE or WebSocket", () => { + const scheduled = new Map(); + const timeouts = []; + let nextHandle = 1; + const watchdog = createCameraStartupWatchdog({ + schedule(callback, timeoutMs) { + const handle = nextHandle; + nextHandle += 1; + scheduled.set(handle, { callback, timeoutMs }); + return handle; + }, + cancel(handle) { + scheduled.delete(handle); + }, + onTimeout(stage) { + timeouts.push(stage); + }, + }); + + watchdog.armFirstMedia(); + watchdog.armFirstMedia(); + assert.equal(watchdog.pendingStage(), "first-media"); + assert.equal(scheduled.size, 1); + const [{ callback, timeoutMs }] = scheduled.values(); + assert.equal(timeoutMs, CAMERA_FIRST_MEDIA_TIMEOUT_MS); + callback(); + + assert.deepEqual(timeouts, ["first-media"]); + assert.equal(watchdog.pendingStage(), null); + assert.match( + cameraStartupWatchdogRecoveryMessage("first-media"), + /не передаёт медиаданные; восстанавливаем/, + ); +}); + +test("first media starts one non-sliding first-playable-frame deadline", () => { + const scheduled = new Map(); + const cancelled = []; + const timeouts = []; + let nextHandle = 10; + const watchdog = createCameraStartupWatchdog({ + schedule(callback, timeoutMs) { + const handle = nextHandle; + nextHandle += 1; + scheduled.set(handle, { callback, timeoutMs }); + return handle; + }, + cancel(handle) { + cancelled.push(handle); + scheduled.delete(handle); + }, + onTimeout(stage) { + timeouts.push(stage); + }, + }); + + watchdog.armFirstMedia(); + watchdog.markMediaReceived(); + const playableHandle = nextHandle - 1; + assert.deepEqual(cancelled, [10]); + assert.equal(watchdog.pendingStage(), "first-playable-frame"); + assert.equal(scheduled.get(playableHandle).timeoutMs, CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS); + + watchdog.markMediaReceived(); + assert.equal(nextHandle - 1, playableHandle, "later fragments must not extend the deadline"); + scheduled.get(playableHandle).callback(); + assert.deepEqual(timeouts, ["first-playable-frame"]); + assert.match( + cameraStartupWatchdogRecoveryMessage("first-playable-frame"), + /первый кадр не воспроизводится; пересоздаём decoder/, + ); +}); + +test("playing clears the camera startup watchdog", () => { + const scheduled = new Map(); + const cancelled = []; + const watchdog = createCameraStartupWatchdog({ + schedule(callback) { + scheduled.set(21, callback); + return 21; + }, + cancel(handle) { + cancelled.push(handle); + scheduled.delete(handle); + }, + onTimeout() { + assert.fail("a playing transport must not time out"); + }, + }); + + watchdog.armFirstMedia(); + watchdog.markMediaReceived(); + watchdog.markPlaying(); + assert.equal(watchdog.pendingStage(), null); + assert.deepEqual(cancelled, [21, 21]); + assert.equal(scheduled.size, 0); +}); + +test("browser append queue absorbs the complete bounded server backlog", () => { + assert.equal( + cameraPendingQueueCanAccept(8 * 1024 * 1024, 64, 1024 * 1024), + true, + ); + assert.equal( + cameraPendingQueueCanAccept(11 * 1024 * 1024, 95, 1024 * 1024), + true, + ); + assert.equal( + cameraPendingQueueCanAccept(12 * 1024 * 1024, 64, 1), + false, + ); + assert.equal( + cameraPendingQueueCanAccept(8 * 1024 * 1024, 96, 1), + false, + ); +}); + +test("server slow-reader close is an automatic browser-only camera recovery", () => { + assert.match( + cameraTransportCloseRecoveryMessage(4_008), + /отстал от эфира; восстанавливаем текущую камеру/, + ); + assert.match(cameraTransportCloseRecoveryMessage(1_008), /переподключаемся/); + assert.match(cameraTransportCloseRecoveryMessage(1_000), /восстанавливаем/); + assert.match(cameraTransportCloseRecoveryMessage(1_011), /восстанавливаем/); +}); + +test("a laptop sleep gap reopens only the exact authoritative live camera transport", () => { + const source = xgridsK1ObservationSources( + cameraStreamingState("sensor.camera.right"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.right"); + assert.ok(source); + const authority = liveCameraPlaybackAuthorityIdentity(source); + assert.ok(authority); + + const initial = initialCameraPlaybackRecoveryState(authority, 1_000); + const wake = reduceCameraPlaybackRecovery(initial, { type: "heartbeat" }, { + activeAuthorityIdentity: authority, + now: 7_000, + documentVisible: true, + networkOnline: true, + }); + assert.equal(wake.reopen, true); + assert.equal(wake.state.lastReopenAt, 7_000); + + const staleSource = reduceCameraPlaybackRecovery(initial, { type: "heartbeat" }, { + activeAuthorityIdentity: `${authority}:replaced`, + now: 7_000, + documentVisible: true, + networkOnline: true, + }); + assert.equal(staleSource.reopen, false); +}); + +test("visibility, pageshow and online wake burst owns one replacement decoder", () => { + const authority = "camera-authority-acquisition-1"; + let state = initialCameraPlaybackRecoveryState(authority, 1_000); + ({ state } = reduceCameraPlaybackRecovery(state, { type: "document-hidden" }, { + activeAuthorityIdentity: authority, + now: 2_000, + documentVisible: false, + networkOnline: true, + })); + + const visible = reduceCameraPlaybackRecovery(state, { type: "document-visible" }, { + activeAuthorityIdentity: authority, + now: 8_000, + documentVisible: true, + networkOnline: true, + }); + assert.equal(visible.reopen, true); + state = visible.state; + + for (const event of [ + { type: "page-restore", persisted: true }, + { type: "network-online" }, + { type: "heartbeat" }, + ]) { + const duplicate = reduceCameraPlaybackRecovery(state, event, { + activeAuthorityIdentity: authority, + now: 8_100, + documentVisible: true, + networkOnline: true, + }); + assert.equal(duplicate.reopen, false); + state = duplicate.state; + } + + ({ state } = reduceCameraPlaybackRecovery(state, { type: "document-hidden" }, { + activeAuthorityIdentity: authority, + now: 12_000, + documentVisible: false, + networkOnline: true, + })); + const secondWake = reduceCameraPlaybackRecovery(state, { type: "document-visible" }, { + activeAuthorityIdentity: authority, + now: 18_000, + documentVisible: true, + networkOnline: true, + }); + assert.equal(secondWake.reopen, true); + assert.equal(secondWake.state.lastReopenAt, 18_000); +}); + +test("healthy visible camera heartbeats do not churn its WebSocket or decoder", () => { + const authority = "camera-authority-acquisition-healthy"; + let state = initialCameraPlaybackRecoveryState(authority, 1_000); + for (const now of [2_000, 3_000, 4_000, 5_000]) { + const heartbeat = reduceCameraPlaybackRecovery(state, { type: "heartbeat" }, { + activeAuthorityIdentity: authority, + now, + documentVisible: true, + networkOnline: true, + }); + assert.equal(heartbeat.reopen, false); + state = heartbeat.state; + } + assert.equal(state.lastReopenAt, null); +}); + +test("callbacks from a replaced camera WebSocket and SourceBuffer are fenced", () => { + assert.equal(cameraTransportCallbackIsCurrent(4, 4, false), true); + assert.equal(cameraTransportCallbackIsCurrent(5, 4, false), false); + assert.equal(cameraTransportCallbackIsCurrent(4, 4, true), false); +}); + +test("an already-stale document opens zero camera transports on mount", () => { + let transportOpenCount = 0; + const uiBuildStaleRef = { current: false }; + const subscribeToAlreadyStaleCoordinator = (listener) => { + listener(); + return () => undefined; + }; + + subscribeToAlreadyStaleCoordinator(() => { + uiBuildStaleRef.current = true; + }); + if (cameraTransportCanOpen(uiBuildStaleRef.current)) transportOpenCount += 1; + + assert.equal(uiBuildStaleRef.current, true); + assert.equal(transportOpenCount, 0); +}); + +test("only the current acquisition authority may schedule a replacement decoder", () => { + const authority = "camera-authority-acquisition-current"; + assert.equal( + cameraTransportRecoveryIsCurrent(authority, authority, 7, 7, false), + true, + ); + assert.equal( + cameraTransportRecoveryIsCurrent(`${authority}:replaced`, authority, 7, 7, false), + false, + ); + assert.equal( + cameraTransportRecoveryIsCurrent(authority, authority, 8, 7, false), + false, + ); + assert.equal( + cameraTransportRecoveryIsCurrent(authority, authority, 7, 7, true), + false, + ); + assert.equal( + cameraTransportRecoveryIsCurrent(null, authority, 7, 7, false), + false, + ); +}); + +test("same camera delivery gets a new browser transport owner for a new acquisition", () => { + const camera = xgridsK1ObservationSources( + cameraStreamingState("sensor.camera.right"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.right"); + assert.ok(camera?.delivery && camera.delivery.kind === "mse-fmp4-websocket"); + const previousAuthority = liveCameraPlaybackAuthorityIdentity(camera); + const nextAuthority = liveCameraPlaybackAuthorityIdentity({ + ...camera, + binding: { + ...camera.binding, + deviceSessionId: "device-session-002", + acquisitionId: "acquisition-002", + }, + }); + assert.ok(previousAuthority && nextAuthority); + assert.notEqual(nextAuthority, previousAuthority); + assert.notEqual( + cameraBrowserTransportIdentity(camera.delivery, nextAuthority), + cameraBrowserTransportIdentity(camera.delivery, previousAuthority), + ); +}); + +test("malformed camera exclusivity or media descriptors cannot own wake recovery", () => { + const camera = xgridsK1ObservationSources( + cameraStreamingState("sensor.camera.left"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(camera?.activation && camera.delivery?.kind === "mse-fmp4-websocket"); + + for (const malformed of [ + { ...camera, activation: { ...camera.activation, groupId: " " } }, + { ...camera, activation: { ...camera.activation, groupId: null } }, + { ...camera, activation: { ...camera.activation, maxActive: 2 } }, + { ...camera, delivery: { ...camera.delivery, mediaType: " " } }, + { ...camera, delivery: { ...camera.delivery, mediaType: null } }, + { ...camera, delivery: { ...camera.delivery, mediaType: "video/webm" } }, + ]) { + assert.equal(liveCameraPlaybackAuthorityIdentity(malformed), null); + } +}); + +test("point-cloud recovery cannot revive a dead or stale camera delivery", () => { + const sources = xgridsK1ObservationSources( + cameraStreamingState("sensor.camera.left"), + model(), + ); + const pointCloud = sources.find(({ modality }) => modality === "point-cloud"); + const camera = sources.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(pointCloud && camera); + assert.equal(pointCloud.availability, "streaming"); + assert.ok(liveCameraPlaybackAuthorityIdentity(camera)); + + assert.equal(liveCameraPlaybackAuthorityIdentity({ + ...camera, + availability: "error", + }), null); + assert.equal(liveCameraPlaybackAuthorityIdentity({ + ...camera, + binding: { ...camera.binding, acquisitionId: "acquisition-replaced" }, + activation: { ...camera.activation, selected: false }, + }), null); +}); + +test("exact active recovery retains the same camera delivery without controls", () => { + const healthy = xgridsK1ObservationSources( + cameraStreamingState("sensor.camera.left"), + model(), + ); + const recovering = xgridsK1ObservationSources( + cameraRecoveringState("sensor.camera.left"), + model(), + ); + const healthyPointCloud = healthy.find(({ modality }) => modality === "point-cloud"); + const recoveryPointCloud = recovering.find(({ modality }) => modality === "point-cloud"); + const healthyCamera = healthy.find(({ sourceId }) => sourceId === "sensor.camera.left"); + const recoveryCamera = recovering.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(healthyPointCloud && recoveryPointCloud && healthyCamera && recoveryCamera); + + assert.equal(recoveryPointCloud.id, healthyPointCloud.id); + assert.equal(recoveryPointCloud.previewUrl, healthyPointCloud.previewUrl); + assert.equal(recoveryPointCloud.availability, "connecting"); + assert.equal(recoveryCamera.id, healthyCamera.id); + assert.deepEqual(recoveryCamera.delivery, healthyCamera.delivery); + assert.equal(recoveryCamera.activation.selected, true); + assert.equal(recoveryCamera.activation.controllable, false); + assert.equal(recoveryCamera.availability, "connecting"); + assert.deepEqual(recoveryCamera.presentationLease, { + kind: "active-stream-recovery", + runtimeId: "runtime-camera-recovery-001", + acquisitionId: "acquisition-001", + acquisitionStateRevision: 3, + producerGeneration: 17, + recoveryGeneration: 6, + }); + + const healthyAuthority = liveCameraPlaybackAuthorityIdentity(healthyCamera); + const recoveryAuthority = liveCameraPlaybackAuthorityIdentity(recoveryCamera); + assert.ok(healthyAuthority && recoveryAuthority); + assert.notEqual(recoveryAuthority, healthyAuthority); + assert.equal( + liveCameraPlaybackAuthorityIdentity( + xgridsK1ObservationSources( + cameraRecoveringState("sensor.camera.left"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.left"), + ), + recoveryAuthority, + "repeated snapshots of the same recovery generation keep one browser owner", + ); +}); + +test("camera recovery stays connecting until the current epoch has durable first media", () => { + const pending = xgridsK1ObservationSources( + cameraRecoveringState("sensor.camera.left"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.left"); + const ready = xgridsK1ObservationSources( + cameraRecoveringState("sensor.camera.left", { + camera_media_state: "ready", + camera_media_ready: true, + camera_epoch: { + generation: 7, + init_committed: true, + init_committed_age_ms: 500, + first_media_committed: true, + first_media_committed_age_ms: 100, + committed_media_segment_count: 1, + last_media_segment_age_ms: 100, + }, + }), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.left"); + + assert.ok(pending?.delivery && ready?.delivery); + assert.equal(pending.activation.selected, true); + assert.equal(pending.availability, "connecting"); + assert.ok(pending.presentationLease); + assert.equal(ready.activation.selected, true); + assert.equal(ready.availability, "streaming"); + assert.ok(ready.presentationLease); +}); + +test("first recovered PCL keeps the exact camera authority until browser media is playable", () => { + const reconnectingState = cameraRecoveringState("sensor.camera.left"); + const recoveredState = cameraRecoveringState( + "sensor.camera.left", + { + state: "recovered", + force_finish_allowed: false, + elapsed_ms: null, + reason_code: null, + }, + { + phase: "live", + connection_supervisor: connectionSupervisor({ dataAuthoritative: true }), + }, + ); + recoveredState.camera_preview = { + ...recoveredState.camera_preview, + phase: "connecting", + }; + recoveredState.sensor_catalog = { + ...recoveredState.sensor_catalog, + streams: recoveredState.sensor_catalog.streams.map((stream) => + stream.source_id === "sensor.camera.left" + ? { ...stream, availability: "connecting" } + : stream), + }; + + const reconnecting = xgridsK1ObservationSources(reconnectingState, model()); + const recovered = xgridsK1ObservationSources(recoveredState, model()); + const reconnectingCamera = reconnecting.find( + ({ sourceId }) => sourceId === "sensor.camera.left", + ); + const recoveredCamera = recovered.find( + ({ sourceId }) => sourceId === "sensor.camera.left", + ); + const reconnectingPointCloud = reconnecting.find( + ({ modality }) => modality === "point-cloud", + ); + const recoveredPointCloud = recovered.find( + ({ modality }) => modality === "point-cloud", + ); + assert.ok( + reconnectingCamera + && recoveredCamera + && reconnectingPointCloud + && recoveredPointCloud, + ); + + assert.equal(recoveredCamera.availability, "connecting"); + assert.equal(recoveredCamera.activation.selected, true); + assert.equal(recoveredCamera.activation.controllable, true); + assert.deepEqual(recoveredCamera.delivery, reconnectingCamera.delivery); + assert.deepEqual(recoveredCamera.presentationLease, reconnectingCamera.presentationLease); + assert.deepEqual( + recoveredPointCloud.presentationLease, + reconnectingPointCloud.presentationLease, + ); + assert.equal( + liveCameraPlaybackAuthorityIdentity(recoveredCamera), + liveCameraPlaybackAuthorityIdentity(reconnectingCamera), + "reconnecting→recovered must not retire the decoder before first playable frame", + ); + const spatialSource = { + id: "acquisition-001", + url: reconnectingPointCloud.previewUrl, + label: "Live", + kind: "rerun-grpc", + }; + assert.equal( + liveRerunRecoveryAuthorityIdentity(recoveredPointCloud, spatialSource), + liveRerunRecoveryAuthorityIdentity(reconnectingPointCloud, spatialSource), + "the first recovered PCL keeps one exact Rerun browser authority", + ); +}); + +test("camera recovery fails closed while the exact spatial lease may continue", () => { + const cameraStreamError = cameraRecoveringState("sensor.camera.left"); + cameraStreamError.sensor_catalog.streams = cameraStreamError.sensor_catalog.streams.map( + (stream) => stream.source_id === "sensor.camera.left" + ? { ...stream, availability: "error" } + : stream, + ); + const mismatchedDelivery = cameraRecoveringState("sensor.camera.left"); + mismatchedDelivery.camera_preview.delivery = { + ...mismatchedDelivery.camera_preview.delivery, + id: "preview-generation-replaced:sensor.camera.left", + }; + const missingDelivery = cameraRecoveringState("sensor.camera.left"); + missingDelivery.camera_preview.delivery = null; + missingDelivery.sensor_catalog.streams = missingDelivery.sensor_catalog.streams.map( + (stream) => stream.source_id === "sensor.camera.left" + ? { ...stream, delivery: null } + : stream, + ); + const cases = [ + cameraRecoveringState("sensor.camera.left", { camera_recovery: "blocked" }), + cameraRecoveringState("sensor.camera.left", {}, { + device_session: { + ...cameraStreamingState("sensor.camera.left").device_session, + device_session_id: "device-session-replaced", + }, + }), + cameraRecoveringState("sensor.camera.left", {}, { + camera_preview: { + ...cameraStreamingState("sensor.camera.left").camera_preview, + phase: "error", + }, + }), + cameraStreamError, + mismatchedDelivery, + missingDelivery, + ]; + + for (const state of cases) { + const sources = xgridsK1ObservationSources(state, model()); + const pointCloud = sources.find(({ modality }) => modality === "point-cloud"); + const camera = sources.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(pointCloud && camera); + assert.equal(pointCloud.previewUrl, "rerun+http://127.0.0.1:9877/proxy"); + assert.equal(pointCloud.availability, "connecting"); + assert.equal(camera.activation.selected, false); + assert.equal(camera.activation.controllable, false); + assert.equal(camera.delivery, null); + assert.equal(camera.presentationLease, null); + assert.equal(liveCameraPlaybackAuthorityIdentity(camera), null); + } +}); + +test("stale and terminal recovery states withdraw both retained transports", () => { + const staleProducer = cameraRecoveringState("sensor.camera.left"); + staleProducer.producer_generation += 1; + const differentAcquisition = cameraRecoveringState("sensor.camera.left", { + acquisition_id: "acquisition-stale", + }); + for (const state of [ + staleProducer, + differentAcquisition, + cameraRecoveringState("sensor.camera.left", { state: "blocked" }), + cameraRecoveringState("sensor.camera.left", { state: "standby" }), + cameraRecoveringState("sensor.camera.left", { state: "fault" }), + ]) { + const sources = xgridsK1ObservationSources(state, model()); + const pointCloud = sources.find(({ modality }) => modality === "point-cloud"); + const camera = sources.find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(pointCloud && camera); + assert.equal(pointCloud.previewUrl, null); + assert.equal(pointCloud.presentationLease, null); + assert.equal(camera.activation.selected, false); + assert.equal(camera.delivery, null); + assert.equal(camera.presentationLease, null); + } +}); + +test("malformed recovery leases cannot own a camera WebSocket or decoder", () => { + const camera = xgridsK1ObservationSources( + cameraRecoveringState("sensor.camera.left"), + model(), + ).find(({ sourceId }) => sourceId === "sensor.camera.left"); + assert.ok(camera?.presentationLease); + + for (const presentationLease of [ + null, + { ...camera.presentationLease, runtimeId: " " }, + { ...camera.presentationLease, acquisitionId: "acquisition-stale" }, + { ...camera.presentationLease, acquisitionStateRevision: 0 }, + { ...camera.presentationLease, producerGeneration: 0 }, + { ...camera.presentationLease, recoveryGeneration: 0 }, + ]) { + assert.equal( + liveCameraPlaybackAuthorityIdentity({ ...camera, presentationLease }), + null, + ); + } +}); + test("layout policy evicts only exclusive camera peers", () => { const sources = xgridsK1ObservationSources( cameraStreamingState("sensor.camera.left"), diff --git a/apps/control-station/test/rerunViewportAtomicAdmission.test.mjs b/apps/control-station/test/rerunViewportAtomicAdmission.test.mjs index 807625e..bb7e5e1 100644 --- a/apps/control-station/test/rerunViewportAtomicAdmission.test.mjs +++ b/apps/control-station/test/rerunViewportAtomicAdmission.test.mjs @@ -5,7 +5,9 @@ import { after, before, test } from "node:test"; import { createServer } from "vite"; let server; +let claimExclusiveLiveViewer; let createRecordedOpenWatchdog; +let createReentrantViewerDisposer; let recordedOpenWatchdogTimeoutMs; let rerunViewerInitialSource; let resolveRecordedViewerSourceUrl; @@ -17,7 +19,9 @@ before(async () => { server: { middlewareMode: true }, }); ({ + claimExclusiveLiveViewer, createRecordedOpenWatchdog, + createReentrantViewerDisposer, recordedOpenWatchdogTimeoutMs, rerunViewerInitialSource, resolveRecordedViewerSourceUrl, @@ -118,6 +122,45 @@ test("complete recorded admission clears its watchdog", () => { assert.deepEqual(cancelled, [23]); }); +test("deferred viewer start cannot reopen after stale unmount", async () => { + let resolveStart; + const start = new Promise((resolve) => { + resolveStart = resolve; + }); + let disposed = false; + let cleanupCount = 0; + let closeCount = 0; + let stopCount = 0; + const diagnostics = []; + const disposeViewer = createReentrantViewerDisposer( + () => { + cleanupCount += 1; + }, + () => { + closeCount += 1; + stopCount += 1; + }, + ); + const pendingMount = (async () => { + await start; + if (disposed) { + disposeViewer(); + return; + } + diagnostics.push("admitted"); + })(); + + disposed = true; + disposeViewer(); + resolveStart(); + await pendingMount; + + assert.equal(cleanupCount, 1); + assert.equal(closeCount, 2); + assert.equal(stopCount, 2); + assert.deepEqual(diagnostics, []); +}); + test("recorded RRD bytes are never split across LogChannel.send_rrd calls", async () => { const source = await readFile( new URL("../src/components/RerunViewport.tsx", import.meta.url), @@ -129,7 +172,7 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn assert.match(source, /viewer\.start\(\s*rerunViewerInitialSource\(resolvedSource\)/s); assert.match( source, - /recordingOpened = true;[\s\S]*clearLiveRecordingOpenTimer\(\);[\s\S]*clearLiveRecordingDiscoveryTimer\(\);/, + /recordingOpened = true;[\s\S]*diagnosticLifecycle\.markAdmitted\(\);/, ); assert.match( source, @@ -145,12 +188,31 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn ); }); +test("one live document owns one native Rerun receiver", async () => { + const releases = []; + const releaseFirstClaim = claimExclusiveLiveViewer(() => releases.push("first")); + const releaseSecondClaim = claimExclusiveLiveViewer(() => releases.push("second")); + + assert.deepEqual(releases, ["first"]); + releaseFirstClaim(); + assert.deepEqual(releases, ["first"]); + releaseSecondClaim(); + + const releaseThirdClaim = claimExclusiveLiveViewer(() => releases.push("third")); + assert.deepEqual(releases, ["first"]); + releaseThirdClaim(); +}); + test("raw replay exercises the same streaming receiver lifecycle as a live scan", async () => { const source = await readFile( new URL("../src/workspaces/Workspaces.tsx", import.meta.url), "utf8", ); assert.match(source, /followLive=\{!recordedReplay && streamActive\}/); + assert.match( + source, + /sourceUrl\.trim\(\) && pointCloudVisible && !intentionalSourceEnd/, + ); }); test("the complete vendor canvas host is hidden during partial and failed admission", async () => { diff --git a/apps/control-station/test/runtimeStateOrdering.test.mjs b/apps/control-station/test/runtimeStateOrdering.test.mjs index 29d728a..d65871a 100644 --- a/apps/control-station/test/runtimeStateOrdering.test.mjs +++ b/apps/control-station/test/runtimeStateOrdering.test.mjs @@ -42,6 +42,113 @@ function snapshot(revision, generation, phase = "streaming", sessionId = "device }; } +function stampedSnapshot({ + runtimeStartedAt = "2026-08-06T10:00:00Z", + runtimeStartedMonotonicNs = "1000000", + runtimeId = "runtime-a", + snapshotRevision = 1, + cameraRevision = snapshotRevision, + generation = 1, +} = {}) { + return { + ...snapshot(cameraRevision, generation), + snapshot_runtime_started_at_utc: runtimeStartedAt, + snapshot_runtime_started_monotonic_ns: runtimeStartedMonotonicNs, + snapshot_runtime_id: runtimeId, + snapshot_revision: snapshotRevision, + }; +} + +test("uses the process snapshot revision before camera-local counters", () => { + const current = stampedSnapshot({ snapshotRevision: 8, cameraRevision: 2 }); + const stale = stampedSnapshot({ snapshotRevision: 7, cameraRevision: 99 }); + const newer = stampedSnapshot({ snapshotRevision: 9, cameraRevision: 1 }); + + assert.equal(selectMonotonicXgridsState(current, stale), current); + assert.equal(selectMonotonicXgridsState(current, newer), newer); +}); + +test("accepts a newer runtime and rejects a delayed snapshot from the old runtime", () => { + const oldRuntime = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T10:00:00Z", + runtimeStartedMonotonicNs: "1000000", + runtimeId: "runtime-old", + snapshotRevision: 300, + }); + const newRuntime = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T10:05:00Z", + runtimeStartedMonotonicNs: "2000000", + runtimeId: "runtime-new", + snapshotRevision: 1, + }); + const delayedOldRuntime = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T10:00:00Z", + runtimeStartedMonotonicNs: "1000000", + runtimeId: "runtime-old", + snapshotRevision: 301, + }); + + assert.equal(selectMonotonicXgridsState(oldRuntime, newRuntime), newRuntime); + assert.equal( + selectMonotonicXgridsState(newRuntime, delayedOldRuntime), + newRuntime, + ); +}); + +test("orders restarts by monotonic time even when UTC moves backwards", () => { + const oldRuntime = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T10:05:00Z", + runtimeStartedMonotonicNs: "2000000", + runtimeId: "runtime-old", + snapshotRevision: 900, + }); + const newRuntime = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T09:55:00Z", + runtimeStartedMonotonicNs: "3000000", + runtimeId: "runtime-new", + snapshotRevision: 1, + }); + + assert.equal(selectMonotonicXgridsState(oldRuntime, newRuntime), newRuntime); + assert.equal(selectMonotonicXgridsState(newRuntime, oldRuntime), newRuntime); +}); + +test("orders two runtimes sharing the same UTC millisecond", () => { + const first = stampedSnapshot({ + runtimeStartedMonotonicNs: "4000000", + runtimeId: "runtime-first", + }); + const second = stampedSnapshot({ + runtimeStartedMonotonicNs: "4000001", + runtimeId: "runtime-second", + }); + + assert.equal(selectMonotonicXgridsState(first, second), second); +}); + +test("a malformed monotonic stamp cannot replace valid runtime authority", () => { + const current = stampedSnapshot({ + runtimeStartedMonotonicNs: "5000000", + runtimeId: "runtime-current", + }); + const malformed = stampedSnapshot({ + runtimeStartedAt: "2026-08-06T11:00:00Z", + runtimeStartedMonotonicNs: "not-a-number", + runtimeId: "runtime-malformed", + snapshotRevision: 9999, + }); + + assert.equal(selectMonotonicXgridsState(current, malformed), current); +}); + +test("does not let an unstamped legacy response replace stamped authority", () => { + const current = stampedSnapshot({ snapshotRevision: 8 }); + const legacy = snapshot(99, 99); + + assert.equal(selectMonotonicXgridsState(current, legacy), current); + assert.equal(selectMonotonicXgridsState(legacy, current), current); +}); + test("accepts the first camera preview snapshot", () => { const incoming = snapshot(1, 1); assert.equal(selectMonotonicXgridsState(null, incoming), incoming); diff --git a/apps/control-station/test/workspaceLayout.test.mjs b/apps/control-station/test/workspaceLayout.test.mjs index b67ae86..1341afa 100644 --- a/apps/control-station/test/workspaceLayout.test.mjs +++ b/apps/control-station/test/workspaceLayout.test.mjs @@ -13,6 +13,9 @@ let projectObservationLayoutSnapshot; let saveObservationWorkspaceLayoutProfile; let WorkspaceLayoutApiError; let WorkspaceLayoutContractError; +let admitLiveDefaultPresentations; +let automaticLivePresentationIdentity; +let livePresentationCloseFence; let observationPresentationSourceAfterLayoutApply; let visibleSourceIdsAfterRecordedCatalogActivation; @@ -33,6 +36,9 @@ before(async () => { WorkspaceLayoutContractError, } = await server.ssrLoadModule("/src/core/observation/workspaceLayout.ts")); ({ + admitLiveDefaultPresentations, + automaticLivePresentationIdentity, + livePresentationCloseFence, observationPresentationSourceAfterLayoutApply, visibleSourceIdsAfterRecordedCatalogActivation, } = await server.ssrLoadModule("/src/core/observation/useObservationLayout.ts")); @@ -213,6 +219,95 @@ test("opening a recorded catalog reveals its sealed cameras beside the point clo ); }); +test("a sequential live acquisition re-arms the same camera without reopening a deliberate close", () => { + const pointCloud = { + id: "k1:sensor.lidar.primary", + sourceId: "sensor.lidar.primary", + modality: "point-cloud", + availability: "streaming", + transport: "rerun-grpc", + previewUrl: "grpc://127.0.0.1:9876/proxy", + delivery: null, + activation: null, + binding: { + deviceId: "k1-a", + deviceSessionId: "device-session-reused", + acquisitionId: "acquisition-a", + }, + capabilities: { defaultVisible: true, overlay: false }, + }; + const camera = (acquisitionId, deliveryId) => ({ + id: "k1:sensor.camera.right", + sourceId: "sensor.camera.right", + modality: "video", + availability: "streaming", + transport: "websocket", + previewUrl: null, + delivery: { + id: deliveryId, + kind: "mse-fmp4-websocket", + url: "/camera-preview/reused", + mediaType: 'video/mp4; codecs="avc1.641028"', + }, + activation: { + groupId: "k1:device-session-reused:camera.preview.decoder", + maxActive: 1, + selected: true, + controllable: true, + }, + binding: { + deviceId: "k1-a", + deviceSessionId: "device-session-reused", + acquisitionId, + }, + capabilities: { defaultVisible: true, overlay: true }, + }); + + const firstCamera = camera("acquisition-a", "camera-preview-2"); + const first = admitLiveDefaultPresentations( + [pointCloud.id], + [pointCloud, firstCamera], + new Set(), + new Set(), + ); + assert.deepEqual(first.visibleIds, [pointCloud.id, firstCamera.id]); + assert.deepEqual(first.admittedIdentities, [ + automaticLivePresentationIdentity(firstCamera), + ]); + + const sameAcquisitionNewDelivery = camera("acquisition-a", "camera-preview-3"); + const closedInFirstAcquisition = new Set([ + livePresentationCloseFence(firstCamera), + ]); + const afterDeliberateClose = admitLiveDefaultPresentations( + [pointCloud.id], + [pointCloud, sameAcquisitionNewDelivery], + new Set(first.admittedIdentities), + closedInFirstAcquisition, + ); + assert.deepEqual(afterDeliberateClose.visibleIds, [pointCloud.id]); + assert.deepEqual(afterDeliberateClose.admittedIdentities, []); + + const nextAcquisitionSameDelivery = camera("acquisition-b", "camera-preview-3"); + assert.notEqual( + automaticLivePresentationIdentity(nextAcquisitionSameDelivery), + automaticLivePresentationIdentity(sameAcquisitionNewDelivery), + ); + const second = admitLiveDefaultPresentations( + [pointCloud.id], + [ + { ...pointCloud, binding: { ...pointCloud.binding, acquisitionId: "acquisition-b" } }, + nextAcquisitionSameDelivery, + ], + new Set(first.admittedIdentities), + closedInFirstAcquisition, + ); + assert.deepEqual(second.visibleIds, [pointCloud.id, nextAcquisitionSameDelivery.id]); + assert.deepEqual(second.admittedIdentities, [ + automaticLivePresentationIdentity(nextAcquisitionSameDelivery), + ]); +}); + test("workspace layout API uses the canonical endpoint and optimistic revision", async () => { const calls = []; const current = decodeObservationWorkspaceLayoutProfile(wireProfile()); diff --git a/docs/04_K1_WIFI_PROVISIONING_PROFILE.md b/docs/04_K1_WIFI_PROVISIONING_PROFILE.md index 1eff7bb..fa95b66 100644 --- a/docs/04_K1_WIFI_PROVISIONING_PROFILE.md +++ b/docs/04_K1_WIFI_PROVISIONING_PROFILE.md @@ -111,6 +111,27 @@ handle from the operator's scan and connects that exact selected handle in the following network action. A fallback lookup remains only for non-UI callers that did not perform discovery first. +The public advertisement cache is deliberately separate from an admitted +device session. Rows from the latest explicit scan generation remain stable +without wall-clock expiry while the operator completes the form. They still +grant no mutation authority without exact retained-handle capture and live +GATT validation. An admitted selected session ends only on proven disconnect, +explicit stop, app/backend restart, selection of another K1, or connection-mode +switch. A later scan may replace unselected candidates but never auto-connects +any of them. + +An explicit Quick Connect to Bridge request for that same device can therefore +continue when K1 no longer advertises after AP activation. The preconditions +are: no active acquisition, no pending evidence cleanup, no active local source, +and a terminal/released control session. A mode switch closes the old selected +session first; the operator then scans, selects the K1, and creates a clean new +GATT session for Bridge. Before the station command the code performs the +normal internal `7f02` baseline read and allows exactly one 99-byte `7f01` +write. A powered-off or unreachable peripheral ends that attempt. An +unobserved post-write result is recorded as terminal `outcome-unknown`; it is +never retried automatically and never blocks a later distinct explicit +scan-select-connect attempt. + The 2026-07-20 prepared-host acceptance installed the exact firmware provider, found one expected K1 candidate, emitted one AP-enable write, observed AP-ready and completed one CoreWLAN association without an iPhone or manual credential. @@ -136,12 +157,19 @@ mode. It never fragments or retries the payload automatically. A completed GATT write only proves transport completion. It does not prove that the K1 joined Wi-Fi or began beaconing. The application polls `7f02`; the -observed response frame contains a fixed-width mode slot, an address slot, a -status byte at offset 50 and the AP-ready flag at offset 51. The stale AP +observed response frame contains a fixed-width text slot, an address slot, a +status byte at offset 50 and the AP-ready flag at offset 51. The text slot is +not a uniform mode enum: AP state uses the `WIFI_AP` control literal, while the +2026-08-08 FW 3.0.2 Bridge observation returned the joined network name. The stale AP baseline reports `WIFI_AP / 192.168.56.1 / byte51=0`; the physically observed ready transition reports the same mode/address with `byte51=1`. -For Bridge/Direct Connect, acceptance requires at least one of: +For Bridge/Direct, acceptance requires the post-write `7f02` text slot to match +the exact requested network name and the address slot to contain a valid +non-AP private IPv4. This proves the desired target even when the K1 was +already joined to the same network before the explicit idempotent command. A +legacy literal-only `WIFI_CLIENT` observation retains the older conservative +cross-family rules and requires at least one of: 1. `7f02` reports a non-AP IPv4 address; 2. the same address appears as a new router/ARP client after the write; @@ -150,6 +178,15 @@ For Bridge/Direct Connect, acceptance requires at least one of: Do not infer success from a write callback alone. +An interrupted attempt with no exact post-write network-name observation is not +made successful by a write callback, changed DHCP address, router/ARP row or +reachable endpoint. Likewise, an already AP-ready baseline alone cannot prove +the outcome of an interrupted Quick-to-Quick attempt. Such an attempt remains +`outcome-unknown` in historical audit and is never replayed automatically. It +does not create a permanent mutation barrier: after the old active operation +and cleanup have terminated, a later explicit operator scan, selection, and +connect is a distinct session with its own single reviewed write. + The Bridge/Direct Connect address is a DHCP lease, not configuration and not device identity. Mission Core re-reads `7f02` without writing before every new LAN control session, implicit-host acquisition and factory-calibration read. @@ -157,6 +194,10 @@ If the value changes, it rotates `device_session_id`; it never retargets an active acquisition. A correlated MQTT `DeviceInfo` response supplies the live model/firmware/serial identity barrier. +The joined network name is used only for exact in-process comparison with the +current explicit request. Durable network audit stores the normalized semantic +family and never stores or publishes the raw network name. + The 2026-07-20 reboot/power-cycle check observed the startup race directly: one read returned the earlier `.54` lease while that exact address had no ARP or application endpoint; a later read returned `.52`, where exact probes found @@ -166,7 +207,9 @@ and why a BLE lease observation alone is not reported as live DeviceInfo. For Quick Connect, host association is not admitted until the canonical byte-51 ready flag is observed. CoreWLAN then searches only for the exact -device-profile SSID for at most 15 seconds and performs at most one association. +device-profile SSID for at most 30 seconds and performs at most one association. +AP-ready is a device-state barrier, not proof that the host has already observed +the RF beacon; a retained successful run required 18.142 seconds of discovery. ## Safety, recovery and stop conditions @@ -176,9 +219,17 @@ device-profile SSID for at most 15 seconds and performs at most one association. secure store. Missing or mismatched firmware material fails before the AP write. Never extrapolate this provider to another firmware or model. - The macOS adapter materializes a device-scoped Keychain item from the exact - firmware source, then performs one association. Standard Wi-Fi Keychain and - native prompt paths remain compatibility fallbacks, not the reviewed - zero-touch path. It never asks the browser for a password. + firmware source before the BLE write, then performs one association using + only that exact profile. Standard Wi-Fi Keychain lookup, native password + prompts and post-write profile rewrites are prohibited. It never asks the + browser for a password. Preflight reads are non-interactive and validate the + exact SSID/source inside the helper before K1 changes network state. +- The prepared-host laboratory adapter launches the reviewed Swift source only + through `/usr/bin/xcrun swift`. Runtime `swiftc` compilation to an ad-hoc + executable is prohibited because its unstable process identity regressed + Keychain ACL and CoreWLAN behavior. Product packaging still requires a + prebuilt, properly signed helper with a stable designated identity and + explicit CoreWLAN authorization. - Do not alter Deco settings, scan the subnet, or guess any credential. - If the status does not change, do not retry automatically. - If the supplied credentials are wrong, reconnect over BLE and overwrite them diff --git a/docs/06_K1_LIVE_VIEWER.md b/docs/06_K1_LIVE_VIEWER.md index d20d73d..b20002a 100644 --- a/docs/06_K1_LIVE_VIEWER.md +++ b/docs/06_K1_LIVE_VIEWER.md @@ -110,12 +110,11 @@ directly and use their sibling metadata receive timestamps when present. ## Connect and stream live 1. Power K1 to its normal steady-green standby state. -2. Confirm the manual power checklist in **Парк → Локальное устройство**. -3. Run the real six-second BLE scan and select the intended device from the +2. Run the real six-second BLE scan and select the intended device from the complete visible-device list. -4. Enter the existing router SSID/password and explicitly authorize the reviewed +3. Enter the existing router SSID/password and explicitly authorize the reviewed provisioning write. The backend does not retry the write automatically. -5. Enter the required project name, confirm operator presence, closed LixelGO, +4. Enter the required project name, confirm operator presence, closed LixelGO, storage/power and steady green, then choose **Запустить сканирование и локальный приём** once. 6. Mission Core emits operations 1–6, waits for their correlated device diff --git a/docs/20_K1_CONNECTION_SUPERVISION_CANON.md b/docs/20_K1_CONNECTION_SUPERVISION_CANON.md new file mode 100644 index 0000000..3b08e9e --- /dev/null +++ b/docs/20_K1_CONNECTION_SUPERVISION_CANON.md @@ -0,0 +1,1141 @@ +# K1 connection supervision canon + +Status: canonical target and product acceptance contract, updated 2026-08-11; +implementation/hardware conformance remains tracked by the acceptance manifest. + +Operator recovery procedure: [`runbooks/K1_CONNECTION_RECOVERY.md`](runbooks/K1_CONNECTION_RECOVERY.md). +Production macOS association-observer boundary: +[`adr/0014-k1-macos-association-observer.md`](adr/0014-k1-macos-association-observer.md). +Physical-state loss and recovery boundary: +[`adr/0015-k1-physical-state-recovery.md`](adr/0015-k1-physical-state-recovery.md). + +This document defines the connection state model for the local Mission Core +runtime. It replaces the earlier assumption that one remembered K1 IPv4 +address, one retained CoreBluetooth object, one open TCP port, or one incoming +point packet means that the device is connected. + +The product goal is a simple operator-owned lifecycle: every app start, +disconnect, explicit stop, or committed network-mode transition ends the old +connection session. A normal connection has one discovery intent and one +application intent: the operator explicitly searches, selects one fresh result, +enters the network data immediately, and presses **Применить** once. Opening the +UI, selecting a result and editing credentials are presentation inputs only. +Changing mode is one explicit local scenario-reset operation; it performs no +device or host I/O and starts no discovery. Historical attempts are audit +evidence only. They never +change a row label or make an available K1 look connected. Durable physical +START/STOP ambiguity remains a separate safety barrier: it can deny Apply and +START until an explicit recovery action reconciles the exact target, but it may +not turn mode, selection or input into hidden recovery work. + +## Product boundary + +### Product-surface brief + +The operator connects the selected K1 inside the existing device plugin section. +The surrounding job/entity/lifecycle model is unchanged. This interaction is +novelty A: a focused improvement to the existing connection surface, not a new +product area. The chosen surface is the existing plugin section; a separate +wizard, modal workflow or mandatory preflight/recovery screen is rejected. + +The surface reuses the canonical shared `Button`, `TextField`, +`ActivityIndicator` and `StatusBadge` components. It introduces no shared +entity, job type or lifecycle state, and it does not implement raw local +`button`/`input` controls or literal local status colors. + +- Bridge/shared-LAN is the product path. +- Quick Connect/device-AP is a prepared-host laboratory and recovery path. +- Direct Connect/controller-hotspot remains a distinct topology and is never + silently treated as Bridge merely because both can expose a private IPv4. +- The browser does not own Bluetooth, Wi-Fi, MQTT, or device state. It renders + the backend snapshot and invokes explicit operator actions. +- Operator errors are secret-free. The UI maps an allowlisted public error code + to canonical copy; an unknown exception/message uses a canonical fallback. + Raw backend messages, credentials, SSIDs, payloads and stack traces are never + rendered. +- No ambient background monitor or passive poll may originate Verify, Scan, + selection, reconnect, BLE writes, Wi-Fi provisioning, START, STOP, or repeat + an operator mutation. Backend events and bounded state polling may passively + refresh already available state; they create no operator progress state and + no new device operation. The sole declared exception is a supervised + read-only control bootstrap already owned by the exact active Apply intent; + after the durable ACK it may finish evidence collection but cannot mutate, + retry, create a UI action or outlive that intent's authority. +- Discovery starts only from one explicit operator action, runs for six seconds + per click, and never auto-connects or auto-selects a device. Every ordinary + connectable result has the same one enabled **Выбрать** action, including an + exact UUID seen in an earlier connection scenario. Selection is a local draft + transition: it immediately reveals the selected card and applicable network + inputs and performs no Scan, GATT connect, Verify, retirement/reopen or other + backend I/O. Fresh discovery results never expose **Переподключиться** or + route through physical-recovery UI. Reconnect belongs only to a previously + established session after a proven interruption. +- One explicit **Применить** click owns the resulting connection intent. It uses + only the selected transport captured by the completed scan, performs no + hidden discovery or read-only preflight, and crosses at most one reviewed + device-mutation boundary. An ended or ambiguous intent is never replayed. +- Normal Bridge Apply never opts into changing the controlling Mac's Wi-Fi + association. Host-network switching is a separate future consequential + operator action, not a hidden part of Apply. Successful K1 provisioning may + therefore end as `network_applied` with `control_not_ready`. +- The exact REST response to Apply is authoritative for that mutation boundary. + When `connection_attempt.phase=network_applied`, the controller accepts the + network intent as completed even when `control_state` is + `control_not_ready` or `unknown` and returns that durable ACK immediately; it + does not wait for WebSocket/poll convergence or full connection-ready. The + service may continue the same intent's supervised control bootstrap after the + ACK, but that continuation is read-only: no BLE/host mutation, mutation retry, + new UI action or second Apply. It is not a hidden frontend Scan or Verify. + Full exact readiness remains a separate gate for control and physical START. + The old Apply and its credentials are spent, but `connection_attempt` is a + read model rather than permanent lifecycle authority: a later new intent may + be admitted only by current server policy and a new explicit operator path. +- App/backend restart, proven BLE disconnect, explicit stop, and a committed + Bridge/Quick network transition close that active session. Passive BLE Scan + is discovery only; it does not by itself provision, switch topology or + resolve a physical command. +- Loss of Mission Core control does not prove that K1 stopped recording. It + revokes local authority and makes the physical device state unknown. +- Physical motion authority is always false in this layer. A mobile platform + requires its own local watchdog and fail-safe stop independent of Mission + Core, the Mac, the router, and the browser. + +### One-intent connection surface + +The existing plugin section uses one progressive two-part surface. Its only +model-bearing heading is **Подключение XGRIDS LixelKity K1**. The parts are +**Подключение** and **Сеть**; saved/original/previous device +terminology, physical-ledger state and protocol recovery names are not operator +ceremonies. + +1. A cold plugin section shows the connection-mode selector and Step 01 + **Подключение** with its explicit Bluetooth search action. Historical K1 + identity never adds a reconnect prompt on cold entry. Opening or polling the + surface performs no action. + Each selector/escape gesture sends one explicit CAS-fenced + `reset_scenario`; it may seal only old local receiver/camera/control owners + and retire their audit lineage, but performs no BLE, device/host Wi-Fi, + MQTT publish, Verify, reconnect, provisioning, START, STOP or automatic + Scan. +2. Bluetooth discovery starts only from the explicit search action; + while it runs the same step shows an activity indicator and visible seconds + countdown. Every completed connectable result keeps one enabled **Выбрать** + action, including an exact prior UUID. No found row renders a reconnect CTA, + a disabled primary action or a historical-device decision. The completed + explicit search is authoritative for this new connection draft; a projected + historical recovery fact cannot hijack it and require a second search. +3. One **Выбрать** click updates browser-local selection only. It keeps the + selected row/card visible and immediately reveals Step 02 **Сеть** for Bridge + and Direct Connect. SSID/password editing is also local-only. Quick Connect + immediately shows its selected-device Apply summary without credential + fields. Selection starts no loader and calls no controller method. +4. One explicit **Применить** click creates the only device connection/mutation + intent. The desired mode already belongs to the acknowledged scenario-reset + revision; Apply may perform at most one reviewed K1 mutation. Its frontend + handler never performs a hidden + Scan or Verify, + reconnect, retirement/reopen, candidate substitution or retry. The supervisor + and ledgers remain the authority before admission and after dispatch. + +The successful Apply response latches its post-mutation state immediately. An +exact REST snapshot whose `connection_attempt.phase` is `network_applied` ends +and spends the old Apply intent even if control is not ready or is unknown. +While its exact service-owned child is `accepted` or `running` with +`safe_next_action=wait-for-current-attempt`, ordinary controls and recovery +actions remain locked behind one passive **Сеть настроена · подтверждаем +управление** indicator. Once that child is terminal, an unready/unknown result +enters the applied-network recovery choice. Delayed WebSocket/poll convergence +cannot turn the accepted mutation into an error or replay the same intent. +Verify is then the recommended read-only choice, not a prerequisite imposed by +the read model. Any new connection path has a new idempotency identity and must +be explicitly admitted from current backend policy. + +The service-owned same-intent bootstrap may still publish a later ready/unready +snapshot after the REST ACK. It owns no blocking Apply loader and cannot call +the browser's Scan/Verify/Apply handlers; the frontend may render only its +passive exact-child settling indicator. Failure simply leaves the spent attempt +in the explicit recovery-choice presentation; it never replays the network +write. + +The ordinary **Выбрать** action does not weaken backend safety because it grants +no authority. An admitted fresh candidate exposes inputs immediately. After an +explicit scenario reset, only a successfully completed successor Scan may make +an exact previously retired transport eligible for this ordinary draft. Apply +still requires that exact current-generation CoreBluetooth handle, live GATT +baseline validation and the existing network-mutation admission; it performs +at most one write. Immediately before that sole write, the backend may append +one exact local `reset-network-intent-read-only-settlement` reopen checkpoint, +bound to the reset, successor Scan, immutable retirement and network request. +This preserves the retirement and original unknown outcome as append-only +audit, grants no START/STOP authority and is never exposed as a reconnect step. +After an applied network result, the same service-owned intent may classify the +reopened record from fresh DeviceInfo plus non-retained DeviceStatus: READY +settles standby, while identity-bound initialized SCANNING settles active without replaying +START and materializes only a STOP-only control shell. A reset-owned settlement +never restarts the retired receiver, camera, evidence writer or acquisition. +Exact operator-facing physical recovery keeps its backend CAS and +transport pinning, but appears only inside an established session after an +actual interruption, never in cold entry or Bluetooth results. + +Outside that reset-owned STOP-only settlement, fresh non-retained READY proof +resolves physical ambiguity as standby; fresh exact same-project SCANNING resolves it as active and exposes only the +separately guarded stop path outside the connection surface. Identity, GATT, +CAS, route/control or policy failure leaves physical outcome unknown. No +password, hidden START/STOP, provisioning or automatic retry bypasses it. + +`physical-command.retire-unavailable` remains a local durable backend primitive +for an unresolved target that is truly unavailable or replaced. It requires a +separate explicit recovery confirmation and exact runtime/operation/revision/ +transport CAS. It performs zero device I/O, starts no scan, preserves the +unknown historical outcome and activates the exact transport deny. Selecting a +row or pressing Apply never invokes it implicitly. + +FW 3.0.2 BLE `7f02` does not expose the stable DeviceInfo identity. Therefore a +physical unit that reappears under a new CoreBluetooth UUID cannot currently be +recognized as the retired identity before provisioning; this is an explicit +protocol/field-acceptance gap, not a closed pre-provision identity invariant. + +Cancel is an exact revision/intent CAS and is local-only: it sends no BLE, +Wi-Fi, MQTT, START or STOP command and does not resurrect an old socket. It is +available only while no scan, Verify, Connect or other lifecycle winner is in +flight. A stale browser tab cannot cancel or complete a newer dialogue. + +The browser may prefill a currently observable SSID when the operating system +provides it, but it must not claim that macOS can silently disclose the Wi-Fi +password. Bridge credentials are explicit operator input, retained only in the +current browser operation memory, and fenced by backend runtime, device, +connection mode, discovery generation and reconfiguration revision/intent. +Any draft-fence change clears the password and submit +authority; a backend runtime or reconfiguration-fence change also clears the +SSID. + +## One connection is a chain of independent evidence + +The word `connected` is reserved for the final, current binding. The following +facts must not be collapsed into one boolean. + +| Fact | What it proves | What it does **not** prove | +| --- | --- | --- | +| Operator `intent` | Which device and topology the current operation requests | Device presence, a completed mutation, host association, or authority | +| Fresh BLE advertisement | A transport UUID was observed by the current explicit scan | Selection, a connected GATT session, that K1 is on a network, or that MQTT works | +| Active selected BLE session | The operator selected one discovered transport and Mission Core established the current reviewed GATT connection | That K1 is on the requested network, that MQTT works, or that the session survives disconnect/restart/stop/mode switch | +| Live GATT baseline/status | The reviewed K1 GATT contract answered during this explicit operation | That a write is safe to repeat or that the Mac followed K1 to its network | +| Semantic device topology (`device_network`) | K1 reported the exact requested AP/STA result and, for LAN modes, a valid target address | That the Mac has a route, TCP is open, DeviceInfo matches, or data is fresh | +| Host association (`host_network`) | The Mac currently has a particular interface, source address, route fingerprint, and host epoch toward the target | That the target is K1 or that port 1883 is reachable | +| TCP endpoint reachability (`endpoint`) | Port 1883 answered for the exact target in the current host epoch | K1 identity, MQTT application dialogue, control authority, or live sensor data | +| DeviceInfo/control proof (`control`) | Current MQTT dialogue returned the expected K1 DeviceInfo for the exact intent, mode, target, and host epoch | Fresh point/camera data or that a physical recording stopped after control loss | +| Live data evidence (`data`) | Point, camera, or telemetry evidence arrived for a bounded acquisition session | Control authority, K1 identity by itself, or permission to start another action | +| Network-attempt audit | What the previous explicit attempt requested, whether its single write boundary was crossed, and how the attempt ended | Presence, a live session, permission for automatic retry, or a veto over a later explicit session | +| `last_known` | Historical troubleshooting context from a previously admitted connection | Presence, reachability, selection, lease, or command authority | + +The authoritative control binding is the exact tuple: + +```text +(intent_id, transport_ref, connection_mode, target_ipv4:port, + host_path_epoch, logical_device_id, compatibility_profile_id, + control_session_id) +``` + +Evidence for a different intent, transport, mode, target, device identity, or +host epoch cannot complete this tuple. "Same mode" is not "same connection"; +Bridge to a different K1 is a new intent and must earn new evidence. + +## Infrastructure failure domains + +The implementation treats every boundary below as independently fallible. A +failure in one row must revoke only the authority that depended on it and must +not fabricate the state of another row. + +| Boundary that can fail | Typical real scenario | Required response | +| --- | --- | --- | +| Browser ↔ local API | app refresh/restart, closed tab, suspended renderer, lost HTTP response | App restart/close ends its operator connection session without sending a device command; a single lost response never repeats an action. The next app session starts with discovery and selection. | +| Mission Core process | crash, upgrade, second process, stale worker thread | Close the active connection session on restart; retain terminal audit only; stable process locks prevent a second controller; old runtime generations cannot publish evidence into a new session. | +| CoreBluetooth discovery/GATT | K1 powers off, Mac sleeps, disconnect callback arrives, scan callback arrives late | Close and clear the selected session on proven disconnect. A later operator scan and selection establish a new GATT connection. Unselected rows remain stable for the latest admitted scan generation and are invalidated only by an explicit successor/reset, runtime-owner teardown or proven exact-target GATT failure. A row never grants mutation authority without exact-handle capture and live GATT validation. | +| Mac Wi-Fi association | Quick AP is left manually, Bridge router changes, same IP is reused by another AP | Rotate host epoch using an opaque OS association identity in addition to route/interface/source IP. Rebuild TCP and protocol proof. | +| IP route ↔ TCP probe | route changes while port 1883 connect is in flight | Admit reachability only for one correlated `route A → TCP → route B` sample where A and B are identical. | +| MQTT control | socket half-open, broker disappears, wrong service answers 1883, late DeviceInfo | Revoke control lease. Require exact current intent, BLE transport, target, host epoch, durable device pin and DeviceInfo before commands. | +| MQTT/data producers | old capture thread drains late, points continue after control loss, new acquisition starts | Stamp every message with an immutable producer generation; stale generations remain non-authoritative and cannot revive the new session. | +| Router/phone hotspot | power loss, LTE/Wi-Fi reset, DHCP reassignment | Preserve only the semantically proven K1 topology. Host/TCP/control/data become offline independently; no provisioning retry follows. | +| Physical K1 | battery removal before/during/after a BLE network write or START/STOP | End the network attempt as interrupted/unknown and allow a later explicit clean network session without replay. For physical START/STOP, preserve the separate durable ambiguity barrier and never infer STOP. | + +This is also the design boundary for a future vehicle. Mission Core may report +loss quickly, but a drive controller must independently enter a locally safe +state when its sensor/control heartbeat expires. A laptop, router, MQTT broker +or UI can never be the only motion watchdog. + +## Product state and authority matrix + +These rows are ordered by proof, not by UI optimism. `Data` is intentionally +shown independently from `Control`. + +| Product state | Device topology | Host path | TCP | DeviceInfo/control | Data | Authority and required presentation | +| --- | --- | --- | --- | --- | --- | --- | +| Not selected | none | any/unknown | unknown | absent | idle | No device or network authority. Offer device discovery/selection. | +| Intent pending | previous only as `last_known` | unknown/current | unknown | revoked | previous stream non-authoritative at most | Show requested topology separately; do not call it connected. | +| Active connection attempt | previous topology is historical only | unknown/current | unknown | revoked | previous stream non-authoritative at most | One selected K1, one mode, one active GATT session and at most one network write for this explicit intent. | +| Attempt ended/interrupted | last observation is audit only | unknown/stale | unknown | revoked | evidence-only at most | Clear the active selection/session. Offer the normal scan-select-connect flow; no dedicated recovery action and no automatic retry. | +| Configured offline | exact intended topology semantically proven | unavailable, wrong, or stale | unknown/unreachable | absent/lost | idle or evidence-only | Preserve the K1 topology fact. Show offline/degraded transport and a read-only host/endpoint verification action. | +| Reachable unverified | exact current topology | fresh exact host epoch | reachable | absent, stale, or mismatch | independent | `degraded`, never `connected`. TCP alone grants no command or acquisition-start authority. | +| Connected/controlled | exact current topology | fresh exact host epoch | reachable and fresh | exact current binding, fresh | idle/healthy/stalled/lost | Control allowed. Acquisition start allowed only when its separate lifecycle preconditions also pass. | +| Control lost, data alive | exact topology remains last proven | current or changing | failed/stale | lost | fresh evidence still arriving | Revoke commands and start authority immediately. Keep data visibly labelled evidence-only; do not send STOP automatically. | +| Standby unconfirmed | exact topology remains last proven | unavailable/stale | unknown/unreachable | STOP acknowledged and SCAN_STOPPING observed, but READY not observed | local receiver closed/evidence-only | Show `standby-unconfirmed`. After cleanup offer explicit bounded exact-target read-only recovery; deny provisioning and START. The always-available scenario reset may retire only the old local/audit lineage and start a clean Step 01, but does not prove standby or replay STOP. | +| Standby unknown after STOP deadline | exact topology remains last proven | unavailable/stale | unknown/unreachable | STOP accepted, but neither READY nor SCAN_STOPPING arrived before the backend operation deadline | local receiver/camera/ingress closed locally; evidence-only archive retained | Terminalize the operation as `timed_out` and present product state `standby-unknown`, with `side_effect_status=unknown`, `safe_to_retry=false`; keep the physical ledger unresolved. Allow passive BLE/read-only recovery after successful cleanup, but deny every mutation, START and STOP until exact reconciliation. | +| Recovery active rebind | exact topology and physical `transport_ref` match | fresh exact epoch | reachable | exactly re-proven DeviceInfo plus fresh same-project SCANNING | old data non-authoritative | Allow passive Scan and one single-use explicit STOP on the still-open recovery binding. An explicit scenario reset supersedes the read-only rebind, locally seals its receiver/camera/control owners and retires the old lineage without a device command; otherwise deny provisioning, START and automatic commands. | +| Control alive, data stalled | exact current topology | fresh exact host epoch | reachable | exact current binding | stalled/lost | Control may remain available; data is not healthy. Offer stop/diagnosis according to backend `allowed_actions`. | +| Closed supervisor | historical facts only | stale/unknown | stale/unknown | revoked | non-authoritative | No actions except reopening the product runtime through its normal lifecycle. | + +`connected` therefore requires all of the following at once: + +1. the current intent exists; +2. K1's exact semantic topology matches that intent; +3. the Mac host-path observation is fresh and belongs to the current epoch; +4. TCP reachability is fresh for the exact target in that epoch; +5. DeviceInfo and MQTT control proof match the exact binding; +6. the proof has not expired and no route/sleep/network transition has revoked + it. + +Live data is not a seventh condition for control connectivity, because data and +control may fail independently. It is a separate condition for declaring an +acquisition healthy or its evidence authoritative. + +## Non-negotiable invariants + +- Ordinary discovery and selection are explicit and never auto-connect. One + search click owns exactly one six-second discovery. The operator selects + exactly one candidate; **Выбрать** is local-only and merely binds the form to + that transport. It starts no backend chain and grants no mutation authority. +- Candidate presentation and active-session liveness are different. The latest + admitted scan generation remains visible without a wall-clock expiry while + the operator reads or completes the form. A successor Scan, explicit scenario + reset, runtime-owner teardown or proven exact-target GATT failure invalidates + that generation. A visible row is never mutation authority: Apply still + requires its exact captured CoreBluetooth object and live GATT validation. + Proven disconnect, explicit stop, app/backend restart, or a committed network + transition ends an admitted session. +- A disconnect callback closes the active session and clears its live + selection. Rediscovery never reconnects the network automatically. UI entry, + row selection and form input perform no I/O. Mode/new-device choice performs + only its explicit local scenario reset and no device/host I/O. The + operator explicitly searches, selects, enters data and presses **Применить**. +- Passive `scan-ble` means non-mutating with respect to K1; CoreBluetooth Scan + is still active discovery, not RF sniffing. It sends no K1 GATT/network or + physical command. After local + receiver/control cleanup has completed and the BLE runtime lease is free, an + unknown or proven-active physical record must **not** disable passive Scan. + It still blocks provisioning and START on the old lineage; scenario reset may + retire that lineage without resolving it. A discovered row is presence + evidence only and grants none of those actions. +- Physical-state recovery is pinned to the durable physical record's exact + `transport_ref`. A fresh candidate for another K1 may be displayed, but it + cannot replace the recovery target, change semantic topology, reconcile the + ledger or authorize any command. Selecting or probing the wrong K1 is a + fail-closed, no-topology-change outcome. +- `physical-command.retire-unavailable` is the only exception for an exact + physical target that the operator confirms is permanently unavailable or + replaced. It is a local ledger/authority transition, not physical + reconciliation and not device takeover: the original outcome remains + unknown, no device I/O or automatic Scan occurs, and the exact retired + transport receives an active deny for provisioning, Verify/adoption, + control, calibration, physical commands and network writes. The retirement + and unknown original outcome remain permanent historical audit even if that + active deny is later reopened. Explicit passive Scan/GATT metadata/MQTT + capture is non-authoritative evidence and cannot by itself restore topology + or control. Only a fresh exact same-UUID row plus the explicit exact-CAS + `physical-command.reopen-retired-reconciliation` transition may remove that + one active retirement deny for read-only reconciliation. Reopen performs no + device I/O. It is normally invoked only by a separate explicit recovery + action. The sole cold-flow exception is the exact local reopen checkpoint + inside a reset-owned, successor-Scan-owned Apply, durably ordered after the + network PREPARED record and before the one write-dispatch edge. Row selection + never invokes either path and no reconnect UI is rendered. A retired durable + identity, once DeviceInfo makes it observable, remains denied control and + successor START outside that exact reopened reconciliation. Because BLE + `7f02` has no stable device identity, a new UUID for the same hardware remains + a pre-provision detection gap. +- Bridge to Quick, Quick to Bridge, same-mode reconnect, and selecting another + K1 all close the old connection session before starting the new one. Old + host, endpoint, control, data and BLE ownership cannot cross that boundary. +- Every explicit network intent creates a new connect, validates the reviewed + GATT contract, and may read internal baseline `7f02` before exactly one + reviewed write. The baseline is internal protocol validation, not a mandatory + operator recovery step or separate product button. +- A fresh Bridge `7f02` observation is provisional when it comes from the + explicit Apply-owned network operation. + It may be projected in memory for route/TCP/DeviceInfo checks, but it does + not replace durable semantic topology until the exact MQTT DeviceInfo has + matched or created the immutable transport/profile identity pin. Identity + mismatch, cancellation, route/control failure or process death leaves the + previous durable device A unchanged, or leaves the store empty on first + contact, and retires only the provisional B session. A previously dispatched + unresolved network write keeps its separate durable-first reconciliation + semantics. +- FW 3.0.2 `7f02` uses its first text slot as `WIFI_AP` in AP state and as the + joined network name in station state. Bridge/Direct admission compares that + post-write name exactly with the current explicit request and a valid non-AP + private IPv4. The parser publishes only the normalized `WIFI_CLIENT` family; + raw network names remain inside the sensitive operation evidence and never + enter the public API or durable secret-free audit. +- A timeout, disconnect, cancellation, browser loss, or process death never + triggers an automatic network write retry. The attempt becomes terminal audit + (`not-dispatched`, `completed`, `failed`, `interrupted`, or `outcome-unknown`) + and releases its right to block a later distinct explicit session. +- Repeating an explicit operator action creates a new intent and may perform + its own one reviewed write after ordinary current-session preconditions pass. + The previous attempt is not replayed and its payload/credentials are not + reused automatically. +- Network-attempt audit is secret-free, atomically recorded, and useful for + diagnosis. It is not durable connection authority and cannot require a + read-only reconciliation UI before another explicit connect. +- Active concurrency remains fail-closed: a process-wide BLE owner and stable + cross-process lease admit at most one discovery/GATT/network operation at a + time. A new operator intent does not bypass an operation that is still + actively executing or native cleanup whose completion is unproved. Once that + active work terminates, historical audit cannot keep the lease fenced. +- Route or MQTT failure after a proven device topology does not make the BLE + write unknown. It produces configured-offline/degraded state with no control + authority. +- Bridge provisioning does not silently move the Mac to another Wi-Fi network. + The ordinary UI sends no host-switch opt-in. `network_applied` is a successful + terminal K1 mutation even when control is not ready; the UI must not repeat + that intent's BLE write. Recommended separately explicit read-only Verify may + establish host route, endpoint and control proof for the applied topology; a + server-policy-admitted new intent follows its own fresh path. Any future + host-switch action requires its own consequential confirmation and intent. +- `network_applied` plus `control_not_ready` or unknown outcome spends the old + Apply. An exact accepted/running service child first keeps ordinary controls + and recovery actions behind a passive settling indicator. After that child is + terminal, the UI presents an explicit recovery choice, independent of the + browser-local mode. It never authorizes an automatic or same-intent replay. + Recommended Verify accepts only the backend `serverBound` current/configured + transport and mode; browser selection, remembered rows and local mode can + never substitute its target. +- A new intent is nevertheless possible when current server policy admits an + explicit path. Bridge uses the distinct `prepare-select-device` local-only + CAS with zero device/host I/O; only after success may the operator start a + fresh six-second scan. Quick and Direct use an explicit policy-gated + `scan-ble`, then the latest fresh row and a new idempotency Apply. A mode or + same-mode new-device request instead uses the local `reset_scenario` CAS, + available across every non-reset owner, and is followed by a fresh explicit + scan. None of these + choices selects a row, scans, Verify, provisions + or continues Apply behind another click; the browser grants no authority. +- A pre-write failure records no network side effect and ends the new attempt. + Previous topology may remain historical context, but no old active session is + resurrected. +- Selecting any mode, including the current one through **Подключить другой + K1**, sends one explicit idempotent `reset_scenario` with exact desired-mode + revision. The first reset supersedes an older non-reset local action; while it + owns `mode`, the visible reset controls dispatch no second intent. It seals + retained receiver/camera/control owners, invalidates BLE candidates, + credentials and topology authority, and durably retires ambiguous old + physical lineage without resolving it. It performs no device command, BLE, + host-network write or automatic Scan. The resulting Step 01 starts clean and + search remains a separate click. Apply remains the only boundary that creates + a new network intent and may cross the reviewed device-write edge. +- Host route/interface epoch changes, manual Wi-Fi switches, sleep/wake, + endpoint loss, MQTT failure, or stale protocol evidence revoke their live + authority immediately. Re-admission is explicit and never writes + automatically. +- A host epoch includes an opaque, process-scoped association token supplied by + the OS helper. Raw SSID/BSSID values never enter API state or logs. Missing + association evidence remains explicitly `unavailable` and never becomes + association proof. The laboratory helper keeps one fallback token only for + the same process/interface/failure scope so an unchanged kernel route is not + falsely rotated every second. Its proven association token is derived from + interface+BSSID, not SSID visibility, so the same BSSID remains stable across + `ssid+bssid` and `bssid-only` observations. For an already exact healthy + DeviceInfo/control binding only, an unproven helper observation with the same + raw kernel fingerprint/interface/source retains the preceding proven host + fingerprint while the monitor still performs TCP and a final raw-route + recheck. This refreshes route/TCP liveness, not association or control proof. + Interface/source/route loss, a proven different BSSID, endpoint/control loss, + proof expiry, intent/target change, or process restart still revokes. An + unverified path remains bounded and fail-closed. Authority still requires + fresh TCP plus exact DeviceInfo/control evidence; the fallback token alone + grants nothing. +- A TCP success is admissible only when host route/association observations + taken immediately before and after the connect are identical. A success + attached to a different or unknown epoch is discarded. +- TCP port 1883 being open is diagnostic only. Correlated K1 protocol evidence + is required for device identity and control authority. +- `connection.endpoint-probe` is the only configured/offline host-only action. + It selects either the exact coherent current supervisor target or the latest + valid durable semantic-topology record on the server, performs one bounded + association-bound host-route/TCP:1883/association-bound host-route sample + through the same process-owned macOS association observer used by the + supervisor, without mutating supervisor evidence, and publishes + `missioncore.xgrids-k1-configured-endpoint-probe/v1`. It never enters the BLE + runtime, never reads DeviceInfo, never changes K1 networking, never retries + automatically and never grants + control authority. The result carries the exact transport plus current intent + or durable semantic revision, so a result from an older target is not rendered + as evidence for a replacement topology. It is diagnostics, not a prerequisite + for the normal scan-select-connect lifecycle. +- The first exact MQTT DeviceInfo accepted for one BLE `transport_ref` creates + a private durable identity pin. A later logical-device/profile mismatch is a + hard identity conflict and cannot overwrite that pin automatically. +- Exactly one local Mission Core process may own the application-control lease. + Process death releases the OS lock, but a second live process cannot open a + competing control dialogue or emit START/STOP. +- The same stable OS lease is acquired at the lowest supported CoreBluetooth + boundary for discovery, status/calibration reads, GATT inspection, AP enable + and Wi-Fi provisioning. A higher-level network transaction may explicitly + borrow its already-held lease, but no service, CLI or read-only helper may + bypass it or acquire a second lock file. +- Caller timeout/cancellation does not release that OS lease. Ownership moves + to the detached native cleanup and is released only when CoreBluetooth has + actually completed. An unproved cleanup poisons the BLE runtime and retains + the lease until process exit; a new loop or request cannot guess that the + adapter is idle. +- Control and data freshness are independent. Either can fail without + inventing the state of the other. +- No command is automatically replayed after timeout, reconnect, wake, browser + refresh, process restart, or network change. +- There is one narrowly scoped service-owned exception to the general + no-background-reconnect rule: an already running, plugin-commanded + acquisition with a composite-confirmed START may rebind its **read-only data + and inspection transports** after transient host-path loss. This exception is + defined below. It is not an operation retry and grants no command or network + mutation authority. +- UI entry, mode, selection and credential input never + start recovery work. Apply is the only normal connection intent. It performs + no hidden discovery, browser Verify, candidate substitution or command + replay. The exact post-reset retired-UUID exception defined above may append + one request-bound local reopen checkpoint inside Apply and settle it through + the service-owned read-only continuation; it is not a UI recovery step and + grants no extra write or START/STOP. There is no loop, timer-driven retry, similar-device fallback, + continuation after completion or background resumption. A later Apply creates + a new bounded intent; the ended one never restarts itself. Exact target + authority comes only from the captured fresh discovery, backend policy, + semantic topology and supervisor/ledger fences; browser memory cannot + manufacture it. +- The read-only observation may open a non-reconnecting pre-START control + generation for the exact target. Outbound ordinal 1 is exactly one canonical + DeviceInfo request; it publishes nothing else and then, when physical + classification is required, waits passively for a fresh, non-retained + DeviceStatus received after the DeviceInfo barrier. Each requested read-only + observation emits zero BLE writes, Wi-Fi provisioning writes, DeviceConfig, + ModelingStatus/status solicitations, time sync, workspace, project, START or + STOP. It is not a network reconnect and grants no mutation authority. +- A denied Apply terminates before device I/O with an explicit stale or + safety-blocked outcome. An Apply whose response is lost after dispatch is + explicitly outcome-unknown and `safe_to_retry=false`; it never reuses the + password or restarts itself. Step 02 remains visible with the selected card so + the operator can understand the result and deliberately start a new scan or + intent. Physical recovery remains pinned to its durable `transport_ref`, so + another K1 cannot be substituted by client inference. +- Fresh canonical READY records cessation/standby without inventing a + successful STOP. Fresh initialized SCANNING for the exact same project may + rebind the successful or ambiguous physical START to the exact new control + generation and expose one single-use, explicitly confirmed STOP. It never + replays START, never sends STOP automatically and never grants a second STOP + from the same confirmation/checkpoint. +- A STOP application acknowledgement is not a READY observation. If + SCAN_STOPPING was observed and Wi-Fi or MQTT is then lost before READY, local + receiver/control cleanup may finish and the product state becomes + `standby-unconfirmed`. Explicit passive Scan and explicit bounded exact-target + read-only recovery remain available; provisioning and START remain blocked on + that lineage. Scenario reset may retire it without claiming READY. +- If STOP was accepted but neither READY nor SCAN_STOPPING arrives before the + backend-owned operation deadline, Mission Core performs local-only + receiver/camera/perception-ingress cleanup, terminalizes the operation as + `timed_out`, and presents the local result as `standby-unknown`, with + `side_effect_status=unknown` and + `safe_to_retry=false`. The durable physical ledger stays unresolved. After + cleanup, passive BLE and read-only observation are allowed; provisioning, + START and STOP remain denied on that lineage until exact reconciliation. + Scenario reset remains available as a local retirement escape. A cleanup + failure retains the local lease/fence and retries + only host cleanup; it never repeats the physical STOP. +- Every runtime message is bound to the acquisition producer generation that + created it. A callback from an older generation is rejected before it can + update data freshness, perception ingress, metrics, or authority. +- In particular, **no automatic BLE network write or retry is allowed after an + ambiguous dispatch**. A later explicit operator attempt is a new session and + is allowed after the old active operation and cleanup have terminated. +- The network-attempt journal and semantic topology store are deliberately + separate. The journal is historical audit; the topology store answers only + what an exact current observation last proved. Neither restores BLE presence, + host, endpoint, DeviceInfo, control or data authority after restart. +- Legacy unresolved network records are terminalized as historical + `abandoned-by-restart`/`outcome-unknown` audit during migration. They are not + silently declared successful and do not authorize replay, but they also do + not veto a new explicit operator session. + +### Sole automatic active-stream read-only rebind exception + +This exception exists only while one process still owns the exact active +acquisition. Admission requires every fact below at the instant data transport +reports connection loss: + +- the physical ledger has one resolved, composite-confirmed START with publish + return, QoS2 completion, successful application response and fresh + non-retained initialized same-project SCANNING; +- the current acquisition id, device id/session, evidence session, snapshot + runtime id, runtime producer generation, connection intent/mode/target, + transport ref and physical operation id all match that START lineage; +- the acquisition-owned right-camera recording root still matches the same + evidence session; and +- no explicit mode/reset/force-finish owner has superseded the lineage. + +When admitted, the runtime remains neutral `reconnecting`; a short route loss +does not terminalize the acquisition. Retry cadence is bounded backoff +`0.5s, 1s, 2s, 4s`, then `5s` capped. There is no automatic terminal timeout: +the owner remains reconnecting/blocked until exact recovery, truthful physical +standby/fault evidence, or explicit local force-finish. A late failure from the +old application-control socket is classified as superseded transport evidence +and cannot raise a global terminal operation while this recovery generation is +current. + +Local force-finish invalidates that recovery generation before closing any +socket or producer. A receiver, camera or archive cleanup failure therefore +cannot resurrect the old acquisition: it is a visible retryable **local-only** +finalization failure, retains the evidence cleanup fence, and leaves the +physical START proof untouched. Only an explicit local cleanup retry or exact +connection-scenario reset may finish those retained host resources; neither +path may publish START/STOP, enter BLE, or write device/host network state. + +Each attempt may perform only: + +1. exact host-route/association and TCP observation for the frozen target; +2. one inspection-only DeviceInfo bootstrap on a new globally fresh control + generation; and +3. passive receipt of a fresh, non-retained DeviceStatus after that DeviceInfo + barrier. + +It performs zero BLE discovery/write, Wi-Fi association/provisioning, +DeviceConfig, time sync, workspace/project mutation, START or STOP. The raw +capture evidence writer is retained across MQTT client replacement. A durable +read-only reconciliation may advance the physical record revision; a failed +first MQTT SUBACK may retry only while every immutable lineage field is equal +and that revision advanced monotonically for the same operation. + +Recovery outcomes are fail-closed: + +- exact same identity and initialized same-project `SCANNING` append a durable + active rebind, adopt the fresh control generation and resubscribe data without + replaying START; +- fresh unbound `READY` appends durable cessation and interrupts/seals local + acquisition owners without STOP; +- fresh `SCAN_OVER` is also durably classified as cessation, but leaves a + `scan-over-awaiting-ready` read-only fence. It is no longer modeled as an + active START, yet a later START remains denied until another fresh exact + `READY` reconciliation; +- foreign identity/same-IP substitution, changed target/session/runtime or + camera lineage blocks recovery for explicit operator handling; and +- device/system fault or unsafe status terminalizes locally as fault without + STOP or retry. + +If the acquisition-owned FFmpeg process exited or has emitted no complete +segment for the stall threshold, recovery may CAS-restart only the same right +camera/source/target/recording session and exact source-generation/active-epoch. +The old archive epoch is sealed `interrupted`, a new epoch is opened once, and +late callbacks from the old epoch are generation-rejected. This is local +producer recovery, not public camera selection or a device command. + +`acquisition.force-finish-local` is the explicit escape hatch for permanent +loss. The request is fenced by snapshot runtime, acquisition id/state revision, +recovery generation and runtime producer generation. The backend invalidates +and cancels the recovery owner before closing local control, receiver, camera +and perception ingress; it preserves physical START evidence and sends no +STOP. A queued mode reset uses the same idempotent helper. Whichever wins the +local lifecycle gate supersedes the other, and no late recovery success may +revive the old acquisition. + +## Network-attempt lifecycle and audit + +The live operation may pass through `prepared`, `dispatching`, and `observing`. +Those stages fence concurrent work only while that explicit operation or its +native cleanup is active. The durable record is an audit trail, not a +cross-session mutation barrier. + +| Stage/outcome | Meaning | Next explicit session | +| --- | --- | --- | +| `prepared` | Current selected session passed preconditions; no write crossed the dispatch boundary | If interrupted, record `not-dispatched`, release active ownership, and allow a new scan-select-connect attempt. | +| `dispatching` | Audit was persisted immediately before the one reviewed BLE write | If the process/transport ends without a result, record `outcome-unknown`; never replay automatically. | +| `observing` | The one write returned and bounded status observation is running | Record the observed outcome or `outcome-unknown`, then release active ownership. | +| `completed` | Current attempt observed its intended topology | Rebuild host/TCP/DeviceInfo for this session; later explicit intents still start cleanly. | +| `failed` / `interrupted` / `outcome-unknown` | Current attempt ended without a usable connected session | Clear selected/live session. The normal operator flow may start a distinct new attempt; no special recovery control is required. | +| Corrupt audit | Historical data cannot be trusted | Quarantine/report the audit record without manufacturing success or replay. If the active process lease is free, corrupt history alone does not make the physical K1 permanently unusable. | + +An unknown result remains honestly unknown in history. Mission Core does not +claim that the command was applied or not applied. That uncertainty prohibits +automatic replay of the old attempt; it does not prohibit a later fresh, +explicit operator intent from sending its own single reviewed command. + +## Transition rules + +### Switching Quick Connect and Bridge + +1. A mode choice or same-mode **Подключить новый K1** click is one explicit, + idempotent `reset_scenario` under desired-mode CAS. It remains available + during active, reconnecting, cleanup-pending and unknown-physical states. + Mission Core locally seals/retire receiver, camera and control ownership, + invalidates candidates/drafts/credentials and retains truthful audit that + the previous K1 may still be scanning. It sends no device command, BLE, + host-network write or automatic Scan. + The top-right refresh-shaped utility is also this explicit reset, not + `state.read`: **Сбросить подключение** sends one reset to the canonical + Bridge default. While pending its label is **Сбрасываем подключение** and it + is non-dispatchable until that bounded reset settles, preventing a second + browser token from abandoning the accepted result. It still supersedes any + other pending local action. The committed revision overwrites even a dirty + browser mode draft with canonical Bridge. +2. Choosing the original mode is a new-device reset, not cancellation of a + browser-local draft. A duplicate `reset_id` replays the committed result; + a newer reset supersedes an older waiting reset before teardown. +3. Explicit Scan is a separate later action and runs one six-second bounded + passive discovery. Mission Core proves that + local runtime/native cleanup has completed and the BLE discovery lease is + free. It may close only a safe pre-START local control session and locally + abort an exactly prepared/no-receiver acquisition without STOP. An unknown + or active physical record does not block this Scan. The committed reset + marker remains `active=true` until a successfully admitted explicit Scan + completes, including an empty successful result. The Scan captures the + exact reset id/revision/mode at action entry, then stores its admitted + discovery generation and sets only that unchanged marker inactive. Failure, + cancellation, generation invalidation or a different + newer pending reset leaves it active. The inactive marker remains available + for exact `reset_id` replay and old-operation suppression. It continues to + fence the saved/reconnect presentation of the retired scenario, including + after reload, but never hides a newly correlated connection attempt admitted + after the reset revision. A new failed or outcome-unknown attempt renders + its current recovery/error surface immediately. +4. Scan does not revoke or replace semantic topology and starts no connect or + Verify. Physical recovery remains pinned to the exact `transport_ref`. +5. The operator selects one ordinary fresh K1 and immediately enters applicable + credentials. Selection is local-only. Apply proves physical standby, exact + mode/discovery/runtime CAS and the selected transport. Physical unknown/ + active blocks Apply and START while leaving separately explicit recovery. +6. Apply retires old live authority, creates the new connection + intent, opens a new GATT session for that selected transport, may read + internal baseline `7f02`, and performs at most one reviewed network write. + It is never retried automatically. +7. Success admits the observed topology. Normal Bridge Apply does not switch Mac + Wi-Fi. Host/TCP/DeviceInfo may therefore remain unready and require explicit + read-only Verify. Failure/disconnect ends the new session; the next operator + attempt starts with explicit discovery and selection. + +Quick to Bridge and Bridge to Quick obey the same transition. Neither path may +carry an old Mac route, TCP result, DeviceInfo response, control session, data +session, or UI `connected` label across the intent boundary. + +### Losing K1, the router, or the Mac route + +- Loss before dispatch records `not-dispatched`; loss at/after dispatch records + `interrupted`/`outcome-unknown`. Both end the active connection session and + permit a later distinct explicit attempt. Neither automatically retries. +- Loss after the target was semantically proven preserves that K1 topology but + revokes host, endpoint, DeviceInfo, control, and acquisition-start authority. +- During acquisition, fresh data that survives control loss is evidence-only. + Mission Core must not infer that recording stopped, nor issue an automatic + STOP over a newly recovered connection. +- Once local receiver/control/native cleanup is complete, physical unknown or + active state leaves passive BLE Scan available. It continues to deny Apply + and START on the old lineage. Explicit read-only recovery targets only the + durable physical record's `transport_ref`; alternatively, scenario reset may + retire that old lineage without resolving it and open a clean new Step 01. +- If that exact target is permanently unavailable, retirement/reopen requires a + separate explicit recovery confirmation with exact CAS. Neither **Выбрать** + nor **Применить** owns that local mutation or subsequent Verify. +- If STOP reached SCAN_STOPPING and the Mac loses Wi-Fi before a fresh READY + status, the local operation is `standby-unconfirmed`, not a generic connected + state and not a proven STOP. If STOP was accepted but neither READY nor + SCAN_STOPPING arrived before the backend operation deadline, local-only + cleanup produces terminal `standby-unknown`, while the physical ledger stays + unresolved. In both cases only exact read-only recovery may classify READY or + same-project SCANNING; no command is replayed automatically. +- A manual Mac Wi-Fi change and an unplanned router/hotspot failure have the + same authority consequence: host epoch changes or expires, so all evidence + bound to the old epoch is rejected. + +### Browser, sleep, wake, and backend process lifecycle + +- Browser app refresh/close emits no device command but ends that operator + connection session. The next page load starts with no selected live K1 and + renders the cold mode + Step 01 **Подключение** surface. Backend-owned history + stays internal; only explicit search, Apply, or a separately rendered exact + recovery action may create I/O. Uniform ordinary **Выбрать** creates only a + browser-local draft. +- Mac sleep/wake always invalidates the old host epoch and lease, even if the + interface name and IPv4 appear unchanged after wake. +- Backend restart closes every old network connection session. A live network + attempt is terminalized as `not-dispatched` when dispatch was not crossed or + `abandoned-by-restart`/`outcome-unknown` otherwise. Backend startup does not + verify, scan, reconnect, provision, START or STOP automatically. A later + operator action may invoke one bounded exact-target read-only attempt. +- A resolved target may restore semantic topology after restart, but it starts + configured/offline. Host path, TCP and DeviceInfo must be observed afresh. +- Restart discards every in-memory CoreBluetooth object, selection and session + token. The UI starts with no selected live K1. Explicit search plus ordinary + **Выбрать** creates only the new local candidate draft; it never internally + re-observes a target. A distinct exact recovery action may perform the one + backend-authorized read-only observation when policy requires it. + Old network-attempt audit has no role in admission. Host association, TCP + reachability, DeviceInfo/control and data remain separate proofs. +- Service shutdown cancels and owns an exact control-bootstrap child while its + owner event loop is running. If synchronous shutdown begins only after that + loop has already stopped or become unavailable, Python cannot physically + join the now-unrunnable Task. The service must still terminalize the durable + child fail-closed and set a closing fence before releasing ownership, so a + retained Task reference cannot later publish success, perform I/O or grant + control authority; the reference may remain until loop/service destruction. + +## Required acceptance scenarios + +Each scenario must assert backend state, durable ledger state, authority, +allowed next action, and product presentation. Passing only the HTTP request or +showing a green lamp is insufficient. + +### Topology transitions and device identity + +| ID | Scenario | Required acceptance | +| --- | --- | --- | +| CONN-01 | Quick Connect to Bridge succeeds | Quick session closes; the operator starts a new Bridge scan, local selection and Apply; one Bridge write at most; exact STA/LAN status commits Bridge; host/TCP/DeviceInfo rebuild; `connected` appears only after the exact new binding. | +| CONN-02 | Bridge to Quick Connect succeeds | Same guarantees in reverse; old Bridge session closes before new Quick discovery, local selection and Apply; AP status commits Quick even before the Mac joins it; old Bridge route/DeviceInfo cannot authorize Quick. | +| CONN-03 | Quick to Bridge fails before dispatch | New attempt terminalizes `not-dispatched`; old Quick facts are historical only, no live session is restored; no BLE side effect, no automatic retry, and a later explicit attempt is allowed. | +| CONN-04 | Bridge to Quick fails before dispatch | New attempt terminalizes `not-dispatched`; old Bridge facts are historical only; active ownership releases and a later explicit attempt is allowed. | +| CONN-05 | Bridge to Bridge, same mode but different K1 | Different transport/device creates a new intent. Old K1 TCP/DeviceInfo is rejected; the new K1 must pass exact discovery, topology, target, identity and control proof. | +| CONN-06 | Same UUID is rediscovered after the prior session ended | It remains a candidate until the operator presses the same **Выбрать** action used by every row. That click updates only the local form and immediately exposes applicable inputs. It performs no Scan, connect, Verify, reopen, network mutation or command. | +| CONN-07 | Wall-clock time passes beyond the legacy candidate TTL after one explicit Scan or admitted Apply | Unselected rows remain stable for the current scan generation; the active GATT session also remains usable. A successor Scan/reset/runtime-owner teardown or proven exact-target GATT failure invalidates the applicable candidate/session, while every Apply still requires exact-handle capture and live GATT validation. | +| CONN-08 | Backend/app restarts after any network-attempt stage | No in-memory CoreBluetooth handle, selection or active network session survives. The previous attempt becomes terminal audit and never causes automatic discovery/write or blocks the normal new scan-select-Apply flow. | + +### Hard K1 power loss by operation phase + +| ID | Fault point | Required acceptance | +| --- | --- | --- | +| CONN-10 | Power removed before `prepared` | No write. Active selection/session clears; previous state is historical only; operator may power K1, scan, select and Apply again. | +| CONN-11 | Power removed after `prepared`, before `dispatching` | Attempt terminalizes `not-dispatched`; no previous live session is restored; active ownership releases and a later explicit attempt is allowed. | +| CONN-12 | Power removed after `dispatching`, before write result | Attempt is terminal `outcome-unknown`, no automatic retry occurs, active selection/session clears, and a later explicit operator scan-select-Apply intent is allowed after current safety gates. Physical START/STOP ambiguity rules are unaffected. | +| CONN-13 | Power removed during `observing` | Last bounded observation remains audit; attempt ends and active ownership releases. No automatic BLE retry and no mandatory read-only recovery UI. | +| CONN-14 | Power removed after target topology was semantically applied, before host association | New device topology stays committed; state is configured-offline, not mutation-unknown and not connected. | +| CONN-15 | Power removed while connected but idle | Host/TCP/control lease expires; topology becomes last proven context; no device command is inferred or replayed. | +| CONN-16 | Power removed during acquisition | Control and data transition independently to lost/stale. A process-owned composite START may enter the sole read-only active-stream recovery generation, but a powered-off K1 cannot satisfy route/TCP/DeviceInfo and remains reconnecting/blocked until explicit local force-finish. Physical recording stays unknown/last-proven-active; no automatic STOP/START or network write occurs. | +| CONN-17 | Process crashes at any START/STOP publish boundary | Durable physical-command state distinguishes `prepared` from `dispatching/observing`. Only `prepared` can become `not-dispatched`; every later ambiguous stage blocks replay and automatic START/STOP after restart. | +| CONN-18 | A proven START survives, but its original MQTT process/socket is lost while the same process-owned acquisition and project are still SCANNING | The sole service-owned active-stream exception retries exact host/TCP plus inspection-only DeviceInfo and passively receives a fresh non-retained initialized SCANNING status for the frozen lineage. The original START remains `succeeded`; the raw writer and producer generation are preserved, START is not replayed, and data resubscribe resumes only after durable same-lineage rebind. | +| CONN-18A | Active-stream recovery observes fresh SCAN_OVER | Append a durable cessation audit for the same START, interrupt/seal local owners without STOP, clear active physical classification and retain a `scan-over-awaiting-ready` read-only fence. A fresh START remains denied until another exact fresh READY reconciliation. | +| CONN-19 | A proven START survives, then K1 is power-cycled and a fresh explicit observation sees READY | The bounded exact-target attempt publishes one DeviceInfo request and passively observes fresh non-retained READY. Record physical cessation without inventing a successful STOP; the original START remains `succeeded` and effective physical state becomes standby. A later START still requires a fresh normal control session and READY baseline on the socket that will publish it. | + +### Host network continuity + +| ID | Scenario | Required acceptance | +| --- | --- | --- | +| CONN-20 | Shared router or phone hotspot disappears | Host epoch/endpoint/control revoke; Bridge topology remains last semantically proven. The plugin section returns to its ordinary cold progressive flow, not provisioning success, a stale connected state or a technical recovery ceremony. | +| CONN-21 | Router returns with the same SSID/IP | Similar-looking network values do not resurrect the old lease. Fresh host observation, TCP probe and exact DeviceInfo/control proof are required. | +| CONN-22 | Operator manually switches Mac Wi-Fi during Bridge | Epoch changes immediately; in-flight old-epoch TCP/DeviceInfo results are rejected; no automatic BLE write follows the switch. | +| CONN-23 | Operator manually leaves K1 AP during Quick Connect | Quick topology remains the device fact, while host/endpoint/control become offline; returning to the AP requires read-only re-verification. | +| CONN-24 | Mac sleeps and wakes on the apparently same network | Old epoch, endpoint and DeviceInfo proof are stale; reacquire them read-only before control returns. | +| CONN-25 | Mac route changes between host probe and TCP result | The result for the earlier epoch is discarded and cannot create reachable or connected state. | +| CONN-26 | TCP 1883 is open on a wrong/non-K1 target | Endpoint may show reachable diagnostic only; missing/mismatched DeviceInfo keeps connectivity degraded and authority false. | +| CONN-27 | MQTT socket is half-open or heartbeat/control proof stops advancing | Route and TCP alone do not preserve authority. Exact-session control proof expires on monotonic and suspend-aware TTL; control/start revoke without sending a command. | +| CONN-28 | Physical-state recovery is required after an ambiguous/interrupted command or a resolved active START whose control socket was lost | Normal row selection stays local and Apply stays denied. A separately explicit recovery action for the exact physical record `transport_ref` may own at most one read-only Verify; another row cannot substitute for it. Each observation generation publishes exactly one canonical DeviceInfo request and passively accepts only a fresh non-retained DeviceStatus after its DeviceInfo barrier. READY records standby/cessation. Exact same-project initialized SCANNING records an active rebind and exposes one single-use explicit STOP outside connection setup. No recovery action scans, provisions, retries or replays a command. | +| CONN-29 | K1 is already SCANNING but there is no matching durable Mission Core START chain, or a different K1 is discovered during recovery | Treat it as external/foreign state, not as our successful START. The foreign candidate may remain visible as passive discovery evidence, but it cannot replace the physical record's `transport_ref`, mutate topology, reconcile the ledger or authorize START/STOP. No automatic STOP is sent; takeover requires a separate explicit durable policy. | +| CONN-29A | Mac Wi-Fi/route disappears for longer than the old control keepalive while one composite-confirmed acquisition remains owned, then returns | Keep acquisition neutral `reconnecting`; retire the late old-control failure as superseded transport. Exact route/TCP/DeviceInfo plus fresh same-lineage SCANNING resumes the receiver. Wrong identity, runtime/session replacement or changed physical operation fails closed. | +| CONN-29B | Operator chooses `Завершить локально` while recovery is pending, concurrently with a mode reset or late recovery success | One lifecycle-gated idempotent cleanup wins, invalidates recovery generation first, seals receiver/camera/control locally and preserves physical evidence. No STOP/START/BLE/network mutation is sent and no late callback revives acquisition. | + +### Durable restart and concurrency + +| ID | Restart/concurrency point | Required acceptance | +| --- | --- | --- | +| CONN-30 | Browser/app refresh or close during a network connection attempt | End the operator connection session without automatic command. Reload starts with no selected live K1 and offers the normal discovery flow; prior attempt remains audit only. | +| CONN-31 | Server restarts with no active operation | No automatic device action and no restored live selection. Historical topology may display as historical/offline context only. | +| CONN-32 | Server restarts at `prepared` | Terminalize as `not-dispatched`, release active ownership, and allow a later explicit new session. | +| CONN-33 | Server restarts at `dispatching` | Terminalize as `abandoned-by-restart`/`outcome-unknown`; never replay automatically; do not block a later explicit new session. | +| CONN-34 | Server restarts at `observing` | Preserve the last observation as audit, terminalize the attempt, release ownership, and allow a later explicit new session. | +| CONN-35 | Server restarts after a completed target observation | Historical topology may display configured/offline, never as live reachability, selection or authority. | +| CONN-36 | Server restarts after `not-dispatched` | Preserve audit/context only; no old active session is restored. | +| CONN-37 | Network-attempt audit JSON is corrupt, oversized, wrong-schema, wrong-permission or symlinked | Quarantine/report the historical record and never invent success/replay. Once active process ownership is known free, corrupt audit alone does not permanently block a fresh explicit session. | +| CONN-38 | Two local server processes request mutations concurrently | Cross-process lease serializes live admission. At most one process owns the selected GATT/write operation; the other performs no BLE write. | +| CONN-39 | A stale process tries to update another operation ID/revision | CAS rejects the stale audit transition and cannot alter the current active operation or emit a BLE write. | + +### Control and data plane independence + +| ID | Scenario | Required acceptance | +| --- | --- | --- | +| CONN-40 | TCP remains open but MQTT DeviceInfo/control dialogue fails | Connectivity is degraded/unverified; control and acquisition-start authority are false. | +| CONN-41 | MQTT control fails while points continue | Control is lost immediately; points may remain visible only as evidence for the current bounded session; no command is issued automatically. | +| CONN-42 | Point/camera stream stalls while control remains healthy | Data state becomes stalled/lost without inventing control loss; operator may stop or diagnose according to `allowed_actions`. | +| CONN-43 | Old acquisition packets arrive after intent/host epoch change | Packets cannot become authoritative for the new binding/session and cannot make the device connected. | +| CONN-44 | DeviceInfo response from old target/epoch arrives late | Exact binding mismatch rejects it; no lease or green connected state is created. | +| CONN-45 | Read-only control observation succeeds after an acquisition interruption | DeviceInfo proves only current identity/control. Physical recovery also requires a fresh non-retained same-generation DeviceStatus after the DeviceInfo barrier. Mission Core does not infer previous physical recording state and does not replay START or STOP; exact same-project SCANNING may expose one explicit single-use STOP only through the durable rebind. | +| CONN-46 | A message from producer generation N arrives after generation N+1 starts | It is rejected before data freshness/perception admission; it cannot mark the new acquisition healthy. | +| CONN-47 | A second local process opens application control while the first is alive | Stable OS lock denies the second process before MQTT commands; no competing START/STOP is emitted. | +| CONN-48 | Camera producer generation N drains after N+1 is bound | The archived N fragment remains evidence, but the live perception ingress rejects it before the new acquisition queue. | +| CONN-49 | Read-only connection monitor raises an unexpected exception | Supervisor revokes authority for that observation, the monitor remains supervised with bounded backoff, and later passive polls can refresh presentation without a process restart. Polling starts no Verify/Scan/reconnect and creates no operator progress state. | + +### Freshness, identity and presentation + +| ID | Scenario | Required acceptance | +| --- | --- | --- | +| CONN-50 | Candidate wall-clock age advances, or Mac sleeps/wakes | Wall-clock age alone removes no row from the latest scan generation and grants no mutation authority. Sleep/wake or an actual disconnect revokes live host/control authority; runtime-owner teardown also invalidates captured handles, so a later Apply must fail cleanly or require explicit rediscovery rather than use UUID memory as authority. | +| CONN-51 | Same route/interface/source IP appears after association changed | Opaque association token changes host epoch; old TCP/DeviceInfo/control proof is rejected. | +| CONN-52 | Logical K1 DeviceInfo changes for a durably pinned BLE transport | Identity conflict is shown; control/start remain blocked; the pin is not overwritten automatically. | +| CONN-53 | Restart with a valid semantic topology record and no live network | UI shows configured/offline historical evidence. It does not show a fresh BLE candidate, reachable endpoint, green control state or live telemetry. | +| CONN-54 | An old interrupted/unknown network-attempt audit and valid older semantic topology coexist | Both may be displayed as historical context. Neither restores a live session, triggers replay, requires a recovery button, or blocks a distinct explicit scan-select-connect attempt. | +| CONN-55 | Plugin container crosses any acceptance width while an error panel is open | Blocks stack without overlap or clipping; the factual state and normal safe next action remain visible; resize fires no device action. | +| CONN-56 | Long translated copy or an unknown future reason code exceeds a panel width | Content reflows and stays reachable. Panels do not use clipping/hidden overflow as an acceptance mechanism; geometry tests detect every escaped interactive element. | +| CONN-57 | The physical K1 is replaced after the previous command chain reached an effective standby state | Identity/profile history is never overwritten in place. Supporting replacement of a resolved-standby identity requires a distinct operator-confirmed archive/rotation transaction; `physical-command.retire-unavailable` applies only to an unresolved unavailable target and is not that transaction. Until resolved-standby rotation exists, the new identity is rejected and no physical command is emitted. | +| CONN-58 | A direct service/CLI/read-only BLE entrypoint runs while another process owns K1 or while native cleanup is detached | The call is rejected before CoreBluetooth admission. Every BLE entrypoint uses the same OS lock; timeout retains it through proven cleanup, and cleanup failure requires process restart. | +| CONN-59 | Network-attempt audit admission succeeds but the in-memory operation journal cannot admit the request | The record is terminalized as failed with `side_effect_status=none`; no BLE call occurs, active ownership releases, and no ghost blocks a later explicit attempt. | +| CONN-60 | A physical START/STOP confirmation is open while topology, device, acquisition or control CAS changes | The modal-open fence becomes stale, all confirmations are discarded and no request is sent. A confirm action may use only the exact immutable checkpoint displayed when the modal opened. | + +### Cross-layer recovery and executable UI policy + +| ID | Scenario | Required acceptance | +| --- | --- | --- | +| CONN-61 | A legacy server left a network record at `dispatching` or `observing` | Under the lifecycle lease, migration terminalizes it as secret-free `abandoned-by-restart`/`outcome-unknown` audit. It never touches BLE or retries the old write, and the terminal historical record does not block a new explicit session. | +| CONN-62 | The browser closes or refreshes while the event WebSocket is active | A proven WebSocket disconnect or closed transport finishes that browser stream quietly and changes no device/backend state. Only the exact closed-send exceptions are normalized; an unrelated runtime failure is still logged/test-visible. No device operation is retried. | +| CONN-63 | One live process/runtime authority blocks an action that the supervisor alone would allow | The composite policy intersects supervisor evidence with active BLE/process ownership, acquisition/runtime state, cleanup, identity pins and the separate physical-command ledger. Active contention is denied; historical network audit is not. The UI does not recreate authority from mode labels or remembered addresses. | +| CONN-64 | Acquisition data is lost while control is healthy, or control and data are both lost | With healthy exact control, a physical STOP remains available only through the durable physical-command/CAS gates. With control lost, physical STOP is denied; local receiver cleanup remains available, the physical K1 state is labelled unknown and manual device follow-up is shown. A nonexistent acknowledgement action never replaces either executable path. | +| CONN-65 | Only durable configured/offline topology remains after restart | A diagnostic endpoint probe may use that exact historical target without entering BLE or mutating K1, but it is optional diagnostics and grants no control. Cold UI entry and polling start no work. A mode/same-mode reset performs one local-only CAS, retires the old lineage and exposes clean Step 01 without automatic Scan or device/host I/O. One explicit six-second scan produces candidates. **Выбрать** is local-only and immediately reveals the applicable Step 02 inputs. Only Apply may create a connection intent; there is no background Verify, discovery, reconnect or mutation. | +| CONN-66 | The one-second connection monitor observes macOS association continuity for a long session | Association identity comes from a bounded compiled or long-lived observer. The monitor does not compile/launch the Swift source twice per poll, cannot hold its shared observation lock for tens of seconds, and revokes freshness while the observer is unavailable instead of delaying the whole control lifecycle. It never invokes Verify, BLE discovery, selection or reconnect and never owns a UI action loader. | +| CONN-67 | A Quick-to-Quick, Bridge-to-Bridge, or cross-mode attempt ends after dispatch with an unknown result | Record the old attempt as terminal `outcome-unknown`, clear its active selected session, and never replay automatically. A later explicit operator scan, selection and Apply is a distinct intent and may perform its own single reviewed write after safety admission; no historical deadlock and no mandatory preflight. Real-K1 repeated reconnect/mode-switch acceptance remains required. | +| CONN-68 | STOP is acknowledged, K1 enters SCAN_STOPPING, and Mac Wi-Fi is lost before fresh READY | Complete local receiver/control cleanup and present `standby-unconfirmed`, not a generic global error and not proven standby. Passive BLE Scan remains allowed after cleanup but starts only from an operator action. An explicit scenario reset may retire the old local/audit lineage and expose clean Step 01 without resolving standby or sending a command; provisioning and START stay denied on the old lineage. Exact read-only recovery may classify fresh non-retained READY or same-project SCANNING; no automatic retry or command. | +| CONN-69 | K1 is hard-powered off during Bridge acquisition, then rebooted while Mission Core remains open | The lost acquisition becomes terminal locally without cache/backend reset; stale control/data cannot revive it. Session recovery remains a separate exact-target surface and passive BLE Scan runs only when requested. A new connection scenario requires explicit reset, then a successor Scan; every admitted result uses **Выбрать**, including the prior UUID. READY records cessation; fresh non-retained identity-bound SCANNING establishes only read-only control proof and exposes one explicit STOP outside connection setup. It never restarts the retired receiver, camera, writer or acquisition. Bridge field acceptance is required; Quick Connect was not exercised by this live incident. | +| CONN-70 | UI enters a disconnected/configured K1 | UI entry, passive polling, row selection and input run no Verify, BLE discovery, connect or mutation. A mode/same-mode gesture owns one visible local scenario-reset action, then clean Step 01 is visible; it never starts discovery automatically. Search runs exactly once for six seconds per explicit click. Every result has **Выбрать**; selection retains the card and immediately reveals applicable Step 02 inputs. One Apply creates one connection intent with at most one K1 mutation and no hidden Scan/Verify/replay. | +| CONN-71 | STOP is accepted, but neither READY nor SCAN_STOPPING arrives before the backend operation deadline | Automatically close only host-owned receiver, camera, perception ingress and control resources. Terminalize the operation as `timed_out` and present `standby-unknown`, with `side_effect_status=unknown`, `safe_to_retry=false`; keep the durable physical record unresolved. After successful cleanup allow a separately explicit exact bounded read-only recovery. An explicit scenario reset instead retires the old local lineage and exposes clean Step 01 while preserving the auditable physical outcome. Its successor Scan uses uniform **Выбрать** rows and never enters recovery from the candidate list. Repeated state polls are idempotent and never replay STOP or start recovery; local cleanup failure retains the local fence and retries cleanup only. | +| CONN-72 | A connected or previously configured Bridge surface is opened | Render the existing plugin section, not a separate wizard or recovery dashboard. Mode, one explicit six-second scan, uniform results, local selection, immediate credentials and one Apply form one predictable path. Backend revision, random intent ID, desired-mode revision, supervisor and ledger CAS remain internal. Only Scan and Apply show truthful action-specific progress. | +| CONN-73 | Apply for fresh cold Bridge candidate B returns a provisional post-write topology before MQTT identity is known | Record the single write exactly once and keep control unready. Commit identity/control authority only after exact DeviceInfo and durable transport/profile pin success. On mismatch or control bootstrap failure, preserve durable A where required, retain B's network-attempt audit, return `network_applied/control_not_ready`, and never repeat the same intent. Recommended read-only Verify may establish control. Alternatively, current server policy may admit an explicitly new intent; the historical attempt itself grants no authority. | +| CONN-74 | The operator chooses **Изменить сеть** while neighboring BLE devices may be advertising | Pin the exact authoritative Bridge transport/mode and reveal credentials immediately. Opening and editing perform no I/O. Apply performs no hidden discovery or Verify and crosses at most one reviewed K1 write-dispatch edge. Reject stale-tab, stale candidate, missing exact device and changed-binding requests before device I/O with an explicit stale outcome. A post-dispatch ambiguous result is explicit outcome-unknown and never replayed. | +| CONN-75 | A safe pre-START holder, active/reconnecting/cleanup-pending acquisition, unresolved durable physical history or stale runtime overlaps a mode/new-device request | The selector, same-mode escape and top-right **Сбросить подключение** utility send one idempotent `reset_scenario` with exact runtime/mode revision; the utility targets canonical Bridge and is never passive `state.read`. The reset queues behind and supersedes old local lifecycle ownership without deadlock, then locally seals receiver/camera/control resources, cancels late recovery generations, invalidates candidates/drafts/credentials and retires the old physical lineage truthfully. It performs zero START/STOP, BLE, device-network, host-network or automatic-Scan I/O. Terminal cleanup failure remains visible and retryable without committing the mode revision. On success the clean Step 01 requires a separate explicit Scan. | +| CONN-76 | An unresolved physical START/STOP target is unavailable or has been replaced while the operator selects a fresh candidate | Selection remains local and cannot retire authority. Without an explicit committed reset plus its successor Scan, Apply is denied before device I/O. `physical-command.retire-unavailable` retains stable idempotency identity and exact runtime/operation/revision/transport CAS, preserves unknown-outcome audit, performs zero device I/O and starts no Scan. Product UI exposes no checker, confirmation ceremony or reconnect action in Bluetooth results. | +| CONN-77 | A fresh post-reset scan returns candidates, including the exact actively retired UUID | Search runs once for six seconds. A successful admitted generation settles only the exact active reset marker captured at Scan entry; a Scan started before a later reset, failed/cancelled Scan or a newer pending reset does not. Every connectable row has one enabled **Выбрать**. Selection immediately exposes the local network draft and performs zero I/O. The explicit reset plus exact successor generation may admit one new network Apply to that rediscovered transport, still gated by exact handle capture and live GATT baseline and limited to one reviewed write. Before dispatch the backend may append one request-bound local reopen checkpoint; it preserves retirement/original-outcome audit and grants no START/STOP. A fresh service-owned DeviceInfo + non-retained READY/SCANNING observation settles that checkpoint without a UI Verify or command replay. SCANNING creates only explicit STOP authority and never resurrects reset-owned receiver/camera/acquisition state. Any other post-reset network dispatch consumes the exception. | +| CONN-78 | One-intent UI contract | A mode, same-mode new-device or top-right **Сбросить подключение** gesture invokes exactly one visible local-only `reset_scenario`; the top-right action changes to **Сбрасываем подключение** while pending and never calls passive refresh instead. While that reset owns `mode`, every reset entry point dispatches no B intent. Its authoritative revision clears the dirty selector and all old click-owned attempt presentation. Row selection and every SSID/password keystroke still invoke no controller I/O. The reset has a stable idempotency identity, sends no device/host command and starts no Scan. One explicit search invokes exactly one six-second scan. One Apply invokes one connect request, no frontend Scan/Verify/reopen and at most one device mutation. Pre-dispatch authority drift is rendered explicitly stale; post-dispatch uncertainty is rendered outcome-unknown and never replayed. `network_applied` plus unready/unknown control spends the old Apply and gates ordinary controls behind an explicit recovery choice. Recommended Verify accepts only the backend server-bound target. Every new Apply has a new idempotency identity, and no new-intent path runs as a hidden frontend or mutating continuation. The already-declared service-owned same-intent read-only bootstrap after the durable ACK is the sole continuation exception; it creates no UI action and cannot mutate or retry. The plugin reuses canonical `Button`, `TextField`, `ActivityIndicator` and `StatusBadge`, with no raw local controls, literal status colors, new shared entities or lifecycle changes. | + +## Composite action policy + +`connection_supervisor.allowed_actions` covers only host, endpoint, identity, +control and data evidence. It is deliberately insufficient to authorize a BLE +or physical mutation. The API therefore publishes +`missioncore.xgrids-k1-connection-policy/v1`, composed from: + +1. the current supervisor revision and exact evidence epochs; +2. the current explicit network intent and active-operation generation; +3. the network-attempt audit/idempotency record, which prevents automatic + replay inside that attempt but is not a cross-session veto; +4. the durable physical-command ledger (highest precedence for START/STOP and + deliberately unaffected by the simpler network lifecycle); +5. the process-wide BLE runtime/quarantine state and the stable cross-process + lifecycle lease; +6. current acquisition/runtime/cleanup state; +7. fresh unselected BLE candidates and, separately, the one admitted active + selected session; +8. the exact `connection_reconfiguration` revision, random intent ID, required + transport/mode and minimum fresh discovery generation; +9. semantic topology and identity-pin store health. + +Every action carries an `allowed` bit, reason codes, target source, required +transport reference and whether a new live GATT validation is mandatory. The +browser may add form-local prerequisites, such as an entered password or an +exact fresh local selection. It may not turn a denied/missing backend +decision into an enabled action. There is no product-level +`recover-current-device-network` prerequisite. Latest-generation candidate rows +and an active selected session do not disappear because wall-clock time elapsed; +generation/runtime/GATT fences and the connection lifecycle clear them at their +respective authority boundaries. + +`scan-ble` is evaluated separately from every mutation. Once local/native +cleanup and the BLE discovery lease are clear, physical unknown/active does not +deny passive Scan. The same physical fact must deny +`provision-fresh-device` and acquisition START on that lineage, but cannot deny +the local-only scenario-reset escape. +Every recovery observation and any recovery STOP are additionally pinned to +the physical record's exact `transport_ref`; a browser-selected or newly +discovered foreign transport cannot substitute for it or change topology. +Recovery is a separately explicit action. Normal row selection remains a local +draft and never admits observation or mutation. + +`physical_command.operator_retirement` is a separate backend projection. Its +`allowed` decision is true only for one exact unresolved post-dispatch target +after acquisition, runtime, control, BLE/native cleanup and cross-process +lifecycle owners are safe. The browser may invoke +`physical-command.retire-unavailable` only from that projection and must echo +the exact backend runtime, operation, revision and transport checkpoint. This +local durable mutation neither inherits `scan-ble` permission nor invokes it. +Success clears only current authority for the retired target and creates an +active exact-transport deny at every adopting or mutating entrypoint, including +provisioning, Verify/adoption, control, calibration, physical commands and +network writes. It applies the retired durable identity fence once DeviceInfo +makes that identity observable. The retirement record and unknown original +outcome remain permanent audit. + +`physical_command.operator_reconciliation_reopen` is a separate backend +projection for exact recovery of a previously established session. The browser +may invoke `physical-command.reopen-retired-reconciliation` only from that +session-recovery projection, never from a cold Bluetooth result row, and must +echo the exact backend runtime, ledger revision, active retirement ID, +transport and discovery generation, plus a stable reopening ID and explicit +operator confirmation. The local durable transition appends a +reopen audit, restores only the retired record's original unresolved stage and +removes only that retirement's active deny. It performs zero BLE, Wi-Fi, MQTT, +DeviceConfig, ModelingStatus, workspace, project, START or STOP I/O and starts +no Scan. The separately explicit recovery intent may then perform one exact +read-only Verify under its original runtime/action/candidate fences; it grants +no provisioning or command authority until fresh READY or exact same-project +SCANNING proof reconciles the record. It never continues into Apply. +Explicit passive Scan/GATT metadata/MQTT capture outside this transition remains +a non-authoritative diagnostic surface and cannot restore topology/control. The +policy does not claim pre-provision identity detection across a changed +CoreBluetooth UUID. + +Physical device commands are fail-closed when the policy is absent, malformed +or denied. Purely local risk-reduction actions are different: stopping or +aborting a local receiver/replay remains possible even when the K1 physical +state is unknown. Such cleanup never claims that K1 received STOP, and the UI +must present the required manual device follow-up. + +## Adaptive UI contract + +The UI is a projection of backend evidence, not a second connection state +machine. + +- Layout responds to the plugin container width, not only the viewport width. +- At every width, connection/topology, project/intake, status, metrics, and + error surfaces either fit side by side or stack in document order. They never + overlap, escape their panel, rely on horizontal clipping, or cover primary + actions. +- Wide composition may use two columns. Narrow composition stacks the network + connection block before project/acquisition controls; fields and actions use + the full available width. +- Requested topology, semantically observed K1 topology, current Mac + association, endpoint reachability, DeviceInfo/control state, and data state + are visually distinct. A single green lamp cannot stand in for all planes. +- `last_known` and stale observations are labelled as + historical/non-authoritative and are never rendered as a live selectable or + connected device. +- The mode selector, same-mode **Подключить новый K1** action and top utility + remain available through every non-reset lifecycle owner and send one explicit + local-only scenario-reset CAS. The first reset may supersede another pending + local action. While that bounded reset owns `mode`, all reset entry points + report cleanup and dispatch no B intent. On success the authoritative reset + revision clears old click-owned presentation and a dirty selector, then shows + clean Step 01 plus truthful warning when the previous K1 may still scan; + search never starts automatically. Provisioning and START retain separate + backend gates, and Apply remains the normal topology/device-mutation boundary. + The reset marker suppresses only the retired scenario: a later correlated + failed or outcome-unknown attempt must render immediately and survive reload + without reviving the old reconnect prompt. +- The only model-bearing plugin-section heading is **Подключение XGRIDS LixelKity K1**. + Below it, copy uses the neutral step names **Подключение** and **Сеть**. It + does not label devices as saved, original, previous or retired, + and it does not surface physical-ledger terminology. +- Progressive disclosure is intent-oriented. A cold surface shows mode and + Step 01 **Подключение** with its explicit Scan action. A completed explicit + scan shows candidates. Selecting one immediately retains its card and reveals + Step 02 inputs (or the Quick Connect summary) without an operation. +- Step 01 search begins only when pressed, runs exactly six seconds, and shows + an `ActivityIndicator` plus a visible seconds countdown. After completion, + every connectable result row has one enabled **Выбрать** action and the base + anatomy, including an exact UUID from an earlier scenario. Selection has no + loader and invokes no controller method. A transport without safe current + draft admission is explicit unavailable evidence, not a disabled primary + affordance or a reconnect CTA. +- Selection retains the candidate card and immediately exposes applicable + fields. Form input invokes no I/O. Candidate/runtime/intent drift produces an + explicit stale state and an explicit search action; it never triggers a scan + or clears the result as a hidden continuation. +- **Изменить сеть** retains one exact device card and reveals credentials + without I/O. **Применить** names the one network operation, performs no hidden + exact-UUID refresh/Verify and crosses at most one device-write boundary. +- Apply accepts the exact REST `network_applied` snapshot immediately for + `control_not_ready` and `unknown`. It does not await WebSocket/poll proof or + full connection-ready. It spends the old Apply and credentials. While the + exact service child is accepted/running, one passive settling indicator is + shown and no recovery action is enabled; terminal unready/unknown state then + presents explicit recovery choices. Verify is recommended and server-bound, + not mandatory. A new server-policy-admitted intent uses Bridge + `prepare-select-device`, Quick/Direct explicit `scan-ble`, or an explicit + scenario reset; every route still requires a later fresh scan and a new + idempotency Apply, with no hidden frontend or mutating continuation. The + declared service-owned same-intent read-only bootstrap is the only post-ACK + continuation and creates no UI action. +- After that fast ACK, the service may finish a supervised read-only control + bootstrap for the same intent. It performs no BLE/host mutation or retry and + creates no frontend Scan/Verify/new-Apply action or blocking Apply loader. Its + exact accepted/running state may own one passive settling indicator only. +- Selecting a fresh Bridge or Direct device reveals SSID/password immediately. + Quick Connect reveals no credential fields. A stale global error from device + A cannot own device B's form, and a backend runtime/fence change invalidates + credentials and renders a clear stale outcome rather than scanning or + replaying automatically. +- The plugin surface uses shared `Button`, `TextField`, `ActivityIndicator` and + `StatusBadge`. Raw local HTML controls and literal local status colors are not + accepted; no new shared entities are introduced. +- During an actively executing `prepared`, `dispatching`, or `observing` + operation, controls that could create a competing mutation are replaced by a + truthful action-specific progress row. When that attempt terminates, its audit + cannot keep the next explicit connect unavailable. An unresolved/active + physical record may hide mutation controls, but it does not deny an explicit + passive BLE Scan after local cleanup. Once that explicit scan completes, Step + 01 shows its complete result set, including unrelated BLE devices, as passive + discovery evidence. Ordinary connectable rows keep one enabled **Выбрать** + action when current backend policy admits a fresh draft; an exact prior UUID + is not special in this list. Another row cannot replace the durable target or + reveal Apply while admission remains denied. A + configured-but-unverified topology or physical recovery target starts no + operation from UI entry or polling. Only explicit search + and Apply create I/O. + Physical recovery stays pinned to the durable `transport_ref`; + each requested observation generation publishes exactly one DeviceInfo + request and passively waits for fresh non-retained status. +- Backend events and bounded state polling converge the presentation after + disconnect, cleanup, read-only recovery success/failure, and power-cycle + recovery. Polling is passive presentation convergence: it starts no device + action, shows no recovery spinner by itself and cannot extend an ended action. + Clearing browser cache/local storage, restarting the backend, or pressing a + generic reset is never a required recovery procedure. Optional state reads + and endpoint diagnostics are read-only and cannot manufacture authority. +- A human-readable error states which product fact failed (discovery, device + topology, Mac association, endpoint, identity/control, data, or active + lifecycle contention). A released local acquisition after acknowledged STOP + plus Wi-Fi loss is rendered as `standby-unconfirmed` with plain recovery + guidance, not as an unexplained global `Ошибка`. A STOP acceptance whose + READY/SCAN_STOPPING deadline expires is rendered as terminal + `standby-unknown`; after local cleanup it offers read-only recovery, while all + mutations remain fenced. Internal stack traces and reason taxonomies remain + in engineering logs, not in the primary product surface. +- Every idle/error state exposes only relevant actions derived from backend + `allowed_actions`. Historical recovery never runs behind **Выбрать** or + **Применить**; UI heuristics do not manufacture authority, loop, or + reinterpret legacy convenience fields. +- Telemetry and acquisition metrics disappear or become explicitly unavailable + when their data source is stale. The UI does not preserve old numbers as live + measurements. +- Browser refresh never invokes provisioning and starts a fresh operator + session with no restored live selection. Resizing and panel expansion do not + change the current session or topology. + +Acceptance widths are container widths `1948`, `1680`, `1481`, `1281`, `1280`, +`1024`, `761`, `760`, and `390` px. At each width, acceptance requires: + +1. no overlap or clipped content; +2. readable requested/observed/authoritative distinctions; +3. reachable primary action and error recovery action; +4. the latest-generation BLE rows remain stable across wall-clock age, are not + presented as mutation authority without exact-handle/live-GATT proof, and no + admitted active session is cleared merely because wall-clock time elapsed; +5. no green connected/control state without the exact backend binding; +6. no stale telemetry presented as current; +7. no action fired by layout change, refresh, or hydration. + +## Operational acceptance rule + +A scenario is complete only when its test evidence covers all applicable +layers: + +1. network-attempt audit transition and active cross-process exclusion; +2. supervisor evidence planes and host epoch handling; +3. exact DeviceInfo/control binding and independent data state; +4. API snapshot and `allowed_actions`; +5. adaptive product presentation at the boundary widths; +6. for real K1 acceptance, one explicitly operated hardware run with redacted + logs and no automatic mutation retry. + +The 2026-08-09 live incidents exercised Bridge only: STOP acknowledgement plus +Mac Wi-Fi loss before READY, and K1 hard power loss/reboot during acquisition. +They are field evidence for `CONN-68`/`CONN-69`, not Quick Connect acceptance. +Quick Connect recovery remains unaccepted until it is exercised separately. + +Synthetic/unit acceptance can prove the state machine and regressions. It does +not by itself claim that a real K1, router, CoreBluetooth stack, or macOS sleep +transition has been physically validated. diff --git a/docs/adr/0003-device-plugin-ui-and-runtime-boundary.md b/docs/adr/0003-device-plugin-ui-and-runtime-boundary.md index 857cfce..b1135b0 100644 --- a/docs/adr/0003-device-plugin-ui-and-runtime-boundary.md +++ b/docs/adr/0003-device-plugin-ui-and-runtime-boundary.md @@ -136,7 +136,7 @@ The current XGRIDS contribution maps its proven internal workflow into those platform states without changing the wire protocol: ```text -confirm power -> scan BLE -> select candidate -> enter Wi-Fi +scan BLE -> select candidate -> enter Wi-Fi -> provision once -> receive LAN address -> start source -> wait for first point frame -> streaming ``` diff --git a/docs/adr/0013-k1-local-connection-matrix.md b/docs/adr/0013-k1-local-connection-matrix.md index 3269e93..d8f84b7 100644 --- a/docs/adr/0013-k1-local-connection-matrix.md +++ b/docs/adr/0013-k1-local-connection-matrix.md @@ -1,6 +1,6 @@ # ADR 0013: explicit K1 local connection matrix -- Status: amended 2026-07-20; Bridge is the product path, Quick Connect retained as a prepared-host laboratory path +- Status: amended 2026-08-08; Bridge is the product path, Quick Connect retained as a prepared-host laboratory path - Date: 2026-07-19 - Extends: ADR 0004, ADR 0005 and ADR 0012 @@ -47,6 +47,23 @@ for the previous session. An active acquisition is never retargeted in place. The later correlated MQTT `DeviceInfo` response supplies model, firmware, serial and vendor identity; IP equality alone cannot identify a K1. +The 2026-08-08 physical Bridge trace corrected the earlier field model: the +first `7f02` text slot is `WIFI_AP` in AP state but contains the joined network +name in FW 3.0.2 station state. Mission Core therefore normalizes that station +response to `WIFI_CLIENT` internally and admits Bridge/Direct only when the raw +post-write name exactly matches the current explicit request plus a valid +non-AP private address. The raw name is not persisted in the secret-free +network audit or published through API state. + +The limitation still applies when recording an interrupted attempt that has no +exact post-write network-name observation. A changed private DHCP address alone +cannot identify the selected network. An already AP-ready baseline likewise +cannot prove the outcome of an interrupted Quick-to-Quick attempt. Mission Core +therefore records that attempt as terminal `outcome-unknown` and never replays it automatically. The +historical uncertainty is not a permanent barrier: after the old active +operation and cleanup have ended, a later explicit operator scan, selection, +and connect is a distinct session with its own one reviewed write. + Product decision on 2026-07-20: Bridge/direct-LAN is the continuing route. Quick Connect remains visible and executable on an already prepared host, but is not a deployment dependency or portability claim. @@ -86,12 +103,65 @@ and the following cold Swift/CoreWLAN process missed the beacon. The corrected implementation holds the selected `BleakClient` open through bounded native SSID discovery and the single association call. -BLE discovery and the selected device action form one host session. A physical -run proved that immediately rediscovering the same K1 by its CoreBluetooth UUID -can fail even though the preceding scan exposed it. Mission Core retains the -non-serializable `BLEDevice` handle process-locally and uses that exact handle -for the next selected network action; it never exposes the handle through API -state or treats the macOS UUID as durable device identity. +One explicit six-second BLE discovery and the later Apply action form one +operator intent without a second discovery. A physical run proved that +immediately rediscovering the same K1 by its CoreBluetooth UUID can fail even +though the preceding scan exposed it. Mission Core retains the non-serializable +`BLEDevice` handle process-locally and uses that exact handle for Apply; it never +exposes the handle through API state or treats the macOS UUID as durable device +identity. UI row selection itself performs no GATT or backend I/O. + +The operator-visible candidate list, local selected draft and admitted active +session are separate contracts. Every explicit scan replaces the candidate +set. Selection only binds a local form to one result from the latest admitted +generation. Wall-clock age does not remove that generation while the operator +completes the form. Apply admits only its exact retained handle and live GATT +validation may create the active session; a remembered UUID is never mutation +authority. Proven disconnect, explicit stop, app/backend restart, another +explicit Scan, or a committed mode transition revokes the applicable candidate +or live session. Rediscovery never auto-connects. + +Quick Connect to Bridge is an explicit topology transition, not another scan +heuristic. Selecting Bridge — or choosing another K1 while Bridge is already +selected — sends one idempotent local `reset_scenario` CAS. It seals retained +receiver/camera/control ownership, invalidates candidates and credentials and +retires old physical lineage truthfully, while sending no device command, BLE, +host-network write or automatic Scan. The next explicit Scan starts the clean +discovery flow, while Apply remains the topology and device-mutation boundary. +The operator selects one discovered K1, sees the Bridge credentials immediately +and submits once. The new GATT +session reads internal baseline `7f02` and emits exactly one reviewed 99-byte +station write. A connect failure ends that attempt. An ambiguous post-write +failure is terminal `outcome-unknown` audit, not a permanent cross-session +fence. There is no automatic BLE or network-write retry. + +Likewise, an exact REST `network_applied` result with unready/unknown control +spends that Apply and its credentials without making the read-model attempt a +permanent topology lock. Recommended Verify is pinned to the backend +current/configured target. A separately explicit new intent still requires +current server policy: Bridge prepares `select-device` before a later fresh +scan; Quick and Direct run an admitted fresh scan, select only its latest row +and create a new idempotency Apply; a mode or same-mode new-device transition +uses one idempotent local-only `reset_scenario` before that fresh scan. None of +these new-intent UI paths reuses the old +intent or runs as a hidden frontend/mutating continuation. The service-owned +same-intent read-only bootstrap declared below is the sole post-ACK exception. + +The Apply REST call returns as soon as the exact durable +`network_applied` proof is available. The service may continue the same +intent's supervised control bootstrap read-only after that ACK. This performs +no BLE/host mutation or retry and creates no frontend Verify, Scan, Apply or +blocking Apply loader. While the exact child is accepted/running, the UI may +show only a passive **Сеть настроена · подтверждаем управление** indicator and +must keep every recovery action disabled. Later control state arrives only as +backend presentation convergence; terminal unready/unknown state then exposes +the explicit server-policy recovery choices. + +Bridge and Quick Connect were physically accepted as separate paths before +this amendment. The combined Quick Connect to Bridge transition has automated +contract coverage but remains a distinct physical acceptance gate; it must not +be reported as field-accepted until one redacted live run records both sides of +the transition. The corrected host boundary derives a non-secret, device-scoped profile ID from the selected SSID. The reviewed client contains per-device `WiFiAP_SSID` and @@ -110,13 +180,40 @@ that opaque source before any BLE write. A missing provider fails closed. The browser, API, argv, logs, manifests and evidence never receive the secret; the importer's short-lived mutable buffer is zeroized after the Keychain handoff. +The 2026-08-06 field regression established that process identity is part of +this prepared-host contract. Runtime `swiftc` compilation produced an ad-hoc +helper with an unstable designated identity. macOS then requested Keychain +authorization repeatedly and the same process context failed to expose the +exact K1 SSID through CoreWLAN even after K1 had acknowledged AP-ready. That +runtime-compiled route is rejected. The laboratory adapter uses the previously +physically accepted Apple-signed interpreter path, +`/usr/bin/xcrun swift `, and validates the source path before +launch. A portable product implementation still requires a packaged, +precompiled and properly signed helper with a stable bundle identifier, +designated requirement, Location/CoreWLAN authorization and Keychain ACL; the +current prepared-host path does not claim that packaging work is complete. + +Before any BLE write, the helper's preflight is non-interactive. It first checks +Keychain item existence through metadata, then validates the selected profile's +SSID and `exact-firmware-profile` provenance inside the helper without returning +secret data. Provider material is also read with interaction disabled if a +missing device profile must be materialized. The association phase accepts only +that already materialized exact profile. It never +falls back to the system Wi-Fi Keychain, rewrites a profile opportunistically, +or opens a password/authorization dialog after K1 has changed network state. +An unavailable or unauthorized profile therefore fails closed with a precise +reason code and no automatic device retry. + The host-network boundary, rather than the XGRIDS frontend, owns platform association. Browsers expose no Wi-Fi join API, and Apple's iOS `NEHotspotConfiguration` consent flow is unavailable on macOS. The current implementation therefore uses a short-lived Swift/CoreWLAN + macOS Keychain helper; Windows Credential Manager and Linux Secret Service adapters remain separate platform work. The helper performs repeated read-only exact-SSID scans -inside one 15-second discovery window and at most one association. It never +inside one 30-second discovery window and at most one association. The larger +window covers the physically observed 18.142-second beacon-discovery case; +AP-ready confirms K1 state but does not prove that macOS has already observed +the RF beacon. It never repeats the BLE command, guesses a password or treats `7f01` as a credential-read command. The credential-bearing 99-byte station-provisioning frame and fixed 100-byte AP-enable frame are separate reviewed payloads. @@ -128,12 +225,13 @@ The owner also observed no explicit device/account pairing in the normal LixelGo onboarding flow; this is consistent with a firmware-defined AP secret, but does not establish account-wide authorization for arbitrary scanners. -Connection verification refreshes the session-scoped lease with the same -read-only BLE status operation. It does not write a characteristic, re-provision +Apply may read BLE baseline internally while establishing a new selected +session. Selection never does so, and the normal flow has no mandatory or hidden +"verify without write" recovery step. The baseline read does not re-provision Wi-Fi, scan the subnet, change a host route, or touch VPN configuration. The -later canonical MQTT session supplies the real data-plane connection and live -`DeviceInfo` identity check. A BLE lease observation is therefore not by itself -a claim that MQTT/RTSP is reachable. +later canonical MQTT +session supplies the real data-plane connection and live `DeviceInfo` identity +check; BLE status alone is not a claim that MQTT/RTSP is reachable. ## Consequences @@ -148,6 +246,10 @@ a claim that MQTT/RTSP is reachable. therefore not scheduled for this Quick Connect path. - Direct Connect requires an already-running hotspot and a controller route; Mission Core does not create or manage that hotspot. +- Discovery never auto-connects devices. Mode, selection and input are local + only. App restart, disconnect, explicit stop and mode transition require a + fresh explicit scan-select-Apply session. Apply performs no hidden rescan or + Verify and may cross at most one device-mutation boundary. - The application-control, START/STOP and raw-first acquisition protocol is unchanged after a target address is admitted. - Direct Connect remains explicitly pending one owner-operated physical diff --git a/docs/adr/0014-k1-macos-association-observer.md b/docs/adr/0014-k1-macos-association-observer.md new file mode 100644 index 0000000..d5d705d --- /dev/null +++ b/docs/adr/0014-k1-macos-association-observer.md @@ -0,0 +1,177 @@ +# ADR 0014: long-lived macOS host-association observer + +Status: planned production boundary; software contract may be developed behind +a disabled feature flag. + +Related acceptance item: `CONN-66` in +[`../20_K1_CONNECTION_SUPERVISION_CANON.md`](../20_K1_CONNECTION_SUPERVISION_CANON.md). + +## Context + +Mission Core must distinguish a K1 that is configured for a network from a Mac +that is currently attached to the same network. Route, TCP, DeviceInfo, +control and data evidence are bound to a host-path epoch; a Wi-Fi switch, +sleep/wake cycle or observer restart must invalidate that epoch before any late +TCP/MQTT result can restore command authority. + +The current laboratory implementation is fail-closed but not a production +observer. One normal connection-monitor poll samples the host path before and +after its TCP probe. Each sample synchronously invokes: + +```text +/usr/bin/xcrun swift plugins/xgrids-k1/macos/associate_wifi.swift +``` + +under one process-local lock with a 30-second timeout. At the one-second +monitor interval this can launch two Swift processes per second. A failed +cycle can occupy the lock for roughly sixty seconds, and cancellation of the +Python `asyncio.to_thread()` waiter does not terminate the native process or +thread. Physical-command validation shares this observation path. Shorter +timeouts, cached shell output or automatic fallback would hide rather than +remove the lifecycle defect. + +## Decision + +Production host-association evidence will come from one signed, long-lived, +read-only agent in the user's macOS login session. + +- The agent owns one `CWWiFiClient` for its process lifetime. +- It observes CoreWLAN link/association/power events and macOS sleep/wake. +- It never scans BLE, changes Wi-Fi, reads K1 credentials, reconnects MQTT or + sends START/STOP. +- It is packaged in a minimal container app and registered with `SMAppService`; + it is not a `LaunchDaemon` and is not launched through `xcrun` at runtime. +- The required Wi-Fi event entitlement and Location authorization are checked + before K1 network mutation is offered. Missing authorization produces + explicit unavailable evidence, not a crash loop or guessed association. +- The existing Wi-Fi mutator remains a separate component under the exclusive + network process lease. Observer authority and mutation authority are never + combined. + +The backend communicates with the observer through bounded local IPC. Each +backend session supplies a random HMAC key. SSID and BSSID remain inside the +agent; only an opaque continuity token is returned and it cannot be correlated +between backend processes. The token material is interface plus BSSID; SSID is +used only to report evidence quality. This keeps one AP identity stable when +macOS alternates between `ssid+bssid` and `bssid-only` disclosure. + +## Observer contract + +```text +schema_version: missioncore.macos-host-association/v2 +agent_instance_id: random 128-bit process instance +sequence: uint64 +association_epoch: uint64 +interface_name: string | null +wifi_interface: true | false | null +state: associated | not-associated | inactive | not-wifi | unavailable +evidence_quality: ssid+bssid | bssid-only | not-wifi | unavailable +continuity_token: 64 lowercase hex | null +reason_code: string | null +observed_monotonic_ns: uint64 +sample_age_ms: uint32 +cause: initial | link-change | association-change | power-change | + permission-change | will-sleep | did-wake | poll-correction | + observer-restart +``` + +`sequence` changes for every event or heartbeat. `association_epoch` changes +when interface, power, state, SSID or BSSID changes. Sleep and wake each create +a barrier even if the visible network looks unchanged afterward. A new agent +instance, IPC reconnect, sequence rollback/gap, malformed frame or timeout is +also a discontinuity. + +The backend adds its own `observer_session_epoch`; the effective host-route +fingerprint includes the agent instance, observer session, association epoch +and opaque token. A response from an old session or sequence is discarded. +Unknown schema/state or incomplete evidence is `unavailable` and immediately +revokes host authority. + +## Timing and failure semantics + +- Heartbeat: 1 second. +- Maximum cached-snapshot age: 750 ms. +- Snapshot RPC deadline: 250 ms. +- Initial handshake deadline: 2 seconds. +- Two missed heartbeats or one invalid IPC frame revoke authority immediately. +- Reconnect backoff: 250 ms, 500 ms, 1 s, 2 s, then at most 5 s. +- There is no automatic fallback to the Swift source runner. +- Agent loss affects only read-only host evidence. It never triggers a K1 + network write, MQTT reconnect or physical command. +- A discontinuity first marks the supervisor host path unavailable and rotates + its epoch. Recovery then requires fresh route, TCP and DeviceInfo/control + evidence in that order. + +## Delivery phases + +Phase A is safe without signing or a physical K1: + +1. Define the Python observer protocol and validate the v2 schema. +2. Add a fake/in-memory transport and backend session/sequence validator. +3. Implement immediate epoch invalidation and bounded cached lookup. +4. Inject the observer into the monitor behind a disabled feature flag. +5. Implement the Swift reducer and local transport as a testable Swift package. +6. Test sleep/wake, timeout, event gaps, delayed replies, crash/restart and + manual network changes. +7. Expose secret-free observer health and next action to the UI. +8. Prove 10,000 samples launch no child process and cause no lock starvation. + +Phase B requires the actual Mac signing and permission environment: + +1. Package and register the user-session agent. +2. Obtain the Wi-Fi events entitlement and complete Location onboarding. +3. Run the observer in shadow mode beside the current fail-closed probe. +4. Cut over only after the physical fault matrix and an eight-hour soak show no + unexplained divergence. + +## Acceptance gate + +- No `xcrun`, `swift` or `swiftc` occurs on the observer path. +- One agent and one CoreWLAN client serve one login session. +- Snapshot p99 is below 50 ms, hard deadline 250 ms, monitor-cycle p99 below + 1.5 seconds. +- No mutex is held across native or IPC calls. +- Sleep, wake, agent restart, sequence gap and timeout always invalidate the + effective host epoch. +- Late TCP/DeviceInfo evidence from an old epoch is rejected. +- SSID, BSSID and credentials never enter IPC logs, API state or artifacts. +- Quick-to-Bridge, Bridge-to-Quick, manual Wi-Fi switch, Wi-Fi off/on, + router loss/return with the same SSID/IP, backend restart and Location denial + all revoke control authority within two seconds and recover only through + fresh route, TCP and DeviceInfo evidence. + +Until this gate passes, the current association probe remains explicitly a +laboratory implementation and `CONN-66` remains open. + +## Laboratory containment while Location evidence is hidden + +The source-runner helper can return `association-identity-unavailable` on a +connected Mac when macOS privacy rules hide SSID and BSSID from the CLI child +process. Rotating a random fallback token on every one-second poll made a +stable route and a successful TCP probe mutually impossible: every following +sample revoked the preceding endpoint result as a fictitious network switch. + +Until the signed observer above replaces the source runner, the laboratory +probe uses one random, process-scoped token for the same interface and +unavailable-evidence scope. This is not promoted to association evidence: + +- the public evidence quality remains `unavailable`; +- interface, source address, kernel route, availability, a proven different + BSSID and process restart remain epoch barriers; +- endpoint reachability alone remains `configured-unverified`; +- only fresh exact DeviceInfo/control evidence can grant control authority; +- `CONN-66`, sleep/wake and same-subnet network-switch acceptance remain open. + +For an already reachable lease whose exact DeviceInfo identity and control +session remain healthy, a temporary helper timeout or privacy-limited +association sample may retain the preceding proven association fingerprint +only while the kernel route fingerprint, interface, source, intent and target +are unchanged. That retained sample still performs TCP contact and a second +kernel-route check, refreshing only route/TCP observation TTLs. Endpoint loss, +control loss, control-proof expiry, target/intent change, a proven association +identity change or any raw route change revokes immediately. A +`configured-unverified` path does not receive this bridge and remains bounded +by the existing technical-failure debounce and transport TTL. + +This containment removes the false per-poll epoch churn observed on the field +Mac without claiming that the planned production observer has been delivered. diff --git a/docs/adr/0015-k1-physical-state-recovery.md b/docs/adr/0015-k1-physical-state-recovery.md new file mode 100644 index 0000000..b2c7da9 --- /dev/null +++ b/docs/adr/0015-k1-physical-state-recovery.md @@ -0,0 +1,309 @@ +# ADR 0015: explicit K1 recovery beside the one-intent connection flow + +Status: accepted product, recovery and presentation contract; executable +coverage and remaining hardware acceptance are tracked in +`docs/k1-connection-acceptance.manifest.json`. + +Related acceptance items: `CONN-16` through `CONN-19`, `CONN-28`, `CONN-29`, +`CONN-65`, and `CONN-68` through `CONN-78` in +[`../20_K1_CONNECTION_SUPERVISION_CANON.md`](../20_K1_CONNECTION_SUPERVISION_CANON.md). + +## Problem + +Loss of K1 power, the router, Mac Wi-Fi, MQTT control or the backend does not +prove whether K1 is physically scanning. Retained points, an open TCP port and a +historical START are insufficient. Replaying START or STOP after an ambiguous +dispatch boundary can create a second physical edge. + +The durable physical-command ledger, exact read-only classification and +fail-closed supervisor must remain. They must not make ordinary connection slow +or surprising. In particular, selecting a device must not secretly connect, +Verify, retire/reopen history or delay network credentials. + +## Decision + +### Existing product surface + +K1 connection stays in the existing device plugin section headed +**Подключение XGRIDS LixelKity K1**. The surrounding job, entity and lifecycle +models do not change. This is novelty A: an improvement to an existing product +surface. A separate wizard, modal flow and mandatory preflight/recovery surface +are rejected. + +The section reuses canonical shared `Button`, `TextField`, `ActivityIndicator` +and `StatusBadge`. It creates no shared entity and uses no raw local HTML +controls or literal local status colors. + +### One-intent normal flow + +The normal flow is: + +1. choose Bridge, Direct Connect or Quick Connect locally; +2. press the explicit Bluetooth search action; +3. wait for exactly one six-second discovery; +4. press **Выбрать** on one result; +5. enter Bridge/Direct credentials immediately, or review the Quick Connect + summary; +6. press **Применить** once. + +Opening the section, changing mode, selecting a row and every +SSID/password keystroke perform zero browser-controller, device or host I/O. +They create no backend operation and show no operation loader. An admitted fresh +selection retains the selected card and exposes applicable inputs immediately. +A candidate without current draft authority is omitted or presented only as +non-actionable evidence; it never receives a misleading disabled primary. + +Each explicit search owns exactly one bounded six-second discovery. It performs +no connect, Verify, selection or mutation. Results are never auto-selected. + +One Apply owns the normal connection intent. It may commit the local desired-mode +draft under backend CAS and may cross at most one reviewed K1 mutation boundary. +Its frontend handler performs no hidden Scan, Verify, reconnect, retirement, +reopen, candidate substitution or retry. Quick, Bridge and Direct use the same single primary +**Применить** action; credentials are required only for Bridge and Direct. + +Ordinary Bridge Apply never opts into changing the controlling Mac's Wi-Fi +association. Host switching is a separate future consequential operator action, +not an Apply substep. K1 provisioning can therefore succeed as +`network_applied` while control is `control_not_ready`. That result must not +repeat that intent's BLE write. Recommended separately explicit read-only +Verify/recovery may establish route, endpoint and DeviceInfo/control evidence +for the applied topology; a new intent remains separately policy-gated. + +The exact REST response owns completion of the Apply mutation. A snapshot with +`connection_attempt.phase=network_applied` is accepted immediately when +`control_state` is `control_not_ready` or `unknown`; the controller does not +wait for WebSocket/poll convergence or call the full connection-ready +requirement. The service may continue supervised same-intent control bootstrap +after this fast durable ACK, but only read-only: no BLE/host mutation, mutation +retry, new UI action or second Apply. This is not a hidden frontend Scan or +Verify. Exact connection-ready remains mandatory before control or physical +START. This separation spends the old intent before a delayed state channel +could invite its duplicate replay. `connection_attempt` is a read model, not +permanent lifecycle authority; current server policy may admit a separately +explicit new intent. + +While the exact service-owned bootstrap child is `accepted` or `running` and +projects `safe_next_action=wait-for-current-attempt`, the UI shows only one +passive **Сеть настроена · подтверждаем управление** indicator. It enables no +Verify, mode change, Scan, row or Apply action. Terminal unready/unknown child +state then exposes the separately explicit policy-gated recovery choices. + +`network_applied` plus unready or unknown control spends the old Apply and gates +ordinary mode change, Scan, row selection and Apply. It first waits passively +for an exact active service child; after terminal settlement it presents an +explicit recovery choice, regardless of browser-local mode. It never authorizes +automatic or same-intent replay. Recommended Verify is pinned to the backend +`serverBound` current/configured transport and mode; it never falls back to a +selected browser row and is not a prerequisite for every new intent. + +Current server policy may admit a distinct, explicit new-intent path. Bridge +uses `prepare-select-device`, a local-only CAS with zero device/host I/O; only +after its success may the operator initiate a fresh six-second Scan. Quick and +Direct use explicit policy-gated `scan-ble`, then the latest fresh row and a new +idempotency Apply. A mode change requires backend `mode_selection` authority and +then a fresh explicit Scan. No recovery choice performs hidden Scan, selection, +Verify, provisioning or continuation of the old Apply, and the browser never +manufactures authority. + +### Freshness and outcome semantics + +Apply is admitted only for the exact selected transport, completed discovery +generation, backend runtime, desired-mode revision, reconfiguration intent and +policy snapshot. Authority drift before dispatch is a terminal, zero-device-I/O +`stale` result. The UI keeps the result understandable, labels it explicitly and +offers a new explicit six-second search. It never starts that search itself. + +A failure before the reviewed mutation boundary is `not-dispatched` or +`failed`, with zero K1 mutation. A lost response, timeout, power failure or +process death after dispatch is `outcome-unknown`, with +`safe_to_retry=false`. The durable network-attempt ledger prevents replay. +Credentials are never reused automatically. A later operator Apply is a new +intent and must pass all current gates. + +## Physical safety remains separate + +Network attempts are disposable; physical START/STOP ambiguity is durable: + +- START and STOP never replay automatically; +- control loss does not prove scanning stopped; +- local receiver/camera/ingress cleanup is not physical STOP; +- a wrong K1/transport/profile/project cannot reconcile the record; +- READY records cessation without rewriting historical command outcome; +- exact same-project SCANNING may mint one single-use confirmed STOP permit on + the still-open exact control binding; +- accepted STOP without READY or SCAN_STOPPING by the backend deadline closes + only host-owned resources, records `timed_out` / `standby-unknown`, preserves + the unresolved ledger and keeps every mutation fenced. + +The composite supervisor and physical-command ledger can disable Apply before +device I/O. Their denial does not turn mode, selection or input into recovery. + +### Explicit read-only recovery + +Recovery is a distinct, explicitly requested exceptional action. It is never a +continuation of row selection or Apply. The browser supplies neither endpoint, +substitute transport nor ledger authority. The backend pins the durable record's +exact transport, identity/profile, operation/revision, acquisition/project, +topology revision and host epoch. + +The non-reconnecting observation is: + +```text +topology-probed + -> pre-start-control-opened + -> device-info-requested (ordinal 1; exactly one publish) + -> device-info-verified + -> awaiting-passive-fresh-status + -> cessation | active-same-project | foreign-active | inconclusive | failed +``` + +It publishes exactly one canonical DeviceInfo request and then accepts only a +fresh non-retained DeviceStatus from the same socket generation after that +barrier. It publishes no status solicitation, DeviceConfig, time sync, +workspace, project, START or STOP; it never scans, reconnects, provisions or +continues into Apply. + +Canonical READY records cessation/standby. Initialized SCANNING may rebind only +when operation/acquisition, identity/profile, transport, host epoch and project +all match; it exposes one separate single-use confirmed STOP checkpoint. +Foreign, stale or inconclusive evidence changes no topology or authority. + +### Explicit retirement and reopen + +`physical-command.retire-unavailable` is a separately confirmed local durable +recovery action for one unresolved target that is truly unavailable or replaced. +Admission requires stable idempotency identity and exact backend runtime, +operation, ledger revision and transport CAS plus safe lifecycle ownership. It +preserves the original unknown outcome, activates the exact-transport deny, +performs zero device/host I/O and starts no discovery. + +`physical-command.reopen-retired-reconciliation` is also separately confirmed. +It requires an exact fresh same-transport candidate, stable `reopening_id`, exact +runtime/revision/retirement/transport/discovery CAS and safe lifecycle ownership. +It preserves retirement audit, removes only that retirement's active deny and +performs zero device/host I/O. The explicit recovery intent may then run one +exact read-only observation. **Выбрать** never invokes retirement, reopen or +Verify. The only Apply exception is an internal, request-bound local reopen +checkpoint for an explicit scenario reset plus its exact successor Scan. It is +ordered after network PREPARED and before the sole dispatch edge, remains +invisible in the wizard and grants no command authority. The same applied +intent may then settle it read-only from fresh DeviceInfo plus non-retained +READY/SCANNING evidence. + +FW 3.0.2 BLE `7f02` contains no stable DeviceInfo identity. Mission Core cannot +prove during BLE-only discovery that the same physical unit has a new +CoreBluetooth UUID. This remains an explicit protocol/hardware gap. + +### Bounded durable audit rollover + +An explicit local scenario reset must not become unavailable merely because +closed retire/reopen history filled the 64 KiB hot ledger. Before a transition +would exceed that bound, Mission Core durably publishes the complete previous +ledger as a private, owner-only, content-addressed archive segment and then +atomically publishes a compact v4 main record. The main record retains every +active retirement deny, the newest lost-response retire/reopen checkpoint, and +all reconciliation/confirmation proof required by the current physical +operation. Compaction never changes a device outcome and performs no device, +network or host I/O. + +Archive segments form a predecessor hash chain with exact sequence and byte +accounting. Reload verifies directory and file ownership/mode, rejects symlink +traversal, bounds total segments and bytes, reparses every embedded ledger and +fails closed for a missing, replayed, reordered or tampered segment. Operation, +reconciliation, verification, confirmation, retirement and reopening identities +remain globally one-use across the hot record and archive. The archive segment +is fsynced before the main-file replace: a crash may leave only an inert orphan, +while retry of the same CAS reuses identical bytes and cannot duplicate the +referenced chain. + +Scenario reset asks the ledger to build the exact prospective retirement or +prepared→not-dispatched plan before closing any local receiver, camera, +control-session or network ownership. That shared planner applies the same hot +serialization, compaction, segment, count and total-byte bounds as commit. When +rollover is required, preflight may idempotently prepublish only the immutable +content-addressed predecessor; the main revision/CAS and physical disposition +remain unchanged. This also proves owner/mode, symlink and content-collision +conditions before teardown. + +Archive publication is restart-safe at the hard-link boundary. A process death +after destination link and directory fsync but before temporary-name unlink may +leave exactly two private names for one inode. Retry removes only a strictly +named, owner-only temporary alias whose bytes and inode exactly match the +expected destination and whose link count is exactly two, fsyncs that cleanup, +then reuses the destination. Any unrelated hard link, extra temporary, symlink, +metadata mismatch or byte mismatch remains a fail-closed corruption condition. + +## Failure and restart semantics + +- UI entry, mode, selection, input, polling, refresh and layout changes + start no device operation. +- Search starts only when pressed, runs once for six seconds and terminalizes. +- Apply starts only when pressed, uses one exact fresh candidate and may perform + at most one K1 mutation. +- Candidate/runtime/intent drift is explicit stale, never hidden rescan. +- Post-dispatch uncertainty is explicit outcome-unknown, never automatic replay. +- K1 power loss revokes the active session without inventing standby. +- Wi-Fi loss and WAN loss are distinct: local LAN control may survive WAN loss; + route/association loss revokes only dependent host/control evidence. +- Browser refresh restores no live local selection and causes no I/O. +- Backend restart restores durable audit and safety ledgers, but no live BLE, + control or operator intent. +- Mac sleep/restart rotates host/runtime authority and rejects late work. + +## Acceptance + +- Mode, selection and input result in zero controller calls. +- Each Search click issues exactly one scan with duration `6`; no effect, timer, + selection or Apply path calls Scan. +- Every result keeps the same ordinary **Выбрать** action. Selection retains + the card, shows applicable inputs immediately, shows no loader and calls no + controller. After an explicit committed scenario reset and its successfully + completed successor Scan, this includes the exact UUID used by the retired + prior scenario; the row never exposes a reconnect/reopen/Verify CTA. +- During unresolved physical recovery, a completed explicit Scan still renders + passive BLE evidence but cannot substitute a foreign target for the durable + recovery record. Exact recovery remains a separate established-session + action outside the cold result list; ordinary **Выбрать** never invokes its + reopen or read-only Verify. A new network flow first requires explicit reset + and a successor Scan. +- Bridge/Direct show SSID/password; Quick Connect does not. +- Exactly one primary **Применить** owns the connection request. Its frontend + handler calls no Scan/Verify/reopen helper and it permits at most one device + mutation. For an exact reset-owned retired UUID, the backend may append only + the internal local settlement checkpoint described above before dispatch. + A later SCANNING settlement grants only explicit STOP authority and never + restarts the reset-owned receiver, camera, writer or acquisition. +- Stale/pre-dispatch and unknown/post-dispatch outcomes are visibly distinct. +- Applied-but-unready/unknown spends the old Apply and gates ordinary mode, + Scan, selection and Apply behind an explicit recovery choice; recommended + Verify has only a server-bound backend target and no browser fallback. +- A new intent remains possible only through current backend policy. Bridge + uses explicit local-only `prepare-select-device`; Quick/Direct use an explicit + admitted Scan and latest fresh row; mode change requires `mode_selection`. + Each route starts no hidden frontend or mutating continuation and ends in a + later fresh Scan/new idempotency Apply. The declared service-owned + same-intent read-only bootstrap after the durable ACK is the sole continuation + exception and creates no UI action. +- The exact Apply REST snapshot with `phase=network_applied` completes the + network intent for both `control_not_ready` and `unknown`, without requiring + connection-ready or waiting for WebSocket/poll convergence. +- A service-owned supervised control bootstrap may continue read-only after + that ACK. It performs no BLE/host mutation or retry and creates no frontend + Scan/Verify/new-Apply action or blocking Apply loader. Its exact + accepted/running state may own one passive settling indicator only. +- Operator error copy comes only from an allowlisted public error-code mapping; + unknown/raw messages use a canonical secret-free fallback and never render + credentials, SSIDs, payloads or stack traces. +- No timeout, disconnect, refresh, restart or state update starts a continuation + or replays an ended action. +- Supervisor, identity pin, network-attempt ledger, physical-command ledger, + process/BLE lease and one-use recovery STOP remain authoritative. +- The plugin uses shared `Button`, `TextField`, `ActivityIndicator` and + `StatusBadge`; contract tests reject raw local controls and literal colors. +- Geometry and long-copy tests keep all actions reachable without overlap. +- Bridge and Quick Connect retain separate real-hardware acceptance. + +This ADR does not itself declare hardware coverage. The manifest may mark a +scenario software-covered only when named executable tests cover the software +invariant; remaining K1/macOS/router and Quick Connect gaps stay explicit. diff --git a/docs/k1-connection-acceptance.manifest.json b/docs/k1-connection-acceptance.manifest.json new file mode 100644 index 0000000..a81a35f --- /dev/null +++ b/docs/k1-connection-acceptance.manifest.json @@ -0,0 +1,93 @@ +{ + "schema_version": "missioncore.k1-connection-acceptance/v1", + "canonical_document": "docs/20_K1_CONNECTION_SUPERVISION_CANON.md", + "meaning": { + "software-covered": "The listed automated tests cover the software invariant; this is not hardware acceptance.", + "partial": "At least one software layer is covered and an explicit remaining gap is listed.", + "planned": "The scenario is specified but does not yet have adequate executable coverage." + }, + "scenarios": [ + {"id":"CONN-01","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs"],"remaining":["real K1 Quick-to-Bridge evidence"]}, + {"id":"CONN-02","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs"],"remaining":["real K1 Bridge-to-Quick evidence"]}, + {"id":"CONN-03","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["hardware pre-dispatch fault injection"]}, + {"id":"CONN-04","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["hardware pre-dispatch fault injection"]}, + {"id":"CONN-05","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_device_identity_pin_store.py"],"remaining":["two-K1 hardware evidence"]}, + {"id":"CONN-06","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_ble_scanner.py","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["CoreBluetooth hardware evidence"]}, + {"id":"CONN-07","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real K1 wait-beyond-TTL acceptance"]}, + {"id":"CONN-08","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real app/backend restart reconnect acceptance"]}, + + {"id":"CONN-10","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["hardware pre-prepare power-loss fault injection"]}, + {"id":"CONN-11","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["process-kill acceptance at the prepared boundary"]}, + {"id":"CONN-12","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real K1 post-dispatch power-loss acceptance"]}, + {"id":"CONN-13","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real K1 observation-loss evidence"]}, + {"id":"CONN-14","status":"software-covered","test_files":["tests/test_xgrids_semantic_topology_store.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["hardware host-association loss"]}, + {"id":"CONN-15","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["hard-power hardware evidence"]}, + {"id":"CONN-16","status":"partial","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_physical_command_ledger.py"],"remaining":["end-to-end passive Scan policy after acquisition power loss","exact-target read-only recovery/rebind integration","Bridge hardware power-loss acceptance","Quick Connect recovery not exercised"]}, + {"id":"CONN-17","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py"],"remaining":["transport dispatch integration","restart acceptance"]}, + {"id":"CONN-18","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_application_mqtt.py"],"remaining":["facade exact-target resolved-active rebind","single-use explicit recovery STOP presentation/action integration","same-project Bridge hardware acceptance","Quick Connect recovery not exercised"]}, + {"id":"CONN-19","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_application_mqtt.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["facade exact-target resolved-active READY cessation integration","Bridge reboot hardware acceptance","Quick Connect recovery not exercised"]}, + + {"id":"CONN-20","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_macos_wifi.py"],"remaining":["router-loss hardware evidence"]}, + {"id":"CONN-21","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_macos_wifi.py"],"remaining":["same-SSID router-return evidence"]}, + {"id":"CONN-22","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_macos_wifi.py"],"remaining":["manual macOS switch evidence"]}, + {"id":"CONN-23","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real Quick AP leave/return"]}, + {"id":"CONN-24","status":"partial","test_files":["tests/test_connection_supervisor.py","tests/test_ble_scanner.py"],"remaining":["macOS sleep/wake hardware acceptance"]}, + {"id":"CONN-25","status":"software-covered","test_files":["tests/test_connection_supervisor.py"],"remaining":["route-race integration evidence"]}, + {"id":"CONN-26","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["wrong-service integration evidence"]}, + {"id":"CONN-27","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_application_mqtt.py","tests/test_xgrids_application_session.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["half-open MQTT hardware acceptance"]}, + {"id":"CONN-28","status":"partial","test_files":["tests/test_xgrids_application_mqtt.py","tests/test_xgrids_physical_command_coordinator.py","tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["full facade policy for passive Scan with unknown/active physical state","physical-record transport_ref pinning across bounded observation and wrong-K1 no-topology-change","resolved-active SCANNING one-STOP integration","real K1 passive READY/SCANNING DeviceStatus acceptance"]}, + {"id":"CONN-29","status":"planned","test_files":[],"remaining":["durable external-active takeover contract","operator-confirmed same-binding STOP"]}, + + {"id":"CONN-30","status":"planned","test_files":[],"remaining":["browser/app close clean-session acceptance at every stage"]}, + {"id":"CONN-31","status":"software-covered","test_files":["tests/test_xgrids_network_mutation_ledger.py","tests/test_xgrids_semantic_topology_store.py"],"remaining":["restart integration acceptance"]}, + {"id":"CONN-32","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real backend restart acceptance from a prepared network mutation"]}, + {"id":"CONN-33","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real backend restart acceptance from a dispatching network mutation"]}, + {"id":"CONN-34","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real backend restart acceptance from an observing network mutation"]}, + {"id":"CONN-35","status":"software-covered","test_files":["tests/test_xgrids_network_mutation_ledger.py","tests/test_xgrids_semantic_topology_store.py"],"remaining":["restart integration acceptance"]}, + {"id":"CONN-36","status":"planned","test_files":[],"remaining":["restart acceptance proving no old live session restoration"]}, + {"id":"CONN-37","status":"planned","test_files":[],"remaining":["corrupt historical audit quarantine without permanent K1 block","operator diagnosis UI"]}, + {"id":"CONN-38","status":"software-covered","test_files":["tests/test_xgrids_network_mutation_ledger.py","tests/test_xgrids_ble_runtime_arbiter.py"],"remaining":["two-service integration acceptance"]}, + {"id":"CONN-39","status":"software-covered","test_files":["tests/test_xgrids_network_mutation_ledger.py"],"remaining":[]}, + + {"id":"CONN-40","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_application_session.py"],"remaining":["real wrong/failed DeviceInfo evidence"]}, + {"id":"CONN-41","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["MQTT fault-injection integration"]}, + {"id":"CONN-42","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["camera and point stall integration"]}, + {"id":"CONN-43","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["late packet integration evidence"]}, + {"id":"CONN-44","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_application_session.py"],"remaining":["late DeviceInfo integration evidence"]}, + {"id":"CONN-45","status":"partial","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_application_control_process_lease.py"],"remaining":["durable physical-command integration"]}, + {"id":"CONN-46","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_camera_gateway.py"],"remaining":["combined MQTT/camera late-producer integration"]}, + {"id":"CONN-47","status":"software-covered","test_files":["tests/test_xgrids_application_control_process_lease.py"],"remaining":["two-service integration acceptance"]}, + {"id":"CONN-48","status":"software-covered","test_files":["tests/test_xgrids_camera_gateway.py"],"remaining":["drain-timeout integration evidence"]}, + {"id":"CONN-49","status":"software-covered","test_files":["tests/test_connection_supervisor.py"],"remaining":["long-running fault-injection acceptance"]}, + + {"id":"CONN-50","status":"partial","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["macOS sleep/wake hardware acceptance"]}, + {"id":"CONN-51","status":"software-covered","test_files":["tests/test_xgrids_macos_wifi.py","tests/test_connection_supervisor.py"],"remaining":["compiled association observer"]}, + {"id":"CONN-52","status":"software-covered","test_files":["tests/test_xgrids_device_identity_pin_store.py","tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["two-identity hardware evidence"]}, + {"id":"CONN-53","status":"software-covered","test_files":["tests/test_xgrids_semantic_topology_store.py","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["restart browser acceptance"]}, + {"id":"CONN-54","status":"planned","test_files":[],"remaining":["historical unknown audit does not block fresh explicit connect","restart browser acceptance"]}, + {"id":"CONN-55","status":"partial","test_files":["apps/control-station/test/devicePluginFrontendBoundary.test.mjs"],"remaining":["automated browser geometry matrix"]}, + {"id":"CONN-56","status":"partial","test_files":["apps/control-station/test/devicePluginFrontendBoundary.test.mjs"],"remaining":["automated long-copy browser geometry"]}, + {"id":"CONN-57","status":"planned","test_files":[],"remaining":["operator-confirmed physical-ledger archive and identity rotation"]}, + {"id":"CONN-58","status":"software-covered","test_files":["tests/test_xgrids_ble_runtime_arbiter.py","tests/test_xgrids_application_control_process_lease.py","tests/test_ble_scanner.py","tests/test_wifi_provisioning.py","tests/test_xgrids_ap_activation.py"],"remaining":["two-service CoreBluetooth hardware acceptance","native cleanup fault injection on macOS"]}, + {"id":"CONN-59","status":"partial","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_network_provisioning_idempotency_journal.py"],"remaining":["prove failed audit admission releases active ownership for a new explicit attempt"]}, + {"id":"CONN-60","status":"software-covered","test_files":["apps/control-station/test/devicePluginContracts.test.mjs"],"remaining":["manual browser confirmation-dismissal acceptance"]}, + {"id":"CONN-61","status":"planned","test_files":[],"remaining":["legacy unresolved record terminalization without BLE or cross-session block","process-kill acceptance"]}, + {"id":"CONN-62","status":"software-covered","test_files":["tests/test_web_validation_security.py"],"remaining":["manual browser refresh/close acceptance"]}, + {"id":"CONN-63","status":"planned","test_files":[],"remaining":["composite policy denies active contention but ignores terminal historical network audit","manual policy presentation acceptance"]}, + {"id":"CONN-64","status":"software-covered","test_files":["tests/test_connection_supervisor.py","tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs"],"remaining":["real K1 control/data loss acceptance"]}, + {"id":"CONN-65","status":"partial","test_files":["tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["resolved-active same-project one-STOP UI action acceptance","real restart/browser host-route and passive DeviceStatus acceptance","Quick Connect recovery not exercised"]}, + {"id":"CONN-66","status":"planned","test_files":[],"remaining":["compiled or long-lived macOS association observer","long-running monitor latency/fault acceptance"]}, + {"id":"CONN-67","status":"partial","test_files":["tests/test_xgrids_acquisition_lifecycle.py"],"remaining":["real K1 repeated same-mode and cross-mode reconnect acceptance"]}, + {"id":"CONN-68","status":"partial","test_files":["tests/test_xgrids_application_session.py","tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["Bridge STOP-ack plus Wi-Fi-loss hardware rerun","Quick Connect recovery not exercised"]}, + {"id":"CONN-69","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["resolved-active same-project one explicit STOP browser acceptance","Bridge hardware rerun with redacted evidence","Quick Connect recovery not exercised"]}, + {"id":"CONN-70","status":"software-covered","test_files":["apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real Bridge one-scan/select/immediate-credentials/Apply acceptance","Quick Connect recovery not exercised"]}, + {"id":"CONN-71","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real Bridge STOP-deadline fault injection","Quick Connect recovery not exercised"]}, + {"id":"CONN-72","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_plugin_runtime.py","apps/control-station/test/devicePluginContracts.test.mjs","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real Bridge existing-plugin-section acceptance","manual two-tab browser acceptance","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-73","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_semantic_topology_store.py","tests/test_xgrids_device_identity_pin_store.py"],"remaining":["real cold Bridge and two-K1 identity-mismatch/restart evidence","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-74","status":"software-covered","test_files":["tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_network_mutation_ledger.py","tests/test_xgrids_network_provisioning_idempotency_journal.py","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real Bridge Apply with no hidden discovery/Verify","post-dispatch hardware fault injection","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-75","status":"software-covered","test_files":["tests/test_xgrids_connection_scenario_reset.py","tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_application_control_process_lease.py","apps/control-station/test/devicePluginFrontendBoundary.test.mjs"],"remaining":["real disconnected/idle desired-mode draft no-I/O acceptance with unresolved durable physical history plus live-owner denial","real pre-START orphan and backend-runtime credential invalidation acceptance","manual top-right emergency-reset acceptance","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-76","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_physical_command_coordinator.py","tests/test_xgrids_acquisition_lifecycle.py","tests/test_xgrids_application_control_process_lease.py","tests/test_xgrids_ble_runtime_arbiter.py","tests/test_xgrids_camera_gateway.py","tests/test_cli.py","tests/test_plugin_runtime.py","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real separate explicit retirement confirmation while selection and Apply remain mutation-free","same-hardware/new-CoreBluetooth-UUID cannot be identified before provisioning because FW 3.0.2 BLE 7f02 exposes no stable DeviceInfo identity","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-77","status":"partial","test_files":["tests/test_xgrids_physical_command_ledger.py","tests/test_xgrids_physical_command_coordinator.py","tests/test_xgrids_acquisition_lifecycle.py","tests/test_plugin_runtime.py","apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["real retired exact-UUID selection remains local and Apply denied","real separately explicit reopen followed by READY and same-project SCANNING outcomes","Quick Connect live acceptance remains separate"]}, + {"id":"CONN-78","status":"software-covered","test_files":["apps/control-station/test/devicePluginFrontendBoundary.test.mjs","apps/control-station/test/k1SupervisorPresentation.test.mjs"],"remaining":["manual Bridge and Quick one-intent timing acceptance","manual top-right idle/pending accessible-label acceptance","real stale-before-dispatch and unknown-after-dispatch fault injection","real fast REST network_applied plus delayed service-owned read-only control-bootstrap convergence","real Bridge prepare-select-device and Quick/Direct scan-new-intent recovery acceptance","manual canonical shared-control visual acceptance"]} + ] +} diff --git a/docs/runbooks/K1_CONNECTION_RECOVERY.md b/docs/runbooks/K1_CONNECTION_RECOVERY.md new file mode 100644 index 0000000..ff516d3 --- /dev/null +++ b/docs/runbooks/K1_CONNECTION_RECOVERY.md @@ -0,0 +1,393 @@ +# K1 connection lifecycle and recovery runbook + +Canonical model: [`../20_K1_CONNECTION_SUPERVISION_CANON.md`](../20_K1_CONNECTION_SUPERVISION_CANON.md). +This runbook is the operator-facing projection of that model. Technical terms in +the internal-safety sections are engineering evidence; they are not wizard copy. + +## One operator wizard + +The connection surface is one progressive wizard, not a recovery dashboard. +Its only model-bearing heading is **Подключение XGRIDS LixelKity K1**. Inside +the wizard the two step names are exactly **Подключение** and **Сеть**. + +### Cold entry + +On a clean cold entry show only: + +- the connection-mode selector; +- Step 01 **Подключение** with the explicit Bluetooth search action. + +Historical K1 identity never adds a reconnect choice to cold entry. If local +session ownership or an older connection scenario exists, one explicit +`reset_scenario` CAS first closes only that local scenario. Only after the reset +is accepted does clean Step 01 expose **Найти по Bluetooth** as a separate +click; reset never starts Scan itself. + +Do not render Step 02 yet and do not start discovery automatically. The mode +selector and same-mode **Подключить новый K1** escape remain available through +every other lifecycle state. The first gesture may supersede another pending +local action. While that one bounded scenario reset itself owns `mode`, all +three reset entry points show pending and dispatch no B intent. Each accepted +gesture sends one idempotent local `reset_scenario` CAS: it queues behind old +local lifecycle ownership, seals retained receiver/camera/control resources, +invalidates candidates/drafts/credentials and retires the old physical lineage +without resolving its outcome. It sends no BLE, device or host network command, +MQTT publish, Verify, provisioning, START, STOP or automatic Scan. Opening and +polling the surface still do nothing. Scan, Verify, provisioning and START +retain separate backend gates. + +The top-right refresh-shaped utility is the same explicit emergency reset, not +a passive state refresh. Its accessible label is **Сбросить подключение**; while +the request owns the current action it reads **Сбрасываем подключение** and +does not dispatch a second reset until that bounded request settles. It remains +available to supersede any other local action. The accepted revision always +returns the new local scenario and even a dirty browser selector to canonical +**Bridge**, clears the old browser/backend presentation and source owners, and +leaves Scan as a separate click. It performs no hidden Scan, Verify, Connect, +START, STOP, BLE or network write and never substitutes a passive `state.read` +for the reset mutation. After settlement the product surface contains no prior +UUID, result count, **Повторить поиск**, reconnect error, selected device, +credentials or recovery card; it returns to **Найти по Bluetooth**. Late +Scan/Verify settlements from the retired scenario cannot repopulate it. +The retained reset marker fences only work that belonged to the retired +scenario. A newly correlated post-reset network attempt that fails or has an +unknown outcome must immediately render its current recovery/error surface; +reload must neither hide that new failure nor resurrect the prior prompt. + +### Step 01 — Подключение + +Step 01 **Подключение** is visible immediately. + +1. Bluetooth discovery starts only after the operator presses the search + action. +2. For the full bounded search, show an activity indicator and a visible + seconds countdown in the same step. +3. After search completes, show the result count or the empty result. Every + connectable result row keeps the same one enabled **Выбрать** action, + including an exact UUID used in an earlier scenario. That action is + local-only and never invokes reopen or Verify. Fresh results never render a + reconnect CTA, a disabled competing primary or an old/new-device decision. + A successful admitted Scan first settles only the reset marker whose + id/revision/mode it captured at action entry, recording the later admitted + discovery generation while retaining the marker for idempotent reset replay. + Failed/cancelled Scan and an older Scan racing a newer pending reset leave + the marker active and do not expose stale recovery authority. +4. After ordinary **Выбрать**, retain the chosen device card and reveal only + the applicable local draft inputs. Selection itself has no loader and makes + no controller call. +5. Only an authoritative successful connection outcome renders Step 01 green + as **Подключение установлено** and advances the normal connection flow. + +The wizard never labels a candidate as saved, original, retired or physically +ambiguous and never exposes ledger, CAS, retirement, reopen or reconnect +terminology in the search list. Exact recovery belongs only to a previously +established session after an actual interruption. + +### Step 02 — Сеть + +Step 02 **Сеть** exists only after Step 01 has a confirmed green connection. + +- If the selected device is already usable on the chosen connection path, + show the network result without asking for credentials. +- If backend policy has safely admitted explicit network setup, show the exact + retained device context and SSID/password fields here, never beside the + candidate list. +- One explicit submit owns any exact hidden revalidation and at most one + reviewed network write. +- **Изменить сеть** belongs only to this step and starts no Bluetooth search + when its form opens. +- A stale tab, changed runtime/binding or policy denial fails before a write and + never exposes a foreign candidate. + +## Ordinary selection + +The ordinary **Выбрать** action is presentation simplification, not relaxed +safety. It creates only a browser-local candidate draft and performs no +controller I/O. It never selects an internal recovery path: + +- an ordinary fresh candidate becomes only the selected local draft; +- an exact prior candidate uses the same **Выбрать** action as every row; +- no candidate selection retires old authority, reopens a ledger record, calls + Verify, connects GATT, scans again or changes topology or the device; +- a foreign, stale, non-connectable or policy-denied candidate remains + unavailable and changes no topology, ledger or device. + +After an explicit committed scenario reset, only its successfully completed +successor Scan may make an exact previously retired transport eligible for a +new network draft. Apply still captures that exact current-generation handle, +validates the live GATT baseline and crosses at most one reviewed write edge. +Selection performs no recovery action. At the final Apply boundary, the backend +may append one exact request-bound local reopen checkpoint after network +PREPARED and before write dispatch. That checkpoint preserves physical +retirement/original-outcome audit, performs no device I/O and authorizes no +START or STOP. The same applied intent then uses fresh DeviceInfo plus a +non-retained DeviceStatus to settle READY as standby or identity-bound SCANNING as +active, without a visible Verify step or command replay. In the SCANNING case +it materializes only explicit STOP authority; it does not restart the retired +receiver, camera, evidence writer or acquisition. + +### Exact internal recovery for an established session + +For an exact target belonging to a previously established session with one +active retirement, +`physical-command.reopen-retired-reconciliation` requires: + +- `operator_confirmed=true`, bound to a separately explicit session-recovery + action outside cold entry and the Bluetooth result list; +- a stable `reopening_id` and reason + `device-returned-for-explicit-reconciliation`; +- exact `expected_snapshot_runtime_id`, `expected_revision`, + `expected_retirement_id`, `expected_transport_ref` and + `expected_discovery_generation` CAS; +- a current connectable candidate and safe lifecycle/process ownership. + +The transaction changes only the local durable ledger. It appends reopen audit, +preserves the retirement and unknown command outcome as history, restores the +original unresolved `dispatching` or `observing` stage and removes only that +retirement's active deny. It performs zero BLE, Wi-Fi, MQTT, DeviceConfig, +ModelingStatus, workspace, project, START or STOP I/O and starts no Scan. + +The same still-current session-recovery action may then own one exact read-only +Verify. Ordinary **Выбрать** never invokes either half. Recovery never replays +historical START/STOP and never silently provisions: + +- fresh non-retained READY resolves standby; +- fresh exact same-project SCANNING resolves active and permits only the + separately guarded stop path; +- identity, GATT, CAS, route/control or policy failure leaves the outcome + unknown and ends the established-session recovery without entering the new + connection wizard. + +If the action response is lost, refreshed state may continue the same click +only when it proves that exact `reopening_id` audit was committed and every +original runtime, candidate and authority fence still matches. A second tab, +new discovery generation, new retirement or different reopening identity cannot +inherit the continuation. + +### Internal retirement + +`physical-command.retire-unavailable` is a local durable primitive for an +unresolved target that is truly unavailable or replaced. It may run only from +its separately confirmed recovery/reset path, never from ordinary candidate +selection, UI entry, polling or a timer. Admission requires explicit +confirmation, stable `retirement_id` and exact backend runtime, +operation, revision and transport CAS while every local owner is safe. + +Retirement preserves the complete old attempt and unknown command outcome, +activates an exact-transport deny and performs zero device I/O or automatic +discovery. Retirement history remains durable even if an exact later recovery +action uses the reopen transaction. The wizard exposes no retirement +transaction or historical label. Any plain-language exact recovery CTA belongs +only to the established-session surface when backend authority permits it. + +Current FW 3.0.2 BLE `7f02` does not expose stable DeviceInfo identity. The same +hardware under a new CoreBluetooth UUID cannot be recognized before DeviceInfo +becomes available. This remains an explicit protocol/hardware acceptance gap; +the wizard must not speculate. + +## Session and freshness rules + +A scan result is an unselected presence candidate owned by the latest explicit +scan generation. Wall-clock age does not remove its row while the operator is +reading or completing the form. A successor Scan, explicit scenario reset, +runtime-owner teardown or proven exact-target GATT failure invalidates it. The +row itself is never network authority: Apply still requires the exact captured +CoreBluetooth object and live GATT validation before any write. + +The selected session ends on proven disconnect, explicit lifecycle stop, a +committed network transition, selection of another device, backend restart or +proven native cleanup. A later connection always requires an explicit search +and **Выбрать**. Polling can update presentation but starts neither operation. + +Bridge, Quick Connect and Direct Connect are separate topologies. In +any state, changing the mode or choosing another K1 in the same mode sends one +local scenario-reset CAS. It can wait for and supersede live/recovery ownership, +seal retained local producers and retire unresolved old lineage, but performs +zero device/host I/O and starts no Scan. Scan, Verify, provisioning and START +remain independently fenced until an explicit candidate intent crosses its +reviewed transition. No old host route, endpoint, control, data or BLE authority +crosses a committed reset boundary. + +## Active scanning: transient host-path recovery + +This is the sole automatic read-only rebind exception. It exists only after +Mission Core itself has a composite-confirmed START and still owns the exact +acquisition/runtime/device/connection/evidence lineage. It does not apply on a +cold connection screen, after backend restart, to an external SCANNING K1 or to +an unresolved/foreign START. + +When the Mac loses Wi-Fi/route or the data socket while that acquisition is +running, the active scanning pane shows a neutral spinner and +**Восстанавливаем соединение** with attempt/elapsed time. Do not show a red +terminal operation banner for the expected late failure of the superseded old +control socket. Keep the acquisition and evidence session owned while the +backend retries exact route/TCP and inspection-only DeviceInfo/status proof. + +The recovery loop never sends BLE, changes Wi-Fi, writes DeviceConfig, repeats +START or sends STOP. Outcomes are: + +- exact same-device/same-project initialized `SCANNING`: silently resume the + point stream/control binding and, when necessary, CAS-restart the dead or + stalled acquisition-owned right-camera FFmpeg epoch; +- fresh `READY`: interrupt/seal host-owned acquisition resources truthfully, + without STOP; +- fresh `SCAN_OVER`: persist cessation, interrupt/seal locally and retain a + read-only `awaiting READY` fence that denies a new START; +- wrong identity/same IP, changed lineage or failed camera CAS: remain blocked + for explicit operator handling; and +- device/system fault or unsafe status: show a truthful terminal fault, with no + command retry. + +While state is `reconnecting` or `blocked`, expose **Завершить локально**. The +action `acquisition.force-finish-local` requires the current snapshot runtime, +acquisition id/state revision, recovery generation, producer generation match, +an idempotency key and explicit confirmation. It cancels recovery first, then +seals only local receiver/camera/control/perception owners. It preserves the +physical START ledger and sends no STOP. If a connection-mode reset races this +action, the shared lifecycle gate makes cleanup idempotent; the loser cannot +overwrite the new mode or revive the old acquisition. + +If receiver, camera or evidence sealing fails, the recovery generation is +still irrevocably cancelled first. The force-finish operation ends with a +visible `local-cleanup-failed` result whose retryability applies only to local +finalization; the terminal acquisition retains `cleanup_pending` and blocks a +replacement session. A later explicit local stop or exact connection-scenario +reset may retry those host resources. It must not retry START, STOP, BLE or a +network write, and a late success from the retired recovery generation remains +fenced. + +## Failure matrix + +| Event | Product result | Operator path | +| --- | --- | --- | +| Cold entry | Mode plus Step 01 and explicit Scan; zero device I/O before Scan | Start search explicitly | +| Disconnected/idle mode or same-mode new-device request with unresolved durable physical history | Local session/audit lineage is retired under one reset CAS; zero device/host I/O and no automatic Scan | Start the clean Step 01 search explicitly; the old physical outcome remains auditable | +| Mode reset while live, reconnecting or terminal cleanup still owns local sources | Reset supersedes recovery and locally seals receiver/camera/control; previous K1 may still scan | Wait for the bounded local cleanup or retry the same reset if local sealing fails | +| Search running | Step 01 spinner and visible countdown | Wait or let the bounded search end | +| Search finds no candidates | Step 01 reports no matches | Repeat search explicitly | +| Search finds one or many candidates | Every connectable row has one enabled **Выбрать**, including the exact prior UUID | Select one row; no reconnect or recovery action appears in search results | +| Wall-clock time passes after Scan before selection | Latest-generation rows remain stable; no operation starts | Select normally; exact capture and live GATT will gate Apply | +| A new Scan/reset/runtime teardown or exact-target GATT failure invalidates the generation | Old rows disappear or the attempted action fails cleanly before mutation | Run one explicit new search if needed | +| Selection is rejected by identity, GATT, CAS, lifecycle or safety policy | Loader ends; nothing changed; Step 02 remains absent | **Повторить** or **Выбрать другое** | +| Selection completes exact device connection | Step 01 turns green | Continue in Step 02 **Сеть** | +| Network setup is safely required | Credentials appear only in Step 02 | Submit once | +| Network write becomes ambiguous after dispatch | Attempt ends unknown; no replay | Wait for cleanup, then create a distinct explicit attempt | +| Device powers off or BLE disconnects | Live selection and authority revoke after proof | Search and select explicitly after the device is available | +| Router, Mac Wi-Fi or MQTT control is lost while idle/pre-START | Host/control authority revokes; data may remain evidence only | Restore reachability, then use the same wizard flow | +| Mac Wi-Fi/route is briefly lost during one composite-confirmed owned acquisition | Active pane remains neutral **Восстанавливаем соединение**; no START/STOP/network retry | Wait for exact automatic read-only rebind or press **Завершить локально** | +| Active recovery returns READY or SCAN_OVER | Local receiver/camera seal without STOP; SCAN_OVER remains fenced until fresh READY | Start another scenario only after backend policy reports it safe | +| Active recovery sees another K1 on the same IP or changed lineage | Recovery blocks fail-closed; no camera/data resurrection | Finish locally or explicitly choose/reset connection scenario | +| A physical START/STOP edge is unresolved | Mutation stays fenced; no technical wizard ceremony | Search/select remains explicit; backend admits only a safe exact path | +| Exact actively retired UUID is present after committed reset and successor Scan | The row exposes the same enabled **Выбрать** as every candidate | Select locally; Apply remains exact-handle/live-GATT gated and may append one internal local settlement checkpoint before its sole write; audit remains append-only and START/STOP stay denied until fresh read-only classification | +| Another candidate is selected while old authority is unavailable and no reset-owned new scenario exists | Selection stays local and Apply remains denied | Start an explicit new connection scenario, then Scan and select again | +| Browser refresh or backend restart | No automatic operation and no restored live selection | Begin from the cold progressive wizard | + +## Physical START/STOP safety remains separate + +The simplified wizard never weakens physical-command safety: + +- loss of control does not prove that K1 stopped recording; +- START and STOP are never replayed automatically; +- local receiver/camera/ingress cleanup is not physical STOP; +- an ambiguous post-dispatch command remains unknown until exact fresh proof; +- read-only recovery is pinned to the durable transport, identity/profile, + host epoch and project; +- each observation publishes exactly one DeviceInfo request and may classify + only a fresh non-retained DeviceStatus after that barrier; +- READY records cessation without inventing a successful STOP; +- SCAN_OVER records cessation without inventing STOP, but keeps a durable + read-only fence until a later fresh unbound READY observation; +- exact same-project SCANNING may mint one single-use, separately confirmed STOP + checkpoint; it does not send STOP automatically; +- a wrong transport/device/project changes no topology or ledger state; +- accepted STOP without READY or SCAN_STOPPING by the backend deadline closes + only host-owned resources, yields `timed_out` / `standby-unknown`, preserves + the unresolved ledger and keeps mutation fenced. + +Engineering logs and state APIs retain these distinctions. The connection +wizard projects only the ordinary progressive flow and a non-technical terminal +selection result. + +## No automatic action rule + +None of these events may scan, select, reconnect, Verify, provision, START or +STOP: + +- opening or resizing the connection surface; +- backend event delivery or state polling; +- an acknowledged scenario reset (it may perform only its explicit local + retirement, never any listed device/network action or automatic Scan); +- candidate list refresh after an ended search; +- browser refresh, sleep/wake or backend restart; +- timeout, disconnect or a historical audit record. + +The only exception is the service-owned active-stream read-only rebind above. +It is triggered by the already-owned receiver's transport loss, not UI entry or +polling, and is limited to route/TCP, DeviceInfo/status inspection, receiver +resubscribe and exact local camera-epoch restart. It never performs discovery, +provisioning, START, STOP or any device/network write. + +Only the currently pressed search, distinct exact recovery CTA, network submit +or separately guarded acquisition control may own corresponding I/O. Ordinary +**Выбрать** owns only a browser-local draft and never owns a loader. Every +loader belongs to the explicit action that created it and ends with it. + +## Hardware acceptance order + +Software tests do not replace a real K1/macOS/router run. Accept sequentially: + +1. Open cold and prove mode plus Step 01 and its explicit Scan action are + visible, while Step 02 is absent and no discovery starts automatically. With both empty and + unresolved durable physical history, change the mode and prove one local + reset CAS, zero device/host calls and no automatic Scan. Repeat from active, + reconnecting and terminal `cleanup_pending` states; prove local sources are + sealed, the old K1 is not claimed stopped, and a local cleanup failure leaves + the exact reset retryable. With an exact prior connection, prove cold entry + contains no historical reconnect prompt; after one reset CAS and zero Scan, + a separate clean **Найти по Bluetooth** action remains clean after reload. +2. Start discovery and prove the spinner and seconds countdown remain visible + for the bounded search, then the exact result count appears. +3. With multiple advertisements, prove every ordinary connectable row keeps + exactly one enabled **Выбрать** action and none auto-selects or auto-connects. + Repeat with the exact prior UUID after reset and prove it has the same + **Выбрать** action, with no reconnect/reopen/Verify path. +4. Select a Bridge device, including that prior UUID, and prove the card remains visible through + **Подключение…**, then Step 01 turns green before Step 02 **Сеть** appears. +5. Prove network fields never coexist with candidate rows, and one explicit + submit owns at most one write. +6. Wait beyond the legacy candidate TTL and prove both the latest-generation + unselected rows and an admitted selected session remain stable; then prove a + missing exact handle/live GATT failure blocks Apply before any write. +7. Exercise identity, GATT, stale-CAS, lifecycle-busy, disconnect and power-off + failures; each ends the loader, leaves Step 02 absent and offers only ordinary + retry/choose-another copy. +8. Retire an unresolved target in controlled fault injection, perform one + scenario reset and rediscover its exact UUID in the successor Scan. Prove its + sole action is **Выбрать**, selection performs no I/O and Step 02 appears + immediately. Apply once and prove exact current-generation handle capture, + live GATT baseline, exactly one request-bound append-only physical reopen + checkpoint and at most one network write. The original retirement/outcome + audit remains immutable; the service-owned continuation uses only DeviceInfo + and non-retained status, with zero START/STOP and no browser Verify. Inject + failed and outcome-unknown network results; each current error/recovery + surface remains visible after reload. +9. Try a different device while old authority is unavailable and prove its + ordinary selection triggers no hidden retirement/reopen/Verify and cannot + bypass the durable target. +10. Prove no row labels a device saved/original/retired, says + **Переподключиться**, or exposes physical-state/ledger terminology. The model name + appears only in the top heading; step names remain **Подключение / Сеть**. +11. During a composite-confirmed live acquisition, remove host Wi-Fi for longer + than the old control keepalive and restore it. Prove neutral reconnecting, + same-lineage SCANNING resume, raw-writer continuity, exact camera epoch + restart when stalled, and zero START/STOP/BLE/network mutation. Repeat with + READY, SCAN_OVER, wrong identity and permanent loss plus + **Завершить локально**. +12. Repeat idle/pre-START Bridge network loss, Mac Wi-Fi switch, sleep/wake, + hard K1 power loss, STOP deadline and backend restart; prove zero automatic + command or retry outside the sole active-stream exception. +13. Repeat the entire acceptance separately for Quick Connect before claiming + Quick coverage. + +The current software contract is not real-hardware acceptance. The acceptance +manifest lists executable coverage and the remaining Bridge/Quick field gaps. diff --git a/experiments/perception/worker/run_e15_shadow_inference.py b/experiments/perception/worker/run_e15_shadow_inference.py index a1d41e9..5007558 100644 --- a/experiments/perception/worker/run_e15_shadow_inference.py +++ b/experiments/perception/worker/run_e15_shadow_inference.py @@ -254,13 +254,11 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]: } if ( not isinstance(replay_source, dict) - or replay_source.get("session_id") - != "20260720T065719Z_viewer_live" + or replay_source.get("session_id") != "20260720T065719Z_viewer_live" or replay_source.get("display_name") != "RAVNOVES00" or replay_source.get("selection") != "complete-recording" or float(replay_source.get("speed", 0)) != 1.0 - or float(replay_source.get("minimum_source_span_seconds", 0)) - < 450 + or float(replay_source.get("minimum_source_span_seconds", 0)) < 450 or replay_source.get("look_ahead") is not False or any( not isinstance(replay_source.get(key), int) @@ -269,9 +267,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]: for key, expected in replay_integer_contract.items() ) ): - raise RuntimeError( - "LAB E28 complete-recording worker replay contract is invalid" - ) + raise RuntimeError("LAB E28 complete-recording worker replay contract is invalid") elif replay_source is not None: raise RuntimeError("LAB E15 non-replay profile carries replay source state") if local_surface is not None: @@ -280,9 +276,7 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]: ) local_acceptance = ( - local_surface.get("acceptance") - if isinstance(local_surface, dict) - else None + local_surface.get("acceptance") if isinstance(local_surface, dict) else None ) expected_profile_sha256 = hashlib.sha256( canonical_json(DEFAULT_K1_LOCAL_SURFACE_PROFILE.to_dict()) @@ -293,29 +287,20 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]: "maximum_runtime_drop_fraction", ) point_capacity = ( - local_surface.get("point_queue_capacity") - if isinstance(local_surface, dict) - else None + local_surface.get("point_queue_capacity") if isinstance(local_surface, dict) else None ) pose_capacity = ( - local_surface.get("pose_buffer_capacity") - if isinstance(local_surface, dict) - else None + local_surface.get("pose_buffer_capacity") if isinstance(local_surface, dict) else None ) result_capacity = ( - local_surface.get("result_capacity") - if isinstance(local_surface, dict) - else None + local_surface.get("result_capacity") if isinstance(local_surface, dict) else None ) if ( - profile.get("mode") - not in {"worker-replay-gate", "physical-shadow-gate"} + profile.get("mode") not in {"worker-replay-gate", "physical-shadow-gate"} or not isinstance(local_surface, dict) or local_surface.get("enabled") is not True - or local_surface.get("profile_id") - != DEFAULT_K1_LOCAL_SURFACE_PROFILE.profile_id - or local_surface.get("profile_sha256") - != expected_profile_sha256 + or local_surface.get("profile_id") != DEFAULT_K1_LOCAL_SURFACE_PROFILE.profile_id + or local_surface.get("profile_sha256") != expected_profile_sha256 or not isinstance(point_capacity, int) or isinstance(point_capacity, bool) or point_capacity not in range(1, 9) @@ -328,27 +313,16 @@ def read_live_profile(path: Path) -> tuple[dict[str, Any], str]: or not 0 <= float(local_surface.get("future_pose_wait_ms", -1)) <= DEFAULT_K1_LOCAL_SURFACE_PROFILE.maximum_pose_binding_ms - or not 0.1 - <= float(local_surface.get("retention_seconds", 0)) - <= 30 + or not 0.1 <= float(local_surface.get("retention_seconds", 0)) <= 30 or float(temporal.get("maximum_pose_point_delta_ms", 0)) != DEFAULT_K1_LOCAL_SURFACE_PROFILE.maximum_pose_binding_ms or not isinstance(local_acceptance, dict) or int(local_acceptance.get("minimum_bound_frames", 0)) < 2 - or any( - not 0 <= float(local_acceptance.get(key, -1)) <= 1 - for key in local_fractions - ) - or float( - local_acceptance.get("maximum_p95_result_age_ms", 0) - ) - <= 0 - or float(local_acceptance.get("minimum_effective_fps", 0)) - <= 0 + or any(not 0 <= float(local_acceptance.get(key, -1)) <= 1 for key in local_fractions) + or float(local_acceptance.get("maximum_p95_result_age_ms", 0)) <= 0 + or float(local_acceptance.get("minimum_effective_fps", 0)) <= 0 ): - raise RuntimeError( - "LAB E28 worker local-surface profile contract is invalid" - ) + raise RuntimeError("LAB E28 worker local-surface profile contract is invalid") fractions = ( "detector_maximum_drop_fraction", "semantic_maximum_drop_fraction", @@ -429,64 +403,45 @@ def _local_surface_acceptance_checks( return { "local_surface_session_initialized": bool(runtime), - "local_surface_closed": snapshot.get("closed") is True - and runtime.get("closed") is True, + "local_surface_closed": snapshot.get("closed") is True and runtime.get("closed") is True, "local_surface_minimum_bound_frames": point_bound >= int(acceptance["minimum_bound_frames"]), - "local_surface_binder_accounting": point_bound - + point_missed - + point_dropped - + point_depth + "local_surface_binder_accounting": point_bound + point_missed + point_dropped + point_depth == point_published, - "local_surface_binder_to_runtime_accounting": point_bound - == runtime_published, + "local_surface_binder_to_runtime_accounting": point_bound == runtime_published, "local_surface_point_buffer_bound": ( int(points.get("capacity", 0)) == int(config["point_queue_capacity"]) - and int(points.get("maximum_depth", 0)) - <= int(points.get("capacity", 0)) + and int(points.get("maximum_depth", 0)) <= int(points.get("capacity", 0)) and point_depth == 0 ), "local_surface_pose_buffer_bound": ( int(poses.get("capacity", 0)) == int(config["pose_buffer_capacity"]) - and int(poses.get("maximum_depth", 0)) - <= int(poses.get("capacity", 0)) + and int(poses.get("maximum_depth", 0)) <= int(poses.get("capacity", 0)) ), - "local_surface_maximum_pose_miss_fraction": point_missed - / max(1, point_published) + "local_surface_maximum_pose_miss_fraction": point_missed / max(1, point_published) <= float(acceptance["maximum_pose_miss_fraction"]), - "local_surface_maximum_point_drop_fraction": point_dropped - / max(1, point_published) + "local_surface_maximum_point_drop_fraction": point_dropped / max(1, point_published) <= float(acceptance["maximum_point_drop_fraction"]), - "local_surface_runtime_accounting": runtime_consumed - + runtime_dropped - + runtime_depth + "local_surface_runtime_accounting": runtime_consumed + runtime_dropped + runtime_depth == runtime_published, - "local_surface_runtime_result_accounting": result_published - + result_failed + "local_surface_runtime_result_accounting": result_published + result_failed == runtime_consumed, "local_surface_runtime_queue_bound": ( int(queue_state.get("capacity", 0)) == int(config["point_queue_capacity"]) - and int(queue_state.get("maximum_depth", 0)) - <= int(queue_state.get("capacity", 0)) + and int(queue_state.get("maximum_depth", 0)) <= int(queue_state.get("capacity", 0)) and runtime_depth == 0 ), - "local_surface_maximum_runtime_drop_fraction": runtime_dropped - / max(1, runtime_published) + "local_surface_maximum_runtime_drop_fraction": runtime_dropped / max(1, runtime_published) <= float(acceptance["maximum_runtime_drop_fraction"]), - "local_surface_minimum_effective_fps": float( - delivery.get("effective_fps", 0) - ) + "local_surface_minimum_effective_fps": float(delivery.get("effective_fps", 0)) >= float(acceptance["minimum_effective_fps"]), "local_surface_zero_runtime_failures": result_failed == 0, "local_surface_maximum_p95_result_age_ms": ( isinstance(p95_result_age, (int, float)) and not isinstance(p95_result_age, bool) - and float(p95_result_age) - <= float(acceptance["maximum_p95_result_age_ms"]) - ), - "local_surface_profile_pinned": ( - runtime_profile.get("profile_id") == config["profile_id"] + and float(p95_result_age) <= float(acceptance["maximum_p95_result_age_ms"]) ), + "local_surface_profile_pinned": (runtime_profile.get("profile_id") == config["profile_id"]), "local_surface_shadow_authority_only": ( snapshot.get("authority") == { @@ -622,6 +577,7 @@ class _TransportState: camera_sequence_gaps: int = 0 last_camera_source_sequence: int | None = None session_id: str | None = None + session_generation: int | None = None session_end_seen: bool = False timed_out: bool = False results_published: int = 0 @@ -956,8 +912,7 @@ class _StageExecutionTelemetry: self._last_frame_by_stage.get(stage_id), ) for stage_id in self._stage_ids - if stage_id in self._native_started - and stage_id not in self._native_failed + if stage_id in self._native_started and stage_id not in self._native_failed ] for stage_id, elapsed_seconds, activations, frame_index in rows: self._emit_native( @@ -1012,9 +967,7 @@ class _StageExecutionTelemetry: "elapsed_seconds": round(elapsed[stage_id], 6), "activations": self._activations[stage_id], "share_percent": ( - round(elapsed[stage_id] / total * 100, 6) - if total > 0 - else None + round(elapsed[stage_id] / total * 100, 6) if total > 0 else None ), } for stage_id in self._stage_ids @@ -1127,11 +1080,20 @@ def _receiver( state.first_ingress_sequence = sequence state.last_ingress_sequence = sequence session_id = str(header["session_id"]) + session_generation_value = header["session_generation"] + if ( + not isinstance(session_generation_value, int) + or isinstance(session_generation_value, bool) + or session_generation_value < 1 + ): + raise ShadowRuntimeError("shadow session generation is invalid") + session_generation = session_generation_value if state.session_id is None: state.session_id = session_id + state.session_generation = session_generation if local_surface is not None: local_surface.begin_session(session_id) - elif state.session_id != session_id: + elif state.session_id != session_id or state.session_generation != session_generation: raise ShadowRuntimeError("shadow session identity changed") modality = str(header["modality"]) state.counts[modality] += 1 @@ -1272,9 +1234,7 @@ def _common(args: argparse.Namespace) -> dict[str, Any]: "k1link/ground_segmentation.py", } if not required_surface_sources <= worker_sources: - raise RuntimeError( - "LAB E28 worker package lacks local-surface runtime" - ) + raise RuntimeError("LAB E28 worker package lacks local-surface runtime") stability = None stability_sha256 = None if args.stability_profile is not None: @@ -1400,11 +1360,7 @@ def run( if not token or len(token) < 40: raise RuntimeError("LAB E15 shadow token is missing") - stage_telemetry = ( - runtime_state.get("_stage_telemetry") - if runtime_state is not None - else None - ) + stage_telemetry = runtime_state.get("_stage_telemetry") if runtime_state is not None else None if not isinstance(stage_telemetry, _StageExecutionTelemetry): stage_telemetry = _StageExecutionTelemetry() if runtime_state is not None: @@ -1461,9 +1417,7 @@ def run( local_surface = K1LocalSurfaceShadowCoordinator( point_capacity=int(local_surface_config["point_queue_capacity"]), pose_capacity=int(local_surface_config["pose_buffer_capacity"]), - future_pose_wait_ms=float( - local_surface_config["future_pose_wait_ms"] - ), + future_pose_wait_ms=float(local_surface_config["future_pose_wait_ms"]), retention_seconds=float(local_surface_config["retention_seconds"]), result_capacity=int(local_surface_config["result_capacity"]), ) @@ -1890,7 +1844,11 @@ def run( optimize=False, ) with stage_telemetry.measure("result-publication", envelope.frame_index): + if transport.session_id is None or transport.session_generation is None: + raise ShadowRuntimeError("shadow result session identity is unavailable") live_result = encode_live_perception_result( + session_id=transport.session_id, + session_generation=transport.session_generation, frame_index=envelope.frame_index, source_frame_index=int(envelope.timeline["source_frame_index"]), session_seconds=frame_seconds, @@ -1969,9 +1927,7 @@ def run( temporal_semantic_summary = ( None if semantic_stabilizer is None else semantic_stabilizer.snapshot() ) - local_surface_snapshot = ( - None if local_surface is None else local_surface.snapshot() - ) + local_surface_snapshot = None if local_surface is None else local_surface.snapshot() acceptance = live["acceptance"] checks = { "minimum_camera_frames": decoded_frame_count >= int(acceptance["minimum_camera_frames"]), @@ -2356,9 +2312,7 @@ def _persistent_run_telemetry_identity( if isinstance(stability, dict) and isinstance(stability.get("profile_id"), str) else "lab-e15-shadow-inference-v1" ) - method_id = ( - INLINE_TEMPORAL_PIPELINE_ID if isinstance(stability, dict) else PIPELINE_ID - ) + method_id = INLINE_TEMPORAL_PIPELINE_ID if isinstance(stability, dict) else PIPELINE_ID return PipelineTelemetryIdentity( contour_id=telemetry.get("contour_id"), agent_id=telemetry.get("agent_id"), @@ -2545,9 +2499,7 @@ def serve(args: argparse.Namespace) -> int: state["last_run_outcome"] = { "request_id": request_id, "state": "failed", - "duration_ms": ( - round(duration_ms, 6) if duration_ms is not None else None - ), + "duration_ms": (round(duration_ms, 6) if duration_ms is not None else None), "exit_code": None, "error_type": type(exc).__name__, } diff --git a/plugins/xgrids-k1/README.md b/plugins/xgrids-k1/README.md index aae6079..d7df916 100644 --- a/plugins/xgrids-k1/README.md +++ b/plugins/xgrids-k1/README.md @@ -95,6 +95,14 @@ Validate the current exact-match profile without device I/O with: uv run python plugins/xgrids-k1/profile_loader.py ``` +Plugin v0.7.0 adds the backend-owned supervised connection lifecycle. Operator +mode choice is a CAS-fenced draft; an explicit Scan commits a safe pre-START +mode switch, while Connect reaches Ready only after the exact current +`DeviceInfo` authority is confirmed. Configured, active and desired modes are +separate facts. Terminal pre-START failures and purely local prepared sessions +self-retire without a device command, and an applied network configuration is +recovered through a separate read-only Verify instead of replaying Wi-Fi. + Plugin v0.6.0 retains the physically accepted v0.5.0 control transport and adds the connection matrix behind the existing explicit `network.provision` action. Bridge remains the default. Direct Connect sends the same single reviewed @@ -102,13 +110,19 @@ Bridge remains the default. Direct Connect sends the same single reviewed Connect accepts no browser/API credential: it sends one reviewed fixed 100-byte AP-enable frame to the selected K1, waits up to 15 seconds for the canonical byte-51 AP-ready flag, and keeps that BLE session alive while the macOS adapter -performs bounded exact-SSID CoreWLAN discovery and one association. Credentials +performs up to 30 seconds of exact-SSID CoreWLAN discovery and one association. +AP-ready does not imply that macOS has already observed the RF beacon. Credentials are resolved by a preinstalled exact `3.0.2` firmware provider. Its optional laboratory importer validates the reviewed official archive, extracts the single AP declaration and installs firmware-scoped material in the OS secure store. The macOS helper then materializes the selected device profile entirely inside Keychain before any BLE write. The secret never enters the browser, API, argv, logs or evidence; the importer's short-lived mutable buffer is zeroized after the stdin handoff. +The prepared-host adapter uses the accepted Apple-signed +`/usr/bin/xcrun swift` runner. It does not runtime-compile an ad-hoc executable, +query the standard Wi-Fi Keychain or open a password dialog after the K1 write. +Production portability still requires a packaged, properly signed helper with +a stable designated identity and explicit CoreWLAN authorization. There is no automatic BLE-write or association retry. A clean host cannot obtain the provider from BLE and the product does not download firmware during connection. Windows/Linux Quick Connect adapters are not planned while that diff --git a/plugins/xgrids-k1/frontend/README.md b/plugins/xgrids-k1/frontend/README.md index 8dc7c95..1cc1ca8 100644 --- a/plugins/xgrids-k1/frontend/README.md +++ b/plugins/xgrids-k1/frontend/README.md @@ -6,17 +6,23 @@ generic application source tree. The contribution contains: -- `K1ProvisioningPipeline` for power confirmation, BLE discovery and the three - explicit local connection directions: Bridge, Quick Connect and Direct - Connect; +- `K1ProvisioningPipeline` for explicit BLE discovery and the three local + connection directions: Bridge, Quick Connect and Direct Connect; - `K1AcquisitionPipeline` for explicit canonical connection/workspace/project/ START checkpoints, local receiver preparation and compatibility file replay; - `K1SpatialControls` for an explicit no-retry STOP followed by the separate READY plus steady-green completion gate; - plugin-local diagnostics, metrics, API state, lifecycle mapping, observation-source mapping and scoped styles; -- typed v0.6.0 local-network and interactive application-control state plus legacy shadow +- typed v0.7.0 supervised connection lifecycle and interactive application-control state plus legacy shadow inspection contracts; +- a click-correlated, non-secret provisioning presentation latch: after Apply, + Steps 01–02 keep their selected-device/form anatomy with disabled controls + until the exact connection attempt becomes reachable or reaches bounded + recovery; the Wi-Fi password is cleared before asynchronous dispatch; +- policy-gated retirement of an unavailable historical K1 as an explicit + local ledger action; it never emits a device command and never bypasses the + public `retire-unavailable-physical-target` decision; - `plugin.ts`, which binds the manifest `device.connection` component key to the runtime provider and connection view. diff --git a/plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx b/plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx index 3036792..aa13fe9 100644 --- a/plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx +++ b/plugins/xgrids-k1/frontend/src/XgridsK1Connection.tsx @@ -1,31 +1,264 @@ -import { Button, StatusBadge, type StatusTone } from "@nodedc/ui-react"; +import { useEffect, useRef, useState } from "react"; +import { StatusBadge, type StatusTone } from "@nodedc/ui-react"; import type { DevicePluginConnectionProps } from "@mission-core/plugin-sdk"; +import { + activeStreamRecoveryPresentation, + suppressGenericErrorDuringActiveStreamRecovery, +} from "./activeStreamRecovery"; import { K1AcquisitionPipeline } from "./components/K1AcquisitionPipeline"; import { K1Diagnostics } from "./components/K1Diagnostics"; import { K1Metrics } from "./components/K1Metrics"; -import { K1ProvisioningPipeline } from "./components/K1ProvisioningPipeline"; +import { K1OperatorError } from "./components/K1OperatorError"; import { + K1ProvisioningPipeline, + unavailablePhysicalRetirementAuthority, +} from "./components/K1ProvisioningPipeline"; +import { + backendConnectionTopology, + connectionAttemptForRuntimeError, + hasControlAuthority, isConfirmedLiveState, + isPhysicalStopRecoverySettling, + isRecoveredPhysicalScanning, + isReleasedTerminalAcquisitionFailure, isSourceRuntimeBusy, + readOnlyConnectionObservationTarget, recoverableAcquisition, + requiresCanonicalStopAfterTerminalLocalFailure, + requiresReadOnlyPhysicalRecovery, sourceStatusLabel, } from "./lifecycle"; -import { localizeRuntimeMessage } from "./messages"; import { phaseLabel, phaseTone } from "./presentation"; -import { useXgridsK1Controller } from "./runtimeContext"; +import { + useXgridsK1Controller, + type XgridsK1Controller, +} from "./runtimeContext"; +import type { XgridsK1State } from "./api"; +import { + DEFAULT_CONNECTION_MODE, + type ConnectionMode, +} from "./configuration"; + +export { K1OperatorError }; + +export function shouldRenderK1GenericRuntimeError( + error: string | null | undefined, + hasCorrelatedConnectionAttempt: boolean, + state: XgridsK1State | null | undefined, + errorAction?: string | null, +): boolean { + return Boolean( + error + && !hasCorrelatedConnectionAttempt + && !suppressGenericErrorDuringActiveStreamRecovery(state, errorAction), + ); +} + +export function physicalRecoveryConnectionDetail( + state: XgridsK1State | null | undefined, +): string | null { + if (!requiresReadOnlyPhysicalRecovery(state)) return null; + const retirementAvailable = Boolean( + unavailablePhysicalRetirementAuthority(state), + ); + const readOnlyVerificationAvailable = Boolean( + readOnlyConnectionObservationTarget(state)?.serverBound, + ); + if (retirementAvailable && readOnlyVerificationAvailable) { + return "Если прежний K1 снова доступен, проверьте его состояние без изменений: проверка читает состояние и не отправляет START, STOP или настройки сети. Если K1 недоступен постоянно или заменён, его можно локально исключить без связи с устройством."; + } + if (readOnlyVerificationAvailable) { + return "Проверьте состояние прежнего K1 без изменений устройства. Проверка использует сохранённое системой подключение и не отправляет START, STOP или настройки сети."; + } + if (retirementAvailable) { + return "Прежний K1 можно локально исключить без связи с устройством: действие не отправляет START, STOP или настройки сети. После этого можно отдельно выбрать другой K1."; + } + return "Безопасная сверка прежнего K1 сейчас недоступна. Обновите состояние; новые команды устройству заблокированы."; +} + +function connectionPhaseFallbackLabel(phase: string | null | undefined): string { + if (phase === "device_selected") return "Выбор выполнен"; + if (phase === "connected") return "Сетевой адрес получен"; + return phaseLabel(phase); +} + +/** + * Keep the disconnected connection job focused on its progressive pipeline. + * Persisted topology is evidence, not live control authority. Operational + * panels return only when they are actionable or required to finish an + * already-started lifecycle, especially STOP and recovery. + */ +export function shouldRenderK1OperationalPanels( + state: XgridsK1State | null | undefined, +): boolean { + return Boolean( + hasControlAuthority(state) + || state?.source_mode === "live" + || state?.source_mode === "replay" + || recoverableAcquisition(state) + || state?.acquisition?.cleanup_pending === true + || requiresCanonicalStopAfterTerminalLocalFailure(state) + || isRecoveredPhysicalScanning(state) + || isPhysicalStopRecoverySettling(state) + || activeStreamRecoveryPresentation(state) !== null + ); +} + +export function K1ConnectionPipelines({ + controller, + desiredConnectionMode, + onDesiredConnectionModeChange, + operationalPanelsVisible, + openSpatialScene, + activateAutomaticSpatialSource, + sourceLabel, +}: { + controller: XgridsK1Controller; + desiredConnectionMode: ConnectionMode; + onDesiredConnectionModeChange: (mode: ConnectionMode) => void | Promise; + operationalPanelsVisible: boolean; + openSpatialScene: () => void; + activateAutomaticSpatialSource: () => void; + sourceLabel: string; +}) { + return ( + <> + {operationalPanelsVisible ? : null} + +
+ + {operationalPanelsVisible ? ( +
+ + +
+ ) : null} +
+ + ); +} export function XgridsK1Connection({ model, host }: DevicePluginConnectionProps) { const controller = useXgridsK1Controller(); - const { state, error, refresh, clearError } = controller; + const { + state, + error, + errorDiagnostic, + errorCorrelation, + refresh, + clearError, + } = controller; + const [desiredConnectionMode, setDesiredConnectionMode] = useState( + DEFAULT_CONNECTION_MODE, + ); + const desiredModeInitialized = useRef(false); + const desiredModeLocallyDirty = useRef(false); + const hydratedScenarioResetKey = useRef(null); const confirmedLive = isConfirmedLiveState(state); const sourceRuntimeBusy = isSourceRuntimeBusy(state); const preparedAcquisition = recoverableAcquisition(state)?.state === "prepared"; - const sourceLabel = sourceStatusLabel(state); - const relevantAcquisitionFailed = state?.source_mode !== "replay" && state?.acquisition?.state === "failed"; + const activeRecoveryPresentation = activeStreamRecoveryPresentation(state); + const sourceLabel = activeRecoveryPresentation?.title ?? sourceStatusLabel(state); + const releasedAcquisitionFailure = isReleasedTerminalAcquisitionFailure(state); + const physicalRecoveryRequired = requiresReadOnlyPhysicalRecovery(state); + const physicalStopRecoverySettling = isPhysicalStopRecoverySettling(state); + const recoveredPhysicalScanning = physicalRecoveryRequired + && state?.application_control_session?.state === "scanning" + && state.application_control_session.can_stop === true; + const physicalRecoveryDetail = physicalRecoveryConnectionDetail(state); + const correlatedConnectionAttempt = connectionAttemptForRuntimeError( + errorCorrelation, + state, + ); + const showGenericRuntimeError = shouldRenderK1GenericRuntimeError( + error, + Boolean(correlatedConnectionAttempt), + state, + errorCorrelation?.action, + ); + const relevantAcquisitionFailed = state?.source_mode !== "replay" + && state?.acquisition?.state === "failed" + && !releasedAcquisitionFailure; + const projectedPhase = releasedAcquisitionFailure && state?.phase === "error" + ? "idle" + : state?.phase; + const connectionTopology = backendConnectionTopology(state); + const effectiveDesiredConnectionMode = desiredModeInitialized.current + ? desiredConnectionMode + : state?.desired_connection_mode + ?? (connectionTopology?.status === "active" + ? connectionTopology.connectionMode + : DEFAULT_CONNECTION_MODE); + + useEffect(() => { + if (!state || desiredModeInitialized.current) return; + desiredModeInitialized.current = true; + setDesiredConnectionMode( + state.desired_connection_mode + ?? (connectionTopology?.status === "active" + ? connectionTopology.connectionMode + : DEFAULT_CONNECTION_MODE), + ); + }, [connectionTopology, state]); + + useEffect(() => { + if (!desiredModeInitialized.current) return; + const backendDesiredMode = state?.desired_connection_mode; + if (!backendDesiredMode) return; + const scenarioReset = state?.connection_scenario_reset; + const scenarioResetKey = scenarioReset + && scenarioReset.revision === state?.desired_connection_mode_revision + && scenarioReset.desired_mode === backendDesiredMode + && state?.snapshot_runtime_id?.trim() + ? `${state.snapshot_runtime_id}:${scenarioReset.revision}` + : null; + if (scenarioResetKey && hydratedScenarioResetKey.current !== scenarioResetKey) { + // The shell emergency reset is an authoritative new backend revision. + // It must retire a locally dirty selector too; an older dirty browser + // draft cannot keep showing Quick/Direct after canonical Bridge won. + hydratedScenarioResetKey.current = scenarioResetKey; + desiredModeLocallyDirty.current = false; + setDesiredConnectionMode(backendDesiredMode); + return; + } + if (backendDesiredMode === desiredConnectionMode) { + desiredModeLocallyDirty.current = false; + return; + } + // Every dropdown gesture is now an explicit backend scenario-reset CAS. + // The callback may publish its accepted mode one render before the hook's + // authoritative snapshot arrives, so passive polling must not overwrite + // that in-flight acknowledgement. Once the backend echoes the exact mode + // above, the dirty fence clears and later authoritative changes hydrate it. + if (desiredModeLocallyDirty.current) return; + setDesiredConnectionMode(backendDesiredMode); + }, [ + desiredConnectionMode, + state?.connection_scenario_reset, + state?.desired_connection_mode, + state?.desired_connection_mode_revision, + state?.snapshot_runtime_id, + ]); + + const updateDesiredConnectionMode = (mode: ConnectionMode) => { + desiredModeLocallyDirty.current = mode !== state?.desired_connection_mode; + setDesiredConnectionMode(mode); + }; const sourceTone: StatusTone = - state?.phase === "error" || relevantAcquisitionFailed + activeRecoveryPresentation + ? activeRecoveryPresentation.tone + : projectedPhase === "error" || relevantAcquisitionFailed ? "danger" : confirmedLive || state?.source_mode === "replay" ? "success" @@ -34,56 +267,88 @@ export function XgridsK1Connection({ model, host }: DevicePluginConnectionProps) : "neutral"; const connectionPhaseLabel = sourceRuntimeBusy || preparedAcquisition ? sourceLabel - : phaseLabel(state?.phase); + : activeRecoveryPresentation + ? activeRecoveryPresentation.title + : physicalStopRecoverySettling + ? "Завершение остановки" + : recoveredPhysicalScanning + ? "Сканирование продолжается" + : physicalRecoveryRequired + ? "Требуется действие" + : projectedPhase === "error" + ? connectionPhaseFallbackLabel(projectedPhase) + : connectionTopology?.status === "active" + ? "Подключение установлено" + : connectionTopology?.status === "configured-unverified" + ? "Подключение отсутствует" + : connectionTopology?.source === "durable" + ? "Подключение отсутствует" + : connectionTopology?.source === "applied" + ? "Подключение отсутствует" + : connectionTopology?.source === "last-known" + ? "Подключение отсутствует" + : connectionPhaseFallbackLabel(projectedPhase); const connectionPhaseTone = sourceRuntimeBusy || preparedAcquisition ? sourceTone - : phaseTone(state?.phase); + : activeRecoveryPresentation + ? activeRecoveryPresentation.tone + : physicalRecoveryRequired + ? "warning" + : projectedPhase === "error" + ? phaseTone(projectedPhase) + : connectionTopology?.status === "active" + ? "success" + : connectionTopology?.status === "configured-unverified" + ? "neutral" + : "neutral"; + const connectionPhaseDetail = physicalStopRecoverySettling + ? "Команда остановки уже принята. Завершение выполняется без повторной команды." + : activeRecoveryPresentation + ? activeRecoveryPresentation.detail + : recoveredPhysicalScanning + ? "Локальная запись остановлена, но сканирование ещё продолжается." + : physicalRecoveryRequired + ? physicalRecoveryDetail + ?? "Безопасное восстановление прежнего K1 сейчас недоступно." + : !sourceRuntimeBusy && connectionTopology?.status === "configured-unverified" + ? "Начните новое подключение." + : !sourceRuntimeBusy && connectionTopology?.status === "active" + ? "Готово к новой сессии." + : "Ожидается состояние локального контура."; + const operationalPanelsVisible = shouldRenderK1OperationalPanels(state); return (
- {error ? ( - + {showGenericRuntimeError && error ? ( + void refresh()} + onClear={clearError} + /> ) : null}
- XGRIDS K1 · PLUGIN UI + ЛОКАЛЬНОЕ ПОДКЛЮЧЕНИЕ

Подключение {model.displayName}

-

BLE/Wi‑Fi provisioning и acquisition pipeline принадлежат этому device plugin; Control Station предоставляет только host slot и переход в пространственную сцену.

+

Выберите способ связи и последовательно установите подключение.

{connectionPhaseLabel} - {localizeRuntimeMessage(state?.message) || "Ожидаем состояние локального контура."} + {connectionPhaseDetail}
- - -
- -
- - -
-
+
); } diff --git a/plugins/xgrids-k1/frontend/src/activeStreamRecovery.ts b/plugins/xgrids-k1/frontend/src/activeStreamRecovery.ts new file mode 100644 index 0000000..e56c7a3 --- /dev/null +++ b/plugins/xgrids-k1/frontend/src/activeStreamRecovery.ts @@ -0,0 +1,278 @@ +import { + isXgridsActiveStreamRecovery, + type XgridsActiveStreamRecovery, + type XgridsK1State, +} from "./api"; + +export interface ActiveStreamRecoveryLineage { + snapshotRuntimeId: string; + acquisitionId: string; + acquisitionStateRevision: number; + recoveryGeneration: number; + runtimeProducerGeneration: number; + recovery: XgridsActiveStreamRecovery; +} + +export type ActiveStreamForceFinishAuthority = ActiveStreamRecoveryLineage; + +export type ActiveStreamRecoveryPresentationAuthority = ActiveStreamRecoveryLineage; + +export type ActiveStreamRecoveryVisibleState = + | "reconnecting" + | "blocked" + | "standby" + | "fault"; + +export interface ActiveStreamRecoveryPresentation { + state: ActiveStreamRecoveryVisibleState; + eyebrow: string; + title: string; + statusLabel: string; + tone: "neutral" | "warning" | "danger"; + detail: string; + progressLabel: string | null; + showSpinner: boolean; + forceFinishAvailable: boolean; +} + +function positiveInteger(value: unknown): value is number { + return Number.isInteger(value) && (value as number) > 0; +} + +/** + * Resolve one exact active-stream lineage from the public runtime snapshot. + * + * A recovery-shaped object alone is not authority. The browser also requires + * the current runtime id, the same acquisition id and the exact producer + * generation on both sides of the projection. This keeps a late recovery + * update from an older producer out of both presentation and mutation gates. + */ +export function exactActiveStreamRecoveryLineage( + state: XgridsK1State | null | undefined, +): ActiveStreamRecoveryLineage | null { + const recovery = state?.connection_recovery; + const acquisition = state?.acquisition; + const snapshotRuntimeId = state?.snapshot_runtime_id?.trim() || null; + const producerGeneration = state?.producer_generation; + const acquisitionId = acquisition?.acquisition_id?.trim() || null; + const recoveryAcquisitionId = recovery?.acquisition_id?.trim() || null; + if ( + !snapshotRuntimeId + || !isXgridsActiveStreamRecovery(recovery) + || !acquisition + || !acquisitionId + || recoveryAcquisitionId !== acquisitionId + || !positiveInteger(acquisition.state_revision) + || !positiveInteger(recovery.generation) + || !positiveInteger(producerGeneration) + || recovery.runtime_producer_generation !== producerGeneration + || recovery.automatic_read_only_rebind !== true + ) return null; + return { + snapshotRuntimeId, + acquisitionId, + acquisitionStateRevision: acquisition.state_revision, + recoveryGeneration: recovery.generation, + runtimeProducerGeneration: producerGeneration, + recovery, + }; +} + +/** Exact, current and backend-policy-admitted authority for local-only finish. */ +export function activeStreamForceFinishAuthority( + state: XgridsK1State | null | undefined, +): ActiveStreamForceFinishAuthority | null { + const lineage = exactActiveStreamRecoveryLineage(state); + if ( + !lineage + || !["reconnecting", "blocked"].includes(lineage.recovery.state) + || lineage.recovery.force_finish_allowed !== true + || state?.phase !== "reconnecting" + || state.source_mode !== "live" + || ![ + "starting", + "awaiting_external_start", + "acquiring", + ].includes(state.acquisition?.state ?? "") + ) return null; + return lineage; +} + +/** + * Exact authority for retaining browser presentation while the backend owns a + * read-only reconnect. This is deliberately narrower than the recovery card: + * terminal/blocked recovery states and an inactive acquisition cannot retain + * a prior spatial or camera transport. + */ +export function activeStreamRecoveryPresentationAuthority( + state: XgridsK1State | null | undefined, +): ActiveStreamRecoveryPresentationAuthority | null { + const lineage = exactActiveStreamRecoveryLineage(state); + if ( + !lineage + || lineage.recovery.state !== "reconnecting" + || state?.phase !== "reconnecting" + || state.source_mode !== "live" + || ![ + "starting", + "awaiting_external_start", + "acquiring", + ].includes(state.acquisition?.state ?? "") + ) return null; + return lineage; +} + +/** + * Keep the exact recovered lineage available to disposable browser receivers + * after the recovery card has disappeared. Spatial admission can complete on + * the first authoritative PCL before the acquisition-owned camera produces + * its first playable frame. This authority carries only the no-write + * presentation lease: it grants neither force-finish nor START/STOP policy. + */ +export function activeStreamRecoveredBrowserAuthority( + state: XgridsK1State | null | undefined, +): ActiveStreamRecoveryPresentationAuthority | null { + const lineage = exactActiveStreamRecoveryLineage(state); + if ( + !lineage + || lineage.recovery.state !== "recovered" + || lineage.recovery.camera_recovery !== "owned" + || state?.phase !== "live" + || state.source_mode !== "live" + || state.acquisition?.state !== "acquiring" + ) return null; + return lineage; +} + +/** + * While a validated recovery contract is active it owns the presentation + * decision. Ordinary supervisor data flags may be stale across the network + * gap, so only an exact reconnect lease can retain browser transports. + */ +export function activeStreamRecoveryOwnsPresentationDecision( + state: XgridsK1State | null | undefined, +): boolean { + const recovery = state?.connection_recovery; + return Boolean( + isXgridsActiveStreamRecovery(recovery) + && !["inactive", "recovered"].includes(recovery.state), + ); +} + +export function activeStreamForceFinishAuthorityMatches( + expected: ActiveStreamForceFinishAuthority, + state: XgridsK1State | null | undefined, +): boolean { + const current = activeStreamForceFinishAuthority(state); + return Boolean( + current + && current.snapshotRuntimeId === expected.snapshotRuntimeId + && current.acquisitionId === expected.acquisitionId + && current.acquisitionStateRevision === expected.acquisitionStateRevision + && current.recoveryGeneration === expected.recoveryGeneration + && current.runtimeProducerGeneration === expected.runtimeProducerGeneration, + ); +} + +export function formatActiveStreamRecoveryElapsed( + elapsedMs: number | null, +): string | null { + if (!Number.isFinite(elapsedMs) || elapsedMs === null || elapsedMs < 0) return null; + const elapsedSeconds = Math.floor(elapsedMs / 1_000); + if (elapsedSeconds < 60) return `${elapsedSeconds} с`; + const minutes = Math.floor(elapsedSeconds / 60); + const seconds = elapsedSeconds % 60; + return seconds > 0 ? `${minutes} мин ${seconds} с` : `${minutes} мин`; +} + +function recoveryProgressLabel( + recovery: XgridsActiveStreamRecovery, +): string | null { + const elapsed = formatActiveStreamRecoveryElapsed(recovery.elapsed_ms); + const attempt = recovery.attempt > 0 + ? `Попытка ${recovery.attempt}` + : "Подготовка проверки"; + return elapsed ? `${attempt} · ${elapsed}` : attempt; +} + +/** + * Present only an exact current lineage. `recovered` deliberately returns + * null so the ordinary confirmed live UI resumes without a transitional card. + */ +export function activeStreamRecoveryPresentation( + state: XgridsK1State | null | undefined, +): ActiveStreamRecoveryPresentation | null { + const lineage = exactActiveStreamRecoveryLineage(state); + if (!lineage) return null; + const recovery = lineage.recovery; + if (recovery.state === "reconnecting") { + return { + state: "reconnecting", + eyebrow: "СВЯЗЬ · АКТИВНЫЙ ПРИЁМ", + title: "Восстанавливаем соединение", + statusLabel: "Восстановление связи", + tone: "neutral", + detail: "Проверяем прежний активный контур только для чтения. START, STOP и настройки сети не отправляются.", + progressLabel: recoveryProgressLabel(recovery), + showSpinner: true, + forceFinishAvailable: activeStreamForceFinishAuthority(state) !== null, + }; + } + if (recovery.state === "blocked") { + return { + state: "blocked", + eyebrow: "СВЯЗЬ · ТРЕБУЕТСЯ ДЕЙСТВИЕ", + title: recovery.camera_recovery === "blocked" + ? "Видеопоток не восстановлен" + : "Связь не восстановлена", + statusLabel: "Восстановление остановлено", + tone: "warning", + detail: recovery.camera_recovery === "blocked" + ? "Связь с K1 проверена, но камера не возобновила передачу. Можно завершить только локальный приём." + : "Автоматическая проверка остановлена. Можно завершить только локальный приём; команда устройству не отправится.", + progressLabel: recoveryProgressLabel(recovery), + showSpinner: false, + forceFinishAvailable: activeStreamForceFinishAuthority(state) !== null, + }; + } + if (recovery.state === "standby") { + return { + state: "standby", + eyebrow: "СВЯЗЬ · СОСТОЯНИЕ ПРОВЕРЕНО", + title: "Устройство перешло в ожидание", + statusLabel: "Приём завершён", + tone: "neutral", + detail: "K1 сообщил, что активное сканирование уже завершено. Локальный приём закрывается без команды STOP.", + progressLabel: recoveryProgressLabel(recovery), + showSpinner: false, + forceFinishAvailable: false, + }; + } + if (recovery.state === "fault") { + return { + state: "fault", + eyebrow: "СВЯЗЬ · СОСТОЯНИЕ ПРОВЕРЕНО", + title: "K1 сообщил об ошибке", + statusLabel: "Восстановление невозможно", + tone: "danger", + detail: "Безопасная проверка обнаружила ошибку устройства. Автоматических команд и повторов нет.", + progressLabel: recoveryProgressLabel(recovery), + showSpinner: false, + forceFinishAvailable: false, + }; + } + return null; +} + +/** + * Only an exact, still-active background reconnect may hide the generic red + * error banner. A failed explicit local finish is operator-facing evidence and + * must remain visible even while the last accepted snapshot says reconnecting. + */ +export function suppressGenericErrorDuringActiveStreamRecovery( + state: XgridsK1State | null | undefined, + errorAction?: string | null, +): boolean { + if (errorAction === "force-finish") return false; + return activeStreamRecoveryPresentationAuthority(state) !== null; +} diff --git a/plugins/xgrids-k1/frontend/src/api.ts b/plugins/xgrids-k1/frontend/src/api.ts index 639cc98..96a55e8 100644 --- a/plugins/xgrids-k1/frontend/src/api.ts +++ b/plugins/xgrids-k1/frontend/src/api.ts @@ -13,6 +13,115 @@ export interface BleDevice { } export type SourceMode = "idle" | "live" | "replay"; +export const XGRIDS_ACTIVE_STREAM_RECOVERY_STATES = [ + "inactive", + "reconnecting", + "blocked", + "recovered", + "standby", + "fault", + "force-finishing", + "force-finished", +] as const; +export type XgridsActiveStreamRecoveryState = + typeof XGRIDS_ACTIVE_STREAM_RECOVERY_STATES[number]; +export const XGRIDS_CAMERA_RECOVERY_STATES = [ + "inactive", + "owned", + "blocked", +] as const; +export type XgridsCameraRecoveryState = + typeof XGRIDS_CAMERA_RECOVERY_STATES[number]; +export const XGRIDS_CAMERA_MEDIA_STATES = [ + "inactive", + "pending-epoch", + "pending-init", + "pending-first-media", + "ready", +] as const; +export type XgridsCameraMediaState = + typeof XGRIDS_CAMERA_MEDIA_STATES[number]; +export const XGRIDS_CONNECTION_MODES = [ + "bridge", + "quick-connect", + "direct-connect", +] as const; +export type XgridsConnectionMode = typeof XGRIDS_CONNECTION_MODES[number]; + +export const XGRIDS_HOST_DIAGNOSTIC_CODES = [ + "host.bluetooth.permission-denied", + "host.bluetooth.adapter-powered-off", + "host.bluetooth.adapter-unavailable", + "host.bluetooth.runtime-unavailable", + "host.bluetooth.operation-timeout", + "host.wifi.permission-denied", + "host.wifi.adapter-powered-off", + "host.wifi.interface-unavailable", + "host.wifi.ssid-unavailable", + "host.wifi.operation-timeout", + "host.wifi.association-failed", + "host.keychain.interaction-required", + "host.keychain.permission-denied", + "host.keychain.unavailable", + "host.route.unavailable", + "host.tcp.connection-refused", + "host.tcp.connection-timeout", + "host.tcp.endpoint-unavailable", + "host.mqtt.connection-timeout", + "host.mqtt.connection-refused", + "host.mqtt.transport-unavailable", + "host.filesystem.permission-denied", + "host.filesystem.ledger-unavailable", +] as const; + +export const XGRIDS_HOST_DIAGNOSTIC_DOMAINS = [ + "corebluetooth", + "corewlan", + "keychain", + "route", + "tcp", + "mqtt", + "filesystem", +] as const; + +export const XGRIDS_HOST_DIAGNOSTIC_IMPACTS = [ + "discovery", + "host-network", + "control", + "durable-safety", +] as const; + +export const XGRIDS_HOST_DIAGNOSTIC_ACTIONS = [ + "grant-bluetooth-permission", + "power-on-bluetooth", + "restore-bluetooth-adapter", + "grant-wifi-permission", + "power-on-wifi", + "restore-wifi-interface", + "unlock-or-authorize-keychain", + "review-keychain-access", + "join-expected-network", + "inspect-host-route", + "verify-broker-endpoint", + "inspect-local-storage", + "restart-local-service", + "explicit-retry", +] as const; + +export type XgridsHostDiagnosticCode = typeof XGRIDS_HOST_DIAGNOSTIC_CODES[number]; +export type XgridsHostDiagnosticDomain = typeof XGRIDS_HOST_DIAGNOSTIC_DOMAINS[number]; +export type XgridsHostDiagnosticImpact = typeof XGRIDS_HOST_DIAGNOSTIC_IMPACTS[number]; +export type XgridsHostDiagnosticAction = typeof XGRIDS_HOST_DIAGNOSTIC_ACTIONS[number]; + +export interface XgridsHostFailureDiagnostic { + schema_version: "missioncore.host-failure-diagnostic/v1"; + code: XgridsHostDiagnosticCode; + domain: XgridsHostDiagnosticDomain; + impact: XgridsHostDiagnosticImpact; + operator_action: XgridsHostDiagnosticAction; + automatic_retry: false; + redacted: true; +} export type AcquisitionState = | "preparing" @@ -58,43 +167,385 @@ export interface XgridsDeviceSession { connectivity?: "unknown" | "offline" | "connecting" | "connected" | "degraded"; } +export const XGRIDS_CONNECTION_VERIFICATION_STATUSES = [ + "not-probed", + "device-network-applied", + "device-network-applied-host-failed", + "adopted", + "host-route-mismatch", + "endpoint-unreachable", + "tcp-reachable-device-info-unverified", + "reachable", + "recovered", + "control-transport-lost", + "unreachable", +] as const; + +export const XGRIDS_CONNECTION_VERIFICATION_LEASE_STATES = [ + "disconnected", + "configured-unverified", + "reachable", +] as const; + +export const XGRIDS_CONNECTION_NETWORK_REACHABILITY = [ + "unknown", + "reachable", + "unreachable", +] as const; + +export type XgridsConnectionVerificationStatus = + typeof XGRIDS_CONNECTION_VERIFICATION_STATUSES[number]; +export type XgridsConnectionVerificationLeaseState = + typeof XGRIDS_CONNECTION_VERIFICATION_LEASE_STATES[number]; +export type XgridsConnectionNetworkReachability = + typeof XGRIDS_CONNECTION_NETWORK_REACHABILITY[number]; + export interface XgridsConnectionVerification { - status?: - | "not-probed" - | "configured" - | "live-address-observed" - | "reachable" - | "recovered" - | "unreachable"; - lease_state?: "disconnected" | "configured" | "reachable"; - lease_generation?: number; - endpoint_validation?: string | null; - network_reachability?: - | "unknown" - | "not-probed" - | "reachable" - | "degraded" - | "unreachable"; + status: XgridsConnectionVerificationStatus; + lease_state: XgridsConnectionVerificationLeaseState; + lease_generation: number; + supervisor_revision: number; + endpoint_validation: string | null; + network_reachability: XgridsConnectionNetworkReachability; + host_route_class?: string | null; + address_source?: string | null; + connection_origin?: string | null; + admission_source?: string | null; address_changed?: boolean; previous_address_present?: boolean; write_performed?: boolean; - observed_at?: string | null; + observed_at: string | null; + last_known_reachable_at?: string | null; reason_code?: string | null; } +export interface XgridsConfiguredEndpointProbe { + schema_version: "missioncore.xgrids-k1-configured-endpoint-probe/v1"; + status: "not-probed" | "reachable" | "endpoint-unreachable" | "host-route-unavailable"; + target_source: "current-supervisor" | "durable-semantic-topology" | null; + connection_mode: XgridsConnectionMode | null; + endpoint: string | null; + transport_ref: string | null; + intent_id: string | null; + semantic_revision: number | null; + host_route_available: boolean | null; + host_route_class: string | null; + tcp_reachable: boolean | null; + identity_validation: "not-performed"; + control_authority_granted: false; + ble_operation_performed: false; + network_mutation_performed: false; + automatic_retry: false; + observed_at: string | null; + reason_code: string | null; +} + export interface XgridsNetworkWriteReconciliation { - status: "device-state-unknown-after-write"; + status: + | "prepared-before-dispatch" + | "device-state-unknown-after-write" + | "durable-ledger-corrupt"; operation_id: string; transport_ref: string; - connection_mode: "bridge" | "quick-connect" | "direct-connect"; - operation_stage: string; + connection_mode: XgridsConnectionMode; + operation_stage: "prepared" | "dispatching" | "observing" | "ledger-corrupt"; reason_code: string; device_write_confirmed: boolean; - required_action: "explicit-read-only-ble-status-observation"; - scope: "process-runtime"; + required_action: + | "restart-service-to-resolve-prepared" + | "explicit-read-only-ble-status-observation" + | "operator-ledger-diagnosis"; + scope: "durable-ledger"; + ledger_revision: number | null; + observed_at: string | null; +} + +export interface XgridsNetworkMutationLedger { + status: "empty" | "unresolved" | "resolved" | "corrupt"; + mutation_allowed: boolean; + reason_code: string | null; + operation_id: string | null; + transport_ref: string | null; + intended_mode: XgridsConnectionMode | null; + stage: "prepared" | "dispatching" | "observing" | "resolved" | null; + revision: number | null; + resolution: + | "not-dispatched" + | "target-observed" + | "interrupted" + | "superseded" + | null; + updated_at_utc: string | null; + diagnostic?: XgridsHostFailureDiagnostic | null; +} + +export interface XgridsCurrentDeviceRecovery { + transport_ref?: string | null; + connection_mode?: XgridsConnectionMode | null; + handle_available?: boolean; + handle_retained?: boolean; + advertised_now?: boolean; + gatt_validated_recently?: boolean; + observed_at?: string | null; +} + +export const XGRIDS_CONNECTION_POLICY_ACTIONS = [ + "scan-ble", + "provision-fresh-device", + "prepare-select-device", + "prepare-change-network", + "cancel-reconfiguration", + "recover-current-device-network", + "observe-fresh-device-network", + "observe-current-device-network", + "observe-configured-device-network", + "inspect-configured-endpoint", + "inspect-host-network", + "probe-endpoint", + "verify-control-device-info", + "start-acquisition", + "stop-acquisition", + "stop-local-receiver", + "retire-unavailable-physical-target", + "acknowledge-data-loss", +] as const; +export type XgridsConnectionPolicyAction = + typeof XGRIDS_CONNECTION_POLICY_ACTIONS[number]; + +export const XGRIDS_CONNECTION_POLICY_TARGET_SOURCES = [ + "none", + "fresh-scan", + "retained-current-process", + "durable-configured-state", + "configured-topology", + "connection-supervisor", + "local-runtime", + "local-prestart-handoff", + "local-reconfiguration-intent", + "durable-physical-command", +] as const; +export type XgridsConnectionPolicyTargetSource = + typeof XGRIDS_CONNECTION_POLICY_TARGET_SOURCES[number]; + +export interface XgridsConnectionPolicyDecision { + allowed: boolean; + reason_codes: string[]; + target_source: XgridsConnectionPolicyTargetSource; + required_transport_ref: string | null; + required_connection_mode?: XgridsConnectionMode | null; + requires_live_gatt_validation: boolean; + automatic_retry: false; + execution_mode?: "capture-only"; + physical_command_allowed?: false; + physical_outcome?: "unknown"; + operator_follow_up?: "manual-device-stop-required"; + device_write_performed?: false; +} + +export interface XgridsConnectionPolicy { + schema_version: "missioncore.xgrids-k1-connection-policy/v1"; + supervisor_revision: number; + network_ledger_revision: number | null; + recommended_action: string; + allowed_actions: XgridsConnectionPolicyAction[]; + actions: Partial>; + facts: { + fresh_transport_refs: string[]; + retained_transport_ref: string | null; + retained_context_is_presence: false; + network_mutation_status: "empty" | "unresolved" | "resolved" | "corrupt"; + network_provisioning_idempotency_status: "empty" | "ready" | "blocked" | "corrupt"; + network_provisioning_idempotency_available: boolean; + network_provisioning_active_operation_id: string | null; + network_provisioning_active_operation_matches_ledger: boolean; + semantic_topology_store_status: "empty" | "available" | "corrupt"; + device_identity_pin_store_status: "empty" | "available" | "corrupt"; + physical_command_status: "empty" | "unresolved" | "resolved" | "corrupt"; + physical_command_requires_reconciliation: boolean; + retired_transport_refs?: string[]; + eligible_fresh_transport_refs?: string[]; + ble_runtime: { + active_operation_kind: string | null; + cleanup_pending: boolean; + poisoned: boolean; + }; + lifecycle_process_lease_holders: Array<"control" | "network">; + control_plane_state: "idle" | "healthy" | "stalled" | "lost"; + data_plane_state: "idle" | "healthy" | "stalled" | "lost"; + physical_network_state: "unknown" | "not-disputed"; + }; +} + +export const XGRIDS_CONNECTION_RECONFIGURATION_INTENTS = [ + "select-device", + "change-network", +] as const; +export type XgridsConnectionReconfigurationIntent = + typeof XGRIDS_CONNECTION_RECONFIGURATION_INTENTS[number]; + +export const XGRIDS_CONNECTION_RECONFIGURATION_STATUSES = [ + "idle", + "awaiting-fresh-scan", + "fresh-scan-completed", +] as const; +export type XgridsConnectionReconfigurationStatus = + typeof XGRIDS_CONNECTION_RECONFIGURATION_STATUSES[number]; + +export interface XgridsConnectionReconfiguration { + schema_version: "missioncore.xgrids-k1-connection-reconfiguration/v1"; + revision: number; + intent_id: string | null; + intent: XgridsConnectionReconfigurationIntent | null; + status: XgridsConnectionReconfigurationStatus; + required_transport_ref: string | null; + required_connection_mode: XgridsConnectionMode | null; + minimum_discovery_generation: number | null; + fresh_discovery_generation: number | null; + required_transport_observed: boolean | null; + prepared_at: string | null; + automatic_retry: false; +} + +export interface XgridsConnectionSupervisorTarget { + ipv4: string; + port: number; +} + +export interface XgridsConnectionSupervisorIntent { + intent_id: string; + requested_mode: XgridsConnectionMode; + expected_device_id: string | null; + requested_at: string; +} + +export interface XgridsConnectionSupervisorHostPath { + epoch: number; + available: boolean; + fingerprint: string | null; + interface: string | null; + source_ipv4: string | null; + route_class: "direct" | "default" | "tunnel" | "unavailable" | "unknown"; + reason_code: string | null; observed_at: string; } +export interface XgridsConnectionSupervisorEndpoint { + target: XgridsConnectionSupervisorTarget | null; + tcp_state: "unknown" | "reachable" | "unreachable"; + intent_id: string | null; + host_path_epoch: number | null; + reason_code: string | null; + observed_at: string | null; +} + +export interface XgridsConnectionSupervisorDeviceNetwork { + state: "unconfigured" | "applied"; + intent_id: string | null; + transport_ref: string | null; + connection_mode: XgridsConnectionMode | null; + target: XgridsConnectionSupervisorTarget | null; + source: "ble-post-write-status" | "ble-read-only-status" | null; + observed_at: string | null; +} + +export interface XgridsConnectionSupervisorDeviceIdentity { + state: "unverified" | "verified" | "stale" | "mismatch"; + intent_id: string | null; + logical_device_id: string | null; + compatibility_profile_id: string | null; + connection_mode: XgridsConnectionMode | null; + source: "mqtt-device-info" | null; + host_path_epoch: number | null; + observed_at: string | null; +} + +export interface XgridsConnectionSupervisorPlane { + state: "idle" | "healthy" | "stalled" | "lost"; + session_id: string | null; + host_path_epoch: number | null; + reason_code: string | null; + observed_at: string | null; +} + +export interface XgridsConnectionSupervisorLease { + state: "absent" | "configured-unverified" | "reachable" | "lost"; + generation: number; + intent_id: string | null; + host_path_epoch: number | null; + connection_mode: XgridsConnectionMode | null; + target: XgridsConnectionSupervisorTarget | null; + logical_device_id: string | null; + reason_code: string | null; + observed_at: string | null; +} + +export interface XgridsConnectionSupervisorAuthority { + network_mutation_allowed: boolean; + control_allowed: boolean; + acquisition_start_allowed: boolean; + data_ingest_authoritative: boolean; + physical_motion_allowed: false; + reason_codes: string[]; +} + +export interface XgridsConnectionSupervisorLastKnown { + connection_mode: XgridsConnectionMode; + target: XgridsConnectionSupervisorTarget; + logical_device_id: string; + compatibility_profile_id: string; + verified_at: string; +} + +export type XgridsConnectionSupervisorAllowedAction = + | "select-connection-intent" + | "inspect-host-network" + | "probe-endpoint" + | "verify-control-device-info" + | "start-acquisition" + | "stop-acquisition" + | "stop-local-receiver" + | "acknowledge-data-loss"; + +export interface XgridsConnectionSupervisor { + schema_version: "missioncore.k1-connection-supervisor/v1"; + revision: number; + closed: boolean; + intent: XgridsConnectionSupervisorIntent | null; + observed: { + device_network: XgridsConnectionSupervisorDeviceNetwork; + host_path: XgridsConnectionSupervisorHostPath; + endpoint: XgridsConnectionSupervisorEndpoint; + device_identity: XgridsConnectionSupervisorDeviceIdentity; + control_plane: XgridsConnectionSupervisorPlane; + data_plane: XgridsConnectionSupervisorPlane; + }; + lease: XgridsConnectionSupervisorLease; + authority: XgridsConnectionSupervisorAuthority; + last_known: XgridsConnectionSupervisorLastKnown | null; + diagnostics?: XgridsHostFailureDiagnostic[]; + allowed_actions: XgridsConnectionSupervisorAllowedAction[]; +} + +export interface XgridsSemanticTopologyRecord { + schema_version: "missioncore.xgrids-k1-semantic-topology/v1"; + revision: number; + transport_ref: string; + connection_mode: XgridsConnectionMode; + ipv4: string; + compatibility_profile_id: string; + firmware_version: string; + source: "ble-post-write-status" | "ble-read-only-status"; + observed_at_utc: string; +} + +export interface XgridsSemanticTopologyStore { + status: "empty" | "available" | "corrupt"; + configured_offline_evidence: boolean; + live_connection_authority: false; + reason_code: string | null; + record: XgridsSemanticTopologyRecord | null; +} + export interface XgridsCompatibilityState { profile_id?: string | null; decision?: "compatible" | "limited" | "unknown" | "incompatible"; @@ -142,10 +593,54 @@ export interface XgridsApplicationControlExecution { can_emit_requests: false; } +export interface XgridsPhysicalCommandState { + status: "empty" | "unresolved" | "resolved" | "corrupt"; + reason_code: string | null; + requires_reconciliation: boolean; + resolved_active_recovery_required?: boolean; + automatic_replay_allowed: false; + normal_session_recovery_supported: false; + recovery_requirement?: string | null; + runtime_bound: boolean; + reconciliation_ready: boolean; + observed_session_state?: "ready" | "scanning" | string | null; + active_operation_id?: string | null; + operator_retirement?: { + allowed: boolean; + reason_codes: string[]; + expected_operation_id: string | null; + expected_revision: number | null; + expected_transport_ref: string | null; + physical_outcome: "unknown"; + device_io_performed: false; + automatic_retry: false; + } | null; + operator_retirement_allowed?: boolean; + operator_retirement_reason_codes?: string[]; + operator_reconciliation_reopen?: { + allowed: boolean; + reason_codes: string[]; + expected_revision: number | null; + expected_retirement_id: string | null; + expected_transport_ref: string | null; + expected_discovery_generation: number | null; + expected_desired_mode: XgridsConnectionMode; + expected_desired_mode_revision: number; + device_io_performed: false; + automatic_retry: false; + } | null; + record?: (Record & { + action?: "start" | "stop"; + stage?: string; + resolution?: string | null; + }) | null; +} + export type XgridsApplicationControlPhase = | "idle" | "connecting" | "connection-ready" + | "active-recovery-requested" | "workspace-requested" | "workspace-ready" | "project-requested" @@ -162,6 +657,10 @@ export type XgridsApplicationControlPhase = export interface XgridsApplicationControlSession { mode: "interactive-canonical"; + inspection_only?: boolean; + inspection_promotion_allowed?: boolean; + session_generation: number; + state_revision: number; state: XgridsApplicationControlPhase; control_socket_open: boolean; can_open: boolean; @@ -222,9 +721,27 @@ export interface XgridsApplicationControlSession { automatic_retry?: false; } | null; safe_to_retry?: boolean; + host_diagnostic?: XgridsHostFailureDiagnostic; } | null; dialogue?: Record | null; transport?: Record | null; + verified_control?: { + logical_device_id: string; + compatibility_profile_id: string; + control_session_id: string; + source: "mqtt-device-info"; + intent_id: string; + transport_ref: string; + host_path_epoch: number; + target_ipv4: string; + target_port: number; + connection_mode: XgridsConnectionMode; + control_proof_revision: number; + control_proof_source: string; + control_proof_fresh: boolean; + control_proof_age_seconds?: number | null; + } | null; + physical_command?: XgridsPhysicalCommandState | null; } export interface XgridsAcquisition { @@ -269,8 +786,70 @@ export interface XgridsOperation { cancellable?: boolean; cancel_requested?: boolean; result?: Record | null; - error?: Record | null; + error?: (Record & { + host_diagnostic?: XgridsHostFailureDiagnostic; + }) | null; evidence_refs?: string[]; + context?: Record; + events?: XgridsOperationEvent[]; +} + +export interface XgridsOperationEvent { + schema_version: "missioncore.operation-event/v1"; + sequence: number; + status: OperationStatus; + stage_code: string; + message_code: string; + observed_at: string; + side_effect_status: string | null; + error_code: string | null; + safe_to_retry: boolean | null; + automatic_retry: false; +} + +export interface XgridsConnectionDiagnosticBundle { + schema_version: "missioncore.xgrids-k1-connection-diagnostic/v1"; + redacted: true; + generated_at_utc: string; + snapshot_runtime_id: string; + attempt: Omit; + network_mutation_ledger: XgridsNetworkMutationLedger; + connection_supervisor: Record; + automatic_retry: false; +} + +export const XGRIDS_CONNECTION_ATTEMPT_PHASES = [ + "network_applied", + "network_not_applied", + "network_outcome_unknown", +] as const; +export type XgridsConnectionAttemptPhase = + typeof XGRIDS_CONNECTION_ATTEMPT_PHASES[number]; + +export interface XgridsConnectionAttempt { + schema_version: "missioncore.xgrids-k1-connection-attempt/v1"; + attempt_id: string; + connection_mode: XgridsConnectionMode; + status: OperationStatus; + stage: string; + public_error_code: string | null; + side_effect_status: string; + phase: XgridsConnectionAttemptPhase; + control_state: "ready" | "control_not_ready" | "unknown"; + safe_next_action: + | "wait-for-current-attempt" + | "continue-with-control-verification" + | "verify-control-read-only" + | "start-acquisition" + | "stop-local-receiver" + | "retire-unavailable-physical-target" + | "scan-select-connect" + | "manual-recovery-required"; + automatic_retry: false; + accepted_at: string | null; + completed_at: string | null; + timeline: XgridsOperationEvent[]; + diagnostic_bundle?: XgridsConnectionDiagnosticBundle; } export interface XgridsK1Metrics { @@ -344,14 +923,129 @@ export interface XgridsSensorCatalog { streams?: XgridsSensorCatalogStream[]; } +export interface XgridsConnectionLifecycleBinding { + binding_key: string; + intent_id: string; + transport_ref: string; + connection_mode: XgridsConnectionMode; + target_ipv4: string; + target_port: number; + host_path_epoch: number; + control_session_id: string; + logical_device_id?: string | null; + compatibility_profile_id?: string | null; + control_proof_source?: string | null; + control_proof_revision?: number | null; +} + +export interface XgridsConnectionLifecycle { + schema_version: "missioncore.xgrids-k1-connection-lifecycle/v1"; + revision: number; + desired_mode: XgridsConnectionMode; + configured_mode: XgridsConnectionMode | null; + active_mode: XgridsConnectionMode | null; + mode_change: { + state: + | "disconnected" + | "awaiting-control" + | "ready" + | "switch-selected" + | "connecting"; + from: XgridsConnectionMode | null; + to: XgridsConnectionMode; + }; + mode_selection: { + allowed: boolean; + reason_codes: string[]; + automatic_retry: false; + }; + active_binding_key: string | null; + active_binding: XgridsConnectionLifecycleBinding | null; + connection_ready: boolean; + ready_to_start: boolean; + operation: XgridsConnectionAttempt | null; + allowed_actions: string[]; + automatic_retry: false; +} + +export interface XgridsActiveStreamRecovery { + schema_version: "missioncore.xgrids-k1-active-stream-recovery/v1"; + state: XgridsActiveStreamRecoveryState; + generation: number; + acquisition_id: string | null; + attempt: number; + started_at_utc: string | null; + elapsed_ms: number | null; + reason_code: string | null; + force_finish_allowed: boolean; + automatic_read_only_rebind: boolean; + automatic_command_retry: false; + start_performed: false; + stop_performed: false; + ble_operation_performed: false; + network_mutation_performed: false; + runtime_producer_generation: number | null; + camera_recovery: XgridsCameraRecoveryState; + camera_media_state: XgridsCameraMediaState; + camera_media_ready: boolean; + camera_epoch: XgridsCameraMediaEpoch | null; +} + +export interface XgridsCameraMediaEpoch { + generation: number; + init_committed: boolean; + init_committed_age_ms: number | null; + first_media_committed: boolean; + first_media_committed_age_ms: number | null; + committed_media_segment_count: number; + last_media_segment_age_ms: number | null; +} + export interface XgridsK1State { contract_version?: string | null; + snapshot_runtime_started_at_utc?: string | null; + snapshot_runtime_started_monotonic_ns?: string | null; + snapshot_runtime_id?: string | null; + snapshot_revision?: number | null; + producer_generation?: number | null; phase?: string | null; message?: string | null; devices?: BleDevice[]; selected_device_id?: string | null; + ble_discovery_generation?: number; k1_ip?: string | null; - connection_mode?: "bridge" | "quick-connect" | "direct-connect" | null; + /** Legacy alias for the last configured mode. */ + connection_mode?: XgridsConnectionMode | null; + configured_connection_mode?: XgridsConnectionMode | null; + active_connection_mode?: XgridsConnectionMode | null; + desired_connection_mode?: XgridsConnectionMode; + desired_connection_mode_revision?: number; + connection_scenario_reset?: { + reset_id: string; + request_revision: number; + revision: number; + desired_mode: XgridsConnectionMode; + /** Missing is treated as active while rolling out the marker lifecycle. */ + active?: boolean; + settled_by_discovery_generation?: number | null; + local_session_closed: true; + previous_device_may_continue_scanning: boolean; + physical_disposition: string; + network_disposition: string | null; + device_command_performed: false; + network_write_performed: false; + automatic_scan: false; + operation_sequence: number; + } | null; + connection_scenario_reset_pending?: { + reset_id: string; + desired_mode: XgridsConnectionMode; + expected_revision: number; + status: "waiting-for-local-lifecycle" | "retiring-local-session"; + device_command_performed: false; + network_write_performed: false; + automatic_scan: false; + } | null; foxglove_ws_url?: string | null; foxglove_viewer_url?: string | null; rerun_grpc_url?: string | null; @@ -362,13 +1056,24 @@ export interface XgridsK1State { modeling_control_safety?: XgridsModelingControlSafety | null; application_control_execution?: XgridsApplicationControlExecution | null; application_control_session?: XgridsApplicationControlSession | null; + physical_command?: XgridsPhysicalCommandState | null; device_ref?: XgridsDeviceRef | null; device_session?: XgridsDeviceSession | null; connection_verification?: XgridsConnectionVerification | null; + configured_endpoint_probe?: XgridsConfiguredEndpointProbe | null; network_write_reconciliation?: XgridsNetworkWriteReconciliation | null; + network_mutation_ledger?: XgridsNetworkMutationLedger | null; + current_device_recovery?: XgridsCurrentDeviceRecovery | null; + connection_supervisor?: XgridsConnectionSupervisor | null; + connection_lifecycle?: XgridsConnectionLifecycle | null; + connection_recovery?: XgridsActiveStreamRecovery | null; + connection_reconfiguration?: XgridsConnectionReconfiguration | null; + connection_policy?: XgridsConnectionPolicy | null; + semantic_topology_store?: XgridsSemanticTopologyStore | null; acquisition?: XgridsAcquisition | null; operations?: XgridsOperation[]; last_operation?: XgridsOperation | null; + connection_attempt?: XgridsConnectionAttempt | null; sensor_catalog?: XgridsSensorCatalog | null; camera_preview?: XgridsCameraPreviewState | null; } @@ -382,6 +1087,7 @@ export interface HealthResponse { export interface ScanRequest { duration_seconds?: number; + operation_id?: string; } export interface CompatibilityAttestation { @@ -397,14 +1103,81 @@ export interface ConnectRequest { connection_mode: "bridge" | "quick-connect" | "direct-connect"; compatibility_attestation: CompatibilityAttestation; operation_id?: string; - idempotency_key?: string; + idempotency_key: string; + expected_mode_revision: number; + expected_discovery_generation: number; + expected_reconfiguration_revision: number; + expected_reconfiguration_intent_id?: string; } -export interface ConnectionVerifyRequest { +export interface SelectConnectionModeRequest { + connection_mode: XgridsConnectionMode; + expected_revision: number; + reset_scenario?: true; + reset_id?: string; +} + +export interface PrepareConnectionReconfigurationRequest { + intent: XgridsConnectionReconfigurationIntent | "cancel"; + expected_reconfiguration_revision: number; + expected_reconfiguration_intent_id: string | null; + expected_desired_mode_revision: number; + expected_active_binding_key: string | null; +} + +interface ConnectionVerifyRequestBase { device_id: string; compatibility_attestation: CompatibilityAttestation; + operation_id?: string; } +export type ConnectionVerifyRequest = + | (ConnectionVerifyRequestBase & { + source: "fresh-scan"; + expected_discovery_generation: number; + expected_reconfiguration_revision: number; + expected_reconfiguration_intent_id?: string; + }) + | (ConnectionVerifyRequestBase & { + source: Extract< + XgridsConnectionPolicyTargetSource, + "retained-current-process" | "durable-configured-state" + >; + expected_discovery_generation?: never; + }); + +export interface ConfiguredEndpointProbeRequest { + operation_id?: string; +} + +export interface RetireUnavailablePhysicalCommandRequest { + retirement_id: string; + expected_operation_id: string; + expected_revision: number; + expected_transport_ref: string; + operator_confirmed: true; + reason: "device-permanently-unavailable-or-replaced"; +} + +export interface ReopenRetiredPhysicalReconciliationRequest { + reopening_id: string; + expected_revision: number; + expected_retirement_id: string; + expected_transport_ref: string; + expected_discovery_generation: number; + expected_desired_mode: XgridsConnectionMode; + expected_desired_mode_revision: number; + operator_confirmed: true; + reason: "device-returned-for-explicit-reconciliation"; +} + +export interface SnapshotRuntimeFenceRequest { + expected_snapshot_runtime_id: string; +} + +export type SnapshotRuntimeFencedRequest = + TRequest & SnapshotRuntimeFenceRequest; + export interface PrepareAcquisitionRequest { project_name: string; mount_type: "handheld"; @@ -415,8 +1188,10 @@ export interface PrepareAcquisitionRequest { evidence_policy?: "required" | "best-effort" | "disabled"; compatibility_attestation: CompatibilityAttestation; operation_id?: string; - idempotency_key?: string; + idempotency_key: string; deadline_seconds?: number; + expected_control_session_generation?: number; + expected_control_state_revision?: number; } export type RequestedStreamId = @@ -429,26 +1204,51 @@ export type RequestedStreamId = export interface StartAcquisitionRequest { acquisition_id: string; expected_state_revision?: number; + expected_control_session_generation: number; + expected_control_state_revision: number; operation_id?: string; - idempotency_key?: string; + idempotency_key: string; deadline_seconds?: number; physical_acceptance?: OperatorPresenceConfirmation; } -export interface StopAcquisitionRequest { +interface StopAcquisitionRequestBase { acquisition_id: string; - mode: "capture-only" | "graceful"; operator_confirmed?: boolean; operation_id?: string; - idempotency_key?: string; + idempotency_key: string; deadline_seconds?: number; physical_acceptance?: OperatorPresenceConfirmation; } +export type StopAcquisitionRequest = + | (StopAcquisitionRequestBase & { + mode: "capture-only"; + expected_control_session_generation?: never; + expected_control_state_revision?: never; + }) + | (StopAcquisitionRequestBase & { + mode: "graceful"; + expected_control_session_generation: number; + expected_control_state_revision: number; + }); + export interface AbortAcquisitionRequest { acquisition_id: string; operation_id?: string; - idempotency_key?: string; + idempotency_key: string; + deadline_seconds?: number; + expected_control_session_generation?: number; + expected_control_state_revision?: number; +} + +export interface ForceFinishAcquisitionRequest { + acquisition_id: string; + expected_state_revision: number; + expected_recovery_generation: number; + operator_confirmed: true; + operation_id?: string; + idempotency_key: string; deadline_seconds?: number; } @@ -496,86 +1296,568 @@ export interface OpenApplicationControlSessionRequest export interface EnterApplicationWorkspaceRequest { operator_confirmed: true; + expected_session_generation: number; + expected_state_revision: number; +} + +export interface CloseApplicationControlSessionRequest { + expected_session_generation: number; + expected_state_revision: number; +} + +export interface ReconcilePhysicalCommandRequest { + reconciliation_id: string; + expected_session_generation: number; + expected_state_revision: number; } export class ApiError extends Error { readonly status: number; readonly transportUnavailable: boolean; + readonly hostDiagnostic: XgridsHostFailureDiagnostic | null; - constructor(message: string, status = 0, transportUnavailable = false) { + constructor( + message: string, + status = 0, + transportUnavailable = false, + hostDiagnostic: unknown = null, + ) { super(message); this.name = "ApiError"; this.status = status; this.transportUnavailable = transportUnavailable; + this.hostDiagnostic = isXgridsHostFailureDiagnostic(hostDiagnostic) + ? hostDiagnostic + : null; } } +export class ApiRequestTimeoutError extends ApiError { + readonly code = "request_timeout_outcome_unknown"; + readonly outcomeUnknown = true; + readonly automaticRetry = false; + readonly timeoutMs: number; + readonly operationLabel: string; + + constructor(operationLabel: string, timeoutMs: number) { + super( + `Локальный запрос «${operationLabel}» не завершился за ${Math.ceil(timeoutMs / 1000)} с. Результат операции неизвестен; автоматический повтор запрещён. Обновите состояние перед отдельным ручным действием.`, + ); + this.name = "ApiRequestTimeoutError"; + this.timeoutMs = timeoutMs; + this.operationLabel = operationLabel; + } +} + +const DEFAULT_REQUEST_TIMEOUT_MS = 8_000; +const ACTION_REQUEST_TIMEOUT_MS: Readonly> = { + [xgridsK1Actions.stateRead]: 8_000, + [xgridsK1Actions.discoveryScan]: 70_000, + // Explicit scenario reset queues behind a bounded in-flight lifecycle and + // then performs local-only retirement. Keep the browser request alive long + // enough to observe that single committed intent. + [xgridsK1Actions.connectionModeSelect]: 360_000, + [xgridsK1Actions.connectionReconfigurePrepare]: 8_000, + // Bridge and Quick are composite operations: exact BLE baseline/write, + // bounded post-write observation and, for Quick, a native CoreWLAN handoff. + // The backend journals a 240-second operation fence. The browser deadline + // must outlive it; aborting the HTTP request earlier cannot cancel the + // native/device work and would present an avoidable ambiguous outcome. + [xgridsK1Actions.networkProvision]: 300_000, + // A canonical verification performs a bounded fresh CoreBluetooth connect + // and 7f02 status read before any route/TCP interpretation. The browser + // must not abandon that read at the generic eight-second HTTP deadline and + // accidentally invite a second operator attempt while native cleanup still + // owns the BLE lifecycle. + [xgridsK1Actions.connectionVerify]: 150_000, + // This is the separate host-only route -> TCP -> route diagnostic. It does + // not enter CoreBluetooth and has a much smaller bounded server deadline. + [xgridsK1Actions.configuredEndpointProbe]: 15_000, + [xgridsK1Actions.applicationControlSessionOpen]: 120_000, + [xgridsK1Actions.applicationControlWorkspaceEnter]: 120_000, + [xgridsK1Actions.applicationControlSessionClose]: 120_000, + [xgridsK1Actions.physicalCommandReconcile]: 120_000, + [xgridsK1Actions.physicalCommandRetireUnavailable]: 8_000, + [xgridsK1Actions.physicalCommandReopenRetiredReconciliation]: 8_000, + [xgridsK1Actions.acquisitionPrepare]: 120_000, + [xgridsK1Actions.acquisitionStart]: 120_000, + [xgridsK1Actions.acquisitionStop]: 120_000, + [xgridsK1Actions.acquisitionForceFinishLocal]: 45_000, +}; + +export function requestTimeoutMsForAction(actionId: string): number { + return ACTION_REQUEST_TIMEOUT_MS[actionId] ?? DEFAULT_REQUEST_TIMEOUT_MS; +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +function isLiteralValue( + value: unknown, + values: Values, +): value is Values[number] { + return typeof value === "string" && values.some((candidate) => candidate === value); +} + +function isOptionalString(value: unknown): value is string | null | undefined { + return value === undefined || value === null || typeof value === "string"; +} + +function isOptionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === "boolean"; +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function isStringOrNull(value: unknown): value is string | null { + return value === null || typeof value === "string"; +} + +function isNonNegativeIntegerOrNull(value: unknown): value is number | null { + return value === null || isNonNegativeInteger(value); +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string"); +} + +export function isXgridsConnectionAttemptPhase( + value: unknown, +): value is XgridsConnectionAttemptPhase { + return isLiteralValue(value, XGRIDS_CONNECTION_ATTEMPT_PHASES); +} + +export function isXgridsHostFailureDiagnostic( + value: unknown, +): value is XgridsHostFailureDiagnostic { + return isRecord(value) + && value.schema_version === "missioncore.host-failure-diagnostic/v1" + && isLiteralValue(value.code, XGRIDS_HOST_DIAGNOSTIC_CODES) + && isLiteralValue(value.domain, XGRIDS_HOST_DIAGNOSTIC_DOMAINS) + && isLiteralValue(value.impact, XGRIDS_HOST_DIAGNOSTIC_IMPACTS) + && isLiteralValue(value.operator_action, XGRIDS_HOST_DIAGNOSTIC_ACTIONS) + && value.automatic_retry === false + && value.redacted === true; +} + +export function isXgridsActiveStreamRecovery( + value: unknown, +): value is XgridsActiveStreamRecovery { + if (!isRecord(value)) return false; + return value.schema_version + === "missioncore.xgrids-k1-active-stream-recovery/v1" + && isLiteralValue(value.state, XGRIDS_ACTIVE_STREAM_RECOVERY_STATES) + && isNonNegativeInteger(value.generation) + && isStringOrNull(value.acquisition_id) + && isNonNegativeInteger(value.attempt) + && isStringOrNull(value.started_at_utc) + && isNonNegativeIntegerOrNull(value.elapsed_ms) + && isStringOrNull(value.reason_code) + && typeof value.force_finish_allowed === "boolean" + && typeof value.automatic_read_only_rebind === "boolean" + && value.automatic_command_retry === false + && value.start_performed === false + && value.stop_performed === false + && value.ble_operation_performed === false + && value.network_mutation_performed === false + && isNonNegativeIntegerOrNull(value.runtime_producer_generation) + && isLiteralValue(value.camera_recovery, XGRIDS_CAMERA_RECOVERY_STATES) + && isXgridsCameraMediaProjection(value); +} + +function isXgridsCameraMediaEpoch(value: unknown): value is XgridsCameraMediaEpoch { + return isRecord(value) + && isNonNegativeInteger(value.generation) + && value.generation > 0 + && typeof value.init_committed === "boolean" + && isNonNegativeIntegerOrNull(value.init_committed_age_ms) + && typeof value.first_media_committed === "boolean" + && isNonNegativeIntegerOrNull(value.first_media_committed_age_ms) + && isNonNegativeInteger(value.committed_media_segment_count) + && isNonNegativeIntegerOrNull(value.last_media_segment_age_ms); +} + +function isXgridsCameraMediaProjection( + value: Record, +): boolean { + if ( + !isLiteralValue(value.camera_media_state, XGRIDS_CAMERA_MEDIA_STATES) + || typeof value.camera_media_ready !== "boolean" + ) return false; + const state = value.camera_media_state; + const epoch = value.camera_epoch; + if (state === "inactive" || state === "pending-epoch") { + return value.camera_media_ready === false && epoch === null; + } + if (!isXgridsCameraMediaEpoch(epoch) || value.camera_media_ready !== (state === "ready")) { + return false; + } + if (state === "pending-init") { + return epoch.init_committed === false + && epoch.init_committed_age_ms === null + && epoch.first_media_committed === false + && epoch.first_media_committed_age_ms === null + && epoch.committed_media_segment_count === 0 + && epoch.last_media_segment_age_ms === null; + } + if (state === "pending-first-media") { + return epoch.init_committed === true + && isNonNegativeInteger(epoch.init_committed_age_ms) + && epoch.first_media_committed === false + && epoch.first_media_committed_age_ms === null + && epoch.committed_media_segment_count === 0 + && epoch.last_media_segment_age_ms === null; + } + return epoch.init_committed === true + && isNonNegativeInteger(epoch.init_committed_age_ms) + && epoch.first_media_committed === true + && isNonNegativeInteger(epoch.first_media_committed_age_ms) + && epoch.committed_media_segment_count > 0 + && isNonNegativeInteger(epoch.last_media_segment_age_ms); +} + +export function isXgridsConnectionVerification( + value: unknown, +): value is XgridsConnectionVerification { + if (!isRecord(value)) return false; + return ( + isLiteralValue(value.status, XGRIDS_CONNECTION_VERIFICATION_STATUSES) + && isLiteralValue( + value.lease_state, + XGRIDS_CONNECTION_VERIFICATION_LEASE_STATES, + ) + && isNonNegativeInteger(value.lease_generation) + && isNonNegativeInteger(value.supervisor_revision) + && isStringOrNull(value.endpoint_validation) + && isLiteralValue( + value.network_reachability, + XGRIDS_CONNECTION_NETWORK_REACHABILITY, + ) + && isOptionalString(value.host_route_class) + && isOptionalString(value.address_source) + && isOptionalString(value.connection_origin) + && isOptionalString(value.admission_source) + && isOptionalBoolean(value.address_changed) + && isOptionalBoolean(value.previous_address_present) + && isOptionalBoolean(value.write_performed) + && isStringOrNull(value.observed_at) + && isOptionalString(value.last_known_reachable_at) + && isOptionalString(value.reason_code) + ); +} + +export function isXgridsConnectionReconfiguration( + value: unknown, +): value is XgridsConnectionReconfiguration { + if (!isRecord(value)) return false; + const intent = value.intent; + const intentId = value.intent_id; + const status = value.status; + const requiredTransportRef = value.required_transport_ref; + const requiredMode = value.required_connection_mode; + const observed = value.required_transport_observed; + return Boolean( + value.schema_version + === "missioncore.xgrids-k1-connection-reconfiguration/v1" + && isNonNegativeInteger(value.revision) + && isStringOrNull(intentId) + && ( + intent === null + || isLiteralValue(intent, XGRIDS_CONNECTION_RECONFIGURATION_INTENTS) + ) + && isLiteralValue( + status, + XGRIDS_CONNECTION_RECONFIGURATION_STATUSES, + ) + && isStringOrNull(requiredTransportRef) + && ( + requiredMode === null + || isLiteralValue(requiredMode, XGRIDS_CONNECTION_MODES) + ) + && isNonNegativeIntegerOrNull(value.minimum_discovery_generation) + && isNonNegativeIntegerOrNull(value.fresh_discovery_generation) + && (observed === null || typeof observed === "boolean") + && isStringOrNull(value.prepared_at) + && value.automatic_retry === false + && ( + status === "idle" + ? intent === null + && intentId === null + && requiredTransportRef === null + && requiredMode === null + && value.minimum_discovery_generation === null + && value.fresh_discovery_generation === null + && observed === null + && value.prepared_at === null + : intent !== null + && Boolean(intentId?.trim()) + && requiredMode === "bridge" + && isNonNegativeInteger(value.minimum_discovery_generation) + && Boolean(value.prepared_at?.trim()) + && ( + intent === "select-device" + ? requiredTransportRef === null + : Boolean(requiredTransportRef?.trim()) + ) + && ( + status === "awaiting-fresh-scan" + ? value.fresh_discovery_generation === null + && observed === null + : isNonNegativeInteger(value.fresh_discovery_generation) + && value.fresh_discovery_generation + >= value.minimum_discovery_generation + && ( + intent === "change-network" + ? typeof observed === "boolean" + : observed === null + ) + ) + ) + ); +} + +export function isXgridsConnectionPolicyDecision( + value: unknown, +): value is XgridsConnectionPolicyDecision { + if (!isRecord(value)) return false; + const requiredMode = value.required_connection_mode; + return ( + typeof value.allowed === "boolean" + && isStringArray(value.reason_codes) + && isLiteralValue(value.target_source, XGRIDS_CONNECTION_POLICY_TARGET_SOURCES) + && isStringOrNull(value.required_transport_ref) + && ( + requiredMode === undefined + || requiredMode === null + || isLiteralValue(requiredMode, XGRIDS_CONNECTION_MODES) + ) + && typeof value.requires_live_gatt_validation === "boolean" + && value.automatic_retry === false + && ( + value.execution_mode === undefined + || value.execution_mode === "capture-only" + ) + && ( + value.physical_command_allowed === undefined + || value.physical_command_allowed === false + ) + && ( + value.physical_outcome === undefined + || value.physical_outcome === "unknown" + ) + && ( + value.operator_follow_up === undefined + || value.operator_follow_up === "manual-device-stop-required" + ) + && ( + value.device_write_performed === undefined + || value.device_write_performed === false + ) + ); +} + +export function isXgridsConnectionPolicy( + value: unknown, +): value is XgridsConnectionPolicy { + if ( + !isRecord(value) + || value.schema_version !== "missioncore.xgrids-k1-connection-policy/v1" + || !isNonNegativeInteger(value.supervisor_revision) + || !isNonNegativeIntegerOrNull(value.network_ledger_revision) + || typeof value.recommended_action !== "string" + || !Array.isArray(value.allowed_actions) + || !isRecord(value.actions) + || !isRecord(value.facts) + || value.facts.retained_context_is_presence !== false + ) { + return false; + } + + const allowedActions: XgridsConnectionPolicyAction[] = []; + const policyActions = value.actions; + for (const action of value.allowed_actions) { + if (!isLiteralValue(action, XGRIDS_CONNECTION_POLICY_ACTIONS)) return false; + allowedActions.push(action); + } + + for (const [action, decision] of Object.entries(policyActions)) { + if ( + !isLiteralValue(action, XGRIDS_CONNECTION_POLICY_ACTIONS) + || !isXgridsConnectionPolicyDecision(decision) + ) { + return false; + } + if (decision.allowed !== allowedActions.includes(action)) return false; + + if ( + action === "observe-current-device-network" + && decision.target_source !== "retained-current-process" + ) { + return false; + } + if ( + action === "observe-configured-device-network" + && decision.target_source !== "durable-configured-state" + ) { + return false; + } + if ( + action === "retire-unavailable-physical-target" + && ( + decision.target_source !== "durable-physical-command" + || decision.physical_command_allowed !== false + || decision.physical_outcome !== "unknown" + || decision.device_write_performed !== false + || decision.requires_live_gatt_validation !== false + ) + ) { + return false; + } + if ( + decision.allowed + && (action === "observe-current-device-network" + || action === "observe-configured-device-network") + && ( + !decision.required_transport_ref?.trim() + || decision.required_connection_mode == null + ) + ) { + return false; + } + } + + return allowedActions.every((action) => { + const decision = policyActions[action]; + return isRecord(decision) && decision.allowed === true; + }); +} + +function hasValidRuntimeContracts( + value: unknown, +): value is XgridsK1State { + if (!isRecord(value)) return false; + const verification = value.connection_verification; + const reconfiguration = value.connection_reconfiguration; + const policy = value.connection_policy; + const attempt = value.connection_attempt; + const activeStreamRecovery = value.connection_recovery; + const verificationValid = verification === undefined + || verification === null + || isXgridsConnectionVerification(verification); + const policyValid = policy === undefined + || policy === null + || isXgridsConnectionPolicy(policy); + const reconfigurationValid = reconfiguration === undefined + || reconfiguration === null + || isXgridsConnectionReconfiguration(reconfiguration); + const attemptValid = attempt === undefined + || attempt === null + || ( + isRecord(attempt) + && attempt.schema_version === "missioncore.xgrids-k1-connection-attempt/v1" + && isXgridsConnectionAttemptPhase(attempt.phase) + && attempt.automatic_retry === false + ); + const activeStreamRecoveryValid = activeStreamRecovery === undefined + || activeStreamRecovery === null + || isXgridsActiveStreamRecovery(activeStreamRecovery); + const producerGenerationValid = value.producer_generation === undefined + || value.producer_generation === null + || isNonNegativeInteger(value.producer_generation); + return verificationValid + && reconfigurationValid + && policyValid + && attemptValid + && activeStreamRecoveryValid + && producerGenerationValid; +} + function unwrapState(payload: unknown): XgridsK1State { const value = isRecord(payload) && isRecord(payload.state) ? payload.state : payload; - if (!isRecord(value)) { + if (!hasValidRuntimeContracts(value)) { throw new ApiError("Локальный сервер вернул некорректное состояние."); } - return value as XgridsK1State; + return value; } -async function requestJson(path: string, init?: RequestInit): Promise { - let response: Response; +async function requestJson( + path: string, + init: RequestInit | undefined, + operationLabel: string, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + controller.abort(); + }, timeoutMs); try { - response = await fetch(path, { + const response = await fetch(path, { ...init, + signal: controller.signal, headers: { Accept: "application/json", ...(init?.body ? { "Content-Type": "application/json" } : {}), ...init?.headers, }, }); - } catch { + const bodyText = await response.text(); + let body: unknown; + + if (bodyText) { + try { + body = JSON.parse(bodyText) as unknown; + } catch { + body = bodyText; + } + } + + if (!response.ok) { + const detail = + isRecord(body) && typeof body.detail === "string" + ? body.detail + : typeof body === "string" && body.trim() + ? body.trim() + : `Запрос к API устройства завершился ошибкой HTTP ${response.status}.`; + throw new ApiError( + detail || `Запрос к API устройства завершился ошибкой HTTP ${response.status}.`, + response.status, + ); + } + + return body; + } catch (error) { + if (error instanceof ApiError) throw error; + if (timedOut) { + throw new ApiRequestTimeoutError(operationLabel, timeoutMs); + } throw new ApiError( "Не удалось подключиться к локальному сервису устройства.", 0, true, ); + } finally { + clearTimeout(timeout); } - - const bodyText = await response.text(); - let body: unknown; - - if (bodyText) { - try { - body = JSON.parse(bodyText) as unknown; - } catch { - body = bodyText; - } - } - - if (!response.ok) { - const detail = - isRecord(body) && typeof body.detail === "string" - ? body.detail - : typeof body === "string" && body.trim() - ? body.trim() - : `Запрос к API устройства завершился ошибкой HTTP ${response.status}.`; - throw new ApiError( - detail || `Запрос к API устройства завершился ошибкой HTTP ${response.status}.`, - response.status, - ); - } - - return body; } -async function postState(path: string, body?: object): Promise { +async function postState( + path: string, + body: object | undefined, + operationLabel: string, + timeoutMs: number, +): Promise { const payload = await requestJson(path, { method: "POST", body: body ? JSON.stringify(body) : undefined, - }); + }, operationLabel, timeoutMs); if (payload === undefined) { return xgridsK1Api.getState(); @@ -588,12 +1870,19 @@ function invokeState(actionId: string, input: object = {}): Promise { - const payload = await requestJson("/api/health"); + const payload = await requestJson( + "/api/health", + undefined, + "health.read", + DEFAULT_REQUEST_TIMEOUT_MS, + ); if (!isRecord(payload)) { throw new ApiError("Локальный сервер вернул некорректный ответ проверки."); } @@ -604,34 +1893,87 @@ export const xgridsK1Api = { return invokeState(xgridsK1Actions.stateRead); }, - scanBle(body: ScanRequest = {}): Promise { + scanBle( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.discoveryScan, body); }, - connect(body: ConnectRequest): Promise { + selectConnectionMode( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.connectionModeSelect, body); + }, + + prepareConnectionReconfiguration( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.connectionReconfigurePrepare, body); + }, + + connect( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.networkProvision, body); }, - verifyConnection(body?: ConnectionVerifyRequest): Promise { + verifyConnection( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.connectionVerify, body); }, - prepareAcquisition(body: PrepareAcquisitionRequest): Promise { + probeConfiguredEndpoint( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.configuredEndpointProbe, body); + }, + + retireUnavailablePhysicalCommand( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.physicalCommandRetireUnavailable, body); + }, + + reopenRetiredPhysicalReconciliation( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState( + xgridsK1Actions.physicalCommandReopenRetiredReconciliation, + body, + ); + }, + + prepareAcquisition( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.acquisitionPrepare, body); }, - startAcquisition(body: StartAcquisitionRequest): Promise { + startAcquisition( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.acquisitionStart, body); }, - stopAcquisition(body: StopAcquisitionRequest): Promise { + stopAcquisition( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.acquisitionStop, body); }, - abortAcquisition(body: AbortAcquisitionRequest): Promise { + abortAcquisition( + body: SnapshotRuntimeFencedRequest, + ): Promise { return invokeState(xgridsK1Actions.acquisitionAbort, body); }, + forceFinishAcquisitionLocally( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.acquisitionForceFinishLocal, body); + }, + startReplay(body: ReplayRequest): Promise { return invokeState(xgridsK1Actions.streamStartReplay, body); }, @@ -640,8 +1982,10 @@ export const xgridsK1Api = { return invokeState(xgridsK1Actions.compatibilityStreamStartLive, body); }, - stopSessionCompatibility(): Promise { - return invokeState(xgridsK1Actions.compatibilityStreamStop); + stopSessionCompatibility( + body: SnapshotRuntimeFenceRequest, + ): Promise { + return invokeState(xgridsK1Actions.compatibilityStreamStop, body); }, selectCameraPreview(body: SelectCameraPreviewRequest): Promise { @@ -671,19 +2015,27 @@ export const xgridsK1Api = { }, openApplicationControlSession( - body: OpenApplicationControlSessionRequest, + body: SnapshotRuntimeFencedRequest, ): Promise { return invokeState(xgridsK1Actions.applicationControlSessionOpen, body); }, enterApplicationWorkspace( - body: EnterApplicationWorkspaceRequest, + body: SnapshotRuntimeFencedRequest, ): Promise { return invokeState(xgridsK1Actions.applicationControlWorkspaceEnter, body); }, - closeApplicationControlSession(): Promise { - return invokeState(xgridsK1Actions.applicationControlSessionClose); + closeApplicationControlSession( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.applicationControlSessionClose, body); + }, + + reconcilePhysicalCommand( + body: SnapshotRuntimeFencedRequest, + ): Promise { + return invokeState(xgridsK1Actions.physicalCommandReconcile, body); }, }; diff --git a/plugins/xgrids-k1/frontend/src/components/ActiveStreamRecoverySurface.tsx b/plugins/xgrids-k1/frontend/src/components/ActiveStreamRecoverySurface.tsx new file mode 100644 index 0000000..846a39c --- /dev/null +++ b/plugins/xgrids-k1/frontend/src/components/ActiveStreamRecoverySurface.tsx @@ -0,0 +1,172 @@ +import { + ActivityIndicator, + Button, + GlassSurface, + StatusBadge, +} from "@nodedc/ui-react"; + +import type { ActiveStreamRecoveryPresentation } from "../activeStreamRecovery"; + +export type ActiveStreamRecoverySurfaceVariant = "panel" | "compact"; + +export interface ActiveStreamRecoverySurfaceProps { + presentation: ActiveStreamRecoveryPresentation | null; + forceFinishing: boolean; + actionBusy: boolean; + onForceFinish: () => void; + variant?: ActiveStreamRecoverySurfaceVariant; +} + +interface ActiveStreamRecoverySurfaceCopy { + eyebrow: string; + title: string; + statusLabel: string; + detail: string; + showSpinner: boolean; + forceFinishAvailable: boolean; +} + +function surfaceCopy({ + presentation, + forceFinishing, +}: Pick< + ActiveStreamRecoverySurfaceProps, + "presentation" | "forceFinishing" +>): ActiveStreamRecoverySurfaceCopy { + return { + eyebrow: forceFinishing + ? "СВЯЗЬ · ЛОКАЛЬНОЕ ЗАВЕРШЕНИЕ" + : presentation?.eyebrow ?? "СВЯЗЬ · АКТИВНЫЙ ПРИЁМ", + title: forceFinishing + ? "Завершаем локальный приём" + : presentation?.title ?? "Восстанавливаем соединение", + statusLabel: forceFinishing + ? "Локальное завершение" + : presentation?.statusLabel ?? "Восстановление связи", + detail: forceFinishing + ? "Закрываем только локальный приём и сохранение. Команда STOP устройству не отправляется." + : presentation?.detail ?? "Проверяем состояние активного приёма.", + showSpinner: forceFinishing || presentation?.showSpinner === true, + forceFinishAvailable: + !forceFinishing && presentation?.forceFinishAvailable === true, + }; +} + +function RecoveryState({ + presentation, + copy, +}: { + presentation: ActiveStreamRecoveryPresentation | null; + copy: ActiveStreamRecoverySurfaceCopy; +}) { + const stateClassName = copy.showSpinner + ? "active-stream-recovery__state" + : "active-stream-recovery__state active-stream-recovery__state--static"; + return ( +
+ {copy.showSpinner ? : null} +
+ {copy.title} + {copy.detail} + {presentation?.progressLabel ? ( + {presentation.progressLabel} + ) : null} +
+
+ ); +} + +function RecoveryAction({ + visible, + actionBusy, + compact, + onForceFinish, +}: { + visible: boolean; + actionBusy: boolean; + compact: boolean; + onForceFinish: () => void; +}) { + if (!visible) return null; + return ( +
+ +

+ Завершит только локальный front/back-приём и сохранение. START, STOP, + Bluetooth и настройки устройства не отправляются. +

+
+ ); +} + +/** + * One shared recovery owner for the connection and spatial workspaces. + * + * The surface never chooses a mutation by itself: its sole callback is the + * explicitly fenced local force-finish action supplied by the K1 controller. + */ +export function ActiveStreamRecoverySurface({ + presentation, + forceFinishing, + actionBusy, + onForceFinish, + variant = "panel", +}: ActiveStreamRecoverySurfaceProps) { + const copy = surfaceCopy({ presentation, forceFinishing }); + const tone = forceFinishing ? "neutral" : presentation?.tone ?? "neutral"; + const content = ( + <> + + + + ); + + if (variant === "compact") { + return ( +
+
+ {copy.eyebrow} + {copy.statusLabel} +
+
+ {content} +
+
+ ); + } + + return ( + +
+
+ {copy.eyebrow} +

{copy.title}

+
+ {copy.statusLabel} +
+
+ {content} +
+ + ); +} diff --git a/plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx b/plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx index b08aacd..a6469dc 100644 --- a/plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx +++ b/plugins/xgrids-k1/frontend/src/components/K1AcquisitionPipeline.tsx @@ -1,5 +1,6 @@ import { useEffect, useMemo, useRef, useState } from "react"; import { + ActivityIndicator, Button, Checker, GlassSurface, @@ -12,6 +13,11 @@ import { } from "@nodedc/ui-react"; import { profileSelectionForConnectionMode } from "../compatibility"; +import { + activeStreamForceFinishAuthority, + activeStreamRecoveryPresentation, + exactActiveStreamRecoveryLineage, +} from "../activeStreamRecovery"; import { SUPPORTED_GNSS_MODE, SUPPORTED_MOUNT_TYPE, @@ -22,48 +28,63 @@ import { } from "../configuration"; import { runAutomaticSpatialSourceStart } from "../automaticSourceStart"; import { + canIssueCanonicalStop, + connectionPolicyAllows, isConfirmedLiveState, - isSoftwareCommandedAcquisition, + isReleasedTerminalAcquisitionFailure, + currentAppliedConnectionTopology, isSourceRuntimeBusy, - isVendorWriteCapable, + isTerminalAcquisitionState, recoverableAcquisition, + requiresCanonicalStopAfterTerminalLocalFailure, sourceStatusLabel, } from "../lifecycle"; -import { normalizeProjectName, validateProjectName } from "../projectName"; +import { connectionPolicyOperatorGuidance } from "../presentation"; +import { + normalizeProjectName, + projectNameAfterConnectionModeSelection, + shouldHydratePreparedProject, + validateProjectName, +} from "../projectName"; import type { XgridsK1Controller } from "../runtimeContext"; -import type { OperatorPresenceConfirmation } from "../api"; +import { + activeStopTarget, + operatorActionPhysicalAcceptance, + preparationTarget, + preparedStartTarget, +} from "../physicalCommandConfirmation"; +import { ActiveStreamRecoverySurface } from "./ActiveStreamRecoverySurface"; type SessionIntent = "live" | "replay"; const sessionItems = [ - { value: "live", label: "Реальное устройство" }, + { value: "live", label: "Прямой приём" }, { value: "replay", label: "Повтор записи" }, ] satisfies Array<{ value: SessionIntent; label: string }>; -const PHYSICAL_ACCEPTANCE = { - operator_present: true, - owner_controlled_device: true, - lixelgo_closed: true, - battery_storage_confirmed: true, - expected_physical_state_confirmed: true, -} satisfies OperatorPresenceConfirmation; - export function K1AcquisitionPipeline({ controller, + desiredConnectionMode = "bridge", openSpatialScene, activateAutomaticSpatialSource, }: { controller: XgridsK1Controller; + desiredConnectionMode?: "bridge" | "quick-connect" | "direct-connect"; openSpatialScene: () => void; activateAutomaticSpatialSource: () => void; }) { const { state, pendingAction, + physicalStopIntentSpent, + physicalStopInFlight, closeApplicationControlSession, - startCanonicalAcquisition, + prepareCanonicalAcquisition, + startPreparedAcquisition, startReplay, stop, + stopLocalReceiver, + forceFinishActiveStreamLocally, abort, } = controller; const [sessionIntent, setSessionIntent] = useState("live"); @@ -75,13 +96,59 @@ export function K1AcquisitionPipeline({ const [mountType, setMountType] = useState(SUPPORTED_MOUNT_TYPE); const [gnssMode, setGnssMode] = useState(SUPPORTED_GNSS_MODE); const hydratedAcquisitionId = useRef(null); + const previousDesiredConnectionMode = useRef(desiredConnectionMode); const activeAcquisition = recoverableAcquisition(state); const preparedAcquisition = activeAcquisition?.state === "prepared" ? activeAcquisition : null; const projectNameValidation = validateProjectName(projectName); - const vendorWriteCapable = isVendorWriteCapable(state); const control = state?.application_control_session; const controlPhase = control?.state ?? "idle"; + const appliedTopology = currentAppliedConnectionTopology(state); + const connectionMode = desiredConnectionMode; + const backendDesiredConnectionMode = state?.desired_connection_mode + ?? desiredConnectionMode; + const configuredConnectionMode = state?.configured_connection_mode + ?? state?.connection_mode + ?? null; + const activeConnectionMode = state?.active_connection_mode + ?? (appliedTopology?.status === "active" ? appliedTopology.connectionMode : null); + const desiredSelectionCommitted = backendDesiredConnectionMode + === desiredConnectionMode; + const desiredModeMatchesActive = desiredSelectionCommitted + && activeConnectionMode === desiredConnectionMode; + const modeSwitchRequired = Boolean( + !desiredSelectionCommitted + || (activeConnectionMode && !desiredModeMatchesActive) + || (configuredConnectionMode && configuredConnectionMode !== desiredConnectionMode), + ); + const connectionConfigured = Boolean( + appliedTopology?.status === "active" + && desiredModeMatchesActive + && state?.connection_lifecycle?.ready_to_start === true, + ); + + useEffect(() => { + if (previousDesiredConnectionMode.current === desiredConnectionMode) return; + previousDesiredConnectionMode.current = desiredConnectionMode; + const projectNameAfterSelection = projectNameAfterConnectionModeSelection( + preparedAcquisition?.project_name, + ); + // A prepared acquisition is immutable backend state, not a draft owned by + // this selector. Preserve its project while the operator previews another + // mode so selecting the active mode again can resume START immediately. + if (preparedAcquisition) { + setProjectName(projectNameAfterSelection); + setProjectNameTouched(false); + return; + } + // Draft project fields belong to the previously selected transport. The + // dropdown sends no physical command; Connect performs the later bounded + // mode transaction, while START remains fenced in the meantime. + setProjectName(projectNameAfterSelection); + setProjectNameTouched(false); + setMountType(SUPPORTED_MOUNT_TYPE); + setGnssMode(SUPPORTED_GNSS_MODE); + }, [desiredConnectionMode, preparedAcquisition]); useEffect(() => { if (state?.source_mode === "live" || state?.source_mode === "replay") { @@ -93,11 +160,35 @@ export function K1AcquisitionPipeline({ useEffect(() => { const acquisitionId = preparedAcquisition?.acquisition_id ?? null; - if (!acquisitionId || hydratedAcquisitionId.current === acquisitionId) return; + if (!shouldHydratePreparedProject({ + acquisitionId, + hydratedAcquisitionId: hydratedAcquisitionId.current, + modeSwitchRequired, + })) return; hydratedAcquisitionId.current = acquisitionId; setProjectName(preparedAcquisition?.project_name ?? ""); setProjectNameTouched(false); - }, [preparedAcquisition?.acquisition_id, preparedAcquisition?.project_name]); + }, [ + modeSwitchRequired, + preparedAcquisition?.acquisition_id, + preparedAcquisition?.project_name, + ]); + + useEffect(() => { + const acquisition = state?.acquisition; + if ( + hydratedAcquisitionId.current === null + || !acquisition + || acquisition.acquisition_id !== hydratedAcquisitionId.current + || !isTerminalAcquisitionState(acquisition.state) + || state?.source_mode !== "idle" + ) return; + hydratedAcquisitionId.current = null; + setProjectName(""); + setProjectNameTouched(false); + setMountType(SUPPORTED_MOUNT_TYPE); + setGnssMode(SUPPORTED_GNSS_MODE); + }, [state?.acquisition, state?.source_mode]); const isBusy = pendingAction !== null; const sourceRuntimeBusy = isSourceRuntimeBusy(state); @@ -108,10 +199,50 @@ export function K1AcquisitionPipeline({ : activeAcquisition ? "live" : sessionIntent; - const sourceLabel = sourceStatusLabel(state); - const relevantAcquisitionFailed = state?.source_mode !== "replay" && state?.acquisition?.state === "failed"; + const gracefulStopTarget = activeStopTarget(state); + const terminalPhysicalStopObserved = + requiresCanonicalStopAfterTerminalLocalFailure(state); + const physicalStopExecutable = Boolean( + gracefulStopTarget + && canIssueCanonicalStop(state, physicalStopIntentSpent), + ); + const physicalStopPresented = physicalStopInFlight || physicalStopExecutable; + const localReceiverStopExecutable = Boolean( + connectionPolicyAllows(state, "stop-local-receiver") + && preparedAcquisition === null, + ); + const terminalPhysicalStopPending = terminalPhysicalStopObserved + && physicalStopInFlight; + const recoveredPhysicalStop = terminalPhysicalStopObserved + && physicalStopExecutable + && !physicalStopInFlight; + const terminalLocalRecovery = terminalPhysicalStopObserved + && !physicalStopPresented + && localReceiverStopExecutable; + const terminalReadOnlyRecovery = terminalPhysicalStopObserved + && !physicalStopPresented + && !localReceiverStopExecutable; + const terminalLocalCapturePending = Boolean( + state?.acquisition?.cleanup_pending === true + || state?.source_mode === "live", + ); + const sourceLabel = terminalPhysicalStopPending + ? "Команда отправлена" + : recoveredPhysicalStop + ? "Требуется остановка" + : terminalLocalRecovery + ? "Локальное завершение доступно" + : terminalReadOnlyRecovery + ? "Действия заблокированы" + : sourceStatusLabel(state); + const releasedAcquisitionFailure = isReleasedTerminalAcquisitionFailure(state); + const relevantAcquisitionFailed = state?.source_mode !== "replay" + && state?.acquisition?.state === "failed" + && !releasedAcquisitionFailure; const sourceTone: StatusTone = - state?.phase === "error" || relevantAcquisitionFailed + terminalPhysicalStopObserved + ? "warning" + : (state?.phase === "error" && !releasedAcquisitionFailure) || relevantAcquisitionFailed ? "danger" : isConfirmedLiveState(state) || state?.source_mode === "replay" ? "success" @@ -122,42 +253,105 @@ export function K1AcquisitionPipeline({ () => sessionItems.map((item) => ({ ...item, disabled: sessionLocked })), [sessionLocked], ); + const activeRecoveryPresentation = activeStreamRecoveryPresentation(state); + const activeRecoveryForceFinishAuthority = activeStreamForceFinishAuthority(state); + const activeRecoveryLineage = exactActiveStreamRecoveryLineage(state); + const recoveredActiveSession = Boolean( + activeRecoveryLineage?.recovery.state === "recovered" + && state?.phase === "live" + && state.source_mode === "live" + && activeAcquisition?.state === "acquiring" + && activeRecoveryLineage.acquisitionId === activeAcquisition.acquisition_id, + ); + const recoveredActiveSessionLabel = activeAcquisition?.project_name?.trim() + || activeAcquisition?.acquisition_id + || "текущая сессия"; + const localForceFinishPending = pendingAction === "force-finish"; + + if (activeRecoveryPresentation || localForceFinishPending) { + return ( + { + if (!activeRecoveryForceFinishAuthority) return; + void forceFinishActiveStreamLocally(); + }} + /> + ); + } const preparedCanonicalLaunch = preparedAcquisition?.control_mode === "plugin-commanded"; const launchBlockedByAcquisition = activeAcquisition !== null && !preparedCanonicalLaunch; const controlRetryBlocked = controlPhase === "failed" && control?.can_open !== true; + const finalStartTarget = preparedStartTarget(state); + const draftPreparationTarget = preparationTarget( + state, + projectNameValidation.value, + ); + const physicalStartAllowed = connectionPolicyAllows(state, "start-acquisition"); + const physicalStartGuidance = finalStartTarget && !physicalStartAllowed + ? connectionPolicyOperatorGuidance(state, "start-acquisition") + : null; + const physicalStopGuidance = gracefulStopTarget + && !physicalStopPresented + && !terminalPhysicalStopObserved + ? connectionPolicyOperatorGuidance(state, "stop-acquisition") + : null; + const physicalStopGuidanceCopy = terminalPhysicalStopPending + ? "Команда остановки устройства уже отправлена. Ждём новое подтверждённое состояние; повторная команда не отправляется." + : terminalLocalRecovery + ? physicalStopIntentSpent + ? "Команда завершилась без нового подтверждённого результата. Повторная команда устройству не отправляется; завершите только локальный приём." + : "Управляющая команда устройству сейчас недоступна. Завершите только разрешённый сервером локальный приём или выполните read-only восстановление." + : terminalReadOnlyRecovery + ? "Управляющие действия сейчас не разрешены. Дождитесь нового подтверждённого состояния или выполните read-only восстановление." + : physicalStopGuidance + ? `${physicalStopGuidance.reason} ${physicalStopGuidance.nextAction}` + : gracefulStopTarget && !physicalStopPresented && physicalStopIntentSpent + ? "Команда завершилась без нового подтверждённого результата. Повторная команда устройству не отправляется; завершите только локальный приём." + : gracefulStopTarget && !physicalStopPresented + ? "Команда устройству недоступна в текущем подтверждённом состоянии. Завершите только локальный приём или выполните read-only восстановление." + : null; - const startLive = async () => { + const submitFinalStart = async () => { + const physicalAcceptance = operatorActionPhysicalAcceptance(); + await runAutomaticSpatialSourceStart( + () => startPreparedAcquisition(physicalAcceptance), + activateAutomaticSpatialSource, + openSpatialScene, + ); + }; + + const requestLivePreparation = async () => { setProjectNameTouched(true); if ( - !state?.k1_ip || + !connectionConfigured || + !connectionMode || sourceRuntimeBusy || launchBlockedByAcquisition || controlRetryBlocked || projectNameValidation.error ) return; - const timezoneName = Intl.DateTimeFormat().resolvedOptions().timeZone || "Etc/UTC"; - await runAutomaticSpatialSourceStart( - () => startCanonicalAcquisition({ - control: { - ...PHYSICAL_ACCEPTANCE, - timezone_name: timezoneName, - }, - acquisition: { - project_name: projectNameValidation.value, - mount_type: SUPPORTED_MOUNT_TYPE, - gnss_mode: SUPPORTED_GNSS_MODE, - compatibility_attestation: profileSelectionForConnectionMode( - state.connection_mode ?? "bridge", - ), - }, - physicalAcceptance: PHYSICAL_ACCEPTANCE, - }), - activateAutomaticSpatialSource, - openSpatialScene, - ); + if (finalStartTarget) { + if (!physicalStartAllowed) return; + await submitFinalStart(); + return; + } + if (!draftPreparationTarget) return; + const prepared = await prepareCanonicalAcquisition({ + acquisition: { + project_name: projectNameValidation.value, + mount_type: SUPPORTED_MOUNT_TYPE, + gnss_mode: SUPPORTED_GNSS_MODE, + compatibility_attestation: profileSelectionForConnectionMode(connectionMode), + }, + }); + if (!prepared) return; + await submitFinalStart(); }; const submitReplay = async () => { @@ -177,8 +371,8 @@ export function K1AcquisitionPipeline({
- {effectiveSessionIntent === "live" ? "ШАГИ 04–05 · ПРОЕКТ И ПРИЁМ" : "СЛУЖЕБНЫЙ РЕЖИМ"} -

{effectiveSessionIntent === "live" ? "Назовите проект и запустите приём" : "Повторите запись"}

+ {terminalPhysicalStopPending ? "ВОССТАНОВЛЕНИЕ · КОМАНДА ОТПРАВЛЕНА" : recoveredPhysicalStop ? "ВОССТАНОВЛЕНИЕ · ОСТАНОВКА" : terminalLocalRecovery ? "ВОССТАНОВЛЕНИЕ · ЛОКАЛЬНЫЙ КОНТУР" : terminalReadOnlyRecovery ? "ВОССТАНОВЛЕНИЕ · ТОЛЬКО ЧТЕНИЕ" : recoveredActiveSession ? "СВЯЗЬ ВОССТАНОВЛЕНА · АКТИВНЫЙ ПРИЁМ" : effectiveSessionIntent === "live" ? "ШАГИ 04–05 · ПРОЕКТ И ПРИЁМ" : "СЛУЖЕБНЫЙ РЕЖИМ"} +

{terminalPhysicalStopPending ? "Ожидаем подтверждение устройства" : recoveredPhysicalStop ? "Сканирование продолжается" : terminalLocalRecovery ? "Завершите локальный приём" : terminalReadOnlyRecovery ? "Ожидайте подтверждённое состояние" : recoveredActiveSession ? "Связь восстановлена · приём продолжается" : effectiveSessionIntent === "live" ? "Назовите проект и запустите приём" : "Повторите запись"}

{sourceLabel}
@@ -188,7 +382,55 @@ export function K1AcquisitionPipeline({ items={selectableSessionItems} onChange={(intent) => { if (!sessionLocked) setSessionIntent(intent); }} /> - {effectiveSessionIntent === "live" ? ( + {terminalPhysicalStopObserved ? ( +
+
+ {terminalPhysicalStopPending ? ( + <> + {terminalLocalCapturePending ? "Локальный приём ещё требует завершения" : "Локальная запись завершена"} + Команда остановки устройства уже отправлена + + Ждём новое подтверждённое состояние K1. Повторная команда устройству не отправляется. + + + ) : recoveredPhysicalStop ? ( + <> + {terminalLocalCapturePending ? "Локальный приём ещё требует завершения" : "Локальная запись завершена"} + Сканирование подтверждено; требуется явный STOP + + Нажмите «Остановить сканирование» ниже или в пространственной сцене. Новый проект, START и настройка сети останутся заблокированы до подтверждённого READY. + + + ) : terminalLocalRecovery ? ( + <> + Команды устройству заблокированы + Доступно локальное завершение приёма + + Повторная команда K1 не отправляется. Завершите локальный приём или выполните read-only восстановление. + + + ) : ( + <> + Управляющие действия заблокированы + Доступно только read-only восстановление + + Дождитесь нового подтверждённого состояния; локальные и управляющие команды сейчас не разрешены. + + + )} +
+
+ ) : recoveredActiveSession ? ( +
+
+ Исходная сессия · {recoveredActiveSessionLabel} + Продолжаем тот же приём без нового START + + Автоматическое восстановление не отправляло START, STOP, Bluetooth или настройки сети. Явная остановка ниже доступна только при текущем подтверждённом праве на STOP. + +
+
+ ) : effectiveSessionIntent === "live" ? (
@@ -228,40 +470,53 @@ export function K1AcquisitionPipeline({ aria-invalid={projectNameTouched && projectNameValidation.error ? true : undefined} description={projectNameTouched && projectNameValidation.error ? projectNameValidation.error - : "Отдельной команды сохранения имени на K1 нет: оно отправляется только при START."} + : "Имя отправляется только при START; отдельной команды сохранения нет."} placeholder="Например, TEST001" />

- Нажатие запуска — явное операторское действие для выбранного K1. Автоматических повторов START нет. + {modeSwitchRequired + ? `Выбран другой способ связи. Сначала установите подключение через ${desiredConnectionMode === "bridge" ? "Bridge" : desiredConnectionMode === "quick-connect" ? "Quick Connect" : "Direct Connect"}.` + : !connectionConfigured + ? "Сначала завершите подключение в выбранном режиме. START не используется для установки связи." + : physicalStartGuidance + ? `${physicalStartGuidance.reason} ${physicalStartGuidance.nextAction}` + : "Одно нажатие выполняет каноническую подготовку и один START после подтверждённого READY. Автоматических повторов команд нет."}

- {control?.control_socket_open && !activeAcquisition && !isBusy ? ( + {control?.control_socket_open && !activeAcquisition && !recoveredPhysicalStop && !isBusy ? (
) : (
- setReplayPath(event.target.value)} spellCheck={false} placeholder="sessions/.../capture.tsv" /> + setReplayPath(event.target.value)} spellCheck={false} placeholder="sessions/.../capture.tsv" /> setReplaySpeed(event.target.value)} />
После последнего кадра начать запись заново. @@ -301,25 +556,47 @@ export function K1AcquisitionPipeline({ )}

- {state?.source_mode === "replay" + {physicalStopGuidanceCopy + ? physicalStopGuidanceCopy + : recoveredPhysicalStop && physicalStopExecutable + ? terminalLocalCapturePending + ? "Локальный приём ещё требует завершения. Эта кнопка отправит ровно один явный STOP и дождётся подтверждённого результата." + : "Локальная запись уже остановлена. Эта кнопка отправит ровно один явный STOP и дождётся READY." + : state?.source_mode === "replay" ? "Остановка завершит фактически запущенный повтор записи." : activeAcquisition || state?.source_mode === "live" - ? vendorWriteCapable && activeAcquisition?.control_mode === "plugin-commanded" - ? "Остановка отправит профилированную команду K1 и дождётся завершения локального сохранения." - : "Остановка завершает только локальный приём и сохранение. Физическое состояние сканера остаётся неизвестным." + ? physicalStopPresented + ? "Остановка отправит профилированную команду и дождётся завершения локального сохранения." + : localReceiverStopExecutable + ? "Остановка завершает только локальный приём и сохранение. Состояние сканирования остаётся неизвестным." + : "Действие остановки сейчас не разрешено. Дождитесь нового подтверждённого состояния или выполните read-only восстановление." : "Активного источника сейчас нет."}

- + {physicalStopPresented || localReceiverStopExecutable ? ( + + ) : null} {activeAcquisition ? ( + ) : null} + {showDefaultActions ? ( + <> + + + + ) : null} +
+ ) : null} + + ); +} diff --git a/plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx b/plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx index feef3c1..47fc2fb 100644 --- a/plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx +++ b/plugins/xgrids-k1/frontend/src/components/K1ProvisioningPipeline.tsx @@ -1,16 +1,34 @@ -import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, + type ReactNode, +} from "react"; +import { + ActivityIndicator, Button, - Checker, GlassSurface, Icon, + IconButton, Select, StatusBadge, TextField, type StatusTone, } from "@nodedc/ui-react"; -import type { BleDevice } from "../api"; +import type { + BleDevice, + ConnectionVerifyRequest, + ReopenRetiredPhysicalReconciliationRequest, + XgridsConnectionAttempt, + XgridsK1State, + XgridsConnectionReconfiguration, + XgridsConnectionReconfigurationIntent, +} from "../api"; import { profileSelectionForConnectionMode } from "../compatibility"; import { DEFAULT_CONNECTION_MODE, @@ -18,13 +36,1053 @@ import { type ConnectionMode, } from "../configuration"; import { + backendConnectionTopology, + activeConnectionReconfiguration, + canAdmitProvisioningConnection, canSubmitProvisioningMutation, + connectionAttemptForRuntimeError, + connectionPolicyAllows, + connectionPolicyDecision, + isRecoveredPhysicalScanning, + isPhysicalStopRecoverySettling, isReachableConnectionLease, + locallyInitiatedBleSessionTarget, + newOperationId, + operationByIdempotencyKey, provisioningCandidateById, + provisioningFailureRequiresFreshCandidate, provisioningIntentKey, + readOnlyConnectionObservationTarget, + readOnlyPhysicalRecoveryBinding, + recommendedConnectionRecoveryObservationTarget, + reconfigurationAllowsFreshDevice, + requiresReadOnlyPhysicalRecovery, + reopenedPhysicalReconciliationMatches, + retiredPhysicalReopenAuthority, + serverBoundAppliedNetworkObservationTarget, + transportRefEquivalenceKey, + trustedConnectionBinding, + type ReadOnlyConnectionObservationTarget, } from "../lifecycle"; import { finiteMetric } from "../presentation"; import type { XgridsK1Controller } from "../runtimeContext"; +import { K1OperatorError } from "./K1OperatorError"; +import { + connectionActionAuthoritySnapshot, + type BleDiscoverySubmitResult, + type ConnectionActionAuthoritySnapshot, +} from "../useXgridsK1Runtime"; + +interface CompletedLocalNetworkIntent { + deviceId: string; + connectionMode: ConnectionMode; + acceptedSessionKey: string | null; +} + +interface DevicePresentationSnapshot { + deviceId: string; + label: string; + rssi: number | null; +} + +export interface PhysicalReopenPresentation { + key: string; + snapshotRuntimeId: string; + request: ReopenRetiredPhysicalReconciliationRequest; + authority: ConnectionActionAuthoritySnapshot; +} + +export function physicalReopenPresentationIsCurrent( + presentation: PhysicalReopenPresentation | null | undefined, + currentAuthority: ConnectionActionAuthoritySnapshot | null | undefined, +): boolean { + return Boolean( + presentation + && currentAuthority + && presentation.snapshotRuntimeId === currentAuthority.snapshotRuntimeId + && connectionActionAuthorityMatches( + presentation.authority, + currentAuthority, + ) + && presentation.request.expected_desired_mode + === currentAuthority.connectionMode + && presentation.request.expected_desired_mode_revision + === currentAuthority.desiredModeRevision + && presentation.request.expected_discovery_generation + === currentAuthority.discoveryGeneration + ); +} + +export function admitPhysicalReopenPresentation( + current: PhysicalReopenPresentation | null | undefined, + next: PhysicalReopenPresentation, + currentAuthority: ConnectionActionAuthoritySnapshot | null | undefined, +): PhysicalReopenPresentation | null { + // Coalesce only the click which still owns the complete live authority. + // A same-runtime reset/reconfiguration/binding/discovery drift releases the + // stale owner synchronously, before React's post-render invalidation effect, + // so a fresh click cannot be blocked by a late request from the old fence. + if (physicalReopenPresentationIsCurrent(current, currentAuthority)) { + return null; + } + return physicalReopenPresentationIsCurrent(next, currentAuthority) + ? next + : null; +} + +export function physicalReopenSettlementIsCurrent( + current: PhysicalReopenPresentation | null | undefined, + actionKey: string, + currentAuthority: ConnectionActionAuthoritySnapshot | null | undefined, +): boolean { + return Boolean( + current?.key === actionKey + && physicalReopenPresentationIsCurrent(current, currentAuthority) + ); +} + +export function physicalReopenClickAuthority( + renderedState: XgridsK1State | null | undefined, + connectionMode: ConnectionMode, + currentAuthority: ConnectionActionAuthoritySnapshot | null | undefined, + reopenAuthority: ReturnType, +): ConnectionActionAuthoritySnapshot | null { + const renderedAuthority = connectionActionAuthoritySnapshot( + renderedState, + connectionMode, + ); + if ( + !renderedAuthority + || !currentAuthority + || !reopenAuthority + || !connectionActionAuthorityMatches(renderedAuthority, currentAuthority) + || renderedAuthority.desiredModeRevision + !== reopenAuthority.expectedDesiredModeRevision + || renderedAuthority.discoveryGeneration + !== reopenAuthority.expectedDiscoveryGeneration + ) return null; + return renderedAuthority; +} + +export interface ScenarioResetPresentationBoundary { + current: boolean; + active: boolean; + key: string | null; +} + +export function scenarioResetPresentationBoundary( + state: XgridsK1State | null | undefined, + connectionMode: ConnectionMode, +): ScenarioResetPresentationBoundary { + const marker = state?.connection_scenario_reset; + const current = Boolean( + marker + && marker.revision === state?.desired_connection_mode_revision + && marker.desired_mode === connectionMode, + ); + const runtimeId = state?.snapshot_runtime_id?.trim() || null; + return { + current, + active: current && marker?.active !== false, + key: current && runtimeId && Number.isInteger(marker?.revision) + ? `${runtimeId}:${marker?.revision}` + : null, + }; +} + +/** + * A reset marker owns only the clean new-device draft it created. Scan is + * presentation-only and deliberately keeps that boundary alive. A current + * backend-owned connection attempt or a new physical recovery state is newer + * authority and must remain visible. OperationRecord.sequence is deliberately + * not used here: it is a per-operation transition counter, not chronology. + */ +export function scenarioResetOwnsCleanConnectionDraft( + state: XgridsK1State | null | undefined, + connectionMode: ConnectionMode, +): boolean { + const boundary = scenarioResetPresentationBoundary(state, connectionMode); + return Boolean( + boundary.current + && !state?.connection_attempt + && !requiresReadOnlyPhysicalRecovery(state), + ); +} + +export function physicalRecoveryPresentationAuthorityKey( + state: XgridsK1State | null | undefined, +): string | null { + if (!requiresReadOnlyPhysicalRecovery(state)) return null; + const target = readOnlyConnectionObservationTarget(state); + const runtimeId = state?.snapshot_runtime_id?.trim(); + const physical = state?.application_control_session?.physical_command + ?? state?.physical_command; + const recordRevision = physical?.record?.revision; + if ( + !runtimeId + || !target?.serverBound + || !Number.isInteger(recordRevision) + ) return null; + return [ + runtimeId, + recordRevision, + transportRefEquivalenceKey(target.deviceId), + target.connectionMode, + target.source, + target.expectedDiscoveryGeneration ?? "durable", + state?.desired_connection_mode_revision ?? "no-mode-revision", + ].join(":"); +} + +export type ProvisioningAttemptViewState = + | "hidden" + | "submitting" + | "settling" + | "failed" + | "connected" + | "retired"; + +export interface ProvisioningAttemptPresentation + extends DevicePresentationSnapshot { + snapshotRuntimeId: string; + connectionMode: ConnectionMode; + idempotencyKey: string; + attemptId: string | null; + /** SSID is presentation-only. The Wi-Fi password is never retained here. */ + ssid: string | null; + localPhase: "submitting" | "settling" | "failed"; + failureMessage: string | null; + freshStartAllowed: boolean; +} + +export function emptyProvisioningAttemptPresentation(): + ProvisioningAttemptPresentation | null { + return null; +} + +const TERMINAL_PROVISIONING_STATUSES = new Set([ + "operator_action_required", + "succeeded", + "failed", + "cancelled", + "timed_out", + "interrupted", +]); + +/** + * Project one click-owned Apply attempt without borrowing an unrelated global + * error or historical connection attempt. The local latch bridges the fast + * HTTP acknowledgement and the later authoritative connection projection. + */ +export function provisioningAttemptViewState( + presentation: ProvisioningAttemptPresentation | null | undefined, + state: XgridsK1State | null | undefined, +): ProvisioningAttemptViewState { + if (!presentation) return "hidden"; + const runtimeId = state?.snapshot_runtime_id?.trim() || null; + if (!runtimeId || runtimeId !== presentation.snapshotRuntimeId) { + return "retired"; + } + if ( + isReachableConnectionLease(state, presentation.connectionMode) + && observedConnectionAuthorityAllowsTarget( + state, + presentation.deviceId, + presentation.connectionMode, + ) + ) { + return "connected"; + } + + const operation = operationByIdempotencyKey( + state, + "network.provision", + presentation.idempotencyKey, + ); + const expectedAttemptId = presentation.attemptId ?? operation?.operation_id ?? null; + const attempt = expectedAttemptId + && state?.connection_attempt?.attempt_id === expectedAttemptId + ? state.connection_attempt + : null; + if (attempt) { + if (["accepted", "running"].includes(attempt.status)) return "settling"; + // The terminal child can arrive one projection before the reachable lease. + // Ready is corroborating success, not a reason to flash a failure card. + if (attempt.status === "succeeded" && attempt.control_state === "ready") { + return "settling"; + } + if (TERMINAL_PROVISIONING_STATUSES.has(attempt.status)) return "failed"; + } + if ( + operation + && ["failed", "cancelled", "timed_out", "interrupted", "operator_action_required"] + .includes(operation.status) + ) { + return "failed"; + } + if (operation && ["accepted", "running"].includes(operation.status)) { + return "settling"; + } + // A succeeded network.provision operation can precede the connection + // attempt/lease projection by one or more snapshots. It is not yet proof of + // a usable network, so keep the same disabled form while authority settles. + if (operation?.status === "succeeded") return "settling"; + return presentation.localPhase; +} + +export interface UnavailablePhysicalRetirementAuthority { + expectedOperationId: string; + expectedRevision: number; + expectedTransportRef: string; +} + +export function connectionAttemptOwnsAppliedNetworkRecovery( + attempt: XgridsConnectionAttempt | null | undefined, +): boolean { + if ( + !attempt + || attempt.phase !== "network_applied" + || attempt.control_state === "ready" + ) return false; + if (["accepted", "running"].includes(attempt.status)) return true; + return [ + "continue-with-control-verification", + "verify-control-read-only", + "manual-recovery-required", + ].includes(attempt.safe_next_action); +} + +export async function dispatchUnavailablePhysicalRetirementForCurrentRuntime( + snapshotRuntimeId: string, + isSnapshotRuntimeCurrent: ((expectedSnapshotRuntimeId: string) => boolean) + | null + | undefined, + dispatch: (expectedSnapshotRuntimeId: string) => Promise, +): Promise< + | { dispatched: false; result: null } + | { dispatched: true; result: T } +> { + // This is the last synchronous mutation boundary. Never let a click rendered + // from runtime A borrow runtime B from the controller after replacement. + if (!isSnapshotRuntimeCurrent?.(snapshotRuntimeId)) { + return { dispatched: false, result: null }; + } + return { + dispatched: true, + result: await dispatch(snapshotRuntimeId), + }; +} + +/** + * The legacy physical projection supplies the exact CAS payload, but only the + * public connection policy may make the local-only retirement executable. + */ +export function unavailablePhysicalRetirementAuthority( + state: XgridsK1State | null | undefined, +): UnavailablePhysicalRetirementAuthority | null { + const decision = connectionPolicyDecision( + state, + "retire-unavailable-physical-target", + ); + const projection = state?.physical_command?.operator_retirement; + const record = state?.physical_command?.record; + const expectedOperationId = projection?.expected_operation_id?.trim() ?? ""; + const expectedTransportRef = projection?.expected_transport_ref?.trim() ?? ""; + const expectedRevision = projection?.expected_revision; + const recordOperationId = record + && typeof record.operation_id === "string" + ? record.operation_id.trim() + : ""; + const recordRevision = record && typeof record.revision === "number" + ? record.revision + : null; + const recordConnection = record + && typeof record.connection === "object" + && record.connection !== null + ? record.connection + : null; + const recordTransportRef = recordConnection + && "transport_ref" in recordConnection + && typeof recordConnection.transport_ref === "string" + ? recordConnection.transport_ref.trim() + : ""; + if ( + !connectionPolicyAllows(state, "retire-unavailable-physical-target") + || decision?.target_source !== "durable-physical-command" + || decision.requires_live_gatt_validation !== false + || decision.physical_command_allowed !== false + || decision.physical_outcome !== "unknown" + || decision.device_write_performed !== false + || !decision.required_transport_ref?.trim() + || projection?.allowed !== true + || projection.reason_codes.length !== 0 + || projection.physical_outcome !== "unknown" + || projection.device_io_performed !== false + || projection.automatic_retry !== false + || !expectedOperationId + || !expectedTransportRef + || !Number.isInteger(expectedRevision) + || (expectedRevision ?? 0) < 1 + || recordOperationId !== expectedOperationId + || recordRevision !== expectedRevision + || transportRefEquivalenceKey(recordTransportRef) + !== transportRefEquivalenceKey(expectedTransportRef) + || transportRefEquivalenceKey(decision.required_transport_ref) + !== transportRefEquivalenceKey(expectedTransportRef) + || !( + state?.physical_command?.requires_reconciliation === true + || state?.physical_command?.resolved_active_recovery_required === true + ) + ) return null; + return { + expectedOperationId, + expectedRevision: expectedRevision as number, + expectedTransportRef, + }; +} + +export interface RuntimeActionFence extends ConnectionActionAuthoritySnapshot { + clickToken: number; +} + +/** Deterministic SSR seam; production never provides this context. */ +export const RuntimeActionFenceTestContext = + createContext(null); + +interface SearchPresentation { + sequence: number; + snapshotRuntimeId: string | null; + connectionMode: ConnectionMode; + desiredModeRevision: number | null; + active: boolean; + completedDiscoveryGeneration: number | null; +} + +export function emptySearchPresentation(): SearchPresentation | null { + return null; +} + +export function localScenarioActionEpochIsCurrent( + capturedEpoch: number, + currentEpoch: number, +): boolean { + return capturedEpoch === currentEpoch; +} + +export function connectionActionAuthorityMatches( + left: ConnectionActionAuthoritySnapshot | null | undefined, + right: ConnectionActionAuthoritySnapshot | null | undefined, +): boolean { + return Boolean( + left + && right + && left.snapshotRuntimeId === right.snapshotRuntimeId + && left.connectionMode === right.connectionMode + && left.desiredModeRevision === right.desiredModeRevision + && left.reconfigurationRevision === right.reconfigurationRevision + && left.reconfigurationIntentId === right.reconfigurationIntentId + && left.activeBindingKey === right.activeBindingKey + && left.discoveryGeneration === right.discoveryGeneration, + ); +} + +const TRANSIENT_RECONFIGURATION_CONTENTION_REASONS = new Set([ + "connection-reconfiguration-lifecycle-busy", + "k1-lifecycle-process-lease-network-owned", +]); + +/** + * Keep a connected Bridge action in its established layout slot while a + * short-lived lifecycle owner prevents this exact click. Durable safety + * blockers still remove the action; the backend remains the final authority. + */ +export function connectedReconfigurationActionApplicable( + state: XgridsK1State | null | undefined, + action: "prepare-select-device" | "prepare-change-network", +): boolean { + if (connectionPolicyAllows(state, action)) return true; + const decision = connectionPolicyDecision(state, action); + return Boolean( + decision + && decision.allowed === false + && decision.automatic_retry === false + && decision.reason_codes.length > 0 + && decision.reason_codes.every((reason) => + TRANSIENT_RECONFIGURATION_CONTENTION_REASONS.has(reason) + ), + ); +} + +export function reconfigurationContinuationAuthority( + fence: ConnectionActionAuthoritySnapshot, + observedState: XgridsK1State | null | undefined, + intent: XgridsConnectionReconfigurationIntent | "cancel", + current: ConnectionActionAuthoritySnapshot | null | undefined, +): ConnectionActionAuthoritySnapshot | null { + if (!observedState) return null; + const observedAuthority = connectionActionAuthoritySnapshot( + observedState, + fence.connectionMode, + ); + const observedReconfiguration = observedState.connection_reconfiguration; + if ( + !observedAuthority + || observedAuthority.snapshotRuntimeId !== fence.snapshotRuntimeId + || observedAuthority.connectionMode !== fence.connectionMode + || observedAuthority.desiredModeRevision !== fence.desiredModeRevision + || observedAuthority.discoveryGeneration !== fence.discoveryGeneration + 1 + || observedAuthority.reconfigurationRevision !== fence.reconfigurationRevision + 1 + || !connectionActionAuthorityMatches(current, observedAuthority) + || ( + intent === "cancel" + ? observedAuthority.activeBindingKey !== fence.activeBindingKey + : observedAuthority.activeBindingKey !== null + || observedState.connection_lifecycle?.active_binding !== null + ) + || ( + intent === "cancel" + ? observedReconfiguration?.status !== "idle" + || observedReconfiguration.intent !== null + || observedAuthority.reconfigurationIntentId !== null + : observedReconfiguration?.intent !== intent + || observedReconfiguration.status === "idle" + || !observedAuthority.reconfigurationIntentId + ) + ) return null; + return observedAuthority; +} + +export function observedConnectionAuthorityAllowsTarget( + observedState: XgridsK1State | null | undefined, + deviceId: string, + connectionMode: ConnectionMode, + { allowUnbound = false }: { allowUnbound?: boolean } = {}, +): boolean { + if (!observedState || !deviceId) return false; + const currentRecovery = observedState.current_device_recovery; + const activeBinding = observedState.connection_lifecycle?.active_binding; + const activeBindingKey = observedState.connection_lifecycle?.active_binding_key + ?.trim() || null; + const referencedTargets = [ + observedState.selected_device_id?.trim() || null, + currentRecovery?.transport_ref?.trim() || null, + activeBinding?.transport_ref?.trim() || null, + ].filter((value): value is string => Boolean(value)); + const deviceKey = transportRefEquivalenceKey(deviceId); + if ( + !deviceKey + || referencedTargets.some( + (value) => transportRefEquivalenceKey(value) !== deviceKey, + ) + ) return false; + if (activeBindingKey && !activeBinding) return false; + if ( + activeBinding + && ( + transportRefEquivalenceKey(activeBinding.transport_ref) !== deviceKey + || activeBinding.connection_mode !== connectionMode + ) + ) return false; + if ( + currentRecovery?.transport_ref?.trim() + && currentRecovery.connection_mode !== connectionMode + ) return false; + if ( + observedState.selected_device_id?.trim() + && observedState.connection_mode + && observedState.connection_mode !== connectionMode + ) return false; + return allowUnbound || referencedTargets.some( + (value) => transportRefEquivalenceKey(value) === deviceKey, + ); +} + +export function connectContinuationAuthority( + fence: ConnectionActionAuthoritySnapshot, + observedState: XgridsK1State | null | undefined, + current: ConnectionActionAuthoritySnapshot | null | undefined, + deviceId: string, + connectionMode: ConnectionMode, +): ConnectionActionAuthoritySnapshot | null { + if (!observedConnectionAuthorityAllowsTarget( + observedState, + deviceId, + connectionMode, + )) return null; + const observedAuthority = connectionActionAuthoritySnapshot( + observedState, + fence.connectionMode, + ); + if ( + !observedAuthority + || observedAuthority.snapshotRuntimeId !== fence.snapshotRuntimeId + || observedAuthority.connectionMode !== fence.connectionMode + || observedAuthority.desiredModeRevision !== fence.desiredModeRevision + || !connectionActionAuthorityMatches(current, observedAuthority) + ) return null; + const observedReconfiguration = observedState?.connection_reconfiguration; + const unchangedOrdinaryConnect = Boolean( + fence.reconfigurationIntentId === null + && observedAuthority.reconfigurationIntentId === null + && observedAuthority.reconfigurationRevision === fence.reconfigurationRevision + && observedAuthority.discoveryGeneration === fence.discoveryGeneration + && observedReconfiguration?.status === "idle" + && observedReconfiguration.intent === null, + ); + const exactConsumedReconfiguration = Boolean( + fence.reconfigurationIntentId !== null + && observedAuthority.reconfigurationIntentId === null + && observedAuthority.reconfigurationRevision === fence.reconfigurationRevision + 1 + && observedAuthority.discoveryGeneration === fence.discoveryGeneration + 1 + && observedReconfiguration?.status === "idle" + && observedReconfiguration.intent === null, + ); + return unchangedOrdinaryConnect || exactConsumedReconfiguration + ? observedAuthority + : null; +} + +export function runtimeActionFenceMatches( + fence: RuntimeActionFence | null | undefined, + currentSnapshotRuntimeId: string | null, + activeFence: RuntimeActionFence | null, + isActionAuthorityCurrent?: (fence: RuntimeActionFence) => boolean, +): boolean { + return Boolean( + fence + && currentSnapshotRuntimeId + && fence.snapshotRuntimeId === currentSnapshotRuntimeId + && activeFence + && activeFence.snapshotRuntimeId === fence.snapshotRuntimeId + && activeFence.clickToken === fence.clickToken + && ( + !isActionAuthorityCurrent + || isActionAuthorityCurrent(fence) + ), + ); +} + +export function currentRuntimeActionRequest( + request: T | null, + currentSnapshotRuntimeId: string | null, + activeFence: RuntimeActionFence | null, + isActionAuthorityCurrent?: (fence: RuntimeActionFence) => boolean, +): T | null { + return runtimeActionFenceMatches( + request, + currentSnapshotRuntimeId, + activeFence, + isActionAuthorityCurrent, + ) ? request : null; +} + +interface PreparingReconfigurationRequest extends RuntimeActionFence { + intent: XgridsConnectionReconfigurationIntent | "cancel"; +} + +export function exactChangeNetworkCandidate( + reconfiguration: XgridsConnectionReconfiguration | null, + devices: readonly BleDevice[], + discoveryGeneration: number | null | undefined, +): BleDevice | null { + const requiredTransportRef = reconfiguration?.required_transport_ref?.trim(); + if ( + reconfiguration?.intent !== "change-network" + || reconfiguration.status !== "fresh-scan-completed" + || reconfiguration.required_connection_mode !== "bridge" + || reconfiguration.required_transport_observed !== true + || !requiredTransportRef + || !Number.isInteger(discoveryGeneration) + || reconfiguration.fresh_discovery_generation !== discoveryGeneration + ) return null; + return provisioningCandidateById(devices, requiredTransportRef); +} + +export interface ExplicitProvisioningDraft { + snapshotRuntimeId: string; + deviceId: string; + connectionMode: ConnectionMode; + desiredModeRevision: number; + discoveryGeneration: number; + reconfigurationRevision: number; + reconfigurationIntentId: string | null; + activeBindingKey: string | null; + requiredTransportRef: string | null; + requiredConnectionMode: ConnectionMode | null; + /** Pre-dispatch evidence expired; only a new explicit scan may re-arm Apply. */ + requiresFreshScanBeforeSubmit: boolean; +} + +export function explicitProvisioningDraftMatches( + draft: ExplicitProvisioningDraft | null, + current: { + snapshotRuntimeId: string | null; + deviceId: string | null; + connectionMode: ConnectionMode; + desiredModeRevision: number | null | undefined; + discoveryGeneration: number | null | undefined; + reconfigurationRevision: number; + reconfigurationIntentId: string | null; + activeBindingKey: string | null; + requiredTransportRef: string | null; + requiredConnectionMode: ConnectionMode | null; + }, +): boolean { + return Boolean( + draft + && draft.snapshotRuntimeId === current.snapshotRuntimeId + && current.deviceId + && transportRefEquivalenceKey(draft.deviceId) + === transportRefEquivalenceKey(current.deviceId) + && draft.connectionMode === current.connectionMode + && draft.desiredModeRevision === current.desiredModeRevision + && draft.discoveryGeneration === current.discoveryGeneration + && draft.reconfigurationRevision === current.reconfigurationRevision + && draft.reconfigurationIntentId === current.reconfigurationIntentId + && draft.activeBindingKey === current.activeBindingKey + && ( + draft.requiredTransportRef === null + ? current.requiredTransportRef === null + : current.requiredTransportRef !== null + && transportRefEquivalenceKey(draft.requiredTransportRef) + === transportRefEquivalenceKey(current.requiredTransportRef) + ) + && draft.requiredConnectionMode === current.requiredConnectionMode, + ); +} + +export function localProvisioningDraftFenceKey({ + snapshotRuntimeId, + reconfigurationRevision, + reconfigurationIntentId, + activeBindingKey, + requiredTransportRef, + requiredConnectionMode, +}: { + snapshotRuntimeId: string | null; + reconfigurationRevision: number; + reconfigurationIntentId: string | null; + activeBindingKey: string | null; + requiredTransportRef: string | null; + requiredConnectionMode: ConnectionMode | null; +}): string { + return [ + snapshotRuntimeId ?? "no-runtime", + reconfigurationRevision, + reconfigurationIntentId ?? "idle", + activeBindingKey ?? "no-binding", + requiredTransportRef + ? transportRefEquivalenceKey(requiredTransportRef) + : "no-required-device", + requiredConnectionMode ?? "no-required-mode", + ].join(":"); +} + +function observationRequest( + target: ReadOnlyConnectionObservationTarget, + reconfigurationRevision: number, + reconfigurationIntentId: string | null, +): ConnectionVerifyRequest | null { + const commonRequest = { + device_id: target.deviceId, + compatibility_attestation: profileSelectionForConnectionMode( + target.connectionMode, + ), + }; + if (target.source === "fresh-scan") { + if (!Number.isInteger(target.expectedDiscoveryGeneration)) return null; + return { + ...commonRequest, + source: "fresh-scan", + expected_discovery_generation: target.expectedDiscoveryGeneration as number, + expected_reconfiguration_revision: reconfigurationRevision, + ...(reconfigurationIntentId + ? { expected_reconfiguration_intent_id: reconfigurationIntentId } + : {}), + }; + } + return { + ...commonRequest, + source: target.source, + }; +} + +export function connectionRecoveryObservationTargetMatches( + expected: ReadOnlyConnectionObservationTarget | null | undefined, + current: ReadOnlyConnectionObservationTarget | null | undefined, +): boolean { + return Boolean( + expected + && current + && expected.action === current.action + && expected.connectionMode === current.connectionMode + && expected.source === current.source + && expected.serverBound === true + && current.serverBound === true + && expected.expectedDiscoveryGeneration + === current.expectedDiscoveryGeneration + && transportRefEquivalenceKey(expected.deviceId) + === transportRefEquivalenceKey(current.deviceId), + ); +} + +export async function dispatchConnectionRecoveryObservationForCurrentRuntime( + snapshotRuntimeId: string, + renderedTarget: ReadOnlyConnectionObservationTarget | null | undefined, + isSnapshotRuntimeCurrent: + | ((expectedSnapshotRuntimeId: string) => boolean) + | null + | undefined, + getCurrentTarget: + | (() => ReadOnlyConnectionObservationTarget | null) + | null + | undefined, + dispatch: (target: ReadOnlyConnectionObservationTarget) => Promise, +): Promise< + | { dispatched: false; result: null } + | { dispatched: true; result: T } +> { + // Cold recovery can legitimately target a connection mode different from + // the browser draft. The mutation boundary is the exact server-bound target + // plus the rendered runtime, never that unrelated draft value. + if ( + !snapshotRuntimeId.trim() + || !renderedTarget?.serverBound + || !isSnapshotRuntimeCurrent?.(snapshotRuntimeId) + || typeof getCurrentTarget !== "function" + || !connectionRecoveryObservationTargetMatches( + renderedTarget, + getCurrentTarget(), + ) + ) { + return { dispatched: false, result: null }; + } + return { + dispatched: true, + result: await dispatch(renderedTarget), + }; +} + +export interface RetiredPhysicalReopenVerificationContext { + target: ReadOnlyConnectionObservationTarget; + authority: ConnectionActionAuthoritySnapshot; +} + +/** + * Admit the read-only half of an explicit retired-device recovery only from + * the current backend projection produced by that exact reopen audit. The + * visible BLE row is presentation evidence and never becomes a Verify target. + */ +export function retiredPhysicalReopenVerificationContext( + observedState: XgridsK1State | null | undefined, + request: ReopenRetiredPhysicalReconciliationRequest, + snapshotRuntimeId: string, + target: ReadOnlyConnectionObservationTarget | null | undefined, + authority: ConnectionActionAuthoritySnapshot | null | undefined, +): RetiredPhysicalReopenVerificationContext | null { + const exactRuntimeId = snapshotRuntimeId.trim(); + if ( + !exactRuntimeId + || observedState?.snapshot_runtime_id?.trim() !== exactRuntimeId + || !reopenedPhysicalReconciliationMatches(observedState, request) + || target?.serverBound !== true + || target.action !== "observe-fresh-device-network" + || target.source !== "fresh-scan" + || target.connectionMode !== request.expected_desired_mode + || target.expectedDiscoveryGeneration + !== request.expected_discovery_generation + || transportRefEquivalenceKey(target.deviceId) + !== transportRefEquivalenceKey(request.expected_transport_ref) + || authority?.snapshotRuntimeId !== exactRuntimeId + || authority.connectionMode !== request.expected_desired_mode + || authority.desiredModeRevision + !== request.expected_desired_mode_revision + || authority.discoveryGeneration + !== request.expected_discovery_generation + ) return null; + return { target, authority }; +} + +export interface RetiredPhysicalReopenSubmitResult { + succeeded: boolean; + observedState: XgridsK1State | null; +} + +export type RetiredPhysicalReopenDispatchResult = + | { + reopenDispatched: false; + reopenResult: null; + verifyDispatched: false; + verifyResult: null; + } + | { + reopenDispatched: true; + reopenResult: RetiredPhysicalReopenSubmitResult; + verifyDispatched: false; + verifyResult: null; + } + | { + reopenDispatched: true; + reopenResult: RetiredPhysicalReopenSubmitResult; + verifyDispatched: true; + verifyResult: TVerify; + }; + +/** + * One explicit click owns one local ledger reopen and, only after exact + * current-state proof, one server-bound read-only Verify. A lost HTTP response + * may continue when the audit proves the same request committed. Runtime or + * same-runtime state drift fails closed before any observation is dispatched. + */ +export async function dispatchRetiredPhysicalReconciliationForCurrentRuntime< + TVerify, +>({ + snapshotRuntimeId, + request, + isSnapshotRuntimeCurrent, + reopen, + getCurrentState, + getCurrentTarget, + getCurrentAuthority, + expectedAuthority, + verify, +}: { + snapshotRuntimeId: string; + request: ReopenRetiredPhysicalReconciliationRequest; + isSnapshotRuntimeCurrent: + | ((expectedSnapshotRuntimeId: string) => boolean) + | null + | undefined; + reopen: ( + request: ReopenRetiredPhysicalReconciliationRequest, + expectedSnapshotRuntimeId: string, + ) => Promise; + getCurrentState: (() => XgridsK1State | null) | null | undefined; + getCurrentTarget: + | (() => ReadOnlyConnectionObservationTarget | null) + | null + | undefined; + getCurrentAuthority: + | ((connectionMode: ConnectionMode) => ConnectionActionAuthoritySnapshot | null) + | null + | undefined; + expectedAuthority: ConnectionActionAuthoritySnapshot; + verify: ( + context: RetiredPhysicalReopenVerificationContext, + expectedSnapshotRuntimeId: string, + ) => Promise; +}): Promise> { + const exactRuntimeId = snapshotRuntimeId.trim(); + const notDispatched = { + reopenDispatched: false, + reopenResult: null, + verifyDispatched: false, + verifyResult: null, + } as const; + if ( + !exactRuntimeId + || !isSnapshotRuntimeCurrent?.(exactRuntimeId) + || typeof getCurrentState !== "function" + || typeof getCurrentTarget !== "function" + || typeof getCurrentAuthority !== "function" + ) return notDispatched; + + const reopenResult = await reopen(request, exactRuntimeId); + const reopenedOnly = { + reopenDispatched: true, + reopenResult, + verifyDispatched: false, + verifyResult: null, + } as const; + if ( + !isSnapshotRuntimeCurrent(exactRuntimeId) + || reopenResult.observedState?.snapshot_runtime_id?.trim() + !== exactRuntimeId + || !reopenedPhysicalReconciliationMatches( + reopenResult.observedState, + request, + ) + ) return reopenedOnly; + + const currentState = getCurrentState(); + const context = retiredPhysicalReopenVerificationContext( + currentState, + request, + exactRuntimeId, + getCurrentTarget(), + getCurrentAuthority(request.expected_desired_mode), + ); + if ( + !context + || !connectionActionAuthorityMatches( + expectedAuthority, + context.authority, + ) + || !isSnapshotRuntimeCurrent(exactRuntimeId) + || getCurrentState() !== currentState + ) return reopenedOnly; + + return { + reopenDispatched: true, + reopenResult, + verifyDispatched: true, + verifyResult: await verify(context, exactRuntimeId), + }; +} + +export async function clearConnectionFailureAfterSuccessfulRefresh( + refresh: () => Promise, + clearError: () => void, +): Promise { + const refreshed = await refresh(); + if (!refreshed) return false; + clearError(); + return true; +} + +export function connectionRecoveryEscapeKey({ + snapshotRuntimeId, + attempt, + target, +}: { + snapshotRuntimeId: string | null | undefined; + attempt: XgridsConnectionAttempt | null | undefined; + target: ReadOnlyConnectionObservationTarget | null | undefined; +}): string | null { + const runtimeId = snapshotRuntimeId?.trim(); + if (!runtimeId || (!attempt && !target)) return null; + const deviceKey = target + ? transportRefEquivalenceKey(target.deviceId) + : "no-target"; + const connectionMode = target?.connectionMode + ?? attempt?.connection_mode + ?? "no-mode"; + // Observation authority may legitimately promote configured -> fresh after + // this exact Scan. Its source/action is therefore presentation evidence, not + // part of the operator escape identity. Runtime, attempt, device and mode + // continue to fence a replacement or a new recovery requirement. + return [ + runtimeId, + attempt?.attempt_id ?? "no-attempt", + attempt?.safe_next_action ?? "no-attempt-action", + deviceKey, + connectionMode, + ].join(":"); +} + +export function connectionRecoveryIsRequired( + recoveryKey: string | null | undefined, + escapedKey: string | null | undefined, +): boolean { + return Boolean(recoveryKey && recoveryKey !== escapedKey); +} + +export function connectionRecoveryEscapeAfterScan( + scanSucceeded: boolean, + recoveryKey: string | null | undefined, +): string | null { + return scanSucceeded && recoveryKey ? recoveryKey : null; +} const connectionCopy: Record = { bridge: { - stepTitle: "Передайте настройки общей сети", + stepTitle: "Настройка общей сети", ssidLabel: "Название общей сети Wi‑Fi", ssidPlaceholder: "Сеть локального контура", - buttonLabel: "Подключить K1 к общей сети", - safetyNote: "K1 получит реквизиты существующей сети одним рассмотренным BLE-запросом без автоматического повтора.", + buttonLabel: "Применить", + safetyNote: "Настройки применяются один раз после явного нажатия.", }, "quick-connect": { - stepTitle: "Включите точку доступа K1 и подключитесь к ней", - buttonLabel: "Включить точку K1 и подключиться", - safetyNote: "Mission Core сначала проверит локальный device-scoped профиль выбранного K1. Если профиль отсутствует, операция остановится до BLE-записи. После preflight Mission Core отправит один рассмотренный AP-enable кадр и найдёт точный SSID выбранного устройства.", + stepTitle: "Прямое подключение", + buttonLabel: "Применить", + safetyNote: "Действие выполняется один раз после явного нажатия.", }, "direct-connect": { - stepTitle: "Подключите K1 к хотспоту контроллера", + stepTitle: "Настройка хотспота контроллера", ssidLabel: "Название хотспота контроллера", - ssidPlaceholder: "SSID управляющего устройства", - buttonLabel: "Подключить K1 к хотспоту", - safetyNote: "Хотспот должен быть уже включён, а управляющее устройство — иметь к нему маршрут. K1 получит его реквизиты одним рассмотренным BLE-запросом.", + ssidPlaceholder: "Название хотспота", + buttonLabel: "Применить", + safetyNote: "Перед продолжением включите хотспот контроллера.", }, }; @@ -78,23 +1136,39 @@ function WizardStep({ ); } -function DeviceRow({ device, selected, onSelect }: { +function DeviceRow({ + device, + selected, + selectionDisabled = false, + selectedLabel = "Выбрано", + disabledLabel = "Недоступно", + actionLabel = "Выбрать", + onSelect, +}: { device: BleDevice; selected: boolean; + selectionDisabled?: boolean; + selectedLabel?: string; + disabledLabel?: string; + actionLabel?: string; onSelect: () => void; }) { return (
@@ -116,210 +1194,2348 @@ function DeviceRow({ device, selected, onSelect }: { export function K1ProvisioningPipeline({ controller, - phaseLabel, - phaseTone, + desiredMode = DEFAULT_CONNECTION_MODE, + onDesiredModeChange = () => undefined, }: { controller: XgridsK1Controller; - phaseLabel: string; - phaseTone: StatusTone; + desiredMode?: ConnectionMode; + onDesiredModeChange?: (mode: ConnectionMode) => void | Promise; }) { - const { state, pendingAction, scan, connect, verifyConnection } = controller; - const [powerConfirmed, setPowerConfirmed] = useState(false); + const { + state, + pendingAction, + error, + errorDiagnostic, + errorCorrelation, + refresh, + clearError, + scanWithResult, + connect, + selectConnectionMode, + verifyConnection, + prepareConnectionReconfigurationWithResult, + isSnapshotRuntimeCurrent, + getConnectionActionAuthority, + getConnectionRecoveryObservationTarget, + isConnectionPolicyActionAllowedCurrent, + isConnectionActionAuthorityCurrent, + } = controller; const [selectedDeviceId, setSelectedDeviceId] = useState(""); + const [modeResetPending, setModeResetPending] = useState<{ + resetId: string; + targetMode: ConnectionMode; + } | null>(null); + const modeResetInFlight = pendingAction === "mode" || modeResetPending !== null; + const [selectedDeviceSnapshot, setSelectedDeviceSnapshot] = useState(null); const [ssid, setSsid] = useState(""); const [password, setPassword] = useState(""); - const [connectionMode, setConnectionMode] = useState( - DEFAULT_CONNECTION_MODE, - ); - const provisioningIntentRef = useRef(null); + const [passwordVisible, setPasswordVisible] = useState(false); + useEffect(() => { + if (password.length === 0 && passwordVisible) { + setPasswordVisible(false); + } + }, [password, passwordVisible]); + const connectionMode = desiredMode; + const [successfulLocalConnect, setSuccessfulLocalConnect] = + useState(null); + const [connectionAttemptPresentation, setConnectionAttemptPresentation] = + useState( + emptyProvisioningAttemptPresentation, + ); + const hydratedScenarioResetPresentationKey = useRef(null); + const [reconfigurationDevicePresentation, setReconfigurationDevicePresentation] = + useState(null); + const [preparingReconfigurationRequest, setPreparingReconfigurationRequest] = + useState(null); + const [candidateUnavailableMessage, setCandidateUnavailableMessage] = + useState(null); + const [scanSecondsRemaining, setScanSecondsRemaining] = useState(null); + const [explicitProvisioningDraft, setExplicitProvisioningDraft] = + useState(null); + const [escapedAppliedAttemptKey, setEscapedAppliedAttemptKey] = + useState(null); + const [escapedConnectionRecoveryKey, setEscapedConnectionRecoveryKey] = + useState(null); + const [failedConnectionRecoveryKey, setFailedConnectionRecoveryKey] = + useState(null); + const [failedPhysicalRecoveryKey, setFailedPhysicalRecoveryKey] = + useState(null); + // Browser-local generation for work which may settle after an awaited + // controller call. A scenario reset does not replace snapshot_runtime_id, + // so runtime identity alone cannot prevent an old Scan/Verify/Retire result + // from repopulating the freshly reset UI. + const localScenarioActionEpoch = useRef(0); + const observedScenarioResetActionKey = useRef(null); + const [searchPresentation, setSearchPresentation] = + useState(emptySearchPresentation); + const searchPresentationSequence = useRef(0); + const resetSearchPresentation = useCallback(() => { + // Invalidate any late completion from a scan which belonged to an older + // runtime, mode or local draft before clearing its visible latch. + searchPresentationSequence.current += 1; + setSearchPresentation(null); + setScanSecondsRemaining(null); + }, []); + const localDraftFence = useRef(null); const devices = state?.devices ?? []; - const isBusy = pendingAction !== null; + const snapshotRuntimeId = state?.snapshot_runtime_id?.trim() || null; + const scenarioResetBoundary = scenarioResetPresentationBoundary( + state, + connectionMode, + ); + const activeScenarioReset = scenarioResetBoundary.active; + const scenarioResetPresentationKey = scenarioResetBoundary.key; + const actionFenceSequence = useRef(0); + const renderedSnapshotRuntimeId = useRef(snapshotRuntimeId); + const injectedRuntimeActionFence = useContext(RuntimeActionFenceTestContext); + const activeRuntimeActionFence = useRef(null); + const connectIntentSequence = useRef(0); + const activeConnectIntent = useRef(null); + if ( + scenarioResetPresentationKey + && observedScenarioResetActionKey.current !== scenarioResetPresentationKey + ) { + // Observe toolbar resets as a synchronous ref boundary. The toolbar owns + // the same backend action as the in-panel reset, but it does not execute + // changeDesiredConnectionMode in this component. + observedScenarioResetActionKey.current = scenarioResetPresentationKey; + localScenarioActionEpoch.current += 1; + activeRuntimeActionFence.current = null; + activeConnectIntent.current = null; + searchPresentationSequence.current += 1; + } + const currentRuntimeActionFence = injectedRuntimeActionFence + ?? activeRuntimeActionFence.current; + const readCurrentRuntimeActionFence = useCallback( + () => injectedRuntimeActionFence ?? activeRuntimeActionFence.current, + [injectedRuntimeActionFence], + ); + if (renderedSnapshotRuntimeId.current !== snapshotRuntimeId) { + renderedSnapshotRuntimeId.current = snapshotRuntimeId; + activeRuntimeActionFence.current = null; + activeConnectIntent.current = null; + } + const activateRuntimeActionFence = useCallback(( + authority: ConnectionActionAuthoritySnapshot, + ): RuntimeActionFence | null => { + const existingFence = readCurrentRuntimeActionFence(); + if (existingFence) { + const existingFenceIsCurrent = Boolean( + typeof isConnectionActionAuthorityCurrent === "function" + ? isConnectionActionAuthorityCurrent(existingFence) + : typeof isSnapshotRuntimeCurrent !== "function" + || isSnapshotRuntimeCurrent(existingFence.snapshotRuntimeId), + ); + if (existingFenceIsCurrent || injectedRuntimeActionFence) return null; + if (activeRuntimeActionFence.current?.clickToken !== existingFence.clickToken) { + return null; + } + // A scenario reset can rotate the full connection authority without + // replacing the process runtime. Reclaim only that provably stale local + // click so an old settlement cannot poison every later explicit action. + activeRuntimeActionFence.current = null; + } + if ( + !snapshotRuntimeId + || renderedSnapshotRuntimeId.current !== snapshotRuntimeId + || authority.snapshotRuntimeId !== snapshotRuntimeId + || ( + typeof isSnapshotRuntimeCurrent === "function" + && !isSnapshotRuntimeCurrent(authority.snapshotRuntimeId) + ) + ) return null; + const fence = { + ...authority, + clickToken: actionFenceSequence.current + 1, + }; + actionFenceSequence.current = fence.clickToken; + activeRuntimeActionFence.current = fence; + return fence; + }, [ + injectedRuntimeActionFence, + isConnectionActionAuthorityCurrent, + isSnapshotRuntimeCurrent, + readCurrentRuntimeActionFence, + snapshotRuntimeId, + ]); + const beginRuntimeActionFence = useCallback(( + authorityMode: ConnectionMode = connectionMode, + ): RuntimeActionFence | null => { + const renderedAuthority = connectionActionAuthoritySnapshot( + state, + authorityMode, + ); + const authority = typeof getConnectionActionAuthority === "function" + ? getConnectionActionAuthority(authorityMode) + : null; + if ( + !snapshotRuntimeId + || renderedSnapshotRuntimeId.current !== snapshotRuntimeId + || !authority + || authority.snapshotRuntimeId !== snapshotRuntimeId + || !connectionActionAuthorityMatches(authority, renderedAuthority) + ) return null; + return activateRuntimeActionFence(authority); + }, [ + activateRuntimeActionFence, + connectionMode, + getConnectionActionAuthority, + snapshotRuntimeId, + state, + ]); + const actionAuthorityIsCurrent = useCallback(( + fence: RuntimeActionFence, + ) => Boolean( + typeof isConnectionActionAuthorityCurrent === "function" + ? isConnectionActionAuthorityCurrent(fence) + : typeof isSnapshotRuntimeCurrent !== "function" + || isSnapshotRuntimeCurrent(fence.snapshotRuntimeId), + ), [isConnectionActionAuthorityCurrent, isSnapshotRuntimeCurrent]); + const runtimeActionIsCurrent = useCallback(( + fence: RuntimeActionFence | null | undefined, + ) => runtimeActionFenceMatches( + fence, + renderedSnapshotRuntimeId.current, + readCurrentRuntimeActionFence(), + actionAuthorityIsCurrent, + ), [actionAuthorityIsCurrent, readCurrentRuntimeActionFence]); + const runtimeClickIsCurrent = useCallback(( + fence: RuntimeActionFence | null | undefined, + ) => Boolean( + runtimeActionFenceMatches( + fence, + renderedSnapshotRuntimeId.current, + readCurrentRuntimeActionFence(), + ) + && fence + && ( + typeof isSnapshotRuntimeCurrent !== "function" + || isSnapshotRuntimeCurrent(fence.snapshotRuntimeId) + ), + ), [isSnapshotRuntimeCurrent, readCurrentRuntimeActionFence]); + const promoteRuntimeActionFence = useCallback(( + fence: RuntimeActionFence, + authority: ConnectionActionAuthoritySnapshot, + ): RuntimeActionFence | null => { + if (!runtimeClickIsCurrent(fence)) return null; + const promotedFence = { ...fence, ...authority }; + if ( + typeof isConnectionActionAuthorityCurrent === "function" + && !isConnectionActionAuthorityCurrent(promotedFence) + ) return null; + activeRuntimeActionFence.current = promotedFence; + return promotedFence; + }, [isConnectionActionAuthorityCurrent, runtimeClickIsCurrent]); + const promoteFenceFromReconfigurationResult = useCallback(( + fence: RuntimeActionFence, + result: { succeeded: boolean; observedState: typeof state }, + intent: XgridsConnectionReconfigurationIntent | "cancel", + ): RuntimeActionFence | null => { + if ( + !runtimeClickIsCurrent(fence) + || !result.succeeded + || !result.observedState + || typeof getConnectionActionAuthority !== "function" + ) { + return null; + } + const observedAuthority = reconfigurationContinuationAuthority( + fence, + result.observedState, + intent, + getConnectionActionAuthority(fence.connectionMode), + ); + return observedAuthority + ? promoteRuntimeActionFence(fence, observedAuthority) + : null; + }, [ + getConnectionActionAuthority, + promoteRuntimeActionFence, + runtimeClickIsCurrent, + ]); + const promoteFenceFromConnectResult = useCallback(( + fence: RuntimeActionFence, + observedState: typeof state, + expectedDeviceId: string, + expectedConnectionMode: ConnectionMode, + ): RuntimeActionFence | null => { + if ( + !runtimeClickIsCurrent(fence) + || !observedState + || typeof getConnectionActionAuthority !== "function" + ) return null; + const continuationAuthority = connectContinuationAuthority( + fence, + observedState, + getConnectionActionAuthority(fence.connectionMode), + expectedDeviceId, + expectedConnectionMode, + ); + return continuationAuthority + ? promoteRuntimeActionFence(fence, continuationAuthority) + : null; + }, [ + getConnectionActionAuthority, + promoteRuntimeActionFence, + runtimeClickIsCurrent, + ]); + const retireRuntimeClickFence = useCallback(( + fence: RuntimeActionFence | null | undefined, + ) => { + if (!runtimeClickIsCurrent(fence)) return; + activeRuntimeActionFence.current = null; + }, [runtimeClickIsCurrent]); + const commitDesiredModeForExplicitAction = useCallback(async ( + explicitMode: ConnectionMode = connectionMode, + ): Promise => { + const currentAuthority = typeof getConnectionActionAuthority === "function" + ? getConnectionActionAuthority(explicitMode) + : null; + if (currentAuthority) return currentAuthority; + const expectedRevision = state?.desired_connection_mode_revision; + if ( + !Number.isInteger(expectedRevision) + || (expectedRevision ?? -1) < 0 + ) return null; + const committed = await selectConnectionMode({ + connection_mode: explicitMode, + expected_revision: expectedRevision as number, + }); + if (!committed || typeof getConnectionActionAuthority !== "function") { + return null; + } + return getConnectionActionAuthority(explicitMode); + }, [ + connectionMode, + getConnectionActionAuthority, + selectConnectionMode, + state?.desired_connection_mode_revision, + ]); + const currentPreparingReconfigurationRequest = currentRuntimeActionRequest( + preparingReconfigurationRequest, + snapshotRuntimeId, + currentRuntimeActionFence, + actionAuthorityIsCurrent, + ); + const preparingReconfigurationIntent = + currentPreparingReconfigurationRequest?.intent ?? null; + const presentedPendingAction = pendingAction && ( + typeof isSnapshotRuntimeCurrent !== "function" + || runtimeActionIsCurrent(currentRuntimeActionFence) + ) ? pendingAction : null; + const reconfiguration = activeConnectionReconfiguration(state); + const reconfigurationRevision = state?.connection_reconfiguration?.revision ?? 0; + const reconfigurationIntentId = reconfiguration?.intent_id ?? null; + const reconfigurationIntent = reconfiguration?.intent ?? null; + const reconfigurationActive = connectionMode === "bridge" + && reconfiguration !== null; + const changeNetworkDialogue = Boolean( + reconfigurationActive && reconfigurationIntent === "change-network", + ); + const changeNetworkRequiredDeviceId = changeNetworkDialogue + ? reconfiguration?.required_transport_ref?.trim() || "" + : ""; + const activeBindingKey = state?.connection_lifecycle?.active_binding_key ?? null; const credentialsReady = connectionMode === "quick-connect" || (ssid.trim().length > 0 && password.length > 0); - const networkWriteReconciliationPending = Boolean( - state?.network_write_reconciliation, - ); - const deviceSummary = useMemo( + const freshDeviceSummary = useMemo( () => provisioningCandidateById(devices, selectedDeviceId), [devices, selectedDeviceId], ); - const canConnect = !networkWriteReconciliationPending && canSubmitProvisioningMutation({ + const localBackendSession = successfulLocalConnect?.connectionMode === connectionMode + && successfulLocalConnect.acceptedSessionKey + ? locallyInitiatedBleSessionTarget( + state, + successfulLocalConnect.deviceId, + connectionMode, + { + requiredSessionKey: successfulLocalConnect.acceptedSessionKey, + }, + ) + : null; + // Only a fresh advertisement is a target for a new network mutation. A + // retained backend session remains useful connection context, but must not + // be rendered as current BLE presence or silently reused as write authority. + const selectedTarget = freshDeviceSummary + ? { + deviceId: freshDeviceSummary.device_id, + label: freshDeviceSummary.name || freshDeviceSummary.device_id, + source: "fresh-scan" as const, + } + : null; + const retainedSessionLabel = localBackendSession + ? selectedDeviceSnapshot?.name?.trim() || localBackendSession.transportRef + : null; + const modeCopy = connectionCopy[connectionMode]; + const requestedModeLabel = connectionModeOptions.find( + (option) => option.value === connectionMode, + )?.label ?? connectionMode; + const backendTopology = backendConnectionTopology(state); + // A backend-owned reconfiguration dialogue wins over a concurrently stale + // reachability projection. Otherwise a resumed intent could render the old + // connected summary and disable every fresh candidate, leaving no Cancel. + const selectedModeConnected = !reconfigurationActive + && isReachableConnectionLease(state, connectionMode); + const selectedModeTopology = backendTopology?.connectionMode === connectionMode + ? backendTopology + : null; + const selectedModeAppliedUnverified = Boolean( + selectedModeTopology?.source === "applied" + && selectedModeTopology.status !== "active", + ); + const desiredModeRevision = state?.desired_connection_mode_revision; + const discoveryGeneration = state?.ble_discovery_generation; + useEffect(() => { + if ( + !scenarioResetPresentationKey + || hydratedScenarioResetPresentationKey.current === scenarioResetPresentationKey + ) return; + // A committed scenario reset is a new local presentation boundary even + // when the backend process/runtime id did not change. Never retain the + // old click-owned settlement or a pending local request across that + // revision. Late promises are fenced by localScenarioActionEpoch. + hydratedScenarioResetPresentationKey.current = scenarioResetPresentationKey; + setConnectionAttemptPresentation(null); + setPreparingReconfigurationRequest(null); + setReconfigurationDevicePresentation(null); + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setSuccessfulLocalConnect(null); + setExplicitProvisioningDraft(null); + setSsid(""); + setPassword(""); + setCandidateUnavailableMessage(null); + setEscapedAppliedAttemptKey(null); + setEscapedConnectionRecoveryKey(null); + setFailedConnectionRecoveryKey(null); + setFailedPhysicalRecoveryKey(null); + setSearchPresentation(null); + setScanSecondsRemaining(null); + }, [scenarioResetPresentationKey]); + const searchPresentationIsCurrent = Boolean( + searchPresentation + && searchPresentation.snapshotRuntimeId === snapshotRuntimeId + && searchPresentation.connectionMode === connectionMode + && searchPresentation.desiredModeRevision === (desiredModeRevision ?? null), + ); + const searchRequested = Boolean( + searchPresentationIsCurrent + && ( + searchPresentation?.active + || ( + Number.isInteger(searchPresentation?.completedDiscoveryGeneration) + && searchPresentation?.completedDiscoveryGeneration === discoveryGeneration + ) + ), + ); + const searchDisplayActive = Boolean( + searchPresentationIsCurrent && searchPresentation?.active, + ); + const connectionAttemptView = provisioningAttemptViewState( + connectionAttemptPresentation, + state, + ); + const connectionAttemptSettling = connectionAttemptView === "submitting" + || connectionAttemptView === "settling"; + const connectionAttemptFailed = connectionAttemptView === "failed"; + const provisioningMutationBusy = Boolean( + presentedPendingAction + || modeResetPending !== null + || currentPreparingReconfigurationRequest + || searchDisplayActive + || connectionAttemptSettling + ); + const isBusy = provisioningMutationBusy; + const activeSearchSequence = searchDisplayActive + ? searchPresentation?.sequence ?? null + : null; + const reconfigurationFreshScanReady = Boolean( + reconfigurationActive + && reconfiguration?.status === "fresh-scan-completed" + && Number.isInteger(discoveryGeneration) + && reconfiguration.fresh_discovery_generation === discoveryGeneration, + ); + const exactChangeNetworkFreshDevice = exactChangeNetworkCandidate( + reconfiguration, + devices, + discoveryGeneration, + ); + const discoveryGenerationCurrent = Number.isInteger(discoveryGeneration) + && (discoveryGeneration ?? -1) >= 0; + const explicitProvisioningDraftContext = { + deviceId: selectedDeviceId || null, + connectionMode, + desiredModeRevision, + discoveryGeneration, + snapshotRuntimeId, + reconfigurationRevision, + reconfigurationIntentId, + activeBindingKey, + requiredTransportRef: reconfiguration?.required_transport_ref ?? null, + requiredConnectionMode: reconfiguration?.required_connection_mode ?? null, + }; + const explicitProvisioningDraftRetained = explicitProvisioningDraftMatches( + explicitProvisioningDraft, + explicitProvisioningDraftContext, + ); + const explicitProvisioningDraftContextRetained = Boolean( + explicitProvisioningDraft + && explicitProvisioningDraftMatches( + explicitProvisioningDraft, + { + ...explicitProvisioningDraftContext, + discoveryGeneration: explicitProvisioningDraft.discoveryGeneration, + }, + ) + ); + const explicitProvisioningDraftStale = Boolean( + explicitProvisioningDraftContextRetained + && ( + !explicitProvisioningDraftRetained + || explicitProvisioningDraft?.requiresFreshScanBeforeSubmit === true + ), + ); + const explicitProvisioningRequested = Boolean( + explicitProvisioningDraftRetained + && selectedTarget + && transportRefEquivalenceKey(explicitProvisioningDraft?.deviceId) + === transportRefEquivalenceKey(selectedTarget.deviceId), + ); + const explicitProvisioningRefreshRequired = Boolean( + explicitProvisioningDraftRetained + && explicitProvisioningDraft?.requiresFreshScanBeforeSubmit, + ); + const backendMutationAllowed = connectionPolicyAllows(state, "provision-fresh-device"); + const appliedNetworkAttempt = state?.connection_attempt?.phase === "network_applied" + ? state.connection_attempt + : null; + const correlatedConnectionAttempt = connectionAttemptForRuntimeError( + errorCorrelation, + state, + ); + const unknownNetworkOutcomeAttempt = state?.connection_attempt?.phase + === "network_outcome_unknown" + && !["accepted", "running"].includes(state.connection_attempt.status) + ? state.connection_attempt + : null; + const connectionRecoveryAttempt = correlatedConnectionAttempt + ?? unknownNetworkOutcomeAttempt; + const connectionRecoveryObservationAllowed = Boolean( + !connectionRecoveryAttempt + || [ + "continue-with-control-verification", + "verify-control-read-only", + ].includes(connectionRecoveryAttempt.safe_next_action), + ); + const appliedAttemptRecoveryKey = appliedNetworkAttempt + && snapshotRuntimeId + ? `${snapshotRuntimeId}:${appliedNetworkAttempt.attempt_id}` + : null; + const appliedRecoveryReconfigurationPrepared = Boolean( + appliedNetworkAttempt?.control_state !== "ready" + && appliedNetworkAttempt?.connection_mode === "bridge" + && reconfiguration?.intent === "select-device" + && reconfiguration.required_connection_mode === "bridge" + && ["awaiting-fresh-scan", "fresh-scan-completed"].includes( + reconfiguration.status, + ), + ); + const appliedRecoveryExplicitlyEscaped = Boolean( + appliedAttemptRecoveryKey + && ( + escapedAppliedAttemptKey === appliedAttemptRecoveryKey + || appliedRecoveryReconfigurationPrepared + ), + ); + const unresolvedAppliedAttempt = appliedNetworkAttempt + && connectionAttemptOwnsAppliedNetworkRecovery(appliedNetworkAttempt) + && !appliedRecoveryExplicitlyEscaped + ? appliedNetworkAttempt + : null; + // A current physical-command ambiguity is stronger than an older + // network-applied record. Never trap a powered-off/failed K1 behind the + // historical connection Verify card. + const physicalRecoveryRequired = requiresReadOnlyPhysicalRecovery(state); + const networkRecoveryRequired = unresolvedAppliedAttempt !== null + && !physicalRecoveryRequired; + // Scan keeps the reset-owned clean draft intact, while the first later + // Connect/Verify operation releases it. This hides only pre-reset saved + // history and never masks a new post-reset failure. + const scenarioResetOwnsCleanDraft = scenarioResetOwnsCleanConnectionDraft( + state, + connectionMode, + ); + const connectionRecoveryTarget = !scenarioResetOwnsCleanDraft + && !physicalRecoveryRequired + && !unresolvedAppliedAttempt + && !selectedModeConnected + && !reconfigurationActive + && !connectionAttemptSettling + && connectionRecoveryObservationAllowed + ? recommendedConnectionRecoveryObservationTarget(state) + : null; + const connectionRecoveryKey = connectionRecoveryEscapeKey({ + snapshotRuntimeId, + attempt: connectionRecoveryAttempt, + target: connectionRecoveryTarget, + }); + const connectionRecoveryRequired = connectionRecoveryIsRequired( + connectionRecoveryKey, + escapedConnectionRecoveryKey, + ) && !(searchRequested && !searchDisplayActive); + const connectionRecoveryVerificationFailed = Boolean( + connectionRecoveryKey + && failedConnectionRecoveryKey === connectionRecoveryKey, + ); + const appliedControlSettlementPending = Boolean( + unresolvedAppliedAttempt + && ["accepted", "running"].includes(unresolvedAppliedAttempt.status) + && unresolvedAppliedAttempt.safe_next_action === "wait-for-current-attempt", + ); + const appliedNetworkRecoveryTarget = unresolvedAppliedAttempt + ? serverBoundAppliedNetworkObservationTarget( + state, + unresolvedAppliedAttempt.connection_mode, + ) + : null; + const networkRecoveryModeLabel = unresolvedAppliedAttempt + ? connectionModeOptions.find( + (option) => option.value === unresolvedAppliedAttempt.connection_mode, + )?.label ?? unresolvedAppliedAttempt.connection_mode + : null; + const localProvisioningPrerequisitesReady = canSubmitProvisioningMutation({ devices, selectedDeviceId, - powerConfirmed, credentialsReady, - isBusy, + isBusy: provisioningMutationBusy, }); - const modeCopy = connectionCopy[connectionMode]; - const selectedModeConnected = isReachableConnectionLease(state, connectionMode); + const physicalRecoveryBinding = physicalRecoveryRequired + ? readOnlyPhysicalRecoveryBinding(state) + : null; + const physicalRecoveryTarget = physicalRecoveryRequired + ? readOnlyConnectionObservationTarget(state) + : null; + const physicalRecoveryAuthorityKey = physicalRecoveryPresentationAuthorityKey( + state, + ); + const physicalRecoveryVerificationFailed = Boolean( + physicalRecoveryAuthorityKey + && failedPhysicalRecoveryKey === physicalRecoveryAuthorityKey, + ); + const physicalReadOnlyVerificationAvailable = Boolean( + physicalRecoveryTarget?.serverBound, + ); + useEffect(() => { + if ( + failedPhysicalRecoveryKey + && failedPhysicalRecoveryKey !== physicalRecoveryAuthorityKey + ) { + setFailedPhysicalRecoveryKey(null); + } + }, [failedPhysicalRecoveryKey, physicalRecoveryAuthorityKey]); + const physicalRecoveryModeLabel = physicalRecoveryBinding + ? connectionModeOptions.find( + (option) => option.value === physicalRecoveryBinding.connectionMode, + )?.label ?? physicalRecoveryBinding.connectionMode + : "Прежнее подключение"; + const reconfigurationTargetAllowed = reconfiguration === null + || ( + selectedTarget !== null + && reconfigurationAllowsFreshDevice( + reconfiguration, + selectedTarget.deviceId, + connectionMode, + ) + ); + const canConnect = discoveryGenerationCurrent + && explicitProvisioningRequested + && !explicitProvisioningDraftStale + && !explicitProvisioningRefreshRequired + && !unresolvedAppliedAttempt + && reconfigurationTargetAllowed + && canAdmitProvisioningConnection({ + policyAllowed: backendMutationAllowed, + targetSource: selectedTarget?.source ?? null, + hasSuccessfulLocalConnect: successfulLocalConnect !== null, + localPrerequisitesReady: localProvisioningPrerequisitesReady, + }); + + const scanAllowedByPolicy = connectionPolicyAllows(state, "scan-ble"); + const backendScanAllowed = scanAllowedByPolicy + && !isBusy + && !networkRecoveryRequired; + const physicalStopRecoverySettling = isPhysicalStopRecoverySettling(state); + const canScan = backendScanAllowed && !physicalStopRecoverySettling; + const canScanAppliedRecovery = Boolean( + unresolvedAppliedAttempt + && !appliedControlSettlementPending + && unresolvedAppliedAttempt.connection_mode !== "bridge" + && scanAllowedByPolicy + && !isBusy + && !physicalStopRecoverySettling, + ); + const trustedBinding = trustedConnectionBinding(state); + const connectedEndpoint = selectedModeConnected + ? selectedModeTopology?.endpoint?.trim() || null + : null; + const connectedDeviceIdentity = selectedModeConnected + ? selectedTarget?.label + || retainedSessionLabel + || (trustedBinding?.connectionMode === connectionMode + ? trustedBinding.deviceId + : null) + || state?.connection_lifecycle?.active_binding?.transport_ref?.trim() + || connectedEndpoint + || "Активное подключение" + : null; + const connectionActionPending = presentedPendingAction === "connect" + || presentedPendingAction === "verify"; + const verificationActionPending = presentedPendingAction === "verify"; + const networkActionPending = presentedPendingAction === "connect" + || presentedPendingAction === "mode" + || Boolean( + changeNetworkDialogue + && presentedPendingAction === "scan" + ); + const selectDeviceAllowedByPolicy = connectionPolicyAllows( + state, + "prepare-select-device", + ); + const changeNetworkAllowedByPolicy = connectionPolicyAllows( + state, + "prepare-change-network", + ); + const canPrepareSelectDevice = connectionMode === "bridge" + && !isBusy + && !networkRecoveryRequired + && selectDeviceAllowedByPolicy; + const canPrepareRecoverySelectDevice = Boolean( + unresolvedAppliedAttempt?.connection_mode === "bridge" + && !appliedControlSettlementPending + && !isBusy + && selectDeviceAllowedByPolicy, + ); + const canPrepareChangeNetwork = connectionMode === "bridge" + && !isBusy + && !networkRecoveryRequired + && changeNetworkAllowedByPolicy; + const selectDeviceActionApplicable = connectionMode === "bridge" + && connectedReconfigurationActionApplicable(state, "prepare-select-device"); + const changeNetworkActionApplicable = connectionMode === "bridge" + && connectedReconfigurationActionApplicable(state, "prepare-change-network"); + const showNetworkStep = Boolean( + !physicalRecoveryRequired + && !connectionRecoveryRequired + && ( + selectedModeConnected + || explicitProvisioningDraftRetained + || explicitProvisioningDraftContextRetained + || unresolvedAppliedAttempt + || connectionAttemptSettling + || connectionAttemptFailed + || (changeNetworkDialogue && Boolean(changeNetworkRequiredDeviceId)) + ) + ); + const requestExplicitProvisioning = useCallback(( + deviceId: string, + { + preservePassword = false, + requiresFreshScanBeforeSubmit = false, + }: { + preservePassword?: boolean; + requiresFreshScanBeforeSubmit?: boolean; + } = {}, + ) => { + if ( + networkRecoveryRequired + || + !Number.isInteger(desiredModeRevision) + || (desiredModeRevision ?? -1) < 0 + || !Number.isInteger(discoveryGeneration) + || (discoveryGeneration ?? -1) < 0 + || snapshotRuntimeId === null + ) return; + setExplicitProvisioningDraft({ + snapshotRuntimeId, + deviceId, + connectionMode, + desiredModeRevision: desiredModeRevision as number, + discoveryGeneration: discoveryGeneration as number, + reconfigurationRevision, + reconfigurationIntentId, + activeBindingKey, + requiredTransportRef: reconfiguration?.required_transport_ref ?? null, + requiredConnectionMode: reconfiguration?.required_connection_mode ?? null, + requiresFreshScanBeforeSubmit, + }); + setCandidateUnavailableMessage(null); + if (!preservePassword) setPassword(""); + }, [ + activeBindingKey, + connectionMode, + desiredModeRevision, + discoveryGeneration, + networkRecoveryRequired, + reconfiguration?.required_connection_mode, + reconfiguration?.required_transport_ref, + reconfigurationIntentId, + reconfigurationRevision, + snapshotRuntimeId, + ]); + useEffect(() => { + if (!searchDisplayActive) { + setScanSecondsRemaining(null); + return; + } + const startedAt = Date.now(); + const updateCountdown = () => { + const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1_000); + setScanSecondsRemaining(Math.max(0, 6 - elapsedSeconds)); + }; + updateCountdown(); + const timer = window.setInterval(updateCountdown, 250); + return () => window.clearInterval(timer); + }, [activeSearchSequence, searchDisplayActive]); + useEffect(() => { + if (!appliedNetworkAttempt) return; + // Once the backend records that the network write reached the K1, local + // credentials and Apply authority are spent even if control verification + // is still unresolved. Recovery below is read-only and server-targeted. + setExplicitProvisioningDraft(null); + setSsid(""); + setPassword(""); + }, [appliedNetworkAttempt?.attempt_id, appliedNetworkAttempt?.phase]); + useEffect(() => { + if (connectionAttemptView !== "retired") return; + setConnectionAttemptPresentation(null); + }, [connectionAttemptView]); + useEffect(() => { + if ( + !successfulLocalConnect + || localBackendSession + || selectedModeAppliedUnverified + ) return; + setSuccessfulLocalConnect(null); + // A lost LAN lease does not erase a fresh BLE target from the operator's + // scan, but it does retire the previous provisioning draft. Recovery tries + // read-only observation first; credentials require a new explicit action. + if (freshDeviceSummary) return; + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setExplicitProvisioningDraft(null); + setPassword(""); + }, [ + freshDeviceSummary, + localBackendSession, + selectedModeAppliedUnverified, + successfulLocalConnect, + ]); + useEffect(() => { + if ( + !selectedDeviceId + || freshDeviceSummary + || localBackendSession + || explicitProvisioningDraftRetained + || explicitProvisioningDraftContextRetained + || connectionAttemptPresentation + ) return; + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setExplicitProvisioningDraft(null); + setPassword(""); + }, [ + connectionAttemptPresentation, + explicitProvisioningDraftRetained, + explicitProvisioningDraftContextRetained, + freshDeviceSummary, + localBackendSession, + selectedDeviceId, + ]); useEffect(() => { - if (selectedDeviceId && !deviceSummary) { - provisioningIntentRef.current = null; - setSelectedDeviceId(""); + if (!explicitProvisioningDraft || explicitProvisioningDraftRetained) return; + if (explicitProvisioningDraftStale) { + setCandidateUnavailableMessage( + "Результат Bluetooth-поиска устарел. Настройки не отправлены; выполните явный повторный поиск.", + ); + return; } - }, [deviceSummary, selectedDeviceId]); + if (connectionAttemptPresentation) return; + // Password entry is authority only for the exact fresh target, discovery + // generation and desired-mode revision that opened it. A later backend/tab + // transition invalidates that draft before another provisioning click can + // reuse it. + setExplicitProvisioningDraft(null); + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setPassword(""); + }, [ + connectionAttemptPresentation, + explicitProvisioningDraft, + explicitProvisioningDraftRetained, + explicitProvisioningDraftStale, + ]); - const canAdoptExistingBridge = connectionMode === "bridge" - && powerConfirmed - && deviceSummary !== null - && deviceSummary.connectable !== false - && !isBusy; + useEffect(() => { + if ( + !changeNetworkDialogue + || !exactChangeNetworkFreshDevice + || ( + transportRefEquivalenceKey(selectedDeviceId) + === transportRefEquivalenceKey(exactChangeNetworkFreshDevice.device_id) + && explicitProvisioningRequested + ) + ) return; + setSelectedDeviceId(exactChangeNetworkFreshDevice.device_id); + setSelectedDeviceSnapshot(exactChangeNetworkFreshDevice); + setSuccessfulLocalConnect(null); + requestExplicitProvisioning( + exactChangeNetworkFreshDevice.device_id, + { + preservePassword: true, + requiresFreshScanBeforeSubmit: false, + }, + ); + }, [ + changeNetworkDialogue, + exactChangeNetworkFreshDevice, + explicitProvisioningRequested, + requestExplicitProvisioning, + selectedDeviceId, + ]); - const resetProvisioningIntent = () => { - provisioningIntentRef.current = null; + useEffect(() => { + const nextFence = localProvisioningDraftFenceKey({ + snapshotRuntimeId, + reconfigurationRevision, + reconfigurationIntentId, + activeBindingKey, + requiredTransportRef: reconfiguration?.required_transport_ref ?? null, + requiredConnectionMode: reconfiguration?.required_connection_mode ?? null, + }); + if (localDraftFence.current === null) { + localDraftFence.current = nextFence; + return; + } + if (localDraftFence.current === nextFence) return; + localDraftFence.current = nextFence; + setSuccessfulLocalConnect(null); + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setExplicitProvisioningDraft(null); + setSsid(""); + setPassword(""); + setConnectionAttemptPresentation((current) => ( + current?.snapshotRuntimeId === snapshotRuntimeId ? current : null + )); + setReconfigurationDevicePresentation((current) => { + const requiredTransportRef = reconfiguration?.required_transport_ref?.trim(); + return reconfiguration?.intent === "change-network" + && requiredTransportRef + && current + && transportRefEquivalenceKey(current.deviceId) + === transportRefEquivalenceKey(requiredTransportRef) + ? current + : null; + }); + setPreparingReconfigurationRequest(null); + setCandidateUnavailableMessage(null); + setEscapedConnectionRecoveryKey(null); + setFailedConnectionRecoveryKey(null); + setFailedPhysicalRecoveryKey(null); + resetSearchPresentation(); + }, [ + activeBindingKey, + reconfiguration?.required_connection_mode, + reconfiguration?.required_transport_ref, + reconfigurationIntentId, + reconfigurationRevision, + resetSearchPresentation, + snapshotRuntimeId, + ]); + + useEffect(() => { + if (!selectedModeConnected || connectionActionPending) return; + setConnectionAttemptPresentation(null); + }, [connectionActionPending, selectedModeConnected]); + + useEffect(() => { + if ( + reconfigurationActive + || presentedPendingAction === "reconfigure" + || preparingReconfigurationIntent + ) return; + setReconfigurationDevicePresentation(null); + }, [ + presentedPendingAction, + preparingReconfigurationIntent, + reconfigurationActive, + ]); + + const repeatDeviceScan = async ({ + preserveDraft = false, + appliedRecoveryEscape = false, + physicalRecoveryScan = false, + connectionRecoveryEscape = false, + }: { + preserveDraft?: boolean; + appliedRecoveryEscape?: boolean; + physicalRecoveryScan?: boolean; + connectionRecoveryEscape?: boolean; + } = {}) => { + const actionEpoch = localScenarioActionEpoch.current; + const recoveryScanAttempt = appliedRecoveryEscape + && canScanAppliedRecovery + ? unresolvedAppliedAttempt + : null; + const exactPhysicalRecoveryScan = physicalRecoveryScan + && physicalRecoveryRequired + ? physicalRecoveryBinding + : null; + const exactConnectionRecoveryKey = connectionRecoveryEscape + && connectionRecoveryRequired + ? connectionRecoveryKey + : null; + if ( + (!canScan && !recoveryScanAttempt) + || ( + typeof isConnectionPolicyActionAllowedCurrent === "function" + && !isConnectionPolicyActionAllowedCurrent("scan-ble") + ) + ) return; + const scanMode = exactPhysicalRecoveryScan?.connectionMode + ?? recoveryScanAttempt?.connection_mode + ?? connectionMode; + const recoveryAttemptKey = recoveryScanAttempt + ? appliedAttemptRecoveryKey + : null; + const preservedDeviceId = preserveDraft ? selectedDeviceId : ""; + const preservedSnapshot = preserveDraft ? selectedDeviceSnapshot : null; + if ( + (recoveryScanAttempt || exactPhysicalRecoveryScan) + && connectionMode !== scanMode + ) { + void onDesiredModeChange(scanMode); + } + const modeAuthority = recoveryScanAttempt + ? typeof getConnectionActionAuthority === "function" + ? getConnectionActionAuthority(scanMode) + : null + : await commitDesiredModeForExplicitAction(scanMode); + if ( + !localScenarioActionEpochIsCurrent( + actionEpoch, + localScenarioActionEpoch.current, + ) + || !modeAuthority + ) return; + const actionFence = activateRuntimeActionFence(modeAuthority); + if (!actionFence || !runtimeActionIsCurrent(actionFence)) return; + const searchSequence = searchPresentationSequence.current + 1; + searchPresentationSequence.current = searchSequence; + setSearchPresentation({ + sequence: searchSequence, + snapshotRuntimeId: modeAuthority.snapshotRuntimeId, + connectionMode: scanMode, + desiredModeRevision: modeAuthority.desiredModeRevision, + active: true, + completedDiscoveryGeneration: null, + }); + setSuccessfulLocalConnect(null); + if (!preserveDraft) { + setSelectedDeviceId(""); + setSelectedDeviceSnapshot(null); + setExplicitProvisioningDraft(null); + setPassword(""); + } + setConnectionAttemptPresentation(null); + setCandidateUnavailableMessage(null); + let scanResult: BleDiscoverySubmitResult | null = null; + try { + if (!runtimeActionIsCurrent(actionFence)) return; + scanResult = await scanWithResult({ durationSeconds: 6 }); + if ( + !localScenarioActionEpochIsCurrent( + actionEpoch, + localScenarioActionEpoch.current, + ) + || !runtimeClickIsCurrent(actionFence) + ) return; + if (scanResult.succeeded && recoveryAttemptKey) { + setEscapedAppliedAttemptKey(recoveryAttemptKey); + } + const settledConnectionRecoveryKey = connectionRecoveryEscapeAfterScan( + scanResult.succeeded, + exactConnectionRecoveryKey, + ); + if (settledConnectionRecoveryKey) { + setEscapedConnectionRecoveryKey(settledConnectionRecoveryKey); + } + if (!preserveDraft || !scanResult.succeeded || !preservedDeviceId) return; + const exactDeviceObserved = scanResult.transportRefs.some( + (transportRef) => transportRefEquivalenceKey(transportRef) + === transportRefEquivalenceKey(preservedDeviceId), + ); + const refreshedAuthority = typeof getConnectionActionAuthority === "function" + ? getConnectionActionAuthority(scanMode) + : null; + if (!exactDeviceObserved || !refreshedAuthority) { + setCandidateUnavailableMessage( + "Выбранный K1 не найден в новом поиске. Настройки не отправлены; повторите поиск или выберите другой результат.", + ); + return; + } + setSelectedDeviceId(preservedDeviceId); + setSelectedDeviceSnapshot(preservedSnapshot); + setExplicitProvisioningDraft({ + snapshotRuntimeId: refreshedAuthority.snapshotRuntimeId, + deviceId: preservedDeviceId, + connectionMode: scanMode, + desiredModeRevision: refreshedAuthority.desiredModeRevision, + discoveryGeneration: refreshedAuthority.discoveryGeneration, + reconfigurationRevision: refreshedAuthority.reconfigurationRevision, + reconfigurationIntentId: refreshedAuthority.reconfigurationIntentId, + activeBindingKey: refreshedAuthority.activeBindingKey, + requiredTransportRef: reconfiguration?.required_transport_ref ?? null, + requiredConnectionMode: reconfiguration?.required_connection_mode ?? null, + requiresFreshScanBeforeSubmit: false, + }); + setCandidateUnavailableMessage(null); + } finally { + setSearchPresentation((current) => ( + searchPresentationSequence.current === searchSequence + && current?.sequence === searchSequence + ? { + ...current, + active: false, + completedDiscoveryGeneration: scanResult?.succeeded + ? scanResult.discoveryGeneration + : null, + } + : current + )); + retireRuntimeClickFence(actionFence); + } }; const submitConnect = async () => { - if (!canConnect || !deviceSummary) return; - const idempotencyKey = provisioningIntentKey(provisioningIntentRef.current); - provisioningIntentRef.current = idempotencyKey; - const networkCredentials = connectionMode === "quick-connect" - ? {} - : { ssid: ssid.trim(), password }; - const succeeded = await connect({ - device_id: deviceSummary.device_id, - ...networkCredentials, - connection_mode: connectionMode, - compatibility_attestation: profileSelectionForConnectionMode(connectionMode), - idempotency_key: idempotencyKey, + const actionEpoch = localScenarioActionEpoch.current; + if ( + !selectedTarget + || !explicitProvisioningDraft + || !canConnect + || unresolvedAppliedAttempt + ) return; + if (activeConnectIntent.current !== null) return; + const connectIntentToken = connectIntentSequence.current + 1; + connectIntentSequence.current = connectIntentToken; + activeConnectIntent.current = connectIntentToken; + const attemptedDeviceId = selectedTarget.deviceId; + const attemptedConnectionMode = connectionMode; + const attemptedSsid = connectionMode === "quick-connect" ? null : ssid.trim(); + const attemptedPassword = password; + const expectedDraftRuntimeId = explicitProvisioningDraft.snapshotRuntimeId; + const expectedDiscoveryGeneration = explicitProvisioningDraft.discoveryGeneration; + const idempotencyKey = provisioningIntentKey(null); + const failureForCurrentAttempt = ( + message: string, + freshStartAllowed: boolean, + ) => { + if (!localScenarioActionEpochIsCurrent( + actionEpoch, + localScenarioActionEpoch.current, + )) return; + setConnectionAttemptPresentation((current) => ( + current?.idempotencyKey === idempotencyKey + ? { + ...current, + localPhase: "failed", + failureMessage: message, + freshStartAllowed, + } + : current + )); + setCandidateUnavailableMessage(message); + }; + let actionFence: RuntimeActionFence | null = null; + + // Spend the editable credentials at the click boundary. The request owns + // only these stack-local values; no password is retained in React state or + // in the presentation latch after dispatch begins. + setConnectionAttemptPresentation({ + snapshotRuntimeId: expectedDraftRuntimeId, + connectionMode: attemptedConnectionMode, + idempotencyKey, + attemptId: null, + deviceId: attemptedDeviceId, + label: selectedDeviceSnapshot?.name?.trim() || selectedTarget.label, + rssi: finiteMetric(selectedDeviceSnapshot?.rssi), + ssid: attemptedSsid, + localPhase: "submitting", + failureMessage: null, + freshStartAllowed: false, }); - if (succeeded) { - provisioningIntentRef.current = null; - setPassword(""); - } else { - // Every later click is a new explicit operator intent, never an - // automatic replay of a consumed failed journal entry. If the prior - // write outcome is ambiguous, the backend reconciliation fence blocks - // this new intent before another device write for both modes. - provisioningIntentRef.current = null; + setExplicitProvisioningDraft(null); + setSsid(""); + setPassword(""); + setCandidateUnavailableMessage(null); + try { + const modeAuthority = await commitDesiredModeForExplicitAction(); + if (!localScenarioActionEpochIsCurrent( + actionEpoch, + localScenarioActionEpoch.current, + )) return; + if (!modeAuthority) { + failureForCurrentAttempt( + "Состояние подключения изменилось до отправки. Настройки не отправлены; начните новый явный поиск.", + true, + ); + return; + } + if (modeAuthority.snapshotRuntimeId !== expectedDraftRuntimeId) { + failureForCurrentAttempt( + "Среда подключения изменилась до отправки. Настройки не отправлены; выберите устройство заново.", + true, + ); + return; + } + if (modeAuthority.discoveryGeneration !== expectedDiscoveryGeneration) { + failureForCurrentAttempt( + "Результат Bluetooth-поиска устарел до отправки. Настройки не отправлены; выполните явный повторный поиск.", + true, + ); + return; + } + actionFence = activateRuntimeActionFence(modeAuthority); + if (!actionFence || !runtimeActionIsCurrent(actionFence)) { + failureForCurrentAttempt( + "Состояние подключения изменилось до отправки. Настройки не отправлены; повторите поиск Bluetooth.", + true, + ); + return; + } + + const networkCredentials = attemptedConnectionMode === "quick-connect" + ? {} + : { ssid: attemptedSsid ?? "", password: attemptedPassword }; + const result = await connect({ + device_id: attemptedDeviceId, + ...networkCredentials, + connection_mode: attemptedConnectionMode, + compatibility_attestation: profileSelectionForConnectionMode( + attemptedConnectionMode, + ), + idempotency_key: idempotencyKey, + expected_mode_revision: modeAuthority.desiredModeRevision, + expected_discovery_generation: modeAuthority.discoveryGeneration, + expected_reconfiguration_revision: modeAuthority.reconfigurationRevision, + ...(modeAuthority.reconfigurationIntentId + ? { + expected_reconfiguration_intent_id: + modeAuthority.reconfigurationIntentId, + } + : {}), + }); + if ( + !localScenarioActionEpochIsCurrent( + actionEpoch, + localScenarioActionEpoch.current, + ) + || !runtimeClickIsCurrent(actionFence) + ) return; + const observedOperation = operationByIdempotencyKey( + result.observedState, + "network.provision", + idempotencyKey, + ); + + if (result.networkIntentCompleted) { + const promotedFence = promoteFenceFromConnectResult( + actionFence, + result.observedState, + attemptedDeviceId, + attemptedConnectionMode, + ); + if (promotedFence) actionFence = promotedFence; + setSuccessfulLocalConnect({ + deviceId: attemptedDeviceId, + connectionMode: attemptedConnectionMode, + acceptedSessionKey: result.acceptedSessionKey, + }); + setReconfigurationDevicePresentation(null); + setConnectionAttemptPresentation((current) => ( + current?.idempotencyKey === idempotencyKey + ? { + ...current, + attemptId: observedOperation?.operation_id ?? current.attemptId, + localPhase: "settling", + } + : current + )); + return; + } + + const freshStartAllowed = result.intentDisposition === "release"; + const failureMessage = freshStartAllowed + && provisioningFailureRequiresFreshCandidate(result.failureReasonCode) + ? "Результат Bluetooth-поиска устарел до отправки. Настройки не отправлены; выполните явный повторный поиск." + : freshStartAllowed + ? "Подключение не началось. Настройки удалены; после устранения причины начните новый явный поиск." + : "Результат отправки неизвестен. Повторное применение заблокировано, чтобы не отправить BLE-команду дважды; дождитесь, пока система определит безопасное продолжение."; + setConnectionAttemptPresentation((current) => ( + current?.idempotencyKey === idempotencyKey + ? { + ...current, + attemptId: observedOperation?.operation_id ?? current.attemptId, + localPhase: "failed", + failureMessage, + freshStartAllowed, + } + : current + )); + setCandidateUnavailableMessage(failureMessage); + } catch { + failureForCurrentAttempt( + "Подключение не подтверждено. Настройки удалены; обновите состояние перед новым явным действием.", + false, + ); + } finally { + if (actionFence && runtimeClickIsCurrent(actionFence)) { + retireRuntimeClickFence(actionFence); + } + if (activeConnectIntent.current === connectIntentToken) { + activeConnectIntent.current = null; + } } }; - const submitExistingBridgeAdoption = async () => { - if (!canAdoptExistingBridge || !deviceSummary) return; - const succeeded = await verifyConnection({ - device_id: deviceSummary.device_id, - compatibility_attestation: profileSelectionForConnectionMode("bridge"), - }); - if (succeeded) { - provisioningIntentRef.current = null; + const verifyAppliedNetwork = async () => { + if ( + !unresolvedAppliedAttempt + || appliedControlSettlementPending + || isBusy + ) return; + const request = appliedNetworkRecoveryTarget + ? observationRequest( + appliedNetworkRecoveryTarget, + reconfigurationRevision, + reconfigurationIntentId, + ) + : null; + if (!request) { + setCandidateUnavailableMessage( + "Попытка настройки завершена, но проверка состояния без изменений сейчас недоступна. Новая BLE-запись заблокирована; обновите состояние.", + ); + return; + } + const actionFence = beginRuntimeActionFence( + unresolvedAppliedAttempt.connection_mode, + ); + if (!actionFence || !runtimeActionIsCurrent(actionFence)) return; + setCandidateUnavailableMessage(null); + try { + const result = await verifyConnection(request); + if (!runtimeClickIsCurrent(actionFence)) return; + if (!result.succeeded) { + setCandidateUnavailableMessage( + "Управляющее подключение не подтверждено. BLE-запись не повторялась; устраните причину и повторите проверку состояния без изменений.", + ); + } + } finally { + retireRuntimeClickFence(actionFence); } }; - return ( - -
-
ПОДКЛЮЧЕНИЕ · ШАГИ 01–03

Подключите устройство к сети

- {phaseLabel} -
-
- - Выберите направление связи. Каждый путь выполняет не более одного сетевого изменения и не повторяет его автоматически. - - void changeDesiredConnectionMode(value)} + disabled={modeResetInFlight} + variant="split" + /> + {modeResetInFlight ? ( +

+ Завершаем прежнюю локальную границу. Новый поиск не начнётся автоматически.

+ ) : null} + {activeScenarioReset ? ( +

+ {state?.connection_scenario_reset + ?.previous_device_may_continue_scanning + ? "Локальный сеанс закрыт; прежний K1 мог продолжить сканирование." + : "Прежний локальный сеанс закрыт. Поиск нового K1 запускается отдельно."} +

+ ) : null} +
+ +
+ 0 + ? `Результатов: ${actionableDevices.length}` + : "Совпадений нет" + : "Ожидает" + } + tone={ + searchActive + ? "accent" + : physicalRecoveryRequired + && verificationActionPending + ? "accent" + : connectionEstablished + ? "success" + : networkRecoveryRequired + || physicalRecoveryRequired + || connectionRecoveryRequired + || explicitProvisioningDraftStale + ? "warning" + : "neutral" + } + > + {searchActive ? ( +
+ + + Поиск Bluetooth · {scanSecondsRemaining ?? 6} с + +
+ ) : networkRecoveryRequired && !connectionAttemptSettling ? ( +
+ + {appliedControlSettlementPending + ? "Команда настройки завершена" + : "Новый выбор временно заблокирован"} + + {networkRecoveryModeLabel} + + {appliedControlSettlementPending + ? "Сервис подтверждает управляющее подключение без повторения BLE-команды" + : appliedNetworkRecoveryTarget?.deviceId + ?? "Сервис не определил точный прежний K1 для безопасной проверки"} + +
+ ) : physicalRecoveryRequired && verificationActionPending ? ( +
+ + Проверяем прежний K1 без повторения START, STOP или настроек сети… +
+ ) : physicalRecoveryRequired ? ( +
+
+ Прежнее подключение не подтверждено + {physicalRecoveryModeLabel} + + {physicalRecoveryBinding?.deviceId + ?? "Прежний K1"} + +
+

+ Переподключитесь к прежнему K1 или начните чистое подключение + нового устройства. +

+ {physicalReadOnlyVerificationAvailable + && !physicalRecoveryVerificationFailed ? ( + + ) : null} + + {candidateUnavailableMessage ? ( +

+ {candidateUnavailableMessage} +

+ ) : null} +
+ ) : connectionRecoveryRequired ? ( +
+ {correlatedConnectionAttempt && error ? ( + void refreshCorrelatedConnectionFailure()} + onClear={clearError} + /> + ) : ( + <> +
+ + {unknownNetworkOutcomeAttempt + ? "Результат применения сети не подтверждён" + : "Сохранённое подключение требует проверки"} + + + {connectionRecoveryModeLabel + ?? connectionRecoveryAttempt?.connection_mode + ?? requestedModeLabel} + + {connectionRecoveryTarget?.deviceId ? ( + {connectionRecoveryTarget.deviceId} + ) : null} +
+

+ {unknownNetworkOutcomeAttempt + ? "Итог прежней попытки неизвестен. Автоматического повтора не было: сначала проверьте сохранённое подключение без изменений либо начните отдельный новый поиск." + : "Проверка читает состояние прежнего K1 и не отправляет настройки сети, START или STOP. Новый поиск — отдельное явное действие для выбора другого устройства."} +

+ {connectionRecoveryActions} + + )} + {candidateUnavailableMessage ? ( +

+ {candidateUnavailableMessage} +

+ ) : null} +
+ ) : connectionEstablished ? ( +
+
+ Подключение установлено + {selectedResultLabel || requestedModeLabel} + {selectedResultDeviceId ? ( + {selectedResultDeviceId} + ) : null} +
+ {( + selectedModeConnected + ? selectDeviceActionApplicable + : Boolean(selectedResult) + ) ? ( + + ) : null} +
+ ) : deviceSelected ? ( +
+
+ Устройство выбрано + {selectedResultLabel} + {selectedResultDeviceId ? ( + {selectedResultDeviceId} + ) : null} +
+ +
+ ) : ( + <> + {canScan ? ( + + ) : null} + + {candidateUnavailableMessage ? ( +

+ {candidateUnavailableMessage} +

+ ) : null} + + {searchRequested && !searchActive ? ( + actionableDevices.length > 0 ? ( +
+ {actionableDevices.map((device) => ( + selectCandidate(device)} + /> + ))} +
+ ) : ( +
+ Подходящих K1 не найдено. Повторите поиск. +
+ ) + ) : null} + + )} + + {showNetworkStep ? ( + + {connectionAttemptOwnsDraft ? ( + provisioningDraftContent + ) : verificationActionPending ? ( +
+ + Проверяем подключение без изменения сети… +
+ ) : networkRecoveryRequired && appliedControlSettlementPending ? ( +
+ +
+ Подтверждаем управляющее подключение +

+ BLE-команда не повторяется. Дождитесь результата одной + сервисной проверки состояния без изменений. +

+
+
+ ) : networkRecoveryRequired ? ( +
+
+ Попытка настройки завершена + + {networkRecoveryModeLabel}: управляющее подключение не подтверждено + + + {appliedNetworkRecoveryTarget?.deviceId + ?? "Сервис не определил точный прежний K1 для безопасной проверки"} + +
+

+ Старая попытка и её BLE-запись не повторяются. Обычные поиск, + выбор и «Применить» заблокированы; отдельные действия ниже либо + проверяют текущую сеть, либо начинают новый явный сценарий. +

+ + {canPrepareRecoverySelectDevice ? ( + + ) : null} + {canScanAppliedRecovery ? ( + + ) : null} + {candidateUnavailableMessage ? ( +

+ {candidateUnavailableMessage} +

+ ) : null} +
+ ) : networkActionPending ? ( +
+ + Настройка сети… +
+ ) : selectedModeConnected && !changeNetworkDialogue ? ( +
+
+ Подключение установлено + {requestedModeLabel} + {connectedEndpoint ? {connectedEndpoint} : null} +
+ {changeNetworkActionApplicable ? ( + + ) : null} +
+ ) : explicitProvisioningDraftStale ? ( +
+ Результат устарел +

+ Результат Bluetooth-поиска изменился до отправки. Данные сети + сохранены только локально; команда устройству не отправлялась. +

+ +
+ ) : ( + provisioningDraftContent + )} + + ) : null}
); diff --git a/plugins/xgrids-k1/frontend/src/components/K1SpatialControls.tsx b/plugins/xgrids-k1/frontend/src/components/K1SpatialControls.tsx index cbb9d93..1816580 100644 --- a/plugins/xgrids-k1/frontend/src/components/K1SpatialControls.tsx +++ b/plugins/xgrids-k1/frontend/src/components/K1SpatialControls.tsx @@ -1,13 +1,22 @@ -import { Button } from "@nodedc/ui-react"; +import { ActivityIndicator, Button } from "@nodedc/ui-react"; import type { DevicePluginConnectionProps } from "@mission-core/plugin-sdk"; import type { AcquisitionState, - OperatorPresenceConfirmation, XgridsAcquisition, + XgridsK1State, } from "../api"; import { + activeStreamForceFinishAuthority, + activeStreamRecoveryPresentation, +} from "../activeStreamRecovery"; +import { + canIssueCanonicalStop, + connectionPolicyAllows, + hasAuthoritativeData, + hasControlAuthority, isSoftwareCommandedAcquisition, + requiresCanonicalStopAfterTerminalLocalFailure, shouldRenderSpatialControls, } from "../lifecycle"; import { @@ -15,7 +24,15 @@ import { formatNumber, spatialActionFailure, } from "../presentation"; -import { useXgridsK1Controller } from "../runtimeContext"; +import { + activeStopTarget, + operatorActionPhysicalAcceptance, +} from "../physicalCommandConfirmation"; +import { + useXgridsK1Controller, + type XgridsK1Controller, +} from "../runtimeContext"; +import { ActiveStreamRecoverySurface } from "./ActiveStreamRecoverySurface"; interface PhasePresentation { label: string; @@ -23,15 +40,31 @@ interface PhasePresentation { busy: boolean; } -const PHYSICAL_ACCEPTANCE = { - operator_present: true, - owner_controlled_device: true, - lixelgo_closed: true, - battery_storage_confirmed: true, - expected_physical_state_confirmed: true, -} satisfies OperatorPresenceConfirmation; +export interface K1SpatialAuthorityState { + controlAuthoritative: boolean; + dataAuthoritative: boolean; + softwareCommanded: boolean; + authorityFailure: string | null; +} -function phasePresentation( +export function k1SpatialAuthorityState( + state: XgridsK1State | null | undefined, +): K1SpatialAuthorityState { + const controlAuthoritative = hasControlAuthority(state); + const dataAuthoritative = hasAuthoritativeData(state); + return { + controlAuthoritative, + dataAuthoritative, + softwareCommanded: controlAuthoritative && isSoftwareCommandedAcquisition(state), + authorityFailure: state?.acquisition?.state === "acquiring" && !dataAuthoritative + ? controlAuthoritative + ? "Поток данных K1 не подтверждён supervisor-ом. Телеметрия скрыта до восстановления data authority." + : "Управляющая сессия K1 потеряна. Локальное завершение доступно, но команды устройству запрещены." + : null, + }; +} + +export function k1SpatialPhasePresentation( acquisition: XgridsAcquisition, softwareCommanded: boolean, ): PhasePresentation { @@ -49,16 +82,20 @@ function phasePresentation( busy: false, }, awaiting_external_start: { - label: "Ожидание запуска на устройстве", - detail: "Запустите сканирование физической кнопкой K1.", + label: softwareCommanded + ? "K1 калибруется и готовит облако точек" + : "Ожидание запуска на устройстве", + detail: softwareCommanded + ? "Первые данные могут появиться через десятки секунд. Не перемещайте устройство." + : "Запустите сканирование физической кнопкой K1.", busy: true, }, starting: { label: softwareCommanded - ? "Калибровка оборудования" + ? "K1 калибруется и готовит облако точек" : "Подготовка локального приёмника", detail: softwareCommanded - ? "Статическая инициализация после запуска — не перемещайте устройство." + ? "Первые данные могут появиться через десятки секунд. Не перемещайте устройство." : "Mission Core запускает запись до физического старта K1.", busy: true, }, @@ -106,43 +143,146 @@ function formatDuration(seconds: number): string { : `${String(minutes).padStart(2, "0")}:${String(remainder).padStart(2, "0")}`; } -export function K1SpatialControls(_props: DevicePluginConnectionProps) { - const controller = useXgridsK1Controller(); - const { state, pendingAction, stop } = controller; +export function runSpatialActiveStreamForceFinish( + controller: Pick< + XgridsK1Controller, + "state" | "forceFinishActiveStreamLocally" + >, +): Promise { + if (!activeStreamForceFinishAuthority(controller.state)) { + return Promise.resolve(false); + } + return controller.forceFinishActiveStreamLocally(); +} + +export function K1SpatialControlsView({ + controller, +}: { + controller: XgridsK1Controller; +}) { + const { + state, + pendingAction, + physicalStopIntentSpent, + physicalStopInFlight, + stop, + stopLocalReceiver, + forceFinishActiveStreamLocally, + } = controller; const acquisition = state?.acquisition; + const activeRecoveryPresentation = activeStreamRecoveryPresentation(state); + const localForceFinishPending = pendingAction === "force-finish"; + + if (activeRecoveryPresentation || localForceFinishPending) { + return ( + { + void runSpatialActiveStreamForceFinish({ + state, + forceFinishActiveStreamLocally, + }); + }} + /> + ); + } + + const physicalStopTarget = activeStopTarget(state); + const localReceiverStopAllowed = connectionPolicyAllows(state, "stop-local-receiver"); + const physicalStopExecutable = Boolean( + physicalStopTarget + && canIssueCanonicalStop(state, physicalStopIntentSpent), + ); + const physicalStopPresented = physicalStopInFlight || physicalStopExecutable; + const stopping = ["awaiting_external_stop", "stopping", "finalizing"].includes( + acquisition?.state ?? "", + ); const cleanupPending = acquisition?.cleanup_pending === true; + if (!acquisition || !shouldRenderSpatialControls(state)) { return null; } + const { + controlAuthoritative, + dataAuthoritative, + authorityFailure, + } = k1SpatialAuthorityState(state); const softwareCommanded = isSoftwareCommandedAcquisition(state); - const phase = phasePresentation(acquisition, softwareCommanded); - const telemetry = deviceTelemetry(state.metrics); - const stopping = ["awaiting_external_stop", "stopping", "finalizing"].includes( - acquisition.state, - ); - const stopDisabled = pendingAction !== null || stopping; + const dataPlaneState = state?.connection_supervisor?.observed.data_plane.state; + const terminalPhysicalStopRequired = requiresCanonicalStopAfterTerminalLocalFailure(state); + const phase = physicalStopInFlight + ? { + label: "Команда остановки устройства отправлена", + detail: "Ждём подтверждённое состояние K1; повторная команда не отправляется.", + busy: true, + } + : terminalPhysicalStopRequired && physicalStopExecutable + ? { + label: "Локальный приём остановился · K1 продолжает работу", + detail: "Остановите устройство явной командой; новый START заблокирован.", + busy: false, + } + : terminalPhysicalStopRequired && localReceiverStopAllowed + ? { + label: "Состояние K1 требует безопасного восстановления", + detail: "Команда устройству не отправляется. Доступно разрешённое сервером локальное завершение или read-only восстановление.", + busy: false, + } + : terminalPhysicalStopRequired + ? { + label: "Управляющие действия заблокированы", + detail: "Дождитесь подтверждённого состояния или выполните read-only восстановление.", + busy: false, + } + : acquisition.state === "acquiring" + && !dataAuthoritative + ? { + label: !controlAuthoritative + ? "Управляющая сессия K1 потеряна" + : dataPlaneState === "lost" + ? "Связь с потоком K1 потеряна" + : dataPlaneState === "stalled" + ? "Поток K1 нестабилен" + : "Ожидаем подтверждённый поток K1", + detail: !controlAuthoritative + ? "Состояние acquisition сохранено как последнее известное; команды устройству не отправляются." + : "Управляющая сессия подтверждена, но живые данные пока не получили авторитетный статус.", + busy: false, + } + : k1SpatialPhasePresentation(acquisition, softwareCommanded); + const telemetry = deviceTelemetry(dataAuthoritative ? state.metrics : undefined); + const stopDisabled = pendingAction !== null + || stopping; const controlFailure = state.application_control_session?.state === "failed" ? state.application_control_session.failure?.message || "Канонический диалог остановлен; автоматический повтор запрещён." : null; - const actionFailure = spatialActionFailure( + const runtimeActionFailure = spatialActionFailure( controller.error ?? controlFailure ?? (cleanupPending - ? "Локальный поток или архив ещё не завершён. Повторите остановку." + ? physicalStopInFlight + ? "Локальный поток или архив ещё не завершён. Команда устройству уже отправлена; дождитесь подтверждённого состояния." + : localReceiverStopAllowed + ? "Локальный поток или архив ещё не завершён. Завершите только разрешённый сервером локальный приём." + : "Локальный поток или архив ещё не завершён. Дождитесь подтверждённого состояния или выполните read-only восстановление." : null), ); - + const actionFailure = runtimeActionFailure ?? spatialActionFailure(authorityFailure); return (
- {phase.busy ?
) : null} - + {physicalStopPresented ? ( + + ) : null} + {!physicalStopPresented && localReceiverStopAllowed && !stopping ? ( + + ) : null}
); } + +export function K1SpatialControls(_props: DevicePluginConnectionProps) { + const controller = useXgridsK1Controller(); + return ; +} diff --git a/plugins/xgrids-k1/frontend/src/configuration.ts b/plugins/xgrids-k1/frontend/src/configuration.ts index 13586cb..6cbb679 100644 --- a/plugins/xgrids-k1/frontend/src/configuration.ts +++ b/plugins/xgrids-k1/frontend/src/configuration.ts @@ -12,17 +12,17 @@ export const connectionModeOptions: Array> = [ { value: "bridge", label: "Общая сеть · Bridge", - description: "Mission Core передаёт K1 реквизиты существующей общей сети.", + description: "Передача реквизитов существующей общей сети.", }, { value: "quick-connect", - label: "Точка доступа K1 · Quick Connect", - description: "Лабораторный режим: Mission Core включает AP K1 и подключает только заранее подготовленный хост. Для обычной работы используйте Bridge.", + label: "Локальная сеть · Quick Connect", + description: "Связь через отдельную локальную сеть. Для обычной работы используйте Bridge.", }, { value: "direct-connect", label: "Хотспот контроллера · Direct Connect", - description: "Mission Core передаёт K1 реквизиты хотспота управляющего устройства.", + description: "Передача реквизитов хотспота контроллера.", }, ]; diff --git a/plugins/xgrids-k1/frontend/src/controlSessionCas.ts b/plugins/xgrids-k1/frontend/src/controlSessionCas.ts new file mode 100644 index 0000000..a6a4cbd --- /dev/null +++ b/plugins/xgrids-k1/frontend/src/controlSessionCas.ts @@ -0,0 +1,72 @@ +import { ApiError, type XgridsK1State } from "./api"; + +export interface ExactApplicationControlCas { + expected_session_generation: number; + expected_state_revision: number; +} + +export interface ExactAcquisitionControlCas { + expected_control_session_generation: number; + expected_control_state_revision: number; +} + +interface ControlSessionVersion { + sessionGeneration: number; + stateRevision: number; +} + +function exactControlSessionVersion( + state: XgridsK1State | null | undefined, + actionLabel: string, +): ControlSessionVersion { + const session = state?.application_control_session; + const sessionGeneration = session?.session_generation; + const stateRevision = session?.state_revision; + if ( + !Number.isSafeInteger(sessionGeneration) + || (sessionGeneration ?? -1) < 0 + || !Number.isSafeInteger(stateRevision) + || (stateRevision ?? -1) < 0 + ) { + throw new ApiError( + `Команда ${actionLabel} не отправлена: последнее принятое состояние не содержит целые session_generation и state_revision управляющей сессии. Обновите состояние K1 и повторите отдельным действием.`, + ); + } + return { + sessionGeneration: sessionGeneration as number, + stateRevision: stateRevision as number, + }; +} + +export function exactApplicationControlCas( + latestAcceptedState: XgridsK1State | null | undefined, + actionLabel: string, +): ExactApplicationControlCas { + const version = exactControlSessionVersion(latestAcceptedState, actionLabel); + return { + expected_session_generation: version.sessionGeneration, + expected_state_revision: version.stateRevision, + }; +} + +export function exactAcquisitionControlCas( + latestAcceptedState: XgridsK1State | null | undefined, + actionLabel: string, +): ExactAcquisitionControlCas { + const version = exactControlSessionVersion(latestAcceptedState, actionLabel); + return { + expected_control_session_generation: version.sessionGeneration, + expected_control_state_revision: version.stateRevision, + }; +} + +export function acquisitionMutationUsesControlSession( + latestAcceptedState: XgridsK1State | null | undefined, +): boolean { + const session = latestAcceptedState?.application_control_session; + return Boolean( + session + && session.mode === "interactive-canonical" + && !["idle", "closed", "completed"].includes(session.state), + ); +} diff --git a/plugins/xgrids-k1/frontend/src/hostDiagnosticPresentation.ts b/plugins/xgrids-k1/frontend/src/hostDiagnosticPresentation.ts new file mode 100644 index 0000000..913e41c --- /dev/null +++ b/plugins/xgrids-k1/frontend/src/hostDiagnosticPresentation.ts @@ -0,0 +1,132 @@ +import { + isXgridsHostFailureDiagnostic, + type XgridsHostDiagnosticCode, + type XgridsHostDiagnosticAction, + type XgridsHostDiagnosticDomain, + type XgridsHostDiagnosticImpact, + type XgridsHostFailureDiagnostic, + type XgridsOperation, +} from "./api"; + +const CODE_LABELS: Record = { + "host.bluetooth.permission-denied": + "macOS не разрешила Mission Core использовать Bluetooth.", + "host.bluetooth.adapter-powered-off": + "Bluetooth на этом Mac выключен.", + "host.bluetooth.adapter-unavailable": + "Системный Bluetooth-адаптер сейчас недоступен.", + "host.bluetooth.runtime-unavailable": + "Локальный Bluetooth runtime не готов к новой операции.", + "host.bluetooth.operation-timeout": + "Bluetooth-операция не завершилась за ограниченное время.", + "host.wifi.permission-denied": + "macOS не разрешила Mission Core читать состояние Wi‑Fi.", + "host.wifi.adapter-powered-off": + "Wi‑Fi на этом Mac выключен.", + "host.wifi.interface-unavailable": + "Системный Wi‑Fi-интерфейс сейчас недоступен.", + "host.wifi.ssid-unavailable": + "macOS не сообщила имя текущей Wi‑Fi-сети.", + "host.wifi.operation-timeout": + "Операция с Wi‑Fi не завершилась за ограниченное время.", + "host.wifi.association-failed": + "Mac не подтвердил подключение к ожидаемой Wi‑Fi-сети.", + "host.keychain.interaction-required": + "Связка ключей требует явного подтверждения оператора.", + "host.keychain.permission-denied": + "macOS запретила чтение профиля подключения.", + "host.keychain.unavailable": + "Профиль подключения сейчас недоступен в связке ключей.", + "host.route.unavailable": + "Прямой локальный маршрут к адресу подключения не найден.", + "host.tcp.connection-refused": + "Управляющий TCP endpoint отклонил соединение.", + "host.tcp.connection-timeout": + "Управляющий TCP endpoint не ответил за ограниченное время.", + "host.tcp.endpoint-unavailable": + "Управляющий TCP endpoint недоступен из текущей сети.", + "host.mqtt.connection-timeout": + "Управляющий MQTT-канал не открылся за ограниченное время.", + "host.mqtt.connection-refused": + "Управляющий MQTT-канал отклонил соединение.", + "host.mqtt.transport-unavailable": + "Транспорт управляющего MQTT-канала недоступен.", + "host.filesystem.permission-denied": + "Mission Core не может записать обязательные данные операции в локальное хранилище.", + "host.filesystem.ledger-unavailable": + "Журнал безопасного результата операции недоступен или не подтверждён.", +}; + +const DOMAIN_LABELS: Record = { + corebluetooth: "Bluetooth macOS", + corewlan: "Wi‑Fi macOS", + keychain: "Связка ключей macOS", + route: "Локальный сетевой маршрут", + tcp: "Управляющий TCP endpoint", + mqtt: "Управляющий канал MQTT", + filesystem: "Локальное хранилище Mission Core", +}; + +const IMPACT_LABELS: Record = { + discovery: "Поиск Bluetooth сейчас недоступен.", + "host-network": "Сетевой путь между этим компьютером и локальным контуром недоступен.", + control: "Управляющая связь не установлена; команды не повторяются автоматически.", + "durable-safety": "Надёжная фиксация результата операции недоступна; новая команда заблокирована.", +}; + +const ACTION_LABELS: Record = { + "grant-bluetooth-permission": + "Разрешите Mission Core доступ к Bluetooth в системных настройках macOS, затем повторите действие вручную.", + "power-on-bluetooth": + "Включите Bluetooth на этом Mac и запустите новый поиск вручную.", + "restore-bluetooth-adapter": + "Восстановите доступность Bluetooth-адаптера macOS и перезапустите локальный сервис перед новой попыткой.", + "grant-wifi-permission": + "Разрешите Mission Core доступ к данным Wi‑Fi в системных настройках macOS, затем повторите действие вручную.", + "power-on-wifi": + "Включите Wi‑Fi на этом Mac и заново выберите требуемый способ подключения.", + "restore-wifi-interface": + "Восстановите системный Wi‑Fi-интерфейс macOS перед новой попыткой подключения.", + "unlock-or-authorize-keychain": + "Разблокируйте связку ключей macOS и подтвердите доступ Mission Core к профилю подключения.", + "review-keychain-access": + "Разрешите Mission Core чтение профиля подключения в связке ключей macOS.", + "join-expected-network": + "Установите связь этого Mac с ожидаемой локальной сетью и повторите действие.", + "inspect-host-route": + "Восстановите прямой локальный маршрут к адресу подключения.", + "verify-broker-endpoint": + "Восстановите доступность управляющего endpoint из текущей сети; команда автоматически не повторяется.", + "inspect-local-storage": + "Освободите место и восстановите доступ к локальному хранилищу Mission Core до следующей операции.", + "restart-local-service": + "Перезапустите канонический локальный сервис Mission Core и после загрузки обновите состояние.", + "explicit-retry": + "После устранения причины повторите действие отдельным нажатием; автоматического повтора нет.", +}; + +export interface HostFailureDiagnosticPresentation { + codeLabel: string; + domainLabel: string; + impactLabel: string; + operatorActionLabel: string; +} + +export function hostFailureDiagnosticPresentation( + value: unknown, +): HostFailureDiagnosticPresentation | null { + if (!isXgridsHostFailureDiagnostic(value)) return null; + return { + codeLabel: CODE_LABELS[value.code], + domainLabel: DOMAIN_LABELS[value.domain], + impactLabel: IMPACT_LABELS[value.impact], + operatorActionLabel: ACTION_LABELS[value.operator_action], + }; +} + +export function operationHostFailureDiagnostic( + operation: XgridsOperation | null | undefined, +): XgridsHostFailureDiagnostic | null { + const diagnostic = operation?.error?.host_diagnostic; + return isXgridsHostFailureDiagnostic(diagnostic) ? diagnostic : null; +} diff --git a/plugins/xgrids-k1/frontend/src/lifecycle.ts b/plugins/xgrids-k1/frontend/src/lifecycle.ts index 5688ded..4ea0b4c 100644 --- a/plugins/xgrids-k1/frontend/src/lifecycle.ts +++ b/plugins/xgrids-k1/frontend/src/lifecycle.ts @@ -5,8 +5,14 @@ import type { BleDevice, XgridsApplicationControlPhase, XgridsAcquisition, + XgridsConnectionAttempt, XgridsK1State, XgridsOperation, + ReopenRetiredPhysicalReconciliationRequest, + XgridsConnectionMode, + XgridsConnectionReconfiguration, + XgridsConnectionPolicyAction, + XgridsConnectionPolicyDecision, } from "./api"; const TERMINAL_ACQUISITION_STATES = new Set([ @@ -24,6 +30,9 @@ const FAILED_OPERATION_STATUSES = new Set([ ]); export type LiveStartPlan = "prepare" | "resume-prepared" | "already-running" | "blocked"; +export type LocalReceiverStopPlan = + | { kind: "acquisition"; acquisitionId: string } + | { kind: "compatibility" }; export type ControlSessionEntryPlan = | "open" | "continue" @@ -50,6 +59,77 @@ export function isTerminalAcquisitionState( return state ? TERMINAL_ACQUISITION_STATES.has(state) : false; } +/** + * Select the exact local runtime cleanup target without inheriting a retained + * terminal acquisition into a replay session. Terminal acquisition state is + * actionable only while its backend-owned local cleanup remains pending. + */ +export function localReceiverStopPlan( + state: XgridsK1State | null | undefined, +): LocalReceiverStopPlan { + const acquisition = state?.acquisition; + const acquisitionId = acquisition?.acquisition_id?.trim(); + const liveOrIdleRuntime = state?.source_mode === "live" || state?.source_mode === "idle"; + if ( + acquisition + && acquisitionId + && liveOrIdleRuntime + && ( + !isTerminalAcquisitionState(acquisition.state) + || acquisition.cleanup_pending === true + ) + ) { + return { kind: "acquisition", acquisitionId }; + } + return { kind: "compatibility" }; +} + +export function isProvenLocalReceiverInactive( + state: XgridsK1State | null | undefined, +): state is XgridsK1State { + const acquisition = state?.acquisition; + const acquisitionReleased = Boolean( + acquisition + && isTerminalAcquisitionState(acquisition.state) + && acquisition.cleanup_pending === false, + ); + return Boolean( + state?.source_mode === "idle" + && (!acquisition || acquisitionReleased), + ); +} + +export function isReleasedTerminalAcquisitionFailure( + state: XgridsK1State | null | undefined, +): boolean { + const controlState = state?.application_control_session?.state; + const acquisition = state?.acquisition; + return Boolean( + state?.source_mode === "idle" + && acquisition + && ["failed", "interrupted"].includes(acquisition.state) + && acquisition.cleanup_pending === false + && [ + "idle", + "connection-ready", + "active-recovery-requested", + "scanning", + "completed", + "closed", + ].includes(controlState ?? ""), + ); +} + +export function shouldSurfaceRuntimeActionError( + action: string, + state: XgridsK1State | null | undefined, +): boolean { + return !( + ["control", "live", "stop", "abort"].includes(action) + && isReleasedTerminalAcquisitionFailure(state) + ); +} + export function shouldRenderSpatialControls( state: XgridsK1State | null | undefined, ): boolean { @@ -57,7 +137,194 @@ export function shouldRenderSpatialControls( if (!acquisition || state?.source_mode === "replay") return false; return ( !isTerminalAcquisitionState(acquisition.state) || - acquisition.cleanup_pending === true + acquisition.cleanup_pending === true || + requiresCanonicalStopAfterTerminalLocalFailure(state) + ); +} + +export function requiresCanonicalStopAfterTerminalLocalFailure( + state: XgridsK1State | null | undefined, +): boolean { + const acquisition = state?.acquisition; + const control = state?.application_control_session; + return Boolean( + acquisition + && isTerminalAcquisitionState(acquisition.state) + && isSoftwareCommandedAcquisition(state) + && control?.state === "scanning" + && control.can_stop === true + ); +} + +export interface PhysicalStopIntentCheckpoint { + snapshotRuntimeId: string; + acquisitionId: string; + deviceId: string; + deviceSessionId: string; + controlSessionGeneration: number; + controlStateRevision: number; +} + +function positiveInteger(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 1; +} + +/** + * Capture the exact backend authority consumed by one physical STOP intent. + * Snapshot identity is retained when available, while the control-session CAS + * is mandatory: a presentation-only error dismissal must never manufacture a + * fresh command intent against the same control checkpoint. + */ +export function physicalStopIntentCheckpoint( + state: XgridsK1State | null | undefined, +): PhysicalStopIntentCheckpoint | null { + const control = state?.application_control_session; + const acquisition = state?.acquisition; + const acquisitionId = acquisition?.acquisition_id?.trim(); + const deviceId = acquisition?.device_id?.trim(); + const deviceSessionId = acquisition?.device_session_id?.trim(); + const runtimeId = state?.snapshot_runtime_id?.trim(); + if ( + !control + || !acquisitionId + || !deviceId + || !deviceSessionId + || !runtimeId + || !positiveInteger(control.session_generation) + || !positiveInteger(control.state_revision) + || control.state !== "scanning" + || control.can_stop !== true + || !connectionPolicyAllows(state, "stop-acquisition") + ) return null; + return { + snapshotRuntimeId: runtimeId, + acquisitionId, + deviceId, + deviceSessionId, + controlSessionGeneration: control.session_generation, + controlStateRevision: control.state_revision, + }; +} + +/** + * A spent physical STOP may be released only by an already-accepted exact + * STOP-authoritative runtime replacement, a distinct acquisition target, or + * an exact control-session CAS transition. A same-runtime polling snapshot + * whose target and control CAS stayed fixed is not new physical-command + * authority. + */ +export function authoritativeStateSupersedesPhysicalStopIntent( + spent: PhysicalStopIntentCheckpoint | null | undefined, + state: XgridsK1State | null | undefined, +): boolean { + if (!spent) return false; + const current = physicalStopIntentCheckpoint(state); + if (!current) return false; + const sameRuntime = current.snapshotRuntimeId === spent.snapshotRuntimeId; + if (!sameRuntime) { + // Snapshot ordering is resolved before this helper is called. A different + // accepted runtime with a complete exact STOP gate is fresh authority even + // when its process-local CAS counters restarted. + return true; + } + const sameTarget = + current.acquisitionId === spent.acquisitionId + && current.deviceId === spent.deviceId + && current.deviceSessionId === spent.deviceSessionId; + if (!sameTarget) { + // A newly accepted acquisition/device/session tuple is a distinct command + // target. The caller has already admitted this state monotonically. + return true; + } + const controlCasAdvanced = + current.controlSessionGeneration > spent.controlSessionGeneration + || ( + current.controlSessionGeneration === spent.controlSessionGeneration + && current.controlStateRevision > spent.controlStateRevision + ); + if (!controlCasAdvanced) return false; + // Within one runtime the exact control CAS must advance; observation-only + // polls remain locked regardless of their snapshot observation revision. + return true; +} + +/** + * Admit one physical STOP button only from the exact current control proof. + * A failed action spends that browser intent independently of its dismissible + * presentation error: the operator may still finish the host receiver + * locally, but the UI must not create a fresh physical STOP mutation from the + * same accepted snapshot/control CAS. + */ +export function canIssueCanonicalStop( + state: XgridsK1State | null | undefined, + physicalStopIntentSpent: boolean | null | undefined, +): boolean { + const control = state?.application_control_session; + return Boolean( + !physicalStopIntentSpent + && physicalStopIntentCheckpoint(state) + && control?.state === "scanning" + && control.can_stop === true + && connectionPolicyAllows(state, "stop-acquisition"), + ); +} + +/** + * A successful read-only Verify may truthfully end in SCANNING rather than + * connection-ready. That state grants exactly one explicit STOP, never START + * or provisioning authority. + */ +export function isRecoveredPhysicalScanning( + state: XgridsK1State | null | undefined, + expectedMode?: XgridsConnectionMode, +): boolean { + const control = state?.application_control_session; + const physical = control?.physical_command ?? state?.physical_command; + const mode = state?.active_connection_mode ?? state?.connection_mode; + return Boolean( + control?.state === "scanning" + && control.can_stop === true + && physical?.requires_reconciliation !== true + && physical?.resolved_active_recovery_required === true + && physical.observed_session_state === "scanning" + && (!expectedMode || mode === expectedMode) + && (!expectedMode || currentAppliedConnectionTopology(state, expectedMode)?.status === "active") + ); +} + +/** + * A STOP that was accepted before the host path disappeared is completed by + * the backend from its durable command ledger. The connection screen must + * wait for that cleanup instead of turning the condition into another device + * action (Scan, START, STOP, or Wi-Fi provisioning). + */ +export function isPhysicalStopRecoverySettling( + state: XgridsK1State | null | undefined, +): boolean { + const control = state?.application_control_session; + const physical = control?.physical_command ?? state?.physical_command; + const record = physical?.record; + const action = record?.action; + const resolution = record?.resolution; + const acquisition = state?.acquisition; + const acquisitionStillSettling = Boolean( + acquisition + && ( + !isTerminalAcquisitionState(acquisition.state) + || acquisition.cleanup_pending === true + ), + ); + const stopOperationStillSettling = Boolean( + state?.operations?.some((operation) => + operation.action === "acquisition.stop" + && ["accepted", "running", "operator_action_required"].includes(operation.status) + ), + ); + return Boolean( + action === "stop" + && resolution !== "stop-standby-observed" + && (physical?.requires_reconciliation === true || physical?.status === "unresolved") + && (acquisitionStillSettling || stopOperationStillSettling), ); } @@ -69,7 +336,11 @@ export function recoverableAcquisition( } export function isConfirmedLiveState(state: XgridsK1State | null | undefined): boolean { - return state?.source_mode === "live" && state.acquisition?.state === "acquiring"; + return Boolean( + state?.source_mode === "live" + && state.acquisition?.state === "acquiring" + && hasAuthoritativeData(state), + ); } export function isSourceRuntimeBusy(state: XgridsK1State | null | undefined): boolean { @@ -126,11 +397,17 @@ export function normalizeRuntimePhase( ): RuntimePhase { const phase = state?.phase; const acquisitionState = effectiveAcquisition(state)?.state; - if (acquisitionState === "failed" || acquisitionState === "interrupted") return "error"; + const releasedFailure = isReleasedTerminalAcquisitionFailure(state); + if ( + !releasedFailure + && (acquisitionState === "failed" || acquisitionState === "interrupted") + ) return "error"; if (acquisitionState === "awaiting_external_start" || acquisitionState === "starting") { return "starting"; } - if (acquisitionState === "acquiring") return "streaming"; + if (acquisitionState === "acquiring") { + return isConfirmedLiveState(state) ? "streaming" : "starting"; + } if ( acquisitionState === "awaiting_external_stop" || acquisitionState === "stopping" || @@ -138,16 +415,21 @@ export function normalizeRuntimePhase( ) { return "stopping"; } - if (acquisitionState === "prepared") return "connected"; - if (phase === "error") return "error"; + if (acquisitionState === "prepared") { + const topology = currentAppliedConnectionTopology(state); + return topology && topology.status !== "configured-offline" + ? "connected" + : "configuring"; + } + if (phase === "error") return releasedFailure ? "idle" : "error"; if (phase === "connected") { - const controlPhase = state?.application_control_session?.state; - return controlPhase && !["idle", "connecting", "closed", "completed", "failed"].includes(controlPhase) + const topology = currentAppliedConnectionTopology(state); + return topology && topology.status !== "configured-offline" ? "connected" : "configuring"; } if (phase === "starting_live") return "starting"; - if (phase === "live") return "streaming"; + if (phase === "live") return isConfirmedLiveState(state) ? "streaming" : "starting"; if (phase === "replay") return "replaying"; if (phase === "stopping") return "stopping"; if (["scanning", "device_selected", "provisioning", "connecting"].includes(phase ?? "")) { @@ -172,6 +454,12 @@ export function sourceStatusLabel(state: XgridsK1State | null | undefined): stri if (state.acquisition?.state === "failed" || state.phase === "error") { return "Ошибка локального приёмника"; } + if (state.connection_supervisor?.observed.data_plane.state === "lost") { + return "Поток данных потерян"; + } + if (state.connection_supervisor?.observed.data_plane.state === "stalled") { + return "Поток данных нестабилен"; + } return "Ожидание реальных данных"; } if (state?.acquisition?.state === "prepared") return "Приём подготовлен"; @@ -201,30 +489,1022 @@ export function operationNeedsReconciliation( return operation.error?.safe_to_retry !== true; } +export function operationAllowsFreshProvisioningIntent( + operation: XgridsOperation | null | undefined, +): boolean { + return Boolean( + operation + && FAILED_OPERATION_STATUSES.has(operation.status) + && operation.error?.safe_to_retry === true + && operation.error.side_effect_status === "none", + ); +} + +const FRESH_CANDIDATE_RETRY_REASON_CODES = new Set([ + "BleakDeviceNotFoundError", + "network-provision-candidate-not-fresh", + "network-provision-candidate-changed", + "network-provision-discovery-generation-conflict", +]); + +/** + * These failures are proven pre-write rejections caused only by an expired + * Bluetooth capture. One explicit network-submit click may refresh discovery + * once and then continue with the exact returned generation. A second failure + * is terminal for that click; this predicate never authorizes an unbounded + * retry or a repeat after an ambiguous/device-write outcome. + */ +export function provisioningFailureRequiresFreshCandidate( + reasonCode: string | null | undefined, +): boolean { + return typeof reasonCode === "string" + && FRESH_CANDIDATE_RETRY_REASON_CODES.has(reasonCode); +} + +export function readOnlyVerificationClearedReconciliation( + previousState: XgridsK1State | null | undefined, + nextState: XgridsK1State | null | undefined, + verifiedDeviceId: string | null | undefined, +): boolean { + const previousFence = previousState?.network_write_reconciliation; + const previousOperationId = previousFence?.operation_id?.trim(); + const nextLedger = nextState?.network_mutation_ledger; + if ( + !verifiedDeviceId + || !previousOperationId + || transportRefEquivalenceKey(previousFence?.transport_ref) + !== transportRefEquivalenceKey(verifiedDeviceId) + // An omitted field or a replacement unresolved fence is never proof that + // this durable operation was reconciled. + || nextState?.network_write_reconciliation !== null + || !nextLedger + || nextLedger.mutation_allowed !== true + ) { + return false; + } + const sameOperationResolved = Boolean( + nextLedger.status === "resolved" + && nextLedger.operation_id === previousOperationId + && nextLedger.stage === "resolved" + && nextLedger.resolution !== null, + ); + const operationExplicitlyAbsent = Boolean( + nextLedger.status === "empty" + && nextLedger.operation_id === null + && nextLedger.stage === null, + ); + return sameOperationResolved || operationExplicitlyAbsent; +} + export function provisioningCandidateById( devices: readonly BleDevice[], selectedDeviceId: string, ): BleDevice | null { - if (!selectedDeviceId) return null; - return devices.find((device) => device.device_id === selectedDeviceId) ?? null; + const selectedKey = transportRefEquivalenceKey(selectedDeviceId); + if (!selectedKey) return null; + return devices.find( + (device) => transportRefEquivalenceKey(device.device_id) === selectedKey, + ) ?? null; +} + +export function currentDeviceTransportRef( + state: XgridsK1State | null | undefined, +): string | null { + const recoveryRef = state?.current_device_recovery?.transport_ref?.trim(); + if (recoveryRef) return recoveryRef; + const selectedRef = state?.selected_device_id?.trim(); + return selectedRef || null; +} + +export interface LocallyInitiatedBleSessionTarget { + transportRef: string; + connectionMode: XgridsConnectionMode; + deviceSessionId: string; + key: string; +} + +export interface LocalBleSessionBindingConstraints { + /** Bind only the exact session accepted by the explicit connect response. */ + requiredSessionKey?: string | null; +} + +export function bleSessionTargetForTransport( + state: XgridsK1State | null | undefined, + transportRef: string | null | undefined, + selectedConnectionMode: XgridsConnectionMode, +): LocallyInitiatedBleSessionTarget | null { + const expectedTransportRef = transportRef?.trim(); + const backendTransportRef = state?.current_device_recovery?.transport_ref?.trim() + || state?.selected_device_id?.trim(); + const connectionMode = state?.current_device_recovery?.connection_mode + ?? state?.connection_mode; + const deviceSessionId = state?.device_session?.device_session_id?.trim(); + if ( + !expectedTransportRef + || !backendTransportRef + || expectedTransportRef !== backendTransportRef + || !deviceSessionId + || connectionMode !== selectedConnectionMode + ) { + return null; + } + return { + transportRef: backendTransportRef, + connectionMode, + deviceSessionId, + key: `${deviceSessionId}:${connectionMode}:${backendTransportRef}`, + }; +} + +export function acceptedBleSessionKeyAfterConnect( + state: XgridsK1State | null | undefined, + transportRef: string | null | undefined, + selectedConnectionMode: XgridsConnectionMode, + sessionKeyBeforeConnect: string | null | undefined, +): string | null { + const target = bleSessionTargetForTransport( + state, + transportRef, + selectedConnectionMode, + ); + if (!target || target.key === sessionKeyBeforeConnect) return null; + return target.key; +} + +/** + * Bind backend session state only to a connection initiated by this UI + * instance. A browser refresh has no local device id and therefore never + * turns an existing backend session into an implicit operator selection. + */ +export function locallyInitiatedBleSessionTarget( + state: XgridsK1State | null | undefined, + locallyInitiatedDeviceId: string | null | undefined, + selectedConnectionMode: XgridsConnectionMode, + constraints: LocalBleSessionBindingConstraints = {}, +): LocallyInitiatedBleSessionTarget | null { + const localRef = locallyInitiatedDeviceId?.trim(); + if (!localRef) return null; + const target = bleSessionTargetForTransport( + state, + localRef, + selectedConnectionMode, + ); + if (!target) return null; + if ( + constraints.requiredSessionKey + && constraints.requiredSessionKey !== target.key + ) return null; + return target; +} + +export interface RetainedBleRecoveryTarget { + transportRef: string; + connectionMode: NonNullable | null; + gattValidatedRecently: boolean; +} + +export function retainedBleRecoveryTarget( + state: XgridsK1State | null | undefined, +): RetainedBleRecoveryTarget | null { + const recovery = state?.current_device_recovery; + const transportRef = recovery?.transport_ref?.trim(); + if ( + !transportRef + || recovery?.handle_retained !== true + || recovery.advertised_now === true + ) { + return null; + } + return { + transportRef, + connectionMode: recovery.connection_mode ?? null, + gattValidatedRecently: recovery.gatt_validated_recently === true, + }; +} + +export function connectionPolicyDecision( + state: XgridsK1State | null | undefined, + action: XgridsConnectionPolicyAction, +): XgridsConnectionPolicyDecision | null { + const policy = state?.connection_policy; + if ( + policy?.schema_version !== "missioncore.xgrids-k1-connection-policy/v1" + || policy.facts.retained_context_is_presence !== false + ) { + return null; + } + return policy.actions[action] ?? null; +} + +export function connectionPolicyAllows( + state: XgridsK1State | null | undefined, + action: XgridsConnectionPolicyAction, +): boolean { + const decision = connectionPolicyDecision(state, action); + return Boolean( + decision?.allowed === true + && decision.automatic_retry === false + && state?.connection_policy?.allowed_actions.includes(action), + ); +} + +export interface ProvisioningNetworkStepDisclosure { + /** A concrete K1 has been admitted or selected by this operator flow. */ + deviceExplicitlySelectedOrAdmitted: boolean; + /** A network/connection intent, rather than discovery alone, has started. */ + networkIntentStarted: boolean; +} + +/** + * Keep Bluetooth discovery entirely inside step 02. + * + * Raw controller settlement is deliberately not an input: an old scan, + * reconfiguration, retirement, or physical-cleanup promise may still be + * unwinding after its authority has gone stale, but that does not mean the + * operator has reached the network step for the current device choice. + */ +export function shouldRevealProvisioningNetworkStep({ + deviceExplicitlySelectedOrAdmitted, + networkIntentStarted, +}: ProvisioningNetworkStepDisclosure): boolean { + return deviceExplicitlySelectedOrAdmitted || networkIntentStarted; +} + +export function activeConnectionReconfiguration( + state: XgridsK1State | null | undefined, +): XgridsConnectionReconfiguration | null { + const reconfiguration = state?.connection_reconfiguration; + if ( + !reconfiguration + || reconfiguration.schema_version + !== "missioncore.xgrids-k1-connection-reconfiguration/v1" + || reconfiguration.intent === null + || reconfiguration.status === "idle" + ) { + return null; + } + return reconfiguration; +} + +/** + * A network-change intent is pinned to the exact device and mode which were + * current when the operator opened it. Selecting another advertisement must + * not turn that intent into a different-device network write. + */ +export function reconfigurationAllowsFreshDevice( + reconfiguration: XgridsConnectionReconfiguration | null, + deviceId: string, + connectionMode: XgridsConnectionMode, +): boolean { + if (!reconfiguration || !deviceId.trim()) return false; + if ( + connectionMode !== "bridge" + || reconfiguration.required_connection_mode !== "bridge" + ) return false; + if (reconfiguration.intent === "select-device") return true; + return Boolean( + reconfiguration.intent === "change-network" + && transportRefEquivalenceKey(reconfiguration.required_transport_ref) + === transportRefEquivalenceKey(deviceId) + && reconfiguration.required_connection_mode === connectionMode, + ); +} + +export function readOnlyObservationShowsNetworkUnavailable( + state: XgridsK1State | null | undefined, +): boolean { + const verification = state?.connection_verification; + if (!verification) return false; + return [ + "device-network-applied-host-failed", + "host-route-mismatch", + "endpoint-unreachable", + "unreachable", + ].includes(verification.status) || Boolean( + verification.lease_state === "configured-unverified" + && verification.network_reachability === "unreachable", + ) || Boolean( + verification.status === "device-network-applied" + && verification.lease_state === "configured-unverified" + && verification.reason_code === "endpoint-target-unconfigured", + ); +} + +const READ_ONLY_NETWORK_UNAVAILABLE_REASON_CODES = new Set([ + "connection-verify-address-unavailable", + "connection-verify-connection-missing", + "connection-verify-route-mismatch", + "connection-verify-mqtt-unreachable", + "configured-endpoint-unavailable", + "endpoint-target-unconfigured", +]); + +/** + * Only failures which prove that the selected device cannot use the current + * Bridge topology may lead from read-only adoption to explicit credentials. + * BLE, identity, lifecycle and CAS failures deliberately stay outside this + * allowlist. + */ +export function readOnlyFailureShowsNetworkUnavailable( + reasonCode: string | null | undefined, +): boolean { + return typeof reasonCode === "string" + && READ_ONLY_NETWORK_UNAVAILABLE_REASON_CODES.has(reasonCode); +} + +export function requiresReadOnlyPhysicalRecovery( + state: XgridsK1State | null | undefined, +): boolean { + const control = state?.application_control_session; + const physical = control?.physical_command ?? state?.physical_command; + return Boolean( + // The coordinator folds unresolved commands and resolved SCAN_OVER into + // requires_reconciliation, while resolved active and explicitly reopened + // rows remain read-only recovery through the separate active flag. + physical?.requires_reconciliation === true + || physical?.resolved_active_recovery_required === true, + ); +} + +export interface ReadOnlyPhysicalRecoveryBinding { + deviceId: string; + connectionMode: XgridsConnectionMode; +} + +export interface TrustedConnectionBinding { + deviceId: string; + connectionMode: XgridsConnectionMode; +} + +/** + * CoreBluetooth UUID text is case-insensitive. Keep the original spelling for + * display and exact CAS payloads, but use this key whenever refs are compared. + */ +export function transportRefEquivalenceKey( + transportRef: string | null | undefined, +): string { + return transportRef?.trim().toLowerCase() ?? ""; +} + +/** + * Resolve the exact K1 remembered by durable/backend-owned state. This is + * selection context only: it can authorize bounded observation, but never a + * device write without the separate connection policy and an explicit click. + */ +export function trustedConnectionBinding( + state: XgridsK1State | null | undefined, +): TrustedConnectionBinding | null { + const physical = readOnlyPhysicalRecoveryBinding(state); + if (physical) return physical; + + const retiredTransportRefs = retiredPhysicalTransportRefs(state); + + const semanticRecord = state?.semantic_topology_store?.record; + const semanticDeviceId = semanticRecord?.transport_ref?.trim(); + const semanticMode = semanticRecord?.connection_mode; + if ( + semanticDeviceId + && !retiredTransportRefs.has(transportRefEquivalenceKey(semanticDeviceId)) + && isConnectionMode(semanticMode) + ) { + return { deviceId: semanticDeviceId, connectionMode: semanticMode }; + } + + const recovery = state?.current_device_recovery; + const recoveryDeviceId = recovery?.transport_ref?.trim(); + const recoveryMode = recovery?.connection_mode; + if ( + recoveryDeviceId + && !retiredTransportRefs.has(transportRefEquivalenceKey(recoveryDeviceId)) + && isConnectionMode(recoveryMode) + ) { + return { deviceId: recoveryDeviceId, connectionMode: recoveryMode }; + } + + return null; +} + +/** Transport refs explicitly retired by the operator must never rehydrate. */ +export function retiredPhysicalTransportRefs( + state: XgridsK1State | null | undefined, +): ReadonlySet { + const policyRetiredRefs = state?.connection_policy?.facts.retired_transport_refs; + if (Array.isArray(policyRetiredRefs)) { + // This backend projection is the authoritative *active* deny-list. The + // durable ledger keeps historical retirement audits even after an explicit + // reconciliation reopen, so unioning every old audit would make a safely + // reopened UUID impossible to use forever. + return new Set( + policyRetiredRefs.map(transportRefEquivalenceKey).filter(Boolean), + ); + } + const record = state?.physical_command?.record; + const retired = new Set(); + if (!record) return retired; + const retirements = Array.isArray(record.operator_retirements) + ? record.operator_retirements + : []; + for (const candidate of retirements) { + if (!candidate || typeof candidate !== "object") continue; + const transportRef = "retired_transport_ref" in candidate + && typeof candidate.retired_transport_ref === "string" + ? candidate.retired_transport_ref.trim() + : ""; + if (transportRef) retired.add(transportRefEquivalenceKey(transportRef)); + } + if (record.resolution === "operator-retired-outcome-unknown") { + const connection = record.connection; + if (connection && typeof connection === "object") { + const transportRef = "transport_ref" in connection + && typeof connection.transport_ref === "string" + ? connection.transport_ref.trim() + : ""; + if (transportRef) retired.add(transportRefEquivalenceKey(transportRef)); + } + } + return retired; +} + +export interface RetiredPhysicalReopenAuthority { + expectedRevision: number; + expectedRetirementId: string; + expectedTransportRef: string; + expectedDiscoveryGeneration: number; + expectedDesiredMode: XgridsConnectionMode; + expectedDesiredModeRevision: number; +} + +/** + * Admit the local-only reopen affordance only for the exact fresh retired row + * and exact backend-projected ledger/discovery CAS. This helper grants no GATT + * or write authority; the separately explicit Verify remains server-fenced. + */ +export function retiredPhysicalReopenAuthority( + state: XgridsK1State | null | undefined, + candidateTransportRef: string, + connectionMode: NonNullable, +): RetiredPhysicalReopenAuthority | null { + const projection = state?.physical_command?.operator_reconciliation_reopen; + const expectedRevision = projection?.expected_revision; + const expectedRetirementId = projection?.expected_retirement_id?.trim() ?? ""; + const expectedTransportRef = projection?.expected_transport_ref?.trim() ?? ""; + const expectedDiscoveryGeneration = projection?.expected_discovery_generation; + const expectedDesiredMode = projection?.expected_desired_mode; + const expectedDesiredModeRevision = + projection?.expected_desired_mode_revision; + const candidateKey = transportRefEquivalenceKey(candidateTransportRef); + const expectedKey = transportRefEquivalenceKey(expectedTransportRef); + const currentGeneration = state?.ble_discovery_generation; + const record = state?.physical_command?.record; + const recordRevision = record && typeof record.revision === "number" + ? record.revision + : null; + const retirements = record && Array.isArray(record.operator_retirements) + ? record.operator_retirements + : []; + const exactRetirementRecorded = retirements.some((candidate) => Boolean( + candidate + && typeof candidate === "object" + && "retirement_id" in candidate + && candidate.retirement_id === expectedRetirementId + && "retired_transport_ref" in candidate + && typeof candidate.retired_transport_ref === "string" + && transportRefEquivalenceKey(candidate.retired_transport_ref) === expectedKey + )); + const freshCandidates = (state?.devices ?? []).filter( + (device) => transportRefEquivalenceKey(device.device_id) === candidateKey, + ); + const freshCandidate = freshCandidates.length === 1 + ? freshCandidates[0] + : null; + const recordConnection = record && typeof record.connection === "object" + && record.connection !== null + ? record.connection + : null; + const recoveryConnectionMode = recordConnection + && "connection_mode" in recordConnection + && typeof recordConnection.connection_mode === "string" + ? recordConnection.connection_mode + : null; + const recoveryMatchesMode = Boolean( + recordConnection + && "transport_ref" in recordConnection + && typeof recordConnection.transport_ref === "string" + && transportRefEquivalenceKey(recordConnection.transport_ref) === expectedKey + && recoveryConnectionMode === connectionMode, + ); + // CoreBluetooth may advertise the exact returned transport while marking + // the passive scan row non-connectable. The backend's exact reopen CAS is + // the only exception: the explicit click is still server-fenced, and every + // unrelated non-connectable row remains blocked by the presentation layer. + if ( + projection?.allowed !== true + || projection.automatic_retry !== false + || projection.device_io_performed !== false + || projection.reason_codes.length !== 0 + || !Number.isInteger(expectedRevision) + || (expectedRevision ?? 0) < 1 + || recordRevision !== expectedRevision + || !expectedRetirementId + || !candidateKey + || candidateKey !== expectedKey + || !Number.isInteger(expectedDiscoveryGeneration) + || expectedDiscoveryGeneration !== currentGeneration + || expectedDesiredMode !== connectionMode + || state?.desired_connection_mode !== expectedDesiredMode + || !Number.isInteger(expectedDesiredModeRevision) + || expectedDesiredModeRevision !== state?.desired_connection_mode_revision + || !retiredPhysicalTransportRefs(state).has(candidateKey) + || !exactRetirementRecorded + || !freshCandidate + || !recoveryMatchesMode + ) return null; + return { + expectedRevision: expectedRevision as number, + expectedRetirementId, + expectedTransportRef, + expectedDiscoveryGeneration: expectedDiscoveryGeneration as number, + expectedDesiredMode, + expectedDesiredModeRevision: expectedDesiredModeRevision as number, + }; +} + +/** + * Prove that a lost reopen response actually committed the exact local-only + * ledger transition. Nothing here grants Verify by itself; the caller must + * also retain the original full connection-action authority. + */ +export function reopenedPhysicalReconciliationMatches( + state: XgridsK1State | null | undefined, + request: ReopenRetiredPhysicalReconciliationRequest, +): boolean { + const record = state?.physical_command?.record; + const recordRevision = record && typeof record.revision === "number" + ? record.revision + : null; + const reopens = record && Array.isArray(record.operator_reconciliation_reopens) + ? record.operator_reconciliation_reopens + : []; + const requestKey = transportRefEquivalenceKey(request.expected_transport_ref); + const exactAudit = reopens.some((candidate) => Boolean( + candidate + && typeof candidate === "object" + && "reopening_id" in candidate + && candidate.reopening_id === request.reopening_id + && "retirement_id" in candidate + && candidate.retirement_id === request.expected_retirement_id + && "retired_record_revision" in candidate + && candidate.retired_record_revision === request.expected_revision + && "reopened_transport_ref" in candidate + && typeof candidate.reopened_transport_ref === "string" + && transportRefEquivalenceKey(candidate.reopened_transport_ref) === requestKey + && "discovery_generation" in candidate + && candidate.discovery_generation === request.expected_discovery_generation + && "reason" in candidate + && candidate.reason === request.reason + )); + const activeRetiredRefs = state?.connection_policy?.facts.retired_transport_refs; + const activeDenyRemoved = Array.isArray(activeRetiredRefs) + && !activeRetiredRefs.some( + (value) => transportRefEquivalenceKey(value) === requestKey, + ); + const freshCandidates = (state?.devices ?? []).filter( + (device) => transportRefEquivalenceKey(device.device_id) === requestKey, + ); + const freshCandidate = freshCandidates.length === 1 + ? freshCandidates[0] + : null; + return Boolean( + request.expected_desired_mode === state?.desired_connection_mode + && request.expected_desired_mode_revision + === state?.desired_connection_mode_revision + && requestKey + && recordRevision === request.expected_revision + 1 + && (record?.stage === "dispatching" || record?.stage === "observing") + && record.resolution === null + && state?.physical_command?.requires_reconciliation === true + && state.ble_discovery_generation === request.expected_discovery_generation + && exactAudit + && activeDenyRemoved + && freshCandidate + && freshCandidate.connectable !== false + ); +} + +/** The exact durable K1 binding that recovery is allowed to observe. */ +export function readOnlyPhysicalRecoveryBinding( + state: XgridsK1State | null | undefined, +): ReadOnlyPhysicalRecoveryBinding | null { + if (!requiresReadOnlyPhysicalRecovery(state)) return null; + const actions: ReadOnlyConnectionObservationAction[] = [ + "observe-fresh-device-network", + "observe-current-device-network", + "observe-configured-device-network", + ]; + for (const action of actions) { + const decision = connectionPolicyDecision(state, action); + const deviceId = decision?.required_transport_ref?.trim(); + const connectionMode = decision?.required_connection_mode ?? null; + if (deviceId && isConnectionMode(connectionMode)) { + return { deviceId, connectionMode }; + } + } + return null; +} + +export function canSelectConnectionMode( + state: XgridsK1State | null | undefined, +): boolean { + const lifecycle = state?.connection_lifecycle; + return Boolean( + lifecycle?.schema_version === "missioncore.xgrids-k1-connection-lifecycle/v1" + && lifecycle.mode_selection.allowed === true + && lifecycle.mode_selection.automatic_retry === false + && lifecycle.allowed_actions.includes("select-connection-mode"), + ); +} + +export type ReadOnlyConnectionObservationAction = Extract< + XgridsConnectionPolicyAction, + | "observe-fresh-device-network" + | "observe-current-device-network" + | "observe-configured-device-network" +>; + +export type ReadOnlyConnectionObservationSource = + | "fresh-scan" + | "retained-current-process" + | "durable-configured-state"; + +export interface ReadOnlyConnectionObservationTarget { + action: ReadOnlyConnectionObservationAction; + deviceId: string; + connectionMode: XgridsConnectionMode; + source: ReadOnlyConnectionObservationSource; + serverBound: boolean; + expectedDiscoveryGeneration: number | null; +} + +function isConnectionMode(value: unknown): value is XgridsConnectionMode { + return value === "bridge" + || value === "quick-connect" + || value === "direct-connect"; +} + +function exactPolicyObservationTarget( + state: XgridsK1State | null | undefined, + action: ReadOnlyConnectionObservationAction, + source: ReadOnlyConnectionObservationSource, +): ReadOnlyConnectionObservationTarget | null { + if (!connectionPolicyAllows(state, action)) return null; + const decision = connectionPolicyDecision(state, action); + const deviceId = decision?.required_transport_ref?.trim(); + const connectionMode = decision?.required_connection_mode ?? null; + if ( + decision?.target_source !== source + || !deviceId + || !isConnectionMode(connectionMode) + ) { + return null; + } + if (source === "fresh-scan") { + const freshDevice = provisioningCandidateById(state?.devices ?? [], deviceId); + if ( + !freshDevice + || freshDevice.connectable === false + || !Number.isInteger(state?.ble_discovery_generation) + || (state?.ble_discovery_generation ?? -1) < 0 + ) return null; + } + return { + action, + deviceId, + connectionMode, + source, + serverBound: true, + expectedDiscoveryGeneration: source === "fresh-scan" + ? state?.ble_discovery_generation as number + : null, + }; +} + +const SERVER_BOUND_RECOVERY_OBSERVATION_PRIORITY: +ReadonlyArray = [ + "observe-current-device-network", + "observe-configured-device-network", + "observe-fresh-device-network", +]; + +/** + * Resolve a recovery Verify target from the public policy, never from a + * browser selection. The backend recommendation wins when it names an exact + * allowed observation; older compatible projections fall back in the same + * current -> configured -> fresh order used by the supervisor. + */ +export function recommendedConnectionRecoveryObservationTarget( + state: XgridsK1State | null | undefined, +): ReadOnlyConnectionObservationTarget | null { + const recommended = state?.connection_policy?.recommended_action; + const orderedActions = SERVER_BOUND_RECOVERY_OBSERVATION_PRIORITY.includes( + recommended as ReadOnlyConnectionObservationAction, + ) + ? [ + recommended as ReadOnlyConnectionObservationAction, + ...SERVER_BOUND_RECOVERY_OBSERVATION_PRIORITY.filter( + (action) => action !== recommended, + ), + ] + : SERVER_BOUND_RECOVERY_OBSERVATION_PRIORITY; + const sources: Record< + ReadOnlyConnectionObservationAction, + ReadOnlyConnectionObservationSource + > = { + "observe-current-device-network": "retained-current-process", + "observe-configured-device-network": "durable-configured-state", + "observe-fresh-device-network": "fresh-scan", + }; + for (const action of orderedActions) { + const target = exactPolicyObservationTarget(state, action, sources[action]); + if (target?.serverBound) return target; + } + return null; +} + +/** + * Resolve the one read-only BLE target authorized by the server policy. + * Unresolved writes never fall back to a browser selection or dropdown mode: + * their UUID, mode and recovery source must be pinned by the same decision. + */ +export function readOnlyConnectionObservationTarget( + state: XgridsK1State | null | undefined, + selectedDeviceId = "", + selectedConnectionMode: XgridsConnectionMode | null = null, +): ReadOnlyConnectionObservationTarget | null { + const exactFresh = exactPolicyObservationTarget( + state, + "observe-fresh-device-network", + "fresh-scan", + ); + if (exactFresh) return exactFresh; + + // Outside reconciliation, the backend deliberately leaves the fresh target + // unpinned because the operator may choose among several current adverts. + // Preserve that reviewed path, but never use it for an unresolved write. + if ( + !hasUnresolvedNetworkMutation(state) + && !requiresReadOnlyPhysicalRecovery(state) + && selectedConnectionMode + && connectionPolicyAllows(state, "observe-fresh-device-network") + ) { + const freshDevice = provisioningCandidateById( + state?.devices ?? [], + selectedDeviceId, + ); + if ( + freshDevice + && freshDevice.connectable !== false + && Number.isInteger(state?.ble_discovery_generation) + && (state?.ble_discovery_generation ?? -1) >= 0 + ) { + return { + action: "observe-fresh-device-network", + deviceId: freshDevice.device_id, + connectionMode: selectedConnectionMode, + source: "fresh-scan", + serverBound: false, + expectedDiscoveryGeneration: state?.ble_discovery_generation as number, + }; + } + } + + return exactPolicyObservationTarget( + state, + "observe-current-device-network", + "retained-current-process", + ) ?? exactPolicyObservationTarget( + state, + "observe-configured-device-network", + "durable-configured-state", + ); +} + +/** + * Resolve recovery for an already-applied network only from a backend-pinned + * current/configured target. A browser-selected advertisement is never an + * authority for this read-only continuation, even when it happens to carry + * the same UUID. + */ +export function serverBoundAppliedNetworkObservationTarget( + state: XgridsK1State | null | undefined, + connectionMode: XgridsConnectionMode, +): ReadOnlyConnectionObservationTarget | null { + const target = exactPolicyObservationTarget( + state, + "observe-current-device-network", + "retained-current-process", + ) ?? exactPolicyObservationTarget( + state, + "observe-configured-device-network", + "durable-configured-state", + ); + return target?.serverBound === true + && target.connectionMode === connectionMode + ? target + : null; +} + +export interface BackendConnectionTopology { + connectionMode: NonNullable; + status: "active" | "configured-unverified" | "configured-offline"; + source: "applied" | "durable" | "last-known"; + endpoint: string | null; +} + +function sameTarget( + left: { ipv4: string; port: number } | null | undefined, + right: { ipv4: string; port: number } | null | undefined, +): boolean { + return Boolean( + left + && right + && left.ipv4 === right.ipv4 + && left.port === right.port, + ); +} + +export function currentAppliedConnectionTopology( + state: XgridsK1State | null | undefined, + connectionMode?: NonNullable, +): BackendConnectionTopology | null { + const supervisor = state?.connection_supervisor; + if (!supervisor || supervisor.closed) return null; + const { intent, lease, observed } = supervisor; + const deviceNetwork = observed.device_network; + const mode = deviceNetwork?.connection_mode; + const target = deviceNetwork?.target; + const currentDeviceNetwork = Boolean( + intent + && mode + && target + && (!connectionMode || mode === connectionMode) + && deviceNetwork.state === "applied" + && deviceNetwork.intent_id === intent.intent_id + && Boolean(deviceNetwork.transport_ref) + && mode === intent.requested_mode + ); + if (!currentDeviceNetwork || !mode || !target) return null; + const currentEndpoint = Boolean( + ["configured-unverified", "reachable"].includes(lease.state) + && lease.intent_id === intent?.intent_id + && lease.connection_mode === mode + && sameTarget(lease.target, target) + && observed.host_path.available === true + && observed.host_path.route_class === "direct" + && lease.host_path_epoch === observed.host_path.epoch + && observed.endpoint.intent_id === intent?.intent_id + && observed.endpoint.host_path_epoch === observed.host_path.epoch + && observed.endpoint.tcp_state === "reachable" + && sameTarget(target, observed.endpoint.target) + ); + const identity = observed.device_identity; + const controlPlane = observed.control_plane; + const lifecycle = state?.connection_lifecycle; + const activeBinding = lifecycle?.active_binding; + const identityExact = Boolean( + identity.state === "verified" + && identity.intent_id === intent?.intent_id + && identity.connection_mode === mode + && identity.host_path_epoch === observed.host_path.epoch + && identity.logical_device_id + && lease.logical_device_id === identity.logical_device_id + && (!intent?.expected_device_id + || identity.logical_device_id === intent.expected_device_id), + ); + const active = Boolean( + currentEndpoint + && lease.state === "reachable" + && supervisor.authority.control_allowed === true + && identityExact + && controlPlane.state === "healthy" + && Boolean(controlPlane.session_id) + && controlPlane.host_path_epoch === observed.host_path.epoch + && lifecycle?.schema_version === "missioncore.xgrids-k1-connection-lifecycle/v1" + && lifecycle.connection_ready === true + && lifecycle.configured_mode === mode + && lifecycle.active_mode === mode + && activeBinding?.connection_mode === mode + && activeBinding.intent_id === intent?.intent_id + && transportRefEquivalenceKey(activeBinding.transport_ref) + === transportRefEquivalenceKey(deviceNetwork.transport_ref) + && activeBinding.target_ipv4 === target.ipv4 + && activeBinding.target_port === target.port + && activeBinding.host_path_epoch === observed.host_path.epoch + && activeBinding.control_session_id === controlPlane.session_id + ); + return { + connectionMode: mode, + status: active + ? "active" + : currentEndpoint + ? "configured-unverified" + : "configured-offline", + source: "applied", + endpoint: target.ipv4, + }; +} + +export function backendConnectionTopology( + state: XgridsK1State | null | undefined, + connectionMode?: NonNullable, +): BackendConnectionTopology | null { + const supervisor = state?.connection_supervisor; + const applied = currentAppliedConnectionTopology(state); + // A current BLE-proved device topology supersedes every persisted or + // historical address, including when the selected UI mode is different. + if (applied) { + return !connectionMode || applied.connectionMode === connectionMode + ? applied + : null; + } + const semanticStore = state?.semantic_topology_store; + const durable = semanticStore?.record; + if ( + semanticStore?.status === "available" + && semanticStore.configured_offline_evidence === true + && semanticStore.live_connection_authority === false + && durable + && durable.schema_version === "missioncore.xgrids-k1-semantic-topology/v1" + && (!connectionMode || durable.connection_mode === connectionMode) + && Boolean(durable.ipv4.trim()) + ) { + const endpointProbe = state?.configured_endpoint_probe; + const durableEndpointReachable = Boolean( + endpointProbe?.status === "reachable" + && endpointProbe.target_source === "durable-semantic-topology" + && endpointProbe.connection_mode === durable.connection_mode + && endpointProbe.endpoint === durable.ipv4 + && transportRefEquivalenceKey(endpointProbe.transport_ref) + === transportRefEquivalenceKey(durable.transport_ref) + && endpointProbe.semantic_revision === durable.revision + && endpointProbe.host_route_available === true + && endpointProbe.host_route_class === "direct" + && endpointProbe.tcp_reachable === true + && endpointProbe.identity_validation === "not-performed" + && endpointProbe.control_authority_granted === false + && endpointProbe.ble_operation_performed === false + && endpointProbe.network_mutation_performed === false + && endpointProbe.automatic_retry === false + ); + return { + connectionMode: durable.connection_mode, + status: durableEndpointReachable + ? "configured-unverified" + : "configured-offline", + source: "durable", + endpoint: durable.ipv4, + }; + } + const lastKnown = supervisor?.last_known; + if ( + !lastKnown + || (connectionMode && lastKnown.connection_mode !== connectionMode) + ) return null; + return { + connectionMode: lastKnown.connection_mode, + status: "configured-offline", + source: "last-known", + endpoint: lastKnown.target.ipv4, + }; +} + +export function hasUnresolvedNetworkMutation( + state: XgridsK1State | null | undefined, +): boolean { + if (state?.network_write_reconciliation) return true; + const ledger = state?.network_mutation_ledger; + return Boolean( + ledger + && ( + ledger.status === "unresolved" + || ledger.status === "corrupt" + || ledger.mutation_allowed !== true + ), + ); } export function canSubmitProvisioningMutation({ devices, selectedDeviceId, - powerConfirmed, credentialsReady, isBusy, }: { devices: readonly BleDevice[]; selectedDeviceId: string; - powerConfirmed: boolean; credentialsReady: boolean; isBusy: boolean; }): boolean { const candidate = provisioningCandidateById(devices, selectedDeviceId); return Boolean( - powerConfirmed && credentialsReady && !isBusy && candidate && @@ -232,16 +1512,198 @@ export function canSubmitProvisioningMutation({ ); } +export function canAdmitProvisioningConnection({ + policyAllowed, + targetSource, + hasSuccessfulLocalConnect, + localPrerequisitesReady, +}: { + policyAllowed: boolean; + targetSource: "fresh-scan" | null; + hasSuccessfulLocalConnect: boolean; + localPrerequisitesReady: boolean; +}): boolean { + return Boolean( + policyAllowed + && targetSource === "fresh-scan" + && !hasSuccessfulLocalConnect + && localPrerequisitesReady, + ); +} + export function isReachableConnectionLease( state: XgridsK1State | null | undefined, connectionMode: NonNullable, ): boolean { - const verification = state?.connection_verification; + const supervisor = state?.connection_supervisor; + if (!supervisor || supervisor.closed) return false; + return currentAppliedConnectionTopology(state, connectionMode)?.status === "active"; +} + +export function isConfiguredConnectionLease( + state: XgridsK1State | null | undefined, + connectionMode: NonNullable, +): boolean { + const topology = backendConnectionTopology(state, connectionMode); + return Boolean(topology && topology.source !== "last-known"); +} + +export function hasControlAuthority( + state: XgridsK1State | null | undefined, +): boolean { + const topology = currentAppliedConnectionTopology(state); + return Boolean(topology?.status === "active"); +} + +export function hasAuthoritativeData( + state: XgridsK1State | null | undefined, +): boolean { + const supervisor = state?.connection_supervisor; + const dataPlane = supervisor?.observed.data_plane; return Boolean( - state?.k1_ip && - state.connection_mode === connectionMode && - verification?.lease_state === "reachable" && - verification.network_reachability === "reachable", + hasControlAuthority(state) + && supervisor?.authority.data_ingest_authoritative === true + && dataPlane?.state === "healthy" + && Boolean(dataPlane.session_id) + && dataPlane.host_path_epoch === supervisor.observed.host_path.epoch, + ); +} + +export function canonicalDeviceConnectivity( + state: XgridsK1State | null | undefined, +): "unknown" | "offline" | "connecting" | "connected" | "degraded" { + const topology = currentAppliedConnectionTopology(state); + if (topology?.status === "active") { + const supervisor = state?.connection_supervisor; + return supervisor + && ["stalled", "lost"].includes(supervisor.observed.data_plane.state) + ? "degraded" + : "connected"; + } + if (topology?.status === "configured-unverified") return "connecting"; + if (topology?.status === "configured-offline") return "offline"; + const fallback = backendConnectionTopology(state); + if (fallback?.source === "last-known") return "degraded"; + if (fallback?.status === "configured-offline") return "offline"; + const supervisor = state?.connection_supervisor; + if (!supervisor || supervisor.closed) return supervisor?.closed ? "offline" : "unknown"; + if (supervisor.lease.state === "lost" || supervisor.last_known) return "degraded"; + return "offline"; +} + +export function activeConnectionEndpointLabel( + state: XgridsK1State | null | undefined, +): string | null { + const topology = currentAppliedConnectionTopology(state); + return topology?.status === "active" ? topology.endpoint : null; +} + +export interface ReachableConnectionLeaseIdentity { + key: string; + runtimeId: string; + leaseGeneration: number; + intentId: string; + hostPathEpoch: number; + connectionMode: NonNullable; +} + +export interface RuntimeErrorCorrelation { + action: string; + runtimeId: string | null; + leaseGeneration: number | null; + connectionAttemptId: string | null; +} + +/** + * Attach connection-attempt diagnostics only to the exact failed Connect + * action which produced them. A global Scan/Verify/Refresh failure must never + * borrow an older durable attempt merely because it remains in the snapshot. + */ +export function connectionAttemptForRuntimeError( + error: RuntimeErrorCorrelation | null | undefined, + state: XgridsK1State | null | undefined, +): XgridsConnectionAttempt | null { + const runtimeId = state?.snapshot_runtime_id?.trim() || null; + const attempt = state?.connection_attempt; + return error?.action === "connect" + && typeof error.runtimeId === "string" + && error.runtimeId === runtimeId + && typeof error.connectionAttemptId === "string" + && attempt + && error.connectionAttemptId === attempt?.attempt_id + && !["accepted", "running"].includes(attempt.status) + ? attempt + : null; +} + +export function reachableConnectionLeaseIdentity( + state: XgridsK1State | null | undefined, +): ReachableConnectionLeaseIdentity | null { + const runtimeId = state?.snapshot_runtime_id; + const supervisor = state?.connection_supervisor; + const connectionMode = supervisor?.lease.connection_mode; + const leaseGeneration = supervisor?.lease.generation; + const intentId = supervisor?.intent?.intent_id; + const hostPathEpoch = supervisor?.lease.host_path_epoch; + if ( + typeof runtimeId !== "string" + || !runtimeId.trim() + || !connectionMode + || !intentId + || !Number.isInteger(leaseGeneration) + || (leaseGeneration ?? -1) < 0 + || !Number.isInteger(hostPathEpoch) + || (hostPathEpoch ?? 0) < 1 + || !isReachableConnectionLease(state, connectionMode) + ) { + return null; + } + return { + key: `${runtimeId}:${intentId}:${hostPathEpoch}:${leaseGeneration}`, + runtimeId, + leaseGeneration: leaseGeneration as number, + intentId, + hostPathEpoch: hostPathEpoch as number, + connectionMode, + }; +} + +export function authoritativeReachableLeaseSupersedesError( + error: RuntimeErrorCorrelation | null | undefined, + state: XgridsK1State | null | undefined, +): boolean { + if ( + !error + || (error.action !== "connect" && error.action !== "verify") + || typeof error.runtimeId !== "string" + || !Number.isInteger(error.leaseGeneration) + // An omitted legacy field is not proof that the process-owned write fence + // was cleared. Only the canonical explicit null may dismiss the banner. + || hasUnresolvedNetworkMutation(state) + ) { + return false; + } + const identity = reachableConnectionLeaseIdentity(state); + return Boolean( + identity + && identity.runtimeId === error.runtimeId + && identity.leaseGeneration > (error.leaseGeneration as number), + ); +} + +/** + * Clear a transient action banner when a later authoritative state proves + * that the failed local acquisition has already been sealed and released. + * Every poll/WebSocket state enters through the same reducer, so recovery + * needs no refresh button or browser-cache reset. + */ +export function authoritativeStateSupersedesRuntimeError( + error: RuntimeErrorCorrelation | null | undefined, + state: XgridsK1State | null | undefined, +): boolean { + return Boolean( + authoritativeReachableLeaseSupersedesError(error, state) + || (error && !shouldSurfaceRuntimeActionError(error.action, state)), ); } @@ -261,6 +1723,33 @@ function defaultUuid(): string { .join("")}-${hex.slice(8, 10).join("")}-${hex.slice(10).join("")}`; } +export function newOperationId(): string { + return `op-${defaultUuid()}`; +} + +export function newPhysicalRetirementId(): string { + return `retirement-${defaultUuid()}`; +} + +export function newPhysicalReopeningId(): string { + return `reopening-${defaultUuid()}`; +} + +export function newMutationContext(action: string): { + operation_id: string; + idempotency_key: string; +} { + const normalizedAction = action.trim(); + if (!normalizedAction) { + throw new Error("Действие операции не задано; безопасный ключ не создан."); + } + const operationId = newOperationId(); + return { + operation_id: operationId, + idempotency_key: `${normalizedAction}:${operationId}`, + }; +} + export function provisioningIntentKey( current: string | null, createUuid: () => string = defaultUuid, diff --git a/plugins/xgrids-k1/frontend/src/manifest.ts b/plugins/xgrids-k1/frontend/src/manifest.ts index 6e1e587..32edd51 100644 --- a/plugins/xgrids-k1/frontend/src/manifest.ts +++ b/plugins/xgrids-k1/frontend/src/manifest.ts @@ -17,12 +17,28 @@ export const xgridsK1Actions = Object.freeze({ xgridsK1Manifest, "calibration.device-snapshot.read", ), + connectionModeSelect: requirePluginAction( + xgridsK1Manifest, + "connection.mode.select", + ), + connectionReconfigurePrepare: requirePluginAction( + xgridsK1Manifest, + "connection.reconfigure.prepare", + ), networkProvision: requirePluginAction(xgridsK1Manifest, "network.provision"), connectionVerify: requirePluginAction(xgridsK1Manifest, "connection.verify"), + configuredEndpointProbe: requirePluginAction( + xgridsK1Manifest, + "connection.endpoint-probe", + ), acquisitionPrepare: requirePluginAction(xgridsK1Manifest, "acquisition.prepare"), acquisitionStart: requirePluginAction(xgridsK1Manifest, "acquisition.start"), acquisitionStop: requirePluginAction(xgridsK1Manifest, "acquisition.stop"), acquisitionAbort: requirePluginAction(xgridsK1Manifest, "acquisition.abort"), + acquisitionForceFinishLocal: requirePluginAction( + xgridsK1Manifest, + "acquisition.force-finish-local", + ), acquisitionStateRead: requirePluginAction(xgridsK1Manifest, "acquisition.state.read"), compatibilityStreamStartLive: requirePluginAction(xgridsK1Manifest, "stream.start-live"), streamStartReplay: requirePluginAction(xgridsK1Manifest, "stream.start-replay"), @@ -54,4 +70,16 @@ export const xgridsK1Actions = Object.freeze({ xgridsK1Manifest, "application-control.session.close", ), + physicalCommandReconcile: requirePluginAction( + xgridsK1Manifest, + "physical-command.reconcile", + ), + physicalCommandRetireUnavailable: requirePluginAction( + xgridsK1Manifest, + "physical-command.retire-unavailable", + ), + physicalCommandReopenRetiredReconciliation: requirePluginAction( + xgridsK1Manifest, + "physical-command.reopen-retired-reconciliation", + ), }); diff --git a/plugins/xgrids-k1/frontend/src/messages.ts b/plugins/xgrids-k1/frontend/src/messages.ts index 0cf447e..17eee43 100644 --- a/plugins/xgrids-k1/frontend/src/messages.ts +++ b/plugins/xgrids-k1/frontend/src/messages.ts @@ -48,7 +48,6 @@ const runtimeMessageReplacements: Array<[RegExp, string]> = [ ], [/Foxglove/gi, "локальный мост визуализации"], [/MacBook/gi, "компьютер"], - [/\bK1\b/g, "устройство"], ]; export function localizeRuntimeMessage(message: string | null | undefined): string | null { diff --git a/plugins/xgrids-k1/frontend/src/observationSources.ts b/plugins/xgrids-k1/frontend/src/observationSources.ts index d6774fd..34ff900 100644 --- a/plugins/xgrids-k1/frontend/src/observationSources.ts +++ b/plugins/xgrids-k1/frontend/src/observationSources.ts @@ -3,15 +3,28 @@ import type { ObservationSourceAvailability, ObservationSourceDelivery, ObservationSourceDescriptor, + ObservationSourcePresentationLease, ObservationSourceProvider, } from "@mission-core/plugin-sdk"; -import { confirmedRuntimeSourceMode, effectiveAcquisition } from "./lifecycle"; +import { + activeStreamRecoveredBrowserAuthority, + activeStreamRecoveryOwnsPresentationDecision, + activeStreamRecoveryPresentationAuthority, + type ActiveStreamRecoveryPresentationAuthority, +} from "./activeStreamRecovery"; +import { + confirmedRuntimeSourceMode, + effectiveAcquisition, + hasAuthoritativeData, + hasControlAuthority, +} from "./lifecycle"; import { xgridsK1Manifest } from "./manifest"; import type { XgridsCameraPreviewDelivery, XgridsK1State, XgridsSensorCatalogStream, } from "./api"; +import { isXgridsActiveStreamRecovery } from "./api"; function providerFor( state: XgridsK1State, @@ -22,30 +35,72 @@ function providerFor( pluginVersion: xgridsK1Manifest.metadata.version, modelId: state.device_ref?.model_id || activeModel.id, compatibilityProfileId: - state.device_session?.compatibility_profile_id ?? state.compatibility?.profile_id ?? null, + (state.connection_supervisor?.observed.device_identity.state === "verified" + ? state.connection_supervisor.observed.device_identity.compatibility_profile_id + : null) + ?? state.device_session?.compatibility_profile_id + ?? state.compatibility?.profile_id + ?? null, }; } -function bindingFor(state: XgridsK1State) { +function bindingFor( + state: XgridsK1State, + recoveryAuthority: ActiveStreamRecoveryPresentationAuthority | null, +) { const acquisition = effectiveAcquisition(state); + const controlAuthoritative = hasControlAuthority(state); + const recoveryAuthoritative = recoveryAuthority !== null; return { - deviceId: state.device_ref?.device_id ?? null, - deviceSessionId: state.device_session?.device_session_id ?? null, + // A legacy snapshot may retain a selected device and session long after + // the control topology has disappeared. Do not publish those values as a + // live host binding until the supervisor has re-attested the topology. + deviceId: recoveryAuthoritative + ? acquisition?.device_id?.trim() || null + : controlAuthoritative ? state.device_ref?.device_id ?? null : null, + deviceSessionId: recoveryAuthoritative + ? acquisition?.device_session_id?.trim() || null + : controlAuthoritative ? state.device_session?.device_session_id ?? null : null, acquisitionId: acquisition?.acquisition_id ?? null, }; } +function recoveryPresentationLease( + authority: ActiveStreamRecoveryPresentationAuthority, +): ObservationSourcePresentationLease { + return { + kind: "active-stream-recovery", + runtimeId: authority.snapshotRuntimeId, + acquisitionId: authority.acquisitionId, + acquisitionStateRevision: authority.acquisitionStateRevision, + producerGeneration: authority.runtimeProducerGeneration, + recoveryGeneration: authority.recoveryGeneration, + }; +} + function catalogDeclares(state: XgridsK1State, streamId: string): boolean { return Boolean(state.sensor_catalog?.streams?.some((stream) => stream.stream_id === streamId)); } -function spatialAvailability(state: XgridsK1State): ObservationSourceAvailability { +function spatialAvailability( + state: XgridsK1State, + recoveryAuthoritative: boolean, + recoveryOwnsPresentation: boolean, +): ObservationSourceAvailability { const mode = confirmedRuntimeSourceMode(state); - if (mode !== "idle" && state.rerun_grpc_url?.trim()) return "streaming"; - if (state.rerun_grpc_url?.trim()) return "available"; - if (state.device_session?.connectivity === "degraded") return "degraded"; - if (state.device_session?.connectivity === "connected") return "available"; - return catalogDeclares(state, "spatial.point-cloud.live") ? "declared" : "unavailable"; + if (mode === "replay" && state.rerun_grpc_url?.trim()) return "streaming"; + const declared = catalogDeclares(state, "spatial.point-cloud.live"); + if (recoveryAuthoritative && state.rerun_grpc_url?.trim()) return "connecting"; + if (recoveryOwnsPresentation) return declared ? "degraded" : "unavailable"; + if (!hasControlAuthority(state)) return declared ? "unverified" : "unavailable"; + if (mode === "live" && state.rerun_grpc_url?.trim() && hasAuthoritativeData(state)) { + return "streaming"; + } + if (["stalled", "lost"].includes( + state.connection_supervisor?.observed.data_plane.state ?? "idle", + )) return "degraded"; + if (state.rerun_grpc_url?.trim() || declared) return "available"; + return "unavailable"; } function catalogAvailability(value: string | null | undefined): ObservationSourceAvailability { @@ -167,15 +222,42 @@ function browserDelivery( return { id, kind: value.kind, url, mediaType }; } +function sameBrowserDelivery( + left: ObservationSourceDelivery | null, + right: ObservationSourceDelivery | null, +): boolean { + return Boolean( + left + && right + && left.kind === "mse-fmp4-websocket" + && right.kind === "mse-fmp4-websocket" + && left.id === right.id + && left.url === right.url + && left.mediaType === right.mediaType, + ); +} + function cameraAvailability( state: XgridsK1State, stream: XgridsSensorCatalogStream, selected: boolean, delivery: ObservationSourceDelivery | null, attested: boolean, + recoverySelected: boolean, + exactCurrentEpochReady: boolean, ): ObservationSourceAvailability { + if (recoverySelected) { + return exactCurrentEpochReady + ? "streaming" + : state.camera_preview?.phase?.trim().toLowerCase() === "degraded" + ? "degraded" + : "connecting"; + } if (!attested) return "unverified"; - if (state.device_session?.connectivity === "degraded") return "degraded"; + if (!hasControlAuthority(state)) return "degraded"; + if (["stalled", "lost"].includes( + state.connection_supervisor?.observed.data_plane.state ?? "idle", + )) return "degraded"; const base = catalogAvailability(stream.availability); if (!selected) return base === "streaming" || base === "connecting" ? "available" : base; @@ -188,12 +270,82 @@ function cameraAvailability( return "connecting"; } +function exactCurrentCameraEpochReady(state: XgridsK1State): boolean { + const recovery = state.connection_recovery; + const previewGeneration = state.camera_preview?.generation; + if ( + !isXgridsActiveStreamRecovery(recovery) + || recovery.camera_media_state !== "ready" + || recovery.camera_media_ready !== true + || !Number.isInteger(previewGeneration) + || (previewGeneration ?? 0) < 1 + ) return false; + const epoch = recovery.camera_epoch; + return Boolean( + epoch + && epoch.generation === previewGeneration + && epoch.init_committed === true + && epoch.first_media_committed === true + && epoch.committed_media_segment_count > 0, + ); +} + +function recoveryCameraTupleIsExact( + state: XgridsK1State, + authority: ActiveStreamRecoveryPresentationAuthority | null, + provider: ObservationSourceProvider, +): boolean { + const acquisition = state.acquisition; + const deviceId = acquisition?.device_id?.trim(); + const deviceSessionId = acquisition?.device_session_id?.trim(); + const compatibilityProfileId = acquisition?.compatibility_profile_id?.trim(); + return Boolean( + authority + && authority.recovery.camera_recovery === "owned" + && acquisition + && acquisition.acquisition_id.trim() === authority.acquisitionId + && deviceId + && deviceSessionId + && compatibilityProfileId + && state.device_ref?.device_id?.trim() === deviceId + && state.device_session?.device_session_id?.trim() === deviceSessionId + && state.device_session?.device_id?.trim() === deviceId + && state.device_session?.compatibility_profile_id?.trim() === compatibilityProfileId + && provider.compatibilityProfileId?.trim() === compatibilityProfileId + ); +} + +function cameraRecoveryPhaseRetainable(state: XgridsK1State): boolean { + return [ + "active", + "buffering", + "connecting", + "degraded", + "ready", + "reconnecting", + "streaming", + ].includes(state.camera_preview?.phase?.trim().toLowerCase() ?? ""); +} + export function xgridsK1ObservationSources( state: XgridsK1State, activeModel: DeviceModelDefinition, ): ObservationSourceDescriptor[] { const provider = providerFor(state, activeModel); - const binding = bindingFor(state); + const recoveryAuthority = activeStreamRecoveryPresentationAuthority(state); + const recoveredBrowserAuthority = activeStreamRecoveredBrowserAuthority(state); + const browserLineageAuthority = recoveryAuthority ?? recoveredBrowserAuthority; + const recoveryOwnsPresentation = activeStreamRecoveryOwnsPresentationDecision(state); + const recoveryAuthoritative = recoveryAuthority !== null; + const presentationLease = browserLineageAuthority + ? recoveryPresentationLease(browserLineageAuthority) + : null; + const binding = bindingFor(state, browserLineageAuthority); + const replayAuthoritative = state.source_mode === "replay"; + const dataAuthoritative = hasAuthoritativeData(state) && !recoveryOwnsPresentation; + const spatialPreviewUrl = replayAuthoritative || dataAuthoritative || recoveryAuthoritative + ? state.rerun_grpc_url?.trim() || null + : null; const clockId = binding.acquisitionId ?? binding.deviceSessionId ?? binding.deviceId ?? null; const descriptorId = (sourceId: string) => `${provider.pluginId}:${provider.modelId}:${sourceId}`; @@ -205,12 +357,22 @@ export function xgridsK1ObservationSources( description: "Облако точек, поза и траектория в общей 3D-сцене", modality: "point-cloud", role: "primary", - availability: spatialAvailability(state), + availability: spatialAvailability( + state, + recoveryAuthoritative, + recoveryOwnsPresentation, + ), transport: "rerun-grpc", endpointLabel: state.rerun_grpc_url?.trim() ? "Rerun gRPC" : "MQTT → Rerun", - previewUrl: state.rerun_grpc_url?.trim() || null, + previewUrl: spatialPreviewUrl, delivery: null, activation: null, + presentationLease: ( + recoveryAuthoritative + || (recoveredBrowserAuthority !== null && dataAuthoritative) + ) && spatialPreviewUrl + ? presentationLease + : null, provider, binding, capabilities: { @@ -234,9 +396,27 @@ export function xgridsK1ObservationSources( const sourceId = stream.source_id?.trim(); if (sourceId) sourceIdCounts.set(sourceId, (sourceIdCounts.get(sourceId) ?? 0) + 1); } - const attested = Boolean(provider.compatibilityProfileId && binding.deviceSessionId); + const supervisor = state.connection_supervisor; + const verifiedControl = state.application_control_session?.verified_control; + const attested = Boolean( + hasControlAuthority(state) + && provider.compatibilityProfileId + && binding.deviceSessionId + && verifiedControl + && supervisor?.observed.control_plane.session_id === verifiedControl.control_session_id + && supervisor.observed.device_identity.logical_device_id + === verifiedControl.logical_device_id + && supervisor.observed.device_identity.compatibility_profile_id + === verifiedControl.compatibility_profile_id, + ); const sessionScope = binding.deviceSessionId ?? binding.deviceId ?? "unbound"; const activeSourceId = state.camera_preview?.active_source_id?.trim() ?? null; + const exactCameraMediaReady = exactCurrentCameraEpochReady(state); + const browserLineageCameraTupleExact = recoveryCameraTupleIsExact( + state, + browserLineageAuthority, + provider, + ); const cameras = cameraRows.flatMap((stream) => { const sourceId = stream.source_id?.trim(); @@ -249,19 +429,58 @@ export function xgridsK1ObservationSources( const activationValid = Boolean( groupId && Number.isInteger(maxActive) && (maxActive ?? 0) > 0, ); - const selected = Boolean( - attested && activationValid && rawActivation?.selected === true && activeSourceId === sourceId, + const normallySelected = Boolean( + !recoveryOwnsPresentation + && attested + && activationValid + && rawActivation?.selected === true + && activeSourceId === sourceId + && exactCameraMediaReady, ); + const streamDelivery = browserDelivery(stream.delivery); + const previewDelivery = browserDelivery(state.camera_preview?.delivery); + const candidateDelivery = stream.delivery ?? state.camera_preview?.delivery; + const retainedDelivery = browserDelivery(candidateDelivery); + const deliveryConsistent = !stream.delivery || !state.camera_preview?.delivery + || sameBrowserDelivery(streamDelivery, previewDelivery); + const streamRecoveryAvailable = [ + "available", + "connecting", + "degraded", + "streaming", + ].includes(catalogAvailability(stream.availability)); + const browserLineageSelected = Boolean( + browserLineageCameraTupleExact + && activationValid + && maxActive === 1 + && rawActivation?.selected === true + && activeSourceId === sourceId + && cameraRecoveryPhaseRetainable(state) + && streamRecoveryAvailable + && retainedDelivery + && deliveryConsistent + ); + const recoverySelected = recoveryAuthority !== null && browserLineageSelected; + const recoveredBrowserSelected = Boolean( + recoveredBrowserAuthority + && browserLineageSelected + ); + const selected = normallySelected || recoverySelected || recoveredBrowserSelected; const activation = activationValid ? { groupId: `${provider.pluginId}:${sessionScope}:${groupId}`, maxActive: maxActive as number, selected, - controllable: Boolean(attested && rawActivation?.controllable), + controllable: Boolean( + !recoveryOwnsPresentation && attested && rawActivation?.controllable, + ), } : null; - const candidateDelivery = stream.delivery ?? state.camera_preview?.delivery; - const delivery = selected ? browserDelivery(candidateDelivery) : null; + const delivery = selected && ( + dataAuthoritative || recoverySelected || recoveredBrowserSelected + ) + ? retainedDelivery + : null; const label = stream.label?.trim() || sourceId; return [{ @@ -272,12 +491,23 @@ export function xgridsK1ObservationSources( description: "Видеоканал, опубликованный активным device-плагином", modality: "video", role: "auxiliary", - availability: cameraAvailability(state, stream, selected, delivery, attested), + availability: cameraAvailability( + state, + stream, + selected, + delivery, + attested, + recoverySelected || recoveredBrowserSelected, + exactCameraMediaReady, + ), transport: delivery ? "websocket" : "other", endpointLabel: safeEndpointLabel(stream.endpoint_label) ?? "Локальный video adapter", previewUrl: null, delivery, activation, + presentationLease: (recoverySelected || recoveredBrowserSelected) && delivery + ? presentationLease + : null, provider, binding, capabilities: { diff --git a/plugins/xgrids-k1/frontend/src/operatorIntentGeneration.ts b/plugins/xgrids-k1/frontend/src/operatorIntentGeneration.ts index d80cf19..8f1c898 100644 --- a/plugins/xgrids-k1/frontend/src/operatorIntentGeneration.ts +++ b/plugins/xgrids-k1/frontend/src/operatorIntentGeneration.ts @@ -6,6 +6,15 @@ export interface OperatorIntentToken extends RuntimeGenerationToken { readonly intentGeneration: number; } +export function isSnapshotRuntimeCurrent( + expectedSnapshotRuntimeId: string, + currentSnapshotRuntimeId: string | null | undefined, +): boolean { + const expected = expectedSnapshotRuntimeId.trim(); + const current = currentSnapshotRuntimeId?.trim() ?? ""; + return Boolean(expected && current && expected === current); +} + /** * Invalidates asynchronous UI work across both plugin activation changes and * successive explicit operator intents. diff --git a/plugins/xgrids-k1/frontend/src/physicalCommandConfirmation.ts b/plugins/xgrids-k1/frontend/src/physicalCommandConfirmation.ts new file mode 100644 index 0000000..09ce477 --- /dev/null +++ b/plugins/xgrids-k1/frontend/src/physicalCommandConfirmation.ts @@ -0,0 +1,430 @@ +import type { + OperatorPresenceConfirmation, + XgridsAcquisition, + XgridsApplicationControlSession, + XgridsConnectionMode, + XgridsK1State, +} from "./api"; +import { currentAppliedConnectionTopology, isSoftwareCommandedAcquisition } from "./lifecycle"; + +export interface PhysicalConfirmationChecks { + operatorPresent: boolean; + ownerControlledDevice: boolean; + lixelgoClosed: boolean; + batteryStorageConfirmed: boolean; + expectedPhysicalStateConfirmed: boolean; +} + +type CompletedPhysicalConfirmationChecks = { + [Key in keyof PhysicalConfirmationChecks]: true; +}; + +export type K1PhysicalConfirmationKind = "prepare" | "start" | "stop"; + +/** + * Semantic state that authorises one physical command confirmation. + * + * Timestamps, the polling snapshot revision and ConnectionSupervisor.revision + * are deliberately absent: the latter is an observation counter and advances + * even when a probe confirms the same semantic route. A read-only refresh must + * not invalidate an operator confirmation. Every field below, however, changes + * the identity, route, CAS authority, acquisition or runtime state of the + * command and therefore closes an already-open modal. + */ +export interface K1PhysicalCommandFence { + kind: K1PhysicalConfirmationKind; + commandDeviceId: string; + commandProjectName: string; + acquisitionId: string; + runtimeId: string | null; + runtimePhase: string | null; + runtimeSourceMode: string | null; + selectedDeviceId: string | null; + deviceRefId: string | null; + deviceSessionId: string | null; + deviceSessionDeviceId: string | null; + deviceSessionConnectivity: string | null; + connectionIntentId: string | null; + requestedConnectionMode: XgridsConnectionMode | null; + expectedDeviceId: string | null; + deviceNetworkState: string | null; + deviceNetworkIntentId: string | null; + transportRef: string | null; + connectionMode: XgridsConnectionMode | null; + targetIpv4: string | null; + targetPort: number | null; + hostPathEpoch: number | null; + hostPathAvailable: boolean | null; + deviceIdentityState: string | null; + deviceIdentityId: string | null; + controlPlaneState: string | null; + controlPlaneSessionId: string | null; + dataPlaneState: string | null; + dataPlaneSessionId: string | null; + leaseState: string | null; + leaseGeneration: number | null; + controlAllowed: boolean | null; + acquisitionStartAllowed: boolean | null; + dataIngestAuthoritative: boolean | null; + controlSessionGeneration: number | null; + controlStateRevision: number | null; + controlState: string | null; + controlSocketOpen: boolean | null; + verifiedControlSessionId: string | null; + controlProofRevision: number | null; + controlProofFresh: boolean | null; + deviceReportedState: string | null; + deviceProjectBound: boolean | null; + deviceInitReady: boolean | null; + acquisitionState: string | null; + acquisitionStateRevision: number | null; + acquisitionDeviceId: string | null; + acquisitionDeviceSessionId: string | null; + acquisitionControlMode: string | null; +} + +export interface K1PhysicalCommandTarget { + deviceId: string; + connection: string; + projectName: string; + acquisitionId: string; + deviceState: string; + fence: K1PhysicalCommandFence; +} + +export interface K1PhysicalCommandCheckpoint { + readonly kind: K1PhysicalConfirmationKind; + readonly target: Readonly>; + readonly fence: Readonly; + readonly fenceKey: string; +} + +export interface K1PhysicalCommandConfirmationPayload { + readonly physicalAcceptance: Readonly; + readonly checkpoint: K1PhysicalCommandCheckpoint; +} + +export function emptyPhysicalConfirmationChecks(): PhysicalConfirmationChecks { + return { + operatorPresent: false, + ownerControlledDevice: false, + lixelgoClosed: false, + batteryStorageConfirmed: false, + expectedPhysicalStateConfirmed: false, + }; +} + +export function physicalConfirmationComplete( + checks: PhysicalConfirmationChecks, +): checks is CompletedPhysicalConfirmationChecks { + return ( + checks.operatorPresent + && checks.ownerControlledDevice + && checks.lixelgoClosed + && checks.batteryStorageConfirmed + && checks.expectedPhysicalStateConfirmed + ); +} + +export function operatorPresenceConfirmation( + checks: PhysicalConfirmationChecks, +): OperatorPresenceConfirmation | null { + if (!physicalConfirmationComplete(checks)) return null; + return { + operator_present: checks.operatorPresent, + owner_controlled_device: checks.ownerControlledDevice, + lixelgo_closed: checks.lixelgoClosed, + battery_storage_confirmed: checks.batteryStorageConfirmed, + expected_physical_state_confirmed: checks.expectedPhysicalStateConfirmed, + }; +} + +/** + * One deliberate click on the local K1 START/STOP action is the operator's + * physical acceptance. The backend still validates the exact control CAS, + * live DeviceInfo/status binding and command ledger before a vendor write; + * this helper only removes the redundant five-checkbox modal. + */ +export function operatorActionPhysicalAcceptance(): OperatorPresenceConfirmation { + return { + operator_present: true, + owner_controlled_device: true, + lixelgo_closed: true, + battery_storage_confirmed: true, + expected_physical_state_confirmed: true, + }; +} + +function recordValue( + record: Record | null | undefined, + key: string, +): unknown { + return record?.[key]; +} + +function trimmed(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function integer(value: unknown): number | null { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 + ? value + : null; +} + +function boolean(value: unknown): boolean | null { + return typeof value === "boolean" ? value : null; +} + +function commandFence( + kind: K1PhysicalConfirmationKind, + state: XgridsK1State | null | undefined, + target: Omit, +): K1PhysicalCommandFence { + const supervisor = state?.connection_supervisor; + const deviceNetwork = supervisor?.observed.device_network; + const hostPath = supervisor?.observed.host_path; + const deviceIdentity = supervisor?.observed.device_identity; + const controlPlane = supervisor?.observed.control_plane; + const dataPlane = supervisor?.observed.data_plane; + const control = state?.application_control_session; + const verifiedControl = control?.verified_control; + const acquisition = state?.acquisition; + + return { + kind, + commandDeviceId: target.deviceId, + commandProjectName: target.projectName, + acquisitionId: target.acquisitionId, + runtimeId: trimmed(state?.snapshot_runtime_id), + runtimePhase: trimmed(state?.phase), + runtimeSourceMode: trimmed(state?.source_mode), + selectedDeviceId: trimmed(state?.selected_device_id), + deviceRefId: trimmed(state?.device_ref?.device_id), + deviceSessionId: trimmed(state?.device_session?.device_session_id), + deviceSessionDeviceId: trimmed(state?.device_session?.device_id), + deviceSessionConnectivity: trimmed(state?.device_session?.connectivity), + connectionIntentId: trimmed(supervisor?.intent?.intent_id), + requestedConnectionMode: supervisor?.intent?.requested_mode ?? null, + expectedDeviceId: trimmed(supervisor?.intent?.expected_device_id), + deviceNetworkState: trimmed(deviceNetwork?.state), + deviceNetworkIntentId: trimmed(deviceNetwork?.intent_id), + transportRef: trimmed(deviceNetwork?.transport_ref), + connectionMode: deviceNetwork?.connection_mode ?? null, + targetIpv4: trimmed(deviceNetwork?.target?.ipv4), + targetPort: integer(deviceNetwork?.target?.port), + hostPathEpoch: integer(hostPath?.epoch), + hostPathAvailable: boolean(hostPath?.available), + deviceIdentityState: trimmed(deviceIdentity?.state), + deviceIdentityId: trimmed(deviceIdentity?.logical_device_id), + controlPlaneState: trimmed(controlPlane?.state), + controlPlaneSessionId: trimmed(controlPlane?.session_id), + dataPlaneState: trimmed(dataPlane?.state), + dataPlaneSessionId: trimmed(dataPlane?.session_id), + leaseState: trimmed(supervisor?.lease.state), + leaseGeneration: integer(supervisor?.lease.generation), + controlAllowed: boolean(supervisor?.authority.control_allowed), + acquisitionStartAllowed: boolean(supervisor?.authority.acquisition_start_allowed), + dataIngestAuthoritative: boolean(supervisor?.authority.data_ingest_authoritative), + controlSessionGeneration: integer(control?.session_generation), + controlStateRevision: integer(control?.state_revision), + controlState: trimmed(control?.state), + controlSocketOpen: boolean(control?.control_socket_open), + verifiedControlSessionId: trimmed(verifiedControl?.control_session_id), + controlProofRevision: integer(verifiedControl?.control_proof_revision), + controlProofFresh: boolean(verifiedControl?.control_proof_fresh), + deviceReportedState: trimmed(recordValue(control?.transport, "latest_device_session_state")), + deviceProjectBound: boolean(recordValue(control?.transport, "latest_device_project_bound")), + deviceInitReady: boolean(recordValue(control?.transport, "latest_device_init_ready")), + acquisitionState: trimmed(acquisition?.state), + acquisitionStateRevision: integer(acquisition?.state_revision), + acquisitionDeviceId: trimmed(acquisition?.device_id), + acquisitionDeviceSessionId: trimmed(acquisition?.device_session_id), + acquisitionControlMode: trimmed(acquisition?.control_mode), + }; +} + +export function physicalCommandFenceKey( + kind: K1PhysicalConfirmationKind, + target: K1PhysicalCommandTarget, +): string { + // Both values are included. This makes a mismatched component kind fail + // closed even if a caller accidentally supplies a target built for another + // physical command. + return JSON.stringify([ + kind, + target.deviceId, + target.connection, + target.projectName, + target.acquisitionId, + target.deviceState, + target.fence, + ]); +} + +export function createPhysicalCommandCheckpoint( + kind: K1PhysicalConfirmationKind, + target: K1PhysicalCommandTarget, +): K1PhysicalCommandCheckpoint { + const fence = Object.freeze({ ...target.fence }); + const targetSnapshot = Object.freeze({ + deviceId: target.deviceId, + connection: target.connection, + projectName: target.projectName, + acquisitionId: target.acquisitionId, + deviceState: target.deviceState, + }); + return Object.freeze({ + kind, + target: targetSnapshot, + fence, + fenceKey: physicalCommandFenceKey(kind, target), + }); +} + +export function physicalCommandCheckpointMatches( + checkpoint: K1PhysicalCommandCheckpoint, + kind: K1PhysicalConfirmationKind, + target: K1PhysicalCommandTarget, +): boolean { + return checkpoint.kind === kind + && checkpoint.fence.kind === kind + && target.fence.kind === kind + && checkpoint.fenceKey === physicalCommandFenceKey(kind, target); +} + +function targetWithFence( + kind: K1PhysicalConfirmationKind, + state: XgridsK1State | null | undefined, + target: Omit, +): K1PhysicalCommandTarget { + return { + ...target, + fence: commandFence(kind, state, target), + }; +} + +function exactReadyState( + control: XgridsApplicationControlSession, +): string | null { + const deviceState = trimmed(recordValue(control.transport, "latest_device_session_state")); + const projectBound = recordValue(control.transport, "latest_device_project_bound"); + const initReady = recordValue(control.transport, "latest_device_init_ready"); + if (deviceState !== "ready" || projectBound !== true || initReady !== false) return null; + return "READY · проект привязан · инициализация не запущена"; +} + +function exactScanningState( + control: XgridsApplicationControlSession | null | undefined, +): string { + const deviceState = trimmed(recordValue(control?.transport, "latest_device_session_state")); + const projectBound = recordValue(control?.transport, "latest_device_project_bound"); + const initReady = recordValue(control?.transport, "latest_device_init_ready"); + if (deviceState === "scanning" && projectBound === true && initReady === true) { + return "SCANNING · проект привязан · инициализация завершена"; + } + return deviceState + ? `${deviceState.toUpperCase()} · последнее подтверждённое состояние K1` + : "Состояние K1 не подтверждено текущим управляющим каналом"; +} + +function exactConnection( + control: XgridsApplicationControlSession | null | undefined, +): string | null { + const verified = control?.verified_control; + if (!verified) return null; + return `${verified.connection_mode} · ${verified.target_ipv4}:${verified.target_port}`; +} + +function acquisitionProject(acquisition: XgridsAcquisition): string { + return trimmed(acquisition.project_name) ?? "Проект без опубликованного имени"; +} + +export function preparedStartTarget( + state: XgridsK1State | null | undefined, +): K1PhysicalCommandTarget | null { + const acquisition = state?.acquisition; + const control = state?.application_control_session; + const verified = control?.verified_control; + const topology = currentAppliedConnectionTopology(state); + const supervisor = state?.connection_supervisor; + const deviceNetwork = supervisor?.observed.device_network; + const hostPath = supervisor?.observed.host_path; + const readyState = control ? exactReadyState(control) : null; + if ( + !acquisition + || acquisition.state !== "prepared" + || acquisition.control_mode !== "plugin-commanded" + || !control + || control.state !== "project-ready" + || control.can_start !== true + || !verified + || verified.control_proof_fresh !== true + || verified.logical_device_id !== acquisition.device_id + || verified.compatibility_profile_id !== acquisition.compatibility_profile_id + || supervisor?.authority.acquisition_start_allowed !== true + || verified.intent_id !== supervisor.intent?.intent_id + || verified.host_path_epoch !== hostPath?.epoch + || verified.transport_ref !== deviceNetwork?.transport_ref + || verified.connection_mode !== deviceNetwork?.connection_mode + || verified.target_ipv4 !== deviceNetwork?.target?.ipv4 + || verified.target_port !== deviceNetwork?.target?.port + || topology?.status !== "active" + || topology.connectionMode !== verified.connection_mode + || topology.endpoint !== verified.target_ipv4 + || state?.connection_lifecycle?.ready_to_start !== true + || !readyState + ) { + return null; + } + return targetWithFence("start", state, { + deviceId: verified.logical_device_id, + connection: `${verified.connection_mode} · ${verified.target_ipv4}:${verified.target_port}`, + projectName: acquisitionProject(acquisition), + acquisitionId: acquisition.acquisition_id, + deviceState: readyState, + }); +} + +export function preparationTarget( + state: XgridsK1State | null | undefined, + projectName: string, +): K1PhysicalCommandTarget | null { + const topology = currentAppliedConnectionTopology(state); + const supervisor = state?.connection_supervisor; + const deviceNetwork = supervisor?.observed.device_network; + if ( + !topology + || !supervisor + || topology.status === "configured-offline" + || !deviceNetwork?.transport_ref + || !deviceNetwork.target + ) { + return null; + } + const logicalDeviceId = supervisor.observed.device_identity.logical_device_id + ?? supervisor.intent?.expected_device_id + ?? deviceNetwork.transport_ref; + return targetWithFence("prepare", state, { + deviceId: logicalDeviceId, + connection: `${topology.connectionMode} · ${deviceNetwork.target.ipv4}:${deviceNetwork.target.port}`, + projectName: projectName.trim(), + acquisitionId: "Будет создана подготовительным этапом; START пока недоступен", + deviceState: "Подготовка не начата · физический START не разрешён", + }); +} + +export function activeStopTarget( + state: XgridsK1State | null | undefined, +): K1PhysicalCommandTarget | null { + const acquisition = state?.acquisition; + if (!acquisition || !isSoftwareCommandedAcquisition(state)) return null; + const control = state?.application_control_session; + return targetWithFence("stop", state, { + deviceId: acquisition.device_id, + connection: exactConnection(control) ?? "Текущий управляющий канал не подтверждён", + projectName: acquisitionProject(acquisition), + acquisitionId: acquisition.acquisition_id, + deviceState: exactScanningState(control), + }); +} diff --git a/plugins/xgrids-k1/frontend/src/presentation.ts b/plugins/xgrids-k1/frontend/src/presentation.ts index 7611d2f..6b3c559 100644 --- a/plugins/xgrids-k1/frontend/src/presentation.ts +++ b/plugins/xgrids-k1/frontend/src/presentation.ts @@ -1,7 +1,245 @@ import type { StatusTone } from "@nodedc/ui-react"; import type { BackendStatus } from "@mission-core/plugin-sdk"; -import type { XgridsK1Metrics } from "./api"; +import type { + XgridsConnectionPolicyAction, + XgridsK1Metrics, + XgridsK1State, +} from "./api"; +import { + connectionPolicyAllows, + connectionPolicyDecision, +} from "./lifecycle"; + +const connectionPolicyReasonCopy: Record = { + "connection-supervisor-closed": "Контур связи K1 закрыт.", + "supervisor-action-not-allowed": "Текущая связь с K1 не подтверждает право на эту физическую команду.", + "network-provision-operation-active": "Предыдущая сетевая операция K1 ещё не завершена.", + "acquisition-active": "Сетевой режим K1 нельзя менять во время активного приёма.", + "acquisition-cleanup-pending": "Локальный приём K1 ещё завершает очистку ресурсов.", + "local-runtime-active": "Локальный исполнительный контур K1 ещё активен.", + "control-session-not-admissible-for-network-change": "Текущая управляющая сессия K1 ещё не допускает смену сети.", + "network-mutation-reconciliation-required": "Результат предыдущей сетевой записи K1 не подтверждён.", + "network-mutation-ledger-corrupt": "Журнал сетевых изменений K1 повреждён.", + "fresh-ble-candidate-required": "Для этого действия нужен K1 из нового Bluetooth-поиска.", + "retained-recovery-context-unavailable": "Сохранённый Bluetooth-контекст текущего K1 больше недоступен.", + "fresh-candidate-supersedes-retained-recovery": "K1 снова виден в свежем поиске; используйте новый найденный экземпляр.", + "reconciliation-target-not-observed": "В свежем Bluetooth-поиске не найден K1, связанный с незавершённой записью.", + "reconciliation-target-not-retained": "Текущий серверный Bluetooth-контекст относится не к тому K1, для которого не завершена сетевая операция.", + "current-device-context-unavailable": "Текущий K1 не подтверждён в оперативном контексте этого процесса.", + "durable-recovery-target-unavailable": "В сохранённом серверном состоянии нет одной точной пары K1 и режима для проверки после перезапуска.", + "fresh-candidate-supersedes-durable-recovery": "Нужный K1 снова найден свежим Bluetooth-поиском; сервер требует проверить именно свежий экземпляр.", + "retained-recovery-supersedes-durable-recovery": "Сервер хранит более свежий контекст текущего K1.", + "configured-endpoint-unavailable": "Нет подтверждённого сохранённого адреса K1 для безопасной проверки.", + "configured-endpoint-probe-lifecycle-busy": "Контур подключения K1 занят другой операцией.", + "configured-endpoint-changed-during-probe": "Адрес K1 изменился во время проверки; результат отброшен.", + "configured-endpoint-probe-failed": "Маршрут и управляющий endpoint K1 не удалось проверить.", + "host-path-unavailable": "На этом компьютере не подтверждён сетевой путь до K1.", + "host-route-not-direct": "Маршрут до K1 проходит не через ожидаемую локальную сеть.", + "endpoint-not-reachable": "Управляющий endpoint K1 сейчас недоступен.", + "device-identity-unverified": "Идентичность подключённого K1 ещё не подтверждена.", + "device-identity-stale": "Подтверждение идентичности K1 устарело.", + "device-identity-mismatch": "Подключённое устройство не совпало с выбранным K1.", + "device-identity-pin-store-corrupt": "Хранилище привязки устройства повреждено.", + "network-provisioning-idempotency-unavailable": "Журнал сетевых намерений K1 недоступен.", + "network-provisioning-idempotency-corrupt": "Журнал сетевых намерений K1 повреждён.", + "network-provisioning-idempotency-invalid": "Журнал сетевых намерений K1 не прошёл проверку.", + "network-provisioning-idempotency-operation-mismatch": "Текущая сетевая операция не совпала с сохранённым намерением.", + "semantic-topology-store-corrupt": "Сохранённая топология K1 повреждена.", + "control-plane-not-healthy": "Управляющий канал K1 не подтверждён.", + "connection-lease-not-reachable": "Текущая сессия связи K1 больше не подтверждена.", + "data-plane-stalled": "Поток данных K1 перестал обновляться.", + "data-plane-lost": "Поток данных K1 потерян.", + "physical-command-reconciliation-required": "Результат предыдущей физической команды K1 не подтверждён.", + "physical-device-already-active": "Последнее подтверждённое состояние K1 — активное сканирование.", + "physical-command-recovery-target-unavailable": "Журнал не содержит точную Bluetooth-цель для восстановления K1.", + "physical-command-recovery-target-not-observed": "Исходный K1 пока не найден в свежем Bluetooth-поиске.", + "physical-command-recovery-target-not-retained": "Сохранённый Bluetooth-контекст относится не к исходному K1.", + "physical-command-recovery-target-mismatch": "Выбрано другое устройство или другой способ связи, чем в незавершённой физической сессии.", + "physical-command-ledger-corrupt": "Журнал физических команд K1 повреждён.", + "physical-command-ledger-unavailable": "Журнал физических команд K1 недоступен.", + "physical-control-authority-unavailable": "Управляющая связь не позволяет безопасно отправить физический STOP.", + "ble-runtime-restart-required": "BLE-контур требует контролируемого перезапуска.", + "ble-runtime-cleanup-pending": "BLE-контур завершает предыдущую операцию.", + "ble-runtime-busy": "BLE-контур занят другой операцией.", + "k1-lifecycle-process-lease-network-owned": "Сетевой переход K1 ещё владеет исполнительным контуром.", + "k1-lifecycle-process-lease-control-owned": "Управляющая сессия K1 ещё владеет исполнительным контуром.", + "local-acquisition-receiver-not-active": "Активного локального приёмника сейчас нет.", + "physical-stop-is-authoritative": "Доступна подтверждённая физическая остановка K1; локальная очистка не должна её подменять.", + "action-not-implemented": "Это действие не реализовано и не может быть выполнено.", + "connection-reconfiguration-lifecycle-busy": "Другое действие подключения ещё не завершено.", + "connection-reconfiguration-bridge-only": "Это действие доступно только для подключения Bridge.", + "connection-reconfiguration-current-device-unavailable": "Нет точной привязки текущего устройства для изменения сети.", + "connection-reconfiguration-active": "Сначала завершите или отмените текущий выбор устройства или сети.", + "connection-reconfiguration-required-device-not-observed": "Исходное устройство не найдено в текущем Bluetooth-поиске.", + "connection-reconfiguration-not-active": "Активного изменения устройства или сети уже нет.", + "connection-reconfiguration-process-lease-busy": "Другой локальный процесс ещё управляет подключением устройства.", + "connection-reconfiguration-acquisition-changed": "Состояние приёма изменилось во время подготовки подключения.", + "connection-reconfiguration-revision-conflict": "Выбор устройства или сети уже изменился в другой вкладке.", + "connection-reconfiguration-binding-conflict": "Активное подключение изменилось до выполнения действия.", + "connection-reconfiguration-fresh-scan-required": "Для этого действия нужен новый Bluetooth-поиск.", + "connection-reconfiguration-discovery-conflict": "Результаты Bluetooth-поиска относятся к предыдущему действию.", + "connection-reconfiguration-target-mismatch": "Изменение сети разрешено только для исходного устройства.", + "acquisition-start-operation-active": "Запуск приёма ещё не завершён.", + "control-session-state-unsafe": "Управляющий диалог ещё не достиг безопасного состояния ожидания.", +}; + +const connectionPolicyNextActionCopy: Record = { + "wait-for-operation": "Дождитесь завершения текущей операции и обновите состояние.", + "diagnose-network-ledger": "Не отправляйте новые команды и проверьте журнал сетевой операции.", + "diagnose-physical-command-ledger": "Не повторяйте команду; сначала проверьте журнал физических команд.", + "restart-ble-runtime": "Контролируемо перезапустите локальный BLE-контур и обновите состояние.", + "scan-ble": "Выполните свежий поиск Bluetooth-устройств.", + "observe-fresh-device-network": "Дождитесь автоматического восстановления связи с выбранным K1.", + "observe-current-device-network": "Дождитесь автоматического восстановления связи с тем же K1.", + "observe-configured-device-network": "Дождитесь автоматического восстановления сохранённого подключения K1.", + "recover-current-device-network": "Нажмите «Подключиться заново».", + "inspect-host-network": "Проверьте активную локальную сеть и маршрут этого компьютера.", + "probe-endpoint": "Дождитесь обновления подключения K1.", + "verify-control-device-info": "Подключитесь заново к выбранному K1.", + "select-connection-intent": "Выберите способ подключения и заново подтвердите текущий K1.", + "start-acquisition": "Повторно откройте финальное подтверждение физического START.", + "stop-acquisition": "Остановите K1 через подтверждённую физическую остановку.", + "stop-local-receiver": "Завершите локальный приём; физическое состояние K1 проверьте вручную.", + "retire-unavailable-physical-target": "Явно исключите недоступный прежний K1 перед новым выбором.", + "manual-recovery-required": "Автоматически безопасного продолжения нет; проверьте состояние K1 вручную.", + "cancel-reconfiguration": "Отмените текущий выбор и вернитесь к обычному восстановлению подключения.", +}; + +export interface ConnectionPolicyOperatorGuidance { + reason: string; + nextAction: string; +} + +const connectionModeSelectionReasonCopy: Record = { + "connection-mode-selection-physical-state-unsafe": + "Предыдущая физическая команда K1 осталась без подтверждённого результата. Поэтому способ подключения пока нельзя изменить.", + "connection-mode-selection-control-state-unsafe": + "Текущий управляющий процесс K1 ещё не завершён. После его завершения способ подключения снова станет доступен.", + "connection-mode-selection-lifecycle-busy": + "Текущее действие подключения ещё завершается. После него способ подключения снова станет доступен.", + "connection-reconfiguration-active": + "Сначала завершите или отмените текущий выбор устройства или сети.", +}; + +const physicalRetirementReasonCopy: Record = { + "physical-command-retirement-operation-conflict": + "Сейчас завершается другая операция с физическим состоянием устройства. Дождитесь её завершения и обновите состояние.", + "physical-command-retirement-state-unsafe": + "Состояние предыдущей физической команды изменилось. Обновите состояние перед новым выбором устройства.", + "physical-command-retirement-not-required": + "Предыдущая физическая команда уже разрешена или больше не удерживает выбор устройства. Обновите состояние и продолжите обычное подключение.", + "physical-command-target-retired": + "Предыдущее устройство уже выведено из текущего контура. Можно сразу выполнить новый явный Bluetooth-поиск.", + "network-provision-operation-active": + "Сейчас завершается подключение устройства к сети. Дождитесь результата перед выбором другого устройства.", + "connection-reconfiguration-active": + "Сначала завершите или отмените текущее изменение устройства или сети.", + "control-local-retirement-pending": + "Управляющая сессия ещё освобождает локальные ресурсы. Дождитесь завершения и обновите состояние.", + "acquisition-active": + "Сканирование ещё активно. Сначала остановите его и дождитесь подтверждённого завершения.", + "acquisition-cleanup-pending": + "Локальный приём ещё освобождает ресурсы после остановки. Дождитесь завершения.", + "acquisition-start-operation-active": + "Запуск сканирования ещё не завершён. Дождитесь его результата перед сменой устройства.", + "acquisition-stop-operation-active": + "Остановка сканирования ещё не завершена. Дождитесь её результата перед сменой устройства.", + "local-runtime-active": + "Локальный поток устройства ещё активен или завершается. Дождитесь перехода в состояние ожидания.", + "control-session-state-unsafe": + "Управляющая сессия устройства ещё не завершена. Дождитесь её закрытия и обновите состояние.", + "ble-runtime-busy": + "Bluetooth занят другой операцией устройства. Дождитесь её завершения; новый поиск автоматически не запустится.", + "ble-runtime-cleanup-pending": + "Bluetooth ещё завершает предыдущую операцию. Дождитесь освобождения соединения и обновите состояние.", + "ble-runtime-restart-required": + "Локальный Bluetooth-контур требует контролируемого перезапуска. Команды устройству не отправлялись.", + "k1-lifecycle-process-lease-active": + "Другой локальный процесс ещё завершает действие с устройством. Дождитесь его завершения и обновите состояние.", + "physical-command-ledger-corrupt": + "Журнал физической команды повреждён. Не выбирайте другое устройство до проверки журнала.", +}; + +const physicalReopenReasonCopy: Record = { + ...physicalRetirementReasonCopy, + "physical-command-reconciliation-reopen-not-required": + "Это устройство больше не требует возврата из предыдущего выбора. Обновите состояние и продолжите обычное подключение.", + "physical-command-reconciliation-reopen-target-not-observed": + "Предыдущее устройство не найдено в последнем Bluetooth-поиске. Обновите поиск после проверки питания K1.", + "physical-command-reconciliation-reopen-target-not-connectable": + "Предыдущее устройство найдено, но сейчас не принимает Bluetooth-подключение. Проверьте питание K1 и повторите явный поиск.", + "physical-command-reconciliation-reopen-candidate-ambiguous": + "Последний Bluetooth-поиск не подтвердил один точный экземпляр предыдущего K1. Повторите поиск рядом только с нужным устройством.", + "physical-command-reconciliation-reopen-operation-conflict": + "Другая операция уже меняет локальное состояние предыдущего устройства. Дождитесь её завершения и обновите поиск.", + "device-calibration-read-active": + "Сейчас читается калибровка устройства. Дождитесь завершения проверки перед повторным использованием K1.", +}; + +/** Human explanation for a backend-disabled topology selector. */ +export function connectionModeSelectionGuidance( + state: XgridsK1State | null | undefined, +): string | null { + const selection = state?.connection_lifecycle?.mode_selection; + if (!selection) return null; + if (selection?.allowed === true) return null; + const reasonCode = selection?.reason_codes.find((code) => code.trim().length > 0); + return reasonCode + ? connectionModeSelectionReasonCopy[reasonCode] + ?? "Способ подключения пока недоступен, потому что состояние K1 не позволяет безопасно изменить его." + : "Способ подключения пока недоступен, потому что состояние K1 не позволяет безопасно изменить его." +} + +/** Human explanation for a currently unavailable local-only device escape. */ +export function physicalRetirementGuidance( + state: XgridsK1State | null | undefined, +): string | null { + const retirement = ( + state?.physical_command + ?? state?.application_control_session?.physical_command + ?? null + )?.operator_retirement; + if (!retirement || retirement.allowed === true) return null; + const reasonCode = retirement.reason_codes.find((code) => code.trim().length > 0); + return reasonCode + ? physicalRetirementReasonCopy[reasonCode] + ?? "Выбор другого устройства пока небезопасен. Обновите состояние после завершения текущей операции." + : "Выбор другого устройства пока небезопасен. Обновите состояние после завершения текущей операции."; +} + +/** Human explanation for a backend-disabled exact retired-device reopen. */ +export function physicalReopenGuidance( + state: XgridsK1State | null | undefined, +): string | null { + const reopen = state?.physical_command?.operator_reconciliation_reopen; + if (!reopen || reopen.allowed === true) return null; + const reasonCode = reopen.reason_codes.find((code) => code.trim().length > 0); + return reasonCode + ? physicalReopenReasonCopy[reasonCode] + ?? "Повторная проверка предыдущего K1 сейчас небезопасна. Дождитесь завершения текущей операции и обновите поиск." + : "Повторная проверка предыдущего K1 сейчас небезопасна. Обновите поиск после завершения текущей операции."; +} + +export function connectionPolicyOperatorGuidance( + state: XgridsK1State | null | undefined, + action: XgridsConnectionPolicyAction, +): ConnectionPolicyOperatorGuidance | null { + if (connectionPolicyAllows(state, action)) return null; + const decision = connectionPolicyDecision(state, action); + const reasonCode = decision?.reason_codes.find((code) => code.trim().length > 0); + const recommendedAction = state?.connection_policy?.recommended_action?.trim(); + return { + reason: reasonCode + ? connectionPolicyReasonCopy[reasonCode] + ?? "Система временно запретила действие до восстановления подтверждённого состояния." + : "Подтверждённая политика действия ещё не получена.", + nextAction: recommendedAction + ? connectionPolicyNextActionCopy[recommendedAction] + ?? "Обновите состояние подключения и следуйте рекомендованному безопасному действию." + : "Обновите состояние подключения перед новым действием.", + }; +} const phaseLabels: Record = { idle: "Ожидание", diff --git a/plugins/xgrids-k1/frontend/src/projectName.ts b/plugins/xgrids-k1/frontend/src/projectName.ts index e1b3f16..9e850fd 100644 --- a/plugins/xgrids-k1/frontend/src/projectName.ts +++ b/plugins/xgrids-k1/frontend/src/projectName.ts @@ -30,3 +30,24 @@ export function validateProjectName(input: string): ProjectNameValidation { } return { value, error: null }; } + +export function projectNameAfterConnectionModeSelection( + preparedProjectName: string | null | undefined, +): string { + return preparedProjectName ?? ""; +} + +export function shouldHydratePreparedProject({ + acquisitionId, + hydratedAcquisitionId, + modeSwitchRequired, +}: { + acquisitionId: string | null; + hydratedAcquisitionId: string | null; + modeSwitchRequired: boolean; +}): boolean { + return Boolean( + acquisitionId + && !(hydratedAcquisitionId === acquisitionId && modeSwitchRequired), + ); +} diff --git a/plugins/xgrids-k1/frontend/src/runtimeContext.tsx b/plugins/xgrids-k1/frontend/src/runtimeContext.tsx index 9c3322f..282ec26 100644 --- a/plugins/xgrids-k1/frontend/src/runtimeContext.tsx +++ b/plugins/xgrids-k1/frontend/src/runtimeContext.tsx @@ -8,11 +8,19 @@ import { type MissionRuntimeState, } from "@mission-core/plugin-sdk"; import { + activeConnectionEndpointLabel, + canonicalDeviceConnectivity, confirmedRuntimeSourceMode, effectiveAcquisition, + hasAuthoritativeData, + hasControlAuthority, normalizeRuntimePhase, spatialSourceId, } from "./lifecycle"; +import { + activeStreamRecoveryOwnsPresentationDecision, + activeStreamRecoveryPresentationAuthority, +} from "./activeStreamRecovery"; import { localizeRuntimeMessage } from "./messages"; import { xgridsK1Manifest } from "./manifest"; import { deviceTelemetry, finiteMetric, pipelineLatency } from "./presentation"; @@ -23,13 +31,21 @@ export type XgridsK1Controller = ReturnType; const XgridsK1RuntimeContext = createContext(null); -function normalizeState( - controller: XgridsK1Controller, +export function normalizeXgridsK1MissionState( + controller: Pick, activeModel: DeviceModelDefinition, ): MissionRuntimeState | null { const state = controller.state; if (!state) return null; - const metrics = state.metrics; + const controlAuthoritative = hasControlAuthority(state); + const recoveryPresentationAuthority = activeStreamRecoveryPresentationAuthority(state); + const recoveryOwnsPresentation = activeStreamRecoveryOwnsPresentationDecision(state); + const dataAuthoritative = hasAuthoritativeData(state) && !recoveryOwnsPresentation; + const recoveryPresentationAuthoritative = recoveryPresentationAuthority !== null; + const replayAuthoritative = state.source_mode === "replay"; + const metrics = replayAuthoritative || dataAuthoritative + ? state.metrics + : undefined; const telemetry = deviceTelemetry(metrics); const deviceRef = state.device_ref; const deviceSession = state.device_session; @@ -43,13 +59,13 @@ function normalizeState( return { phase: normalizeRuntimePhase(state), message: localizeRuntimeMessage(state.message), - activeDevice: deviceRef + activeDevice: deviceRef && controlAuthoritative ? { pluginId: xgridsK1Manifest.metadata.id, modelId: deviceRef.model_id || activeModel.id, displayName: activeModel.displayName, instanceId: deviceRef.device_id, - endpointLabel: state.k1_ip, + endpointLabel: activeConnectionEndpointLabel(state), } : null, deviceSession: deviceSession @@ -57,7 +73,7 @@ function normalizeState( sessionId: deviceSession.device_session_id, deviceId: deviceSession.device_id, compatibilityProfileId: deviceSession.compatibility_profile_id, - connectivity: deviceSession.connectivity, + connectivity: canonicalDeviceConnectivity(state), } : null, acquisition: acquisition @@ -80,7 +96,9 @@ function normalizeState( stageCode: operation.stage_code, messageCode: operation.message_code, })), - spatialSource: sourceUrl && resolvedSpatialSourceId + spatialSource: sourceUrl + && resolvedSpatialSourceId + && (replayAuthoritative || dataAuthoritative || recoveryPresentationAuthoritative) ? { id: resolvedSpatialSourceId, url: sourceUrl, @@ -102,7 +120,9 @@ function normalizeState( range: null, }, viewerSettings: state.viewer_settings, - sourceMode: confirmedRuntimeSourceMode(state), + sourceMode: recoveryPresentationAuthoritative + ? "live" + : confirmedRuntimeSourceMode(state), metrics: { publishedFrameCount: ( Number.isSafeInteger(metrics?.pcl_frames) && @@ -140,10 +160,13 @@ export function XgridsK1RuntimeProvider({ const inheritedRuntime = useMissionRuntime(); const controller = useXgridsK1Runtime(active); const missionRuntime: MissionRuntimeController = { - state: activeModel ? normalizeState(controller, activeModel) : null, + state: activeModel + ? normalizeXgridsK1MissionState(controller, activeModel) + : null, backendStatus: controller.backendStatus, pendingAction: controller.pendingAction, - refresh: controller.refresh, + refresh: () => controller.refresh().then(() => undefined), + resetConnectionScenario: controller.resetConnectionScenario, updateViewerSettings: controller.updateViewerSettings, setObservationSourceActive: controller.setObservationSourceActive, }; diff --git a/plugins/xgrids-k1/frontend/src/stateOrdering.ts b/plugins/xgrids-k1/frontend/src/stateOrdering.ts index 11c88d3..0d80178 100644 --- a/plugins/xgrids-k1/frontend/src/stateOrdering.ts +++ b/plugins/xgrids-k1/frontend/src/stateOrdering.ts @@ -10,6 +10,66 @@ function deviceSessionScope(state: XgridsK1State): string | null { return typeof sessionId === "string" && sessionId.trim() ? sessionId : null; } +interface RuntimeSnapshotStamp { + startedAtMonotonicNs: bigint | null; + startedAtEpochMs: number | null; + runtimeId: string; + revision: number; +} + +function monotonicNanoseconds(value: string | null | undefined): bigint | null { + if (typeof value !== "string" || !/^(0|[1-9][0-9]*)$/.test(value)) return null; + try { + return BigInt(value); + } catch { + return null; + } +} + +function runtimeSnapshotStamp(state: XgridsK1State): RuntimeSnapshotStamp | null { + const startedAt = state.snapshot_runtime_started_at_utc; + const startedAtMonotonicNs = monotonicNanoseconds( + state.snapshot_runtime_started_monotonic_ns, + ); + const runtimeId = state.snapshot_runtime_id; + const revision = monotonicInteger(state.snapshot_revision); + if ( + typeof runtimeId !== "string" + || !runtimeId.trim() + || revision === null + ) { + return null; + } + const parsedEpochMs = typeof startedAt === "string" ? Date.parse(startedAt) : Number.NaN; + const startedAtEpochMs = Number.isFinite(parsedEpochMs) ? parsedEpochMs : null; + if (startedAtMonotonicNs === null && startedAtEpochMs === null) return null; + return { startedAtMonotonicNs, startedAtEpochMs, runtimeId, revision }; +} + +function stampedSnapshotIsAtLeastAsNew( + current: RuntimeSnapshotStamp, + incoming: RuntimeSnapshotStamp, +): boolean { + if (incoming.runtimeId === current.runtimeId) { + return incoming.revision >= current.revision; + } + if (incoming.startedAtMonotonicNs !== null || current.startedAtMonotonicNs !== null) { + if (incoming.startedAtMonotonicNs === null) return false; + if (current.startedAtMonotonicNs === null) return true; + return incoming.startedAtMonotonicNs > current.startedAtMonotonicNs; + } + if ( + incoming.startedAtEpochMs !== null + && current.startedAtEpochMs !== null + && incoming.startedAtEpochMs !== current.startedAtEpochMs + ) { + return incoming.startedAtEpochMs > current.startedAtEpochMs; + } + // Legacy UTC-only process identities with equal timestamps cannot be + // ordered safely. Keep the already accepted authority. + return false; +} + function cameraSnapshotIsAtLeastAsNew( current: XgridsCameraPreviewState, incoming: XgridsCameraPreviewState, @@ -43,6 +103,17 @@ export function selectMonotonicXgridsState( current: XgridsK1State | null, incoming: XgridsK1State, ): XgridsK1State { + if (!current) return incoming; + const currentStamp = runtimeSnapshotStamp(current); + const incomingStamp = runtimeSnapshotStamp(incoming); + if (currentStamp || incomingStamp) { + if (!currentStamp) return incoming; + if (!incomingStamp) return current; + return stampedSnapshotIsAtLeastAsNew(currentStamp, incomingStamp) + ? incoming + : current; + } + if (current && deviceSessionScope(current) !== deviceSessionScope(incoming)) { return incoming; } diff --git a/plugins/xgrids-k1/frontend/src/styles.css b/plugins/xgrids-k1/frontend/src/styles.css index b23944d..ca037df 100644 --- a/plugins/xgrids-k1/frontend/src/styles.css +++ b/plugins/xgrids-k1/frontend/src/styles.css @@ -1,9 +1,27 @@ /* All selectors below are scoped to the XGRIDS frontend contribution. */ .xgrids-k1-plugin { +container: xgrids-k1 / inline-size; +width: 100%; +min-width: 0; +max-width: 100%; +box-sizing: border-box; + +*, +*::before, +*::after { + box-sizing: border-box; +} + +> * { + min-width: 0; + max-width: 100%; +} + .device-workspace__grid { display: grid; min-width: 0; - grid-template-columns: minmax(23rem, 0.78fr) minmax(34rem, 1.22fr); + max-width: 100%; + grid-template-columns: minmax(0, 1fr); align-items: start; gap: 0.85rem; } @@ -11,6 +29,7 @@ .device-workspace__side { display: grid; min-width: 0; + max-width: 100%; gap: 0.85rem; } @@ -18,9 +37,114 @@ .status-panel, .latency-panel, .session-panel { + min-width: 0; + max-width: 100%; background: var(--station-panel); } +/* Every layout hop between the plugin root and the canonical controls must be + shrinkable. A single auto min-size in this chain lets topology/status text + establish a wider intrinsic track and paint the provisioning job over the + acquisition job even though the outer grid itself uses minmax(0, 1fr). */ +.workspace-lead, +.metrics-grid, +.error-banner, +.wizard-list, +.wizard-step, +.field-stack, +.session-form, +.scan-configuration-grid, +.device-list, +.device-row, +.diagnostics-grid, +.detail-list { + min-width: 0; + max-width: 100%; +} + +.metrics-grid > *, +.device-workspace__grid > *, +.device-workspace__side > *, +.scan-configuration-grid > *, +.diagnostics-grid > * { + min-width: 0; + max-width: 100%; +} + +.error-banner > div { + min-width: 0; +} + +.error-banner__copy { + display: grid; + gap: 0.2rem; +} + +.error-banner--compact { + align-items: start; + padding-block: 0.7rem; +} + +.error-banner__recovery-actions { + display: grid; + gap: 0.45rem; + margin-top: 0.55rem; +} + +.error-banner__details { + margin-top: 0.35rem; + color: var(--nodedc-text-secondary); + font-size: 0.66rem; +} + +.error-banner__details summary { + width: fit-content; + color: var(--nodedc-text-tertiary); + cursor: pointer; +} + +.workspace-lead__status, +.workspace-lead__status > span, +.panel-heading > div, +.panel-heading h2, +.wizard-step__content, +.wizard-step__content > header, +.wizard-step__content > header h3, +.connection-summary span, +.nodedc-field__description, +.empty-device-list, +.retained-recovery-target small, +.session-footer p { + min-width: 0; +} + +.workspace-lead__status > span, +.panel-heading h2, +.wizard-step__content > header h3, +.connection-summary span, +.nodedc-field__description, +.empty-device-list, +.retained-recovery-target small, +.session-footer p { + overflow-wrap: anywhere; +} + +.workspace-lead p, +.error-banner p, +.step-copy, +.safety-note, +.live-instruction { + overflow-wrap: anywhere; +} + +.connection-panel { + container: k1-connection-panel / inline-size; +} + +.session-panel { + container: k1-session-panel / inline-size; +} + .error-banner { display: grid; grid-template-columns: auto minmax(0, 1fr) auto; @@ -58,12 +182,54 @@ line-height: 1.45; } +.error-banner__diagnostic { + display: grid; + min-width: 0; + max-width: 100%; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.65rem; + margin: 0.45rem 0 0; +} + +.error-banner__diagnostic > div { + min-width: 0; +} + +.error-banner__diagnostic dt, +.error-banner__diagnostic dd { + margin: 0; + overflow-wrap: anywhere; +} + +.error-banner__diagnostic dt { + color: var(--nodedc-text-tertiary); + font-size: 0.58rem; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.error-banner__diagnostic dd { + margin-top: 0.12rem; + color: var(--nodedc-text-secondary); + font-size: 0.66rem; + line-height: 1.4; +} + .error-banner__actions { display: flex; + min-width: 0; + max-width: 100%; align-items: center; + flex-wrap: wrap; gap: 0.35rem; } +.error-banner__actions > .nodedc-button { + min-width: 0; + max-width: 100%; + overflow-wrap: anywhere; +} + .wizard-list { display: grid; margin-top: 1.4rem; @@ -71,6 +237,8 @@ .configuration-anchor { display: grid; + min-width: 0; + max-width: 100%; gap: 0.55rem; margin-top: 1.2rem; border-radius: 0.95rem; @@ -81,6 +249,35 @@ .configuration-anchor .nodedc-select-anchor, .configuration-field .nodedc-select-anchor { width: 100%; + min-width: 0; + max-width: 100%; +} + +.connection-topology-summary { + display: grid; + min-width: 0; + max-width: 100%; + gap: 0.42rem; +} + +.connection-topology-summary .connection-summary { + min-width: 0; + max-width: 100%; + margin: 0; +} + +.connection-summary__value { + display: flex; + min-width: 0; + max-width: 100%; + align-items: center; + justify-content: flex-end; + gap: 0.5rem; +} + +.connection-summary__value strong { + min-width: 0; + max-width: 100%; } .wizard-step { @@ -219,19 +416,26 @@ } .device-row__identity strong { - overflow: hidden; + min-width: 0; + max-width: 100%; + overflow: visible; + overflow-wrap: anywhere; font-size: 0.69rem; - text-overflow: ellipsis; - white-space: nowrap; + text-overflow: clip; + white-space: normal; } .device-row code, .detail-row code { - overflow: hidden; + display: block; + min-width: 0; + max-width: 100%; + overflow: visible; + overflow-wrap: anywhere; color: var(--nodedc-text-muted); font-size: 0.58rem; - text-overflow: ellipsis; - white-space: nowrap; + text-overflow: clip; + white-space: normal; } .device-row__signal { @@ -242,7 +446,7 @@ background: var(--nodedc-text-muted); } -.device-row[data-compatible="true"] .device-row__signal { +.device-row[data-likely-k1="true"] .device-row__signal { background: rgb(var(--nodedc-success-rgb)); } @@ -260,14 +464,81 @@ line-height: 1.45; } +.retained-recovery-target { + display: grid; + min-width: 0; + gap: 0.42rem; + margin-top: 0.62rem; + border-radius: 0.95rem; + background: rgb(255 255 255 / 0.025); + padding: 0.78rem; +} + +.retained-recovery-target > div { + display: flex; + min-width: 0; + align-items: center; + justify-content: space-between; + gap: 0.55rem; +} + +.retained-recovery-target span, +.retained-recovery-target small { + color: var(--nodedc-text-muted); + font-size: 0.58rem; + line-height: 1.45; +} + +.retained-recovery-target code { + min-width: 0; + overflow-wrap: anywhere; + color: var(--nodedc-text-secondary); + font-size: 0.58rem; +} + .field-stack, .session-form { display: grid; gap: 0.85rem; } +.password-field-row { + display: grid; + min-width: 0; + grid-template-columns: minmax(0, 1fr) auto; + align-items: end; + gap: 0.55rem; +} + +.connection-recovery-choice { + display: grid; + min-width: 0; + max-width: 100%; + gap: 0.65rem; + border: 1px solid rgb(255 255 255 / 0.08); + border-radius: 0.95rem; + background: rgb(255 255 255 / 0.025); + padding: 0.85rem; +} + +.connection-recovery-choice > strong { + color: var(--nodedc-text-primary); + font-size: 0.72rem; + line-height: 1.4; +} + +.connection-recovery-choice > .safety-note { + margin: 0; +} + +.connection-recovery-choice--retirement { + border-color: rgb(var(--nodedc-danger-rgb) / 0.24); +} + .connection-summary { display: flex; + min-width: 0; + max-width: 100%; align-items: center; justify-content: space-between; gap: 1rem; @@ -283,6 +554,8 @@ } .connection-summary strong { + min-width: 0; + max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; @@ -296,6 +569,61 @@ margin-top: 1rem; } +.active-stream-recovery { + display: grid; + gap: 1rem; + margin-top: 1rem; +} + +.active-stream-recovery__state { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + align-items: start; + gap: 0.75rem; + border-radius: 0.85rem; + background: rgb(255 255 255 / 0.03); + padding: 0.85rem; +} + +.active-stream-recovery__state--static { + grid-template-columns: minmax(0, 1fr); +} + +.active-stream-recovery__state > .nodedc-activity-indicator { + margin-top: 0.12rem; +} + +.active-stream-recovery__copy { + display: grid; + min-width: 0; + gap: 0.28rem; +} + +.active-stream-recovery__copy strong { + color: var(--nodedc-text-primary); + font-size: 0.72rem; + line-height: 1.4; +} + +.active-stream-recovery__copy span, +.active-stream-recovery__copy small, +.active-stream-recovery__actions p { + margin: 0; + color: var(--nodedc-text-muted); + font-size: 0.62rem; + line-height: 1.5; + overflow-wrap: anywhere; +} + +.active-stream-recovery__copy small { + color: var(--nodedc-text-secondary); +} + +.active-stream-recovery__actions { + display: grid; + gap: 0.55rem; +} + .scan-configuration-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); @@ -375,11 +703,12 @@ } .detail-row dd { - overflow: hidden; + overflow: visible; + overflow-wrap: anywhere; color: var(--nodedc-text-secondary); text-align: right; - text-overflow: ellipsis; - white-space: nowrap; + text-overflow: clip; + white-space: normal; } .inline-state { @@ -449,25 +778,106 @@ } } -@media (max-width: 1480px) { +/* Keep both jobs usable before admitting the split composition: the + provisioning column retains 32 rem and the acquisition column 38 rem. + Below their combined working width the panels stack instead of squeezing + and letting intrinsic text paint into the neighbouring surface. */ +@container xgrids-k1 (min-width: 78rem) { .xgrids-k1-plugin .device-workspace__grid { - grid-template-columns: minmax(21rem, 0.76fr) minmax(30rem, 1.24fr); + grid-template-columns: minmax(32rem, 0.8fr) minmax(38rem, 1.2fr); } } -@media (max-width: 1280px) { - .xgrids-k1-plugin .device-workspace__grid { - grid-template-columns: 1fr; +@container k1-connection-panel (max-width: 48rem) { + .xgrids-k1-plugin .panel-heading, + .xgrids-k1-plugin .wizard-step__content > header { + min-width: 0; + align-items: flex-start; + flex-wrap: wrap; } + + .xgrids-k1-plugin .panel-heading > div, + .xgrids-k1-plugin .wizard-step__content > header h3 { + min-width: 0; + max-width: 100%; + overflow-wrap: anywhere; + } + + .xgrids-k1-plugin .panel-heading > .nodedc-status, + .xgrids-k1-plugin .wizard-step__content > header > .nodedc-status, + .xgrids-k1-plugin .connection-summary__value > .nodedc-status, + .xgrids-k1-plugin .retained-recovery-target .nodedc-status { + max-width: 100%; + flex: 0 1 auto; + line-height: 1.35; + text-align: left; + white-space: normal; + } + + .xgrids-k1-plugin .connection-summary, + .xgrids-k1-plugin .connection-summary--topology, + .xgrids-k1-plugin .connection-summary__value { + align-items: stretch; + flex-direction: column; + } + + .xgrids-k1-plugin .connection-summary__value { + justify-content: flex-start; + } + + .xgrids-k1-plugin .connection-summary strong { + overflow-wrap: anywhere; + text-overflow: clip; + white-space: normal; + } + + .xgrids-k1-plugin .device-row__action { + align-items: stretch; + flex-direction: column; + } + + .xgrids-k1-plugin .device-row__name { + flex-wrap: wrap; + } + + .xgrids-k1-plugin .device-row__name small { + flex: 1 1 100%; + } + + .xgrids-k1-plugin .retained-recovery-target > div { + align-items: flex-start; + flex-direction: column; + } + } -@media (max-width: 1040px) { +@container xgrids-k1 (max-width: 65rem) { .xgrids-k1-plugin .diagnostics-grid { grid-template-columns: 1fr; } } -@media (max-width: 760px) { +@container k1-session-panel (max-width: 48rem) { + .xgrids-k1-plugin .panel-heading { + min-width: 0; + align-items: flex-start; + flex-wrap: wrap; + } + + .xgrids-k1-plugin .panel-heading > div { + min-width: 0; + max-width: 100%; + overflow-wrap: anywhere; + } + + .xgrids-k1-plugin .panel-heading > .nodedc-status { + max-width: 100%; + flex: 0 1 auto; + line-height: 1.35; + text-align: left; + white-space: normal; + } + .xgrids-k1-plugin .scan-configuration-grid { grid-template-columns: 1fr; } @@ -480,12 +890,33 @@ grid-column: auto; } - .xgrids-k1-plugin .session-footer, - .xgrids-k1-plugin .error-banner { + .xgrids-k1-plugin .session-footer { align-items: stretch; - grid-template-columns: 1fr; flex-direction: column; } +} + +@container xgrids-k1 (max-width: 48rem) { + .xgrids-k1-plugin .workspace-lead, + .xgrids-k1-plugin .panel-heading, + .xgrids-k1-plugin .wizard-step__content > header { + min-width: 0; + align-items: flex-start; + flex-wrap: wrap; + } + + .xgrids-k1-plugin .workspace-lead > div, + .xgrids-k1-plugin .workspace-lead__status, + .xgrids-k1-plugin .panel-heading > div, + .xgrids-k1-plugin .wizard-step__content > header h3 { + min-width: 0; + max-width: 100%; + } + + .xgrids-k1-plugin .workspace-lead__status { + align-items: flex-start; + text-align: left; + } .xgrids-k1-plugin .error-banner { grid-template-columns: auto minmax(0, 1fr); @@ -493,12 +924,57 @@ .xgrids-k1-plugin .error-banner__actions { grid-column: 2; + min-width: 0; + flex-wrap: wrap; justify-content: flex-end; } - .xgrids-k1-plugin .device-row__action { + .xgrids-k1-plugin .error-banner__diagnostic { + grid-template-columns: 1fr; + gap: 0.4rem; + } + + .xgrids-k1-plugin .retained-recovery-target > div { + align-items: flex-start; + flex-direction: column; + } +} + +@container xgrids-k1 (max-width: 32rem) { + .xgrids-k1-plugin .wizard-step { + grid-template-columns: 1.75rem minmax(0, 1fr); + gap: 0.55rem; + } + + .xgrids-k1-plugin .wizard-step__rail span { + width: 1.75rem; + height: 1.75rem; + } + + .xgrids-k1-plugin .detail-row { + grid-template-columns: 1fr; + gap: 0.25rem; + } + + .xgrids-k1-plugin .detail-row dd { + overflow-wrap: anywhere; + text-align: left; + white-space: normal; + } + + .xgrids-k1-plugin .error-banner { + grid-template-columns: 1fr; + } + + .xgrids-k1-plugin .error-banner__dot { + display: none; + } + + .xgrids-k1-plugin .error-banner__actions { + grid-column: 1; align-items: stretch; flex-direction: column; + justify-content: flex-start; } } @@ -517,6 +993,57 @@ backdrop-filter: blur(20px); } +.xgrids-k1-spatial-controls--recovery { + display: grid; + min-width: min(46rem, 100%); + grid-template-columns: minmax(0, 1fr); + gap: 0.45rem; + padding: 0.65rem 0.75rem; +} + +.active-stream-recovery__compact-heading { + display: flex; + min-width: 0; + align-items: center; + justify-content: space-between; + gap: 0.75rem; +} + +.active-stream-recovery__compact-heading > span:first-child { + overflow: hidden; + color: var(--nodedc-text-muted); + font-size: 0.5rem; + font-weight: 650; + letter-spacing: 0.14em; + text-overflow: ellipsis; + white-space: nowrap; +} + +.active-stream-recovery--compact { + min-width: 0; + grid-template-columns: minmax(0, 1fr) auto; + align-items: center; + gap: 0.65rem; + margin-top: 0; +} + +.active-stream-recovery--compact .active-stream-recovery__state { + min-width: 0; + background: transparent; + padding: 0; +} + +.active-stream-recovery--compact .active-stream-recovery__actions { + max-width: 15rem; + grid-template-columns: auto; + gap: 0.25rem; +} + +.active-stream-recovery--compact .active-stream-recovery__actions p { + font-size: 0.5rem; + line-height: 1.35; +} + .xgrids-k1-spatial-controls__phase { display: flex; min-width: 11rem; @@ -547,16 +1074,6 @@ font-size: 0.53rem; } -.xgrids-k1-spatial-controls__spinner { - width: 0.82rem; - height: 0.82rem; - flex: 0 0 0.82rem; - border: 1px solid rgb(255 255 255 / 0.16); - border-top-color: var(--nodedc-text-primary); - border-radius: 50%; - animation: xgrids-k1-spin 900ms linear infinite; -} - .xgrids-k1-spatial-controls__telemetry { display: flex; flex: 0 1 auto; @@ -598,8 +1115,32 @@ white-space: nowrap; } -@keyframes xgrids-k1-spin { - to { transform: rotate(360deg); } +.xgrids-k1-spatial-controls__action-label { + display: block; + width: 9.75rem; + font-size: 0.66rem; + line-height: 1.08; + text-align: center; + white-space: normal; +} + +.xgrids-k1-spatial-controls__action-label--local { + width: 8.75rem; +} + +.connection-action-progress { + display: flex; + min-height: 2.75rem; + width: 100%; + align-items: center; + justify-content: center; + gap: 0.65rem; + color: var(--nodedc-text-secondary); +} + +.connection-action-progress strong { + font-size: 0.78rem; + font-weight: 600; } @media (max-width: 960px) { @@ -612,4 +1153,16 @@ .xgrids-k1-spatial-controls__error small { display: none; } + + .active-stream-recovery--compact { + grid-template-columns: minmax(0, 1fr); + } + + .active-stream-recovery--compact .active-stream-recovery__actions { + max-width: none; + } + + .active-stream-recovery--compact .active-stream-recovery__actions p { + display: none; + } } diff --git a/plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts b/plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts index 3e6f464..f0f432a 100644 --- a/plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts +++ b/plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts @@ -11,46 +11,238 @@ import { type EventSocketStatus, type OpenApplicationControlSessionRequest, type OperatorPresenceConfirmation, + type PrepareConnectionReconfigurationRequest, type PrepareAcquisitionRequest, + type ReopenRetiredPhysicalReconciliationRequest, type ReplayRequest, + type RetireUnavailablePhysicalCommandRequest, + type SelectConnectionModeRequest, type XgridsOperation, type XgridsApplicationControlPhase, + type XgridsConnectionMode, + type XgridsConnectionPolicyAction, + type XgridsHostFailureDiagnostic, type XgridsK1State, } from "./api"; import { - controlSessionEntryPlan, + acceptedBleSessionKeyAfterConnect, + authoritativeStateSupersedesPhysicalStopIntent, + bleSessionTargetForTransport, + connectionPolicyAllows, + currentAppliedConnectionTopology, + authoritativeStateSupersedesRuntimeError, + isRecoveredPhysicalScanning, + isProvenLocalReceiverInactive, isTerminalAcquisitionState, liveStartPlan, + localReceiverStopPlan, + newMutationContext, + newOperationId, + operationAllowsFreshProvisioningIntent, operationByIdempotencyKey, operationNeedsReconciliation, + physicalStopIntentCheckpoint, + recommendedConnectionRecoveryObservationTarget, + readOnlyVerificationClearedReconciliation, + requiresCanonicalStopAfterTerminalLocalFailure, isSoftwareCommandedAcquisition, + shouldSurfaceRuntimeActionError, + transportRefEquivalenceKey, + type PhysicalStopIntentCheckpoint, + type RuntimeErrorCorrelation, } from "./lifecycle"; import { localizeRuntimeMessage } from "./messages"; +import { + acquisitionMutationUsesControlSession, + exactAcquisitionControlCas, + exactApplicationControlCas, +} from "./controlSessionCas"; import { awaitWhileIntentCurrent, + isSnapshotRuntimeCurrent as snapshotRuntimeIdsMatch, OperatorIntentGeneration, } from "./operatorIntentGeneration"; import { selectMonotonicXgridsState } from "./stateOrdering"; +import { operationHostFailureDiagnostic } from "./hostDiagnosticPresentation"; +import { activeStreamForceFinishAuthority } from "./activeStreamRecovery"; +import { DEFAULT_CONNECTION_MODE } from "./configuration"; export type PendingAction = | "scan" + | "mode" + | "reconfigure" | "connect" | "verify" + | "retire" + | "reopen" + | "probe" | "control" | "live" | "replay" | "stop" + | "force-finish" | "abort" | "camera" | "viewer"; -export interface CanonicalLiveStartRequest { - control: OpenApplicationControlSessionRequest; - acquisition: PrepareAcquisitionRequest; - physicalAcceptance: OperatorPresenceConfirmation; +export interface RuntimeActionToken { + runtimeGeneration: number; + actionSequence: number; +} + +export class SnapshotRuntimeActionArbiter { + private current: RuntimeActionToken | null = null; + private sequence = 0; + + begin( + runtimeGeneration: number, + supersedeCurrent = false, + ): RuntimeActionToken | null { + if ( + this.current?.runtimeGeneration === runtimeGeneration + && !supersedeCurrent + ) return null; + this.sequence += 1; + this.current = { + runtimeGeneration, + actionSequence: this.sequence, + }; + return this.current; + } + + isCurrent(token: RuntimeActionToken): boolean { + return this.current?.runtimeGeneration === token.runtimeGeneration + && this.current.actionSequence === token.actionSequence; + } + + retireForSnapshotChange( + previousSnapshotRuntimeId: string | null, + acceptedSnapshotRuntimeId: string | null, + ): boolean { + if ( + !previousSnapshotRuntimeId + || !acceptedSnapshotRuntimeId + || previousSnapshotRuntimeId === acceptedSnapshotRuntimeId + ) return false; + this.current = null; + return true; + } + + settle(token: RuntimeActionToken): boolean { + if (!this.isCurrent(token)) return false; + this.current = null; + return true; + } +} + +export type AcquisitionPreparationDraft = Omit< + PrepareAcquisitionRequest, + | "operation_id" + | "idempotency_key" + | "expected_control_session_generation" + | "expected_control_state_revision" +>; + +export interface CanonicalLivePreparationRequest { + acquisition: AcquisitionPreparationDraft; +} + +export interface ProvisioningSubmitResult { + /** The exact Apply response was accepted, including applied-but-unready control. */ + succeeded: boolean; + /** The explicit network mutation completed even if the later control proof failed. */ + networkIntentCompleted: boolean; + intentDisposition: "retain" | "release"; + /** Exact journaled failure for this provisioning attempt. */ + failureReasonCode: string | null; + acceptedSessionKey: string | null; + observedState: XgridsK1State | null; +} + +export interface ConnectionVerificationSubmitResult { + succeeded: boolean; + reconciliationCompleted: boolean; + observedState: XgridsK1State | null; + /** Exact journaled failure for this Verify operation, never an unrelated last error. */ + reasonCode: string | null; +} + +export interface RuntimeActionOptions { + /** Keep a bounded background observation inside its owning surface. */ + surfaceErrors?: boolean; + /** Explicit BLE discovery window owned by the visible Scan action. */ + durationSeconds?: number; + /** Exact durable connection attempt owned by this action, if any. */ + connectionAttemptId?: () => string | null; + /** Retire an older UI callback; the backend remains the mutation authority. */ + supersedePending?: boolean; + /** + * Pin a composite UI action to the runtime rendered at its explicit click. + * The literal id is checked again at the dispatch boundary; it is never + * replaced with a newer runtime discovered while the action is settling. + */ + expectedSnapshotRuntimeId?: string; +} + +export interface ConnectionActionAuthoritySnapshot { + snapshotRuntimeId: string; + connectionMode: XgridsConnectionMode; + desiredModeRevision: number; + reconfigurationRevision: number; + reconfigurationIntentId: string | null; + activeBindingKey: string | null; + discoveryGeneration: number; +} + +export interface BleDiscoverySubmitResult { + succeeded: boolean; + snapshotRuntimeId: string | null; + discoveryGeneration: number | null; + transportRefs: readonly string[]; +} + +export interface ConnectionReconfigurationSubmitResult { + succeeded: boolean; + observedState: XgridsK1State | null; +} + +export function connectionActionAuthoritySnapshot( + state: XgridsK1State | null | undefined, + connectionMode: XgridsConnectionMode, +): ConnectionActionAuthoritySnapshot | null { + const snapshotRuntimeId = state?.snapshot_runtime_id?.trim() || null; + const desiredModeRevision = state?.desired_connection_mode_revision; + const discoveryGeneration = state?.ble_discovery_generation; + const reconfigurationRevision = state?.connection_reconfiguration?.revision ?? 0; + const reconfiguration = state?.connection_reconfiguration; + const reconfigurationIntentId = reconfiguration + && reconfiguration.intent !== null + && reconfiguration.status !== "idle" + ? reconfiguration.intent_id + : null; + if ( + !snapshotRuntimeId + || state?.desired_connection_mode !== connectionMode + || !Number.isInteger(desiredModeRevision) + || (desiredModeRevision ?? -1) < 0 + || !Number.isInteger(discoveryGeneration) + || (discoveryGeneration ?? -1) < 0 + || !Number.isInteger(reconfigurationRevision) + || reconfigurationRevision < 0 + ) return null; + return { + snapshotRuntimeId, + connectionMode, + desiredModeRevision: desiredModeRevision as number, + reconfigurationRevision, + reconfigurationIntentId, + activeBindingKey: state?.connection_lifecycle?.active_binding_key ?? null, + discoveryGeneration: discoveryGeneration as number, + }; } const CONTROL_STATE_READ_INTERVAL_MS = 250; +const CONTROL_PHASE_WAIT_TIMEOUT_MS = 35_000; function controlPhase(state: XgridsK1State): XgridsApplicationControlPhase { return state.application_control_session?.state ?? "idle"; @@ -93,7 +285,7 @@ function controlFailure(state: XgridsK1State): ApiError { response_rejected: "Сканер отклонил подготовительную операцию.", compatibility_profile_mismatch: - "Живой DeviceInfo не соответствует выбранному профилю модели, platform type, прошивки или активации.", + "Ответ подключённого устройства не соответствует выбранной модели K1, версии прошивки или состоянию активации.", scan_initialization_timeout: "После подтверждённого START сканер не завершил инициализацию в безопасный срок.", operation_reuse_forbidden: @@ -145,6 +337,9 @@ function controlFailure(state: XgridsK1State): ApiError { : ""; return new ApiError( `${reasonDetail || localizedDetail || "Канонический диалог K1 остановлен."} Этап: ${stage}. ${failedOperation} ${exchanges} ${commandStatus} ${diagnostics} ${retryStatus}`, + 0, + false, + failure?.host_diagnostic, ); } @@ -152,9 +347,16 @@ async function waitForControlPhase( expected: XgridsApplicationControlPhase, acceptState: (state: XgridsK1State) => void, assertOperatorIntentCurrent: () => void, + timeoutMs = CONTROL_PHASE_WAIT_TIMEOUT_MS, ): Promise { + const deadline = Date.now() + timeoutMs; for (;;) { assertOperatorIntentCurrent(); + if (Date.now() >= deadline) { + throw new ApiError( + `K1 не подтвердил этап «${expected}» за ${Math.ceil(timeoutMs / 1_000)} с. START не отправлялся.`, + ); + } const nextState = await awaitWhileIntentCurrent( assertOperatorIntentCurrent, () => xgridsK1Api.getState(), @@ -179,6 +381,137 @@ async function waitForControlPhase( } } +function hasExactConnectionReady( + state: XgridsK1State, + connectionMode: NonNullable, +): boolean { + return Boolean( + state.connection_lifecycle?.schema_version + === "missioncore.xgrids-k1-connection-lifecycle/v1" + && state.connection_lifecycle.connection_ready === true + && state.connection_lifecycle.desired_mode === connectionMode + && state.connection_lifecycle.configured_mode === connectionMode + && state.connection_lifecycle.active_mode === connectionMode + && state.desired_connection_mode === connectionMode + && state.active_connection_mode === connectionMode + && state.application_control_session?.state === "connection-ready" + && currentAppliedConnectionTopology(state, connectionMode)?.status === "active", + ); +} + +function requireExactConnectionReady( + state: XgridsK1State, + connectionMode: NonNullable, +): XgridsK1State { + if (hasExactConnectionReady(state, connectionMode)) return state; + if (state.application_control_session?.state === "failed") { + throw controlFailure(state); + } + throw new ApiError( + "Подключение к выбранному K1 не завершено. START не отправлялся; вернитесь в «Парк» и завершите подключение устройства.", + ); +} + +/** + * Prove that the exact Apply request crossed its one network-mutation + * boundary, even when the separate control bootstrap is not ready yet. + * This is intentionally stricter than `phase=network_applied` alone: the + * attempt, idempotent operation, durable ledger and BLE-observed target must + * all name the same operation, transport and mode. + */ +export function exactAppliedNetworkIntentCompleted( + state: XgridsK1State, + request: ConnectRequest, + operation: XgridsOperation | null | undefined, +): boolean { + const attempt = state.connection_attempt; + const ledger = state.network_mutation_ledger; + const deviceNetwork = state.connection_supervisor?.observed.device_network; + const operationMode = operation?.context + && typeof operation.context.connection_mode === "string" + ? operation.context.connection_mode + : null; + const operationPhase = operation?.result + && typeof operation.result.phase === "string" + ? operation.result.phase + : null; + const requestTransportKey = transportRefEquivalenceKey(request.device_id); + return Boolean( + attempt?.schema_version === "missioncore.xgrids-k1-connection-attempt/v1" + && attempt.phase === "network_applied" + && attempt.connection_mode === request.connection_mode + && operation?.action === "network.provision" + && operation.status === "succeeded" + && operation.idempotency_key === request.idempotency_key + && attempt.attempt_id === operation.operation_id + && operationMode === request.connection_mode + && operationPhase === "network_applied" + && ledger?.status === "resolved" + && ledger.mutation_allowed === true + && ledger.operation_id === operation.operation_id + && ledger.intended_mode === request.connection_mode + && ledger.resolution === "target-observed" + && transportRefEquivalenceKey(ledger.transport_ref) === requestTransportKey + && deviceNetwork?.state === "applied" + && deviceNetwork.connection_mode === request.connection_mode + && transportRefEquivalenceKey(deviceNetwork.transport_ref) + === requestTransportKey, + ); +} + +function requireExactReadOnlyVerificationOutcome( + state: XgridsK1State, + connectionMode: NonNullable, +): XgridsK1State { + if ( + hasExactConnectionReady(state, connectionMode) + || isRecoveredPhysicalScanning(state, connectionMode) + ) return state; + return requireExactConnectionReady(state, connectionMode); +} + +function connectionModeForVerification( + request: ConnectionVerifyRequest, +): XgridsConnectionMode { + if (request.compatibility_attestation.topology === "device-ap") { + return "quick-connect"; + } + if (request.compatibility_attestation.topology === "controller-hotspot") { + return "direct-connect"; + } + return "bridge"; +} + +async function waitForPhysicalReconciliationProof( + acceptState: (state: XgridsK1State) => void, + assertOperatorIntentCurrent: () => void, +): Promise { + for (;;) { + assertOperatorIntentCurrent(); + const nextState = await awaitWhileIntentCurrent( + assertOperatorIntentCurrent, + () => xgridsK1Api.getState(), + ); + acceptState(nextState); + const phase = controlPhase(nextState); + if (phase === "failed") throw controlFailure(nextState); + if (phase !== "connection-ready") { + throw new ApiError( + "Управляющая сессия изменилась до завершения read-only сверки физического состояния K1.", + ); + } + const physical = nextState.application_control_session?.physical_command; + if (physical?.requires_reconciliation !== true) return nextState; + if (physical.reconciliation_ready) return nextState; + await awaitWhileIntentCurrent( + assertOperatorIntentCurrent, + () => new Promise((resolve) => { + window.setTimeout(resolve, CONTROL_STATE_READ_INTERVAL_MS); + }), + ); + } +} + function messageFor(error: unknown): string { if (error instanceof ApiError) { // Domain errors are already written for the operator. Only HTTP details @@ -193,6 +526,25 @@ function messageFor(error: unknown): string { return "Запрос к локальному сервису устройства завершился ошибкой."; } +function apiErrorForOperation( + message: string, + operation: XgridsOperation | null | undefined, +): ApiError { + return new ApiError( + message, + 0, + false, + operationHostFailureDiagnostic(operation), + ); +} + +const CONNECT_SESSION_RESET_COPY = + "Автоматический повтор команды K1 не отправлялся. Сессия подключения в интерфейсе сброшена. Выполните новый поиск Bluetooth, выберите K1 и запустите подключение ещё раз."; + +function resetConnectSessionMessage(detail: string): string { + return `${detail} ${CONNECT_SESSION_RESET_COPY}`; +} + export function networkProvisionFailureMessage( operation: XgridsOperation | null | undefined, ): string | null { @@ -200,44 +552,183 @@ export function networkProvisionFailureMessage( const code = operation.error?.code; if (typeof code !== "string") return null; - if ( - code === "host-wifi-helper-build-timeout" - || code === "host-wifi-helper-build-failed" - ) { - const failedBeforeDeviceWrite = operation.error?.side_effect_status === "none" - && operation.error?.safe_to_retry === true; - const failure = code === "host-wifi-helper-build-timeout" - ? "не успел собраться за отведённое время" - : "не удалось собрать"; - if (failedBeforeDeviceWrite) { - return `Локальный компонент Wi‑Fi ${failure}. Команда K1 не отправлялась; подготовьте локальный компонент и повторите подключение отдельным действием.`; + if (code === "BleakGATTProtocolError") { + const attCode = operation.error?.ble_att_error_code; + const attName = operation.error?.ble_att_error_name; + const attDetail = typeof attCode === "number" && typeof attName === "string" + ? ` Код периферии: ATT ${attCode} ${attName}.` + : ""; + if (operation.error?.device_write_attempted === false) { + return `Bluetooth-сеанс K1 завершился ошибкой до команды изменения сети.${attDetail} Запись сетевого профиля не выполнялась; выполните новый поиск после освобождения Bluetooth.`; } - return `Локальный компонент Wi‑Fi ${failure} уже после начала операции с K1. Состояние устройства нельзя выводить из этой локальной ошибки; автоматического повтора команды не было. Выполните read-only проверку K1 перед новым подключением.`; + if (typeof attCode === "number" && typeof attName === "string") { + return resetConnectSessionMessage( + `Bluetooth-периферия завершила сетевую операцию ошибкой.${attDetail} Команда могла быть принята K1; итог текущей попытки не подтверждён.`, + ); + } + return resetConnectSessionMessage( + "Bluetooth-периферия завершила сетевую операцию ошибкой. Команда могла быть принята K1; итог текущей попытки не подтверждён.", + ); + } + + if (code === "network-not-found") { + const attemptCount = operation.error?.scan_attempt_count; + const elapsedMs = operation.error?.scan_elapsed_ms; + const scanDetail = typeof attemptCount === "number" && typeof elapsedMs === "number" + ? ` macOS выполнила ${attemptCount} проверок за ${(elapsedMs / 1000).toFixed(1)} с.` + : ""; + return `K1 принял команду Quick Connect и подтвердил готовность точки доступа, но macOS не увидела её Wi‑Fi-сеть за отведённое время.${scanDetail} Quick Connect не установлен; автоматического повтора не было. Выполните новый поиск перед следующей явной попыткой или используйте Bridge.`; + } + + if ( + code === "keychain-authorization-required" + || code === "keychain-authorization-denied" + || code === "keychain-authorization-cancelled" + || code === "keychain-access-failed" + ) { + const failedBeforeDeviceWrite = operation.error?.side_effect_status === "none"; + return failedBeforeDeviceWrite + ? "Локальный профиль K1 недоступен в связке ключей. Команда устройству не отправлялась; подготовьте разрешение профиля отдельным действием и затем повторите подключение." + : resetConnectSessionMessage( + "После подтверждённого включения точки K1 локальный профиль стал недоступен в связке ключей. Дополнительный пароль не запрашивался; итог текущей попытки не подтверждён.", + ); } const messages: Record = { - BleakGATTProtocolError: - "Сканер отклонил запись сетевого профиля. Результат изменения сети неизвестен; автоматический повтор запрещён. Проверьте текущее состояние K1 или подхватите существующее подключение без изменения настроек Wi‑Fi.", - "network-not-found": - "Точка доступа выбранного K1 не найдена. Команда включения точки не повторялась; проверьте питание и состояние K1.", - "credential-entry-cancelled": - "Первичная регистрация пароля K1 отменена. Получите пароль сохранённой сети этого K1 на авторизованном устройстве и повторите подключение отдельным действием.", - "credential-invalid": - "Пароль точки доступа K1 имеет недопустимую длину. Получите сохранённый пароль этого K1 в LixelGO/iPhone и повторите подключение.", "host-wifi-operation-timeout": - "Локальная операция подготовки Wi‑Fi не завершилась вовремя. Это могло произойти до изменения состояния K1; наличие сохранённого пароля этим кодом не подтверждается и не опровергается. Проверьте состояние K1 и повторите подключение отдельным действием.", + resetConnectSessionMessage("Локальная операция подготовки Wi‑Fi не завершилась вовремя; итог текущей попытки подключения не подтверждён."), "profile-ssid-mismatch": - "Сохранённый профиль относится к другому K1. Подключение остановлено без повторной команды сканеру.", + "Сохранённый профиль относится к другому устройству. Подключение остановлено без повторной команды сканеру.", + "profile-credential-source-mismatch": + "Сохранённый профиль K1 не подтверждён для точной версии прошивки. Автоматического выбора другого пароля нет; подготовьте профиль отдельно перед новой попыткой.", + "profile-unavailable": + resetConnectSessionMessage("Локальный профиль выбранного K1 отсутствует; текущая попытка подключения завершилась ошибкой."), "corewlan-error": "macOS не смогла подключиться к точке доступа K1. Проверьте пароль сохранённой сети этого K1; автоматического повтора не было.", "wifi-interface-unavailable": "Системный Wi-Fi-интерфейс macOS недоступен. Команда сканеру автоматически не повторялась.", "unsupported-platform": "Для этой операционной системы адаптер подключения к точке K1 ещё не реализован.", + "credential-source-unavailable": + "Локальный профиль выбранного K1 не готов. Команда устройству не отправлялась; после подготовки профиля разрешена новая явная попытка.", + "network-provision-candidate-not-fresh": + "Результат Bluetooth-поиска отсутствует или устарел. Команда K1 не отправлялась; выполните один свежий поиск.", + "network-provision-candidate-changed": + "Bluetooth-кандидат изменился до команды K1. Записи не было; выполните один свежий поиск.", + "network-provision-candidate-name-unavailable": + "K1 не сообщил имя своей точки доступа. Команда устройству не отправлялась; выполните новый поиск.", + "network-provision-target-not-distinguishable-from-baseline": + resetConnectSessionMessage("После BLE-команды K1 вернул сетевой статус, неотличимый от исходного; итог текущей попытки подключения не подтверждён."), + "network-provision-lifecycle-busy": + "Сетевая операция заблокирована активной сессией или локальной очисткой. Команда K1 не отправлялась; завершите текущую сессию и повторите явно.", }; return messages[code] ?? null; } +export function discoveryScanFailureMessage( + operation: XgridsOperation | null | undefined, +): string | null { + if (!operation || operation.action !== "discovery.scan" || operation.status !== "failed") { + return null; + } + const code = operation.error?.code; + if (typeof code !== "string") return null; + const messages: Record = { + "ble-discovery-already-running": + "Поиск Bluetooth уже выполняется в другой вкладке. Текущий запрос не запускал второй системный скан; дождитесь завершения первого и обновите состояние.", + "ble-runtime-busy": + "Bluetooth занят другой локальной операцией K1. Второй системный сеанс не запускался; дождитесь завершения текущей операции и обновите состояние.", + "ble-runtime-cleanup-pending": + "Предыдущий Bluetooth-сеанс ещё подтверждает отключение. Новый поиск не запускался; дождитесь завершения очистки и обновите состояние.", + "ble-runtime-owner-loop-conflict": + "Локальный Bluetooth runtime привязан к другому активному циклу. Перезапустите локальный Mission Core; команды K1 не отправлялись.", + "ble-runtime-restart-required": + "Локальный Bluetooth runtime не подтвердил очистку предыдущего сеанса. Перезапустите Mission Core перед новым поиском; команды K1 не отправлялись.", + "ble-discovery-blocked-by-provisioning": + "Поиск Bluetooth не запускался: сейчас выполняется операция подключения K1.", + "ble-discovery-timeout": + "Системный поиск Bluetooth не завершился вовремя и был принудительно остановлен. Команды K1 не отправлялись; повторный поиск безопасен.", + "ble-discovery-cancelled": + "Поиск Bluetooth отменён и системный скан остановлен. Команды K1 не отправлялись.", + }; + return messages[code] + ?? "Поиск Bluetooth завершился ошибкой до обращения к K1. Команды устройству не отправлялись; проверьте доступ macOS к Bluetooth и повторите поиск."; +} + +export function connectionVerificationFailureMessage( + operation: XgridsOperation | null | undefined, +): string | null { + if (!operation || operation.action !== "connection.verify" || operation.status !== "failed") { + return null; + } + const code = operation.error?.code; + if (typeof code !== "string") return null; + const messages: Record = { + "connection-verify-candidate-not-fresh": + "Сохранённое подключение не удалось восстановить: результат поиска K1 отсутствует или устарел. Выполните новый поиск Bluetooth.", + "connection-verify-device-not-connectable": + "K1 был в списке поиска, но сейчас не принимает Bluetooth-подключение. Убедитесь, что другое приложение не держит устройство, затем выполните новый поиск.", + "connection-verify-device-not-rediscovered": + "Mission Core не получил объявление сохранённого CoreBluetooth-устройства в отведённое окно. Команды K1 не отправлялись; повторите явную read-only сверку.", + "connection-verify-runtime-loop-unavailable": + "Локальный Bluetooth runtime недоступен. Перезапустите локальный Mission Core; команды K1 не отправлялись.", + "connection-verify-status-read-timeout": + "Mission Core не завершил read-only этап Bluetooth вовремя. Настройки K1 не менялись и команды не отправлялись; этап сохранён в журнале операции.", + "connection-verify-exact-uuid-scan-timeout": + "Mission Core не получил объявление точного сохранённого CoreBluetooth UUID в отведённое окно. Команды K1 не отправлялись; это не является выводом о состоянии устройства.", + "connection-verify-address-unavailable": + "K1 ответил, но не сообщил адрес в общей сети. Bridge пока не подключён; настройки устройства не менялись.", + "connection-verify-target-not-distinguishable-from-baseline": + "K1 ответил, но приложение не смогло подтвердить, что прежние настройки сети были применены. Автоматического повтора и новой записи не было.", + "connection-verify-route-mismatch": + "K1 сообщил локальный адрес, но компьютер находится в другой сети. Подключите компьютер к той же сети; настройки K1 не изменялись.", + "connection-verify-mqtt-unreachable": + "Старый адрес K1 недоступен. Устройство найдено по Bluetooth: можно заново применить настройки общей сети. Команда Wi-Fi не отправлялась.", + "connection-verify-local-address-conflict": + "K1 сообщил адрес этого компьютера вместо собственного; подключение не принято и запись устройству не выполнялась.", + "connection-verify-status-read-invalid": + "Ответ не принадлежит выбранному K1. Подключение не принято; настройки устройства не менялись.", + "connection-verify-candidate-changed": + "Список Bluetooth изменился во время подключения. Результат отброшен без изменения K1; выполните новый поиск.", + "connection-verify-lease-changed": + "Подключение K1 изменилось во время восстановления. Результат отброшен без изменения устройства; обновите состояние и подключитесь заново.", + "connection-verify-busy": + "Другая операция подключения уже выполняется. Вторая Bluetooth-сессия не запускалась.", + "connection-verify-cleanup-pending": + "Предыдущая Bluetooth-сессия ещё завершается. Дождитесь её закрытия и выполните новый поиск.", + "connection-verify-lifecycle-busy": + "Подключение нельзя менять во время активного сканирования K1. Новые команды устройству не отправлялись.", + "connection-verify-connection-missing": + "В локальном runtime нет подключения K1, которое можно обновить. Выполните новый поиск Bluetooth.", + "connection-verify-status-read-failed": + "Mission Core не завершил read-only чтение Bluetooth. Настройки K1 не менялись и команды не отправлялись; этап сбоя сохранён в журнале операции.", + "application-connection-binding-lost": + "Сеть изменилась во время подключения. Команды устройству не отправлялись; после восстановления сети нажмите «Подключиться заново».", + "physical-command-reconciliation-proof-timeout": + "K1 подключился, но не сообщил текущее состояние вовремя. START и STOP не отправлялись; нажмите «Подключиться заново».", + }; + return messages[code] + ?? "Подключение не восстановлено. Настройки K1 не менялись; выполните новый поиск Bluetooth."; +} + +export function operationById( + state: XgridsK1State, + action: string, + operationId: string, +): XgridsOperation | null { + const operations = state.operations ?? []; + for (let index = operations.length - 1; index >= 0; index -= 1) { + const operation = operations[index]; + if (operation?.action === action && operation.operation_id === operationId) { + return operation; + } + } + return state.last_operation?.action === action + && state.last_operation.operation_id === operationId + ? state.last_operation + : null; +} + function measuredLatency(state: XgridsK1State | null): number | null { if (state?.source_mode !== "live") return null; const metrics = state?.metrics; @@ -260,32 +751,172 @@ export function useXgridsK1Runtime(enabled: boolean) { const [eventStatus, setEventStatus] = useState("connecting"); const [pendingAction, setPendingAction] = useState(null); const [error, setError] = useState(null); + const [errorDiagnostic, setErrorDiagnostic] = + useState(null); + const [presentedErrorCorrelation, setPresentedErrorCorrelation] = + useState(null); const [latencyHistory, setLatencyHistory] = useState([]); + const [physicalStopIntentSpent, setPhysicalStopIntentSpent] = useState(false); + const [physicalStopInFlight, setPhysicalStopInFlight] = useState(false); + const latestState = useRef(null); + const errorCorrelation = useRef(null); + const spentPhysicalStopIntent = useRef(null); + const physicalStopPresentationOwner = useRef(null); const operatorIntents = useRef(new OperatorIntentGeneration()); - const actionSequence = useRef(0); - const actionInFlight = useRef<{ - runtimeGeneration: number; - actionSequence: number; - } | null>(null); + const runtimeActionArbiter = useRef(new SnapshotRuntimeActionArbiter()); - const acceptState = useCallback((nextState: XgridsK1State) => { - setState((currentState) => selectMonotonicXgridsState(currentState, nextState)); - setBackendStatus("online"); + const spendPhysicalStopIntent = useCallback(( + checkpoint: PhysicalStopIntentCheckpoint, + ): boolean => { + if ( + authoritativeStateSupersedesPhysicalStopIntent( + checkpoint, + latestState.current, + ) + ) return false; + spentPhysicalStopIntent.current = checkpoint; + setPhysicalStopIntentSpent(true); + return true; }, []); - const refresh = useCallback(async (reportErrors = true) => { + const acceptState = useCallback((nextState: XgridsK1State) => { + const previousState = latestState.current; + const previousSnapshotRuntimeId = + previousState?.snapshot_runtime_id?.trim() || null; + const acceptedState = selectMonotonicXgridsState(previousState, nextState); + if ( + previousState + && acceptedState === previousState + && acceptedState !== nextState + ) return false; + const acceptedSnapshotRuntimeId = + acceptedState.snapshot_runtime_id?.trim() || null; + if ( + previousSnapshotRuntimeId + && acceptedSnapshotRuntimeId + && previousSnapshotRuntimeId !== acceptedSnapshotRuntimeId + ) { + // A backend runtime replacement retires the old in-flight UI action + // synchronously. Its finally block is token-guarded, so it cannot clear + // a newer action started against the accepted runtime. + runtimeActionArbiter.current.retireForSnapshotChange( + previousSnapshotRuntimeId, + acceptedSnapshotRuntimeId, + ); + setPendingAction(null); + physicalStopPresentationOwner.current = null; + setPhysicalStopInFlight(false); + } + latestState.current = acceptedState; + setState(acceptedState); + if ( + authoritativeStateSupersedesPhysicalStopIntent( + spentPhysicalStopIntent.current, + acceptedState, + ) + ) { + spentPhysicalStopIntent.current = null; + setPhysicalStopIntentSpent(false); + } + if ( + authoritativeStateSupersedesRuntimeError( + errorCorrelation.current, + acceptedState, + ) + ) { + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); + setError(null); + setErrorDiagnostic(null); + } + setBackendStatus("online"); + return true; + }, []); + + const isSnapshotRuntimeCurrent = useCallback(( + expectedSnapshotRuntimeId: string, + ): boolean => { + const currentSnapshotRuntimeId = + latestState.current?.snapshot_runtime_id?.trim() || null; + return snapshotRuntimeIdsMatch( + expectedSnapshotRuntimeId, + currentSnapshotRuntimeId, + ); + }, []); + + const getConnectionActionAuthority = useCallback(( + connectionMode: XgridsConnectionMode, + ): ConnectionActionAuthoritySnapshot | null => + connectionActionAuthoritySnapshot(latestState.current, connectionMode), []); + + const getCurrentState = useCallback( + (): XgridsK1State | null => latestState.current, + [], + ); + + const getConnectionRecoveryObservationTarget = useCallback( + () => recommendedConnectionRecoveryObservationTarget(latestState.current), + [], + ); + + const isConnectionPolicyActionAllowedCurrent = useCallback(( + action: XgridsConnectionPolicyAction, + ) => connectionPolicyAllows(latestState.current, action), []); + + const isConnectionActionAuthorityCurrent = useCallback(( + expected: ConnectionActionAuthoritySnapshot, + ): boolean => { + const current = connectionActionAuthoritySnapshot( + latestState.current, + expected.connectionMode, + ); + return Boolean( + current + && current.snapshotRuntimeId === expected.snapshotRuntimeId + && current.desiredModeRevision === expected.desiredModeRevision + && current.reconfigurationRevision === expected.reconfigurationRevision + && current.reconfigurationIntentId === expected.reconfigurationIntentId + && current.activeBindingKey === expected.activeBindingKey + && current.discoveryGeneration === expected.discoveryGeneration, + ); + }, []); + + const expectedSnapshotRuntimeId = useCallback((): string => { + const snapshotRuntimeId = latestState.current?.snapshot_runtime_id?.trim() || null; + if (!snapshotRuntimeId) { + throw new ApiError( + "Состояние локального сервиса обновилось. Обновите страницу перед следующим действием.", + ); + } + return snapshotRuntimeId; + }, []); + + const refresh = useCallback(async ( + reportErrors = true, + ): Promise => { const runtimeToken = operatorIntents.current.captureRuntime(); - if (!enabled || !runtimeToken) return; + if (!enabled || !runtimeToken) return null; const [healthResult, stateResult] = await Promise.allSettled([ xgridsK1Api.getHealth(), xgridsK1Api.getState(), ]); - if (!operatorIntents.current.isRuntimeCurrent(runtimeToken)) return; + if (!operatorIntents.current.isRuntimeCurrent(runtimeToken)) return null; + let acceptedRefreshedState: XgridsK1State | null = null; if (stateResult.status === "fulfilled") { acceptState(stateResult.value); - if (reportErrors) setError(null); + // The event socket can win the race and publish the committed reopen + // before this REST refresh returns. In that case the duplicate REST + // snapshot is correctly rejected by monotonic ordering, while the + // already-accepted latest state is still the proof this click needs. + acceptedRefreshedState = latestState.current; + if (reportErrors && errorCorrelation.current?.action === "refresh") { + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); + setError(null); + setErrorDiagnostic(null); + } } if (healthResult.status === "fulfilled") { @@ -297,36 +928,81 @@ export function useXgridsK1Runtime(enabled: boolean) { } if (stateResult.status === "rejected" && reportErrors) { + errorCorrelation.current = { + action: "refresh", + runtimeId: latestState.current?.snapshot_runtime_id ?? null, + leaseGeneration: + latestState.current?.connection_supervisor?.lease.generation ?? null, + connectionAttemptId: null, + }; + setPresentedErrorCorrelation(errorCorrelation.current); setError(messageFor(stateResult.reason)); + setErrorDiagnostic(null); } + return acceptedRefreshedState; }, [acceptState, enabled]); const run = useCallback( - async (action: PendingAction, operation: () => Promise) => { + async ( + action: PendingAction, + operation: () => Promise, + options: RuntimeActionOptions = {}, + ) => { const runtimeToken = operatorIntents.current.captureRuntime(); if (!enabled || !runtimeToken) return false; - if ( - actionInFlight.current?.runtimeGeneration - === runtimeToken.runtimeGeneration - ) return false; - actionSequence.current += 1; - const actionToken = { - runtimeGeneration: runtimeToken.runtimeGeneration, - actionSequence: actionSequence.current, - }; - actionInFlight.current = actionToken; + const actionToken = runtimeActionArbiter.current.begin( + runtimeToken.runtimeGeneration, + options.supersedePending === true, + ); + if (!actionToken) return false; setPendingAction(action); + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); setError(null); + setErrorDiagnostic(null); try { - if (!operatorIntents.current.isRuntimeCurrent(runtimeToken)) return false; + if ( + !operatorIntents.current.isRuntimeCurrent(runtimeToken) + || !runtimeActionArbiter.current.isCurrent(actionToken) + ) return false; const nextState = await operation(); - if (!operatorIntents.current.isRuntimeCurrent(runtimeToken)) return false; - acceptState(nextState); + if ( + !operatorIntents.current.isRuntimeCurrent(runtimeToken) + || !runtimeActionArbiter.current.isCurrent(actionToken) + ) return false; + if (!acceptState(nextState)) return false; + if (!runtimeActionArbiter.current.isCurrent(actionToken)) return false; return true; } catch (operationError) { - if (operatorIntents.current.isRuntimeCurrent(runtimeToken)) { - setError(messageFor(operationError)); + if ( + operatorIntents.current.isRuntimeCurrent(runtimeToken) + && runtimeActionArbiter.current.isCurrent(actionToken) + ) { + if ( + options.surfaceErrors !== false + && shouldSurfaceRuntimeActionError(action, latestState.current) + ) { + errorCorrelation.current = { + action, + runtimeId: latestState.current?.snapshot_runtime_id ?? null, + leaseGeneration: + latestState.current?.connection_supervisor?.lease.generation ?? null, + connectionAttemptId: options.connectionAttemptId?.() ?? null, + }; + setPresentedErrorCorrelation(errorCorrelation.current); + setError(messageFor(operationError)); + setErrorDiagnostic( + operationError instanceof ApiError + ? operationError.hostDiagnostic + : null, + ); + } else { + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); + setError(null); + setErrorDiagnostic(null); + } if ( operationError instanceof ApiError && operationError.transportUnavailable @@ -336,11 +1012,7 @@ export function useXgridsK1Runtime(enabled: boolean) { } return false; } finally { - if ( - actionInFlight.current?.runtimeGeneration === actionToken.runtimeGeneration - && actionInFlight.current.actionSequence === actionToken.actionSequence - ) { - actionInFlight.current = null; + if (runtimeActionArbiter.current.settle(actionToken)) { if (operatorIntents.current.isRuntimeCurrent(runtimeToken)) { setPendingAction(null); } @@ -350,29 +1022,322 @@ export function useXgridsK1Runtime(enabled: boolean) { [acceptState, enabled], ); + const scanWithResult = useCallback( + async ( + options: RuntimeActionOptions = {}, + ): Promise => { + let scannedState: XgridsK1State | null = null; + const succeeded = await run("scan", async () => { + const operationId = newOperationId(); + try { + const nextState = await xgridsK1Api.scanBle({ + duration_seconds: options.durationSeconds ?? 6, + operation_id: operationId, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + scannedState = nextState; + return nextState; + } catch (scanError) { + let failedState: XgridsK1State; + try { + failedState = await xgridsK1Api.getState(); + } catch { + throw scanError; + } + acceptState(failedState); + const failedOperation = operationById( + failedState, + "discovery.scan", + operationId, + ); + const failureMessage = discoveryScanFailureMessage(failedOperation); + if (failureMessage) { + throw apiErrorForOperation(failureMessage, failedOperation); + } + throw scanError; + } + }, options); + if (!succeeded || !scannedState) { + return { + succeeded: false, + snapshotRuntimeId: null, + discoveryGeneration: null, + transportRefs: [], + }; + } + const acceptedScanState = scannedState as XgridsK1State; + const snapshotRuntimeId = + acceptedScanState.snapshot_runtime_id?.trim() || null; + const discoveryGeneration = Number.isInteger( + acceptedScanState.ble_discovery_generation, + ) ? acceptedScanState.ble_discovery_generation as number : null; + return { + succeeded: Boolean(snapshotRuntimeId && discoveryGeneration !== null), + snapshotRuntimeId, + discoveryGeneration, + transportRefs: (acceptedScanState.devices ?? []) + .map((device) => device.device_id?.trim()) + .filter((deviceId): deviceId is string => Boolean(deviceId)), + }; + }, + [acceptState, expectedSnapshotRuntimeId, run], + ); + const scan = useCallback( - () => run("scan", () => xgridsK1Api.scanBle({ duration_seconds: 6 })), - [run], + async (options: RuntimeActionOptions = {}) => + (await scanWithResult(options)).succeeded, + [scanWithResult], + ); + + const selectConnectionMode = useCallback( + (request: SelectConnectionModeRequest) => + run("mode", async () => { + try { + return await xgridsK1Api.selectConnectionMode({ + ...request, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + } catch (selectionError) { + // A second tab may have advanced the process-local CAS. Adopt the + // current backend draft before surfacing the conflict; this read is + // non-mutating and avoids a manual Refresh ceremony. + try { + acceptState(await xgridsK1Api.getState()); + } catch { + // Preserve the original selection error when state is unavailable. + } + throw selectionError; + } + }, { + supersedePending: request.reset_scenario === true, + }), + [acceptState, expectedSnapshotRuntimeId, run], + ); + + const resetConnectionScenario = useCallback((): Promise => { + const expectedRevision = latestState.current?.desired_connection_mode_revision; + if ( + !Number.isInteger(expectedRevision) + || (expectedRevision ?? -1) < 0 + || !latestState.current?.snapshot_runtime_id?.trim() + ) { + return Promise.resolve(false); + } + return selectConnectionMode({ + connection_mode: DEFAULT_CONNECTION_MODE, + expected_revision: expectedRevision as number, + reset_scenario: true, + reset_id: newOperationId(), + }); + }, [selectConnectionMode]); + + const prepareConnectionReconfigurationWithResult = useCallback( + async ( + request: PrepareConnectionReconfigurationRequest, + ): Promise => { + let observedState: XgridsK1State | null = null; + const succeeded = await run( + "reconfigure", + async () => { + const nextState = await xgridsK1Api.prepareConnectionReconfiguration({ + ...request, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + observedState = nextState; + return nextState; + }, + ); + return { succeeded, observedState }; + }, + [expectedSnapshotRuntimeId, run], + ); + + const prepareConnectionReconfiguration = useCallback( + async (request: PrepareConnectionReconfigurationRequest) => + (await prepareConnectionReconfigurationWithResult(request)).succeeded, + [prepareConnectionReconfigurationWithResult], + ); + + const retireUnavailablePhysicalCommandWithResult = useCallback( + async ( + request: RetireUnavailablePhysicalCommandRequest, + actionSnapshotRuntimeId: string, + ): Promise => { + let observedState: XgridsK1State | null = null; + const exactSnapshotRuntimeId = actionSnapshotRuntimeId.trim(); + if ( + !exactSnapshotRuntimeId + || !isSnapshotRuntimeCurrent(exactSnapshotRuntimeId) + ) { + return { succeeded: false, observedState: latestState.current }; + } + const succeeded = await run("retire", async () => { + if (!isSnapshotRuntimeCurrent(exactSnapshotRuntimeId)) { + throw new ApiError( + "Состояние локального сервиса обновилось. Прежний K1 не исключён.", + ); + } + const nextState = await xgridsK1Api.retireUnavailablePhysicalCommand({ + ...request, + expected_snapshot_runtime_id: exactSnapshotRuntimeId, + }); + observedState = nextState; + return nextState; + }, { surfaceErrors: false }); + if (!succeeded) { + // A 409 means the local safety facts changed between projection and + // click. Re-read them so the UI can explain the exact safe next step. + // A lost response may also have committed the same retirement; this + // read discovers that result without issuing a second mutation. + observedState = (await refresh(false)) ?? latestState.current; + } else { + observedState = latestState.current; + } + return { succeeded, observedState }; + }, + [isSnapshotRuntimeCurrent, latestState, refresh, run], + ); + + const retireUnavailablePhysicalCommand = useCallback( + async ( + request: RetireUnavailablePhysicalCommandRequest, + actionSnapshotRuntimeId: string, + ) => ( + await retireUnavailablePhysicalCommandWithResult( + request, + actionSnapshotRuntimeId, + ) + ).succeeded, + [retireUnavailablePhysicalCommandWithResult], + ); + + const reopenRetiredPhysicalReconciliation = useCallback( + async ( + request: ReopenRetiredPhysicalReconciliationRequest, + actionSnapshotRuntimeId: string, + ): Promise => { + let observedState: XgridsK1State | null = null; + const exactSnapshotRuntimeId = actionSnapshotRuntimeId.trim(); + if ( + !exactSnapshotRuntimeId + || !isSnapshotRuntimeCurrent(exactSnapshotRuntimeId) + ) { + return { succeeded: false, observedState: latestState.current }; + } + const succeeded = await run( + "reopen", + async () => { + if (!isSnapshotRuntimeCurrent(exactSnapshotRuntimeId)) { + throw new ApiError( + "Состояние локального сервиса обновилось. Прежняя сверка не открыта.", + ); + } + const nextState = await xgridsK1Api.reopenRetiredPhysicalReconciliation({ + ...request, + expected_snapshot_runtime_id: exactSnapshotRuntimeId, + }); + observedState = nextState; + return nextState; + }, + { surfaceErrors: false, supersedePending: true }, + ); + if (!succeeded) { + observedState = (await refresh(false)) ?? latestState.current; + } else { + // `run` may have accepted the reopen REST snapshot and then yielded + // after a newer same-runtime WebSocket state was already accepted. + // Continue only from the current monotonic truth; returning the stale + // REST proof here could issue one unnecessary Verify after another + // tab explicitly re-retired the device. + observedState = latestState.current; + } + return { succeeded, observedState }; + }, + [isSnapshotRuntimeCurrent, latestState, refresh, run], ); const connect = useCallback( - (request: ConnectRequest) => - run("connect", async () => { + async (request: ConnectRequest): Promise => { + let intentDisposition: ProvisioningSubmitResult["intentDisposition"] = "retain"; + let failureReasonCode: string | null = null; + let acceptedSessionKey: string | null = null; + let networkIntentCompleted = false; + let observedState: XgridsK1State | null = null; + let failedConnectionAttemptId: string | null = null; + const sessionKeyBeforeConnect = bleSessionTargetForTransport( + state, + request.device_id, + request.connection_mode, + )?.key ?? null; + const acceptSuccessfulConnectState = (nextState: XgridsK1State) => { + networkIntentCompleted = true; + observedState = nextState; + acceptedSessionKey = acceptedBleSessionKeyAfterConnect( + nextState, + request.device_id, + request.connection_mode, + sessionKeyBeforeConnect, + ); + return nextState; + }; + const captureFailureReason = ( + operation: XgridsOperation | null | undefined, + ) => { + const code = operation?.error?.code; + failureReasonCode = typeof code === "string" ? code : null; + failedConnectionAttemptId = operation?.action === "network.provision" + && operation.idempotency_key === request.idempotency_key + && typeof operation.operation_id === "string" + ? operation.operation_id + : null; + }; + const succeeded = await run("connect", async () => { + const intentToken = operatorIntents.current.beginOperatorIntent(); + if (!intentToken) { + throw new ApiError( + "Экран подключения закрыт; дальнейшая проверка K1 не запускалась.", + ); + } + const assertOperatorIntentCurrent = () => { + if (!operatorIntents.current.isOperatorIntentCurrent(intentToken)) { + throw new ApiError( + "Подключение заменено новым действием оператора; START не отправлялся.", + ); + } + }; const previous = operationByIdempotencyKey( state, "network.provision", request.idempotency_key, ); - if (previous?.status === "succeeded" && state) return state; if (operationNeedsReconciliation(previous)) { - throw new ApiError( - "Предыдущая запись настроек завершилась с неопределённым результатом. Автоматический повтор заблокирован; измените параметры только после проверки устройства.", + captureFailureReason(previous); + throw apiErrorForOperation( + resetConnectSessionMessage( + "Текущая попытка подключения не получила подтверждённого результата.", + ), + previous, + ); + } + if (operationAllowsFreshProvisioningIntent(previous)) { + intentDisposition = "release"; + captureFailureReason(previous); + const failureMessage = networkProvisionFailureMessage(previous); + throw apiErrorForOperation( + failureMessage + ?? "Подготовительный этап завершился до команды K1. После устранения причины разрешена новая явная попытка.", + previous, ); } let nextState: XgridsK1State; - try { - nextState = await xgridsK1Api.connect(request); + if (previous?.status === "succeeded" && state) { + nextState = state; + } else try { + nextState = await xgridsK1Api.connect({ + ...request, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); } catch (connectError) { // A failed network action is terminal and persisted by the backend. // Re-read state only; never replay the device command. This replaces @@ -389,9 +1354,55 @@ export function useXgridsK1Runtime(enabled: boolean) { "network.provision", request.idempotency_key, ); - const failureMessage = networkProvisionFailureMessage(failedOperation); - if (failureMessage) throw new ApiError(failureMessage); - throw connectError; + // The response body can be lost after the backend has already + // committed the exact idempotent operation. The read-only journal is + // authoritative: acknowledge that success instead of presenting a + // false failure which could invite another operator click. + if (failedOperation?.status === "succeeded") { + nextState = failedState; + if ( + !exactAppliedNetworkIntentCompleted( + nextState, + request, + failedOperation, + ) + && !hasExactConnectionReady(nextState, request.connection_mode) + ) { + throw connectError; + } + } else { + captureFailureReason(failedOperation); + if (operationNeedsReconciliation(failedOperation)) { + throw apiErrorForOperation( + resetConnectSessionMessage( + "Текущая попытка подключения завершилась с неизвестным результатом.", + ), + failedOperation, + ); + } + if (operationAllowsFreshProvisioningIntent(failedOperation)) { + intentDisposition = "release"; + const failureMessage = networkProvisionFailureMessage(failedOperation); + throw apiErrorForOperation( + failureMessage + ?? "Подготовительный этап завершился до команды K1. После устранения причины разрешена новая явная попытка.", + failedOperation, + ); + } + const failureMessage = networkProvisionFailureMessage(failedOperation); + if (failureMessage) { + throw apiErrorForOperation(failureMessage, failedOperation); + } + if (failedOperation) { + throw apiErrorForOperation( + resetConnectSessionMessage( + "Текущая попытка подключения не получила подтверждённого результата.", + ), + failedOperation, + ); + } + throw connectError; + } } const operation = operationByIdempotencyKey( nextState, @@ -399,46 +1410,218 @@ export function useXgridsK1Runtime(enabled: boolean) { request.idempotency_key, ); if (operationNeedsReconciliation(operation)) { - throw new ApiError( - "Результат записи настроек требует ручной проверки. Повторная аппаратная запись не выполнена.", + captureFailureReason(operation); + throw apiErrorForOperation( + resetConnectSessionMessage( + "Текущая попытка подключения завершилась с неизвестным результатом.", + ), + operation, + ); + } + if (operationAllowsFreshProvisioningIntent(operation)) { + intentDisposition = "release"; + captureFailureReason(operation); + const failureMessage = networkProvisionFailureMessage(operation); + throw apiErrorForOperation( + failureMessage + ?? "Подготовительный этап завершился до команды K1. После устранения причины разрешена новая явная попытка.", + operation, ); } if (operation && operation.status !== "succeeded") { - throw new ApiError("Запись настроек ещё выполняется; дождитесь обновления состояния."); + captureFailureReason(operation); + throw apiErrorForOperation( + resetConnectSessionMessage( + "Текущая попытка подключения завершилась без подтверждения успеха.", + ), + operation, + ); } + const exactNetworkIntentCompleted = exactAppliedNetworkIntentCompleted( + nextState, + request, + operation, + ); + if ( + !exactNetworkIntentCompleted + && !hasExactConnectionReady(nextState, request.connection_mode) + ) { + return requireExactConnectionReady(nextState, request.connection_mode); + } + nextState = acceptSuccessfulConnectState(nextState); + assertOperatorIntentCurrent(); return nextState; - }), - [acceptState, run, state], + }, { + connectionAttemptId: () => failedConnectionAttemptId, + }); + return { + succeeded, + networkIntentCompleted, + intentDisposition, + failureReasonCode, + acceptedSessionKey: networkIntentCompleted ? acceptedSessionKey : null, + observedState, + }; + }, + [acceptState, expectedSnapshotRuntimeId, run, state], ); const verifyConnection = useCallback( - (request?: ConnectionVerifyRequest) => - run("verify", () => xgridsK1Api.verifyConnection(request)), - [run], + async ( + request: ConnectionVerifyRequest, + options: RuntimeActionOptions = {}, + ): Promise => { + let reconciliationCompleted = false; + let observedState: XgridsK1State | null = null; + let reasonCode: string | null = null; + const requestedConnectionMode = connectionModeForVerification(request); + const actionSnapshotRuntimeId = + options.expectedSnapshotRuntimeId?.trim() || null; + if ( + actionSnapshotRuntimeId + && !isSnapshotRuntimeCurrent(actionSnapshotRuntimeId) + ) { + return { + succeeded: false, + reconciliationCompleted: false, + observedState: latestState.current, + reasonCode: null, + }; + } + const succeeded = await run("verify", async () => { + if ( + actionSnapshotRuntimeId + && !isSnapshotRuntimeCurrent(actionSnapshotRuntimeId) + ) { + throw new ApiError( + "Состояние локального сервиса обновилось. Прежняя сверка не запущена.", + ); + } + const operationId = newOperationId(); + try { + const verifiedState = await xgridsK1Api.verifyConnection({ + ...request, + operation_id: operationId, + expected_snapshot_runtime_id: + actionSnapshotRuntimeId ?? expectedSnapshotRuntimeId(), + }); + observedState = verifiedState; + reconciliationCompleted = readOnlyVerificationClearedReconciliation( + state, + verifiedState, + request?.device_id, + ); + return requireExactReadOnlyVerificationOutcome( + verifiedState, + requestedConnectionMode, + ); + } catch (verificationError) { + let failedState: XgridsK1State; + try { + failedState = await xgridsK1Api.getState(); + } catch { + throw verificationError; + } + reconciliationCompleted = readOnlyVerificationClearedReconciliation( + state, + failedState, + request?.device_id, + ); + observedState = failedState; + acceptState(failedState); + const failedOperation = operationById( + failedState, + "connection.verify", + operationId, + ); + reasonCode = typeof failedOperation?.error?.code === "string" + ? failedOperation.error.code + : typeof failedState.connection_verification?.reason_code === "string" + ? failedState.connection_verification.reason_code + : null; + // The HTTP response can be lost after the backend completed the BLE + // observation. That journal proves only the read-only BLE stage; the + // composite action is successful only with exact DeviceInfo/control + // Ready, or a verified SCANNING adoption that grants STOP only. + if (failedOperation?.status === "succeeded") { + if ( + hasExactConnectionReady(failedState, requestedConnectionMode) + || isRecoveredPhysicalScanning(failedState, requestedConnectionMode) + ) { + return failedState; + } + throw verificationError; + } + const failureMessage = connectionVerificationFailureMessage(failedOperation); + if (failureMessage) { + throw apiErrorForOperation(failureMessage, failedOperation); + } + throw verificationError; + } + }, options); + return { succeeded, reconciliationCompleted, observedState, reasonCode }; + }, + [ + acceptState, + expectedSnapshotRuntimeId, + isSnapshotRuntimeCurrent, + latestState, + run, + state, + ], + ); + + const probeConfiguredEndpoint = useCallback( + () => run("probe", () => xgridsK1Api.probeConfiguredEndpoint({ + operation_id: newOperationId(), + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + })), + [expectedSnapshotRuntimeId, run], ); const openApplicationControlSession = useCallback( (request: OpenApplicationControlSessionRequest) => - run("control", () => xgridsK1Api.openApplicationControlSession(request)), - [run], + run("control", () => xgridsK1Api.openApplicationControlSession({ + ...request, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + })), + [expectedSnapshotRuntimeId, run], ); const enterApplicationWorkspace = useCallback( () => - run("control", () => - xgridsK1Api.enterApplicationWorkspace({ operator_confirmed: true }), - ), - [run], + run("control", () => { + const controlCas = exactApplicationControlCas( + latestState.current, + "ENTER рабочего пространства", + ); + return xgridsK1Api.enterApplicationWorkspace({ + operator_confirmed: true, + ...controlCas, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + }), + [expectedSnapshotRuntimeId, run], ); const closeApplicationControlSession = useCallback( - () => run("control", () => xgridsK1Api.closeApplicationControlSession()), - [run], + () => run("control", () => { + const controlCas = exactApplicationControlCas( + latestState.current, + "CLOSE управляющей сессии", + ); + return xgridsK1Api.closeApplicationControlSession({ + ...controlCas, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + }), + [expectedSnapshotRuntimeId, run], ); - const startCanonicalAcquisition = useCallback( - (request: CanonicalLiveStartRequest) => + const prepareCanonicalAcquisition = useCallback( + (request: CanonicalLivePreparationRequest) => run("live", async () => { + const actionSnapshotRuntimeId = expectedSnapshotRuntimeId(); const intentToken = operatorIntents.current.beginOperatorIntent(); if (!intentToken) { throw new ApiError( @@ -446,9 +1629,12 @@ export function useXgridsK1Runtime(enabled: boolean) { ); } const assertOperatorIntentCurrent = () => { - if (!operatorIntents.current.isOperatorIntentCurrent(intentToken)) { + if ( + !operatorIntents.current.isOperatorIntentCurrent(intentToken) + || !isSnapshotRuntimeCurrent(actionSnapshotRuntimeId) + ) { throw new ApiError( - "Операторское действие завершено или заменено; дальнейшие команды канонического диалога не отправлялись.", + "Операторское действие или локальный сервис изменились; дальнейшие команды канонического диалога не отправлялись.", ); } }; @@ -456,8 +1642,13 @@ export function useXgridsK1Runtime(enabled: boolean) { assertOperatorIntentCurrent, () => xgridsK1Api.getState(), ); - let openedControlSession = false; acceptState(nextState); + assertOperatorIntentCurrent(); + if (!currentAppliedConnectionTopology(nextState)) { + throw new ApiError( + "K1 ещё не подключён. Вернитесь в «Парк» и завершите подключение; START не используется для подключения.", + ); + } const plan = liveStartPlan(nextState); if (plan === "blocked") { throw new ApiError("Сначала завершите текущий приём или повтор записи."); @@ -466,62 +1657,97 @@ export function useXgridsK1Runtime(enabled: boolean) { for (;;) { assertOperatorIntentCurrent(); + // WebSocket/polling may have accepted a newer monotonic snapshot + // while the previous stage was awaiting its response. Every next + // decision and CAS therefore starts from the latest accepted state, + // never from the closure or an earlier stage response. + nextState = latestState.current ?? nextState; const phase = controlPhase(nextState); const acquisition = nextState.acquisition; - const entryPlan = controlSessionEntryPlan( - phase, - openedControlSession, - nextState.application_control_session?.can_open === true, - ); - - if (entryPlan === "failed") { + if (phase === "failed") { // A failed dialogue always ends this operator intent. Even when // backend reconciliation says a fresh attempt may be safe, that // attempt requires another explicit click. throw controlFailure(nextState); } - if (entryPlan === "duplicate-open") { - throw new ApiError( - "Управляющая сессия завершилась сразу после открытия. Автоматический повтор заблокирован; проверьте состояние и повторите только отдельным нажатием.", - ); - } - - if (entryPlan === "open") { - if (acquisition && !isTerminalAcquisitionState(acquisition.state)) { - throw new ApiError( - "Незавершённая подготовка не привязана к открытой control-сессии. Отмените её перед новым запуском.", - ); - } - openedControlSession = true; - nextState = await awaitWhileIntentCurrent( - assertOperatorIntentCurrent, - () => xgridsK1Api.openApplicationControlSession(request.control), - ); - acceptState(nextState); - continue; - } - if (phase === "connecting") { - nextState = await awaitWhileIntentCurrent( - assertOperatorIntentCurrent, - () => waitForControlPhase( - "connection-ready", - acceptState, - assertOperatorIntentCurrent, - ), + throw new ApiError( + "Подключение K1 ещё выполняется. Дождитесь результата; START не продолжает незавершённое подключение.", + ); + } + + if (["idle", "closed", "completed"].includes(phase)) { + throw new ApiError( + "K1 ещё не подключён. Сначала завершите подключение устройства в «Парке».", ); - continue; } if (phase === "connection-ready") { + const physical = nextState.application_control_session?.physical_command + ?? nextState.physical_command; + if (physical?.requires_reconciliation === true) { + if (!physical.reconciliation_ready) { + nextState = await awaitWhileIntentCurrent( + assertOperatorIntentCurrent, + () => waitForPhysicalReconciliationProof( + acceptState, + assertOperatorIntentCurrent, + ), + ); + } + const reconciliationPhysical = + nextState.application_control_session?.physical_command + ?? nextState.physical_command; + if (reconciliationPhysical?.requires_reconciliation === true) { + if (!reconciliationPhysical.reconciliation_ready) { + throw new ApiError( + "K1 не сообщил текущее состояние вовремя. Новая команда устройству не отправлялась.", + ); + } + nextState = await awaitWhileIntentCurrent( + assertOperatorIntentCurrent, + () => { + const controlCas = exactApplicationControlCas( + latestState.current, + "восстановления физического состояния", + ); + return xgridsK1Api.reconcilePhysicalCommand({ + reconciliation_id: newOperationId(), + ...controlCas, + expected_snapshot_runtime_id: actionSnapshotRuntimeId, + }); + }, + ); + acceptState(nextState); + assertOperatorIntentCurrent(); + } + const observedPhysical = + nextState.application_control_session?.physical_command + ?? nextState.physical_command; + if (observedPhysical?.observed_session_state !== "ready") { + throw new ApiError( + "K1 всё ещё сканирует. Новый START заблокирован; сначала остановите текущую запись.", + ); + } + continue; + } nextState = await awaitWhileIntentCurrent( assertOperatorIntentCurrent, - () => xgridsK1Api.enterApplicationWorkspace({ - operator_confirmed: true, - }), + () => { + const controlCas = exactApplicationControlCas( + latestState.current, + "ENTER рабочего пространства", + ); + return xgridsK1Api.enterApplicationWorkspace({ + operator_confirmed: true, + ...controlCas, + expected_snapshot_runtime_id: actionSnapshotRuntimeId, + }); + }, ); acceptState(nextState); + assertOperatorIntentCurrent(); continue; } @@ -541,9 +1767,21 @@ export function useXgridsK1Runtime(enabled: boolean) { if (!acquisition || isTerminalAcquisitionState(acquisition.state)) { nextState = await awaitWhileIntentCurrent( assertOperatorIntentCurrent, - () => xgridsK1Api.prepareAcquisition(request.acquisition), + () => { + const controlCas = exactAcquisitionControlCas( + latestState.current, + "PREPARE acquisition", + ); + return xgridsK1Api.prepareAcquisition({ + ...request.acquisition, + ...newMutationContext("acquisition.prepare"), + ...controlCas, + expected_snapshot_runtime_id: actionSnapshotRuntimeId, + }); + }, ); acceptState(nextState); + assertOperatorIntentCurrent(); continue; } if (acquisition.state !== "prepared" || acquisition.control_mode !== "plugin-commanded") { @@ -576,17 +1814,12 @@ export function useXgridsK1Runtime(enabled: boolean) { if (phase === "project-ready") { if (!acquisition || acquisition.state !== "prepared") { - throw new ApiError("Локальный приём не подготовлен к каноническому START."); + throw new ApiError("Локальный приём не подготовлен к финальному подтверждению START."); } - nextState = await awaitWhileIntentCurrent( - assertOperatorIntentCurrent, - () => xgridsK1Api.startAcquisition({ - acquisition_id: acquisition.acquisition_id, - expected_state_revision: acquisition.state_revision, - physical_acceptance: request.physicalAcceptance, - }), - ); - acceptState(nextState); + // Preparation deliberately stops on an exact + // prepared/project-ready snapshot. The same explicit operator + // action may then call startPreparedAcquisition, whose fresh CAS + // and supervisor authority checks remain the physical write gate. return nextState; } @@ -597,43 +1830,73 @@ export function useXgridsK1Runtime(enabled: boolean) { throw new ApiError(`Запуск K1 недоступен из состояния «${phase}».`); } }), - [acceptState, run], + [acceptState, expectedSnapshotRuntimeId, run], ); const prepareAcquisition = useCallback( - (request: PrepareAcquisitionRequest) => + (request: AcquisitionPreparationDraft) => run("live", async () => { - const plan = liveStartPlan(state); + const currentState = latestState.current; + const plan = liveStartPlan(currentState); if (plan === "blocked") { throw new ApiError( "Сначала завершите текущий приём или повтор записи.", ); } - if (plan === "already-running" && state) return state; + if (plan === "already-running" && currentState) return currentState; return ( - plan === "resume-prepared" && state - ? state - : await xgridsK1Api.prepareAcquisition(request) + plan === "resume-prepared" && currentState + ? currentState + : await xgridsK1Api.prepareAcquisition( + acquisitionMutationUsesControlSession(currentState) + ? { + ...request, + ...newMutationContext("acquisition.prepare"), + ...exactAcquisitionControlCas( + latestState.current, + "PREPARE acquisition", + ), + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + } + : { + ...request, + ...newMutationContext("acquisition.prepare"), + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }, + ) ); }), - [run, state], + [expectedSnapshotRuntimeId, run], ); const startPreparedAcquisition = useCallback( (physicalAcceptance: OperatorPresenceConfirmation) => run("live", async () => { - const acquisition = state?.acquisition; + const currentState = latestState.current; + const acquisition = currentState?.acquisition; if (!acquisition?.acquisition_id || acquisition.state !== "prepared") { throw new ApiError("Сначала сохраните проект и подготовьте локальный приём."); } + if (acquisition.control_mode !== "plugin-commanded") { + throw new ApiError( + "Физический START недоступен: подготовленная acquisition не принадлежит управляющей сессии плагина.", + ); + } + const controlCas = exactAcquisitionControlCas( + latestState.current, + "START acquisition", + ); return xgridsK1Api.startAcquisition({ acquisition_id: acquisition.acquisition_id, expected_state_revision: acquisition.state_revision, + ...newMutationContext("acquisition.start"), + ...controlCas, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), physical_acceptance: physicalAcceptance, }); }), - [run, state], + [expectedSnapshotRuntimeId, run], ); const startReplay = useCallback( @@ -642,39 +1905,192 @@ export function useXgridsK1Runtime(enabled: boolean) { ); const stop = useCallback( - (physicalAcceptance?: OperatorPresenceConfirmation) => - run("stop", () => { - const acquisition = state?.acquisition; - const acquisitionTerminal = isTerminalAcquisitionState(acquisition?.state); - if (acquisition && !acquisitionTerminal) { - const softwareCommanded = isSoftwareCommandedAcquisition(state); - if (softwareCommanded && !physicalAcceptance) { + async (physicalAcceptance?: OperatorPresenceConfirmation) => { + let stopPresentationOwner: object | null = null; + try { + return await run("stop", async () => { + const currentState = latestState.current; + const acquisition = currentState?.acquisition; + const acquisitionTerminal = isTerminalAcquisitionState(acquisition?.state); + if ( + acquisition + && ( + !acquisitionTerminal + || requiresCanonicalStopAfterTerminalLocalFailure(currentState) + ) + ) { + const softwareCommanded = isSoftwareCommandedAcquisition(currentState); + if (softwareCommanded && !physicalAcceptance) { + throw new ApiError( + "Подтвердите присутствие рядом с K1 перед каноническим STOP.", + ); + } + if (softwareCommanded) { + if (spentPhysicalStopIntent.current) { + throw new ApiError( + "Предыдущий физический STOP уже израсходовал текущий control checkpoint. Дождитесь нового подтверждённого состояния K1; повторная команда не отправлялась.", + ); + } + // Re-read the atomic authority immediately at the mutation + // boundary. UI admission and an earlier closure snapshot are + // never sufficient authority for a physical device command. + const dispatchState = latestState.current; + const dispatchAcquisition = dispatchState?.acquisition; + if ( + !dispatchAcquisition + || dispatchAcquisition.acquisition_id !== acquisition.acquisition_id + || !isSoftwareCommandedAcquisition(dispatchState) + ) { + throw new ApiError( + "Цель STOP изменилась до отправки. Команда устройству не отправлялась.", + ); + } + const checkpoint = physicalStopIntentCheckpoint(dispatchState); + if (!checkpoint) { + throw new ApiError( + "Точный SCANNING/can_stop/policy snapshot для STOP отсутствует. Команда устройству не отправлялась.", + ); + } + const controlCas = exactAcquisitionControlCas( + dispatchState, + "STOP acquisition", + ); + if ( + controlCas.expected_control_session_generation + !== checkpoint.controlSessionGeneration + || controlCas.expected_control_state_revision + !== checkpoint.controlStateRevision + ) { + throw new ApiError( + "Control checkpoint изменился до отправки STOP. Команда устройству не отправлялась.", + ); + } + if (!spendPhysicalStopIntent(checkpoint)) { + throw new ApiError( + "Состояние K1 изменилось до отправки STOP. Команда устройству не отправлялась.", + ); + } + stopPresentationOwner = { checkpoint }; + physicalStopPresentationOwner.current = stopPresentationOwner; + setPhysicalStopInFlight(true); + return await xgridsK1Api.stopAcquisition({ + acquisition_id: dispatchAcquisition.acquisition_id, + mode: "graceful", + ...newMutationContext("acquisition.stop"), + ...controlCas, + expected_snapshot_runtime_id: checkpoint.snapshotRuntimeId, + physical_acceptance: physicalAcceptance, + }); + } + if (!connectionPolicyAllows(currentState, "stop-local-receiver")) { + if (isProvenLocalReceiverInactive(currentState)) return currentState; + throw new ApiError( + "Текущее состояние K1 больше не разрешает остановку локального приёма. Команда не отправлялась.", + ); + } + return await xgridsK1Api.stopAcquisition({ + acquisition_id: acquisition.acquisition_id, + mode: "capture-only", + ...newMutationContext("acquisition.stop"), + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + ...(physicalAcceptance + ? { physical_acceptance: physicalAcceptance } + : {}), + }); + } + if (!connectionPolicyAllows(currentState, "stop-local-receiver")) { + if (isProvenLocalReceiverInactive(currentState)) return currentState; throw new ApiError( - "Подтвердите присутствие рядом с K1 перед каноническим STOP.", + "Текущее состояние K1 больше не разрешает остановку локального приёма. Команда не отправлялась.", ); } - return xgridsK1Api.stopAcquisition({ - acquisition_id: acquisition.acquisition_id, - mode: softwareCommanded ? "graceful" : "capture-only", - ...(physicalAcceptance - ? { physical_acceptance: physicalAcceptance } - : {}), + return await xgridsK1Api.stopSessionCompatibility({ + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), }); + }); + } finally { + if ( + stopPresentationOwner + && physicalStopPresentationOwner.current === stopPresentationOwner + ) { + physicalStopPresentationOwner.current = null; + setPhysicalStopInFlight(false); } - return xgridsK1Api.stopSessionCompatibility(); - }), - [run, state], + } + }, + [expectedSnapshotRuntimeId, run, spendPhysicalStopIntent], + ); + + const stopLocalReceiver = useCallback( + () => run("stop", () => { + const currentState = latestState.current; + if (!connectionPolicyAllows(currentState, "stop-local-receiver")) { + if (isProvenLocalReceiverInactive(currentState)) { + return Promise.resolve(currentState); + } + throw new ApiError( + "Текущее состояние K1 больше не разрешает остановку локального приёма. Команда не отправлялась.", + ); + } + const snapshotRuntimeId = expectedSnapshotRuntimeId(); + const stopPlan = localReceiverStopPlan(currentState); + if (stopPlan.kind === "acquisition") { + return xgridsK1Api.stopAcquisition({ + acquisition_id: stopPlan.acquisitionId, + mode: "capture-only", + ...newMutationContext("acquisition.stop"), + expected_snapshot_runtime_id: snapshotRuntimeId, + }); + } + return xgridsK1Api.stopSessionCompatibility({ + expected_snapshot_runtime_id: snapshotRuntimeId, + }); + }), + [expectedSnapshotRuntimeId, run], + ); + + const forceFinishActiveStreamLocally = useCallback( + () => run("force-finish", () => { + // Re-read the complete public authority at the mutation boundary. A + // rendered button or an earlier poll is never sufficient authority for + // local cleanup of an active acquisition. + const authority = activeStreamForceFinishAuthority(latestState.current); + if (!authority) { + throw new ApiError( + "Активное восстановление уже изменилось. Локальный приём не завершён; обновите состояние.", + ); + } + return xgridsK1Api.forceFinishAcquisitionLocally({ + acquisition_id: authority.acquisitionId, + expected_state_revision: authority.acquisitionStateRevision, + expected_recovery_generation: authority.recoveryGeneration, + operator_confirmed: true, + ...newMutationContext("acquisition.force-finish-local"), + deadline_seconds: 30, + expected_snapshot_runtime_id: authority.snapshotRuntimeId, + }); + }), + [run], ); const abort = useCallback(() => { - const acquisition = state?.acquisition; + const currentState = latestState.current; + const acquisition = currentState?.acquisition; if (!acquisition || isTerminalAcquisitionState(acquisition.state)) { return Promise.resolve(false); } - return run("abort", () => - xgridsK1Api.abortAcquisition({ acquisition_id: acquisition.acquisition_id }), - ); - }, [run, state?.acquisition]); + return run("abort", () => { + const controlCas = acquisitionMutationUsesControlSession(currentState) + ? exactAcquisitionControlCas(latestState.current, "ABORT acquisition") + : {}; + return xgridsK1Api.abortAcquisition({ + acquisition_id: acquisition.acquisition_id, + ...newMutationContext("acquisition.abort"), + ...controlCas, + expected_snapshot_runtime_id: expectedSnapshotRuntimeId(), + }); + }); + }, [expectedSnapshotRuntimeId, run]); const setObservationSourceActive = useCallback( (sourceId: string, active: boolean) => @@ -682,6 +2098,20 @@ export function useXgridsK1Runtime(enabled: boolean) { if (!state) { throw new ApiError("Состояние устройства ещё не загружено."); } + const supervisor = state.connection_supervisor; + const verifiedControl = state.application_control_session?.verified_control; + if ( + supervisor?.authority.control_allowed !== true + || !verifiedControl + || supervisor.observed.control_plane.session_id + !== verifiedControl.control_session_id + || supervisor.observed.device_identity.logical_device_id + !== verifiedControl.logical_device_id + ) { + throw new ApiError( + "Управляющая сессия выбранного K1 не подтверждена; команда камеры не отправлялась.", + ); + } const deviceSessionId = state.device_session?.device_session_id?.trim(); if (!deviceSessionId) { throw new ApiError("Для камеры нет активной сессии устройства."); @@ -729,10 +2159,15 @@ export function useXgridsK1Runtime(enabled: boolean) { useEffect(() => { if (!enabled) { operatorIntents.current.deactivateRuntime(); + latestState.current = null; + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); setState(null); setBackendStatus("checking"); setEventStatus("closed"); setPendingAction(null); + physicalStopPresentationOwner.current = null; + setPhysicalStopInFlight(false); setError(null); setLatencyHistory([]); return; @@ -793,20 +2228,46 @@ export function useXgridsK1Runtime(enabled: boolean) { eventStatus, pendingAction, error, + errorDiagnostic, + errorCorrelation: presentedErrorCorrelation, + physicalStopIntentSpent, + physicalStopInFlight, latencyHistory, + isSnapshotRuntimeCurrent, + getCurrentState, + getConnectionActionAuthority, + getConnectionRecoveryObservationTarget, + isConnectionPolicyActionAllowedCurrent, + isConnectionActionAuthorityCurrent, refresh: () => refresh(true), - clearError: () => setError(null), + clearError: () => { + errorCorrelation.current = null; + setPresentedErrorCorrelation(null); + setError(null); + setErrorDiagnostic(null); + }, scan, + scanWithResult, + selectConnectionMode, + resetConnectionScenario, + prepareConnectionReconfiguration, + prepareConnectionReconfigurationWithResult, + retireUnavailablePhysicalCommand, + retireUnavailablePhysicalCommandWithResult, + reopenRetiredPhysicalReconciliation, connect, verifyConnection, + probeConfiguredEndpoint, openApplicationControlSession, enterApplicationWorkspace, closeApplicationControlSession, - startCanonicalAcquisition, + prepareCanonicalAcquisition, prepareAcquisition, startPreparedAcquisition, startReplay, stop, + stopLocalReceiver, + forceFinishActiveStreamLocally, abort, setObservationSourceActive, updateViewerSettings, diff --git a/plugins/xgrids-k1/macos/associate_wifi.swift b/plugins/xgrids-k1/macos/associate_wifi.swift index 1ac5cc1..dced9b4 100644 --- a/plugins/xgrids-k1/macos/associate_wifi.swift +++ b/plugins/xgrids-k1/macos/associate_wifi.swift @@ -1,6 +1,7 @@ -import AppKit import CoreWLAN +import CryptoKit import Foundation +import LocalAuthentication import Security private let keychainService = "NODEDC Mission Core Host Wi-Fi Profiles" @@ -14,6 +15,8 @@ private struct HostWifiRequest: Decodable { let password: String? let scanTimeoutSeconds: Double? let credentialSourceID: String? + let interfaceName: String? + let continuityKeyHex: String? enum CodingKeys: String, CodingKey { case action @@ -22,6 +25,8 @@ private struct HostWifiRequest: Decodable { case password case scanTimeoutSeconds = "scan_timeout_seconds" case credentialSourceID = "credential_source_id" + case interfaceName = "interface_name" + case continuityKeyHex = "continuity_key_hex" } } @@ -60,6 +65,9 @@ private struct HostWifiResponse: Encodable { let scanAttemptCount: Int? let scanElapsedMilliseconds: Int? let credentialSource: String? + let wifiInterface: Bool? + let associationIdentity: String? + let associationEvidence: String? let reasonCode: String? enum CodingKeys: String, CodingKey { @@ -73,6 +81,9 @@ private struct HostWifiResponse: Encodable { case scanAttemptCount = "scan_attempt_count" case scanElapsedMilliseconds = "scan_elapsed_ms" case credentialSource = "credential_source" + case wifiInterface = "wifi_interface" + case associationIdentity = "association_identity" + case associationEvidence = "association_evidence" case reasonCode = "reason_code" } } @@ -88,6 +99,9 @@ private func emit( scanAttemptCount: Int? = nil, scanElapsedMilliseconds: Int? = nil, credentialSource: String? = nil, + wifiInterface: Bool? = nil, + associationIdentity: String? = nil, + associationEvidence: String? = nil, reasonCode: String? = nil, exitCode: Int32 ) -> Never { @@ -102,6 +116,9 @@ private func emit( scanAttemptCount: scanAttemptCount, scanElapsedMilliseconds: scanElapsedMilliseconds, credentialSource: credentialSource, + wifiInterface: wifiInterface, + associationIdentity: associationIdentity, + associationEvidence: associationEvidence, reasonCode: reasonCode ) if let data = try? JSONEncoder().encode(response) { @@ -148,10 +165,75 @@ private func materialKeychainQuery(sourceID: String) -> [String: Any] { return keychainQuery(service: credentialMaterialKeychainService, account: sourceID) } -private func loadProfile(profileID: String) throws -> StoredProfile { +private func nonInteractiveAuthenticationContext() -> LAContext { + let context = LAContext() + context.interactionNotAllowed = true + return context +} + +private func keychainItemExists(service: String, account: String) throws -> Bool { + var query = keychainQuery(service: service, account: account) + query[kSecReturnAttributes as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitOne + // Preflight is deliberately non-interactive. Authorization prompts belong + // only to an explicit enrollment/migration step, never to a K1 network + // mutation that has already been admitted by the browser. + query[kSecUseAuthenticationContext as String] = nonInteractiveAuthenticationContext() + + var item: CFTypeRef? + let status = SecItemCopyMatching(query as CFDictionary, &item) + if status == errSecSuccess { + return true + } + if status == errSecItemNotFound { + return false + } + throw NSError(domain: "HostWifiKeychainMetadata", code: Int(status)) +} + +private func keychainReasonCode(_ error: Error, missing: String) -> String { + let status = OSStatus((error as NSError).code) + switch status { + case errSecItemNotFound: + return missing + case errSecInteractionNotAllowed: + return "keychain-authorization-required" + case errSecAuthFailed: + return "keychain-authorization-denied" + case errSecUserCanceled: + return "keychain-authorization-cancelled" + default: + return "keychain-access-failed" + } +} + +private func coreWLANReasonCode(_ error: Error) -> String { + let nsError = error as NSError + guard nsError.domain == CWErrorDomain else { + return "corewlan-error" + } + // Stable CWErr values from Apple's CoreWLANTypes contract. Export only a + // reviewed failure class; NSError descriptions may contain host details. + switch nsError.code { + case -3930: // kCWOperationNotPermittedErr + return "corewlan-authorization-denied" + case -3905, -3925: // kCWTimeoutErr, kCWSupplicantTimeoutErr + return "host-wifi-operation-timeout" + default: + return "corewlan-error" + } +} + +private func loadProfile( + profileID: String, + interactionAllowed: Bool = true +) throws -> StoredProfile { var query = profileKeychainQuery(profileID: profileID) query[kSecReturnData as String] = true query[kSecMatchLimit as String] = kSecMatchLimitOne + if !interactionAllowed { + query[kSecUseAuthenticationContext as String] = nonInteractiveAuthenticationContext() + } var item: CFTypeRef? let status = SecItemCopyMatching(query as CFDictionary, &item) @@ -191,10 +273,16 @@ private func storeProfile(profileID: String, profile: StoredProfile) throws { } } -private func loadCredentialMaterial(sourceID: String) throws -> StoredCredentialMaterial { +private func loadCredentialMaterial( + sourceID: String, + interactionAllowed: Bool = true +) throws -> StoredCredentialMaterial { var query = materialKeychainQuery(sourceID: sourceID) query[kSecReturnData as String] = true query[kSecMatchLimit as String] = kSecMatchLimitOne + if !interactionAllowed { + query[kSecUseAuthenticationContext as String] = nonInteractiveAuthenticationContext() + } var item: CFTypeRef? let status = SecItemCopyMatching(query as CFDictionary, &item) @@ -237,25 +325,6 @@ private func storeCredentialMaterial( } } -private func loadSystemWiFiProfile(ssid: String, ssidData: Data) -> StoredProfile? { - var password: NSString? - let status = CWKeychainFindWiFiPassword( - CWKeychainDomain.user, - ssidData, - &password - ) - guard status == errSecSuccess, let password else { - return nil - } - let profile = StoredProfile( - schemaVersion: 1, - ssid: ssid, - password: password as String, - credentialSource: "system-wifi-keychain" - ) - return profileIsValid(profile) ? profile : nil -} - private struct TargetedScanResult { let network: CWNetwork? let attemptCount: Int @@ -298,26 +367,57 @@ private func scanForExpectedNetwork( } } -private func promptForDevicePassword(ssid: String) -> String? { - let application = NSApplication.shared - application.setActivationPolicy(.accessory) - - let passwordField = NSSecureTextField(frame: NSRect(x: 0, y: 0, width: 360, height: 24)) - passwordField.placeholderString = "Пароль точки доступа K1" - - let alert = NSAlert() - alert.alertStyle = .informational - alert.messageText = "Первое подключение к \(ssid)" - alert.informativeText = "macOS не нашла локальный профиль этой точки доступа. Если credential вам неизвестен, нажмите «Отмена» и выполните авторизованный импорт device-профиля LixelGO. Введённое значение будет сохранено только в Keychain этого Mac и не попадёт в браузер, API, журнал или evidence Mission Core." - alert.accessoryView = passwordField - alert.addButton(withTitle: "Подключиться") - alert.addButton(withTitle: "Отмена") - - application.activate(ignoringOtherApps: true) - guard alert.runModal() == .alertFirstButtonReturn else { +private func decodeContinuityKey(_ value: String) -> Data? { + let bytes = Array(value.utf8) + guard bytes.count == 64 else { return nil } - return passwordField.stringValue + func nibble(_ byte: UInt8) -> UInt8? { + switch byte { + case 48 ... 57: + return byte - 48 + case 97 ... 102: + return byte - 87 + default: + return nil + } + } + var decoded = Data(capacity: 32) + for offset in stride(from: 0, to: bytes.count, by: 2) { + guard let high = nibble(bytes[offset]), let low = nibble(bytes[offset + 1]) else { + return nil + } + decoded.append((high << 4) | low) + } + return decoded +} + +private func appendLengthPrefixed(_ value: String, to material: inout Data) { + let data = Data(value.utf8) + var length = UInt32(data.count).bigEndian + withUnsafeBytes(of: &length) { bytes in + material.append(contentsOf: bytes) + } + material.append(data) +} + +private func associationIdentity( + continuityKey: Data, + interfaceName: String, + bssid: String +) -> String { + // BSSID is the association identity. SSID visibility is permission- and + // timing-dependent on macOS, so folding it into this token would rotate a + // healthy binding when the same AP alternates between `ssid+bssid` and + // `bssid-only` evidence. + var material = Data("mission-core/host-wifi-association/v2".utf8) + appendLengthPrefixed(interfaceName, to: &material) + appendLengthPrefixed(bssid.lowercased(), to: &material) + let digest = HMAC.authenticationCode( + for: material, + using: SymmetricKey(data: continuityKey) + ) + return digest.map { String(format: "%02x", $0) }.joined() } private let input = FileHandle.standardInput.readDataToEndOfFile() @@ -335,6 +435,75 @@ do { emit(ok: false, reasonCode: "profile-id-invalid", exitCode: 1) } + if request.action == "inspect-association" { + guard let interfaceName = request.interfaceName, + (1 ... 32).contains(interfaceName.count), + interfaceName.allSatisfy({ + $0.isASCII && ($0.isLetter || $0.isNumber || ".-_".contains($0)) + }), + let continuityKeyHex = request.continuityKeyHex, + let continuityKey = decodeContinuityKey(continuityKeyHex) + else { + emit(ok: false, reasonCode: "association-inspection-invalid", exitCode: 1) + } + + guard let interface = CWWiFiClient.shared().interface(withName: interfaceName) else { + emit( + ok: true, + adapter: "CoreWLAN", + wifiInterface: false, + associationIdentity: associationIdentity( + continuityKey: continuityKey, + interfaceName: interfaceName, + bssid: "not-wifi-interface" + ), + associationEvidence: "not-wifi", + exitCode: 0 + ) + } + guard interface.powerOn(), interface.serviceActive() else { + emit( + ok: true, + adapter: "CoreWLAN", + wifiInterface: true, + associationEvidence: "unavailable", + reasonCode: "wifi-interface-inactive", + exitCode: 0 + ) + } + let currentSSID = interface.ssid()?.trimmingCharacters(in: .whitespacesAndNewlines) + let currentBSSID = interface.bssid()?.trimmingCharacters(in: .whitespacesAndNewlines) + guard let currentBSSID, !currentBSSID.isEmpty else { + // SSID alone is not an exact association identity: two APs may use + // the same network name. Returning no digest forces the Python + // caller to rotate its fail-closed continuity token. + emit( + ok: true, + adapter: "CoreWLAN", + wifiInterface: true, + associationEvidence: "unavailable", + reasonCode: "association-identity-unavailable", + exitCode: 0 + ) + } + emit( + ok: true, + adapter: "CoreWLAN", + wifiInterface: true, + associationIdentity: associationIdentity( + continuityKey: continuityKey, + interfaceName: interfaceName, + bssid: currentBSSID + ), + associationEvidence: ( + currentSSID == nil || currentSSID?.isEmpty == true + ? "bssid-only" + : "ssid+bssid" + ), + exitCode: 0 + ) + } + if request.action == "store-profile" { guard let ssid = request.ssid, let password = request.password else { emit(ok: false, reasonCode: "credential-missing", exitCode: 1) @@ -364,20 +533,27 @@ do { if request.action == "check-credential-material" { do { - _ = try loadCredentialMaterial(sourceID: request.profileID) + let available = try keychainItemExists( + service: credentialMaterialKeychainService, + account: request.profileID + ) emit( ok: true, adapter: "macOS Keychain", - profileAvailable: true, - credentialSource: "exact-firmware-profile", + profileAvailable: available, + credentialSource: available ? "exact-firmware-profile" : nil, exitCode: 0 ) } catch { emit( - ok: true, + ok: false, adapter: "macOS Keychain", profileAvailable: false, - exitCode: 0 + reasonCode: keychainReasonCode( + error, + missing: "credential-source-unavailable" + ), + exitCode: 1 ) } } @@ -395,55 +571,77 @@ do { emit(ok: false, reasonCode: "credential-source-invalid", exitCode: 1) } - let material: StoredCredentialMaterial do { - material = try loadCredentialMaterial(sourceID: sourceID) + let profileAvailable = try keychainItemExists( + service: keychainService, + account: request.profileID + ) + if profileAvailable { + let profile = try loadProfile( + profileID: request.profileID, + interactionAllowed: false + ) + guard profile.ssid == ssid else { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + profileEnrolled: false, + reasonCode: "profile-ssid-mismatch", + exitCode: 1 + ) + } + guard profile.credentialSource == "exact-firmware-profile" else { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + profileEnrolled: false, + reasonCode: "profile-credential-source-mismatch", + exitCode: 1 + ) + } + emit( + ok: true, + adapter: "macOS Keychain", + profileAvailable: true, + profileEnrolled: false, + credentialSource: "exact-firmware-profile", + exitCode: 0 + ) + } } catch { emit( - ok: true, + ok: false, adapter: "macOS Keychain", profileAvailable: false, profileEnrolled: false, - reasonCode: "credential-source-unavailable", - exitCode: 0 + reasonCode: keychainReasonCode(error, missing: "profile-unavailable"), + exitCode: 1 + ) + } + + let material: StoredCredentialMaterial + do { + material = try loadCredentialMaterial( + sourceID: sourceID, + interactionAllowed: false + ) + } catch { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + profileEnrolled: false, + reasonCode: keychainReasonCode( + error, + missing: "credential-source-unavailable" + ), + exitCode: 1 ) } do { - let existing = try loadProfile(profileID: request.profileID) - guard existing.ssid == ssid else { - emit( - ok: false, - adapter: "macOS Keychain", - profileAvailable: false, - profileEnrolled: false, - reasonCode: "profile-ssid-mismatch", - exitCode: 1 - ) - } - if existing.password == material.password, - existing.credentialSource != "exact-firmware-profile" { - try storeProfile( - profileID: request.profileID, - profile: StoredProfile( - schemaVersion: 1, - ssid: ssid, - password: existing.password, - credentialSource: "exact-firmware-profile" - ) - ) - } - emit( - ok: true, - adapter: "macOS Keychain", - profileAvailable: true, - profileEnrolled: false, - credentialSource: existing.password == material.password - ? "exact-firmware-profile" - : (existing.credentialSource ?? "mission-core-keychain"), - exitCode: 0 - ) - } catch { try storeProfile( profileID: request.profileID, profile: StoredProfile( @@ -461,39 +659,82 @@ do { credentialSource: "exact-firmware-profile", exitCode: 0 ) - } - } - - if request.action == "check-profile" { - let profile: StoredProfile - do { - profile = try loadProfile(profileID: request.profileID) } catch { - emit( - ok: true, - adapter: "macOS Keychain", - profileAvailable: false, - exitCode: 0 - ) - } - if let expectedSSID = request.ssid, profile.ssid != expectedSSID { emit( ok: false, adapter: "macOS Keychain", profileAvailable: false, - reasonCode: "profile-ssid-mismatch", + profileEnrolled: false, + reasonCode: keychainReasonCode(error, missing: "profile-unavailable"), exitCode: 1 ) } - emit( - ok: true, - adapter: "macOS Keychain", - profileAvailable: true, - exitCode: 0 - ) } - guard request.action == "associate" || request.action == "scan-profile" else { + if request.action == "check-profile" { + guard let expectedSSID = request.ssid, + let expectedSSIDData = expectedSSID.data(using: .utf8), + (1 ... 32).contains(expectedSSIDData.count) + else { + emit(ok: false, reasonCode: "ssid-invalid", exitCode: 1) + } + do { + let available = try keychainItemExists( + service: keychainService, + account: request.profileID + ) + if !available { + emit( + ok: true, + adapter: "macOS Keychain", + profileAvailable: false, + exitCode: 0 + ) + } + let profile = try loadProfile( + profileID: request.profileID, + interactionAllowed: false + ) + guard profile.ssid == expectedSSID else { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + reasonCode: "profile-ssid-mismatch", + exitCode: 1 + ) + } + guard profile.credentialSource == "exact-firmware-profile" else { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + reasonCode: "profile-credential-source-mismatch", + exitCode: 1 + ) + } + emit( + ok: true, + adapter: "macOS Keychain", + profileAvailable: true, + credentialSource: "exact-firmware-profile", + exitCode: 0 + ) + } catch { + emit( + ok: false, + adapter: "macOS Keychain", + profileAvailable: false, + reasonCode: keychainReasonCode(error, missing: "profile-unavailable"), + exitCode: 1 + ) + } + } + + guard request.action == "associate" + || request.action == "associate-ephemeral" + || request.action == "scan-profile" + else { emit(ok: false, reasonCode: "action-unsupported", exitCode: 1) } guard let interface = CWWiFiClient.shared().interface() else { @@ -528,7 +769,7 @@ do { ) } - guard request.action == "associate" else { + guard request.action == "associate" || request.action == "associate-ephemeral" else { emit(ok: false, reasonCode: "action-unsupported", exitCode: 1) } guard let expectedSSID = request.ssid, @@ -554,83 +795,76 @@ do { ) } - var profileEnrolled = false - var profileNeedsStore = false let credentialSource: String - let profile: StoredProfile - do { - profile = try loadProfile(profileID: request.profileID) - credentialSource = profile.credentialSource ?? "mission-core-keychain" - } catch { - if let systemProfile = loadSystemWiFiProfile( - ssid: expectedSSID, - ssidData: expectedSSIDData - ) { - profile = systemProfile - credentialSource = "system-wifi-keychain" - profileNeedsStore = true - } else { - guard let password = promptForDevicePassword(ssid: expectedSSID) else { - emit( - ok: false, - scanAttemptCount: scan.attemptCount, - scanElapsedMilliseconds: scan.elapsedMilliseconds, - reasonCode: "credential-entry-cancelled", - exitCode: 1 - ) - } - guard let passwordData = password.data(using: .utf8), - (1 ... 64).contains(passwordData.count) - else { - emit( - ok: false, - scanAttemptCount: scan.attemptCount, - scanElapsedMilliseconds: scan.elapsedMilliseconds, - reasonCode: "credential-invalid", - exitCode: 1 - ) - } - profile = StoredProfile( - schemaVersion: 1, - ssid: expectedSSID, - password: password, - credentialSource: "native-secure-prompt" + let associationPassword: String + if request.action == "associate-ephemeral" { + guard let password = request.password, + let passwordData = password.data(using: .utf8), + (1 ... 64).contains(passwordData.count) + else { + emit( + ok: false, + scanAttemptCount: scan.attemptCount, + scanElapsedMilliseconds: scan.elapsedMilliseconds, + reasonCode: "credential-missing", + exitCode: 1 ) - credentialSource = "native-secure-prompt" - profileNeedsStore = true } - } - guard profile.ssid == expectedSSID else { - emit( - ok: false, - scanAttemptCount: scan.attemptCount, - scanElapsedMilliseconds: scan.elapsedMilliseconds, - reasonCode: "profile-ssid-mismatch", - exitCode: 1 - ) + credentialSource = "operation-memory" + associationPassword = password + } else { + let profile: StoredProfile + do { + // The AP write has already happened. A prepared-host Quick action must + // never trigger a Keychain authorization sheet at this stage. + profile = try loadProfile( + profileID: request.profileID, + interactionAllowed: false + ) + guard profile.credentialSource == "exact-firmware-profile" else { + emit( + ok: false, + scanAttemptCount: scan.attemptCount, + scanElapsedMilliseconds: scan.elapsedMilliseconds, + reasonCode: "profile-credential-source-mismatch", + exitCode: 1 + ) + } + credentialSource = "exact-firmware-profile" + } catch { + emit( + ok: false, + scanAttemptCount: scan.attemptCount, + scanElapsedMilliseconds: scan.elapsedMilliseconds, + reasonCode: keychainReasonCode(error, missing: "profile-unavailable"), + exitCode: 1 + ) + } + guard profile.ssid == expectedSSID else { + emit( + ok: false, + scanAttemptCount: scan.attemptCount, + scanElapsedMilliseconds: scan.elapsedMilliseconds, + reasonCode: "profile-ssid-mismatch", + exitCode: 1 + ) + } + associationPassword = profile.password } - if interface.ssid() == profile.ssid { - if profileNeedsStore { - try storeProfile(profileID: request.profileID, profile: profile) - profileEnrolled = true - } + if interface.ssid() == expectedSSID { emit( ok: true, adapter: "CoreWLAN", alreadyAssociated: true, - profileEnrolled: profileEnrolled, + profileEnrolled: false, scanAttemptCount: scan.attemptCount, scanElapsedMilliseconds: scan.elapsedMilliseconds, credentialSource: credentialSource, exitCode: 0 ) } - try interface.associate(to: network, password: profile.password) - if profileNeedsStore { - try storeProfile(profileID: request.profileID, profile: profile) - profileEnrolled = true - } + try interface.associate(to: network, password: associationPassword) // CoreWLAN's synchronous association call throws on failure. Reading the // current SSID again would require Location authorization on recent macOS // versions and could turn a successful association into a false negative. @@ -638,12 +872,12 @@ do { ok: true, adapter: "CoreWLAN", alreadyAssociated: false, - profileEnrolled: profileEnrolled, + profileEnrolled: false, scanAttemptCount: scan.attemptCount, scanElapsedMilliseconds: scan.elapsedMilliseconds, credentialSource: credentialSource, exitCode: 0 ) } catch { - emit(ok: false, reasonCode: "corewlan-error", exitCode: 1) + emit(ok: false, reasonCode: coreWLANReasonCode(error), exitCode: 1) } diff --git a/plugins/xgrids-k1/plugin.manifest.json b/plugins/xgrids-k1/plugin.manifest.json index 6321012..b460bff 100644 --- a/plugins/xgrids-k1/plugin.manifest.json +++ b/plugins/xgrids-k1/plugin.manifest.json @@ -3,7 +3,7 @@ "kind": "DevicePlugin", "metadata": { "id": "nodedc.device.xgrids-lixelkity-k1", - "version": "0.6.0", + "version": "0.7.5", "displayName": "XGRIDS K1 Integration" }, "spec": { @@ -37,11 +37,15 @@ { "id": "sensor.catalog.read", "mutating": false, "secretFields": [] }, { "id": "calibration.device-snapshot.read", "mutating": false, "secretFields": [] }, { "id": "network.provision", "mutating": true, "secretFields": ["password"] }, + { "id": "connection.mode.select", "mutating": true, "secretFields": [] }, + { "id": "connection.reconfigure.prepare", "mutating": true, "secretFields": [] }, { "id": "connection.verify", "mutating": false, "secretFields": [] }, + { "id": "connection.endpoint-probe", "mutating": false, "secretFields": [] }, { "id": "acquisition.prepare", "mutating": true, "secretFields": [] }, { "id": "acquisition.start", "mutating": true, "secretFields": [] }, { "id": "acquisition.stop", "mutating": true, "secretFields": [] }, { "id": "acquisition.abort", "mutating": true, "secretFields": [] }, + { "id": "acquisition.force-finish-local", "mutating": true, "secretFields": [] }, { "id": "acquisition.state.read", "mutating": false, "secretFields": [] }, { "id": "stream.start-live", "mutating": true, "secretFields": [] }, { "id": "stream.start-replay", "mutating": true, "secretFields": [] }, @@ -54,7 +58,10 @@ { "id": "application-control.shadow-disarm", "mutating": true, "secretFields": [] }, { "id": "application-control.session.open", "mutating": true, "secretFields": [] }, { "id": "application-control.workspace.enter", "mutating": true, "secretFields": [] }, - { "id": "application-control.session.close", "mutating": true, "secretFields": [] } + { "id": "application-control.session.close", "mutating": true, "secretFields": [] }, + { "id": "physical-command.reconcile", "mutating": true, "secretFields": [] }, + { "id": "physical-command.retire-unavailable", "mutating": true, "secretFields": [] }, + { "id": "physical-command.reopen-retired-reconciliation", "mutating": true, "secretFields": [] } ], "models": [ { diff --git a/scripts/build_m4_worker_shadow_artifact.py b/scripts/build_m4_worker_shadow_artifact.py index 408d9ec..2c85543 100644 --- a/scripts/build_m4_worker_shadow_artifact.py +++ b/scripts/build_m4_worker_shadow_artifact.py @@ -25,7 +25,7 @@ WHEEL_NAME = "nodedc_mission_core-0.1.0-py3-none-any.whl" RUNNER_NAME = RUNNER.name PATCH_ID = re.compile(r"^[A-Za-z0-9._-]{1,96}$") EXPECTED_BASELINE_SHA256 = "ea10359339e6cce31b5780a2710299771cab7cc0c1c2a2b56a1621f786b31fa8" -EXPECTED_WHEEL_SHA256 = "ac0ee30446130d3e309cd01e875bec81171a30e057d11d8558a12bb8aec9bf26" +EXPECTED_WHEEL_SHA256 = "9a60efa68eadf2267fffe3dbb89fb58d5e69e672fa3cdf7529474c9f47416acb" PAYLOAD_FILES = ( RUNNER_NAME, WHEEL_NAME, diff --git a/src/k1link/artifacts.py b/src/k1link/artifacts.py index abf523b..3a9dab6 100644 --- a/src/k1link/artifacts.py +++ b/src/k1link/artifacts.py @@ -14,7 +14,12 @@ def utc_now_iso() -> str: def write_json_atomic(path: Path, payload: Any) -> None: - """Write JSON without exposing a partially written artifact.""" + """Write JSON without exposing or acknowledging a partial commit. + + Flushing the temporary file protects its contents, but a crash can still + lose the directory entry created by ``replace``. The parent directory is + therefore flushed after the atomic rename as the second durability edge. + """ path = path.expanduser() path.parent.mkdir(parents=True, exist_ok=True) serialized = json.dumps(payload, ensure_ascii=False, indent=2) + "\n" @@ -34,6 +39,13 @@ def write_json_atomic(path: Path, payload: Any) -> None: stream.flush() os.fsync(stream.fileno()) Path(temp_name).replace(path) + directory_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + directory_flags |= getattr(os, "O_DIRECTORY", 0) + directory_descriptor = os.open(path.parent, directory_flags) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) finally: if temp_name is not None: Path(temp_name).unlink(missing_ok=True) diff --git a/src/k1link/compute/l34a_assisted_yolox_error_audit.py b/src/k1link/compute/l34a_assisted_yolox_error_audit.py index a29a8b0..02c579b 100644 --- a/src/k1link/compute/l34a_assisted_yolox_error_audit.py +++ b/src/k1link/compute/l34a_assisted_yolox_error_audit.py @@ -142,7 +142,10 @@ def build_l34a_assisted_yolox_error_audit( }, "limitations": [ "the review was seeded from the same frozen candidate and is not independent truth", - "precision, recall and F1 are assisted diagnostic alignment metrics, not acceptance metrics", + ( + "precision, recall and F1 are assisted diagnostic alignment metrics, " + "not acceptance metrics" + ), "custom labels are retained as proposed ontology terms and were not adjudicated", "the result is source-scoped to 32 RAVNOVES00 right-camera frames", ], diff --git a/src/k1link/compute/live_perception.py b/src/k1link/compute/live_perception.py index d942484..e411b94 100644 --- a/src/k1link/compute/live_perception.py +++ b/src/k1link/compute/live_perception.py @@ -10,7 +10,7 @@ import threading import time import zlib from collections import deque -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from hashlib import sha256 from typing import Any, Final, Literal @@ -33,7 +33,7 @@ LiveIngressModality = Literal[ LIVE_INGRESS_SCHEMA: Final = "missioncore.live-perception-ingress/v1" LIVE_INGRESS_WIRE_SCHEMA: Final = "missioncore.live-perception-wire/v1" -LIVE_RESULT_WIRE_SCHEMA: Final = "missioncore.live-perception-result-wire/v1" +LIVE_RESULT_WIRE_SCHEMA: Final = "missioncore.live-perception-result-wire/v2" LIVE_RESULT_MAGIC: Final = b"MCPR" LIVE_RESULT_MAX_HEADER_BYTES: Final = 256 * 1024 LIVE_RESULT_MAX_PAYLOAD_BYTES: Final = 2 * 1024 * 1024 @@ -41,6 +41,8 @@ LIVE_RESULT_MAX_PAYLOAD_BYTES: Final = 2 * 1024 * 1024 @dataclass(frozen=True, slots=True) class LivePerceptionResultFrame: + session_id: str + session_generation: int frame_index: int source_frame_index: int session_seconds: float @@ -53,6 +55,8 @@ class LivePerceptionResultFrame: def encode_live_perception_result( *, + session_id: str, + session_generation: int, frame_index: int, source_frame_index: int, session_seconds: float, @@ -65,7 +69,11 @@ def encode_live_perception_result( """Encode one bounded, non-authoritative worker-to-viewer result frame.""" if ( - frame_index < 0 + not session_id + or len(session_id) > 160 + or isinstance(session_generation, bool) + or session_generation < 1 + or frame_index < 0 or source_frame_index < 0 or captured_at_epoch_ns < 0 or not math.isfinite(session_seconds) @@ -91,6 +99,8 @@ def encode_live_perception_result( raise ValueError("live perception result payload exceeds the bound") header = { "schema_version": LIVE_RESULT_WIRE_SCHEMA, + "session_id": session_id, + "session_generation": session_generation, "frame_index": frame_index, "source_frame_index": source_frame_index, "session_seconds": session_seconds, @@ -192,11 +202,19 @@ def decode_live_perception_result(encoded: bytes) -> LivePerceptionResultFrame: mask = np.frombuffer(raw_mask, dtype=np.uint8).reshape((600, 800)).copy() normalized_objects = tuple(_normalize_live_result_object(value) for value in objects) frame_index = header.get("frame_index") + session_id = header.get("session_id") + session_generation = header.get("session_generation") source_frame_index = header.get("source_frame_index") session_seconds = header.get("session_seconds") captured_at_epoch_ns = header.get("captured_at_epoch_ns") if ( - not isinstance(frame_index, int) + not isinstance(session_id, str) + or not session_id + or len(session_id) > 160 + or not isinstance(session_generation, int) + or isinstance(session_generation, bool) + or session_generation < 1 + or not isinstance(frame_index, int) or isinstance(frame_index, bool) or frame_index < 0 or not isinstance(source_frame_index, int) @@ -212,6 +230,8 @@ def decode_live_perception_result(encoded: bytes) -> LivePerceptionResultFrame: ): raise ValueError("live perception result time identity is invalid") return LivePerceptionResultFrame( + session_id=session_id, + session_generation=session_generation, frame_index=frame_index, source_frame_index=source_frame_index, session_seconds=float(session_seconds), @@ -297,6 +317,7 @@ class LiveIngressEvent: ingress_sequence: int session_id: str + session_generation: int modality: LiveIngressModality source_id: str source_sequence: int @@ -310,6 +331,7 @@ class LiveIngressEvent: "schema_version": LIVE_INGRESS_WIRE_SCHEMA, "ingress_sequence": self.ingress_sequence, "session_id": self.session_id, + "session_generation": self.session_generation, "modality": self.modality, "source_id": self.source_id, "source_sequence": self.source_sequence, @@ -382,9 +404,13 @@ class LivePerceptionIngress: } self._ingress_sequence = 0 self._session_id: str | None = None + self._session_generation = 0 self._active = False self._closed = False self._consumer_id: str | None = None + self._results_accepted = 0 + self._results_rejected_stale = 0 + self._results_rejected_receiver = 0 def begin_session(self, session_id: str) -> None: if not session_id or len(session_id) > 160: @@ -402,6 +428,7 @@ class LivePerceptionIngress: for queue in self._queues.values(): queue.items.clear() self._session_id = session_id + self._session_generation += 1 self._active = True self._publish_locked( modality="control", @@ -470,6 +497,31 @@ class LivePerceptionIngress: self._consumer_id = None self._condition.notify_all() + def admit_result( + self, + *, + session_id: str, + session_generation: int, + receiver: Callable[[], bool], + ) -> bool: + """Atomically reject a late worker result before viewer admission.""" + + with self._condition: + if ( + self._closed + or not self._active + or self._session_id != session_id + or self._session_generation != session_generation + ): + self._results_rejected_stale += 1 + return False + accepted = receiver() + if accepted: + self._results_accepted += 1 + else: + self._results_rejected_receiver += 1 + return accepted + def take_next( self, consumer_id: str, @@ -509,7 +561,11 @@ class LivePerceptionIngress: "mode": "shadow-diagnostic-only", "active": self._active, "session_id": self._session_id, + "session_generation": self._session_generation, "consumer_connected": self._consumer_id is not None, + "results_accepted": self._results_accepted, + "results_rejected_stale": self._results_rejected_stale, + "results_rejected_receiver": self._results_rejected_receiver, "commands_enabled": False, "navigation_or_safety_accepted": False, "closed": self._closed, @@ -548,6 +604,7 @@ class LivePerceptionIngress: event = LiveIngressEvent( ingress_sequence=self._ingress_sequence, session_id=session_id, + session_generation=self._session_generation, modality=modality, source_id=source_id, source_sequence=source_sequence, diff --git a/src/k1link/device_plugins/xgrids_k1/active_acquisition_recovery_checkpoint.py b/src/k1link/device_plugins/xgrids_k1/active_acquisition_recovery_checkpoint.py new file mode 100644 index 0000000..2f1a344 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/active_acquisition_recovery_checkpoint.py @@ -0,0 +1,5368 @@ +from __future__ import annotations + +import fcntl +import hashlib +import ipaddress +import json +import math +import os +import re +import stat +import tempfile +import threading +import unicodedata +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, Final, Literal, cast + +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + PhysicalCommandClassifiedStopAncestryProof, +) +from k1link.sessions.store import resolve_missioncore_data_dir + +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA: Final = ( + "missioncore.xgrids-k1-active-acquisition-recovery-checkpoint/v5" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA: Final = ( + "missioncore.xgrids-k1-active-acquisition-recovery-checkpoint/v4" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA: Final = ( + "missioncore.xgrids-k1-active-acquisition-recovery-checkpoint/v3" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA: Final = ( + "missioncore.xgrids-k1-active-acquisition-recovery-checkpoint/v2" +) +ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA: Final = ( + "missioncore.xgrids-k1-physical-command/v5" +) +ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA: Final = ( + "missioncore.xgrids-k1-active-acquisition-start-origin-proof/v1" +) +ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA: Final = ( + "missioncore.xgrids-k1-first-published-pcl-proof/v1" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_FILENAME = ( + "active-acquisition-recovery-checkpoint.json" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_LOCK_FILENAME = ( + ".active-acquisition-recovery-checkpoint.lock" +) +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES = 32 * 1024 +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION = (1 << 63) - 1 +ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS = 32 + +ActiveAcquisitionRecoveryCheckpointState = Literal["prepared", "active", "ceased"] +ActiveAcquisitionRecoveryCheckpointStatus = Literal[ + "empty", + "prepared", + "active", + "ceased", + "corrupt", +] +ActiveAcquisitionRecoveryConnectionMode = Literal[ + "bridge", + "quick-connect", + "direct-connect", +] +ActiveAcquisitionRecoveryEvidencePolicy = Literal["required", "best-effort", "disabled"] +ActiveAcquisitionRecoveryMountType = Literal["handheld"] +ActiveAcquisitionRecoveryGnssMode = Literal["none"] +ActiveAcquisitionRecoveryProofSource = Literal[ + "live-control-session", + "explicit-read-only-reconciliation", +] +ActiveAcquisitionRecoveryProofState = Literal["scanning", "ready", "scan_over"] +ActiveAcquisitionRecoveryPhysicalAction = Literal["start", "stop"] +ActiveAcquisitionRecoveryPhysicalResolution = Literal[ + "not-dispatched", + "start-active-observed", + "stop-standby-observed", + "physical-active-observed", + "physical-standby-observed", +] +ActiveAcquisitionRecoveryPhysicalReconciliationKind = Literal[ + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", +] +ActiveAcquisitionRecoveryStopFence = Literal[ + "none", + "definitely-not-dispatched", +] +ActiveAcquisitionRecoveryStartOriginKind = Literal[ + "composite-resolved", + "ambiguous-reconciled", +] +ActiveAcquisitionRecoveryStartAttemptStage = Literal[ + "dispatching", + "observing", + "resolved", +] +ActiveAcquisitionRecoveryProjectEvidenceStrength = Literal[ + "exact-vendor-project-id", + "edge-correlated-vendor-project-id", +] +ActiveAcquisitionRecoveryTransitionKind = Literal[ + "prepare", + "activate", + "activate-reconciled", + "record-gap", + "rebind-active", + "cease", + "cease-active-reconciled-standby", + "cease-prepared-reconciled", + "cease-prepared-resolved-start-standby", +] + +_CHECKPOINT_STATES = frozenset({"prepared", "active", "ceased"}) +_CONNECTION_MODES = frozenset({"bridge", "quick-connect", "direct-connect"}) +_EVIDENCE_POLICIES = frozenset({"required", "best-effort", "disabled"}) +_MOUNT_TYPES = frozenset({"handheld"}) +_GNSS_MODES = frozenset({"none"}) +_PROOF_SOURCES = frozenset( + {"live-control-session", "explicit-read-only-reconciliation"} +) +_PROOF_STATE_CODES = {"ready": 300, "scanning": 302, "scan_over": 304} +_PHYSICAL_ACTIONS = frozenset({"start", "stop"}) +_PHYSICAL_RESOLUTIONS = frozenset( + { + "not-dispatched", + "start-active-observed", + "stop-standby-observed", + "physical-active-observed", + "physical-standby-observed", + } +) +_PHYSICAL_RECONCILIATION_KINDS = frozenset( + { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", + } +) +_STOP_FENCES = frozenset({"none", "definitely-not-dispatched"}) +_START_ORIGIN_KINDS = frozenset( + {"composite-resolved", "ambiguous-reconciled"} +) +_START_ATTEMPT_STAGES = frozenset({"dispatching", "observing", "resolved"}) +_PROJECT_EVIDENCE_STRENGTHS = frozenset( + {"exact-vendor-project-id", "edge-correlated-vendor-project-id"} +) +_TRANSITION_KINDS = frozenset( + { + "prepare", + "activate", + "activate-reconciled", + "record-gap", + "rebind-active", + "cease", + "cease-active-reconciled-standby", + "cease-prepared-reconciled", + "cease-prepared-resolved-start-standby", + } +) +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_MAX_REQUESTED_STREAMS = 32 + + +class ActiveAcquisitionRecoveryCheckpointError(RuntimeError): + """Base error for the durable, authority-free recovery checkpoint.""" + + reason_code = "active-acquisition-recovery-checkpoint-error" + + +class ActiveAcquisitionRecoveryCheckpointCorrupt( + ActiveAcquisitionRecoveryCheckpointError +): + """The checkpoint or its process lock cannot be trusted.""" + + reason_code = "active-acquisition-recovery-checkpoint-corrupt" + + +class StaleActiveAcquisitionRecoveryCheckpoint( + ActiveAcquisitionRecoveryCheckpointError +): + """A CAS writer no longer descends from the durable revision.""" + + reason_code = "active-acquisition-recovery-checkpoint-stale" + + +class ActiveAcquisitionRecoveryCheckpointTransitionError( + ActiveAcquisitionRecoveryCheckpointError +): + """A requested local checkpoint transition is semantically invalid.""" + + reason_code = "active-acquisition-recovery-checkpoint-transition-invalid" + + +class ActiveAcquisitionRecoveryCheckpointIdempotencyConflict( + ActiveAcquisitionRecoveryCheckpointError +): + """A transition id was already committed with different exact bytes.""" + + reason_code = "active-acquisition-recovery-checkpoint-idempotency-conflict" + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryIdentity: + logical_device_id: str + vendor_device_id_sha256: str + device_serial_sha256: str + + def __post_init__(self) -> None: + _validate_identifier(self.logical_device_id, field_name="logical_device_id") + _validate_sha256( + self.vendor_device_id_sha256, + field_name="vendor_device_id_sha256", + ) + _validate_sha256( + self.device_serial_sha256, + field_name="device_serial_sha256", + ) + + def as_dict(self) -> dict[str, object]: + return { + "logical_device_id": self.logical_device_id, + "vendor_device_id_sha256": self.vendor_device_id_sha256, + "device_serial_sha256": self.device_serial_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryConnection: + transport_ref: str + connection_mode: ActiveAcquisitionRecoveryConnectionMode + target_ipv4: str + target_port: int + + def __post_init__(self) -> None: + _validate_identifier(self.transport_ref, field_name="transport_ref") + _validate_connection_mode(self.connection_mode) + _validate_ipv4(self.target_ipv4, field_name="target_ipv4") + _validate_port(self.target_port, field_name="target_port") + + def as_dict(self) -> dict[str, object]: + return { + "transport_ref": self.transport_ref, + "connection_mode": self.connection_mode, + "target_ipv4": self.target_ipv4, + "target_port": self.target_port, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryTransportBinding: + """One exact process-local verified control binding. + + host_path_epoch and producer_generation are deliberately not ordered across + runtime_instance_id values. Durable ordering belongs only to the checkpoint + transport_revision. + """ + + runtime_instance_id: str + intent_id: str + transport_ref: str + connection_mode: ActiveAcquisitionRecoveryConnectionMode + target_ipv4: str + target_port: int + host_path_epoch: int + control_session_id: str + producer_generation: int + logical_device_id: str + compatibility_profile_id: str + vendor_device_id_sha256: str + device_serial_sha256: str + + def __post_init__(self) -> None: + for field_name, value in ( + ("runtime_instance_id", self.runtime_instance_id), + ("intent_id", self.intent_id), + ("transport_ref", self.transport_ref), + ("control_session_id", self.control_session_id), + ("logical_device_id", self.logical_device_id), + ("compatibility_profile_id", self.compatibility_profile_id), + ): + _validate_identifier(value, field_name=field_name) + _validate_connection_mode(self.connection_mode) + _validate_ipv4(self.target_ipv4, field_name="target_ipv4") + _validate_port(self.target_port, field_name="target_port") + _validate_positive_int(self.host_path_epoch, field_name="host_path_epoch") + _validate_positive_int( + self.producer_generation, + field_name="producer_generation", + ) + _validate_sha256( + self.vendor_device_id_sha256, + field_name="vendor_device_id_sha256", + ) + _validate_sha256( + self.device_serial_sha256, + field_name="device_serial_sha256", + ) + + def as_dict(self) -> dict[str, object]: + return { + "runtime_instance_id": self.runtime_instance_id, + "intent_id": self.intent_id, + "transport_ref": self.transport_ref, + "connection_mode": self.connection_mode, + "target_ipv4": self.target_ipv4, + "target_port": self.target_port, + "host_path_epoch": self.host_path_epoch, + "control_session_id": self.control_session_id, + "producer_generation": self.producer_generation, + "logical_device_id": self.logical_device_id, + "compatibility_profile_id": self.compatibility_profile_id, + "vendor_device_id_sha256": self.vendor_device_id_sha256, + "device_serial_sha256": self.device_serial_sha256, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryStatusProof: + source: ActiveAcquisitionRecoveryProofSource + session_state: ActiveAcquisitionRecoveryProofState + session_state_code: int + binding: ActiveAcquisitionRecoveryTransportBinding + evidence_session_id: str + project_id_sha256: str | None + init_ready: bool + status_message_sha256: str + mqtt_retained: Literal[False] + observed_at_utc: str + + def __post_init__(self) -> None: + _validate_proof_source(self.source) + _validate_proof_state(self.session_state, self.session_state_code) + if not isinstance(self.binding, ActiveAcquisitionRecoveryTransportBinding): + raise TypeError("binding must be ActiveAcquisitionRecoveryTransportBinding") + _validate_identifier( + self.evidence_session_id, + field_name="evidence_session_id", + ) + if self.session_state == "scanning": + if self.project_id_sha256 is None: + raise ValueError("SCANNING proof requires project_id_sha256") + _validate_sha256(self.project_id_sha256, field_name="project_id_sha256") + if self.init_ready is not True: + raise ValueError("SCANNING proof requires init_ready=true") + else: + if self.project_id_sha256 is not None: + raise ValueError("READY/SCAN_OVER proof must be project-unbound") + if self.init_ready is not False: + raise ValueError("READY/SCAN_OVER proof requires init_ready=false") + _validate_sha256( + self.status_message_sha256, + field_name="status_message_sha256", + ) + if self.mqtt_retained is not False: + raise ValueError("checkpoint proof must be fresh and non-retained") + _canonical_timestamp_value( + self.observed_at_utc, + field_name="observed_at_utc", + ) + + def as_dict(self) -> dict[str, object]: + return { + "source": self.source, + "session_state": self.session_state, + "session_state_code": self.session_state_code, + "binding": self.binding.as_dict(), + "evidence_session_id": self.evidence_session_id, + "project_id_sha256": self.project_id_sha256, + "init_ready": self.init_ready, + "status_message_sha256": self.status_message_sha256, + "mqtt_retained": self.mqtt_retained, + "observed_at_utc": self.observed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryPhysicalLineageProof: + """Typed projection of one exact terminal physical-ledger head. + + ``action`` and ``resolution`` always describe the actual current ledger + record. A later read-only observation is represented separately by + ``reconciliation_kind`` and ``reconciliation_resolution``. This prevents a + RESOLVED/not-dispatched STOP from being rewritten as a synthetic START merely + because its latest fresh observation proves that the original scan is still + active. + """ + + ledger_schema_version: Literal["missioncore.xgrids-k1-physical-command/v5"] + ledger_revision: int + proof_id: str + operation_id: str + original_start_operation_id: str + parent_operation_id: str | None + acquisition_id: str + action: ActiveAcquisitionRecoveryPhysicalAction + resolution: ActiveAcquisitionRecoveryPhysicalResolution + payload_sha256: str + original_start_payload_sha256: str + reconciliation_kind: ActiveAcquisitionRecoveryPhysicalReconciliationKind | None + reconciliation_resolution: ActiveAcquisitionRecoveryPhysicalResolution | None + status_message_sha256: str | None + observed_session_state: ActiveAcquisitionRecoveryProofState | None + binding: ActiveAcquisitionRecoveryTransportBinding + composite_complete: bool + edge_terminal: Literal[True] + late_start_excluded: Literal[True] + stop_fence: ActiveAcquisitionRecoveryStopFence + observed_at_utc: str + ancestor_chain: PhysicalCommandClassifiedStopAncestryProof | None = None + # Hash of the immutable command-attempt audit embedded by the *latest* + # reconciliation. Older v5 checkpoint documents omit this field; they + # remain readable, but a new crash-era PREPARED activation requires it. + reconciliation_original_attempt_sha256: str | None = None + + def __post_init__(self) -> None: + if ( + self.ledger_schema_version + != ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA + ): + raise ValueError("unsupported physical lineage schema") + _validate_positive_revision(self.ledger_revision, field_name="ledger_revision") + for field_name, value in ( + ("proof_id", self.proof_id), + ("operation_id", self.operation_id), + ("original_start_operation_id", self.original_start_operation_id), + ("acquisition_id", self.acquisition_id), + ): + _validate_identifier(value, field_name=field_name) + if self.parent_operation_id is not None: + _validate_identifier( + self.parent_operation_id, + field_name="parent_operation_id", + ) + _validate_physical_action(self.action) + _validate_physical_resolution(self.resolution) + _validate_sha256(self.payload_sha256, field_name="payload_sha256") + _validate_sha256( + self.original_start_payload_sha256, + field_name="original_start_payload_sha256", + ) + if (self.reconciliation_kind is None) != ( + self.reconciliation_resolution is None + ): + raise ValueError( + "physical reconciliation kind and resolution must be paired" + ) + if self.reconciliation_kind is not None: + _validate_physical_reconciliation_kind(self.reconciliation_kind) + assert self.reconciliation_resolution is not None + _validate_physical_resolution(self.reconciliation_resolution) + if self.reconciliation_original_attempt_sha256 is not None: + if self.reconciliation_kind is None: + raise ValueError( + "non-reconciled physical lineage cannot carry an attempt hash" + ) + _validate_sha256( + self.reconciliation_original_attempt_sha256, + field_name="reconciliation_original_attempt_sha256", + ) + if not isinstance(self.binding, ActiveAcquisitionRecoveryTransportBinding): + raise TypeError("binding must be ActiveAcquisitionRecoveryTransportBinding") + if not isinstance(self.composite_complete, bool): + raise TypeError("composite_complete must be bool") + if self.edge_terminal is not True or self.late_start_excluded is not True: + raise ValueError("physical lineage must be terminal and late-edge fenced") + _validate_stop_fence(self.stop_fence) + effective_resolution = self.effective_resolution + if self.action == "start" and self.operation_id == ( + self.original_start_operation_id + ) and self.original_start_payload_sha256 != self.payload_sha256: + raise ValueError("original START payload must match the START ledger head") + if self.action == "stop": + if self.operation_id == self.original_start_operation_id: + raise ValueError("STOP lineage cannot reuse the original START") + if self.parent_operation_id is None: + raise ValueError("STOP lineage requires an immediate parent") + if self.parent_operation_id == self.original_start_operation_id: + if self.ancestor_chain is not None: + raise ValueError( + "direct START parent must not carry an ancestry certificate" + ) + else: + ancestry = self.ancestor_chain + if not isinstance( + ancestry, + PhysicalCommandClassifiedStopAncestryProof, + ): + raise ValueError( + "chained STOP lineage requires a typed ancestry certificate" + ) + if ( + ancestry.ledger_revision != self.ledger_revision + or ancestry.head_operation_id != self.operation_id + or ancestry.head_parent_operation_id != self.parent_operation_id + or ancestry.root_start_operation_id + != self.original_start_operation_id + or ancestry.acquisition_id != self.acquisition_id + ): + raise ValueError( + "chained STOP ancestry does not match the exact physical head" + ) + if self.resolution == "not-dispatched": + if self.stop_fence != "definitely-not-dispatched": + raise ValueError( + "undispatched STOP requires a definitely-not-dispatched fence" + ) + elif self.stop_fence != "none": + raise ValueError("terminal dispatched STOP cannot retain an active fence") + elif self.stop_fence != "none": + raise ValueError("START lineage cannot contain a STOP fence") + elif self.ancestor_chain is not None: + raise ValueError("START lineage cannot contain STOP ancestry") + if self.resolution == "not-dispatched" and self.action == "start": + if ( + self.operation_id != self.original_start_operation_id + or self.status_message_sha256 is not None + or self.observed_session_state is not None + or self.composite_complete + or self.reconciliation_kind is not None + ): + raise ValueError("not-dispatched lineage is internally inconsistent") + else: + if self.status_message_sha256 is None: + raise ValueError("observed physical lineage requires a status digest") + _validate_sha256( + self.status_message_sha256, + field_name="status_message_sha256", + ) + if self.observed_session_state is None: + raise ValueError("observed physical lineage requires a session state") + _validate_proof_state( + self.observed_session_state, + _PROOF_STATE_CODES[self.observed_session_state], + ) + if effective_resolution in { + "start-active-observed", + "physical-active-observed", + } and self.observed_session_state != "scanning": + raise ValueError("active physical lineage must carry fresh SCANNING") + if self.resolution == "start-active-observed" and not self.composite_complete: + raise ValueError("ordinary START resolution must be composite-complete") + if self.resolution == "stop-standby-observed" and ( + self.action != "stop" + or self.observed_session_state not in {"ready", "scan_over"} + or not self.composite_complete + ): + raise ValueError("ordinary STOP resolution must be composite-complete") + if self.resolution == "physical-standby-observed" and ( + self.observed_session_state not in {"ready", "scan_over"} + or self.composite_complete + ): + raise ValueError("standby reconciliation is internally inconsistent") + if self.resolution == "physical-active-observed" and self.composite_complete: + raise ValueError("active reconciliation is not a composite command proof") + if self.reconciliation_kind == "prepared-stop-classification" and not ( + self.action == "stop" + and self.resolution == "not-dispatched" + and self.reconciliation_resolution + in {"physical-active-observed", "physical-standby-observed"} + and not self.composite_complete + and self.stop_fence == "definitely-not-dispatched" + ): + raise ValueError("prepared STOP classification is internally inconsistent") + if self.reconciliation_kind == "resolved-active-rebind" and not ( + self.reconciliation_resolution == "physical-active-observed" + and self.observed_session_state == "scanning" + and ( + ( + self.action == "start" + and self.operation_id == self.original_start_operation_id + and self.resolution == "start-active-observed" + and self.composite_complete + and self.stop_fence == "none" + ) + or ( + self.action == "start" + and self.operation_id == self.original_start_operation_id + and self.resolution == "physical-active-observed" + and not self.composite_complete + and self.stop_fence == "none" + ) + or ( + self.action == "stop" + and self.resolution == "not-dispatched" + and not self.composite_complete + and self.stop_fence == "definitely-not-dispatched" + ) + ) + ): + raise ValueError("resolved active rebind is internally inconsistent") + if self.reconciliation_kind == "resolved-active-cessation" and not ( + self.reconciliation_resolution == "physical-standby-observed" + and self.observed_session_state in {"ready", "scan_over"} + ): + raise ValueError("resolved active cessation is internally inconsistent") + if self.reconciliation_kind == "ambiguous-outcome" and not ( + ( + self.action == "start" + and self.operation_id == self.original_start_operation_id + ) + or ( + self.action == "stop" + and self.operation_id != self.original_start_operation_id + and self.parent_operation_id is not None + ) + ): + raise ValueError("ambiguous command classification has invalid lineage") + if self.reconciliation_kind == "ambiguous-outcome" and not ( + self.resolution + in {"physical-active-observed", "physical-standby-observed"} + and self.reconciliation_resolution == self.resolution + and not self.composite_complete + and self.stop_fence == "none" + ): + raise ValueError("ambiguous command classification is internally inconsistent") + _canonical_timestamp_value( + self.observed_at_utc, + field_name="observed_at_utc", + ) + + @property + def effective_resolution(self) -> ActiveAcquisitionRecoveryPhysicalResolution: + """Return the latest observed physical result without rewriting the head.""" + + return self.reconciliation_resolution or self.resolution + + def as_dict(self) -> dict[str, object]: + document: dict[str, object] = { + "ledger_schema_version": self.ledger_schema_version, + "ledger_revision": self.ledger_revision, + "proof_id": self.proof_id, + "operation_id": self.operation_id, + "original_start_operation_id": self.original_start_operation_id, + "parent_operation_id": self.parent_operation_id, + "acquisition_id": self.acquisition_id, + "action": self.action, + "resolution": self.resolution, + "payload_sha256": self.payload_sha256, + "original_start_payload_sha256": self.original_start_payload_sha256, + "reconciliation_kind": self.reconciliation_kind, + "reconciliation_resolution": self.reconciliation_resolution, + "status_message_sha256": self.status_message_sha256, + "observed_session_state": self.observed_session_state, + "binding": self.binding.as_dict(), + "composite_complete": self.composite_complete, + "edge_terminal": self.edge_terminal, + "late_start_excluded": self.late_start_excluded, + "stop_fence": self.stop_fence, + "observed_at_utc": self.observed_at_utc, + "ancestor_chain": ( + self.ancestor_chain.as_dict() + if self.ancestor_chain is not None + else None + ), + } + if self.reconciliation_original_attempt_sha256 is not None: + document["reconciliation_original_attempt_sha256"] = ( + self.reconciliation_original_attempt_sha256 + ) + return document + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryStartOriginProof: + """Exact archived original START attempt paired with its current proof.""" + + schema_version: Literal[ + "missioncore.xgrids-k1-active-acquisition-start-origin-proof/v1" + ] + ledger_schema_version: Literal[ + "missioncore.xgrids-k1-physical-command/v5" + ] + ledger_revision: int + physical_proof_id: str + reconciliation_id: str + origin_kind: ActiveAcquisitionRecoveryStartOriginKind + original_attempt_revision: int + original_attempt_stage: ActiveAcquisitionRecoveryStartAttemptStage + original_attempt_resolution: Literal["start-active-observed"] | None + original_attempt_sha256: str + operation_id: str + parent_operation_id: str | None + acquisition_id: str + payload_sha256: str + baseline_status_proof: ActiveAcquisitionRecoveryStatusProof + original_project_id_sha256: str | None + automatic_replay_allowed: Literal[False] + project_evidence_strength: ActiveAcquisitionRecoveryProjectEvidenceStrength + # Project observed on the active, non-retained reconciliation edge. It is + # distinct from `original_project_id_sha256`: an ambiguous START did not + # itself prove the project, but every later read-only rebind must preserve + # this edge-correlated project identity. + reconciled_active_project_id_sha256: str | None = None + + def __post_init__(self) -> None: + if ( + self.schema_version + != ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA + ): + raise ValueError("unsupported START origin proof schema") + if ( + self.ledger_schema_version + != ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA + ): + raise ValueError("unsupported START origin physical ledger schema") + _validate_positive_revision(self.ledger_revision, field_name="ledger_revision") + _validate_positive_revision( + self.original_attempt_revision, + field_name="original_attempt_revision", + ) + if self.original_attempt_revision >= self.ledger_revision: + raise ValueError("START origin attempt must precede its reconciliation head") + for field_name, value in ( + ("physical_proof_id", self.physical_proof_id), + ("reconciliation_id", self.reconciliation_id), + ("operation_id", self.operation_id), + ("acquisition_id", self.acquisition_id), + ): + _validate_identifier(value, field_name=field_name) + if self.parent_operation_id is not None: + _validate_identifier( + self.parent_operation_id, + field_name="parent_operation_id", + ) + if self.origin_kind not in _START_ORIGIN_KINDS: + raise ValueError("unsupported START origin kind") + if self.original_attempt_stage not in _START_ATTEMPT_STAGES: + raise ValueError("unsupported START origin attempt stage") + _validate_sha256( + self.original_attempt_sha256, + field_name="original_attempt_sha256", + ) + _validate_sha256(self.payload_sha256, field_name="payload_sha256") + if not isinstance( + self.baseline_status_proof, + ActiveAcquisitionRecoveryStatusProof, + ): + raise TypeError( + "baseline_status_proof must be ActiveAcquisitionRecoveryStatusProof" + ) + if not ( + self.baseline_status_proof.source == "live-control-session" + and self.baseline_status_proof.session_state == "ready" + and self.baseline_status_proof.project_id_sha256 is None + and self.baseline_status_proof.init_ready is False + and self.baseline_status_proof.mqtt_retained is False + ): + raise ValueError( + "START origin baseline must be fresh unbound live-session READY" + ) + if self.original_project_id_sha256 is not None: + _validate_sha256( + self.original_project_id_sha256, + field_name="original_project_id_sha256", + ) + if self.reconciled_active_project_id_sha256 is not None: + _validate_sha256( + self.reconciled_active_project_id_sha256, + field_name="reconciled_active_project_id_sha256", + ) + if self.automatic_replay_allowed is not False: + raise ValueError("reconciled START origin never permits command replay") + if self.project_evidence_strength not in _PROJECT_EVIDENCE_STRENGTHS: + raise ValueError("unsupported START project evidence strength") + if self.origin_kind == "composite-resolved": + if not ( + self.original_attempt_stage == "resolved" + and self.original_attempt_resolution == "start-active-observed" + and self.original_project_id_sha256 is not None + and self.project_evidence_strength == "exact-vendor-project-id" + ): + raise ValueError("composite START origin is internally inconsistent") + elif not ( + self.original_attempt_stage in {"dispatching", "observing"} + and self.original_attempt_resolution is None + and self.original_project_id_sha256 is None + and self.project_evidence_strength + == "edge-correlated-vendor-project-id" + ): + raise ValueError("ambiguous START origin is internally inconsistent") + + def as_dict(self) -> dict[str, object]: + document: dict[str, object] = { + "schema_version": self.schema_version, + "ledger_schema_version": self.ledger_schema_version, + "ledger_revision": self.ledger_revision, + "physical_proof_id": self.physical_proof_id, + "reconciliation_id": self.reconciliation_id, + "origin_kind": self.origin_kind, + "original_attempt_revision": self.original_attempt_revision, + "original_attempt_stage": self.original_attempt_stage, + "original_attempt_resolution": self.original_attempt_resolution, + "original_attempt_sha256": self.original_attempt_sha256, + "operation_id": self.operation_id, + "parent_operation_id": self.parent_operation_id, + "acquisition_id": self.acquisition_id, + "payload_sha256": self.payload_sha256, + "baseline_status_proof": self.baseline_status_proof.as_dict(), + "original_project_id_sha256": self.original_project_id_sha256, + "automatic_replay_allowed": self.automatic_replay_allowed, + "project_evidence_strength": self.project_evidence_strength, + } + if self.reconciled_active_project_id_sha256 is not None: + document["reconciled_active_project_id_sha256"] = ( + self.reconciled_active_project_id_sha256 + ) + return document + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryFirstPublishedPclProof: + """Receipt for the first non-empty PCL frame already published to Rerun.""" + + schema_version: Literal[ + "missioncore.xgrids-k1-first-published-pcl-proof/v1" + ] + proof_id: str + runtime_instance_id: str + evidence_session_id: str + viewer_producer_generation: int + frame_sequence: int + captured_at_epoch_ns: int + point_count: int + published_pcl_frames: int + published_at_utc: str + + def __post_init__(self) -> None: + if ( + self.schema_version + != ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA + ): + raise ValueError("unsupported first-published-PCL proof schema") + for field_name, value in ( + ("proof_id", self.proof_id), + ("runtime_instance_id", self.runtime_instance_id), + ("evidence_session_id", self.evidence_session_id), + ): + _validate_identifier(value, field_name=field_name) + _validate_positive_int( + self.viewer_producer_generation, + field_name="viewer_producer_generation", + ) + _validate_positive_int(self.frame_sequence, field_name="frame_sequence") + _validate_nonnegative_int_value( + self.captured_at_epoch_ns, + field_name="captured_at_epoch_ns", + ) + _validate_positive_int(self.point_count, field_name="point_count") + _validate_positive_int( + self.published_pcl_frames, + field_name="published_pcl_frames", + ) + _canonical_timestamp_value( + self.published_at_utc, + field_name="published_at_utc", + ) + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "proof_id": self.proof_id, + "runtime_instance_id": self.runtime_instance_id, + "evidence_session_id": self.evidence_session_id, + "viewer_producer_generation": self.viewer_producer_generation, + "frame_sequence": self.frame_sequence, + "captured_at_epoch_ns": self.captured_at_epoch_ns, + "point_count": self.point_count, + "published_pcl_frames": self.published_pcl_frames, + "published_at_utc": self.published_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryTransitionReceipt: + transition_id_sha256: str + kind: ActiveAcquisitionRecoveryTransitionKind + request_fingerprint_sha256: str + committed_revision: int + committed_at_utc: str + + def __post_init__(self) -> None: + _validate_sha256( + self.transition_id_sha256, + field_name="transition_id_sha256", + ) + _validate_transition_kind(self.kind) + _validate_sha256( + self.request_fingerprint_sha256, + field_name="request_fingerprint_sha256", + ) + _validate_positive_revision( + self.committed_revision, + field_name="committed_revision", + ) + _canonical_timestamp_value( + self.committed_at_utc, + field_name="committed_at_utc", + ) + + def as_dict(self) -> dict[str, object]: + return { + "transition_id_sha256": self.transition_id_sha256, + "kind": self.kind, + "request_fingerprint_sha256": self.request_fingerprint_sha256, + "committed_revision": self.committed_revision, + "committed_at_utc": self.committed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryCheckpoint: + schema_version: Literal[ + "missioncore.xgrids-k1-active-acquisition-recovery-checkpoint/v5" + ] + revision: int + state: ActiveAcquisitionRecoveryCheckpointState + transport_revision: int + physical_lineage_head_revision: int | None + acquisition_id: str + original_start_operation_id: str + start_payload_sha256: str + identity: ActiveAcquisitionRecoveryIdentity + connection: ActiveAcquisitionRecoveryConnection + compatibility_profile_id: str + project_name: str + project_name_wire_sha256: str + active_project_id_sha256: str | None + original_evidence_session_id: str + current_evidence_session_id: str + duration_seconds: float | None + requested_streams: tuple[str, ...] + evidence_policy: ActiveAcquisitionRecoveryEvidencePolicy + mount_type: ActiveAcquisitionRecoveryMountType + gnss_mode: ActiveAcquisitionRecoveryGnssMode + prepared_binding: ActiveAcquisitionRecoveryTransportBinding + current_binding: ActiveAcquisitionRecoveryTransportBinding + created_at_utc: str + updated_at_utc: str + activated_at_utc: str | None + ceased_at_utc: str | None + last_gap_started_at_utc: str | None + last_gap_started_transport_revision: int | None + last_gap_failed_binding: ActiveAcquisitionRecoveryTransportBinding | None + last_gap_recovered_at_utc: str | None + last_gap_recovered_transport_revision: int | None + activation_status_proof: ActiveAcquisitionRecoveryStatusProof | None + activation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None + current_active_status_proof: ActiveAcquisitionRecoveryStatusProof | None + current_active_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None + last_rebind_status_proof: ActiveAcquisitionRecoveryStatusProof | None + last_rebind_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof | None + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None + prepared_resolution_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None + reconciled_start_origin_proof: ActiveAcquisitionRecoveryStartOriginProof | None + first_published_pcl_proof: ActiveAcquisitionRecoveryFirstPublishedPclProof | None + prepared_resolved_start_reconciliation_id: str | None + prepared_resolved_start_original_attempt_sha256: str | None + prepared_resolved_start_original_project_id_sha256: str | None + transition_receipts: tuple[ActiveAcquisitionRecoveryTransitionReceipt, ...] + + @property + def restart_resume_authority(self) -> Literal[False]: + return False + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "revision": self.revision, + "state": self.state, + "transport_revision": self.transport_revision, + "physical_lineage_head_revision": self.physical_lineage_head_revision, + "acquisition_id": self.acquisition_id, + "original_start_operation_id": self.original_start_operation_id, + "start_payload_sha256": self.start_payload_sha256, + "identity": self.identity.as_dict(), + "connection": self.connection.as_dict(), + "compatibility_profile_id": self.compatibility_profile_id, + "project_name": self.project_name, + "project_name_wire_sha256": self.project_name_wire_sha256, + "active_project_id_sha256": self.active_project_id_sha256, + "original_evidence_session_id": self.original_evidence_session_id, + "current_evidence_session_id": self.current_evidence_session_id, + "duration_seconds": self.duration_seconds, + "requested_streams": list(self.requested_streams), + "evidence_policy": self.evidence_policy, + "mount_type": self.mount_type, + "gnss_mode": self.gnss_mode, + "prepared_binding": self.prepared_binding.as_dict(), + "current_binding": self.current_binding.as_dict(), + "created_at_utc": self.created_at_utc, + "updated_at_utc": self.updated_at_utc, + "activated_at_utc": self.activated_at_utc, + "ceased_at_utc": self.ceased_at_utc, + "last_gap_started_at_utc": self.last_gap_started_at_utc, + "last_gap_started_transport_revision": ( + self.last_gap_started_transport_revision + ), + "last_gap_failed_binding": ( + self.last_gap_failed_binding.as_dict() + if self.last_gap_failed_binding is not None + else None + ), + "last_gap_recovered_at_utc": self.last_gap_recovered_at_utc, + "last_gap_recovered_transport_revision": ( + self.last_gap_recovered_transport_revision + ), + "activation_status_proof": ( + self.activation_status_proof.as_dict() + if self.activation_status_proof is not None + else None + ), + "activation_physical_proof": ( + self.activation_physical_proof.as_dict() + if self.activation_physical_proof is not None + else None + ), + "current_active_status_proof": ( + self.current_active_status_proof.as_dict() + if self.current_active_status_proof is not None + else None + ), + "current_active_physical_proof": ( + self.current_active_physical_proof.as_dict() + if self.current_active_physical_proof is not None + else None + ), + "last_rebind_status_proof": ( + self.last_rebind_status_proof.as_dict() + if self.last_rebind_status_proof is not None + else None + ), + "last_rebind_physical_proof": ( + self.last_rebind_physical_proof.as_dict() + if self.last_rebind_physical_proof is not None + else None + ), + "cessation_status_proof": ( + self.cessation_status_proof.as_dict() + if self.cessation_status_proof is not None + else None + ), + "cessation_physical_proof": ( + self.cessation_physical_proof.as_dict() + if self.cessation_physical_proof is not None + else None + ), + "prepared_resolution_proof": ( + self.prepared_resolution_proof.as_dict() + if self.prepared_resolution_proof is not None + else None + ), + "reconciled_start_origin_proof": ( + self.reconciled_start_origin_proof.as_dict() + if self.reconciled_start_origin_proof is not None + else None + ), + "first_published_pcl_proof": ( + self.first_published_pcl_proof.as_dict() + if self.first_published_pcl_proof is not None + else None + ), + "prepared_resolved_start_reconciliation_id": ( + self.prepared_resolved_start_reconciliation_id + ), + "prepared_resolved_start_original_attempt_sha256": ( + self.prepared_resolved_start_original_attempt_sha256 + ), + "prepared_resolved_start_original_project_id_sha256": ( + self.prepared_resolved_start_original_project_id_sha256 + ), + "transition_receipts": [ + receipt.as_dict() for receipt in self.transition_receipts + ], + } + + +@dataclass(frozen=True, slots=True) +class ActiveAcquisitionRecoveryCheckpointSnapshot: + status: ActiveAcquisitionRecoveryCheckpointStatus + checkpoint: ActiveAcquisitionRecoveryCheckpoint | None + reason_code: str | None + + @property + def restart_resume_authority(self) -> Literal[False]: + return False + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + "status": self.status, + "restart_resume_authority": self.restart_resume_authority, + "reason_code": self.reason_code, + "checkpoint": ( + self.checkpoint.as_dict() if self.checkpoint is not None else None + ), + } + + +class ActiveAcquisitionRecoveryCheckpointStore: + """Secret-free durable lineage store; never recovery or command authority.""" + + def __init__( + self, + repository_root: Path, + *, + clock: Callable[[], datetime] | None = None, + ) -> None: + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = ( + data_dir + / "xgrids-k1" + / ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_FILENAME + ) + self._process_lock_path = ( + data_dir + / "xgrids-k1" + / ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_LOCK_FILENAME + ) + self._data_dir = data_dir + self._clock = clock or (lambda: datetime.now(UTC)) + self._thread_lock = threading.RLock() + self._checkpoint: ActiveAcquisitionRecoveryCheckpoint | None = None + self._corrupt = False + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> ActiveAcquisitionRecoveryCheckpointSnapshot: + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return ActiveAcquisitionRecoveryCheckpointSnapshot( + status="corrupt", + checkpoint=None, + reason_code=ActiveAcquisitionRecoveryCheckpointCorrupt.reason_code, + ) + if self._checkpoint is None: + return ActiveAcquisitionRecoveryCheckpointSnapshot( + status="empty", + checkpoint=None, + reason_code=None, + ) + return ActiveAcquisitionRecoveryCheckpointSnapshot( + status=self._checkpoint.state, + checkpoint=self._checkpoint, + reason_code=None, + ) + + def prepare( + self, + *, + transition_id: str, + predecessor_revision: int, + acquisition_id: str, + original_start_operation_id: str, + start_payload_sha256: str, + identity: ActiveAcquisitionRecoveryIdentity, + connection: ActiveAcquisitionRecoveryConnection, + compatibility_profile_id: str, + project_name: str, + project_name_wire_sha256: str, + original_evidence_session_id: str, + duration_seconds: float | None, + requested_streams: tuple[str, ...], + evidence_policy: ActiveAcquisitionRecoveryEvidencePolicy, + mount_type: ActiveAcquisitionRecoveryMountType, + gnss_mode: ActiveAcquisitionRecoveryGnssMode, + prepared_binding: ActiveAcquisitionRecoveryTransportBinding, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Persist PREPARED before physical START dispatch.""" + + _validate_identifier(transition_id, field_name="transition_id") + _validate_nonnegative_revision( + predecessor_revision, + field_name="predecessor_revision", + ) + _validate_identifier(acquisition_id, field_name="acquisition_id") + _validate_identifier( + original_start_operation_id, + field_name="original_start_operation_id", + ) + _validate_sha256(start_payload_sha256, field_name="start_payload_sha256") + if not isinstance(identity, ActiveAcquisitionRecoveryIdentity): + raise TypeError("identity must be ActiveAcquisitionRecoveryIdentity") + if not isinstance(connection, ActiveAcquisitionRecoveryConnection): + raise TypeError("connection must be ActiveAcquisitionRecoveryConnection") + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + exact_project_name = _exact_project_name(project_name) + _validate_sha256( + project_name_wire_sha256, + field_name="project_name_wire_sha256", + ) + if project_name_wire_sha256 != active_acquisition_project_name_sha256( + exact_project_name + ): + raise ValueError("project_name_wire_sha256 does not bind exact wire bytes") + _validate_identifier( + original_evidence_session_id, + field_name="original_evidence_session_id", + ) + canonical_duration_seconds = _optional_duration_seconds( + duration_seconds, + field_name="duration_seconds", + ) + canonical_streams = _validated_requested_streams(requested_streams) + _validate_evidence_policy(evidence_policy) + _validate_mount_type(mount_type) + _validate_gnss_mode(gnss_mode) + if not isinstance(prepared_binding, ActiveAcquisitionRecoveryTransportBinding): + raise TypeError( + "prepared_binding must be ActiveAcquisitionRecoveryTransportBinding" + ) + _require_binding_matches_checkpoint_values( + identity=identity, + connection=connection, + compatibility_profile_id=compatibility_profile_id, + binding=prepared_binding, + ) + fingerprint = _request_fingerprint( + "prepare", + { + "acquisition_id": acquisition_id, + "original_start_operation_id": original_start_operation_id, + "start_payload_sha256": start_payload_sha256, + "identity": identity.as_dict(), + "connection": connection.as_dict(), + "compatibility_profile_id": compatibility_profile_id, + "project_name": exact_project_name, + "project_name_wire_sha256": project_name_wire_sha256, + "original_evidence_session_id": original_evidence_session_id, + "duration_seconds": canonical_duration_seconds, + "requested_streams": list(canonical_streams), + "evidence_policy": evidence_policy, + "mount_type": mount_type, + "gnss_mode": gnss_mode, + "prepared_binding": prepared_binding.as_dict(), + }, + ) + + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="prepare", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current_revision = current.revision if current is not None else 0 + if predecessor_revision != current_revision: + raise StaleActiveAcquisitionRecoveryCheckpoint( + "checkpoint predecessor revision is no longer current" + ) + if current is not None and current.state != "ceased": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "an unceased acquisition checkpoint cannot be replaced" + ) + revision = _next_revision(current_revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc if current is not None else None, + ) + receipts = _append_transition_receipt( + (), + transition_id=transition_id, + kind="prepare", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ) + candidate = ActiveAcquisitionRecoveryCheckpoint( + schema_version=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + revision=revision, + state="prepared", + transport_revision=1, + physical_lineage_head_revision=None, + acquisition_id=acquisition_id, + original_start_operation_id=original_start_operation_id, + start_payload_sha256=start_payload_sha256, + identity=identity, + connection=connection, + compatibility_profile_id=compatibility_profile_id, + project_name=exact_project_name, + project_name_wire_sha256=project_name_wire_sha256, + active_project_id_sha256=None, + original_evidence_session_id=original_evidence_session_id, + current_evidence_session_id=original_evidence_session_id, + duration_seconds=canonical_duration_seconds, + requested_streams=canonical_streams, + evidence_policy=evidence_policy, + mount_type=mount_type, + gnss_mode=gnss_mode, + prepared_binding=prepared_binding, + current_binding=prepared_binding, + created_at_utc=now, + updated_at_utc=now, + activated_at_utc=None, + ceased_at_utc=None, + last_gap_started_at_utc=None, + last_gap_started_transport_revision=None, + last_gap_failed_binding=None, + last_gap_recovered_at_utc=None, + last_gap_recovered_transport_revision=None, + activation_status_proof=None, + activation_physical_proof=None, + current_active_status_proof=None, + current_active_physical_proof=None, + last_rebind_status_proof=None, + last_rebind_physical_proof=None, + cessation_status_proof=None, + cessation_physical_proof=None, + prepared_resolution_proof=None, + reconciled_start_origin_proof=None, + first_published_pcl_proof=None, + prepared_resolved_start_reconciliation_id=None, + prepared_resolved_start_original_attempt_sha256=None, + prepared_resolved_start_original_project_id_sha256=None, + transition_receipts=receipts, + ) + return self._persist_locked(candidate) + + def activate( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Activate only an exact composite-resolved START plus fresh SCANNING.""" + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if not isinstance(status_proof, ActiveAcquisitionRecoveryStatusProof): + raise TypeError("status_proof must be ActiveAcquisitionRecoveryStatusProof") + if not isinstance( + physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "physical_proof must be ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + if status_proof.session_state != "scanning": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "activation requires fresh canonical SCANNING" + ) + fingerprint = _request_fingerprint( + "activate", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "status_proof": status_proof.as_dict(), + "physical_proof": physical_proof.as_dict(), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="activate", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "prepared": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "only a prepared checkpoint can become active" + ) + _require_status_binding(current, status_proof) + if status_proof.evidence_session_id != current.original_evidence_session_id: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "activation status must belong to the original evidence session" + ) + _require_activation_physical_proof( + current, + status_proof=status_proof, + physical_proof=physical_proof, + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + assert status_proof.project_id_sha256 is not None + candidate = replace( + current, + revision=revision, + state="active", + active_project_id_sha256=status_proof.project_id_sha256, + physical_lineage_head_revision=physical_proof.ledger_revision, + current_binding=status_proof.binding, + updated_at_utc=now, + activated_at_utc=status_proof.observed_at_utc, + activation_status_proof=status_proof, + activation_physical_proof=physical_proof, + current_active_status_proof=status_proof, + current_active_physical_proof=physical_proof, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="activate", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def activate_reconciled( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + gap_started_at_utc: str, + origin_proof: ActiveAcquisitionRecoveryStartOriginProof, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + first_pcl_proof: ActiveAcquisitionRecoveryFirstPublishedPclProof, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Atomically promote a crash-era PREPARED START after first PCL.""" + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + gap_started = _canonical_timestamp_value( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + if not isinstance(origin_proof, ActiveAcquisitionRecoveryStartOriginProof): + raise TypeError("origin_proof must be ActiveAcquisitionRecoveryStartOriginProof") + if not isinstance(status_proof, ActiveAcquisitionRecoveryStatusProof): + raise TypeError("status_proof must be ActiveAcquisitionRecoveryStatusProof") + if not isinstance( + physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "physical_proof must be ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + if not isinstance( + first_pcl_proof, + ActiveAcquisitionRecoveryFirstPublishedPclProof, + ): + raise TypeError( + "first_pcl_proof must be ActiveAcquisitionRecoveryFirstPublishedPclProof" + ) + fingerprint = _request_fingerprint( + "activate-reconciled", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "gap_started_at_utc": gap_started, + "origin_proof": origin_proof.as_dict(), + "status_proof": status_proof.as_dict(), + "physical_proof": physical_proof.as_dict(), + "first_pcl_proof": first_pcl_proof.as_dict(), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="activate-reconciled", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "prepared": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "reconciled activation requires an exact prepared checkpoint" + ) + _require_reconciled_start_active( + current, + origin_proof=origin_proof, + status_proof=status_proof, + physical_proof=physical_proof, + ) + _require_first_published_pcl( + status_proof=status_proof, + first_pcl_proof=first_pcl_proof, + ) + transport_revision = _next_transport_revision( + current.transport_revision + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + assert status_proof.project_id_sha256 is not None + candidate = replace( + current, + revision=revision, + state="active", + transport_revision=transport_revision, + physical_lineage_head_revision=physical_proof.ledger_revision, + active_project_id_sha256=status_proof.project_id_sha256, + current_evidence_session_id=status_proof.evidence_session_id, + current_binding=status_proof.binding, + updated_at_utc=now, + activated_at_utc=first_pcl_proof.published_at_utc, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=current.transport_revision, + last_gap_failed_binding=current.prepared_binding, + last_gap_recovered_at_utc=status_proof.observed_at_utc, + last_gap_recovered_transport_revision=transport_revision, + activation_status_proof=status_proof, + activation_physical_proof=physical_proof, + current_active_status_proof=status_proof, + current_active_physical_proof=physical_proof, + reconciled_start_origin_proof=origin_proof, + first_published_pcl_proof=first_pcl_proof, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="activate-reconciled", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def record_gap( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + failed_binding: ActiveAcquisitionRecoveryTransportBinding, + gap_started_at_utc: str, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Record one gap against the exact current binding and durable revision.""" + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if not isinstance(failed_binding, ActiveAcquisitionRecoveryTransportBinding): + raise TypeError( + "failed_binding must be ActiveAcquisitionRecoveryTransportBinding" + ) + gap_started = _canonical_timestamp_value( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + fingerprint = _request_fingerprint( + "record-gap", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "failed_binding": failed_binding.as_dict(), + "gap_started_at_utc": gap_started, + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="record-gap", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "active": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "only an active checkpoint can record a gap" + ) + if _has_open_gap(current): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "the current transport gap is already open" + ) + if failed_binding != current.current_binding: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "gap must bind the exact current process-local transport" + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + candidate = replace( + current, + revision=revision, + updated_at_utc=now, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=current.transport_revision, + last_gap_failed_binding=failed_binding, + last_gap_recovered_at_utc=None, + last_gap_recovered_transport_revision=None, + last_rebind_status_proof=None, + last_rebind_physical_proof=None, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="record-gap", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def rebind_active( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Close a gap while preserving the process-scoped evidence writer. + + A new binding in the same runtime instance may keep its evidence + session because capture recovery reuses the open writer. A different + runtime instance cannot reopen those exclusive artifacts and therefore + must bind a successor evidence session. + """ + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if not isinstance(status_proof, ActiveAcquisitionRecoveryStatusProof): + raise TypeError("status_proof must be ActiveAcquisitionRecoveryStatusProof") + if not isinstance( + physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "physical_proof must be ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + if status_proof.session_state != "scanning": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active rebind requires fresh canonical SCANNING" + ) + fingerprint = _request_fingerprint( + "rebind-active", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "status_proof": status_proof.as_dict(), + "physical_proof": physical_proof.as_dict(), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="rebind-active", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "active" or not _has_open_gap(current): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active rebind requires one exact open gap" + ) + _require_status_binding(current, status_proof) + _require_active_rebind_physical_proof( + current, + status_proof=status_proof, + physical_proof=physical_proof, + ) + if status_proof.project_id_sha256 != current.active_project_id_sha256: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "rebind project hash does not match the active acquisition" + ) + if status_proof.binding == current.current_binding: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "rebind requires a new exact process-local transport binding" + ) + failed_binding = current.last_gap_failed_binding + assert failed_binding is not None + crosses_runtime_instance = ( + status_proof.binding.runtime_instance_id + != failed_binding.runtime_instance_id + ) + if crosses_runtime_instance and status_proof.evidence_session_id == ( + current.current_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "rebind across a runtime instance requires a successor evidence session" + ) + transport_revision = _next_transport_revision(current.transport_revision) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + candidate = replace( + current, + revision=revision, + transport_revision=transport_revision, + physical_lineage_head_revision=physical_proof.ledger_revision, + current_binding=status_proof.binding, + current_evidence_session_id=status_proof.evidence_session_id, + updated_at_utc=now, + last_gap_recovered_at_utc=status_proof.observed_at_utc, + last_gap_recovered_transport_revision=transport_revision, + last_rebind_status_proof=status_proof, + last_rebind_physical_proof=physical_proof, + current_active_status_proof=status_proof, + current_active_physical_proof=physical_proof, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="rebind-active", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def cease_active_reconciled_standby( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + gap_started_at_utc: str, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Atomically settle an ACTIVE acquisition found standby after restart. + + The transition either opens and closes the restart gap in one durable + replacement or closes the exact gap already recorded before the + process ended. It never exposes an intermediate open-gap revision. + """ + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + gap_started = _canonical_timestamp_value( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + if not isinstance( + cessation_status_proof, + ActiveAcquisitionRecoveryStatusProof, + ): + raise TypeError( + "cessation_status_proof must be ActiveAcquisitionRecoveryStatusProof" + ) + if not isinstance( + cessation_physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "cessation_physical_proof must be " + "ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + fingerprint = _request_fingerprint( + "cease-active-reconciled-standby", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "gap_started_at_utc": gap_started, + "cessation_status_proof": cessation_status_proof.as_dict(), + "cessation_physical_proof": cessation_physical_proof.as_dict(), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="cease-active-reconciled-standby", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "active": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires an exact active checkpoint" + ) + + if _has_open_gap(current): + if current.last_gap_started_at_utc != gap_started: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement must close the exact open gap" + ) + failed_binding = current.last_gap_failed_binding + gap_started_transport_revision = ( + current.last_gap_started_transport_revision + ) + assert failed_binding is not None + assert gap_started_transport_revision is not None + validation_checkpoint = current + else: + failed_binding = current.current_binding + gap_started_transport_revision = current.transport_revision + validation_checkpoint = replace( + current, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=current.transport_revision, + last_gap_failed_binding=failed_binding, + last_gap_recovered_at_utc=None, + last_gap_recovered_transport_revision=None, + last_rebind_status_proof=None, + last_rebind_physical_proof=None, + ) + + failed_evidence_session_id = current.current_evidence_session_id + _require_active_cessation( + validation_checkpoint, + status_proof=cessation_status_proof, + physical_proof=cessation_physical_proof, + ) + _require_active_reconciled_standby_shape( + current, + failed_binding=failed_binding, + failed_evidence_session_id=failed_evidence_session_id, + gap_started_at_utc=gap_started, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + ) + + transport_revision = _next_transport_revision( + current.transport_revision + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + candidate = replace( + current, + revision=revision, + state="ceased", + transport_revision=transport_revision, + physical_lineage_head_revision=( + cessation_physical_proof.ledger_revision + ), + current_binding=cessation_status_proof.binding, + current_evidence_session_id=( + cessation_status_proof.evidence_session_id + ), + updated_at_utc=now, + ceased_at_utc=cessation_status_proof.observed_at_utc, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=( + gap_started_transport_revision + ), + last_gap_failed_binding=failed_binding, + last_gap_recovered_at_utc=( + cessation_status_proof.observed_at_utc + ), + last_gap_recovered_transport_revision=transport_revision, + last_rebind_status_proof=None, + last_rebind_physical_proof=None, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="cease-active-reconciled-standby", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def cease( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + status_proof: ActiveAcquisitionRecoveryStatusProof | None, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Cease only with terminal physical lineage and exact current status.""" + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if not isinstance( + physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "physical_proof must be ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + if status_proof is not None and not isinstance( + status_proof, + ActiveAcquisitionRecoveryStatusProof, + ): + raise TypeError("status_proof must be ActiveAcquisitionRecoveryStatusProof") + fingerprint = _request_fingerprint( + "cease", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "physical_proof": physical_proof.as_dict(), + "status_proof": ( + status_proof.as_dict() if status_proof is not None else None + ), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="cease", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state not in {"prepared", "active"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "only a prepared or active checkpoint can cease" + ) + _require_physical_lineage_base(current, physical_proof) + transport_revision = current.transport_revision + current_binding = current.current_binding + gap_recovered_at = current.last_gap_recovered_at_utc + gap_recovered_revision = current.last_gap_recovered_transport_revision + prepared_resolution: ActiveAcquisitionRecoveryPhysicalLineageProof | None = None + if current.state == "prepared": + _require_prepared_cessation( + current, + status_proof=status_proof, + physical_proof=physical_proof, + ) + prepared_resolution = physical_proof + current_binding = physical_proof.binding + else: + assert status_proof is not None + _require_active_cessation( + current, + status_proof=status_proof, + physical_proof=physical_proof, + ) + if _has_open_gap(current): + if status_proof.binding == current.current_binding: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "terminal recovery across an open gap requires a new binding" + ) + failed_binding = current.last_gap_failed_binding + assert failed_binding is not None + _require_gap_cessation_evidence_session( + failed_binding=failed_binding, + failed_evidence_session_id=( + current.current_evidence_session_id + ), + status_proof=status_proof, + ) + transport_revision = _next_transport_revision( + current.transport_revision + ) + gap_recovered_at = status_proof.observed_at_utc + gap_recovered_revision = transport_revision + current_binding = status_proof.binding + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + ceased_at = ( + status_proof.observed_at_utc + if status_proof is not None + else physical_proof.observed_at_utc + ) + candidate = replace( + current, + revision=revision, + state="ceased", + transport_revision=transport_revision, + physical_lineage_head_revision=physical_proof.ledger_revision, + current_binding=current_binding, + current_evidence_session_id=( + status_proof.evidence_session_id + if status_proof is not None + else current.current_evidence_session_id + ), + updated_at_utc=now, + ceased_at_utc=ceased_at, + last_gap_recovered_at_utc=gap_recovered_at, + last_gap_recovered_transport_revision=gap_recovered_revision, + cessation_status_proof=status_proof, + cessation_physical_proof=physical_proof, + prepared_resolution_proof=prepared_resolution, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="cease", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def cease_prepared_reconciled( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + gap_started_at_utc: str, + origin_proof: ActiveAcquisitionRecoveryStartOriginProof, + active_status_proof: ActiveAcquisitionRecoveryStatusProof, + active_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Close a reconciled PREPARED START when STOP wins before first PCL.""" + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + gap_started = _canonical_timestamp_value( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + if not isinstance(origin_proof, ActiveAcquisitionRecoveryStartOriginProof): + raise TypeError("origin_proof must be ActiveAcquisitionRecoveryStartOriginProof") + for field_name, status_candidate in ( + ("active_status_proof", active_status_proof), + ("cessation_status_proof", cessation_status_proof), + ): + if not isinstance( + status_candidate, + ActiveAcquisitionRecoveryStatusProof, + ): + raise TypeError( + f"{field_name} must be ActiveAcquisitionRecoveryStatusProof" + ) + for field_name, physical_candidate in ( + ("active_physical_proof", active_physical_proof), + ("cessation_physical_proof", cessation_physical_proof), + ): + if not isinstance( + physical_candidate, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + f"{field_name} must be ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + fingerprint = _request_fingerprint( + "cease-prepared-reconciled", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "gap_started_at_utc": gap_started, + "origin_proof": origin_proof.as_dict(), + "active_status_proof": active_status_proof.as_dict(), + "active_physical_proof": active_physical_proof.as_dict(), + "cessation_status_proof": cessation_status_proof.as_dict(), + "cessation_physical_proof": cessation_physical_proof.as_dict(), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="cease-prepared-reconciled", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "prepared": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "pre-PCL cessation requires an exact prepared checkpoint" + ) + _require_reconciled_start_active( + current, + origin_proof=origin_proof, + status_proof=active_status_proof, + physical_proof=active_physical_proof, + ) + _require_prepared_reconciled_cessation( + current, + active_status_proof=active_status_proof, + active_physical_proof=active_physical_proof, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + ) + transport_revision = _next_transport_revision( + current.transport_revision + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + assert active_status_proof.project_id_sha256 is not None + candidate = replace( + current, + revision=revision, + state="ceased", + transport_revision=transport_revision, + physical_lineage_head_revision=( + cessation_physical_proof.ledger_revision + ), + active_project_id_sha256=active_status_proof.project_id_sha256, + current_evidence_session_id=( + cessation_status_proof.evidence_session_id + ), + current_binding=cessation_status_proof.binding, + updated_at_utc=now, + ceased_at_utc=cessation_status_proof.observed_at_utc, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=current.transport_revision, + last_gap_failed_binding=current.prepared_binding, + last_gap_recovered_at_utc=( + cessation_status_proof.observed_at_utc + ), + last_gap_recovered_transport_revision=transport_revision, + current_active_status_proof=active_status_proof, + current_active_physical_proof=active_physical_proof, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + prepared_resolution_proof=cessation_physical_proof, + reconciled_start_origin_proof=origin_proof, + first_published_pcl_proof=None, + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="cease-prepared-reconciled", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + def cease_prepared_resolved_start_standby( + self, + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, + gap_started_at_utc: str, + origin_proof: ActiveAcquisitionRecoveryStartOriginProof, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + reconciliation_id: str, + reconciliation_original_attempt_sha256: str, + reconciliation_original_project_id_sha256: str, + ) -> ActiveAcquisitionRecoveryCheckpoint: + """Cease a proven START that became standby before restart PCL. + + This is deliberately distinct from ``cease_prepared_reconciled``: + READY/SCAN_OVER proves that the old successful START is no longer + active, so there is no transient ACTIVE checkpoint, capture promotion, + first-PCL receipt, or invented STOP edge. + """ + + _validate_mutation_request( + transition_id=transition_id, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + gap_started = _canonical_timestamp_value( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + if not isinstance(origin_proof, ActiveAcquisitionRecoveryStartOriginProof): + raise TypeError("origin_proof must be ActiveAcquisitionRecoveryStartOriginProof") + if not isinstance( + cessation_status_proof, + ActiveAcquisitionRecoveryStatusProof, + ): + raise TypeError( + "cessation_status_proof must be ActiveAcquisitionRecoveryStatusProof" + ) + if not isinstance( + cessation_physical_proof, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ): + raise TypeError( + "cessation_physical_proof must be " + "ActiveAcquisitionRecoveryPhysicalLineageProof" + ) + _validate_identifier(reconciliation_id, field_name="reconciliation_id") + _validate_sha256( + reconciliation_original_attempt_sha256, + field_name="reconciliation_original_attempt_sha256", + ) + _validate_sha256( + reconciliation_original_project_id_sha256, + field_name="reconciliation_original_project_id_sha256", + ) + fingerprint = _request_fingerprint( + "cease-prepared-resolved-start-standby", + { + "acquisition_id": expected_acquisition_id, + "original_start_operation_id": expected_start_operation_id, + "gap_started_at_utc": gap_started, + "origin_proof": origin_proof.as_dict(), + "cessation_status_proof": cessation_status_proof.as_dict(), + "cessation_physical_proof": cessation_physical_proof.as_dict(), + "reconciliation_id": reconciliation_id, + "reconciliation_original_attempt_sha256": ( + reconciliation_original_attempt_sha256 + ), + "reconciliation_original_project_id_sha256": ( + reconciliation_original_project_id_sha256 + ), + }, + ) + with self._thread_lock, self._process_lock_locked(): + current, replayed = self._begin_transition_locked( + transition_id=transition_id, + kind="cease-prepared-resolved-start-standby", + fingerprint=fingerprint, + ) + if replayed: + assert current is not None + return current + current = _require_current_checkpoint( + current, + expected_revision=expected_revision, + expected_acquisition_id=expected_acquisition_id, + expected_start_operation_id=expected_start_operation_id, + ) + if current.state != "prepared": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires an exact prepared checkpoint" + ) + _require_prepared_resolved_start_standby( + current, + gap_started_at_utc=gap_started, + origin_proof=origin_proof, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + reconciliation_id=reconciliation_id, + reconciliation_original_attempt_sha256=( + reconciliation_original_attempt_sha256 + ), + reconciliation_original_project_id_sha256=( + reconciliation_original_project_id_sha256 + ), + ) + transport_revision = _next_transport_revision( + current.transport_revision + ) + revision = _next_revision(current.revision) + now = _nondecreasing_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + assert origin_proof.original_project_id_sha256 is not None + candidate = replace( + current, + revision=revision, + state="ceased", + transport_revision=transport_revision, + physical_lineage_head_revision=( + cessation_physical_proof.ledger_revision + ), + active_project_id_sha256=( + origin_proof.original_project_id_sha256 + ), + current_evidence_session_id=( + cessation_status_proof.evidence_session_id + ), + current_binding=cessation_status_proof.binding, + updated_at_utc=now, + ceased_at_utc=cessation_status_proof.observed_at_utc, + last_gap_started_at_utc=gap_started, + last_gap_started_transport_revision=current.transport_revision, + last_gap_failed_binding=current.prepared_binding, + last_gap_recovered_at_utc=( + cessation_status_proof.observed_at_utc + ), + last_gap_recovered_transport_revision=transport_revision, + cessation_status_proof=cessation_status_proof, + cessation_physical_proof=cessation_physical_proof, + prepared_resolution_proof=cessation_physical_proof, + reconciled_start_origin_proof=origin_proof, + first_published_pcl_proof=None, + prepared_resolved_start_reconciliation_id=reconciliation_id, + prepared_resolved_start_original_attempt_sha256=( + reconciliation_original_attempt_sha256 + ), + prepared_resolved_start_original_project_id_sha256=( + reconciliation_original_project_id_sha256 + ), + transition_receipts=_append_transition_receipt( + current.transition_receipts, + transition_id=transition_id, + kind="cease-prepared-resolved-start-standby", + fingerprint=fingerprint, + committed_revision=revision, + committed_at_utc=now, + ), + ) + return self._persist_locked(candidate) + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + try: + data_created = _ensure_private_directory(self._data_dir, parents=True) + checkpoint_dir_created = _ensure_private_directory( + self.path.parent, + parents=False, + ) + except (OSError, ValueError) as exc: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint directories are not private and owner-controlled" + ) from exc + if data_created: + _fsync_directory(self._data_dir.parent) + if checkpoint_dir_created: + _fsync_directory(self._data_dir) + + flags = os.O_RDWR | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self._process_lock_path, flags) + lock_created = False + except FileNotFoundError: + try: + descriptor = os.open( + self._process_lock_path, + flags | os.O_CREAT | os.O_EXCL, + 0o600, + ) + lock_created = True + except FileExistsError: + try: + descriptor = os.open(self._process_lock_path, flags) + lock_created = False + except OSError as exc: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint process lock cannot be opened safely" + ) from exc + except OSError as exc: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint process lock cannot be created safely" + ) from exc + except OSError as exc: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint process lock cannot be opened safely" + ) from exc + + stream: IO[bytes] | None = None + try: + try: + before = os.fstat(descriptor) + _require_private_regular_file( + before, + label="checkpoint lock", + empty=True, + ) + stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + after = os.fstat(stream.fileno()) + path_metadata = self._process_lock_path.lstat() + _require_private_regular_file( + after, + label="checkpoint lock", + empty=True, + ) + if (after.st_dev, after.st_ino) != ( + path_metadata.st_dev, + path_metadata.st_ino, + ): + raise ValueError("checkpoint process lock identity changed") + if lock_created: + _fsync_directory(self.path.parent) + except (OSError, ValueError) as exc: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint process lock failed closed" + ) from exc + try: + yield + finally: + if stream is not None: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + if stream is not None: + stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _begin_transition_locked( + self, + *, + transition_id: str, + kind: ActiveAcquisitionRecoveryTransitionKind, + fingerprint: str, + ) -> tuple[ActiveAcquisitionRecoveryCheckpoint | None, bool]: + self._reload_locked() + self._require_healthy_locked() + current = self._checkpoint + transition_hash = _transition_id_sha256(transition_id) + if current is None: + return None, False + matching = next( + ( + receipt + for receipt in current.transition_receipts + if receipt.transition_id_sha256 == transition_hash + ), + None, + ) + if matching is None: + return current, False + if matching.kind == kind and matching.request_fingerprint_sha256 == fingerprint: + return current, True + raise ActiveAcquisitionRecoveryCheckpointIdempotencyConflict( + "transition id already belongs to different exact request bytes" + ) + + def _require_healthy_locked(self) -> None: + if self._corrupt: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint is corrupt; persisted acquisition evidence was not replaced" + ) + + def _persist_locked( + self, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + ) -> ActiveAcquisitionRecoveryCheckpoint: + _validate_checkpoint_semantics(checkpoint) + _write_private_json_atomic( + self.path, + checkpoint.as_dict(), + data_dir=self._data_dir, + ) + self._checkpoint = checkpoint + self._corrupt = False + return checkpoint + + def _reload_locked(self) -> None: + try: + payload = _read_private_json(self.path) + except FileNotFoundError: + self._checkpoint = None + self._corrupt = False + return + except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): + self._checkpoint = None + self._corrupt = True + return + try: + checkpoint = _checkpoint_from_mapping(payload) + _validate_checkpoint_semantics(checkpoint) + except ( + TypeError, + ValueError, + ActiveAcquisitionRecoveryCheckpointError, + ): + self._checkpoint = None + self._corrupt = True + return + self._checkpoint = checkpoint + self._corrupt = False + + +def active_acquisition_project_name_sha256(project_name: str) -> str: + """Hash exact already-canonical START project-name wire bytes.""" + + exact = _exact_project_name(project_name) + return hashlib.sha256(exact.encode("utf-8")).hexdigest() + + +def _validate_mutation_request( + *, + transition_id: str, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, +) -> None: + _validate_identifier(transition_id, field_name="transition_id") + _validate_positive_revision(expected_revision, field_name="expected_revision") + _validate_identifier( + expected_acquisition_id, + field_name="expected_acquisition_id", + ) + _validate_identifier( + expected_start_operation_id, + field_name="expected_start_operation_id", + ) + + +def _require_current_checkpoint( + current: ActiveAcquisitionRecoveryCheckpoint | None, + *, + expected_revision: int, + expected_acquisition_id: str, + expected_start_operation_id: str, +) -> ActiveAcquisitionRecoveryCheckpoint: + if ( + current is None + or current.revision != expected_revision + or current.acquisition_id != expected_acquisition_id + or current.original_start_operation_id != expected_start_operation_id + ): + raise StaleActiveAcquisitionRecoveryCheckpoint( + "checkpoint revision or acquisition lineage is no longer current" + ) + return current + + +def _require_binding_matches_checkpoint_values( + *, + identity: ActiveAcquisitionRecoveryIdentity, + connection: ActiveAcquisitionRecoveryConnection, + compatibility_profile_id: str, + binding: ActiveAcquisitionRecoveryTransportBinding, +) -> None: + if ( + binding.logical_device_id != identity.logical_device_id + or binding.vendor_device_id_sha256 != identity.vendor_device_id_sha256 + or binding.device_serial_sha256 != identity.device_serial_sha256 + or binding.compatibility_profile_id != compatibility_profile_id + or binding.transport_ref != connection.transport_ref + or binding.connection_mode != connection.connection_mode + or binding.target_ipv4 != connection.target_ipv4 + or binding.target_port != connection.target_port + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "transport binding does not match exact checkpoint identity and target" + ) + + +def _require_status_binding( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + proof: ActiveAcquisitionRecoveryStatusProof, +) -> None: + _require_binding_matches_checkpoint_values( + identity=checkpoint.identity, + connection=checkpoint.connection, + compatibility_profile_id=checkpoint.compatibility_profile_id, + binding=proof.binding, + ) + + +def _require_gap_cessation_evidence_session( + *, + failed_binding: ActiveAcquisitionRecoveryTransportBinding, + failed_evidence_session_id: str, + status_proof: ActiveAcquisitionRecoveryStatusProof, +) -> None: + crosses_runtime_instance = ( + status_proof.binding.runtime_instance_id + != failed_binding.runtime_instance_id + ) + preserves_evidence_session = ( + status_proof.evidence_session_id == failed_evidence_session_id + ) + if crosses_runtime_instance and preserves_evidence_session: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "terminal recovery across a runtime instance requires a successor " + "evidence session" + ) + if not crosses_runtime_instance and not preserves_evidence_session: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "same-runtime terminal recovery must preserve the exact evidence session" + ) + + +def _require_physical_lineage_base( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if ( + proof.acquisition_id != checkpoint.acquisition_id + or proof.original_start_operation_id + != checkpoint.original_start_operation_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "physical proof does not match exact acquisition/START lineage" + ) + _require_binding_matches_checkpoint_values( + identity=checkpoint.identity, + connection=checkpoint.connection, + compatibility_profile_id=checkpoint.compatibility_profile_id, + binding=proof.binding, + ) + + +def _stop_lineage_reaches_checkpoint_start( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> bool: + if proof.action != "stop" or proof.parent_operation_id is None: + return False + if proof.parent_operation_id == checkpoint.original_start_operation_id: + return proof.ancestor_chain is None + ancestry = proof.ancestor_chain + return bool( + isinstance(ancestry, PhysicalCommandClassifiedStopAncestryProof) + and ancestry.ledger_revision == proof.ledger_revision + and ancestry.head_operation_id == proof.operation_id + and ancestry.head_parent_operation_id == proof.parent_operation_id + and ancestry.root_start_operation_id + == checkpoint.original_start_operation_id + and ancestry.acquisition_id == checkpoint.acquisition_id + ) + + +def _require_matching_observed_proofs( + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if ( + physical_proof.binding != status_proof.binding + or physical_proof.status_message_sha256 + != status_proof.status_message_sha256 + or physical_proof.observed_session_state != status_proof.session_state + or physical_proof.observed_at_utc != status_proof.observed_at_utc + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "physical lineage and fresh status proof do not describe one observation" + ) + + +def _require_activation_physical_proof( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + _require_physical_lineage_base(checkpoint, physical_proof) + _require_matching_observed_proofs( + status_proof=status_proof, + physical_proof=physical_proof, + ) + if ( + physical_proof.operation_id != checkpoint.original_start_operation_id + or physical_proof.action != "start" + or physical_proof.resolution != "start-active-observed" + or physical_proof.reconciliation_kind is not None + or physical_proof.reconciliation_resolution is not None + or not physical_proof.composite_complete + or physical_proof.stop_fence != "none" + or physical_proof.payload_sha256 != checkpoint.start_payload_sha256 + or physical_proof.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + or physical_proof.binding != checkpoint.prepared_binding + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "activation requires exact composite-resolved original START" + ) + + +def _require_reconciled_start_active( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + origin_proof: ActiveAcquisitionRecoveryStartOriginProof, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if status_proof.session_state != "scanning": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "reconciled START requires fresh canonical SCANNING" + ) + if status_proof.source != "explicit-read-only-reconciliation": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "reconciled START requires explicit read-only status" + ) + _require_status_binding(checkpoint, status_proof) + _require_physical_lineage_base(checkpoint, physical_proof) + _require_matching_observed_proofs( + status_proof=status_proof, + physical_proof=physical_proof, + ) + baseline = origin_proof.baseline_status_proof + if ( + origin_proof.operation_id != checkpoint.original_start_operation_id + or origin_proof.acquisition_id != checkpoint.acquisition_id + or origin_proof.payload_sha256 != checkpoint.start_payload_sha256 + or physical_proof.payload_sha256 != checkpoint.start_payload_sha256 + or physical_proof.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + or origin_proof.ledger_revision != physical_proof.ledger_revision + or origin_proof.physical_proof_id != physical_proof.proof_id + or baseline.binding != checkpoint.prepared_binding + or baseline.evidence_session_id != checkpoint.original_evidence_session_id + or ( + physical_proof.reconciliation_original_attempt_sha256 is not None + and physical_proof.reconciliation_original_attempt_sha256 + != origin_proof.original_attempt_sha256 + ) + or ( + origin_proof.reconciled_active_project_id_sha256 is not None + and status_proof.project_id_sha256 + != origin_proof.reconciled_active_project_id_sha256 + ) + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "START origin does not bind the exact prepared acquisition" + ) + if ( + status_proof.binding.runtime_instance_id + == checkpoint.prepared_binding.runtime_instance_id + or status_proof.binding.control_session_id + == checkpoint.prepared_binding.control_session_id + or status_proof.evidence_session_id + == checkpoint.original_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "backend restart requires a new runtime, control and evidence session" + ) + if physical_proof.operation_id != checkpoint.original_start_operation_id: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "reconciled activation must retain the original START head" + ) + composite = bool( + origin_proof.origin_kind == "composite-resolved" + and physical_proof.action == "start" + and physical_proof.resolution == "start-active-observed" + and physical_proof.reconciliation_kind == "resolved-active-rebind" + and physical_proof.reconciliation_resolution == "physical-active-observed" + and physical_proof.composite_complete + and physical_proof.stop_fence == "none" + and status_proof.project_id_sha256 + == origin_proof.original_project_id_sha256 + and origin_proof.reconciled_active_project_id_sha256 + in {None, origin_proof.original_project_id_sha256} + ) + ambiguous = bool( + origin_proof.origin_kind == "ambiguous-reconciled" + and physical_proof.action == "start" + and physical_proof.resolution == "physical-active-observed" + and physical_proof.reconciliation_kind + in {"ambiguous-outcome", "resolved-active-rebind"} + and physical_proof.reconciliation_resolution == "physical-active-observed" + and not physical_proof.composite_complete + and physical_proof.stop_fence == "none" + and ( + physical_proof.reconciliation_kind == "ambiguous-outcome" + or ( + physical_proof.reconciliation_original_attempt_sha256 + == origin_proof.original_attempt_sha256 + and origin_proof.reconciled_active_project_id_sha256 + == status_proof.project_id_sha256 + ) + ) + ) + if not (composite or ambiguous): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "reconciled START lacks exact composite or ambiguous active lineage" + ) + + +def _require_first_published_pcl( + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + first_pcl_proof: ActiveAcquisitionRecoveryFirstPublishedPclProof, +) -> None: + if ( + first_pcl_proof.runtime_instance_id + != status_proof.binding.runtime_instance_id + or first_pcl_proof.evidence_session_id + != status_proof.evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "first PCL proof does not belong to the reconciled runtime/evidence session" + ) + if _validated_timestamp( + first_pcl_proof.published_at_utc, + field_name="published_at_utc", + ) < _validated_timestamp( + status_proof.observed_at_utc, + field_name="observed_at_utc", + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "first PCL publication precedes reconciled SCANNING" + ) + + +def _require_prepared_reconciled_cessation( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + active_status_proof: ActiveAcquisitionRecoveryStatusProof, + active_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if cessation_status_proof.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "pre-PCL cessation requires fresh READY/SCAN_OVER" + ) + if cessation_status_proof.source != "explicit-read-only-reconciliation": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "pre-PCL cessation requires explicit read-only status" + ) + _require_status_binding(checkpoint, cessation_status_proof) + _require_physical_lineage_base(checkpoint, cessation_physical_proof) + _require_matching_observed_proofs( + status_proof=cessation_status_proof, + physical_proof=cessation_physical_proof, + ) + if ( + cessation_status_proof.binding != active_status_proof.binding + or cessation_status_proof.evidence_session_id + != active_status_proof.evidence_session_id + or cessation_physical_proof.ledger_revision + <= active_physical_proof.ledger_revision + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "pre-PCL cessation must advance the exact reconciled active binding" + ) + ordinary_stop = bool( + cessation_physical_proof.action == "stop" + and _stop_lineage_reaches_checkpoint_start( + checkpoint, + cessation_physical_proof, + ) + and cessation_physical_proof.resolution == "stop-standby-observed" + and cessation_physical_proof.composite_complete + and cessation_physical_proof.reconciliation_kind is None + and cessation_physical_proof.stop_fence == "none" + ) + reconciled_stop = bool( + cessation_physical_proof.action == "stop" + and _stop_lineage_reaches_checkpoint_start( + checkpoint, + cessation_physical_proof, + ) + and cessation_physical_proof.effective_resolution + == "physical-standby-observed" + and cessation_physical_proof.reconciliation_kind + in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-cessation", + } + and not cessation_physical_proof.composite_complete + ) + if not (ordinary_stop or reconciled_stop): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "pre-PCL cessation lacks an exact terminal STOP descendant" + ) + + +def _require_prepared_resolved_start_standby( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + gap_started_at_utc: str, + origin_proof: ActiveAcquisitionRecoveryStartOriginProof, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + reconciliation_id: str, + reconciliation_original_attempt_sha256: str, + reconciliation_original_project_id_sha256: str, +) -> None: + """Validate restart standby without manufacturing an ACTIVE interval.""" + + if cessation_status_proof.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "resolved START standby settlement requires fresh READY/SCAN_OVER" + ) + if cessation_status_proof.source != "explicit-read-only-reconciliation": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "resolved START standby settlement requires explicit read-only status" + ) + _require_status_binding(checkpoint, cessation_status_proof) + _require_physical_lineage_base(checkpoint, cessation_physical_proof) + _require_matching_observed_proofs( + status_proof=cessation_status_proof, + physical_proof=cessation_physical_proof, + ) + baseline = origin_proof.baseline_status_proof + if not ( + origin_proof.origin_kind == "composite-resolved" + and origin_proof.operation_id == checkpoint.original_start_operation_id + and origin_proof.acquisition_id == checkpoint.acquisition_id + and origin_proof.payload_sha256 == checkpoint.start_payload_sha256 + and origin_proof.original_attempt_stage == "resolved" + and origin_proof.original_attempt_resolution == "start-active-observed" + and origin_proof.original_project_id_sha256 is not None + and origin_proof.project_evidence_strength == "exact-vendor-project-id" + and origin_proof.automatic_replay_allowed is False + and origin_proof.ledger_revision == cessation_physical_proof.ledger_revision + and origin_proof.physical_proof_id == cessation_physical_proof.proof_id + and origin_proof.reconciliation_id == reconciliation_id + and origin_proof.original_attempt_sha256 + == reconciliation_original_attempt_sha256 + and origin_proof.original_project_id_sha256 + == reconciliation_original_project_id_sha256 + and baseline.binding == checkpoint.prepared_binding + and baseline.evidence_session_id + == checkpoint.original_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "resolved START origin does not bind the exact prepared acquisition" + ) + if ( + cessation_status_proof.binding.runtime_instance_id + == checkpoint.prepared_binding.runtime_instance_id + or cessation_status_proof.binding.control_session_id + == checkpoint.prepared_binding.control_session_id + or cessation_status_proof.evidence_session_id + == checkpoint.original_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires new runtime, control and evidence sessions" + ) + if not ( + cessation_physical_proof.operation_id + == checkpoint.original_start_operation_id + and cessation_physical_proof.action == "start" + and cessation_physical_proof.resolution == "start-active-observed" + and cessation_physical_proof.payload_sha256 + == checkpoint.start_payload_sha256 + and cessation_physical_proof.original_start_payload_sha256 + == checkpoint.start_payload_sha256 + and cessation_physical_proof.reconciliation_kind + == "resolved-active-cessation" + and cessation_physical_proof.reconciliation_resolution + == "physical-standby-observed" + and cessation_physical_proof.composite_complete + and cessation_physical_proof.stop_fence == "none" + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "resolved START standby settlement lacks exact terminal lineage" + ) + if _validated_timestamp( + cessation_status_proof.observed_at_utc, + field_name="cessation observed_at_utc", + ) < _validated_timestamp( + gap_started_at_utc, + field_name="gap_started_at_utc", + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby observation precedes the recorded transport gap" + ) + + +def _require_active_rebind_physical_proof( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + _require_active_rebind_physical_proof_shape( + checkpoint, + status_proof=status_proof, + physical_proof=physical_proof, + ) + if ( + checkpoint.physical_lineage_head_revision is None + or physical_proof.ledger_revision + <= checkpoint.physical_lineage_head_revision + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "rebind physical ledger revision must advance active lineage" + ) + + +def _require_active_rebind_physical_proof_shape( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + _require_physical_lineage_base(checkpoint, physical_proof) + _require_matching_observed_proofs( + status_proof=status_proof, + physical_proof=physical_proof, + ) + start_head = bool( + physical_proof.operation_id == checkpoint.original_start_operation_id + and physical_proof.action == "start" + and physical_proof.resolution == "start-active-observed" + and physical_proof.payload_sha256 == checkpoint.start_payload_sha256 + and physical_proof.composite_complete + and physical_proof.stop_fence == "none" + and physical_proof.reconciliation_kind == "resolved-active-rebind" + ) + ambiguous_start_head = bool( + checkpoint.reconciled_start_origin_proof is not None + and checkpoint.reconciled_start_origin_proof.origin_kind + == "ambiguous-reconciled" + and physical_proof.operation_id == checkpoint.original_start_operation_id + and physical_proof.action == "start" + and physical_proof.resolution == "physical-active-observed" + and physical_proof.payload_sha256 == checkpoint.start_payload_sha256 + and not physical_proof.composite_complete + and physical_proof.stop_fence == "none" + and physical_proof.reconciliation_kind == "resolved-active-rebind" + ) + undispatched_stop_head = bool( + physical_proof.operation_id != checkpoint.original_start_operation_id + and _stop_lineage_reaches_checkpoint_start(checkpoint, physical_proof) + and physical_proof.action == "stop" + and physical_proof.resolution == "not-dispatched" + and not physical_proof.composite_complete + and physical_proof.stop_fence == "definitely-not-dispatched" + and physical_proof.reconciliation_kind + in {"prepared-stop-classification", "resolved-active-rebind"} + ) + if ( + physical_proof.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + or physical_proof.reconciliation_resolution != "physical-active-observed" + or physical_proof.effective_resolution != "physical-active-observed" + or not (start_head or ambiguous_start_head or undispatched_stop_head) + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "rebind requires exact active physical lineage with a typed STOP fence" + ) + + +def _require_prepared_cessation( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof | None, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if ( + physical_proof.operation_id != checkpoint.original_start_operation_id + or physical_proof.action != "start" + or physical_proof.payload_sha256 != checkpoint.start_payload_sha256 + or physical_proof.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "prepared resolution must close the exact original START edge" + ) + if physical_proof.resolution == "not-dispatched": + if ( + status_proof is not None + or physical_proof.binding != checkpoint.prepared_binding + or physical_proof.stop_fence != "none" + or physical_proof.reconciliation_kind is not None + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "not-dispatched resolution cannot attach device status or another binding" + ) + return + if not ( + physical_proof.resolution == "physical-standby-observed" + and physical_proof.reconciliation_kind == "ambiguous-outcome" + and physical_proof.reconciliation_resolution == "physical-standby-observed" + and physical_proof.stop_fence == "none" + and not physical_proof.composite_complete + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "prepared ambiguity requires not-dispatched or terminal standby reconciliation" + ) + if status_proof is None or status_proof.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "prepared terminal reconciliation requires fresh READY/SCAN_OVER" + ) + _require_status_binding(checkpoint, status_proof) + _require_matching_observed_proofs( + status_proof=status_proof, + physical_proof=physical_proof, + ) + + +def _require_active_cessation( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + _require_active_cessation_shape( + checkpoint, + status_proof=status_proof, + physical_proof=physical_proof, + ) + if ( + checkpoint.physical_lineage_head_revision is None + or physical_proof.ledger_revision + <= checkpoint.physical_lineage_head_revision + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "cessation physical ledger revision must advance active lineage" + ) + + +def _require_active_cessation_shape( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + status_proof: ActiveAcquisitionRecoveryStatusProof, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + if status_proof.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active cessation requires fresh READY/SCAN_OVER" + ) + _require_status_binding(checkpoint, status_proof) + _require_matching_observed_proofs( + status_proof=status_proof, + physical_proof=physical_proof, + ) + ordinary_stop = bool( + physical_proof.action == "stop" + and _stop_lineage_reaches_checkpoint_start(checkpoint, physical_proof) + and physical_proof.resolution == "stop-standby-observed" + and physical_proof.composite_complete + and physical_proof.reconciliation_kind is None + and physical_proof.stop_fence == "none" + ) + reconciled_start_standby = bool( + physical_proof.action == "start" + and physical_proof.operation_id == checkpoint.original_start_operation_id + and physical_proof.resolution == "start-active-observed" + and physical_proof.composite_complete + and physical_proof.reconciliation_kind == "resolved-active-cessation" + and physical_proof.reconciliation_resolution == "physical-standby-observed" + and physical_proof.stop_fence == "none" + ) + reconciled_ambiguous_start_standby = bool( + checkpoint.reconciled_start_origin_proof is not None + and checkpoint.reconciled_start_origin_proof.origin_kind + == "ambiguous-reconciled" + and physical_proof.action == "start" + and physical_proof.operation_id == checkpoint.original_start_operation_id + and physical_proof.resolution == "physical-active-observed" + and not physical_proof.composite_complete + and physical_proof.reconciliation_kind == "resolved-active-cessation" + and physical_proof.reconciliation_resolution + == "physical-standby-observed" + and physical_proof.stop_fence == "none" + ) + reconciled_undispatched_stop_standby = bool( + physical_proof.action == "stop" + and physical_proof.operation_id != checkpoint.original_start_operation_id + and _stop_lineage_reaches_checkpoint_start(checkpoint, physical_proof) + and physical_proof.resolution == "not-dispatched" + and not physical_proof.composite_complete + and physical_proof.reconciliation_kind + in {"prepared-stop-classification", "resolved-active-cessation"} + and physical_proof.reconciliation_resolution == "physical-standby-observed" + and physical_proof.stop_fence == "definitely-not-dispatched" + ) + if not ( + ordinary_stop + or reconciled_start_standby + or reconciled_ambiguous_start_standby + or reconciled_undispatched_stop_standby + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active cessation lacks terminal physical standby lineage" + ) + if physical_proof.original_start_payload_sha256 != checkpoint.start_payload_sha256: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "START reconciliation does not match the original physical edge" + ) + if not _has_open_gap(checkpoint) and status_proof.binding != checkpoint.current_binding: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active cessation must use the exact current binding" + ) + if not _has_open_gap(checkpoint) and status_proof.evidence_session_id != ( + checkpoint.current_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "active cessation must use the exact current evidence session" + ) + + +def _require_active_reconciled_standby_shape( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + *, + failed_binding: ActiveAcquisitionRecoveryTransportBinding, + failed_evidence_session_id: str, + gap_started_at_utc: str, + cessation_status_proof: ActiveAcquisitionRecoveryStatusProof, + cessation_physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> None: + """Validate the restart-only portion of an atomic ACTIVE settlement.""" + + if cessation_status_proof.source != "explicit-read-only-reconciliation": + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires explicit read-only status" + ) + if cessation_status_proof.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires fresh READY/SCAN_OVER" + ) + _require_status_binding(checkpoint, cessation_status_proof) + _require_physical_lineage_base(checkpoint, cessation_physical_proof) + _require_matching_observed_proofs( + status_proof=cessation_status_proof, + physical_proof=cessation_physical_proof, + ) + if not ( + cessation_physical_proof.reconciliation_kind + in {"prepared-stop-classification", "resolved-active-cessation"} + and cessation_physical_proof.reconciliation_resolution + == "physical-standby-observed" + and cessation_physical_proof.effective_resolution + == "physical-standby-observed" + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires exact terminal reconciliation lineage" + ) + + prior_status = checkpoint.current_active_status_proof + prior_physical = checkpoint.current_active_physical_proof + if prior_status is None or prior_physical is None: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement lacks the exact prior active proof" + ) + if ( + prior_status.binding != failed_binding + or prior_status.evidence_session_id != failed_evidence_session_id + or prior_status.session_state != "scanning" + or checkpoint.active_project_id_sha256 is None + or prior_status.project_id_sha256 != checkpoint.active_project_id_sha256 + or prior_physical.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement does not preserve the failed active lineage" + ) + if cessation_physical_proof.ledger_revision <= prior_physical.ledger_revision: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby physical proof must advance the active lineage" + ) + if ( + cessation_status_proof.binding.runtime_instance_id + == failed_binding.runtime_instance_id + or cessation_status_proof.binding.control_session_id + == failed_binding.control_session_id + or cessation_status_proof.evidence_session_id + == failed_evidence_session_id + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby settlement requires new runtime, control and evidence sessions" + ) + + gap_started = _validated_timestamp( + gap_started_at_utc, + field_name="gap_started_at_utc", + ) + if gap_started < _validated_timestamp( + prior_status.observed_at_utc, + field_name="prior active observed_at_utc", + ): + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart gap cannot precede the prior active proof" + ) + if _validated_timestamp( + cessation_status_proof.observed_at_utc, + field_name="cessation observed_at_utc", + ) < gap_started: + raise ActiveAcquisitionRecoveryCheckpointTransitionError( + "restart standby observation precedes the recorded transport gap" + ) + + +def _has_open_gap(checkpoint: ActiveAcquisitionRecoveryCheckpoint) -> bool: + return bool( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ) + + +def _transition_id_sha256(transition_id: str) -> str: + _validate_identifier(transition_id, field_name="transition_id") + return hashlib.sha256(transition_id.encode("utf-8")).hexdigest() + + +def _request_fingerprint( + kind: ActiveAcquisitionRecoveryTransitionKind, + payload: Mapping[str, object], +) -> str: + _validate_transition_kind(kind) + return hashlib.sha256( + _canonical_json_bytes({"kind": kind, "payload": dict(payload)}) + ).hexdigest() + + +def _append_transition_receipt( + receipts: tuple[ActiveAcquisitionRecoveryTransitionReceipt, ...], + *, + transition_id: str, + kind: ActiveAcquisitionRecoveryTransitionKind, + fingerprint: str, + committed_revision: int, + committed_at_utc: str, +) -> tuple[ActiveAcquisitionRecoveryTransitionReceipt, ...]: + # Idempotent replay is deliberately bounded to this receipt horizon. Once + # evicted, an old request has no replay authority and must pass the current + # acquisition/state/revision CAS like a new request (normally failing stale). + retained = receipts[-(ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS - 1) :] + return ( + *retained, + ActiveAcquisitionRecoveryTransitionReceipt( + transition_id_sha256=_transition_id_sha256(transition_id), + kind=kind, + request_fingerprint_sha256=fingerprint, + committed_revision=committed_revision, + committed_at_utc=committed_at_utc, + ), + ) + + +def _next_revision(current_revision: int) -> int: + if current_revision >= ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint revision is exhausted" + ) + return current_revision + 1 + + +def _next_transport_revision(current_revision: int) -> int: + if current_revision >= ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION: + raise ActiveAcquisitionRecoveryCheckpointCorrupt( + "checkpoint transport revision is exhausted" + ) + return current_revision + 1 + + +def _read_private_json(path: Path) -> object: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except FileNotFoundError: + raise + except OSError as exc: + raise ValueError("checkpoint file cannot be opened safely") from exc + try: + before = os.fstat(descriptor) + _require_private_regular_file(before, label="checkpoint file", empty=False) + if not 0 < before.st_size <= ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES: + raise ValueError("checkpoint file exceeds its bounded size") + raw = _read_bounded( + descriptor, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES, + ) + after = os.fstat(descriptor) + path_metadata = path.lstat() + _require_private_regular_file(after, label="checkpoint file", empty=False) + if (after.st_dev, after.st_ino) != ( + path_metadata.st_dev, + path_metadata.st_ino, + ): + raise ValueError("checkpoint file identity changed while reading") + if ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_uid, + before.st_nlink, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_uid, + after.st_nlink, + after.st_size, + after.st_mtime_ns, + ): + raise ValueError("checkpoint file metadata changed while reading") + finally: + os.close(descriptor) + return json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object) + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = _canonical_json_bytes(payload) + if len(serialized) > ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES: + raise ValueError("checkpoint payload exceeds its bounded size") + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(path.parent, parents=False) + previous_identity = _existing_private_file_identity(path) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + published_identity = os.fstat(stream.fileno()) + _require_unchanged_existing_path(path, previous_identity) + os.replace(temp_path, path) + destination = path.lstat() + _require_private_regular_file( + destination, + label="published checkpoint", + empty=False, + ) + if (destination.st_dev, destination.st_ino) != ( + published_identity.st_dev, + published_identity.st_ino, + ): + raise OSError("checkpoint atomic identity mismatch") + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _canonical_json_bytes(payload: Mapping[str, object]) -> bytes: + return ( + json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + + +def _existing_private_file_identity(path: Path) -> tuple[int, int] | None: + try: + metadata = path.lstat() + except FileNotFoundError: + return None + _require_private_regular_file(metadata, label="checkpoint file", empty=False) + return metadata.st_dev, metadata.st_ino + + +def _require_unchanged_existing_path( + path: Path, + expected: tuple[int, int] | None, +) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + if expected is None: + return + raise ValueError("checkpoint file disappeared during publication") from None + _require_private_regular_file(metadata, label="checkpoint file", empty=False) + observed = metadata.st_dev, metadata.st_ino + if expected is None or observed != expected: + raise ValueError("checkpoint file changed during publication") + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + metadata = path.lstat() + _require_private_directory(metadata, label="checkpoint directory") + return True + _require_private_directory(metadata, label="checkpoint directory") + return False + + +def _require_private_directory(metadata: os.stat_result, *, label: str) -> None: + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o700 + or metadata.st_uid != os.getuid() + ): + raise ValueError(f"{label} is not private and owner-controlled") + + +def _require_private_regular_file( + metadata: os.stat_result, + *, + label: str, + empty: bool, +) -> None: + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_uid != os.getuid() + or metadata.st_nlink != 1 + or (empty and metadata.st_size != 0) + ): + raise ValueError(f"{label} is not a private regular file") + + +def _read_bounded(descriptor: int, limit: int) -> bytes: + chunks: list[bytes] = [] + remaining = limit + 1 + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 16 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + if len(payload) > limit: + raise ValueError("checkpoint file exceeds its bounded size") + return payload + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("checkpoint contains duplicate fields") + document[key] = value + return document + + +def _checkpoint_from_mapping(value: object) -> ActiveAcquisitionRecoveryCheckpoint: + legacy_keys = { + "schema_version", + "revision", + "state", + "transport_revision", + "physical_lineage_head_revision", + "acquisition_id", + "original_start_operation_id", + "start_payload_sha256", + "identity", + "connection", + "compatibility_profile_id", + "project_name", + "project_name_wire_sha256", + "active_project_id_sha256", + "original_evidence_session_id", + "current_evidence_session_id", + "duration_seconds", + "requested_streams", + "evidence_policy", + "mount_type", + "gnss_mode", + "prepared_binding", + "current_binding", + "created_at_utc", + "updated_at_utc", + "activated_at_utc", + "ceased_at_utc", + "last_gap_started_at_utc", + "last_gap_started_transport_revision", + "last_gap_failed_binding", + "last_gap_recovered_at_utc", + "last_gap_recovered_transport_revision", + "activation_status_proof", + "activation_physical_proof", + "current_active_status_proof", + "current_active_physical_proof", + "last_rebind_status_proof", + "last_rebind_physical_proof", + "cessation_status_proof", + "cessation_physical_proof", + "prepared_resolution_proof", + "transition_receipts", + } + if not isinstance(value, dict): + raise ValueError("checkpoint does not match the secret-free checkpoint schema") + source_schema = value.get("schema_version") + legacy_v2 = source_schema == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA + legacy_before_v4 = source_schema in { + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA, + } + legacy_before_v5 = source_schema in { + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA, + } + if source_schema not in { + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA, + }: + raise ValueError("unsupported checkpoint schema") + keys = set(legacy_keys) + if not legacy_before_v4: + keys.update( + {"reconciled_start_origin_proof", "first_published_pcl_proof"} + ) + if not legacy_before_v5: + keys.update( + { + "prepared_resolved_start_reconciliation_id", + "prepared_resolved_start_original_attempt_sha256", + "prepared_resolved_start_original_project_id_sha256", + } + ) + document = _exact_mapping(value, keys, label="checkpoint") + state = _required_string(document["state"], field_name="state") + if state not in _CHECKPOINT_STATES: + raise ValueError("unsupported checkpoint state") + project_name = _exact_project_name( + _required_string(document["project_name"], field_name="project_name") + ) + checkpoint = ActiveAcquisitionRecoveryCheckpoint( + schema_version=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + revision=_positive_revision(document["revision"], field_name="revision"), + state=cast(ActiveAcquisitionRecoveryCheckpointState, state), + transport_revision=_positive_revision( + document["transport_revision"], + field_name="transport_revision", + ), + physical_lineage_head_revision=_optional_positive_revision( + document["physical_lineage_head_revision"], + field_name="physical_lineage_head_revision", + ), + acquisition_id=_identifier_from_value( + document["acquisition_id"], + field_name="acquisition_id", + ), + original_start_operation_id=_identifier_from_value( + document["original_start_operation_id"], + field_name="original_start_operation_id", + ), + start_payload_sha256=_sha256_from_value( + document["start_payload_sha256"], + field_name="start_payload_sha256", + ), + identity=_identity_from_mapping(document["identity"]), + connection=_connection_from_mapping(document["connection"]), + compatibility_profile_id=_identifier_from_value( + document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ), + project_name=project_name, + project_name_wire_sha256=_sha256_from_value( + document["project_name_wire_sha256"], + field_name="project_name_wire_sha256", + ), + active_project_id_sha256=_optional_sha256( + document["active_project_id_sha256"], + field_name="active_project_id_sha256", + ), + original_evidence_session_id=_identifier_from_value( + document["original_evidence_session_id"], + field_name="original_evidence_session_id", + ), + current_evidence_session_id=_identifier_from_value( + document["current_evidence_session_id"], + field_name="current_evidence_session_id", + ), + duration_seconds=_optional_duration_seconds( + document["duration_seconds"], + field_name="duration_seconds", + ), + requested_streams=_requested_streams_from_value(document["requested_streams"]), + evidence_policy=_evidence_policy_from_value(document["evidence_policy"]), + mount_type=_mount_type_from_value(document["mount_type"]), + gnss_mode=_gnss_mode_from_value(document["gnss_mode"]), + prepared_binding=_binding_from_mapping(document["prepared_binding"]), + current_binding=_binding_from_mapping(document["current_binding"]), + created_at_utc=_canonical_timestamp_value( + document["created_at_utc"], + field_name="created_at_utc", + ), + updated_at_utc=_canonical_timestamp_value( + document["updated_at_utc"], + field_name="updated_at_utc", + ), + activated_at_utc=_optional_timestamp( + document["activated_at_utc"], + field_name="activated_at_utc", + ), + ceased_at_utc=_optional_timestamp( + document["ceased_at_utc"], + field_name="ceased_at_utc", + ), + last_gap_started_at_utc=_optional_timestamp( + document["last_gap_started_at_utc"], + field_name="last_gap_started_at_utc", + ), + last_gap_started_transport_revision=_optional_positive_revision( + document["last_gap_started_transport_revision"], + field_name="last_gap_started_transport_revision", + ), + last_gap_failed_binding=_optional_binding( + document["last_gap_failed_binding"] + ), + last_gap_recovered_at_utc=_optional_timestamp( + document["last_gap_recovered_at_utc"], + field_name="last_gap_recovered_at_utc", + ), + last_gap_recovered_transport_revision=_optional_positive_revision( + document["last_gap_recovered_transport_revision"], + field_name="last_gap_recovered_transport_revision", + ), + activation_status_proof=_optional_status_proof( + document["activation_status_proof"] + ), + activation_physical_proof=_optional_physical_proof( + document["activation_physical_proof"], + legacy_v2=legacy_v2, + ), + current_active_status_proof=_optional_status_proof( + document["current_active_status_proof"] + ), + current_active_physical_proof=_optional_physical_proof( + document["current_active_physical_proof"], + legacy_v2=legacy_v2, + ), + last_rebind_status_proof=_optional_status_proof( + document["last_rebind_status_proof"] + ), + last_rebind_physical_proof=_optional_physical_proof( + document["last_rebind_physical_proof"], + legacy_v2=legacy_v2, + ), + cessation_status_proof=_optional_status_proof( + document["cessation_status_proof"] + ), + cessation_physical_proof=_optional_physical_proof( + document["cessation_physical_proof"], + legacy_v2=legacy_v2, + ), + prepared_resolution_proof=_optional_physical_proof( + document["prepared_resolution_proof"], + legacy_v2=legacy_v2, + ), + reconciled_start_origin_proof=( + None + if legacy_before_v4 + else _optional_start_origin_proof( + document["reconciled_start_origin_proof"] + ) + ), + first_published_pcl_proof=( + None + if legacy_before_v4 + else _optional_first_published_pcl_proof( + document["first_published_pcl_proof"] + ) + ), + prepared_resolved_start_reconciliation_id=( + None + if legacy_before_v5 + else _optional_identifier_from_value( + document["prepared_resolved_start_reconciliation_id"], + field_name="prepared_resolved_start_reconciliation_id", + ) + ), + prepared_resolved_start_original_attempt_sha256=( + None + if legacy_before_v5 + else _optional_sha256( + document[ + "prepared_resolved_start_original_attempt_sha256" + ], + field_name=( + "prepared_resolved_start_original_attempt_sha256" + ), + ) + ), + prepared_resolved_start_original_project_id_sha256=( + None + if legacy_before_v5 + else _optional_sha256( + document[ + "prepared_resolved_start_original_project_id_sha256" + ], + field_name=( + "prepared_resolved_start_original_project_id_sha256" + ), + ) + ), + transition_receipts=_receipts_from_value(document["transition_receipts"]), + ) + return checkpoint + + +def _identity_from_mapping(value: object) -> ActiveAcquisitionRecoveryIdentity: + document = _exact_mapping( + value, + {"logical_device_id", "vendor_device_id_sha256", "device_serial_sha256"}, + label="identity", + ) + return ActiveAcquisitionRecoveryIdentity( + logical_device_id=_identifier_from_value( + document["logical_device_id"], + field_name="logical_device_id", + ), + vendor_device_id_sha256=_sha256_from_value( + document["vendor_device_id_sha256"], + field_name="vendor_device_id_sha256", + ), + device_serial_sha256=_sha256_from_value( + document["device_serial_sha256"], + field_name="device_serial_sha256", + ), + ) + + +def _connection_from_mapping(value: object) -> ActiveAcquisitionRecoveryConnection: + document = _exact_mapping( + value, + {"transport_ref", "connection_mode", "target_ipv4", "target_port"}, + label="connection", + ) + mode = _connection_mode_from_value(document["connection_mode"]) + return ActiveAcquisitionRecoveryConnection( + transport_ref=_identifier_from_value( + document["transport_ref"], + field_name="transport_ref", + ), + connection_mode=mode, + target_ipv4=_required_string( + document["target_ipv4"], + field_name="target_ipv4", + ), + target_port=_positive_int(document["target_port"], field_name="target_port"), + ) + + +def _binding_from_mapping(value: object) -> ActiveAcquisitionRecoveryTransportBinding: + document = _exact_mapping( + value, + { + "runtime_instance_id", + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + "logical_device_id", + "compatibility_profile_id", + "vendor_device_id_sha256", + "device_serial_sha256", + }, + label="transport binding", + ) + return ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=_identifier_from_value( + document["runtime_instance_id"], + field_name="runtime_instance_id", + ), + intent_id=_identifier_from_value( + document["intent_id"], + field_name="intent_id", + ), + transport_ref=_identifier_from_value( + document["transport_ref"], + field_name="transport_ref", + ), + connection_mode=_connection_mode_from_value(document["connection_mode"]), + target_ipv4=_required_string( + document["target_ipv4"], + field_name="target_ipv4", + ), + target_port=_positive_int(document["target_port"], field_name="target_port"), + host_path_epoch=_positive_int( + document["host_path_epoch"], + field_name="host_path_epoch", + ), + control_session_id=_identifier_from_value( + document["control_session_id"], + field_name="control_session_id", + ), + producer_generation=_positive_int( + document["producer_generation"], + field_name="producer_generation", + ), + logical_device_id=_identifier_from_value( + document["logical_device_id"], + field_name="logical_device_id", + ), + compatibility_profile_id=_identifier_from_value( + document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ), + vendor_device_id_sha256=_sha256_from_value( + document["vendor_device_id_sha256"], + field_name="vendor_device_id_sha256", + ), + device_serial_sha256=_sha256_from_value( + document["device_serial_sha256"], + field_name="device_serial_sha256", + ), + ) + + +def _optional_binding( + value: object, +) -> ActiveAcquisitionRecoveryTransportBinding | None: + return None if value is None else _binding_from_mapping(value) + + +def _status_proof_from_mapping(value: object) -> ActiveAcquisitionRecoveryStatusProof: + document = _exact_mapping( + value, + { + "source", + "session_state", + "session_state_code", + "binding", + "evidence_session_id", + "project_id_sha256", + "init_ready", + "status_message_sha256", + "mqtt_retained", + "observed_at_utc", + }, + label="status proof", + ) + source = _proof_source_from_value(document["source"]) + state = _proof_state_from_value(document["session_state"]) + retained = _required_bool(document["mqtt_retained"], field_name="mqtt_retained") + if retained: + raise ValueError("checkpoint proof must be fresh and non-retained") + return ActiveAcquisitionRecoveryStatusProof( + source=source, + session_state=state, + session_state_code=_nonnegative_int( + document["session_state_code"], + field_name="session_state_code", + ), + binding=_binding_from_mapping(document["binding"]), + evidence_session_id=_identifier_from_value( + document["evidence_session_id"], + field_name="evidence_session_id", + ), + project_id_sha256=_optional_sha256( + document["project_id_sha256"], + field_name="project_id_sha256", + ), + init_ready=_required_bool(document["init_ready"], field_name="init_ready"), + status_message_sha256=_sha256_from_value( + document["status_message_sha256"], + field_name="status_message_sha256", + ), + mqtt_retained=False, + observed_at_utc=_canonical_timestamp_value( + document["observed_at_utc"], + field_name="observed_at_utc", + ), + ) + + +def _optional_status_proof( + value: object, +) -> ActiveAcquisitionRecoveryStatusProof | None: + return None if value is None else _status_proof_from_mapping(value) + + +def _classified_stop_ancestry_from_mapping( + value: object, +) -> PhysicalCommandClassifiedStopAncestryProof: + document = _exact_mapping( + value, + { + "schema_version", + "ledger_revision", + "head_operation_id", + "head_parent_operation_id", + "root_start_operation_id", + "acquisition_id", + "depth", + "chain_sha256", + }, + label="classified STOP ancestry", + ) + if document["schema_version"] != PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA: + raise ValueError("unsupported classified STOP ancestry schema") + return PhysicalCommandClassifiedStopAncestryProof( + schema_version=PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + ledger_revision=_positive_revision( + document["ledger_revision"], + field_name="ledger_revision", + ), + head_operation_id=_identifier_from_value( + document["head_operation_id"], + field_name="head_operation_id", + ), + head_parent_operation_id=_identifier_from_value( + document["head_parent_operation_id"], + field_name="head_parent_operation_id", + ), + root_start_operation_id=_identifier_from_value( + document["root_start_operation_id"], + field_name="root_start_operation_id", + ), + acquisition_id=_identifier_from_value( + document["acquisition_id"], + field_name="acquisition_id", + ), + depth=_positive_int(document["depth"], field_name="depth"), + chain_sha256=_sha256_from_value( + document["chain_sha256"], + field_name="chain_sha256", + ), + ) + + +def _optional_classified_stop_ancestry( + value: object, +) -> PhysicalCommandClassifiedStopAncestryProof | None: + return None if value is None else _classified_stop_ancestry_from_mapping(value) + + +def _physical_proof_from_mapping( + value: object, + *, + legacy_v2: bool = False, +) -> ActiveAcquisitionRecoveryPhysicalLineageProof: + keys = { + "ledger_schema_version", + "ledger_revision", + "proof_id", + "operation_id", + "original_start_operation_id", + "parent_operation_id", + "acquisition_id", + "action", + "resolution", + "payload_sha256", + "original_start_payload_sha256", + "reconciliation_kind", + "reconciliation_resolution", + "status_message_sha256", + "observed_session_state", + "binding", + "composite_complete", + "edge_terminal", + "late_start_excluded", + "stop_fence", + "observed_at_utc", + } + if not legacy_v2: + keys.add("ancestor_chain") + if isinstance(value, Mapping) and ( + "reconciliation_original_attempt_sha256" in value + ): + keys.add("reconciliation_original_attempt_sha256") + document = _exact_mapping( + value, + keys, + label="physical lineage proof", + ) + if document["ledger_schema_version"] != ( + ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA + ): + raise ValueError("unsupported physical lineage schema") + return ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=_positive_revision( + document["ledger_revision"], + field_name="ledger_revision", + ), + proof_id=_identifier_from_value(document["proof_id"], field_name="proof_id"), + operation_id=_identifier_from_value( + document["operation_id"], + field_name="operation_id", + ), + original_start_operation_id=_identifier_from_value( + document["original_start_operation_id"], + field_name="original_start_operation_id", + ), + parent_operation_id=_optional_identifier_from_value( + document["parent_operation_id"], + field_name="parent_operation_id", + ), + acquisition_id=_identifier_from_value( + document["acquisition_id"], + field_name="acquisition_id", + ), + action=_physical_action_from_value(document["action"]), + resolution=_physical_resolution_from_value(document["resolution"]), + payload_sha256=_sha256_from_value( + document["payload_sha256"], + field_name="payload_sha256", + ), + original_start_payload_sha256=_sha256_from_value( + document["original_start_payload_sha256"], + field_name="original_start_payload_sha256", + ), + reconciliation_kind=_optional_physical_reconciliation_kind_from_value( + document["reconciliation_kind"] + ), + reconciliation_resolution=_optional_physical_resolution_from_value( + document["reconciliation_resolution"] + ), + status_message_sha256=_optional_sha256( + document["status_message_sha256"], + field_name="status_message_sha256", + ), + observed_session_state=_optional_proof_state( + document["observed_session_state"] + ), + binding=_binding_from_mapping(document["binding"]), + composite_complete=_required_bool( + document["composite_complete"], + field_name="composite_complete", + ), + edge_terminal=cast( + Literal[True], + _required_true(document["edge_terminal"], field_name="edge_terminal"), + ), + late_start_excluded=cast( + Literal[True], + _required_true( + document["late_start_excluded"], + field_name="late_start_excluded", + ), + ), + stop_fence=_stop_fence_from_value(document["stop_fence"]), + observed_at_utc=_canonical_timestamp_value( + document["observed_at_utc"], + field_name="observed_at_utc", + ), + ancestor_chain=( + None + if legacy_v2 + else _optional_classified_stop_ancestry(document["ancestor_chain"]) + ), + reconciliation_original_attempt_sha256=_optional_sha256( + document.get("reconciliation_original_attempt_sha256"), + field_name="reconciliation_original_attempt_sha256", + ), + ) + + +def _optional_physical_proof( + value: object, + *, + legacy_v2: bool = False, +) -> ActiveAcquisitionRecoveryPhysicalLineageProof | None: + return ( + None + if value is None + else _physical_proof_from_mapping(value, legacy_v2=legacy_v2) + ) + + +def _start_origin_proof_from_mapping( + value: object, +) -> ActiveAcquisitionRecoveryStartOriginProof: + keys = { + "schema_version", + "ledger_schema_version", + "ledger_revision", + "physical_proof_id", + "reconciliation_id", + "origin_kind", + "original_attempt_revision", + "original_attempt_stage", + "original_attempt_resolution", + "original_attempt_sha256", + "operation_id", + "parent_operation_id", + "acquisition_id", + "payload_sha256", + "baseline_status_proof", + "original_project_id_sha256", + "automatic_replay_allowed", + "project_evidence_strength", + } + if isinstance(value, Mapping) and "reconciled_active_project_id_sha256" in value: + keys.add("reconciled_active_project_id_sha256") + document = _exact_mapping( + value, + keys, + label="START origin proof", + ) + if document["schema_version"] != ( + ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA + ): + raise ValueError("unsupported START origin proof schema") + if document["ledger_schema_version"] != ( + ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA + ): + raise ValueError("unsupported START origin physical ledger schema") + origin_kind = _required_string(document["origin_kind"], field_name="origin_kind") + if origin_kind not in _START_ORIGIN_KINDS: + raise ValueError("unsupported START origin kind") + attempt_stage = _required_string( + document["original_attempt_stage"], + field_name="original_attempt_stage", + ) + if attempt_stage not in _START_ATTEMPT_STAGES: + raise ValueError("unsupported START origin attempt stage") + resolution = document["original_attempt_resolution"] + if resolution not in {None, "start-active-observed"}: + raise ValueError("unsupported original START resolution") + strength = _required_string( + document["project_evidence_strength"], + field_name="project_evidence_strength", + ) + if strength not in _PROJECT_EVIDENCE_STRENGTHS: + raise ValueError("unsupported START project evidence strength") + replay = _required_bool( + document["automatic_replay_allowed"], + field_name="automatic_replay_allowed", + ) + if replay: + raise ValueError("reconciled START origin never permits command replay") + return ActiveAcquisitionRecoveryStartOriginProof( + schema_version=ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA, + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=_positive_revision( + document["ledger_revision"], + field_name="ledger_revision", + ), + physical_proof_id=_identifier_from_value( + document["physical_proof_id"], + field_name="physical_proof_id", + ), + reconciliation_id=_identifier_from_value( + document["reconciliation_id"], + field_name="reconciliation_id", + ), + origin_kind=cast(ActiveAcquisitionRecoveryStartOriginKind, origin_kind), + original_attempt_revision=_positive_revision( + document["original_attempt_revision"], + field_name="original_attempt_revision", + ), + original_attempt_stage=cast( + ActiveAcquisitionRecoveryStartAttemptStage, + attempt_stage, + ), + original_attempt_resolution=cast( + Literal["start-active-observed"] | None, + resolution, + ), + original_attempt_sha256=_sha256_from_value( + document["original_attempt_sha256"], + field_name="original_attempt_sha256", + ), + operation_id=_identifier_from_value( + document["operation_id"], + field_name="operation_id", + ), + parent_operation_id=_optional_identifier_from_value( + document["parent_operation_id"], + field_name="parent_operation_id", + ), + acquisition_id=_identifier_from_value( + document["acquisition_id"], + field_name="acquisition_id", + ), + payload_sha256=_sha256_from_value( + document["payload_sha256"], + field_name="payload_sha256", + ), + baseline_status_proof=_status_proof_from_mapping( + document["baseline_status_proof"] + ), + original_project_id_sha256=_optional_sha256( + document["original_project_id_sha256"], + field_name="original_project_id_sha256", + ), + automatic_replay_allowed=False, + project_evidence_strength=cast( + ActiveAcquisitionRecoveryProjectEvidenceStrength, + strength, + ), + reconciled_active_project_id_sha256=_optional_sha256( + document.get("reconciled_active_project_id_sha256"), + field_name="reconciled_active_project_id_sha256", + ), + ) + + +def _optional_start_origin_proof( + value: object, +) -> ActiveAcquisitionRecoveryStartOriginProof | None: + return None if value is None else _start_origin_proof_from_mapping(value) + + +def _first_published_pcl_proof_from_mapping( + value: object, +) -> ActiveAcquisitionRecoveryFirstPublishedPclProof: + document = _exact_mapping( + value, + { + "schema_version", + "proof_id", + "runtime_instance_id", + "evidence_session_id", + "viewer_producer_generation", + "frame_sequence", + "captured_at_epoch_ns", + "point_count", + "published_pcl_frames", + "published_at_utc", + }, + label="first published PCL proof", + ) + if document["schema_version"] != ( + ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA + ): + raise ValueError("unsupported first-published-PCL proof schema") + return ActiveAcquisitionRecoveryFirstPublishedPclProof( + schema_version=ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA, + proof_id=_identifier_from_value( + document["proof_id"], + field_name="proof_id", + ), + runtime_instance_id=_identifier_from_value( + document["runtime_instance_id"], + field_name="runtime_instance_id", + ), + evidence_session_id=_identifier_from_value( + document["evidence_session_id"], + field_name="evidence_session_id", + ), + viewer_producer_generation=_positive_int( + document["viewer_producer_generation"], + field_name="viewer_producer_generation", + ), + frame_sequence=_positive_int( + document["frame_sequence"], + field_name="frame_sequence", + ), + captured_at_epoch_ns=_nonnegative_int( + document["captured_at_epoch_ns"], + field_name="captured_at_epoch_ns", + ), + point_count=_positive_int( + document["point_count"], + field_name="point_count", + ), + published_pcl_frames=_positive_int( + document["published_pcl_frames"], + field_name="published_pcl_frames", + ), + published_at_utc=_canonical_timestamp_value( + document["published_at_utc"], + field_name="published_at_utc", + ), + ) + + +def _optional_first_published_pcl_proof( + value: object, +) -> ActiveAcquisitionRecoveryFirstPublishedPclProof | None: + return None if value is None else _first_published_pcl_proof_from_mapping(value) + + +def _receipt_from_mapping( + value: object, +) -> ActiveAcquisitionRecoveryTransitionReceipt: + document = _exact_mapping( + value, + { + "transition_id_sha256", + "kind", + "request_fingerprint_sha256", + "committed_revision", + "committed_at_utc", + }, + label="transition receipt", + ) + return ActiveAcquisitionRecoveryTransitionReceipt( + transition_id_sha256=_sha256_from_value( + document["transition_id_sha256"], + field_name="transition_id_sha256", + ), + kind=_transition_kind_from_value(document["kind"]), + request_fingerprint_sha256=_sha256_from_value( + document["request_fingerprint_sha256"], + field_name="request_fingerprint_sha256", + ), + committed_revision=_positive_revision( + document["committed_revision"], + field_name="committed_revision", + ), + committed_at_utc=_canonical_timestamp_value( + document["committed_at_utc"], + field_name="committed_at_utc", + ), + ) + + +def _receipts_from_value( + value: object, +) -> tuple[ActiveAcquisitionRecoveryTransitionReceipt, ...]: + if not isinstance(value, list) or not 1 <= len(value) <= ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS + ): + raise ValueError("transition_receipts must be a bounded non-empty array") + return tuple(_receipt_from_mapping(item) for item in value) + + +def _validate_checkpoint_semantics( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, +) -> None: + if checkpoint.schema_version != ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA: + raise ValueError("unsupported checkpoint schema") + _validate_positive_revision(checkpoint.revision, field_name="revision") + _validate_positive_revision( + checkpoint.transport_revision, + field_name="transport_revision", + ) + if checkpoint.physical_lineage_head_revision is not None: + _validate_positive_revision( + checkpoint.physical_lineage_head_revision, + field_name="physical_lineage_head_revision", + ) + _validate_identifier(checkpoint.acquisition_id, field_name="acquisition_id") + _validate_identifier( + checkpoint.original_start_operation_id, + field_name="original_start_operation_id", + ) + _validate_sha256(checkpoint.start_payload_sha256, field_name="start_payload_sha256") + _validate_identifier( + checkpoint.compatibility_profile_id, + field_name="compatibility_profile_id", + ) + if checkpoint.project_name != _exact_project_name(checkpoint.project_name): + raise ValueError("project_name is not exact canonical wire text") + if checkpoint.project_name_wire_sha256 != active_acquisition_project_name_sha256( + checkpoint.project_name + ): + raise ValueError("project_name_wire_sha256 does not match exact wire name") + if checkpoint.active_project_id_sha256 is not None: + _validate_sha256( + checkpoint.active_project_id_sha256, + field_name="active_project_id_sha256", + ) + _validate_identifier( + checkpoint.original_evidence_session_id, + field_name="original_evidence_session_id", + ) + _validate_identifier( + checkpoint.current_evidence_session_id, + field_name="current_evidence_session_id", + ) + _optional_duration_seconds( + checkpoint.duration_seconds, + field_name="duration_seconds", + ) + if checkpoint.requested_streams != _validated_requested_streams( + checkpoint.requested_streams + ): + raise ValueError("requested_streams are not canonical") + _validate_evidence_policy(checkpoint.evidence_policy) + _validate_mount_type(checkpoint.mount_type) + _validate_gnss_mode(checkpoint.gnss_mode) + for binding in (checkpoint.prepared_binding, checkpoint.current_binding): + _require_binding_matches_checkpoint_values( + identity=checkpoint.identity, + connection=checkpoint.connection, + compatibility_profile_id=checkpoint.compatibility_profile_id, + binding=binding, + ) + if checkpoint.last_gap_failed_binding is not None: + _require_binding_matches_checkpoint_values( + identity=checkpoint.identity, + connection=checkpoint.connection, + compatibility_profile_id=checkpoint.compatibility_profile_id, + binding=checkpoint.last_gap_failed_binding, + ) + _canonical_timestamp_value(checkpoint.created_at_utc, field_name="created_at_utc") + _canonical_timestamp_value(checkpoint.updated_at_utc, field_name="updated_at_utc") + if _validated_timestamp( + checkpoint.updated_at_utc, + field_name="updated_at_utc", + ) < _validated_timestamp( + checkpoint.created_at_utc, + field_name="created_at_utc", + ): + raise ValueError("checkpoint updated_at_utc cannot precede created_at_utc") + for field_name, value in ( + ("activated_at_utc", checkpoint.activated_at_utc), + ("ceased_at_utc", checkpoint.ceased_at_utc), + ("last_gap_started_at_utc", checkpoint.last_gap_started_at_utc), + ("last_gap_recovered_at_utc", checkpoint.last_gap_recovered_at_utc), + ): + if value is not None: + _canonical_timestamp_value(value, field_name=field_name) + + receipts = checkpoint.transition_receipts + if not 1 <= len(receipts) <= ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS: + raise ValueError("checkpoint requires a bounded transition receipt journal") + if len({item.transition_id_sha256 for item in receipts}) != len(receipts): + raise ValueError("transition receipt ids must be unique") + previous_revision = 0 + previous_committed_at: datetime | None = None + for receipt in receipts: + if receipt.committed_revision <= previous_revision: + raise ValueError("transition receipt revisions must be strictly increasing") + committed_at = _validated_timestamp( + receipt.committed_at_utc, + field_name="receipt committed_at_utc", + ) + if previous_committed_at is not None and committed_at < previous_committed_at: + raise ValueError("transition receipt audit timestamps cannot decrease") + previous_revision = receipt.committed_revision + previous_committed_at = committed_at + if receipts[-1].committed_revision != checkpoint.revision: + raise ValueError("last transition receipt must match checkpoint revision") + if receipts[-1].kind not in ( + {"prepare"} + if checkpoint.state == "prepared" + else { + "cease", + "cease-active-reconciled-standby", + "cease-prepared-reconciled", + "cease-prepared-resolved-start-standby", + } + if checkpoint.state == "ceased" + else {receipts[-1].kind} + ) and checkpoint.state in {"prepared", "ceased"}: + raise ValueError("terminal checkpoint state does not match last transition") + + gap_start = ( + checkpoint.last_gap_started_at_utc, + checkpoint.last_gap_started_transport_revision, + checkpoint.last_gap_failed_binding, + ) + gap_recovery = ( + checkpoint.last_gap_recovered_at_utc, + checkpoint.last_gap_recovered_transport_revision, + ) + if len({value is None for value in gap_start}) != 1: + raise ValueError("gap start audit fields must be all present or all absent") + if len({value is None for value in gap_recovery}) != 1: + raise ValueError("gap recovery audit fields must be paired") + if gap_recovery[0] is not None: + if gap_start[0] is None: + raise ValueError("gap recovery cannot exist without a gap start") + assert checkpoint.last_gap_started_transport_revision is not None + assert checkpoint.last_gap_recovered_transport_revision is not None + if checkpoint.last_gap_recovered_transport_revision != ( + checkpoint.last_gap_started_transport_revision + 1 + ): + raise ValueError("one recovered gap must advance transport revision exactly once") + if checkpoint.transport_revision != ( + checkpoint.last_gap_recovered_transport_revision + ): + raise ValueError("current transport revision must match recovered gap") + elif gap_start[0] is not None: + if checkpoint.last_gap_started_transport_revision != checkpoint.transport_revision: + raise ValueError("open gap must bind current durable transport revision") + + activation_status = checkpoint.activation_status_proof + activation_physical = checkpoint.activation_physical_proof + origin_proof = checkpoint.reconciled_start_origin_proof + first_pcl_proof = checkpoint.first_published_pcl_proof + resolved_start_reconciliation_id = ( + checkpoint.prepared_resolved_start_reconciliation_id + ) + resolved_start_attempt_sha256 = ( + checkpoint.prepared_resolved_start_original_attempt_sha256 + ) + resolved_start_project_sha256 = ( + checkpoint.prepared_resolved_start_original_project_id_sha256 + ) + if len( + { + value is None + for value in ( + resolved_start_reconciliation_id, + resolved_start_attempt_sha256, + resolved_start_project_sha256, + ) + } + ) != 1: + raise ValueError( + "prepared resolved START settlement audit fields must be all present or absent" + ) + if checkpoint.state == "prepared": + if checkpoint.transport_revision != 1 or checkpoint.current_binding != ( + checkpoint.prepared_binding + ) or checkpoint.physical_lineage_head_revision is not None: + raise ValueError("prepared checkpoint must remain on initial transport revision") + if any( + value is not None + for value in ( + checkpoint.active_project_id_sha256, + checkpoint.activated_at_utc, + checkpoint.ceased_at_utc, + *gap_start, + *gap_recovery, + activation_status, + activation_physical, + checkpoint.current_active_status_proof, + checkpoint.current_active_physical_proof, + checkpoint.last_rebind_status_proof, + checkpoint.last_rebind_physical_proof, + checkpoint.cessation_status_proof, + checkpoint.cessation_physical_proof, + checkpoint.prepared_resolution_proof, + origin_proof, + first_pcl_proof, + resolved_start_reconciliation_id, + resolved_start_attempt_sha256, + resolved_start_project_sha256, + ) + ): + raise ValueError("prepared checkpoint contains later-state evidence") + if checkpoint.current_evidence_session_id != ( + checkpoint.original_evidence_session_id + ): + raise ValueError("prepared checkpoint must retain original evidence session") + return + + if activation_status is None or activation_physical is None: + if checkpoint.prepared_resolution_proof is None: + raise ValueError("post-prepare checkpoint lacks terminal physical lineage") + else: + _require_status_binding(checkpoint, activation_status) + if origin_proof is None: + if activation_status.evidence_session_id != ( + checkpoint.original_evidence_session_id + ): + raise ValueError("activation proof is outside original evidence session") + _require_activation_physical_proof( + checkpoint, + status_proof=activation_status, + physical_proof=activation_physical, + ) + else: + _require_reconciled_start_active( + checkpoint, + origin_proof=origin_proof, + status_proof=activation_status, + physical_proof=activation_physical, + ) + if first_pcl_proof is None: + raise ValueError("reconciled activation lacks first published PCL") + _require_first_published_pcl( + status_proof=activation_status, + first_pcl_proof=first_pcl_proof, + ) + if ( + checkpoint.activated_at_utc + != ( + first_pcl_proof.published_at_utc + if first_pcl_proof is not None + else activation_status.observed_at_utc + ) + or checkpoint.active_project_id_sha256 + != activation_status.project_id_sha256 + ): + raise ValueError("activation audit does not match activation proof") + current_active_status = checkpoint.current_active_status_proof + current_active_physical = checkpoint.current_active_physical_proof + if (current_active_status is None) != (current_active_physical is None): + raise ValueError("current active status and physical proofs must be paired") + if current_active_status is not None: + assert current_active_physical is not None + _require_status_binding(checkpoint, current_active_status) + _require_matching_observed_proofs( + status_proof=current_active_status, + physical_proof=current_active_physical, + ) + if ( + origin_proof is not None + and ( + current_active_physical == activation_physical + or ( + activation_physical is None + and checkpoint.state == "ceased" + and checkpoint.prepared_resolution_proof is not None + and first_pcl_proof is None + ) + ) + ): + _require_reconciled_start_active( + checkpoint, + origin_proof=origin_proof, + status_proof=current_active_status, + physical_proof=current_active_physical, + ) + elif current_active_physical.reconciliation_kind is None: + _require_activation_physical_proof( + checkpoint, + status_proof=current_active_status, + physical_proof=current_active_physical, + ) + else: + _require_active_rebind_physical_proof_shape( + checkpoint, + status_proof=current_active_status, + physical_proof=current_active_physical, + ) + if ( + current_active_status.session_state != "scanning" + or current_active_status.project_id_sha256 + != checkpoint.active_project_id_sha256 + or ( + checkpoint.state == "active" + and current_active_status.binding != checkpoint.current_binding + ) + or ( + checkpoint.state == "active" + and current_active_status.evidence_session_id + != checkpoint.current_evidence_session_id + ) + or current_active_physical.original_start_payload_sha256 + != checkpoint.start_payload_sha256 + ): + raise ValueError("current active physical head is inconsistent") + if (checkpoint.last_rebind_status_proof is None) != ( + checkpoint.last_rebind_physical_proof is None + ): + raise ValueError("rebind status and physical lineage proofs must be paired") + if checkpoint.last_rebind_status_proof is not None: + rebind = checkpoint.last_rebind_status_proof + physical_rebind = checkpoint.last_rebind_physical_proof + assert physical_rebind is not None + _require_status_binding(checkpoint, rebind) + _require_active_rebind_physical_proof_shape( + checkpoint, + status_proof=rebind, + physical_proof=physical_rebind, + ) + if ( + rebind.session_state != "scanning" + or rebind.project_id_sha256 != checkpoint.active_project_id_sha256 + or rebind.binding != checkpoint.current_binding + or rebind.evidence_session_id + != checkpoint.current_evidence_session_id + or rebind.observed_at_utc != checkpoint.last_gap_recovered_at_utc + ): + raise ValueError("last rebind proof does not match current recovery binding") + + if checkpoint.state == "active": + if ( + checkpoint.ceased_at_utc is not None + or checkpoint.cessation_status_proof is not None + or checkpoint.cessation_physical_proof is not None + or checkpoint.prepared_resolution_proof is not None + or resolved_start_reconciliation_id is not None + or resolved_start_attempt_sha256 is not None + or resolved_start_project_sha256 is not None + or checkpoint.active_project_id_sha256 is None + or current_active_physical is None + or checkpoint.physical_lineage_head_revision + != current_active_physical.ledger_revision + ): + raise ValueError("active checkpoint contains terminal or incomplete evidence") + if gap_recovery[0] is not None and ( + checkpoint.last_rebind_status_proof is None + or checkpoint.last_rebind_physical_proof is None + ) and not (origin_proof is not None and first_pcl_proof is not None): + raise ValueError("active recovered gap lacks its exact rebind proof") + return + + if checkpoint.ceased_at_utc is None or checkpoint.cessation_physical_proof is None: + raise ValueError("ceased checkpoint lacks terminal physical proof") + cessation_physical = checkpoint.cessation_physical_proof + if checkpoint.physical_lineage_head_revision != cessation_physical.ledger_revision: + raise ValueError("ceased physical lineage head does not match cessation proof") + _require_physical_lineage_base(checkpoint, cessation_physical) + if resolved_start_reconciliation_id is not None: + if not ( + checkpoint.prepared_resolution_proof == cessation_physical + and origin_proof is not None + and first_pcl_proof is None + and activation_status is None + and activation_physical is None + and checkpoint.current_active_status_proof is None + and checkpoint.current_active_physical_proof is None + and checkpoint.last_rebind_status_proof is None + and checkpoint.last_rebind_physical_proof is None + and checkpoint.cessation_status_proof is not None + and checkpoint.activated_at_utc is None + and checkpoint.last_gap_started_at_utc is not None + and resolved_start_attempt_sha256 is not None + and resolved_start_project_sha256 is not None + ): + raise ValueError( + "prepared resolved START standby settlement evidence is incomplete" + ) + _require_prepared_resolved_start_standby( + checkpoint, + gap_started_at_utc=checkpoint.last_gap_started_at_utc, + origin_proof=origin_proof, + cessation_status_proof=checkpoint.cessation_status_proof, + cessation_physical_proof=cessation_physical, + reconciliation_id=resolved_start_reconciliation_id, + reconciliation_original_attempt_sha256=( + resolved_start_attempt_sha256 + ), + reconciliation_original_project_id_sha256=( + resolved_start_project_sha256 + ), + ) + if ( + checkpoint.current_evidence_session_id + != checkpoint.cessation_status_proof.evidence_session_id + or checkpoint.active_project_id_sha256 + != origin_proof.original_project_id_sha256 + or receipts[-1].kind + != "cease-prepared-resolved-start-standby" + ): + raise ValueError( + "prepared resolved START standby settlement is inconsistent" + ) + return + if checkpoint.prepared_resolution_proof is not None: + if origin_proof is not None: + if ( + first_pcl_proof is not None + or activation_status is not None + or activation_physical is not None + or checkpoint.current_active_status_proof is None + or checkpoint.current_active_physical_proof is None + or checkpoint.cessation_status_proof is None + ): + raise ValueError("pre-PCL reconciled cessation evidence is incomplete") + _require_reconciled_start_active( + checkpoint, + origin_proof=origin_proof, + status_proof=checkpoint.current_active_status_proof, + physical_proof=checkpoint.current_active_physical_proof, + ) + _require_prepared_reconciled_cessation( + checkpoint, + active_status_proof=checkpoint.current_active_status_proof, + active_physical_proof=checkpoint.current_active_physical_proof, + cessation_status_proof=checkpoint.cessation_status_proof, + cessation_physical_proof=cessation_physical, + ) + if checkpoint.current_evidence_session_id != ( + checkpoint.cessation_status_proof.evidence_session_id + ): + raise ValueError("pre-PCL cessation evidence session is inconsistent") + return + if ( + activation_status is not None + or activation_physical is not None + or checkpoint.cessation_status_proof is not None + and checkpoint.prepared_resolution_proof.resolution == "not-dispatched" + or checkpoint.prepared_resolution_proof != cessation_physical + ): + raise ValueError("prepared cessation evidence is inconsistent") + _require_prepared_cessation( + checkpoint, + status_proof=checkpoint.cessation_status_proof, + physical_proof=cessation_physical, + ) + expected_evidence_session = ( + checkpoint.cessation_status_proof.evidence_session_id + if checkpoint.cessation_status_proof is not None + else checkpoint.original_evidence_session_id + ) + if checkpoint.current_evidence_session_id != expected_evidence_session: + raise ValueError("prepared cessation evidence session is inconsistent") + else: + status = checkpoint.cessation_status_proof + if status is None: + raise ValueError("active cessation requires exact fresh status proof") + _require_active_cessation_shape( + checkpoint, + status_proof=status, + physical_proof=cessation_physical, + ) + if checkpoint.current_evidence_session_id != status.evidence_session_id: + raise ValueError("active cessation evidence session is inconsistent") + prior_active_status = checkpoint.current_active_status_proof + if prior_active_status is None: + raise ValueError("active cessation lacks its prior active status proof") + if status.binding != prior_active_status.binding: + if ( + checkpoint.last_gap_failed_binding != prior_active_status.binding + or checkpoint.last_gap_started_at_utc is None + or checkpoint.last_gap_recovered_at_utc != status.observed_at_utc + ): + raise ValueError( + "gap cessation does not close the exact prior active binding" + ) + _require_gap_cessation_evidence_session( + failed_binding=prior_active_status.binding, + failed_evidence_session_id=( + prior_active_status.evidence_session_id + ), + status_proof=status, + ) + if receipts[-1].kind == "cease-active-reconciled-standby": + gap_started_at_utc = checkpoint.last_gap_started_at_utc + failed_binding = checkpoint.last_gap_failed_binding + if gap_started_at_utc is None or failed_binding is None: + raise ValueError( + "restart standby settlement lacks its atomic gap audit" + ) + _require_active_reconciled_standby_shape( + checkpoint, + failed_binding=failed_binding, + failed_evidence_session_id=( + prior_active_status.evidence_session_id + ), + gap_started_at_utc=gap_started_at_utc, + cessation_status_proof=status, + cessation_physical_proof=cessation_physical, + ) + + +def _exact_mapping( + value: object, + keys: set[str], + *, + label: str, +) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"{label} does not match the secret-free checkpoint schema") + return cast(Mapping[str, object], value) + + +def _exact_project_name(value: str) -> str: + if not isinstance(value, str): + raise TypeError("project_name must be a string") + if not value or len(value) > 96: + raise ValueError("project_name must contain 1..96 exact characters") + normalized = unicodedata.normalize("NFKC", value) + if normalized != value or value.strip() != value: + raise ValueError("project_name must already be exact canonical wire text") + try: + value.encode("utf-8", errors="strict") + except UnicodeEncodeError as exc: + raise ValueError("project_name contains invalid Unicode") from exc + if any(unicodedata.category(character) in {"Cc", "Cs"} for character in value): + raise ValueError("project_name contains control characters") + return value + + +def _validated_requested_streams(value: tuple[str, ...]) -> tuple[str, ...]: + if not isinstance(value, tuple) or not 1 <= len(value) <= _MAX_REQUESTED_STREAMS: + raise ValueError("requested_streams must be a bounded non-empty tuple") + if len(set(value)) != len(value): + raise ValueError("requested_streams must be unique") + for stream_id in value: + _validate_identifier(stream_id, field_name="requested_stream") + return value + + +def _requested_streams_from_value(value: object) -> tuple[str, ...]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError("requested_streams must be a string array") + return _validated_requested_streams(tuple(cast(list[str], value))) + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _identifier_from_value(value: object, *, field_name: str) -> str: + result = _required_string(value, field_name=field_name) + _validate_identifier(result, field_name=field_name) + return result + + +def _optional_identifier_from_value( + value: object, + *, + field_name: str, +) -> str | None: + return None if value is None else _identifier_from_value(value, field_name=field_name) + + +def _validate_sha256(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{field_name} must be a lowercase SHA-256 digest") + + +def _sha256_from_value(value: object, *, field_name: str) -> str: + result = _required_string(value, field_name=field_name) + _validate_sha256(result, field_name=field_name) + return result + + +def _optional_sha256(value: object, *, field_name: str) -> str | None: + return None if value is None else _sha256_from_value(value, field_name=field_name) + + +def _validate_connection_mode(value: str) -> None: + if not isinstance(value, str) or value not in _CONNECTION_MODES: + raise ValueError("unsupported checkpoint connection mode") + + +def _connection_mode_from_value( + value: object, +) -> ActiveAcquisitionRecoveryConnectionMode: + result = _required_string(value, field_name="connection_mode") + _validate_connection_mode(result) + return cast(ActiveAcquisitionRecoveryConnectionMode, result) + + +def _validate_evidence_policy(value: str) -> None: + if not isinstance(value, str) or value not in _EVIDENCE_POLICIES: + raise ValueError("unsupported checkpoint evidence policy") + + +def _evidence_policy_from_value( + value: object, +) -> ActiveAcquisitionRecoveryEvidencePolicy: + result = _required_string(value, field_name="evidence_policy") + _validate_evidence_policy(result) + return cast(ActiveAcquisitionRecoveryEvidencePolicy, result) + + +def _validate_mount_type(value: str) -> None: + if not isinstance(value, str) or value not in _MOUNT_TYPES: + raise ValueError("unsupported checkpoint mount_type") + + +def _mount_type_from_value(value: object) -> ActiveAcquisitionRecoveryMountType: + result = _required_string(value, field_name="mount_type") + _validate_mount_type(result) + return cast(ActiveAcquisitionRecoveryMountType, result) + + +def _validate_gnss_mode(value: str) -> None: + if not isinstance(value, str) or value not in _GNSS_MODES: + raise ValueError("unsupported checkpoint gnss_mode") + + +def _gnss_mode_from_value(value: object) -> ActiveAcquisitionRecoveryGnssMode: + result = _required_string(value, field_name="gnss_mode") + _validate_gnss_mode(result) + return cast(ActiveAcquisitionRecoveryGnssMode, result) + + +def _validate_proof_source(value: str) -> None: + if not isinstance(value, str) or value not in _PROOF_SOURCES: + raise ValueError("unsupported checkpoint proof source") + + +def _proof_source_from_value(value: object) -> ActiveAcquisitionRecoveryProofSource: + result = _required_string(value, field_name="source") + _validate_proof_source(result) + return cast(ActiveAcquisitionRecoveryProofSource, result) + + +def _validate_proof_state(value: str, code: int) -> None: + expected = _PROOF_STATE_CODES.get(value) + if expected is None or code != expected: + raise ValueError("checkpoint proof state and code do not match") + + +def _proof_state_from_value(value: object) -> ActiveAcquisitionRecoveryProofState: + result = _required_string(value, field_name="session_state") + if result not in _PROOF_STATE_CODES: + raise ValueError("unsupported checkpoint proof state") + return cast(ActiveAcquisitionRecoveryProofState, result) + + +def _optional_proof_state( + value: object, +) -> ActiveAcquisitionRecoveryProofState | None: + return None if value is None else _proof_state_from_value(value) + + +def _validate_physical_action(value: str) -> None: + if not isinstance(value, str) or value not in _PHYSICAL_ACTIONS: + raise ValueError("unsupported physical lineage action") + + +def _physical_action_from_value( + value: object, +) -> ActiveAcquisitionRecoveryPhysicalAction: + result = _required_string(value, field_name="action") + _validate_physical_action(result) + return cast(ActiveAcquisitionRecoveryPhysicalAction, result) + + +def _validate_physical_resolution(value: str) -> None: + if not isinstance(value, str) or value not in _PHYSICAL_RESOLUTIONS: + raise ValueError("unsupported physical lineage resolution") + + +def _physical_resolution_from_value( + value: object, +) -> ActiveAcquisitionRecoveryPhysicalResolution: + result = _required_string(value, field_name="resolution") + _validate_physical_resolution(result) + return cast(ActiveAcquisitionRecoveryPhysicalResolution, result) + + +def _optional_physical_resolution_from_value( + value: object, +) -> ActiveAcquisitionRecoveryPhysicalResolution | None: + return None if value is None else _physical_resolution_from_value(value) + + +def _validate_physical_reconciliation_kind(value: str) -> None: + if not isinstance(value, str) or value not in _PHYSICAL_RECONCILIATION_KINDS: + raise ValueError("unsupported physical lineage reconciliation kind") + + +def _optional_physical_reconciliation_kind_from_value( + value: object, +) -> ActiveAcquisitionRecoveryPhysicalReconciliationKind | None: + if value is None: + return None + result = _required_string(value, field_name="reconciliation_kind") + _validate_physical_reconciliation_kind(result) + return cast(ActiveAcquisitionRecoveryPhysicalReconciliationKind, result) + + +def _validate_stop_fence(value: str) -> None: + if not isinstance(value, str) or value not in _STOP_FENCES: + raise ValueError("unsupported physical STOP fence") + + +def _stop_fence_from_value(value: object) -> ActiveAcquisitionRecoveryStopFence: + result = _required_string(value, field_name="stop_fence") + _validate_stop_fence(result) + return cast(ActiveAcquisitionRecoveryStopFence, result) + + +def _validate_transition_kind(value: str) -> None: + if not isinstance(value, str) or value not in _TRANSITION_KINDS: + raise ValueError("unsupported checkpoint transition kind") + + +def _transition_kind_from_value( + value: object, +) -> ActiveAcquisitionRecoveryTransitionKind: + result = _required_string(value, field_name="kind") + _validate_transition_kind(result) + return cast(ActiveAcquisitionRecoveryTransitionKind, result) + + +def _validate_ipv4(value: str, *, field_name: str) -> None: + try: + parsed = ipaddress.ip_address(value) + except ValueError as exc: + raise ValueError(f"{field_name} must be an IPv4 address") from exc + if not isinstance(parsed, ipaddress.IPv4Address) or str(parsed) != value: + raise ValueError(f"{field_name} must be canonical IPv4") + + +def _validate_port(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= 65535: + raise ValueError(f"{field_name} must be within 1..65535") + + +def _validate_positive_int(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{field_name} must be a positive integer") + + +def _positive_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{field_name} must be a positive integer") + return value + + +def _validate_optional_positive_int(value: int | None, *, field_name: str) -> None: + if value is not None: + _validate_positive_int(value, field_name=field_name) + + +def _optional_positive_int(value: object, *, field_name: str) -> int | None: + return None if value is None else _positive_int(value, field_name=field_name) + + +def _optional_duration_seconds(value: object, *, field_name: str) -> float | None: + """Accept JSON integer/number duration while rejecting bool and non-finite values.""" + + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{field_name} must be a finite number or null") + try: + duration = float(value) + except OverflowError as exc: + raise ValueError(f"{field_name} must be a finite number or null") from exc + if not math.isfinite(duration) or duration < 1.0: + raise ValueError(f"{field_name} must be finite and at least 1.0") + return duration + + +def _nonnegative_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a nonnegative integer") + return value + + +def _validate_nonnegative_int_value(value: int, *, field_name: str) -> None: + _nonnegative_int(value, field_name=field_name) + + +def _validate_positive_revision(value: int, *, field_name: str) -> None: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or value > ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ): + raise ValueError(f"{field_name} must be a bounded positive revision") + + +def _positive_revision(value: object, *, field_name: str) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or value > ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ): + raise ValueError(f"{field_name} must be a bounded positive revision") + return value + + +def _optional_positive_revision(value: object, *, field_name: str) -> int | None: + return None if value is None else _positive_revision(value, field_name=field_name) + + +def _validate_nonnegative_revision(value: int, *, field_name: str) -> None: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value > ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ): + raise ValueError(f"{field_name} must be a bounded nonnegative revision") + + +def _required_bool(value: object, *, field_name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{field_name} must be bool") + return value + + +def _required_true(value: object, *, field_name: str) -> bool: + result = _required_bool(value, field_name=field_name) + if result is not True: + raise ValueError(f"{field_name} must be true") + return result + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _validated_timestamp(value: object, *, field_name: str) -> datetime: + raw = _required_string(value, field_name=field_name) + if len(raw) > 32 or not raw.endswith("Z"): + raise ValueError(f"{field_name} must be UTC") + try: + parsed = datetime.fromisoformat(raw.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise ValueError(f"{field_name} is invalid") from exc + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): + raise ValueError(f"{field_name} must be UTC") + return parsed.astimezone(UTC) + + +def _canonical_timestamp_value(value: object, *, field_name: str) -> str: + raw = _required_string(value, field_name=field_name) + parsed = _validated_timestamp(raw, field_name=field_name) + canonical = _timestamp(parsed) + if raw != canonical: + raise ValueError(f"{field_name} must use canonical millisecond UTC") + return canonical + + +def _optional_timestamp(value: object, *, field_name: str) -> str | None: + return None if value is None else _canonical_timestamp_value(value, field_name=field_name) + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("checkpoint clock must be timezone-aware") + return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _nondecreasing_timestamp(value: datetime, *, floor: str | None) -> str: + candidate = _validated_timestamp(_timestamp(value), field_name="checkpoint clock") + if floor is None: + return _timestamp(candidate) + floor_value = _validated_timestamp(floor, field_name="checkpoint audit floor") + return _timestamp(max(candidate, floor_value)) diff --git a/src/k1link/device_plugins/xgrids_k1/application_control_process_lease.py b/src/k1link/device_plugins/xgrids_k1/application_control_process_lease.py new file mode 100644 index 0000000..e246dce --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/application_control_process_lease.py @@ -0,0 +1,265 @@ +from __future__ import annotations + +import fcntl +import os +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from k1link.sessions.store import resolve_missioncore_data_dir + +APPLICATION_CONTROL_LOCK_FILENAME = ".application-control.lock" + + +class ApplicationControlProcessLeaseError(RuntimeError): + """The process-wide K1 control fence cannot be trusted.""" + + reason_code = "application-control-process-lease-error" + + +class ApplicationControlProcessLeaseUnavailable(ApplicationControlProcessLeaseError): + """Another Mission Core process owns the canonical K1 control dialogue.""" + + reason_code = "application-control-process-lease-unavailable" + + +ApplicationControlProcessLeaseReleaseState = Literal["owned", "released", "ambiguous"] +ApplicationControlProcessLeaseReleaseDisposition = Literal[ + "released", + "already-released", +] + + +class ApplicationControlProcessLeaseReleaseAmbiguous(ApplicationControlProcessLeaseError): + """Neither strict unlock nor close proved the OS fence disposition.""" + + reason_code = "application-control-process-lease-release-ambiguous" + + +@dataclass(frozen=True, slots=True) +class ApplicationControlProcessLeaseReleaseOutcome: + """Terminal release result, including non-retryable syscall diagnostics.""" + + disposition: ApplicationControlProcessLeaseReleaseDisposition + unlock_error_code: str | None = None + close_error_code: str | None = None + + +@dataclass(slots=True) +class ApplicationControlProcessLease: + """Non-persistent ownership of the one canonical K1 control dialogue. + + The lock file is deliberately stable while ownership lives only in the OS + lock attached to ``_descriptor``. A process crash normally releases + control admission without manufacturing any K1 command or durable recovery + claim. A target-owning child may deliberately inherit a duplicate of that + exact description; after a parent crash the kernel then preserves the fence + until the child itself exits. + """ + + path: Path + _descriptor: int + _identity: tuple[int, int] + _released: bool = False + _release_ambiguous: bool = False + + @property + def release_state(self) -> ApplicationControlProcessLeaseReleaseState: + if self._release_ambiguous: + return "ambiguous" + return "released" if self._released else "owned" + + @classmethod + def acquire(cls, repository_root: Path) -> ApplicationControlProcessLease: + data_dir = resolve_missioncore_data_dir(repository_root) + lock_dir = data_dir / "xgrids-k1" + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(lock_dir, parents=False) + path = lock_dir / APPLICATION_CONTROL_LOCK_FILENAME + + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags, 0o600) + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock cannot be opened safely" + ) from exc + + locked = False + try: + opened = os.fstat(descriptor) + _validate_private_lock_file(opened) + try: + current = path.lstat() + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock identity is unavailable" + ) from exc + _validate_private_lock_file(current) + identity = (opened.st_dev, opened.st_ino) + if identity != (current.st_dev, current.st_ino): + raise ApplicationControlProcessLeaseError( + "application control process lock changed while opening" + ) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + locked = True + except BlockingIOError as exc: + raise ApplicationControlProcessLeaseUnavailable( + "another Mission Core process owns K1 application control" + ) from exc + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock cannot be acquired safely" + ) from exc + try: + locked_path = path.lstat() + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock disappeared after acquisition" + ) from exc + _validate_private_lock_file(locked_path) + if identity != (locked_path.st_dev, locked_path.st_ino): + raise ApplicationControlProcessLeaseError( + "application control process lock changed during acquisition" + ) + return cls(path=path, _descriptor=descriptor, _identity=identity) + except BaseException: + if locked: + _unlock_descriptor(descriptor) + os.close(descriptor) + raise + + def release(self) -> ApplicationControlProcessLeaseReleaseOutcome: + if self._released: + return ApplicationControlProcessLeaseReleaseOutcome( + disposition="already-released" + ) + if self._release_ambiguous: + raise ApplicationControlProcessLeaseReleaseAmbiguous( + "application control process lock release remains ambiguous" + ) + unlock_error: OSError | None = None + close_error: OSError | None = None + try: + fcntl.flock(self._descriptor, fcntl.LOCK_UN) + except OSError as exc: + unlock_error = exc + try: + os.close(self._descriptor) + except OSError as exc: + close_error = exc + if unlock_error is not None and close_error is not None: + # There is no portable ownership answer after two failing syscalls. + # Never retry either syscall on this descriptor: quarantine the + # process-local object and require process restart. + self._release_ambiguous = True + error = ApplicationControlProcessLeaseReleaseAmbiguous( + "application control process lock release outcome is ambiguous" + ) + error.add_note( + "explicit unlock failed: " + f"{type(unlock_error).__name__}: {unlock_error}" + ) + error.add_note( + "descriptor close failed: " + f"{type(close_error).__name__}: {close_error}" + ) + raise error from close_error + # Either strict LOCK_UN or close proved that the flock can no longer + # be reused through this object. A diagnostic on the other syscall is + # terminal, not a retry instruction. + self._released = True + return ApplicationControlProcessLeaseReleaseOutcome( + disposition="released", + unlock_error_code=_os_error_code(unlock_error), + close_error_code=_os_error_code(close_error), + ) + + def duplicate_descriptor_for_child(self) -> int: + """Duplicate the live flock description for one target-owning child. + + ``flock`` ownership follows the open file description across ``dup`` + and ``exec``. A camera adapter that inherits this duplicate therefore + keeps the K1 lifecycle fence after an abrupt parent-process exit. The + caller must pass the returned descriptor through ``pass_fds`` and + close its parent-side duplicate immediately after spawning. + + Normal shutdown must still terminate and reap the child before calling + :meth:`release`: an explicit ``LOCK_UN`` on any duplicate unlocks the + shared description for every process. + """ + + if self._released or self._release_ambiguous: + raise ApplicationControlProcessLeaseError( + "released or quarantined application control lease cannot be inherited" + ) + try: + descriptor = os.dup(self._descriptor) + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock cannot be duplicated safely" + ) from exc + try: + metadata = os.fstat(descriptor) + _validate_private_lock_file(metadata) + if (metadata.st_dev, metadata.st_ino) != self._identity: + raise ApplicationControlProcessLeaseError( + "duplicated application control process lock changed identity" + ) + return descriptor + except BaseException: + os.close(descriptor) + raise + + def __enter__(self) -> ApplicationControlProcessLease: + return self + + def __exit__(self, *_: object) -> None: + self.release() + + +def _ensure_private_directory(path: Path, *, parents: bool) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + metadata = path.lstat() + except OSError as exc: + raise ApplicationControlProcessLeaseError( + "application control process lock directory is unavailable" + ) from exc + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + raise ApplicationControlProcessLeaseError( + "application control process lock directory is not private" + ) + + +def _validate_private_lock_file(metadata: os.stat_result) -> None: + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + raise ApplicationControlProcessLeaseError( + "application control process lock is not a private regular file" + ) + + +def _os_error_code(error: OSError | None) -> str | None: + if error is None: + return None + return f"{type(error).__name__}:{error.errno if error.errno is not None else 'unknown'}" + + +def _unlock_descriptor(descriptor: int) -> None: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError: + return diff --git a/src/k1link/device_plugins/xgrids_k1/ble/ap_activation.py b/src/k1link/device_plugins/xgrids_k1/ble/ap_activation.py index cd66aff..57d6cb4 100644 --- a/src/k1link/device_plugins/xgrids_k1/ble/ap_activation.py +++ b/src/k1link/device_plugins/xgrids_k1/ble/ap_activation.py @@ -1,7 +1,7 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from contextlib import AsyncExitStack, asynccontextmanager from importlib.metadata import version from time import monotonic @@ -11,7 +11,19 @@ from bleak import BleakClient, BleakScanner from bleak.exc import BleakDeviceNotFoundError, BleakError from k1link.artifacts import utc_now_iso -from k1link.device_plugins.xgrids_k1.ble.scanner import discovered_device_selection +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationProgress, + run_ble_operation_session, +) +from k1link.device_plugins.xgrids_k1.ble.scanner import ( + CapturedDiscoveredDevice, + captured_device_handle, + connected_device_capture, + demote_connected_device_handle_after_gatt_failure, + discovered_device_selection, + mark_captured_device_gatt_validated, + retrieve_connected_device_capture, +) from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( AP_FALLBACK_IPV4, SERVICE_UUID, @@ -23,6 +35,7 @@ from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( WifiStatus, WriteMode, _annotate_ble_operation_error, + _optional_int_attribute, parse_wifi_status, ) @@ -30,6 +43,7 @@ PROFILE_ID = "xgrids-k1-fw3-quick-connect-ap-v1" FRAME_LENGTH = 100 COMMAND_OFFSET = 99 ENABLE_AP_COMMAND = 1 +BLE_AP_ENABLE_HARD_TIMEOUT_GRACE_SECONDS = 25.0 ApActivationOutcome = Literal[ "already_active", @@ -56,7 +70,7 @@ class ApActivationResult(TypedDict): write_performed: bool write_mode: ResolvedWriteMode | None write_without_response_advertised: bool - max_write_without_response_size: int + max_write_without_response_size: int | None frame_length: int baseline_status: WifiStatus observations: list[StatusObservation] @@ -100,11 +114,16 @@ def _outcome( @asynccontextmanager -async def device_ap_activation_session( +async def _device_ap_activation_session_impl( device_macos_uuid: str, timeout_seconds: float = 15.0, poll_interval_seconds: float = 0.5, write_mode: WriteMode = "auto", + *, + captured_device: CapturedDiscoveredDevice | None, + recovery_device_session_id: str | None, + on_write_dispatch: Callable[[WifiStatus, ResolvedWriteMode], None] | None, + progress: BleOperationProgress, ) -> AsyncIterator[ApActivationResult]: """Keep BLE connected around one reviewed Quick Connect AP-enable write. @@ -130,6 +149,12 @@ async def device_ap_activation_session( operation_stage: BleOperationStage = "resolution" device_write_attempted = False device_write_confirmed = False + resolved_write_mode_for_error: ResolvedWriteMode | None = None + write_characteristic_properties: tuple[str, ...] | None = None + max_without_response: int | None = None + active_captured_device: CapturedDiscoveredDevice | None = None + gatt_baseline_validated = False + progress.operation_stage = operation_stage try: try: @@ -137,9 +162,44 @@ async def device_ap_activation_session( # Keep the explicit scan and AP activation in one CoreBluetooth # lifecycle. Re-looking up the UUID here lost a physically present # K1 during acceptance, while the retained BLEDevice connected. - selection = discovered_device_selection(device_macos_uuid) - device = selection.device - if device is None and not selection.from_fresh_scan: + if captured_device is not None: + device = ( + captured_device_handle(captured_device) + if captured_device.macos_uuid.casefold() + == device_macos_uuid.casefold() + else None + ) + if device is not None: + active_captured_device = captured_device + selection_from_fresh_scan = False + elif recovery_device_session_id is not None: + # Retrieval is part of the same serialized ap-enable + # lease as connect, baseline read and the one command. + active_captured_device = connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + if active_captured_device is None: + active_captured_device = await retrieve_connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + device = ( + captured_device_handle(active_captured_device) + if active_captured_device is not None + else None + ) + selection_from_fresh_scan = False + else: + selection = discovered_device_selection(device_macos_uuid) + device = selection.device + selection_from_fresh_scan = selection.from_fresh_scan + if ( + device is None + and captured_device is None + and recovery_device_session_id is None + and not selection_from_fresh_scan + ): # Preserve a bounded fallback for non-UI callers that did not # establish a fresh explicit scan lease. device = await BleakScanner.find_device_by_address( @@ -157,21 +217,34 @@ async def device_ap_activation_session( operation_stage=operation_stage, device_write_attempted=device_write_attempted, device_write_confirmed=device_write_confirmed, + resolved_write_mode=resolved_write_mode_for_error, + write_characteristic_properties=write_characteristic_properties, + max_write_without_response_size=max_without_response, + frame_length=len(frame), ) raise async with AsyncExitStack() as client_stack: operation_stage = "connect" + progress.operation_stage = operation_stage try: client = await client_stack.enter_async_context( BleakClient(device, timeout=timeout_seconds, pair=False) ) except Exception as exc: + if active_captured_device is not None: + demote_connected_device_handle_after_gatt_failure( + active_captured_device + ) _annotate_ble_operation_error( exc, operation_stage=operation_stage, device_write_attempted=device_write_attempted, device_write_confirmed=device_write_confirmed, + resolved_write_mode=resolved_write_mode_for_error, + write_characteristic_properties=write_characteristic_properties, + max_write_without_response_size=max_without_response, + frame_length=len(frame), ) raise @@ -179,6 +252,7 @@ async def device_ap_activation_session( async with asyncio.timeout(timeout_seconds + 10.0): device_name = client.name operation_stage = "gatt-contract" + progress.operation_stage = operation_stage service = client.services.get_service(SERVICE_UUID) write_characteristic = client.services.get_characteristic( WRITE_CHARACTERISTIC_UUID @@ -210,7 +284,11 @@ async def device_ap_activation_session( raise ValueError("Reviewed K1 status characteristic is not readable") properties = set(write_characteristic.properties) - max_without_response = write_characteristic.max_write_without_response_size + write_characteristic_properties = tuple(sorted(properties)) + max_without_response = _optional_int_attribute( + write_characteristic, + "max_write_without_response_size", + ) resolved_write_mode: ResolvedWriteMode if write_mode == "auto": if "write-without-response" in properties: @@ -226,17 +304,32 @@ async def device_ap_activation_session( ) resolved_write_mode = "with_response" else: + resolved_write_mode = "without_response" + + resolved_write_mode_for_error = resolved_write_mode + if resolved_write_mode == "without_response": + if max_without_response is None: + raise ValueError( + "Negotiated write-without-response size is unavailable" + ) if len(frame) > max_without_response: raise ValueError( "AP activation frame exceeds the negotiated " "write-without-response size" ) - resolved_write_mode = "without_response" operation_stage = "baseline-read" + progress.operation_stage = operation_stage baseline = parse_wifi_status( bytes(await client.read_gatt_char(status_characteristic)) ) + if active_captured_device is not None and not ( + mark_captured_device_gatt_validated(active_captured_device) + ): + raise RuntimeError( + "Exact BLE recovery handle changed before AP validation" + ) + gatt_baseline_validated = True # WIFI_AP is a control-mode status, not proof that the radio is # still beaconing. A physical run found the exact SSID shortly # after AP-enable, then found no beacon while 7f02 continued to @@ -246,17 +339,26 @@ async def device_ap_activation_session( # a stale-ready status. There is still no automatic retry. operation_stage = "gatt-write" + # Persist the write barrier before handing the frame to + # CoreBluetooth. The callback is deliberately synchronous + # and secret-free; failure here prevents the device write. + if on_write_dispatch is not None: + on_write_dispatch(baseline, resolved_write_mode) device_write_attempted = True + progress.operation_stage = operation_stage + progress.device_write_attempted = True await client.write_gatt_char( write_characteristic, frame, response=resolved_write_mode == "with_response", ) device_write_confirmed = resolved_write_mode == "with_response" + progress.device_write_confirmed = device_write_confirmed write_completed = monotonic() deadline = write_completed + timeout_seconds operation_stage = "status-poll" + progress.operation_stage = operation_stage while monotonic() < deadline: try: status = parse_wifi_status( @@ -306,11 +408,19 @@ async def device_ap_activation_session( "outcome": _outcome(baseline, observations, disconnected), } except Exception as exc: + if active_captured_device is not None and not gatt_baseline_validated: + demote_connected_device_handle_after_gatt_failure( + active_captured_device + ) _annotate_ble_operation_error( exc, operation_stage=operation_stage, device_write_attempted=device_write_attempted, device_write_confirmed=device_write_confirmed, + resolved_write_mode=resolved_write_mode_for_error, + write_characteristic_properties=write_characteristic_properties, + max_write_without_response_size=max_without_response, + frame_length=len(frame), ) raise @@ -321,14 +431,62 @@ async def device_ap_activation_session( # annotated as BLE failures when they are thrown back through yield. yield result finally: + # The runtime arbiter enforces a hard deadline by cancellation. Even + # when that bypasses the normal exception annotator, a captured + # recovery object that never passed baseline GATT validation must be + # discarded. + if active_captured_device is not None and not gatt_baseline_validated: + demote_connected_device_handle_after_gatt_failure(active_captured_device) frame[:] = b"\x00" * len(frame) +@asynccontextmanager +async def device_ap_activation_session( + device_macos_uuid: str, + timeout_seconds: float = 15.0, + poll_interval_seconds: float = 0.5, + write_mode: WriteMode = "auto", + *, + captured_device: CapturedDiscoveredDevice | None = None, + recovery_device_session_id: str | None = None, + on_write_dispatch: Callable[[WifiStatus, ResolvedWriteMode], None] | None = None, +) -> AsyncIterator[ApActivationResult]: + """Own one process BLE lease through AP-ready and host Wi-Fi handoff.""" + + if captured_device is not None and recovery_device_session_id is not None: + raise ValueError( + "captured_device and recovery_device_session_id are mutually exclusive" + ) + if recovery_device_session_id == "": + raise ValueError("recovery_device_session_id must not be empty") + progress = BleOperationProgress(operation_stage="resolution") + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=(timeout_seconds + BLE_AP_ENABLE_HARD_TIMEOUT_GRACE_SECONDS), + operation=lambda operation_progress: _device_ap_activation_session_impl( + device_macos_uuid, + timeout_seconds=timeout_seconds, + poll_interval_seconds=poll_interval_seconds, + write_mode=write_mode, + captured_device=captured_device, + recovery_device_session_id=recovery_device_session_id, + on_write_dispatch=on_write_dispatch, + progress=operation_progress, + ), + progress=progress, + ) as result: + yield result + + async def activate_device_ap_once( device_macos_uuid: str, timeout_seconds: float = 15.0, poll_interval_seconds: float = 0.5, write_mode: WriteMode = "auto", + *, + captured_device: CapturedDiscoveredDevice | None = None, + recovery_device_session_id: str | None = None, + on_write_dispatch: Callable[[WifiStatus, ResolvedWriteMode], None] | None = None, ) -> ApActivationResult: """Run one AP activation and release BLE immediately after its result. @@ -341,5 +499,8 @@ async def activate_device_ap_once( timeout_seconds=timeout_seconds, poll_interval_seconds=poll_interval_seconds, write_mode=write_mode, + captured_device=captured_device, + recovery_device_session_id=recovery_device_session_id, + on_write_dispatch=on_write_dispatch, ) as result: return result diff --git a/src/k1link/device_plugins/xgrids_k1/ble/gatt.py b/src/k1link/device_plugins/xgrids_k1/ble/gatt.py index 3c9a58a..6dd56a9 100644 --- a/src/k1link/device_plugins/xgrids_k1/ble/gatt.py +++ b/src/k1link/device_plugins/xgrids_k1/ble/gatt.py @@ -8,6 +8,10 @@ from bleak import BleakClient, BleakScanner from bleak.exc import BleakDeviceNotFoundError from k1link.artifacts import utc_now_iso +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationProgress, + run_ble_operation, +) class DescriptorRecord(TypedDict): @@ -48,57 +52,68 @@ async def dump_metadata(device_macos_uuid: str, timeout_seconds: float) -> GattD if timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") - started_at = utc_now_iso() - async with asyncio.timeout(timeout_seconds): - device = await BleakScanner.find_device_by_address( - device_macos_uuid, - timeout=min(20.0, timeout_seconds), - ) - if device is None: - raise BleakDeviceNotFoundError( + async def perform(progress: BleOperationProgress) -> GattDumpResult: + started_at = utc_now_iso() + async with asyncio.timeout(timeout_seconds): + progress.operation_stage = "gatt-metadata-discovery" + device = await BleakScanner.find_device_by_address( device_macos_uuid, - "Device was not rediscovered; keep the K1 powered and nearby.", + timeout=min(20.0, timeout_seconds), ) + if device is None: + raise BleakDeviceNotFoundError( + device_macos_uuid, + "Device was not rediscovered; keep the K1 powered and nearby.", + ) - async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: - services: list[ServiceRecord] = [] - for service in client.services: - characteristics: list[CharacteristicRecord] = [] - for characteristic in service.characteristics: - descriptors: list[DescriptorRecord] = [] - for descriptor in characteristic.descriptors: - descriptors.append( + progress.operation_stage = "gatt-metadata-connect" + async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + progress.operation_stage = "gatt-metadata-enumeration" + services: list[ServiceRecord] = [] + for service in client.services: + characteristics: list[CharacteristicRecord] = [] + for characteristic in service.characteristics: + descriptors: list[DescriptorRecord] = [] + for descriptor in characteristic.descriptors: + descriptors.append( + { + "uuid": descriptor.uuid, + "handle": descriptor.handle, + "description": descriptor.description, + } + ) + characteristics.append( { - "uuid": descriptor.uuid, - "handle": descriptor.handle, - "description": descriptor.description, + "uuid": characteristic.uuid, + "handle": characteristic.handle, + "description": characteristic.description, + "properties": sorted(characteristic.properties), + "descriptors": descriptors, } ) - characteristics.append( + services.append( { - "uuid": characteristic.uuid, - "handle": characteristic.handle, - "description": characteristic.description, - "properties": sorted(characteristic.properties), - "descriptors": descriptors, + "uuid": service.uuid, + "handle": service.handle, + "description": service.description, + "characteristics": characteristics, } ) - services.append( - { - "uuid": service.uuid, - "handle": service.handle, - "description": service.description, - "characteristics": characteristics, - } - ) - return { - "schema_version": 1, - "started_at_utc": started_at, - "completed_at_utc": utc_now_iso(), - "adapter": "CoreBluetooth", - "bleak_version": version("bleak"), - "device_macos_uuid": device_macos_uuid, - "device_name": client.name, - "metadata_only": True, - "services": services, - } + progress.operation_stage = "gatt-metadata-complete" + return { + "schema_version": 1, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "adapter": "CoreBluetooth", + "bleak_version": version("bleak"), + "device_macos_uuid": device_macos_uuid, + "device_name": client.name, + "metadata_only": True, + "services": services, + } + + return await run_ble_operation( + "status-read", + hard_timeout_seconds=timeout_seconds + 5.0, + operation=perform, + ) diff --git a/src/k1link/device_plugins/xgrids_k1/ble/reader.py b/src/k1link/device_plugins/xgrids_k1/ble/reader.py index e5a229a..0e77ea6 100644 --- a/src/k1link/device_plugins/xgrids_k1/ble/reader.py +++ b/src/k1link/device_plugins/xgrids_k1/ble/reader.py @@ -8,6 +8,10 @@ from bleak import BleakClient, BleakScanner from bleak.exc import BleakDeviceNotFoundError from k1link.artifacts import utc_now_iso +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationProgress, + run_ble_operation, +) class CharacteristicReadResult(TypedDict): @@ -33,36 +37,47 @@ async def read_characteristic_once( if timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") - started_at = utc_now_iso() - async with asyncio.timeout(timeout_seconds): - device = await BleakScanner.find_device_by_address( - device_macos_uuid, - timeout=min(20.0, timeout_seconds), - ) - if device is None: - raise BleakDeviceNotFoundError( + async def perform(progress: BleOperationProgress) -> CharacteristicReadResult: + started_at = utc_now_iso() + async with asyncio.timeout(timeout_seconds): + progress.operation_stage = "gatt-characteristic-discovery" + device = await BleakScanner.find_device_by_address( device_macos_uuid, - "Device was not rediscovered; keep the K1 powered and nearby.", + timeout=min(20.0, timeout_seconds), ) + if device is None: + raise BleakDeviceNotFoundError( + device_macos_uuid, + "Device was not rediscovered; keep the K1 powered and nearby.", + ) - async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: - characteristic = client.services.get_characteristic(characteristic_uuid) - if characteristic is None: - raise ValueError(f"Characteristic not found: {characteristic_uuid}") - if "read" not in characteristic.properties: - raise ValueError(f"Characteristic is not readable: {characteristic_uuid}") - value = bytes(await client.read_gatt_char(characteristic)) + progress.operation_stage = "gatt-characteristic-connect" + async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + characteristic = client.services.get_characteristic(characteristic_uuid) + if characteristic is None: + raise ValueError(f"Characteristic not found: {characteristic_uuid}") + if "read" not in characteristic.properties: + raise ValueError(f"Characteristic is not readable: {characteristic_uuid}") + progress.operation_stage = "gatt-characteristic-read" + value = bytes(await client.read_gatt_char(characteristic)) - return { - "schema_version": 1, - "started_at_utc": started_at, - "completed_at_utc": utc_now_iso(), - "adapter": "CoreBluetooth", - "bleak_version": version("bleak"), - "device_macos_uuid": device_macos_uuid, - "device_name": client.name, - "characteristic_uuid": characteristic.uuid, - "operation": "single_gatt_read_no_pair_no_write", - "value_length": len(value), - "value_hex": value.hex(), - } + progress.operation_stage = "gatt-characteristic-complete" + return { + "schema_version": 1, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "adapter": "CoreBluetooth", + "bleak_version": version("bleak"), + "device_macos_uuid": device_macos_uuid, + "device_name": client.name, + "characteristic_uuid": characteristic.uuid, + "operation": "single_gatt_read_no_pair_no_write", + "value_length": len(value), + "value_hex": value.hex(), + } + + return await run_ble_operation( + "status-read", + hard_timeout_seconds=timeout_seconds + 5.0, + operation=perform, + ) diff --git a/src/k1link/device_plugins/xgrids_k1/ble/runtime_arbiter.py b/src/k1link/device_plugins/xgrids_k1/ble/runtime_arbiter.py new file mode 100644 index 0000000..aefa624 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/ble/runtime_arbiter.py @@ -0,0 +1,745 @@ +from __future__ import annotations + +import asyncio +import threading +from collections.abc import AsyncIterator, Callable, Coroutine, Iterator +from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, TypedDict + +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + APPLICATION_CONTROL_LOCK_FILENAME, + ApplicationControlProcessLease, +) +from k1link.sessions.store import resolve_missioncore_data_dir + +BleOperationKind = Literal["scan", "status-read", "wifi-provision", "ap-enable"] +BleRuntimeIdleCallbackDisposition = Literal["released", "deferred", "poisoned"] + + +class BleRuntimeBusy(RuntimeError): + """Another process-owned CoreBluetooth operation still owns the adapter.""" + + def __init__( + self, + *, + active_operation_kind: BleOperationKind, + cleanup_pending: bool, + ) -> None: + message = ( + "предыдущая BLE-операция ещё завершает безопасную очистку" + if cleanup_pending + else "другая BLE-операция уже выполняется в этом процессе" + ) + super().__init__(message) + self.reason_code = "ble-runtime-cleanup-pending" if cleanup_pending else "ble-runtime-busy" + self.active_operation_kind = active_operation_kind + self.cleanup_pending = cleanup_pending + + +class BleRuntimeOwnerLoopConflict(RuntimeError): + """The process BLE runtime is still owned by another live event loop.""" + + def __init__(self) -> None: + super().__init__("BLE runtime уже привязан к другому активному event loop") + self.reason_code = "ble-runtime-owner-loop-conflict" + + +class BleRuntimePoisoned(RuntimeError): + """A closed owner loop abandoned an operation whose cleanup is unproved.""" + + def __init__(self) -> None: + super().__init__( + "BLE runtime требует перезапуска: предыдущая очистка CoreBluetooth не подтверждена" + ) + self.reason_code = "ble-runtime-restart-required" + + +class BleRuntimeProcessLeaseNotConfigured(RuntimeError): + """Low-level BLE was entered before its canonical OS lock was configured.""" + + def __init__(self) -> None: + super().__init__("BLE runtime process lease repository root is not configured") + self.reason_code = "ble-runtime-process-lease-not-configured" + + +class BleRuntimeProcessLeaseConfigurationConflict(RuntimeError): + """The canonical OS lock target changed while BLE ownership was live.""" + + def __init__(self) -> None: + super().__init__("BLE runtime process lease configuration cannot change while active") + self.reason_code = "ble-runtime-process-lease-configuration-conflict" + + +class BleRuntimeProcessLeaseBorrowInvalid(RuntimeError): + """An explicit higher-level OS lease borrow is stale or mismatched.""" + + def __init__(self) -> None: + super().__init__("BLE runtime process lease borrow is not active for this lock target") + self.reason_code = "ble-runtime-process-lease-borrow-invalid" + + +@dataclass(slots=True) +class BleOperationProgress: + """Non-secret facts copied onto a hard-deadline exception.""" + + operation_stage: str = "pending" + device_write_attempted: bool = False + device_write_confirmed: bool = False + owner_epoch: int = 0 + + +class BleOperationHardTimeout(TimeoutError): + """A hard caller deadline elapsed while cleanup continues in the background.""" + + def __init__( + self, + operation_kind: BleOperationKind, + progress: BleOperationProgress, + ) -> None: + super().__init__("BLE-операция не завершилась в отведённое время") + self.reason_code = { + "scan": "ble-discovery-timeout", + "status-read": "ble-status-read-timeout", + "wifi-provision": "ble-provisioning-timeout", + "ap-enable": "ble-ap-enable-timeout", + }[operation_kind] + self.operation_kind = operation_kind + self.operation_stage = progress.operation_stage + self.device_write_attempted = progress.device_write_attempted + self.device_write_confirmed = progress.device_write_confirmed + + +class BleRuntimeSnapshot(TypedDict): + owner_epoch: int + owner_loop_bound: bool + active_operation_kind: BleOperationKind | None + cleanup_pending: bool + poisoned: bool + + +@dataclass(slots=True) +class BleRuntimeProcessLeaseBorrowToken: + """An opaque, context-local proof that a higher layer owns the OS lease.""" + + repository_root: Path + lock_path: Path + configuration_epoch: int + _lease: ApplicationControlProcessLease + _active: bool = True + + +@dataclass(slots=True) +class _ActiveLease: + token: int + operation_kind: BleOperationKind + owner_epoch: int + cleanup_pending: bool = False + task: asyncio.Task[Any] | None = None + owned_process_lease: ApplicationControlProcessLease | None = None + borrowed_process_lease: ApplicationControlProcessLease | None = None + + +class _ProcessBleRuntimeArbiter: + """One fail-fast process lease shared by every supported BLE entrypoint.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._owner_loop: asyncio.AbstractEventLoop | None = None + self._owner_epoch = 0 + self._next_token = 0 + self._active: _ActiveLease | None = None + self._poisoned = False + self._idle_callbacks: list[Callable[[], None]] = [] + self._process_lease_repository_root: Path | None = None + self._process_lease_lock_path: Path | None = None + self._process_lease_configuration_epoch = 0 + + def configure_process_lease(self, repository_root: Path) -> int: + """Pin every low-level BLE entrypoint to one canonical OS lock file.""" + + resolved_root = repository_root.expanduser().resolve() + lock_path = _process_lease_path(resolved_root) + with self._lock: + if ( + self._process_lease_repository_root == resolved_root + and self._process_lease_lock_path == lock_path + ): + return self._process_lease_configuration_epoch + if self._active is not None or self._poisoned or self._idle_callbacks: + raise BleRuntimeProcessLeaseConfigurationConflict() + self._process_lease_configuration_epoch += 1 + self._process_lease_repository_root = resolved_root + self._process_lease_lock_path = lock_path + return self._process_lease_configuration_epoch + + def create_process_lease_borrow( + self, + lease: ApplicationControlProcessLease, + ) -> BleRuntimeProcessLeaseBorrowToken: + """Validate an already-held higher-level lease before context borrowing.""" + + with self._lock: + repository_root = self._process_lease_repository_root + lock_path = self._process_lease_lock_path + if repository_root is None or lock_path is None: + raise BleRuntimeProcessLeaseNotConfigured() + if lease.path != lock_path or lease.release_state != "owned": + raise BleRuntimeProcessLeaseBorrowInvalid() + return BleRuntimeProcessLeaseBorrowToken( + repository_root=repository_root, + lock_path=lock_path, + configuration_epoch=self._process_lease_configuration_epoch, + _lease=lease, + ) + + def deactivate_process_lease_borrow( + self, + borrow: BleRuntimeProcessLeaseBorrowToken, + ) -> None: + with self._lock: + borrow._active = False + + def bind_owner_loop(self, loop: asyncio.AbstractEventLoop) -> int: + if loop.is_closed(): + raise BleRuntimeOwnerLoopConflict() + with self._lock: + current = self._owner_loop + if current is loop: + # Facade dispatch binds every action so state/STOP/close remain + # available after an unproved BLE teardown. Only the next BLE + # lease acquisition is rejected while this loop stays owner. + return self._owner_epoch + if self._poisoned: + raise BleRuntimePoisoned() + if current is not None and not current.is_closed(): + raise BleRuntimeOwnerLoopConflict() + if self._active is not None: + # Its done callback cannot be trusted after the owning loop has + # closed. Never admit a replacement native CoreBluetooth task. + self._poisoned = True + raise BleRuntimePoisoned() + self._owner_epoch += 1 + self._owner_loop = loop + return self._owner_epoch + + def owner_epoch_for_loop(self, loop: asyncio.AbstractEventLoop) -> int | None: + with self._lock: + if self._poisoned or self._owner_loop is not loop or loop.is_closed(): + return None + return self._owner_epoch + + def invalidate_owner_loop( + self, + expected_loop: asyncio.AbstractEventLoop | None, + ) -> int: + with self._lock: + if expected_loop is not None and self._owner_loop is not expected_loop: + return self._owner_epoch + self._owner_epoch += 1 + self._owner_loop = None + if self._active is not None: + self._poisoned = True + return self._owner_epoch + + def acquire( + self, + operation_kind: BleOperationKind, + *, + owner_epoch: int, + process_lease_borrow: BleRuntimeProcessLeaseBorrowToken | None, + ) -> int: + with self._lock: + if self._poisoned: + raise BleRuntimePoisoned() + if self._owner_epoch != owner_epoch: + raise BleRuntimeOwnerLoopConflict() + if self._active is not None: + raise BleRuntimeBusy( + active_operation_kind=self._active.operation_kind, + cleanup_pending=self._active.cleanup_pending, + ) + repository_root = self._process_lease_repository_root + lock_path = self._process_lease_lock_path + if repository_root is None or lock_path is None: + raise BleRuntimeProcessLeaseNotConfigured() + + owned_process_lease: ApplicationControlProcessLease | None = None + borrowed_process_lease: ApplicationControlProcessLease | None = None + if process_lease_borrow is None: + # Detect an environment-driven data-root change before opening + # or creating a lock at a different path than configuration. + if _process_lease_path(repository_root) != lock_path: + raise BleRuntimeProcessLeaseConfigurationConflict() + owned_process_lease = ApplicationControlProcessLease.acquire(repository_root) + if owned_process_lease.path != lock_path: + owned_process_lease.release() + raise BleRuntimeProcessLeaseConfigurationConflict() + else: + if ( + not process_lease_borrow._active + or process_lease_borrow.configuration_epoch + != self._process_lease_configuration_epoch + or process_lease_borrow.repository_root != repository_root + or process_lease_borrow.lock_path != lock_path + or process_lease_borrow._lease.path != lock_path + or process_lease_borrow._lease.release_state != "owned" + ): + raise BleRuntimeProcessLeaseBorrowInvalid() + borrowed_process_lease = process_lease_borrow._lease + self._next_token += 1 + token = self._next_token + self._active = _ActiveLease( + token=token, + operation_kind=operation_kind, + owner_epoch=owner_epoch, + owned_process_lease=owned_process_lease, + borrowed_process_lease=borrowed_process_lease, + ) + return token + + def attach_task(self, token: int, task: asyncio.Task[Any]) -> None: + with self._lock: + if self._active is None or self._active.token != token: + raise RuntimeError("BLE runtime lease was invalidated before task attachment") + self._active.task = task + + def mark_cleanup_pending(self, token: int) -> None: + with self._lock: + if self._active is not None and self._active.token == token: + self._active.cleanup_pending = True + + def release(self, token: int) -> None: + """Finish one lease and drain callbacks before admitting another lease. + + The completed task remains represented by ``_active`` while callbacks + run. Consequently a concurrent local BLE acquisition still fails + closed until the external lifecycle barriers retained by those + callbacks have actually been released. Callbacks must be bounded and + must not call back into this arbiter. + """ + + while True: + with self._lock: + active = self._active + if active is None or active.token != token or self._poisoned: + return + callbacks = tuple(self._idle_callbacks) + self._idle_callbacks.clear() + if not callbacks: + process_lease = active.owned_process_lease + if process_lease is not None: + try: + # This is a non-blocking unlock/close. Keep the + # arbiter lock held so neither a local acquisition + # nor a newly registered external release can race + # the final OS ownership transition. + process_lease.release() + except Exception: + active.cleanup_pending = True + self._poisoned = True + return + self._active = None + return + + for index, callback in enumerate(callbacks): + try: + callback() + except Exception: + # An external release that cannot be proven complete is a + # process-wide safety failure. Keep this lease and every + # unexecuted callback retained until process restart; never + # retry a callback whose partial effects are unknowable. + with self._lock: + current = self._active + if current is not None and current.token == token: + current.cleanup_pending = True + self._poisoned = True + self._idle_callbacks[0:0] = callbacks[index:] + return + + def poison_cleanup_failure(self, token: int) -> None: + """Retain ownership when native session teardown is not proven clean.""" + + with self._lock: + if self._active is not None and self._active.token == token: + self._active.cleanup_pending = True + self._poisoned = True + + def defer_until_idle( + self, + callback: Callable[[], None], + ) -> BleRuntimeIdleCallbackDisposition: + """Run a bounded callback now or at the next proven-idle transition. + + Registration and the idle decision are serialized with acquisition and + release. When a BLE operation is active, its lease stays active while + the callback runs, so another local operation cannot enter the native + runtime between teardown and release of an external process lease. + Poisoned runtimes retain callbacks without ever invoking them: only a + process restart may safely release the corresponding OS ownership. + + The callback must be quick, non-blocking, and must not call this arbiter. + """ + + with self._lock: + if self._poisoned: + self._idle_callbacks.append(callback) + return "poisoned" + if self._active is not None: + self._idle_callbacks.append(callback) + return "deferred" + try: + callback() + except Exception: + # There is no caller-independent way to know whether an + # external release partially succeeded. Preserve the callback + # and prohibit a later BLE admission until process restart. + self._poisoned = True + self._idle_callbacks.append(callback) + raise + return "released" + + def snapshot(self) -> BleRuntimeSnapshot: + with self._lock: + active = self._active + return { + "owner_epoch": self._owner_epoch, + "owner_loop_bound": ( + self._owner_loop is not None and not self._owner_loop.is_closed() + ), + "active_operation_kind": (active.operation_kind if active is not None else None), + "cleanup_pending": bool(active is not None and active.cleanup_pending), + "poisoned": self._poisoned, + } + + def reset_for_tests(self) -> None: + """Forget singleton state without letting an old callback release a new lease.""" + + with self._lock: + owned_process_lease = ( + self._active.owned_process_lease if self._active is not None else None + ) + self._owner_epoch += 1 + self._next_token += 1 + self._owner_loop = None + self._active = None + self._poisoned = False + self._idle_callbacks.clear() + self._process_lease_configuration_epoch += 1 + self._process_lease_repository_root = None + self._process_lease_lock_path = None + if owned_process_lease is not None: + # Test reset deliberately models process exit for synthetic poison. + owned_process_lease.release() + + +_BLE_PROCESS_ARBITER = _ProcessBleRuntimeArbiter() +_BLE_PROCESS_LEASE_BORROW_CONTEXT: ContextVar[ + BleRuntimeProcessLeaseBorrowToken | None +] = ContextVar("ble_runtime_process_lease_borrow", default=None) + + +def _process_lease_path(repository_root: Path) -> Path: + return ( + resolve_missioncore_data_dir(repository_root) + / "xgrids-k1" + / APPLICATION_CONTROL_LOCK_FILENAME + ) + + +def _annotate_cancellation( + exc: asyncio.CancelledError, + progress: BleOperationProgress, +) -> None: + """Preserve side-effect facts when caller cancellation crosses BLE I/O.""" + + exc.operation_stage = progress.operation_stage # type: ignore[attr-defined] + exc.device_write_attempted = progress.device_write_attempted # type: ignore[attr-defined] + exc.device_write_confirmed = progress.device_write_confirmed # type: ignore[attr-defined] + + +def bind_ble_runtime_owner_loop( + loop: asyncio.AbstractEventLoop | None = None, +) -> int: + """Bind process CoreBluetooth ownership to the current persistent loop.""" + + return _BLE_PROCESS_ARBITER.bind_owner_loop(loop or asyncio.get_running_loop()) + + +def ble_runtime_owner_epoch_for_current_loop() -> int | None: + """Return the owner epoch only when called on the bound live loop.""" + + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return None + return _BLE_PROCESS_ARBITER.owner_epoch_for_loop(loop) + + +def invalidate_ble_runtime_owner_loop( + loop: asyncio.AbstractEventLoop | None = None, +) -> int: + """Invalidate process loop ownership without releasing an active BLE lease.""" + + return _BLE_PROCESS_ARBITER.invalidate_owner_loop(loop) + + +def ble_runtime_snapshot() -> BleRuntimeSnapshot: + """Expose a non-secret process snapshot for diagnostics and focused tests.""" + + return _BLE_PROCESS_ARBITER.snapshot() + + +def reset_ble_runtime_arbiter_for_tests() -> None: + """Reset process-global state between synthetic tests only.""" + + _BLE_PROCESS_ARBITER.reset_for_tests() + _BLE_PROCESS_LEASE_BORROW_CONTEXT.set(None) + + +def configure_ble_runtime_process_lease(repository_root: Path) -> int: + """Configure the canonical OS lifecycle lock used by all BLE entrypoints.""" + + return _BLE_PROCESS_ARBITER.configure_process_lease(repository_root) + + +@contextmanager +def borrow_ble_runtime_process_lease( + lease: ApplicationControlProcessLease, +) -> Iterator[BleRuntimeProcessLeaseBorrowToken]: + """Borrow a higher-level lifecycle lease without a second flock attempt. + + The caller must keep ``lease`` owned until every BLE operation admitted in + this context has reached proven idle. A copied context cannot admit a new + operation after this manager exits because the opaque token is invalidated. + """ + + borrow = _BLE_PROCESS_ARBITER.create_process_lease_borrow(lease) + context_token = _BLE_PROCESS_LEASE_BORROW_CONTEXT.set(borrow) + try: + yield borrow + finally: + _BLE_PROCESS_ARBITER.deactivate_process_lease_borrow(borrow) + _BLE_PROCESS_LEASE_BORROW_CONTEXT.reset(context_token) + + +def defer_until_ble_runtime_idle( + callback: Callable[[], None], +) -> BleRuntimeIdleCallbackDisposition: + """Release an external barrier only after native BLE ownership is idle. + + ``"released"`` means the callback ran synchronously because no BLE lease + existed. ``"deferred"`` means it is owned by the active lease and will run + exactly once after native task completion. ``"poisoned"`` means it is + intentionally retained without execution until process restart. + """ + + return _BLE_PROCESS_ARBITER.defer_until_idle(callback) + + +async def wait_for_ble_runtime_idle(timeout_seconds: float = 1.0) -> bool: + """Wait without blocking the owner loop until the active lease is released.""" + + if timeout_seconds < 0: + raise ValueError("timeout_seconds must be non-negative") + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_seconds + while ble_runtime_snapshot()["active_operation_kind"] is not None: + if loop.time() >= deadline: + return False + await asyncio.sleep(min(0.01, max(0.0, deadline - loop.time()))) + return True + + +async def run_ble_operation[T]( + operation_kind: BleOperationKind, + *, + hard_timeout_seconds: float, + operation: Callable[[BleOperationProgress], Coroutine[Any, Any, T]], + progress: BleOperationProgress | None = None, +) -> T: + """Run one BLE task behind a process lease and a non-blocking hard deadline.""" + + if hard_timeout_seconds <= 0: + raise ValueError("hard_timeout_seconds must be positive") + loop = asyncio.get_running_loop() + owner_epoch = bind_ble_runtime_owner_loop(loop) + operation_progress = progress or BleOperationProgress() + operation_progress.owner_epoch = owner_epoch + token = _BLE_PROCESS_ARBITER.acquire( + operation_kind, + owner_epoch=owner_epoch, + process_lease_borrow=_BLE_PROCESS_LEASE_BORROW_CONTEXT.get(), + ) + try: + task: asyncio.Task[T] = loop.create_task(operation(operation_progress)) + except BaseException: + _BLE_PROCESS_ARBITER.release(token) + raise + _BLE_PROCESS_ARBITER.attach_task(token, task) + cleanup_requested = False + + def release_after_completion(completed: asyncio.Future[T]) -> None: + cleanup_failed = False + try: + completed_exception = None if completed.cancelled() else completed.exception() + cleanup_failed = cleanup_requested and completed_exception is not None + except BaseException: + # The caller observes the original result/exception. This callback + # only consumes detached cleanup outcomes and releases ownership. + cleanup_failed = cleanup_requested + finally: + if cleanup_failed: + _BLE_PROCESS_ARBITER.poison_cleanup_failure(token) + else: + _BLE_PROCESS_ARBITER.release(token) + + task.add_done_callback(release_after_completion) + try: + completed, _ = await asyncio.wait({task}, timeout=hard_timeout_seconds) + except asyncio.CancelledError as exc: + _annotate_cancellation(exc, operation_progress) + if not task.done(): + _BLE_PROCESS_ARBITER.mark_cleanup_pending(token) + cleanup_requested = True + task.cancel() + raise + if completed: + return task.result() + + _BLE_PROCESS_ARBITER.mark_cleanup_pending(token) + cleanup_requested = True + task.cancel() + raise BleOperationHardTimeout(operation_kind, operation_progress) + + +@asynccontextmanager +async def run_ble_operation_session[T]( + operation_kind: BleOperationKind, + *, + hard_setup_timeout_seconds: float, + hard_cleanup_timeout_seconds: float = 5.0, + operation: Callable[ + [BleOperationProgress], + AbstractAsyncContextManager[T], + ], + progress: BleOperationProgress | None = None, +) -> AsyncIterator[T]: + """Hold one process BLE lease across setup, caller work, and cleanup. + + Only setup is subject to the hard deadline. Once the inner session yields, + its native BLE client remains alive until the caller leaves this context. + Cancellation never frees the process lease early: a detached native cleanup + continues to own the lease until its task has actually completed. + """ + + if hard_setup_timeout_seconds <= 0: + raise ValueError("hard_setup_timeout_seconds must be positive") + if hard_cleanup_timeout_seconds <= 0: + raise ValueError("hard_cleanup_timeout_seconds must be positive") + loop = asyncio.get_running_loop() + owner_epoch = bind_ble_runtime_owner_loop(loop) + operation_progress = progress or BleOperationProgress() + operation_progress.owner_epoch = owner_epoch + token = _BLE_PROCESS_ARBITER.acquire( + operation_kind, + owner_epoch=owner_epoch, + process_lease_borrow=_BLE_PROCESS_LEASE_BORROW_CONTEXT.get(), + ) + ready: asyncio.Future[T] = loop.create_future() + release_requested = asyncio.Event() + setup_cleanup_requested = False + + async def session_task() -> None: + async with operation(operation_progress) as value: + if not ready.done(): + ready.set_result(value) + await release_requested.wait() + + try: + task = loop.create_task(session_task()) + except BaseException: + _BLE_PROCESS_ARBITER.release(token) + raise + _BLE_PROCESS_ARBITER.attach_task(token, task) + + def release_after_completion(completed: asyncio.Future[None]) -> None: + cleanup_failed = False + try: + completed_exception = None if completed.cancelled() else completed.exception() + cleanup_failed = bool( + ( + ready.done() + and not ready.cancelled() + and release_requested.is_set() + and (completed.cancelled() or completed_exception is not None) + ) + or (setup_cleanup_requested and completed_exception is not None) + ) + except BaseException: + # The active caller observes setup/cleanup failures directly. This + # callback only consumes detached outcomes before releasing ownership. + cleanup_failed = setup_cleanup_requested or bool( + ready.done() and not ready.cancelled() and release_requested.is_set() + ) + finally: + if cleanup_failed: + _BLE_PROCESS_ARBITER.poison_cleanup_failure(token) + else: + _BLE_PROCESS_ARBITER.release(token) + + task.add_done_callback(release_after_completion) + setup_waiters: set[asyncio.Future[Any]] = {ready, task} + try: + completed, _ = await asyncio.wait( + setup_waiters, + timeout=hard_setup_timeout_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + except asyncio.CancelledError as exc: + _annotate_cancellation(exc, operation_progress) + if not task.done(): + _BLE_PROCESS_ARBITER.mark_cleanup_pending(token) + setup_cleanup_requested = True + task.cancel() + raise + + if ready not in completed: + if task in completed: + # Setup failed before the inner session became available. + task.result() + raise RuntimeError("BLE session ended before setup completed") + _BLE_PROCESS_ARBITER.mark_cleanup_pending(token) + setup_cleanup_requested = True + task.cancel() + raise BleOperationHardTimeout(operation_kind, operation_progress) + + try: + try: + yield ready.result() + except asyncio.CancelledError as exc: + _annotate_cancellation(exc, operation_progress) + raise + finally: + # From this point the setup deadline no longer applies. The caller may + # perform a bounded host-side handoff while the same BLE client remains + # connected. On exit, ownership is retained until __aexit__ really ends. + # A wedged native disconnect must not hold the HTTP caller forever: + # detach it after the cleanup deadline while the task and process lease + # remain quarantined until CoreBluetooth actually acknowledges cleanup. + _BLE_PROCESS_ARBITER.mark_cleanup_pending(token) + release_requested.set() + completed, _ = await asyncio.wait( + {task}, + timeout=hard_cleanup_timeout_seconds, + ) + if completed: + task.result() + # Do not cancel a disconnect already in progress. A second cancellation + # can make an otherwise responsive ``__aexit__`` finish as cancelled + # without proving that CoreBluetooth acknowledged the native teardown. + # The detached task therefore keeps the process lease until its natural + # completion callback releases it. diff --git a/src/k1link/device_plugins/xgrids_k1/ble/scanner.py b/src/k1link/device_plugins/xgrids_k1/ble/scanner.py index 555113f..9d6efa6 100644 --- a/src/k1link/device_plugins/xgrids_k1/ble/scanner.py +++ b/src/k1link/device_plugins/xgrids_k1/ble/scanner.py @@ -1,34 +1,58 @@ from __future__ import annotations -from dataclasses import dataclass +import asyncio +import math +from collections.abc import Callable +from dataclasses import dataclass, replace from importlib.metadata import version from threading import Lock from time import monotonic -from typing import TypedDict +from time import time as suspend_aware_time +from typing import Literal, TypedDict +from uuid import UUID from bleak import BleakScanner from bleak.backends.device import BLEDevice from bleak.backends.scanner import AdvertisementData from k1link.artifacts import utc_now_iso - -# The operator-visible candidate lease is the admission contract. The exact -# CoreBluetooth handle gets a small internal grace window because it is -# published just before the facade timestamps the same scan result. This -# guarantees that a UI-admissible candidate can never fall into a second UUID -# lookup at the millisecond boundary. -BLE_DISCOVERY_CANDIDATE_LEASE_TTL_SECONDS = 60.0 -BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS = ( - BLE_DISCOVERY_CANDIDATE_LEASE_TTL_SECONDS + 5.0 +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationProgress, + ble_runtime_owner_epoch_for_current_loop, + ble_runtime_snapshot, + run_ble_operation, ) + +# Compatibility constants retained for public imports and diagnostics. A scan +# generation is now invalidated only by an explicit new scan, runtime-owner +# teardown or a proven GATT failure. Wall-clock age alone must never make an +# operator-selected K1 disappear while they are completing the form. +BLE_DISCOVERY_CANDIDATE_LEASE_TTL_SECONDS = 60.0 +BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS = BLE_DISCOVERY_CANDIDATE_LEASE_TTL_SECONDS + 5.0 # Compatibility alias for callers that historically treated this as the # low-level retained-handle lifetime. BLE_DISCOVERY_LEASE_TTL_SECONDS = BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS +BLE_SCAN_HARD_TIMEOUT_GRACE_SECONDS = 5.0 _runtime_handle_lock = Lock() _runtime_handles: dict[str, BLEDevice] = {} _runtime_handle_generation = 0 _runtime_handle_observed_at_monotonic: float | None = None +_runtime_handle_observed_at_suspend_aware: float | None = None +_runtime_handle_owner_epoch: int | None = None +_connected_runtime_handle: _ConnectedRuntimeHandle | None = None +_pending_gatt_validations: dict[ + tuple[int, int, str, int], + _PendingGattValidation, +] = {} +_invalidated_captured_handles: set[tuple[int, int, str, int]] = set() + +# A successful baseline read may precede the facade's device-session commit by +# a few synchronous instructions. Keep that evidence only long enough for the +# exact captured CoreBluetooth object to be pinned. This is validation +# evidence, never presence or connection authority. +BLE_GATT_VALIDATION_RECENCY_TTL_SECONDS = 10.0 +BLE_GATT_VALIDATION_HANDOFF_TTL_SECONDS = BLE_GATT_VALIDATION_RECENCY_TTL_SECONDS @dataclass(frozen=True) @@ -39,6 +63,62 @@ class DiscoveredDeviceSelection: from_fresh_scan: bool +@dataclass(frozen=True) +class CapturedDiscoveredDevice: + """Opaque proof that one exact BLEDevice came from a fresh owner scan.""" + + device: BLEDevice + macos_uuid: str + owner_epoch: int + scan_generation: int = 0 + captured_at_monotonic: float = 0.0 + captured_at_suspend_aware: float | None = None + source: Literal[ + "fresh-scan", + "retained-session", + "retrieved-session", + "retrieved-durable", + ] = "fresh-scan" + + +@dataclass(frozen=True) +class _FreshnessStamp: + """One lease timestamp measured by both relevant macOS clocks. + + ``time.monotonic`` prevents wall-clock corrections from extending a lease, + while ``time.time`` advances across macOS suspend. A lease is usable only + while both clocks agree that it is fresh. + """ + + monotonic: float + suspend_aware: float + + +@dataclass(frozen=True) +class _PendingGattValidation: + """Short handoff proof retaining the exact validated transport object.""" + + captured: CapturedDiscoveredDevice + validated_at: _FreshnessStamp + + +@dataclass(frozen=True) +class _ConnectedRuntimeHandle: + """One private recovery handle scoped to the admitted device session.""" + + captured: CapturedDiscoveredDevice + device_session_id: str + retained_at: _FreshnessStamp + last_gatt_validated_at: _FreshnessStamp | None = None + + +class ConnectedDeviceRecoverySnapshot(TypedDict): + status: Literal["retained", "unavailable"] + scope: Literal["owner-epoch-device-session"] + advertised_now: bool + gatt_validated_recently: bool + + class BleDeviceRecord(TypedDict): macos_uuid: str id_kind: str @@ -87,77 +167,1001 @@ def advertisement_record(device: BLEDevice, advertisement: AdvertisementData) -> } -def _invalidate_runtime_handles_locked() -> None: +def _freshness_now() -> _FreshnessStamp: + """Read a decision clock pair; wall is sampled last to expose suspend.""" + + monotonic_value = monotonic() + suspend_aware_value = suspend_aware_time() + return _validated_freshness_stamp(monotonic_value, suspend_aware_value) + + +def _freshness_observed_at() -> _FreshnessStamp: + """Timestamp evidence; wall is sampled first so suspend cannot mint freshness.""" + + suspend_aware_value = suspend_aware_time() + monotonic_value = monotonic() + return _validated_freshness_stamp(monotonic_value, suspend_aware_value) + + +def _validated_freshness_stamp( + monotonic_value: object, + suspend_aware_value: object, +) -> _FreshnessStamp: + if ( + not isinstance(monotonic_value, (int, float)) + or isinstance(monotonic_value, bool) + or not math.isfinite(monotonic_value) + or not isinstance(suspend_aware_value, (int, float)) + or isinstance(suspend_aware_value, bool) + or not math.isfinite(suspend_aware_value) + ): + raise RuntimeError("BLE freshness clocks must return finite numbers") + return _FreshnessStamp( + monotonic=float(monotonic_value), + suspend_aware=float(suspend_aware_value), + ) + + +def _stamp_is_fresh( + stamp: _FreshnessStamp, + *, + now: _FreshnessStamp, + ttl_seconds: float, +) -> bool: + monotonic_age = now.monotonic - stamp.monotonic + suspend_aware_age = now.suspend_aware - stamp.suspend_aware + return bool( + 0.0 <= monotonic_age <= ttl_seconds + and 0.0 <= suspend_aware_age <= ttl_seconds + ) + + +def _scan_observation_stamp_locked(now: _FreshnessStamp) -> _FreshnessStamp | None: + """Return the current scan stamp, adopting old synthetic test fixtures. + + Production scans always publish both clocks atomically. A few downstream + tests historically seed the private monotonic field directly; giving those + fixtures a wall-clock origin at first observation preserves compatibility + without weakening real scan leases. + """ + + global _runtime_handle_observed_at_suspend_aware + + observed_monotonic = _runtime_handle_observed_at_monotonic + if observed_monotonic is None: + return None + if _runtime_handle_observed_at_suspend_aware is None: + _runtime_handle_observed_at_suspend_aware = now.suspend_aware + return _FreshnessStamp( + monotonic=observed_monotonic, + suspend_aware=_runtime_handle_observed_at_suspend_aware, + ) + + +def _captured_scan_lease_is_fresh( + captured: CapturedDiscoveredDevice, + *, + now: _FreshnessStamp, +) -> bool: + # Production captures always contain both values. ``None`` preserves old + # synthetic captures created directly by unit tests; they still require a + # matching owner epoch and exact object identity. + if captured.captured_at_suspend_aware is None: + return True + return _stamp_is_fresh( + _FreshnessStamp( + monotonic=captured.captured_at_monotonic, + suspend_aware=captured.captured_at_suspend_aware, + ), + now=now, + ttl_seconds=BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS, + ) + + +def _captured_with_freshness( + captured: CapturedDiscoveredDevice, + *, + stamp: _FreshnessStamp, + source: Literal["retained-session", "retrieved-session"], +) -> CapturedDiscoveredDevice: + """Refresh one exact handle lease without claiming advertisement or GATT.""" + + return replace( + captured, + captured_at_monotonic=stamp.monotonic, + captured_at_suspend_aware=stamp.suspend_aware, + source=source, + ) + + +def _expired_freshness(now: _FreshnessStamp) -> _FreshnessStamp: + """Create an unusable handle stamp while retaining its recovery identity.""" + + expired_age = BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 1.0 + return _FreshnessStamp( + monotonic=now.monotonic - expired_age, + suspend_aware=now.suspend_aware - expired_age, + ) + + +def _invalidate_scan_handles_locked() -> None: global _runtime_handle_observed_at_monotonic + global _runtime_handle_observed_at_suspend_aware, _runtime_handle_owner_epoch _runtime_handles.clear() _runtime_handle_observed_at_monotonic = None + _runtime_handle_observed_at_suspend_aware = None + _runtime_handle_owner_epoch = None + _pending_gatt_validations.clear() -def _begin_scan_generation() -> int: +def _invalidate_runtime_handles_locked() -> None: + global _connected_runtime_handle + + _invalidate_scan_handles_locked() + _connected_runtime_handle = None + _invalidated_captured_handles.clear() + + +def invalidate_runtime_handles() -> None: + """Drop retained CoreBluetooth handles before changing owner event loops.""" + global _runtime_handle_generation with _runtime_handle_lock: + # Supersede an in-flight scan as well as clearing its last published + # handles. A late callback from the previous owner loop cannot restore + # the invalidated CoreBluetooth objects. _runtime_handle_generation += 1 _invalidate_runtime_handles_locked() + + +def invalidate_connected_device_session( + macos_uuid: str, + *, + device_session_id: str, +) -> bool: + """Drop only one exact pinned connection session. + + Loss of an already-open connection must not erase a newer explicit scan + generation. This primitive therefore leaves public scan handles and + unrelated fresh-scan validation handoffs untouched; the exact retained + ``UUID + device_session_id`` binding and its object-bound handoff are + retired. + """ + + if not macos_uuid or not device_session_id: + return False + global _connected_runtime_handle + with _runtime_handle_lock: + retained = _connected_runtime_handle + if ( + retained is None + or retained.captured.macos_uuid != macos_uuid + or retained.device_session_id != device_session_id + ): + return False + captured_key = _captured_handle_key(retained.captured) + _pending_gatt_validations.pop(captured_key, None) + _invalidated_captured_handles.add(captured_key) + _connected_runtime_handle = None + return True + + +def reset_runtime_handles_for_tests() -> None: + """Reset retained-handle state without reusing an in-flight generation.""" + + invalidate_runtime_handles() + + +def _begin_scan_generation(owner_epoch: int) -> int: + global _runtime_handle_generation, _runtime_handle_owner_epoch + + with _runtime_handle_lock: + _runtime_handle_generation += 1 + # Public discovery is generation-scoped. The separately pinned handle + # represents an already admitted device session and therefore survives + # later scans which may legitimately omit a non-advertising K1. + _invalidate_scan_handles_locked() + _runtime_handle_owner_epoch = owner_epoch return _runtime_handle_generation def _finish_failed_scan(scan_generation: int) -> None: with _runtime_handle_lock: if _runtime_handle_generation == scan_generation: - _invalidate_runtime_handles_locked() + _invalidate_scan_handles_locked() def _publish_scan_handles( scan_generation: int, + owner_epoch: int, handles: dict[str, BLEDevice], ) -> None: global _runtime_handle_observed_at_monotonic + global _runtime_handle_observed_at_suspend_aware + + if ble_runtime_owner_epoch_for_current_loop() != owner_epoch: + return with _runtime_handle_lock: - if _runtime_handle_generation != scan_generation: + if ( + _runtime_handle_generation != scan_generation + or _runtime_handle_owner_epoch != owner_epoch + ): return + observed_at = _freshness_observed_at() _runtime_handles.update(handles) - _runtime_handle_observed_at_monotonic = monotonic() + _runtime_handle_observed_at_monotonic = observed_at.monotonic + _runtime_handle_observed_at_suspend_aware = observed_at.suspend_aware + # A fresh scan is candidate/presence evidence for the next explicit + # intent. It must never mutate the immutable CoreBluetooth object bound + # to an older admitted device-session with the same reusable UUID. + + +def _captured_handle_key(captured: CapturedDiscoveredDevice) -> tuple[int, int, str, int]: + return ( + captured.owner_epoch, + captured.scan_generation, + captured.macos_uuid, + id(captured.device), + ) + + +def _same_captured_transport( + left: CapturedDiscoveredDevice, + right: CapturedDiscoveredDevice, +) -> bool: + """Match one exact CoreBluetooth object, not merely its reusable UUID.""" + + return ( + left.owner_epoch == right.owner_epoch + and left.scan_generation == right.scan_generation + and left.macos_uuid == right.macos_uuid + and left.device is right.device + ) + + +def _prune_pending_gatt_validations_locked(now: _FreshnessStamp) -> None: + stale = [ + key + for key, validation in _pending_gatt_validations.items() + if not _stamp_is_fresh( + validation.validated_at, + now=now, + ttl_seconds=BLE_GATT_VALIDATION_HANDOFF_TTL_SECONDS, + ) + ] + for key in stale: + _pending_gatt_validations.pop(key, None) def discovered_device_selection(macos_uuid: str) -> DiscoveredDeviceSelection: - """Resolve a device against the latest unexpired explicit scan generation. + """Resolve a device against the latest explicit scan generation. ``from_fresh_scan`` distinguishes a fresh scan that did not contain the requested device from a caller that has no usable explicit scan lease. A mutating caller may perform fallback discovery only in the latter case. """ + current_owner_epoch = ble_runtime_owner_epoch_for_current_loop() + global _runtime_handle_owner_epoch + with _runtime_handle_lock: - observed_at = _runtime_handle_observed_at_monotonic + now = _freshness_now() + observed_at = _scan_observation_stamp_locked(now) if observed_at is None: return DiscoveredDeviceSelection(device=None, from_fresh_scan=False) - age_seconds = monotonic() - observed_at - if age_seconds < 0.0 or age_seconds > BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS: - _invalidate_runtime_handles_locked() + # Production handles are always epoch-tagged by ``scan``. The None + # branch only preserves synthetic fixtures that seed this private cache + # directly; adopt them onto the first bound test loop. + if _runtime_handle_owner_epoch is None and current_owner_epoch is not None: + _runtime_handle_owner_epoch = current_owner_epoch + elif ( + _runtime_handle_owner_epoch is not None + and _runtime_handle_owner_epoch != current_owner_epoch + ): + # This lookup owns only the public scan lease. The separately + # pinned connection handle is invalidated only by the explicit + # owner-runtime transition/close path. Merely resolving the public + # scan cache -- including from a different or unbound loop -- must + # never revoke session authority. + _invalidate_scan_handles_locked() return DiscoveredDeviceSelection(device=None, from_fresh_scan=False) - return DiscoveredDeviceSelection( - device=_runtime_handles.get(macos_uuid), - from_fresh_scan=True, - ) + device = _runtime_handles.get(macos_uuid) + if ( + device is not None + and current_owner_epoch is not None + and ( + current_owner_epoch, + _runtime_handle_generation, + macos_uuid, + id(device), + ) + in _invalidated_captured_handles + ): + # The facade may deliberately retain the operator-visible row + # after a losing read-only Verify. Session teardown still revoked + # this exact CoreBluetooth object, so the same generation must not + # masquerade as a usable selection. ``from_fresh_scan`` stays + # true to forbid the low-level UUID lookup fallback; only a new + # explicit scan may publish a replacement object. + device = None + return DiscoveredDeviceSelection(device=device, from_fresh_scan=True) def discovered_device(macos_uuid: str) -> BLEDevice | None: - """Return a CoreBluetooth handle only while its explicit scan lease is fresh.""" + """Return a handle from the latest explicit scan generation.""" return discovered_device_selection(macos_uuid).device -async def scan(duration_seconds: float) -> BleScanResult: +def capture_discovered_device(macos_uuid: str) -> CapturedDiscoveredDevice | None: + """Capture one exact fresh handle before a potentially long network action.""" + + selection = discovered_device_selection(macos_uuid) + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + if selection.device is None or not selection.from_fresh_scan or owner_epoch is None: + return None + with _runtime_handle_lock: + # Close the lookup-to-capture race without weakening the exact-object + # fence. A concurrent generation change, owner transition or teardown + # must make this capture fail; it must never relabel the old object as + # belonging to the new scan generation. + if ( + _runtime_handle_owner_epoch != owner_epoch + or _runtime_handles.get(macos_uuid) is not selection.device + ): + return None + scan_generation = _runtime_handle_generation + now = _freshness_now() + captured_at = _scan_observation_stamp_locked(now) + captured = CapturedDiscoveredDevice( + device=selection.device, + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + scan_generation=scan_generation, + captured_at_monotonic=( + captured_at.monotonic if captured_at is not None else now.monotonic + ), + captured_at_suspend_aware=( + captured_at.suspend_aware if captured_at is not None else now.suspend_aware + ), + source="fresh-scan", + ) + if _captured_handle_key(captured) in _invalidated_captured_handles: + return None + return captured + + +def pin_connected_device_handle( + captured: CapturedDiscoveredDevice, + *, + device_session_id: str, +) -> None: + """Promote a previously captured fresh handle after connection admission. + + The caller cannot manufacture a new CoreBluetooth object here: the exact + capture must still belong to the current process owner epoch. The handle is + private runtime state and is never added to operator-visible scan results. + """ + + if not device_session_id: + raise ValueError("device_session_id must not be empty") + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + if owner_epoch is None or captured.owner_epoch != owner_epoch: + raise RuntimeError("BLE recovery handle belongs to another owner epoch") + if captured.device.address != captured.macos_uuid: + raise RuntimeError("BLE recovery handle identity changed after capture") + global _connected_runtime_handle + with _runtime_handle_lock: + captured_key = _captured_handle_key(captured) + if captured_key in _invalidated_captured_handles: + raise RuntimeError("BLE recovery handle was invalidated by session teardown") + now = _freshness_now() + _prune_pending_gatt_validations_locked(now) + pending_validation = _pending_gatt_validations.pop( + captured_key, + None, + ) + validated_at = ( + pending_validation.validated_at + if pending_validation is not None + and _same_captured_transport(pending_validation.captured, captured) + else None + ) + retained = _connected_runtime_handle + retained_is_exact = bool( + retained is not None + and _same_captured_transport(retained.captured, captured) + ) + retained_validation = ( + retained.last_gatt_validated_at + if retained_is_exact and retained is not None + else None + ) + if ( + validated_at is None + and retained_validation is not None + and _stamp_is_fresh( + retained_validation, + now=now, + ttl_seconds=BLE_GATT_VALIDATION_HANDOFF_TTL_SECONDS, + ) + ): + validated_at = retained_validation + if captured.source in {"retrieved-session", "retrieved-durable"} and ( + validated_at is None + ): + raise RuntimeError( + "Retrieved BLE recovery handle requires exact GATT validation before admission" + ) + effective_capture = ( + retained.captured + if retained_is_exact and retained is not None and validated_at is not None + else captured + ) + _connected_runtime_handle = _ConnectedRuntimeHandle( + captured=_captured_with_freshness( + effective_capture, + stamp=now, + source="retained-session", + ), + device_session_id=device_session_id, + retained_at=now, + last_gatt_validated_at=validated_at, + ) + + +def mark_captured_device_gatt_validated( + captured: CapturedDiscoveredDevice, +) -> bool: + """Record one exact successful GATT contract and baseline read. + + A fresh connection may validate immediately before its device-session is + committed. In that case a short, object-identity-bound handoff token lets + :func:`pin_connected_device_handle` carry the fact into the new session. + If the handle is already pinned, its diagnostic timestamp is updated in + place. Neither branch proves advertisement, presence or command authority. + """ + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + if ( + owner_epoch is None + or captured.owner_epoch != owner_epoch + or captured.device.address != captured.macos_uuid + ): + return False + + global _connected_runtime_handle + with _runtime_handle_lock: + now = _freshness_now() + _prune_pending_gatt_validations_locked(now) + retained = _connected_runtime_handle + if retained is not None and _same_captured_transport( + retained.captured, + captured, + ): + _connected_runtime_handle = replace( + retained, + captured=_captured_with_freshness( + captured, + stamp=now, + source="retained-session", + ), + retained_at=now, + last_gatt_validated_at=now, + ) + # Always publish a short exact-object handoff. A successful baseline + # may be immediately followed by a facade device-session rotation; the + # caller still holds its pre-read capture while the retained projection + # above now carries refreshed timestamps. Retaining the object inside + # this token also prevents Python ``id`` reuse from authorizing another + # BLEDevice during the handoff window. + _pending_gatt_validations[_captured_handle_key(captured)] = ( + _PendingGattValidation(captured=captured, validated_at=now) + ) + return True + + +def demote_connected_device_handle_after_gatt_failure( + captured: CapturedDiscoveredDevice, +) -> bool: + """Expire the exact transport object that failed live GATT validation. + + Matching includes owner epoch, scan generation and Python object identity, + so a late failure from an older operation cannot demote a replacement + ``BLEDevice`` published by a newer explicit scan. The process-scoped exact + UUID/session token remains available for a later explicit CoreBluetooth + retrieval; neither failure nor retrieval is advertisement evidence. + """ + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + if owner_epoch is None or captured.owner_epoch != owner_epoch: + return False + + global _connected_runtime_handle + with _runtime_handle_lock: + _pending_gatt_validations.pop(_captured_handle_key(captured), None) + # A failed exact connect/GATT exchange invalidates this selection even + # when it had not yet been promoted to a device session. The next + # operator attempt must come from a new explicit scan generation, not + # silently reuse the same failed CoreBluetooth object. + _invalidated_captured_handles.add(_captured_handle_key(captured)) + retained = _connected_runtime_handle + if retained is None or not _same_captured_transport( + retained.captured, + captured, + ): + return False + _connected_runtime_handle = None + return True + + +def mark_connected_device_gatt_validated( + captured: CapturedDiscoveredDevice, + *, + device_session_id: str, +) -> None: + """Record a live GATT preflight without turning it into write authority.""" + + global _connected_runtime_handle + with _runtime_handle_lock: + retained = _connected_runtime_handle + if retained is None: + return + if ( + retained.device_session_id != device_session_id + or not _same_captured_transport(retained.captured, captured) + ): + return + now = _freshness_now() + _connected_runtime_handle = replace( + retained, + captured=_captured_with_freshness( + captured, + stamp=now, + source="retained-session", + ), + retained_at=now, + last_gatt_validated_at=now, + ) + + +def captured_device_handle(captured: CapturedDiscoveredDevice) -> BLEDevice | None: + """Resolve an opaque capture only on its original live owner epoch.""" + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + if owner_epoch is None or captured.owner_epoch != owner_epoch: + return None + if captured.device.address != captured.macos_uuid: + return None + with _runtime_handle_lock: + if _captured_handle_key(captured) in _invalidated_captured_handles: + return None + return captured.device + + +def connected_device_capture( + macos_uuid: str, + *, + device_session_id: str, +) -> CapturedDiscoveredDevice | None: + """Resolve only the exact current owner/session recovery handle.""" + + global _connected_runtime_handle + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + with _runtime_handle_lock: + pinned = _connected_runtime_handle + if pinned is None: + return None + if owner_epoch is None or pinned.captured.owner_epoch != owner_epoch: + return None + if ( + pinned.captured.macos_uuid != macos_uuid + or pinned.device_session_id != device_session_id + ): + return None + return pinned.captured + + +def connected_device_recovery_name( + macos_uuid: str, + *, + device_session_id: str, +) -> str | None: + """Return the exact process-token name as context, never as presence. + + Unlike :func:`connected_device_capture`, this deliberately does not require + a fresh transport-object lease. It is only last-known context bound to the + exact owner epoch, CoreBluetooth UUID and device session; every physical + action still requires explicit retrieval and a new reviewed GATT baseline. + """ + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + runtime = ble_runtime_snapshot() + with _runtime_handle_lock: + pinned = _connected_runtime_handle + if ( + owner_epoch is None + or not macos_uuid + or not device_session_id + or runtime["owner_epoch"] != owner_epoch + or not runtime["owner_loop_bound"] + or runtime["poisoned"] + or pinned is None + or pinned.captured.owner_epoch != owner_epoch + or pinned.captured.macos_uuid.casefold() != macos_uuid.casefold() + or pinned.device_session_id != device_session_id + ): + return None + name = pinned.captured.device.name + normalized = str(name).strip() if name is not None else "" + return normalized or None + + +async def _retrieve_corebluetooth_device( + captured: CapturedDiscoveredDevice, +) -> BLEDevice | None: + """Retrieve one UUID through the exact CoreBluetooth manager that observed it.""" + + details = captured.device.details + if not isinstance(details, tuple) or len(details) != 2: + return None + peripheral, manager = details + try: + identifier = peripheral.identifier() + identifier_value = str(identifier.UUIDString()) + except (AttributeError, TypeError, ValueError): + return None + if identifier_value.casefold() != captured.macos_uuid.casefold(): + return None + if getattr(manager, "event_loop", None) is not asyncio.get_running_loop(): + return None + wait_until_ready = getattr(manager, "wait_until_ready", None) + central_manager = getattr(manager, "central_manager", None) + if wait_until_ready is None or central_manager is None: + return None + + try: + await wait_until_ready() + retrieved = central_manager.retrievePeripheralsWithIdentifiers_([identifier]) + except Exception: + # Retrieval is an explicit, read-only recovery attempt. A transient + # CoreBluetooth failure must not consume the process session token or + # silently fall through to discovery/connect retries. + return None + for candidate in retrieved or (): + try: + candidate_identifier = str(candidate.identifier().UUIDString()) + except (AttributeError, TypeError, ValueError): + continue + if candidate_identifier.casefold() != captured.macos_uuid.casefold(): + continue + try: + name_value = candidate.name() + except (AttributeError, TypeError): + name_value = None + return BLEDevice( + captured.macos_uuid, + str(name_value) if name_value is not None else captured.device.name, + details=(candidate, manager), + ) + return None + + +def _new_corebluetooth_retrieval_context(macos_uuid: str) -> tuple[object, object] | None: + """Create a native manager and NSUUID for a durable read-only lookup. + + Imports stay local because this module is also imported by non-macOS test + and tooling processes. The returned manager is retained by the + ``BLEDevice.details`` tuple for the complete GATT operation. + """ + + try: + UUID(macos_uuid) + from bleak.backends.corebluetooth.CentralManagerDelegate import ( + CentralManagerDelegate, + ) + from Foundation import NSUUID # type: ignore[import-untyped] + + identifier = NSUUID.UUIDWithString_(macos_uuid) + if identifier is None: + return None + return CentralManagerDelegate(), identifier + except (ImportError, TypeError, ValueError): + return None + + +async def retrieve_known_device_capture_for_status_read( + macos_uuid: str, +) -> CapturedDiscoveredDevice | None: + """Retrieve one server-pinned CoreBluetooth UUID inside a status read. + + This is deliberately narrower than the current-session recovery token. It + is admitted only while the process BLE arbiter owns an explicit + ``status-read`` operation. CoreBluetooth retrieval does not scan, connect, + prove presence, validate identity, grant control or authorize a write; the + caller must still perform a new reviewed GATT connect and 7f02 read. + """ + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + runtime = ble_runtime_snapshot() + if ( + owner_epoch is None + or not macos_uuid + or runtime["owner_epoch"] != owner_epoch + or not runtime["owner_loop_bound"] + or runtime["active_operation_kind"] != "status-read" + or runtime["poisoned"] + ): + return None + context = _new_corebluetooth_retrieval_context(macos_uuid) + if context is None: + return None + manager, identifier = context + wait_until_ready = getattr(manager, "wait_until_ready", None) + central_manager = getattr(manager, "central_manager", None) + if wait_until_ready is None or central_manager is None: + return None + try: + await wait_until_ready() + retrieved = central_manager.retrievePeripheralsWithIdentifiers_([identifier]) + except Exception: + return None + + runtime_after = ble_runtime_snapshot() + if ( + ble_runtime_owner_epoch_for_current_loop() != owner_epoch + or runtime_after["owner_epoch"] != owner_epoch + or not runtime_after["owner_loop_bound"] + or runtime_after["active_operation_kind"] != "status-read" + or runtime_after["poisoned"] + ): + return None + for peripheral in retrieved or (): + try: + candidate_uuid = str(peripheral.identifier().UUIDString()) + except (AttributeError, TypeError, ValueError): + continue + if candidate_uuid.casefold() != macos_uuid.casefold(): + continue + try: + name_value = peripheral.name() + except (AttributeError, TypeError): + name_value = None + now = _freshness_observed_at() + return CapturedDiscoveredDevice( + device=BLEDevice( + macos_uuid, + str(name_value) if name_value is not None else None, + details=(peripheral, manager), + ), + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + scan_generation=0, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + return None + + +async def discover_known_device_capture_for_status_read( + macos_uuid: str, + *, + timeout_seconds: float, +) -> CapturedDiscoveredDevice | None: + """Observe one exact durable CoreBluetooth UUID without publishing discovery. + + The unfiltered advertisement scan is admitted only inside the current + ``status-read`` arbiter lease. It neither updates the operator-visible + discovery generation nor falls back to another identifier. The caller + must still perform exactly one reviewed GATT connect and 7f02 read before + this transport object can be admitted to a device session. + """ + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + runtime = ble_runtime_snapshot() + if ( + owner_epoch is None + or not macos_uuid + or not math.isfinite(timeout_seconds) + or timeout_seconds <= 0 + or runtime["owner_epoch"] != owner_epoch + or not runtime["owner_loop_bound"] + or runtime["active_operation_kind"] != "status-read" + or runtime["poisoned"] + ): + return None + try: + UUID(macos_uuid) + except (TypeError, ValueError): + return None + + candidate = await BleakScanner.find_device_by_address( + macos_uuid, + timeout=timeout_seconds, + ) + + runtime_after = ble_runtime_snapshot() + if ( + ble_runtime_owner_epoch_for_current_loop() != owner_epoch + or runtime_after["owner_epoch"] != owner_epoch + or not runtime_after["owner_loop_bound"] + or runtime_after["active_operation_kind"] != "status-read" + or runtime_after["poisoned"] + or candidate is None + or candidate.address.casefold() != macos_uuid.casefold() + ): + return None + + # Preserve the exact native CoreBluetooth transport observed by the scan, + # while canonicalizing its public address to the spelling of the durable + # identity pin. Downstream capture checks intentionally require exact + # string equality as well as object identity. + device = BLEDevice( + macos_uuid, + candidate.name, + details=candidate.details, + ) + now = _freshness_observed_at() + return CapturedDiscoveredDevice( + device=device, + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + scan_generation=0, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + + +async def retrieve_connected_device_capture( + macos_uuid: str, + *, + device_session_id: str, +) -> CapturedDiscoveredDevice | None: + """Explicitly retrieve a process token without claiming presence or GATT. + + The token must match the exact current owner epoch, CoreBluetooth UUID and + device session. Retrieval uses the manager retained by that token on its + owner event loop. Failure leaves the token untouched so another explicit + operator attempt remains possible; there is no scan, connect or retry here. + """ + + global _connected_runtime_handle + + owner_epoch = ble_runtime_owner_epoch_for_current_loop() + runtime = ble_runtime_snapshot() + operation_kind = runtime["active_operation_kind"] + if ( + owner_epoch is None + or not macos_uuid + or not device_session_id + or runtime["owner_epoch"] != owner_epoch + or not runtime["owner_loop_bound"] + or operation_kind not in {"status-read", "wifi-provision", "ap-enable"} + or runtime["poisoned"] + ): + return None + with _runtime_handle_lock: + pinned = _connected_runtime_handle + if ( + pinned is None + or pinned.captured.owner_epoch != owner_epoch + or pinned.captured.macos_uuid.casefold() != macos_uuid.casefold() + or pinned.device_session_id != device_session_id + ): + return None + + retrieved_device = await _retrieve_corebluetooth_device(pinned.captured) + if ( + retrieved_device is None + or retrieved_device.address.casefold() != macos_uuid.casefold() + ): + return None + runtime_after = ble_runtime_snapshot() + if ( + ble_runtime_owner_epoch_for_current_loop() != owner_epoch + or runtime_after["owner_epoch"] != owner_epoch + or not runtime_after["owner_loop_bound"] + or runtime_after["active_operation_kind"] != operation_kind + or runtime_after["poisoned"] + ): + return None + + with _runtime_handle_lock: + current = _connected_runtime_handle + if current is not pinned: + return None + now = _freshness_observed_at() + retrieved_capture = CapturedDiscoveredDevice( + device=retrieved_device, + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + scan_generation=pinned.captured.scan_generation, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-session", + ) + _connected_runtime_handle = replace( + pinned, + captured=retrieved_capture, + retained_at=now, + last_gatt_validated_at=None, + ) + return retrieved_capture + + +def connected_device_recovery_snapshot( + macos_uuid: str | None, + *, + device_session_id: str | None, +) -> ConnectedDeviceRecoverySnapshot: + """Return a non-secret availability fact without exposing BLEDevice state.""" + + runtime = ble_runtime_snapshot() + with _runtime_handle_lock: + now = _freshness_now() + pinned = _connected_runtime_handle + retained_available = bool( + pinned is not None + and macos_uuid is not None + and device_session_id is not None + and pinned.captured.macos_uuid == macos_uuid + and pinned.device_session_id == device_session_id + and pinned.captured.owner_epoch == runtime["owner_epoch"] + and runtime["owner_loop_bound"] + and not runtime["poisoned"] + ) + observed_at = _scan_observation_stamp_locked(now) + advertised_now = bool( + retained_available + and macos_uuid in _runtime_handles + and observed_at is not None + ) + validated_at = ( + pinned.last_gatt_validated_at + if retained_available and pinned is not None + else None + ) + gatt_validated_recently = bool( + validated_at is not None + and _stamp_is_fresh( + validated_at, + now=now, + ttl_seconds=BLE_GATT_VALIDATION_RECENCY_TTL_SECONDS, + ) + ) + return { + # "retained" deliberately does not mean connected, present, or safe + # to mutate. Every mutation still performs a new GATT connect and an + # exact baseline status read before its single write. + "status": "retained" if retained_available else "unavailable", + "scope": "owner-epoch-device-session", + "advertised_now": advertised_now, + "gatt_validated_recently": gatt_validated_recently, + } + + +async def _scan_impl( + duration_seconds: float, + progress: BleOperationProgress, + on_admitted: Callable[[], None] | None, +) -> BleScanResult: + owner_epoch = progress.owner_epoch if duration_seconds <= 0: raise ValueError("duration_seconds must be positive") + if on_admitted is not None: + on_admitted() started_at = utc_now_iso() - scan_generation = _begin_scan_generation() + progress.operation_stage = "discovery" + scan_generation = _begin_scan_generation(owner_epoch) try: discovered = await BleakScanner.discover(timeout=duration_seconds, return_adv=True) handles = {device.address: device for device, _advertisement in discovered.values()} @@ -188,5 +1192,20 @@ async def scan(duration_seconds: float) -> BleScanResult: # A slower, superseded scan may return useful data to its own caller, but # it must never replace the handle lease published by a newer generation. - _publish_scan_handles(scan_generation, handles) + _publish_scan_handles(scan_generation, owner_epoch, handles) return result + + +async def scan( + duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, +) -> BleScanResult: + if duration_seconds <= 0: + raise ValueError("duration_seconds must be positive") + return await run_ble_operation( + "scan", + hard_timeout_seconds=(duration_seconds + BLE_SCAN_HARD_TIMEOUT_GRACE_SECONDS), + operation=lambda progress: _scan_impl(duration_seconds, progress, on_admitted), + progress=BleOperationProgress(operation_stage="discovery"), + ) diff --git a/src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py b/src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py index e678e18..be6bd03 100644 --- a/src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py +++ b/src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py @@ -2,6 +2,8 @@ from __future__ import annotations import asyncio import ipaddress +import math +from collections.abc import Callable from importlib.metadata import version from time import monotonic from typing import Literal, TypedDict @@ -10,8 +12,20 @@ from bleak import BleakClient, BleakScanner from bleak.exc import BleakDeviceNotFoundError, BleakError, BleakGATTProtocolError from k1link.artifacts import utc_now_iso +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationProgress, + run_ble_operation, +) from k1link.device_plugins.xgrids_k1.ble.scanner import ( + CapturedDiscoveredDevice, + captured_device_handle, + connected_device_capture, + demote_connected_device_handle_after_gatt_failure, + discover_known_device_capture_for_status_read, discovered_device_selection, + mark_captured_device_gatt_validated, + retrieve_connected_device_capture, + retrieve_known_device_capture_for_status_read, ) PROFILE_ID = "xgrids-k1-fw3-wifi-v1" @@ -22,6 +36,8 @@ FRAME_LENGTH = 99 SSID_SLOT_LENGTH = 32 PASSWORD_SLOT_LENGTH = 64 AP_FALLBACK_IPV4 = "192.168.56.1" +BLE_STATUS_HARD_TIMEOUT_GRACE_SECONDS = 5.0 +BLE_PROVISION_HARD_TIMEOUT_GRACE_SECONDS = 25.0 ProvisioningOutcome = Literal[ "lan_address_observed", "status_changed", @@ -38,11 +54,28 @@ BleOperationStage = Literal[ "gatt-write", "status-poll", ] +StatusReadOperationStage = Literal[ + "resolution", + "exact-uuid-scan", + "connect", + "gatt-contract", + "status-read", +] +_STATUS_READ_OPERATION_STAGES: frozenset[str] = frozenset( + { + "resolution", + "exact-uuid-scan", + "connect", + "gatt-contract", + "status-read", + } +) class WifiStatus(TypedDict): value_length: int mode: str | None + network_name: str | None ipv4: str | None status_code: int reserved: int | None @@ -70,7 +103,7 @@ class WifiProvisioningResult(TypedDict): operation: str write_mode: ResolvedWriteMode write_without_response_advertised: bool - max_write_without_response_size: int + max_write_without_response_size: int | None frame_length: int baseline_status: WifiStatus observations: list[StatusObservation] @@ -86,6 +119,10 @@ class WifiStatusReadResult(TypedDict): device_macos_uuid: str device_name: str service_uuid: str + write_characteristic_uuid: str + write_characteristic_properties: list[str] + max_write_without_response_size: int | None + mtu_size: int | None status_characteristic_uuid: str operation: Literal["single_reviewed_wifi_status_read"] write_performed: Literal[False] @@ -98,17 +135,61 @@ def _annotate_ble_operation_error( operation_stage: BleOperationStage, device_write_attempted: bool, device_write_confirmed: bool, + resolved_write_mode: ResolvedWriteMode | None, + write_characteristic_properties: tuple[str, ...] | None, + max_write_without_response_size: int | None, + frame_length: int, ) -> None: """Attach non-secret transport facts while preserving the exception type.""" exc.operation_stage = operation_stage # type: ignore[attr-defined] exc.device_write_attempted = device_write_attempted # type: ignore[attr-defined] exc.device_write_confirmed = device_write_confirmed # type: ignore[attr-defined] + exc.resolved_write_mode = resolved_write_mode # type: ignore[attr-defined] + exc.write_characteristic_properties = ( # type: ignore[attr-defined] + write_characteristic_properties + ) + exc.max_write_without_response_size = ( # type: ignore[attr-defined] + max_write_without_response_size + ) + exc.frame_length = frame_length # type: ignore[attr-defined] if isinstance(exc, BleakGATTProtocolError): exc.att_error_code = int(exc.code) # type: ignore[attr-defined] exc.att_error_name = exc.code.name # type: ignore[attr-defined] +def _annotate_status_read_error(exc: Exception, operation_stage: str) -> None: + """Attach only one sanitized read-only stage to a transport failure.""" + + try: + existing_stage = getattr(exc, "operation_stage", None) + except Exception: + existing_stage = None + if existing_stage in _STATUS_READ_OPERATION_STAGES: + return + sanitized_stage = ( + operation_stage + if operation_stage in _STATUS_READ_OPERATION_STAGES + else "resolution" + ) + try: + exc.operation_stage = sanitized_stage # type: ignore[attr-defined] + except Exception: + # A third-party exception may forbid dynamic attributes. Preserve its + # original type and traceback rather than replacing the BLE failure. + return + + +def _optional_int_attribute(source: object, name: str) -> int | None: + """Read optional backend metadata without making diagnostics operationally required.""" + + try: + value = getattr(source, name, None) + except Exception: + return None + return value if isinstance(value, int) and not isinstance(value, bool) else None + + def build_wifi_provisioning_frame(ssid: str, password: str) -> bytearray: """Build the deterministic 99-byte frame used by LixelGO for K1 Wi-Fi setup.""" ssid_bytes = ssid.encode("utf-8") @@ -133,17 +214,40 @@ def build_wifi_provisioning_frame(ssid: str, password: str) -> bytearray: def parse_wifi_status(value: bytes) -> WifiStatus: - """Parse the non-secret status frame returned by the K1 read characteristic.""" + """Parse the status frame returned by the K1 read characteristic. + + The first 32-byte text slot is not a mode enum in station mode. Physical + K1 FW 3.0.2 evidence shows that it contains the joined Wi-Fi network name + (for example a lab SSID), while AP mode uses the control literal + ``WIFI_AP``. Keep ``mode`` as the normalized semantic family so callers do + not have to mistake an operator network name for a protocol enum, and + expose ``network_name`` only for exact, in-process target comparison. + """ if len(value) < 51: raise ValueError("K1 Wi-Fi status must contain at least 51 bytes") - mode_length = value[0] - if mode_length > SSID_SLOT_LENGTH: - raise ValueError("K1 Wi-Fi status mode length is invalid") + text_length = value[0] + if text_length > SSID_SLOT_LENGTH: + raise ValueError("K1 Wi-Fi status text length is invalid") try: - mode = value[1 : 1 + mode_length].decode("utf-8") if mode_length else None + status_text = ( + value[1 : 1 + text_length].decode("utf-8") if text_length else None + ) except UnicodeDecodeError as exc: - raise ValueError("K1 Wi-Fi status mode is not valid UTF-8") from exc + raise ValueError("K1 Wi-Fi status text is not valid UTF-8") from exc + + if status_text == "WIFI_AP": + mode = "WIFI_AP" + network_name = None + elif status_text: + # Older synthetic fixtures and possible legacy firmware may still + # report the literal WIFI_CLIENT. It identifies the station family + # but supplies no exact network discriminator. + mode = "WIFI_CLIENT" + network_name = None if status_text == "WIFI_CLIENT" else status_text + else: + mode = None + network_name = None address_length = value[33] address_start = 34 @@ -163,6 +267,7 @@ def parse_wifi_status(value: bytes) -> WifiStatus: return { "value_length": len(value), "mode": mode, + "network_name": network_name, "ipv4": ipv4, "status_code": value[50], "reserved": value[51] if len(value) > 51 else None, @@ -186,25 +291,95 @@ def _outcome( return "no_status_change_before_timeout" -async def read_wifi_status_once( +async def _read_wifi_status_impl( device_macos_uuid: str, *, - timeout_seconds: float = 20.0, - rediscover: bool = False, + timeout_seconds: float, + exact_scan_timeout_seconds: float, + rediscover: bool, + captured_device: CapturedDiscoveredDevice | None, + recovery_device_session_id: str | None, + allow_known_device_retrieval: bool, + on_gatt_validated: Callable[[CapturedDiscoveredDevice], None] | None, + progress: BleOperationProgress, ) -> WifiStatusReadResult: - """Read the K1's current DHCP status over BLE without writing a characteristic.""" - - if timeout_seconds <= 0: - raise ValueError("timeout_seconds must be positive") - async with asyncio.timeout(timeout_seconds + 5.0): - # A still-live explicit scan lease is authoritative even for a caller - # requesting recovery. Physical acceptance proved that immediately - # looking the same CoreBluetooth UUID up again can lose a present K1. - # ``rediscover`` therefore permits fallback only after that short lease - # has expired; it never discards a fresh retained BLEDevice. - selection = discovered_device_selection(device_macos_uuid) - device = selection.device - if device is None and not selection.from_fresh_scan: + active_captured_device: CapturedDiscoveredDevice | None = None + gatt_baseline_validated = False + try: + progress.operation_stage = "resolution" + if captured_device is not None: + # Explicit recovery is fail-closed: only the exact retrieved + # CoreBluetooth object may be used. Never replace it with a scan, + # UUID lookup, or automatic retry behind the operator's back. + device = ( + captured_device_handle(captured_device) + if captured_device.macos_uuid.casefold() + == device_macos_uuid.casefold() + else None + ) + if device is not None: + active_captured_device = captured_device + selection_from_fresh_scan = False + elif recovery_device_session_id is not None: + # Same-process recovery is resolved inside the status-read BLE + # lease. Retrieval, connect, reviewed GATT contract and 7f02 are + # therefore one serialized operation; no scan row or UUID lookup + # can race between those steps. + active_captured_device = connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + if active_captured_device is None: + active_captured_device = await retrieve_connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + device = ( + captured_device_handle(active_captured_device) + if active_captured_device is not None + else None + ) + selection_from_fresh_scan = False + elif allow_known_device_retrieval: + # Durable physical recovery deliberately bypasses public scan + # generations. ``rediscover`` means one primary, unfiltered + # advertisement wait for this exact CoreBluetooth UUID; it never + # attempts the potentially stale cached peripheral first and never + # falls back to it after a timeout or failed connect. + if rediscover: + progress.operation_stage = "exact-uuid-scan" + active_captured_device = ( + await discover_known_device_capture_for_status_read( + device_macos_uuid, + timeout_seconds=exact_scan_timeout_seconds, + ) + ) + else: + active_captured_device = ( + await retrieve_known_device_capture_for_status_read( + device_macos_uuid, + ) + ) + device = ( + captured_device_handle(active_captured_device) + if active_captured_device is not None + else None + ) + selection_from_fresh_scan = False + else: + # A still-live explicit scan lease is authoritative. Physical + # acceptance proved that immediately looking the same UUID up + # again can lose a present K1. + selection = discovered_device_selection(device_macos_uuid) + device = selection.device + selection_from_fresh_scan = selection.from_fresh_scan + if ( + device is None + and captured_device is None + and recovery_device_session_id is None + and not selection_from_fresh_scan + and not allow_known_device_retrieval + ): device = await BleakScanner.find_device_by_address( device_macos_uuid, timeout=timeout_seconds, @@ -212,23 +387,61 @@ async def read_wifi_status_once( if device is None: raise BleakDeviceNotFoundError( device_macos_uuid, - "Device was not rediscovered; keep the K1 powered and nearby.", + "Exact BLE device is unavailable; run an explicit recovery or scan.", ) + progress.operation_stage = "connect" async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + progress.operation_stage = "gatt-contract" service = client.services.get_service(SERVICE_UUID) - status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID) + write_characteristic = client.services.get_characteristic( + WRITE_CHARACTERISTIC_UUID + ) + status_characteristic = client.services.get_characteristic( + STATUS_CHARACTERISTIC_UUID + ) if service is None: raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}") + if write_characteristic is None: + raise ValueError( + "Reviewed K1 write characteristic not found: " + f"{WRITE_CHARACTERISTIC_UUID}" + ) if status_characteristic is None: raise ValueError( - f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}" + "Reviewed K1 status characteristic not found: " + f"{STATUS_CHARACTERISTIC_UUID}" + ) + if write_characteristic.service_uuid != service.uuid: + raise ValueError( + "K1 write characteristic is attached to an unexpected service" ) if status_characteristic.service_uuid != service.uuid: - raise ValueError("K1 status characteristic is attached to an unexpected service") + raise ValueError( + "K1 status characteristic is attached to an unexpected service" + ) if "read" not in status_characteristic.properties: raise ValueError("Reviewed K1 status characteristic is not readable") + write_properties = sorted( + {str(item) for item in write_characteristic.properties} + ) + max_without_response = _optional_int_attribute( + write_characteristic, + "max_write_without_response_size", + ) + mtu_size = _optional_int_attribute(client, "mtu_size") + progress.operation_stage = "status-read" value = bytes(await client.read_gatt_char(status_characteristic)) + status = parse_wifi_status(value) + if active_captured_device is not None and not ( + mark_captured_device_gatt_validated(active_captured_device) + ): + raise RuntimeError( + "Exact BLE recovery handle changed before status validation" + ) + gatt_baseline_validated = True + if active_captured_device is not None and on_gatt_validated is not None: + on_gatt_validated(active_captured_device) return { "schema_version": 1, "profile_id": PROFILE_ID, @@ -238,11 +451,325 @@ async def read_wifi_status_once( "device_macos_uuid": device_macos_uuid, "device_name": client.name or "", "service_uuid": service.uuid, + "write_characteristic_uuid": write_characteristic.uuid, + "write_characteristic_properties": write_properties, + "max_write_without_response_size": max_without_response, + "mtu_size": mtu_size, "status_characteristic_uuid": status_characteristic.uuid, "operation": "single_reviewed_wifi_status_read", "write_performed": False, - "status": parse_wifi_status(value), + "status": status, } + except Exception as exc: + _annotate_status_read_error(exc, progress.operation_stage) + raise + finally: + # Connection/contract/read failure revokes only this transport lease; + # the process-scoped UUID/session token remains available for another + # explicit CoreBluetooth retrieval attempt. + if active_captured_device is not None and not gatt_baseline_validated: + demote_connected_device_handle_after_gatt_failure(active_captured_device) + + +async def read_wifi_status_once( + device_macos_uuid: str, + *, + timeout_seconds: float = 20.0, + exact_scan_timeout_seconds: float = 30.0, + rediscover: bool = False, + captured_device: CapturedDiscoveredDevice | None = None, + recovery_device_session_id: str | None = None, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[CapturedDiscoveredDevice], None] | None = None, +) -> WifiStatusReadResult: + """Read the K1's current DHCP status over BLE without writing a characteristic.""" + + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + if captured_device is not None and recovery_device_session_id is not None: + raise ValueError( + "captured_device and recovery_device_session_id are mutually exclusive" + ) + if recovery_device_session_id == "": + raise ValueError("recovery_device_session_id must not be empty") + if recovery_device_session_id is not None and allow_known_device_retrieval: + raise ValueError( + "same-process recovery and durable known-device retrieval are mutually exclusive" + ) + use_exact_uuid_scan = bool( + rediscover + and allow_known_device_retrieval + and captured_device is None + and recovery_device_session_id is None + ) + if use_exact_uuid_scan and ( + not math.isfinite(exact_scan_timeout_seconds) + or exact_scan_timeout_seconds <= 0 + ): + raise ValueError("exact_scan_timeout_seconds must be positive and finite") + exact_scan_budget = exact_scan_timeout_seconds if use_exact_uuid_scan else 0.0 + return await run_ble_operation( + "status-read", + hard_timeout_seconds=( + exact_scan_budget + + timeout_seconds + + BLE_STATUS_HARD_TIMEOUT_GRACE_SECONDS + ), + operation=lambda progress: _read_wifi_status_impl( + device_macos_uuid, + timeout_seconds=timeout_seconds, + exact_scan_timeout_seconds=exact_scan_timeout_seconds, + rediscover=rediscover, + captured_device=captured_device, + recovery_device_session_id=recovery_device_session_id, + allow_known_device_retrieval=allow_known_device_retrieval, + on_gatt_validated=on_gatt_validated, + progress=progress, + ), + progress=BleOperationProgress(operation_stage="resolution"), + ) + + +async def _provision_wifi_impl( + device_macos_uuid: str, + ssid: str, + password: str, + *, + timeout_seconds: float, + poll_interval_seconds: float, + write_mode: WriteMode, + captured_device: CapturedDiscoveredDevice | None, + recovery_device_session_id: str | None, + on_write_dispatch: Callable[[WifiStatus, ResolvedWriteMode], None] | None, + progress: BleOperationProgress, +) -> WifiProvisioningResult: + frame = build_wifi_provisioning_frame(ssid, password) + started_at = utc_now_iso() + observations: list[StatusObservation] = [] + disconnected = False + operation_stage: BleOperationStage = "resolution" + device_write_attempted = False + device_write_confirmed = False + resolved_write_mode_for_error: ResolvedWriteMode | None = None + write_characteristic_properties: tuple[str, ...] | None = None + max_without_response: int | None = None + active_captured_device: CapturedDiscoveredDevice | None = None + gatt_baseline_validated = False + + try: + progress.operation_stage = operation_stage + # The explicit UI scan and its selected network action are one + # CoreBluetooth lifecycle. A supplied capture is fail-closed: never + # replace an expired/mismatched object with a scan or UUID lookup. + if captured_device is not None: + device = ( + captured_device_handle(captured_device) + if captured_device.macos_uuid.casefold() + == device_macos_uuid.casefold() + else None + ) + if device is not None: + active_captured_device = captured_device + selection_from_fresh_scan = False + elif recovery_device_session_id is not None: + # Resolve an exact retained UUID/session token only after the + # wifi-provision arbiter lease has been admitted. This prevents a + # concurrent scan/status operation and removes the preflight-to- + # write lease gap. + active_captured_device = connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + if active_captured_device is None: + active_captured_device = await retrieve_connected_device_capture( + device_macos_uuid, + device_session_id=recovery_device_session_id, + ) + device = ( + captured_device_handle(active_captured_device) + if active_captured_device is not None + else None + ) + selection_from_fresh_scan = False + else: + selection = discovered_device_selection(device_macos_uuid) + device = selection.device + selection_from_fresh_scan = selection.from_fresh_scan + if ( + device is None + and captured_device is None + and recovery_device_session_id is None + and not selection_from_fresh_scan + ): + # Non-UI callers without a current explicit scan retain the + # bounded lookup fallback. A fresh scan missing this device is + # authoritative and must not be silently replaced here. + device = await BleakScanner.find_device_by_address( + device_macos_uuid, + timeout=min(20.0, timeout_seconds), + ) + if device is None: + raise BleakDeviceNotFoundError( + device_macos_uuid, + "Device was not rediscovered; keep the K1 powered and nearby.", + ) + + operation_stage = "connect" + progress.operation_stage = operation_stage + async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: + device_name = client.name + operation_stage = "gatt-contract" + progress.operation_stage = operation_stage + service = client.services.get_service(SERVICE_UUID) + write_characteristic = client.services.get_characteristic(WRITE_CHARACTERISTIC_UUID) + status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID) + if service is None: + raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}") + if write_characteristic is None: + raise ValueError( + f"Reviewed K1 write characteristic not found: {WRITE_CHARACTERISTIC_UUID}" + ) + if status_characteristic is None: + raise ValueError( + f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}" + ) + if write_characteristic.service_uuid != service.uuid: + raise ValueError("K1 write characteristic is attached to an unexpected service") + if status_characteristic.service_uuid != service.uuid: + raise ValueError("K1 status characteristic is attached to an unexpected service") + if "read" not in status_characteristic.properties: + raise ValueError("Reviewed K1 status characteristic is not readable") + + properties = set(write_characteristic.properties) + write_characteristic_properties = tuple(sorted(properties)) + max_without_response = _optional_int_attribute( + write_characteristic, + "max_write_without_response_size", + ) + resolved_write_mode: ResolvedWriteMode + if write_mode == "auto": + if "write-without-response" in properties: + resolved_write_mode = "without_response" + elif "write" in properties: + resolved_write_mode = "with_response" + else: + raise ValueError("Reviewed K1 characteristic is not writable") + elif write_mode == "with_response": + if "write" not in properties: + raise ValueError( + "Reviewed K1 characteristic does not advertise writes with response" + ) + resolved_write_mode = "with_response" + else: + resolved_write_mode = "without_response" + + resolved_write_mode_for_error = resolved_write_mode + if resolved_write_mode == "without_response": + if max_without_response is None: + raise ValueError("Negotiated write-without-response size is unavailable") + if len(frame) > max_without_response: + raise ValueError( + "Provisioning frame exceeds the negotiated write-without-response size" + ) + + operation_stage = "baseline-read" + progress.operation_stage = operation_stage + baseline_value = bytes(await client.read_gatt_char(status_characteristic)) + baseline = parse_wifi_status(baseline_value) + if active_captured_device is not None and not ( + mark_captured_device_gatt_validated(active_captured_device) + ): + raise RuntimeError( + "Exact BLE recovery handle changed before provisioning validation" + ) + gatt_baseline_validated = True + + operation_stage = "gatt-write" + # This callback is the durable side-effect boundary. It must finish + # before CoreBluetooth receives the frame, so a process crash can + # only create a conservative false-positive fence, never an unsafe + # forgotten write. It receives no SSID, password, or frame bytes. + if on_write_dispatch is not None: + on_write_dispatch(baseline, resolved_write_mode) + device_write_attempted = True + progress.operation_stage = operation_stage + progress.device_write_attempted = True + await client.write_gatt_char( + write_characteristic, + frame, + response=resolved_write_mode == "with_response", + ) + device_write_confirmed = resolved_write_mode == "with_response" + progress.device_write_confirmed = device_write_confirmed + write_completed = monotonic() + deadline = write_completed + timeout_seconds + + operation_stage = "status-poll" + progress.operation_stage = operation_stage + while monotonic() < deadline: + try: + value = bytes(await client.read_gatt_char(status_characteristic)) + except BleakError: + if not client.is_connected: + disconnected = True + break + raise + status = parse_wifi_status(value) + observation: StatusObservation = { + "observed_at_utc": utc_now_iso(), + "seconds_after_write": round(monotonic() - write_completed, 3), + "status": status, + } + if not observations or status != observations[-1]["status"]: + observations.append(observation) + if status["ipv4"] not in (None, AP_FALLBACK_IPV4): + break + await asyncio.sleep(poll_interval_seconds) + + return { + "schema_version": 1, + "profile_id": PROFILE_ID, + "started_at_utc": started_at, + "completed_at_utc": utc_now_iso(), + "adapter": "CoreBluetooth", + "bleak_version": version("bleak"), + "device_macos_uuid": device_macos_uuid, + "device_name": device_name, + "service_uuid": service.uuid, + "write_characteristic_uuid": write_characteristic.uuid, + "status_characteristic_uuid": status_characteristic.uuid, + "operation": "single_reviewed_wifi_provisioning_write", + "write_mode": resolved_write_mode, + "write_without_response_advertised": ("write-without-response" in properties), + "max_write_without_response_size": max_without_response, + "frame_length": len(frame), + "baseline_status": baseline, + "observations": observations, + "outcome": _outcome(baseline, observations, disconnected), + } + except Exception as exc: + if active_captured_device is not None and not gatt_baseline_validated: + demote_connected_device_handle_after_gatt_failure(active_captured_device) + _annotate_ble_operation_error( + exc, + operation_stage=operation_stage, + device_write_attempted=device_write_attempted, + device_write_confirmed=device_write_confirmed, + resolved_write_mode=resolved_write_mode_for_error, + write_characteristic_properties=write_characteristic_properties, + max_write_without_response_size=max_without_response, + frame_length=len(frame), + ) + raise + finally: + # Hard-timeout cancellation may bypass ``except Exception``. The + # failed live transport lease must still be demoted when the exact + # captured object never completed the reviewed baseline read; the + # UUID/session recovery token itself remains available for a later + # explicit attempt. + if active_captured_device is not None and not gatt_baseline_validated: + demote_connected_device_handle_after_gatt_failure(active_captured_device) + frame[:] = b"\x00" * len(frame) async def provision_wifi_once( @@ -252,160 +779,40 @@ async def provision_wifi_once( timeout_seconds: float = 45.0, poll_interval_seconds: float = 1.0, write_mode: WriteMode = "auto", + *, + captured_device: CapturedDiscoveredDevice | None = None, + recovery_device_session_id: str | None = None, + on_write_dispatch: Callable[[WifiStatus, ResolvedWriteMode], None] | None = None, ) -> WifiProvisioningResult: """Perform one reviewed provisioning write and poll the K1 status characteristic.""" + if timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") if poll_interval_seconds <= 0: raise ValueError("poll_interval_seconds must be positive") if write_mode not in ("auto", "with_response", "without_response"): raise ValueError(f"Unsupported write mode: {write_mode}") - - frame = build_wifi_provisioning_frame(ssid, password) - started_at = utc_now_iso() - observations: list[StatusObservation] = [] - disconnected = False - operation_stage: BleOperationStage = "resolution" - device_write_attempted = False - device_write_confirmed = False - - try: - async with asyncio.timeout(timeout_seconds + 25.0): - # The explicit UI scan and its selected network action are one - # CoreBluetooth lifecycle. Physical acceptance proved that a - # second UUID lookup can fail moments after a successful scan, so - # use the exact retained handle while its short lease is fresh. - selection = discovered_device_selection(device_macos_uuid) - device = selection.device - if device is None and not selection.from_fresh_scan: - # Non-UI callers without a current explicit scan retain the - # bounded lookup fallback. A fresh scan missing this device is - # authoritative and must not be silently replaced here. - device = await BleakScanner.find_device_by_address( - device_macos_uuid, - timeout=min(20.0, timeout_seconds), - ) - if device is None: - raise BleakDeviceNotFoundError( - device_macos_uuid, - "Device was not rediscovered; keep the K1 powered and nearby.", - ) - - operation_stage = "connect" - async with BleakClient(device, timeout=timeout_seconds, pair=False) as client: - device_name = client.name - operation_stage = "gatt-contract" - service = client.services.get_service(SERVICE_UUID) - write_characteristic = client.services.get_characteristic(WRITE_CHARACTERISTIC_UUID) - status_characteristic = client.services.get_characteristic( - STATUS_CHARACTERISTIC_UUID - ) - if service is None: - raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}") - if write_characteristic is None: - raise ValueError( - f"Reviewed K1 write characteristic not found: {WRITE_CHARACTERISTIC_UUID}" - ) - if status_characteristic is None: - raise ValueError( - f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}" - ) - if write_characteristic.service_uuid != service.uuid: - raise ValueError("K1 write characteristic is attached to an unexpected service") - if status_characteristic.service_uuid != service.uuid: - raise ValueError( - "K1 status characteristic is attached to an unexpected service" - ) - if "read" not in status_characteristic.properties: - raise ValueError("Reviewed K1 status characteristic is not readable") - - properties = set(write_characteristic.properties) - max_without_response = write_characteristic.max_write_without_response_size - resolved_write_mode: ResolvedWriteMode - if write_mode == "auto": - if "write-without-response" in properties: - resolved_write_mode = "without_response" - elif "write" in properties: - resolved_write_mode = "with_response" - else: - raise ValueError("Reviewed K1 characteristic is not writable") - elif write_mode == "with_response": - if "write" not in properties: - raise ValueError( - "Reviewed K1 characteristic does not advertise writes with response" - ) - resolved_write_mode = "with_response" - else: - if len(frame) > max_without_response: - raise ValueError( - "Provisioning frame exceeds the negotiated write-without-response size" - ) - resolved_write_mode = "without_response" - - operation_stage = "baseline-read" - baseline_value = bytes(await client.read_gatt_char(status_characteristic)) - baseline = parse_wifi_status(baseline_value) - - operation_stage = "gatt-write" - device_write_attempted = True - await client.write_gatt_char( - write_characteristic, - frame, - response=resolved_write_mode == "with_response", - ) - device_write_confirmed = resolved_write_mode == "with_response" - write_completed = monotonic() - deadline = write_completed + timeout_seconds - - operation_stage = "status-poll" - while monotonic() < deadline: - try: - value = bytes(await client.read_gatt_char(status_characteristic)) - except BleakError: - if not client.is_connected: - disconnected = True - break - raise - status = parse_wifi_status(value) - observation: StatusObservation = { - "observed_at_utc": utc_now_iso(), - "seconds_after_write": round(monotonic() - write_completed, 3), - "status": status, - } - if not observations or status != observations[-1]["status"]: - observations.append(observation) - if status["ipv4"] not in (None, AP_FALLBACK_IPV4): - break - await asyncio.sleep(poll_interval_seconds) - - return { - "schema_version": 1, - "profile_id": PROFILE_ID, - "started_at_utc": started_at, - "completed_at_utc": utc_now_iso(), - "adapter": "CoreBluetooth", - "bleak_version": version("bleak"), - "device_macos_uuid": device_macos_uuid, - "device_name": device_name, - "service_uuid": service.uuid, - "write_characteristic_uuid": write_characteristic.uuid, - "status_characteristic_uuid": status_characteristic.uuid, - "operation": "single_reviewed_wifi_provisioning_write", - "write_mode": resolved_write_mode, - "write_without_response_advertised": ("write-without-response" in properties), - "max_write_without_response_size": max_without_response, - "frame_length": len(frame), - "baseline_status": baseline, - "observations": observations, - "outcome": _outcome(baseline, observations, disconnected), - } - except Exception as exc: - _annotate_ble_operation_error( - exc, - operation_stage=operation_stage, - device_write_attempted=device_write_attempted, - device_write_confirmed=device_write_confirmed, + if captured_device is not None and recovery_device_session_id is not None: + raise ValueError( + "captured_device and recovery_device_session_id are mutually exclusive" ) - raise - finally: - frame[:] = b"\x00" * len(frame) + if recovery_device_session_id == "": + raise ValueError("recovery_device_session_id must not be empty") + progress = BleOperationProgress(operation_stage="resolution") + return await run_ble_operation( + "wifi-provision", + hard_timeout_seconds=(timeout_seconds + BLE_PROVISION_HARD_TIMEOUT_GRACE_SECONDS), + operation=lambda operation_progress: _provision_wifi_impl( + device_macos_uuid, + ssid, + password, + timeout_seconds=timeout_seconds, + poll_interval_seconds=poll_interval_seconds, + write_mode=write_mode, + captured_device=captured_device, + recovery_device_session_id=recovery_device_session_id, + on_write_dispatch=on_write_dispatch, + progress=operation_progress, + ), + progress=progress, + ) diff --git a/src/k1link/device_plugins/xgrids_k1/camera.py b/src/k1link/device_plugins/xgrids_k1/camera.py index d1cf518..9a9f4c8 100644 --- a/src/k1link/device_plugins/xgrids_k1/camera.py +++ b/src/k1link/device_plugins/xgrids_k1/camera.py @@ -6,8 +6,9 @@ import queue import signal import subprocess import threading +import time from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import suppress from dataclasses import dataclass, field from pathlib import Path @@ -17,6 +18,7 @@ from fastapi import APIRouter, WebSocket, WebSocketDisconnect from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4 from k1link.web.camera_archive import ( + CAMERA_ARCHIVE_SCHEMA, CameraArchiveError, CameraArchiveKind, CameraArchiveStatus, @@ -36,9 +38,162 @@ CAMERA_SOURCE_LABELS: Final[dict[CameraSourceId, str]] = { CAMERA_MEDIA_TYPE: Final = 'video/mp4; codecs="avc1.641028"' CAMERA_EXCLUSIVE_GROUP: Final = "camera.preview.decoder" MAX_FMP4_BOX_BYTES: Final = 8 * 1024 * 1024 -MAX_FMP4_SEGMENT_BYTES: Final = 1024 * 1024 -MAX_QUEUED_SEGMENTS: Final = 4 +# The archive recovery boundary already admits an 8 MiB fragment. Keep the +# live parser on that same reviewed envelope: a complete moof..mdat fragment at +# or below the limit is durable evidence; a larger one is rejected before it +# can be archived or exposed to a browser. +MAX_FMP4_SEGMENT_BYTES: Final = 8 * 1024 * 1024 +MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS: Final = 64 +# One reader must be able to hold the cached init plus one maximum-sized valid +# media fragment. This remains a strict per-reader bound and matches the +# frontend's reviewed 12 MiB receive envelope. +MAX_CAMERA_PREVIEW_QUEUED_BYTES: Final = 12 * 1024 * 1024 +MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: Final = 3.0 +MAX_CAMERA_PREVIEW_CONSUMERS: Final = 8 +CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS: Final = 3.0 CAMERA_DRAIN_TIMEOUT_SECONDS: Final = 5.0 +# K1 reports SCANNING before its RTSP encoder is necessarily ready to emit the +# first SPS/PPS/IDR group. LixelGO's manual camera selection naturally happens +# after that gap; Mission Core selects the right camera immediately. Keep one +# RTSP attempt, but allow the device's camera pipeline to finish starting. +CAMERA_SOURCE_IO_TIMEOUT_MICROSECONDS: Final = 30_000_000 +CAMERA_PRODUCER_STALL_MILLISECONDS: Final = 10_000 +CAMERA_PRODUCER_WATCHDOG_INTERVAL_SECONDS: Final = 1.0 +CAMERA_TRANSPORT_RECOVERY_ERROR_CODES: Final = frozenset( + {"camera-source-ended", "incomplete-fmp4-fragment"} +) + + +def classify_camera_recording_health( + camera: Mapping[str, Any], + *, + stall_milliseconds: int = CAMERA_PRODUCER_STALL_MILLISECONDS, +) -> str | None: + """Classify only exact acquisition-camera transport stalls. + + A process and a newly allocated archive epoch are not media evidence. The + connecting states are split by durable init/media commits so recovery can + distinguish an RTSP/codec bootstrap stall from a post-init IDR/media stall. + Storage and parser-integrity failures remain terminal and are never + disguised as reconnectable transport loss. + """ + + if stall_milliseconds < 1: + raise ValueError("camera stall threshold must be positive") + recording = camera.get("recording") + if not isinstance(recording, Mapping) or recording.get("active") is not True: + return None + if camera.get("phase") == "error": + error = camera.get("error") + code = error.get("code") if isinstance(error, Mapping) else None + return str(code) if code in CAMERA_TRANSPORT_RECOVERY_ERROR_CODES else None + + current_epoch = recording.get("current_epoch") + if camera.get("phase") == "connecting": + producer_age_ms = recording.get("producer_age_ms") + if not ( + isinstance(producer_age_ms, int) + and not isinstance(producer_age_ms, bool) + and producer_age_ms >= stall_milliseconds + ): + return None + if not isinstance(current_epoch, Mapping) or current_epoch.get( + "init_committed" + ) is not True: + return "camera-connecting-no-init-stalled" + init_committed_age_ms = current_epoch.get("init_committed_age_ms") + if ( + current_epoch.get("first_media_committed") is not True + and isinstance(init_committed_age_ms, int) + and not isinstance(init_committed_age_ms, bool) + and init_committed_age_ms >= stall_milliseconds + ): + return "camera-connecting-no-media-stalled" + return None + last_media_age_ms = recording.get("last_media_segment_age_ms") + if last_media_age_ms is None: + # Compatibility for a retained/synthetic v1alpha1 snapshot. Once the + # phase is streaming, the legacy last-segment clock is necessarily a + # media commit because init alone never enters that phase. + last_media_age_ms = recording.get("last_segment_age_ms") + if ( + camera.get("phase") == "streaming" + and isinstance(last_media_age_ms, int) + and not isinstance(last_media_age_ms, bool) + and last_media_age_ms >= stall_milliseconds + ): + return "camera-stream-stalled" + return None + + +class _CameraPreviewSegmentQueue: + """A byte- and fragment-bounded handoff for one disposable browser. + + A K1 camera normally emits about ten fragments per second. A four-item + ``queue.Queue`` therefore treated an ordinary sub-second browser/event-loop + pause as a dead consumer and closed an otherwise healthy preview. Keep a + few seconds of bounded headroom without ever back-pressuring the archive + producer. Once either bound is reached the owning lease is still dropped; + the browser can attach a fresh reader and receive the cached init segment. + """ + + def __init__(self, *, clock: Callable[[], float] = time.monotonic) -> None: + self._condition = threading.Condition() + self._segments: deque[tuple[float, tuple[str, bytes]]] = deque() + self._queued_bytes = 0 + self._closed = False + self._clock = clock + + @property + def queued_bytes(self) -> int: + with self._condition: + return self._queued_bytes + + @property + def queued_segments(self) -> int: + with self._condition: + return len(self._segments) + + def offer(self, segment: tuple[str, bytes]) -> bool: + payload_size = len(segment[1]) + with self._condition: + if self._closed: + return False + now = self._clock() + if self._segments and now - self._segments[0][0] > MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: + return False + if len(self._segments) >= MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS: + return False + if self._queued_bytes + payload_size > MAX_CAMERA_PREVIEW_QUEUED_BYTES: + return False + self._segments.append((now, segment)) + self._queued_bytes += payload_size + self._condition.notify() + return True + + def get(self, timeout: float | None = None) -> tuple[str, bytes] | None: + deadline = None if timeout is None else time.monotonic() + timeout + with self._condition: + while not self._segments and not self._closed: + if deadline is None: + self._condition.wait() + continue + remaining = deadline - time.monotonic() + if remaining <= 0: + raise queue.Empty + self._condition.wait(remaining) + if self._segments: + _, segment = self._segments.popleft() + self._queued_bytes -= len(segment[1]) + return segment + return None + + def close(self) -> None: + with self._condition: + self._segments.clear() + self._queued_bytes = 0 + self._closed = True + self._condition.notify_all() @dataclass @@ -47,9 +202,7 @@ class CameraProcessLease: source_id: CameraSourceId process: subprocess.Popen[bytes] stderr_tail: deque[str] - segments: queue.Queue[tuple[str, bytes] | None] = field( - default_factory=lambda: queue.Queue(maxsize=MAX_QUEUED_SEGMENTS) - ) + segments: _CameraPreviewSegmentQueue = field(default_factory=_CameraPreviewSegmentQueue) failure_code: str | None = None @@ -67,6 +220,10 @@ class CommittedCameraSegment: CommittedCameraSegmentObserver = Callable[[CommittedCameraSegment], None] +CameraProducerStallObserver = Callable[[str, int], None] +CameraProducerCommitFence = Callable[[Callable[[], bool]], bool] +CameraProducerPreDetachFence = Callable[[Callable[[], bool]], bool] +CameraProducerCommittedObserver = Callable[[Mapping[str, Any]], None] @dataclass @@ -76,13 +233,29 @@ class _CameraProducer: process: subprocess.Popen[bytes] archive: CameraArchiveWriter | None stderr_tail: deque[str] = field(default_factory=lambda: deque(maxlen=12)) - delivery: CameraProcessLease | None = None + # Browser windows are disposable readers of one acquisition-owned producer. + # Keep one bounded queue per reader so a stale tab, React replacement mount, + # or slow decoder cannot monopolize the only live camera lease. + deliveries: dict[int, CameraProcessLease] = field(default_factory=dict) init_segment: bytes | None = None failure_code: str | None = None stop_requested: bool = False drain_requested: bool = False reader_started: bool = False reader_done: threading.Event = field(default_factory=threading.Event) + started_monotonic: float = field(default_factory=time.monotonic) + last_segment_monotonic: float | None = None + init_committed_monotonic: float | None = None + first_media_committed_monotonic: float | None = None + last_media_committed_monotonic: float | None = None + committed_media_segment_count: int = 0 + + +@dataclass(frozen=True, slots=True) +class _PreparedCameraProducer: + producer: _CameraProducer + target_host: str + recording_root: Path | None class XgridsK1CameraGateway: @@ -101,7 +274,17 @@ class XgridsK1CameraGateway: plugin_id: str, *, committed_segment_observer: CommittedCameraSegmentObserver | None = None, + process_fence_descriptor_factory: Callable[[], int] | None = None, + producer_stall_observer: CameraProducerStallObserver | None = None, + producer_stall_milliseconds: int = CAMERA_PRODUCER_STALL_MILLISECONDS, + producer_watchdog_interval_seconds: float = ( + CAMERA_PRODUCER_WATCHDOG_INTERVAL_SECONDS + ), ) -> None: + if producer_stall_milliseconds < 1: + raise ValueError("camera stall threshold must be positive") + if not 0.01 <= producer_watchdog_interval_seconds <= 60.0: + raise ValueError("camera watchdog interval must be within 0.01..60 seconds") self._repository_root = repository_root.resolve() self._plugin_id = plugin_id self._lock = threading.RLock() @@ -113,16 +296,76 @@ class XgridsK1CameraGateway: self._target_host: str | None = None self._producer: _CameraProducer | None = None self._recording_root: Path | None = None + # Acquisition-session lifetime proof. It deliberately survives an + # FFmpeg epoch detach/retry so the transient gap before archive + # finalization cannot erase whether any real frame was committed. + self._recording_media_segment_count = 0 self._expected_source_end_generation: int | None = None self._archive_summaries: list[dict[str, Any]] = [] - self._error: dict[str, str] | None = None + self._error: dict[str, Any] | None = None self._closed = False self._committed_segment_observer = committed_segment_observer + self._process_fence_descriptor_factory = process_fence_descriptor_factory + self._producer_stall_observer = producer_stall_observer + self._producer_stall_milliseconds = producer_stall_milliseconds + self._producer_watchdog_interval_seconds = producer_watchdog_interval_seconds self._committed_segment_observer_errors = 0 self._ffmpeg_path, self._ffmpeg_source = _resolve_ffmpeg(self._repository_root) def snapshot(self) -> dict[str, Any]: with self._lock: + now = time.monotonic() + producer = self._producer + producer_alive = producer is not None and producer.process.poll() is None + producer_age_ms = ( + int(max(now - producer.started_monotonic, 0.0) * 1000) + if producer is not None + else None + ) + last_segment_age_ms = ( + int(max(now - producer.last_segment_monotonic, 0.0) * 1000) + if producer is not None and producer.last_segment_monotonic is not None + else None + ) + last_media_segment_age_ms = ( + int(max(now - producer.last_media_committed_monotonic, 0.0) * 1000) + if producer is not None + and producer.last_media_committed_monotonic is not None + else None + ) + current_epoch = ( + { + "generation": producer.generation, + "init_committed": producer.init_committed_monotonic is not None, + "init_committed_age_ms": ( + int(max(now - producer.init_committed_monotonic, 0.0) * 1000) + if producer.init_committed_monotonic is not None + else None + ), + "first_media_committed": ( + producer.first_media_committed_monotonic is not None + ), + "first_media_committed_age_ms": ( + int( + max(now - producer.first_media_committed_monotonic, 0.0) + * 1000 + ) + if producer.first_media_committed_monotonic is not None + else None + ), + "committed_media_segment_count": ( + producer.committed_media_segment_count + ), + "last_media_segment_age_ms": last_media_segment_age_ms, + } + if producer is not None and producer.archive is not None + else None + ) + media_ready = bool( + isinstance(current_epoch, Mapping) + and current_epoch["init_committed"] is True + and current_epoch["first_media_committed"] is True + ) delivery = None if ( self._source_id is not None @@ -150,17 +393,29 @@ class XgridsK1CameraGateway: }, "recording": { "active": self._recording_root is not None, - "source_end_expected": ( - self._expected_source_end_generation is not None - ), + "source_end_expected": (self._expected_source_end_generation is not None), "session": ( self._recording_root.name if self._recording_root is not None else None ), "active_epoch": ( - self._producer.generation - if self._producer is not None and self._producer.archive is not None + producer.generation + if producer is not None and producer.archive is not None else None ), + "committed_media_segment_count": ( + self._recording_media_segment_count + if self._recording_root is not None + else None + ), + "producer_alive": producer_alive, + "producer_age_ms": producer_age_ms, + "last_segment_age_ms": last_segment_age_ms, + "last_media_segment_age_ms": last_media_segment_age_ms, + "current_epoch": current_epoch, + "media_ready": media_ready, + "preview_consumer_count": ( + len(producer.deliveries) if producer is not None else 0 + ), "completed_epochs": len(self._archive_summaries), "last_summary": ( dict(self._archive_summaries[-1]) if self._archive_summaries else None @@ -200,7 +455,7 @@ class XgridsK1CameraGateway: ): return self.snapshot() - old_producer, old_delivery = self._detach_producer_locked() + old_producer, old_deliveries = self._detach_producer_locked() self._generation += 1 self._revision += 1 self._source_id = source_id @@ -212,7 +467,7 @@ class XgridsK1CameraGateway: self._shutdown_producer( old_producer, - old_delivery, + old_deliveries, status="complete", failure_code="source-switch", ) @@ -220,6 +475,222 @@ class XgridsK1CameraGateway: self._spawn_selected_producer() return self.snapshot() + def restart_recording_producer( + self, + source_id: CameraSourceId, + target_host: str, + *, + expected_generation: int, + expected_recording_session: str, + expected_active_epoch: int | None, + expected_recording_media_segment_count: int, + pre_detach_fence: CameraProducerPreDetachFence, + commit_fence: CameraProducerCommitFence | None = None, + committed_before_start: CameraProducerCommittedObserver | None = None, + ) -> dict[str, Any]: + """CAS-restart one exact acquisition-owned local FFmpeg epoch. + + This method preserves the selected source, target and recording root. + It never contacts the K1 control plane. The old archive epoch is sealed + as interrupted before a new generation is spawned, and a stale caller + cannot churn a producer already replaced by another recovery owner. + """ + + if source_id not in CAMERA_SOURCE_PATHS: + raise ValueError("неизвестный camera source") + target = validate_private_ipv4(target_host) + if expected_generation < 1: + raise ValueError("camera generation устарело") + if not expected_recording_session: + raise ValueError("camera recording session отсутствует") + if ( + isinstance(expected_recording_media_segment_count, bool) + or expected_recording_media_segment_count < 0 + ): + raise ValueError("camera recovery media progress устарел") + reservation: list[ + tuple[_CameraProducer | None, tuple[CameraProcessLease, ...]] + ] = [] + + def reserve_and_detach() -> bool: + # The facade invokes this closure while owning camera priority and + # the acquisition-lineage gate. The closure then acquires the + # camera lifecycle in the canonical outer->inner order. The + # callback itself is invoked with no camera lock held, avoiding a + # camera->facade inversion. + with self._lifecycle_lock: # noqa: SIM117 - lock order is intentional + with self._lock: + self._require_open_locked() + recording_root = self._recording_root + producer = self._producer + active_epoch = ( + producer.generation + if producer is not None and producer.archive is not None + else None + ) + sealed_transport_end = bool( + expected_active_epoch is None + and producer is None + and self._sealed_transport_ending_epoch_matches_locked( + source_id=source_id, + expected_generation=expected_generation, + ) + ) + if ( + self._source_id != source_id + or self._target_host != target + or self._generation != expected_generation + or recording_root is None + or recording_root.name != expected_recording_session + or active_epoch != expected_active_epoch + or (expected_active_epoch is None and not sealed_transport_end) + # Durable media may advance while a watchdog callback + # waits to enter this lifecycle lock. Generation/epoch + # alone do not change on that edge, so the session + # commit counter is the exact progress CAS. + or self._recording_media_segment_count + != expected_recording_media_segment_count + ): + raise ValueError("camera recovery lineage устарела") + old_producer, old_deliveries = self._detach_producer_locked() + self._generation += 1 + self._revision += 1 + self._expected_source_end_generation = None + self._phase = "selected" + self._error = None + reservation.append((old_producer, old_deliveries)) + return True + + # No detach, generation, error, archive-summary or file mutation may + # precede this outer authority fence. A denied STOP-first reservation + # therefore leaves both memory and durable epoch bytes untouched. + if not pre_detach_fence(reserve_and_detach): + raise ValueError("camera recovery lineage устарела") + if len(reservation) != 1: + raise RuntimeError("camera recovery reservation did not detach exactly once") + old_producer, old_deliveries = reservation[0] + + self._shutdown_producer( + old_producer, + old_deliveries, + status="interrupted", + failure_code="active-stream-connection-recovery", + ) + prepared = self._prepare_selected_producer() + + def commit() -> bool: + with self._lifecycle_lock: + if not self._commit_prepared_producer(prepared): + return False + committed_snapshot = self.snapshot() + if committed_before_start is not None: + committed_before_start(committed_snapshot) + with self._lifecycle_lock: + if self._producer is not prepared.producer: + return False + self._start_producer_threads(prepared.producer) + return True + + try: + committed = commit_fence(commit) if commit_fence is not None else commit() + except BaseException: + self._discard_prepared_producer(prepared, failure_code="restart-commit-failed") + raise + if not committed: + self._discard_prepared_producer(prepared, failure_code="stale-restart-commit") + raise ValueError("camera recovery lineage устарела") + return self.snapshot() + + def _sealed_transport_ending_epoch_matches_locked( + self, + *, + source_id: CameraSourceId, + expected_generation: int, + ) -> bool: + """Prove ``active_epoch=None`` is one exact finalized transport end.""" + + error = self._error + summary = self._archive_summaries[-1] if self._archive_summaries else None + error_code = error.get("code") if isinstance(error, Mapping) else None + return bool( + self._recording_root is not None + and self._producer is None + and self._phase == "error" + and error_code == "camera-source-ended" + and isinstance(summary, Mapping) + and summary.get("schema_version") == CAMERA_ARCHIVE_SCHEMA + and summary.get("source_id") == source_id + and summary.get("codec_epoch") == expected_generation + and summary.get("status") == "interrupted" + and summary.get("failure_code") == error_code + ) + + def activate_recording_producer( + self, + source_id: CameraSourceId, + target_host: str, + session_dir: Path, + *, + commit_fence: CameraProducerCommitFence, + committed_before_start: CameraProducerCommittedObserver | None = None, + ) -> dict[str, Any]: + """Prepare the first acquisition camera outside every shared gate.""" + + if source_id not in CAMERA_SOURCE_PATHS: + raise ValueError("неизвестный camera source") + target = validate_private_ipv4(target_host) + root = session_dir.expanduser().resolve() + if not root.is_dir(): + raise ValueError("observation session directory does not exist") + if not root.is_relative_to(self._repository_root): + raise ValueError("camera recording root must stay inside the repository") + with self._lifecycle_lock: # noqa: SIM117 - lock order is intentional + with self._lock: + self._require_open_locked() + if self._ffmpeg_path is None: + raise RuntimeError("локальный camera adapter FFmpeg не найден") + if self._producer is not None or self._recording_root is not None: + raise RuntimeError("camera acquisition producer уже активен") + if self._source_id is not None and ( + self._source_id != source_id or self._target_host != target + ): + raise RuntimeError("для camera gateway уже выбран другой source") + if self._source_id is None: + self._generation += 1 + self._source_id = source_id + self._target_host = target + self._recording_root = root + self._recording_media_segment_count = 0 + self._archive_summaries = [] + self._expected_source_end_generation = None + self._phase = "selected" + self._error = None + self._revision += 1 + prepared = self._prepare_selected_producer() + + def commit() -> bool: + with self._lifecycle_lock: + if not self._commit_prepared_producer(prepared): + return False + committed_snapshot = self.snapshot() + if committed_before_start is not None: + committed_before_start(committed_snapshot) + with self._lifecycle_lock: + if self._producer is not prepared.producer: + return False + self._start_producer_threads(prepared.producer) + return True + + try: + committed = commit_fence(commit) + except BaseException: + self._discard_prepared_producer(prepared, failure_code="activation-commit-failed") + raise + if not committed: + self._discard_prepared_producer(prepared, failure_code="stale-activation-commit") + raise ValueError("camera activation lineage устарела") + return self.snapshot() + def stop(self, generation: int) -> dict[str, Any]: with self._lifecycle_lock: with self._lock: @@ -227,7 +698,7 @@ class XgridsK1CameraGateway: return self.snapshot() if generation != self._generation: raise ValueError("camera preview generation устарело") - producer, delivery = self._detach_producer_locked() + producer, deliveries = self._detach_producer_locked() self._revision += 1 self._phase = "idle" self._source_id = None @@ -235,7 +706,7 @@ class XgridsK1CameraGateway: self._error = None self._shutdown_producer( producer, - delivery, + deliveries, status="complete", failure_code="source-stopped", ) @@ -244,7 +715,7 @@ class XgridsK1CameraGateway: def stop_current(self) -> dict[str, Any]: with self._lifecycle_lock: with self._lock: - producer, delivery = self._detach_producer_locked() + producer, deliveries = self._detach_producer_locked() changed = self._source_id is not None or self._phase != "idle" if changed: self._revision += 1 @@ -256,7 +727,7 @@ class XgridsK1CameraGateway: self._error = None self._shutdown_producer( producer, - delivery, + deliveries, status="interrupted", failure_code="gateway-stop", ) @@ -278,8 +749,9 @@ class XgridsK1CameraGateway: if self._recording_root == root: return self.snapshot() raise RuntimeError("для camera gateway уже активна другая acquisition-сессия") - producer, delivery = self._detach_producer_locked() + producer, deliveries = self._detach_producer_locked() self._recording_root = root + self._recording_media_segment_count = 0 self._expected_source_end_generation = None self._archive_summaries = [] selected = self._source_id is not None @@ -291,7 +763,7 @@ class XgridsK1CameraGateway: # retrospectively. Restart it at a clean codec epoch instead. self._shutdown_producer( producer, - delivery, + deliveries, status="interrupted", failure_code="recording-start-restart", ) @@ -299,6 +771,83 @@ class XgridsK1CameraGateway: self._spawn_selected_producer() return self.snapshot() + def retry_recording_producer( + self, + source_id: CameraSourceId, + target_host: str, + *, + expected_generation: int, + expected_recording_session: str, + pre_retry_fence: CameraProducerPreDetachFence, + commit_fence: CameraProducerCommitFence | None = None, + committed_before_start: CameraProducerCommittedObserver | None = None, + ) -> dict[str, Any]: + """Retry an exact partially-armed recording without device commands. + + A failed local FFmpeg spawn may leave selection and recording root + committed while no producer epoch exists. ``start_recording`` is + deliberately idempotent for that same root, so post-PCL activation + needs this narrower CAS to clear the local error and retry the spawn. + It cannot replace a living/archive-owning producer or a newer source. + """ + + if source_id not in CAMERA_SOURCE_PATHS: + raise ValueError("неизвестный camera source") + target = validate_private_ipv4(target_host) + if expected_generation < 1: + raise ValueError("camera generation устарело") + if not expected_recording_session: + raise ValueError("camera recording session отсутствует") + def reserve_retry() -> bool: + with self._lifecycle_lock: # noqa: SIM117 - lock order is intentional + with self._lock: + self._require_open_locked() + recording_root = self._recording_root + if ( + self._source_id != source_id + or self._target_host != target + or self._generation != expected_generation + or recording_root is None + or recording_root.name != expected_recording_session + or self._producer is not None + ): + raise ValueError("camera activation retry lineage устарела") + # A failed spawn may already have created and sealed + # epoch-{generation}. Never reopen or overwrite that + # evidence; the retry owns a fresh codec/archive generation. + self._generation += 1 + self._phase = "selected" + self._error = None + self._revision += 1 + return True + + if not pre_retry_fence(reserve_retry): + raise ValueError("camera activation retry lineage устарела") + prepared = self._prepare_selected_producer() + + def commit() -> bool: + with self._lifecycle_lock: + if not self._commit_prepared_producer(prepared): + return False + committed_snapshot = self.snapshot() + if committed_before_start is not None: + committed_before_start(committed_snapshot) + with self._lifecycle_lock: + if self._producer is not prepared.producer: + return False + self._start_producer_threads(prepared.producer) + return True + + try: + committed = commit_fence(commit) if commit_fence is not None else commit() + except BaseException: + self._discard_prepared_producer(prepared, failure_code="retry-commit-failed") + raise + if not committed: + self._discard_prepared_producer(prepared, failure_code="stale-retry-commit") + raise ValueError("camera activation retry lineage устарела") + return self.snapshot() + def expect_source_end_for_device_stop(self) -> dict[str, Any]: """Classify one clean FFmpeg EOF as the camera tail of canonical STOP.""" @@ -332,7 +881,7 @@ class XgridsK1CameraGateway: with self._lifecycle_lock: with self._lock: - producer, delivery = self._detach_producer_locked() + producer, deliveries = self._detach_producer_locked() recording_was_active = self._recording_root is not None self._recording_root = None self._expected_source_end_generation = None @@ -342,7 +891,7 @@ class XgridsK1CameraGateway: self._revision += 1 self._shutdown_producer( producer, - delivery, + deliveries, status=status, failure_code=failure_code, ) @@ -355,29 +904,45 @@ class XgridsK1CameraGateway: if generation != self._generation or self._source_id is None: raise ValueError("camera preview generation не активно") producer = self._producer + recording_active = self._recording_root is not None if producer is None: + # An acquisition owns its producer lifecycle. Once an epoch is + # sealed, a disposable browser reconnect must not resurrect the + # same generation against the already-finalized archive. Only + # the backend recovery owner may advance that exact lineage via + # restart_recording_producer(). Preview-only mode preserves its + # legacy lazy-spawn behavior below. + if recording_active: + raise RuntimeError( + "camera acquisition producer ожидает backend recovery" + ) producer = self._spawn_selected_producer() with self._lock: if self._producer is not producer or producer.generation != generation: raise ValueError("camera preview generation не активно") - if producer.delivery is not None: - raise RuntimeError("для camera preview уже открыт browser consumer") lease = CameraProcessLease( generation=producer.generation, source_id=producer.source_id, process=producer.process, stderr_tail=producer.stderr_tail, ) - producer.delivery = lease - if producer.init_segment is not None: - lease.segments.put_nowait(("init", producer.init_segment)) + while len(producer.deliveries) >= MAX_CAMERA_PREVIEW_CONSUMERS: + _, superseded = next(iter(producer.deliveries.items())) + producer.deliveries.pop(id(superseded), None) + superseded.failure_code = "consumer-superseded" + _close_segment_queue(superseded) + producer.deliveries[id(lease)] = lease + if producer.init_segment is not None and not lease.segments.offer( + ("init", producer.init_segment) + ): + raise RuntimeError("camera preview init segment exceeds buffer bounds") self._revision += 1 return lease def mark_streaming(self, lease: CameraProcessLease) -> None: with self._lock: producer = self._producer - if producer is None or producer.delivery is not lease: + if producer is None or producer.deliveries.get(id(lease)) is not lease: return self._mark_streaming_locked(producer) @@ -386,33 +951,57 @@ class XgridsK1CameraGateway: with self._lifecycle_lock: with self._lock: producer = self._producer - if producer is None or producer.delivery is not lease: - return - producer.delivery = None - self._revision += 1 - # During acquisition the browser is a disposable observer. In - # legacy preview-only mode retain the old lazy-owner behavior. - if self._recording_root is None: - producer.stop_requested = True - self._producer = None - producer_to_stop = producer - self._phase = "selected" - self._error = None + if producer is not None and producer.deliveries.get(id(lease)) is lease: + producer.deliveries.pop(id(lease), None) + self._revision += 1 + # During acquisition the browser is a disposable observer. In + # legacy preview-only mode retain the old lazy-owner behavior. + if self._recording_root is None and not producer.deliveries: + producer.stop_requested = True + self._producer = None + producer_to_stop = producer + self._phase = "selected" + self._error = None + # An ASGI cancellation does not stop a worker already blocked in + # ``asyncio.to_thread(queue.get)``. Close every released lease even + # when another path already retired it, so no executor thread can + # remain orphaned after a browser remount or service shutdown. + _close_segment_queue(lease) if producer_to_stop is not None: self._shutdown_producer( producer_to_stop, - None, + (), status="interrupted", failure_code=(lease.failure_code or "browser-disconnected"), ) + def retire_slow_delivery( + self, + lease: CameraProcessLease, + *, + failure_code: str = "consumer-too-slow", + ) -> None: + """Retire only one disposable browser reader, never the producer.""" + + with self._lock: + producer = self._producer + if producer is None: + lease.failure_code = failure_code + _close_segment_queue(lease) + return + self._drop_slow_delivery( + producer, + lease, + failure_code=failure_code, + ) + def close(self) -> None: with self._lifecycle_lock: with self._lock: if self._closed: return self._closed = True - producer, delivery = self._detach_producer_locked() + producer, deliveries = self._detach_producer_locked() self._revision += 1 self._phase = "idle" self._source_id = None @@ -422,12 +1011,26 @@ class XgridsK1CameraGateway: self._error = None self._shutdown_producer( producer, - delivery, + deliveries, status="interrupted", failure_code="gateway-closed", ) def _spawn_selected_producer(self) -> _CameraProducer: + with self._lock: + existing = self._producer + if existing is not None: + return existing + prepared = self._prepare_selected_producer() + if not self._commit_prepared_producer(prepared): + self._discard_prepared_producer(prepared, failure_code="stale-generation") + raise RuntimeError("camera generation изменилась во время запуска adapter") + self._start_producer_threads(prepared.producer) + return prepared.producer + + def _prepare_selected_producer(self) -> _PreparedCameraProducer: + """Open one private candidate without publishing it as the live owner.""" + with self._lock: self._require_open_locked() source_id = self._source_id @@ -438,7 +1041,7 @@ class XgridsK1CameraGateway: if source_id is None or target_host is None or ffmpeg_path is None: raise RuntimeError("camera preview runtime не готов") if self._producer is not None: - return self._producer + raise RuntimeError("camera producer изменился во время подготовки adapter") archive: CameraArchiveWriter | None = None if recording_root is not None: @@ -453,7 +1056,12 @@ class XgridsK1CameraGateway: raise RuntimeError( "Не удалось открыть долговременное хранилище camera stream." ) from exc + inherited_fence_descriptor: int | None = None try: + if self._process_fence_descriptor_factory is not None: + if os.name != "posix": + raise RuntimeError("camera lifecycle fence inheritance requires POSIX pass_fds") + inherited_fence_descriptor = self._process_fence_descriptor_factory() process = subprocess.Popen( _build_ffmpeg_argv(ffmpeg_path, target_host, source_id), stdin=subprocess.DEVNULL, @@ -461,8 +1069,11 @@ class XgridsK1CameraGateway: stderr=subprocess.PIPE, shell=False, start_new_session=(os.name == "posix"), + pass_fds=( + (inherited_fence_descriptor,) if inherited_fence_descriptor is not None else () + ), ) - except (OSError, ValueError) as exc: + except (OSError, ValueError, RuntimeError) as exc: if archive is not None: with suppress(CameraArchiveError): self._record_archive_summary( @@ -474,6 +1085,9 @@ class XgridsK1CameraGateway: "Не удалось запустить локальный camera adapter.", ) raise RuntimeError("Не удалось запустить локальный camera adapter.") from exc + finally: + if inherited_fence_descriptor is not None: + os.close(inherited_fence_descriptor) if process.stdout is None or process.stderr is None: _terminate_process(process) @@ -494,29 +1108,33 @@ class XgridsK1CameraGateway: process=process, archive=archive, ) + return _PreparedCameraProducer( + producer=producer, + target_host=target_host, + recording_root=recording_root, + ) + + def _commit_prepared_producer(self, prepared: _PreparedCameraProducer) -> bool: + producer = prepared.producer with self._lock: if ( self._closed - or generation != self._generation - or source_id != self._source_id - or target_host != self._target_host + or producer.generation != self._generation + or producer.source_id != self._source_id + or prepared.target_host != self._target_host + or prepared.recording_root != self._recording_root + or self._producer is not None ): producer.stop_requested = True - stale = True - else: - self._producer = producer - self._revision += 1 - self._phase = "connecting" - self._error = None - stale = False - if stale: - self._shutdown_producer( - producer, - None, - status="interrupted", - failure_code="stale-generation", - ) - raise RuntimeError("camera generation изменилась во время запуска adapter") + return False + self._producer = producer + self._revision += 1 + self._phase = "connecting" + self._error = None + return True + + def _start_producer_threads(self, producer: _CameraProducer) -> None: + generation = producer.generation threading.Thread( target=_drain_stderr, @@ -531,7 +1149,31 @@ class XgridsK1CameraGateway: name=f"k1-camera-fmp4-{generation}", daemon=True, ).start() - return producer + if self._producer_stall_observer is not None: + threading.Thread( + target=self._watch_producer_health, + args=(producer,), + name=f"k1-camera-watchdog-{generation}", + daemon=True, + ).start() + + def _discard_prepared_producer( + self, + prepared: _PreparedCameraProducer, + *, + failure_code: str, + ) -> None: + producer = prepared.producer + with self._lock: + if self._producer is producer: + self._producer = None + producer.stop_requested = True + self._shutdown_producer( + producer, + (), + status="interrupted", + failure_code=failure_code, + ) def _publish_segment( self, @@ -544,6 +1186,7 @@ class XgridsK1CameraGateway: producer, "segment-too-large", "Camera adapter отклонил слишком большой video segment.", + observed_segment_bytes=len(payload), ) return False @@ -565,6 +1208,20 @@ class XgridsK1CameraGateway: "Долговременная запись camera stream завершилась ошибкой.", ) return False + if kind == "media": + # Session-lifetime evidence, independent of browser delivery + # and producer detach/finalize timing. Update it before any + # derived observer can inspect the gateway snapshot. + with self._lock: + self._recording_media_segment_count += 1 + producer.committed_media_segment_count += 1 + committed_monotonic = time.monotonic() + if producer.first_media_committed_monotonic is None: + producer.first_media_committed_monotonic = committed_monotonic + producer.last_media_committed_monotonic = committed_monotonic + else: + with self._lock: + producer.init_committed_monotonic = time.monotonic() observer = self._committed_segment_observer if observer is not None: try: @@ -592,28 +1249,30 @@ class XgridsK1CameraGateway: ) if not producer_owned or producer.stop_requested: return False + producer.last_segment_monotonic = time.monotonic() if kind == "init": producer.init_segment = payload else: self._mark_streaming_locked(producer) - delivery = producer.delivery - if delivery is None: + deliveries = tuple(producer.deliveries.values()) + if not deliveries: return True - try: - delivery.segments.put_nowait((kind, payload)) - except queue.Full: - self._drop_slow_delivery(producer, delivery) + for delivery in deliveries: + if not delivery.segments.offer((kind, payload)): + self._drop_slow_delivery(producer, delivery) return True def _drop_slow_delivery( self, producer: _CameraProducer, delivery: CameraProcessLease, + *, + failure_code: str = "consumer-too-slow", ) -> None: with self._lock: - if self._producer is producer and producer.delivery is delivery: - producer.delivery = None - delivery.failure_code = "consumer-too-slow" + if self._producer is producer and producer.deliveries.get(id(delivery)) is delivery: + producer.deliveries.pop(id(delivery), None) + delivery.failure_code = failure_code self._revision += 1 _close_segment_queue(delivery) @@ -622,20 +1281,30 @@ class XgridsK1CameraGateway: producer: _CameraProducer, code: str, message: str, + *, + observed_segment_bytes: int | None = None, ) -> None: with self._lock: producer_owned = self._producer is producer or producer.drain_requested if not producer_owned or producer.stop_requested: return producer.failure_code = code - self._set_error_locked(code, message) + facts = ( + { + "observed_size_bytes": observed_segment_bytes, + "maximum_size_bytes": MAX_FMP4_SEGMENT_BYTES, + } + if observed_segment_bytes is not None + else None + ) + self._set_error_locked(code, message, facts=facts) with suppress(OSError): producer.process.terminate() def _producer_ended(self, producer: _CameraProducer) -> None: with self._lock: - delivery = producer.delivery - producer.delivery = None + deliveries = tuple(producer.deliveries.values()) + producer.deliveries.clear() owns_shutdown = self._producer is producer and not producer.stop_requested expected_clean_end = ( owns_shutdown @@ -657,11 +1326,9 @@ class XgridsK1CameraGateway: "camera-source-ended", _safe_ffmpeg_message(producer.stderr_tail), ) - if delivery is not None: + for delivery in deliveries: delivery.failure_code = ( - None - if expected_clean_end - else producer.failure_code or "camera-source-ended" + None if expected_clean_end else producer.failure_code or "camera-source-ended" ) _close_segment_queue(delivery) if not owns_shutdown: @@ -686,24 +1353,68 @@ class XgridsK1CameraGateway: "camera-storage-finalize-failed", "Не удалось завершить долговременную запись camera stream.", ) + self._notify_producer_stall_observer(producer) + + def _watch_producer_health(self, producer: _CameraProducer) -> None: + """Wake recovery once per continuous stall in this exact generation.""" + + reported_reason: str | None = None + while True: + time.sleep(self._producer_watchdog_interval_seconds) + with self._lock: + if self._closed or self._producer is not producer: + return + camera = self.snapshot() + reason = classify_camera_recording_health( + camera, + stall_milliseconds=self._producer_stall_milliseconds, + ) + if reason is None: + reported_reason = None + continue + if reason == reported_reason: + continue + reported_reason = reason + self._notify_producer_stall_observer(producer, reason=reason) + + def _notify_producer_stall_observer( + self, + producer: _CameraProducer, + *, + reason: str | None = None, + ) -> None: + observer = self._producer_stall_observer + if observer is None: + return + current_reason = reason or classify_camera_recording_health( + self.snapshot(), + stall_milliseconds=self._producer_stall_milliseconds, + ) + if current_reason is None: + return + try: + observer(current_reason, producer.generation) + except Exception: + # Recovery wake is derived supervision. It cannot fail archive + # ownership or the FFmpeg reader which produced the evidence. + return def _shutdown_producer( self, producer: _CameraProducer | None, - delivery: CameraProcessLease | None, + deliveries: tuple[CameraProcessLease, ...], *, status: CameraArchiveStatus, failure_code: str | None, ) -> None: - if delivery is not None: + for delivery in deliveries: delivery.failure_code = producer.failure_code if producer is not None else failure_code _close_segment_queue(delivery) if producer is None: return _terminate_process(producer.process, close_streams=False) - drained = ( - not producer.reader_started - or producer.reader_done.wait(timeout=CAMERA_DRAIN_TIMEOUT_SECONDS) + drained = not producer.reader_started or producer.reader_done.wait( + timeout=CAMERA_DRAIN_TIMEOUT_SECONDS ) _close_process_streams(producer.process) if not drained: @@ -721,7 +1432,7 @@ class XgridsK1CameraGateway: if producer.failure_code is not None else status ) - if delivery is not None: + for delivery in deliveries: delivery.failure_code = effective_failure_code try: self._finalize_archive( @@ -755,26 +1466,32 @@ class XgridsK1CameraGateway: def _detach_producer_locked( self, - ) -> tuple[_CameraProducer | None, CameraProcessLease | None]: + ) -> tuple[_CameraProducer | None, tuple[CameraProcessLease, ...]]: producer = self._producer self._producer = None if producer is None: - return None, None + return None, () producer.drain_requested = producer.archive is not None producer.stop_requested = not producer.drain_requested - delivery = producer.delivery - producer.delivery = None - return producer, delivery + deliveries = tuple(producer.deliveries.values()) + producer.deliveries.clear() + return producer, deliveries def _mark_streaming_locked(self, producer: _CameraProducer) -> None: if self._producer is producer and self._phase != "streaming": self._revision += 1 self._phase = "streaming" - def _set_error_locked(self, code: str, message: str) -> None: + def _set_error_locked( + self, + code: str, + message: str, + *, + facts: Mapping[str, Any] | None = None, + ) -> None: self._revision += 1 self._phase = "error" - self._error = {"code": code, "message": message} + self._error = {"code": code, "message": message, **dict(facts or {})} def _require_open_locked(self) -> None: if self._closed: @@ -826,7 +1543,7 @@ def _build_ffmpeg_argv( "-allowed_media_types", "video", "-timeout", - "5000000", + str(CAMERA_SOURCE_IO_TIMEOUT_MICROSECONDS), "-probesize", "4000000", "-analyzeduration", @@ -864,6 +1581,13 @@ def _read_exact(stream: IO[bytes], size: int) -> bytes: return b"".join(chunks) +class _Fmp4BoxTooLarge(ValueError): + def __init__(self, *, box_type: bytes, observed_bytes: int) -> None: + super().__init__("invalid or unbounded ISO-BMFF box") + self.box_type = box_type + self.observed_bytes = observed_bytes + + def _read_mp4_box(stream: IO[bytes]) -> tuple[bytes, bytes]: header = _read_exact(stream, 8) size = int.from_bytes(header[:4], "big") @@ -872,21 +1596,15 @@ def _read_mp4_box(stream: IO[bytes]) -> tuple[bytes, bytes]: extended = _read_exact(stream, 8) size = int.from_bytes(extended, "big") header += extended - if size == 0 or size < len(header) or size > MAX_FMP4_BOX_BYTES: + if size > MAX_FMP4_BOX_BYTES: + raise _Fmp4BoxTooLarge(box_type=box_type, observed_bytes=size) + if size == 0 or size < len(header): raise ValueError("invalid or unbounded ISO-BMFF box") return box_type, header + _read_exact(stream, size - len(header)) def _close_segment_queue(lease: CameraProcessLease) -> None: - try: - lease.segments.put_nowait(None) - except queue.Full: - while True: - try: - lease.segments.get_nowait() - except queue.Empty: - break - lease.segments.put_nowait(None) + lease.segments.close() def _read_fmp4_stdout( @@ -909,6 +1627,7 @@ def _read_fmp4_stdout( init_parts: list[bytes] = [] fragment_parts: list[bytes] = [] init_sent = False + fragment_size = 0 try: while True: box_type, box = _read_mp4_box(stream) @@ -926,9 +1645,19 @@ def _read_fmp4_stdout( if box_type == b"moof": fragment_parts = [box] + fragment_size = len(box) continue if fragment_parts: fragment_parts.append(box) + fragment_size += len(box) + if fragment_size > MAX_FMP4_SEGMENT_BYTES: + gateway._mark_producer_failure( + producer, + "segment-too-large", + "Camera adapter отклонил слишком большой video segment.", + observed_segment_bytes=fragment_size, + ) + return if box_type == b"mdat": if not gateway._publish_segment( producer, @@ -937,17 +1666,33 @@ def _read_fmp4_stdout( ): return fragment_parts = [] + fragment_size = 0 continue # `styp`/`sidx` are valid media-segment prefixes. Preserve them and # wait for the following moof+mdat instead of forwarding raw boxes. if box_type in {b"styp", b"sidx"}: fragment_parts.append(box) + fragment_size += len(box) + except _Fmp4BoxTooLarge as exc: + if init_sent: + gateway._mark_producer_failure( + producer, + "segment-too-large", + "Camera adapter отклонил слишком большой video segment.", + observed_segment_bytes=fragment_size + exc.observed_bytes, + ) + else: + gateway._mark_producer_failure( + producer, + "invalid-fmp4", + _safe_ffmpeg_message(producer.stderr_tail), + ) except EOFError: if not init_sent: gateway._mark_producer_failure( producer, "invalid-fmp4", - "Camera adapter вернул некорректный fMP4 stream.", + _safe_ffmpeg_message(producer.stderr_tail), ) elif fragment_parts: gateway._mark_producer_failure( @@ -959,7 +1704,7 @@ def _read_fmp4_stdout( gateway._mark_producer_failure( producer, "invalid-fmp4", - "Camera adapter вернул некорректный fMP4 stream.", + _safe_ffmpeg_message(producer.stderr_tail), ) finally: try: @@ -1051,15 +1796,35 @@ def build_xgrids_k1_camera_router( return client_closed = False + close_code = 1000 + close_reason = "" try: while True: segment = await asyncio.to_thread(lease.segments.get) if segment is None: + if lease.failure_code in { + "consumer-too-slow", + "consumer-send-timeout", + }: + close_code = 4008 + close_reason = "preview-consumer-lagged" break kind, payload = segment if kind == "media": gateway.mark_streaming(lease) - await websocket.send_bytes(payload) + try: + await asyncio.wait_for( + websocket.send_bytes(payload), + timeout=CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS, + ) + except TimeoutError: + gateway.retire_slow_delivery( + lease, + failure_code="consumer-send-timeout", + ) + close_code = 4008 + close_reason = "preview-consumer-lagged" + break except WebSocketDisconnect: client_closed = True except RuntimeError: @@ -1067,6 +1832,6 @@ def build_xgrids_k1_camera_router( finally: gateway.release_delivery(lease, client_closed=client_closed) with suppress(RuntimeError): - await websocket.close() + await websocket.close(code=close_code, reason=close_reason) return router diff --git a/src/k1link/device_plugins/xgrids_k1/cli.py b/src/k1link/device_plugins/xgrids_k1/cli.py index 28ffc4f..9274729 100644 --- a/src/k1link/device_plugins/xgrids_k1/cli.py +++ b/src/k1link/device_plugins/xgrids_k1/cli.py @@ -1,13 +1,21 @@ from __future__ import annotations import asyncio +import fcntl +import http.client import json +import os import platform import shutil +import stat import subprocess import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path -from typing import Annotated, TypedDict +from typing import Annotated, Literal, TypedDict +from uuid import UUID import typer import uvicorn @@ -35,8 +43,17 @@ from k1link.device_plugins.xgrids_k1.analyze import ( run_calibrated_overlay_experiment, summarize_mqtt_streams, ) +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + ApplicationControlProcessLease, + ApplicationControlProcessLeaseError, +) from k1link.device_plugins.xgrids_k1.archive import discover_legacy_viewer_sessions from k1link.device_plugins.xgrids_k1.ble.gatt import dump_metadata +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + borrow_ble_runtime_process_lease, + configure_ble_runtime_process_lease, + defer_until_ble_runtime_idle, +) from k1link.device_plugins.xgrids_k1.ble.scanner import scan from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( PROFILE_ID, @@ -54,6 +71,10 @@ from k1link.device_plugins.xgrids_k1.mqtt import ( capture_mqtt, ) from k1link.device_plugins.xgrids_k1.net.snapshot import snapshot +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PhysicalCommandLedger, + active_operator_retirements, +) from k1link.device_plugins.xgrids_k1.protocol.application_authority import ( ApplicationAuthorityLoadError, MacOSKeychainApplicationAuthorityProvisioner, @@ -93,6 +114,243 @@ app.add_typer(compute_app, name="compute") app.add_typer(lab_app, name="lab") app.add_typer(artifact_app, name="artifact") +_CANONICAL_MISSION_CORE_PORT = 8000 +_MISSION_CORE_SERVE_LOCK_FILENAME = ".serve.lock" + + +class _MissionCoreServeLeaseError(RuntimeError): + """The canonical backend singleton lock cannot be trusted.""" + + +class _MissionCoreServeLeaseUnavailable(_MissionCoreServeLeaseError): + """Another live process owns canonical backend startup or runtime.""" + + +@dataclass(slots=True) +class _MissionCoreServeLease: + """Stable OS-owned lease for the complete canonical Uvicorn lifetime.""" + + path: Path + _descriptor: int + _identity: tuple[int, int] + _released: bool = False + + @classmethod + def acquire(cls, repository_root: Path) -> _MissionCoreServeLease: + runtime_dir = repository_root.expanduser().resolve() / ".runtime" + _ensure_serve_runtime_directory(runtime_dir) + lock_dir = runtime_dir / "mission-core" + _ensure_private_serve_lock_directory(lock_dir) + path = lock_dir / _MISSION_CORE_SERVE_LOCK_FILENAME + common_flags = os.O_RDWR | getattr(os, "O_CLOEXEC", 0) + common_flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, common_flags | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + try: + descriptor = os.open(path, common_flags) + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock cannot be opened safely" + ) from exc + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock cannot be created safely" + ) from exc + + locked = False + try: + opened = os.fstat(descriptor) + _validate_private_serve_lock_file(opened) + try: + current = path.lstat() + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock identity is unavailable" + ) from exc + _validate_private_serve_lock_file(current) + identity = (opened.st_dev, opened.st_ino) + if identity != (current.st_dev, current.st_ino): + raise _MissionCoreServeLeaseError( + "Mission Core serve lock changed while opening" + ) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + locked = True + except BlockingIOError as exc: + raise _MissionCoreServeLeaseUnavailable( + "another process owns Mission Core startup or runtime" + ) from exc + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock cannot be acquired safely" + ) from exc + try: + locked_path = path.lstat() + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock disappeared after acquisition" + ) from exc + _validate_private_serve_lock_file(locked_path) + if identity != (locked_path.st_dev, locked_path.st_ino): + raise _MissionCoreServeLeaseError( + "Mission Core serve lock changed during acquisition" + ) + return cls(path=path, _descriptor=descriptor, _identity=identity) + except BaseException: + if locked: + _unlock_serve_descriptor(descriptor) + os.close(descriptor) + raise + + def release(self) -> None: + if self._released: + return + try: + _unlock_serve_descriptor(self._descriptor) + finally: + os.close(self._descriptor) + self._released = True + + def __enter__(self) -> _MissionCoreServeLease: + return self + + def __exit__(self, *_: object) -> None: + self.release() + + +def _ensure_serve_runtime_directory(path: Path) -> None: + try: + path.mkdir(mode=0o700, parents=False, exist_ok=False) + except FileExistsError: + pass + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core runtime directory is unavailable" + ) from exc + try: + metadata = path.lstat() + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core runtime directory identity is unavailable" + ) from exc + if not stat.S_ISDIR(metadata.st_mode): + raise _MissionCoreServeLeaseError( + "Mission Core runtime directory is not a regular directory" + ) + + +def _ensure_private_serve_lock_directory(path: Path) -> None: + try: + path.mkdir(mode=0o700, parents=False, exist_ok=False) + except FileExistsError: + pass + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock directory is unavailable" + ) from exc + try: + metadata = path.lstat() + except OSError as exc: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock directory identity is unavailable" + ) from exc + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + raise _MissionCoreServeLeaseError( + "Mission Core serve lock directory is not private" + ) + + +def _validate_private_serve_lock_file(metadata: os.stat_result) -> None: + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_nlink != 1 + ): + raise _MissionCoreServeLeaseError( + "Mission Core serve lock is not a private regular file" + ) + + +def _unlock_serve_descriptor(descriptor: int) -> None: + try: + fcntl.flock(descriptor, fcntl.LOCK_UN) + except OSError: + return + + +def _acquire_mission_core_serve_lease(repository_root: Path) -> _MissionCoreServeLease: + return _MissionCoreServeLease.acquire(repository_root) + + +def _configure_ble_process_lease() -> Path: + """Pin every standalone CLI BLE call to Mission Core's global K1 lock.""" + + repository_root = Path(__file__).resolve().parents[4] + configure_ble_runtime_process_lease(repository_root) + return repository_root + + +class _CliWifiProvisioningBlocked(RuntimeError): + """The standalone mutating BLE command lacks safe physical authority.""" + + +def _canonical_corebluetooth_uuid(value: str) -> str: + try: + return str(UUID(str(value).strip())).upper() + except (AttributeError, ValueError) as exc: + raise _CliWifiProvisioningBlocked( + "BLE device target is not a canonical CoreBluetooth UUID" + ) from exc + + +def _require_cli_wifi_target_not_retired( + repository_root: Path, + device: str, +) -> str: + canonical_device = _canonical_corebluetooth_uuid(device) + snapshot = PhysicalCommandLedger(repository_root).snapshot() + if snapshot.status == "corrupt": + raise _CliWifiProvisioningBlocked( + "physical command audit is unavailable; Wi-Fi write is blocked" + ) + record = snapshot.record + if record is not None: + for retirement in active_operator_retirements(record): + retired_device = _canonical_corebluetooth_uuid( + retirement.retired_transport_ref + ) + if retired_device == canonical_device: + raise _CliWifiProvisioningBlocked( + "the selected BLE UUID belongs to an operator-retired K1" + ) + return canonical_device + + +@contextmanager +def _cli_wifi_mutation_lease(device: str) -> Iterator[str]: + """Fence one explicit CLI Wi-Fi write against retirement and Mission Core.""" + + repository_root = _configure_ble_process_lease() + lease = ApplicationControlProcessLease.acquire(repository_root) + try: + # This durable check is intentionally after acquiring the same global + # flock as the backend retirement action. Retirement either commits + # first and this write is denied, or this explicit write owns the fence + # through credential entry and the complete native BLE lifecycle. + canonical_device = _require_cli_wifi_target_not_retired( + repository_root, + device, + ) + with borrow_ble_runtime_process_lease(lease): + yield canonical_device + finally: + # A hard CoreBluetooth timeout may return before its native cleanup + # task is terminal. Never expose the retirement boundary until the + # shared BLE arbiter proves idle; poison intentionally keeps it held + # until process restart. + defer_until_ble_runtime_idle(lease.release) + class ToolStatus(TypedDict): name: str @@ -837,25 +1095,133 @@ def publish_e26_lab( def serve_console( port: Annotated[ int, - typer.Option(min=1024, max=65535, help="Loopback HTTP port for the local console."), + typer.Option( + min=1024, + max=65535, + help="Canonical loopback HTTP port; only 8000 is accepted.", + ), ] = 8000, ) -> None: """Serve the built Mission Core Control Station and loopback control API.""" - frontend = Path(__file__).resolve().parents[4] / "apps" / "control-station" / "dist" - if not frontend.is_dir(): + if port != _CANONICAL_MISSION_CORE_PORT: console.print( - "[red]Frontend build is missing.[/red] Run npm install && npm run build " - "inside apps/control-station." + f"[red]Mission Core запускается только на каноническом порту " + f"{_CANONICAL_MISSION_CORE_PORT}.[/red]" ) + console.print("Другой локальный backend не создан.") raise typer.Exit(code=2) - console.print(f"NODEDC MISSION CORE: http://127.0.0.1:{port}") - console.print("The credential endpoint is bound to this Mac only.") - uvicorn.run( - "k1link.web.app:app", - host="127.0.0.1", - port=port, - log_level="info", - access_log=True, + + repository_root = Path(__file__).resolve().parents[4] + try: + lease = _acquire_mission_core_serve_lease(repository_root) + except _MissionCoreServeLeaseUnavailable: + local_server = _local_server_status(_CANONICAL_MISSION_CORE_PORT) + if local_server == "mission-core": + _print_existing_mission_core(_CANONICAL_MISSION_CORE_PORT) + return + if local_server == "free": + console.print( + "[yellow]Mission Core уже запускается или завершает работу.[/yellow]" + ) + else: + console.print( + "[red]Запуск Mission Core уже выполняется, но канонический health endpoint " + "пока не подтверждён.[/red]" + ) + console.print( + "Второй backend не создан. Повторите запуск после завершения текущего перехода." + ) + raise typer.Exit(code=2) from None + except _MissionCoreServeLeaseError as exc: + console.print( + "[red]Не удалось безопасно получить блокировку канонического Mission Core.[/red]" + ) + console.print(f"Второй backend не создан: {exc}") + raise typer.Exit(code=2) from exc + + with lease: + local_server = _local_server_status(_CANONICAL_MISSION_CORE_PORT) + if local_server == "mission-core": + _print_existing_mission_core(_CANONICAL_MISSION_CORE_PORT) + return + if local_server == "occupied": + console.print( + f"[red]Порт 127.0.0.1:{_CANONICAL_MISSION_CORE_PORT} уже занят другим " + "или неготовым процессом.[/red]" + ) + console.print( + "Mission Core не стал создавать второй backend. Остановите точный " + "процесс-владелец порта и повторите запуск." + ) + raise typer.Exit(code=2) + + frontend = repository_root / "apps" / "control-station" / "dist" + if not frontend.is_dir(): + console.print( + "[red]Frontend build is missing.[/red] Run npm install && npm run build " + "inside apps/control-station." + ) + raise typer.Exit(code=2) + console.print( + f"NODEDC MISSION CORE: http://127.0.0.1:{_CANONICAL_MISSION_CORE_PORT}" + ) + console.print("The credential endpoint is bound to this Mac only.") + uvicorn.run( + "k1link.web.app:app", + host="127.0.0.1", + port=_CANONICAL_MISSION_CORE_PORT, + log_level="info", + access_log=True, + ) + + +def _print_existing_mission_core(port: int) -> None: + console.print( + f"[green]NODEDC MISSION CORE уже запущен:[/green] http://127.0.0.1:{port}" + ) + console.print("Используется единственный локальный backend; второй процесс не создан.") + + +def _local_server_status(port: int) -> Literal["free", "mission-core", "occupied"]: + """Classify the loopback listener before starting the singleton backend. + + A healthy Mission Core listener makes ``k1link serve`` idempotent. Any + other listener fails with a short operator-facing error instead of letting + Uvicorn start application resources and then emit an opaque ``Errno 48``. + The check is observational only and never kills or replaces a process. + """ + + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=0.75) + try: + connection.request( + "GET", + "/api/health", + headers={"Accept": "application/json", "Connection": "close"}, + ) + response = connection.getresponse() + if response.status != 200: + return "occupied" + body = response.read(16_385) + if len(body) > 16_384: + return "occupied" + payload = json.loads(body.decode("utf-8")) + except ConnectionRefusedError: + return "free" + except ( + OSError, + TimeoutError, + UnicodeDecodeError, + json.JSONDecodeError, + http.client.HTTPException, + ): + return "occupied" + finally: + connection.close() + return ( + "mission-core" + if isinstance(payload, dict) + and payload.get("service") == "mission-core-control-plane" + else "occupied" ) @@ -871,6 +1237,7 @@ def ble_scan( ] = 30.0, ) -> None: """Discover BLE advertisements without connecting or changing device configuration.""" + _configure_ble_process_lease() try: result = asyncio.run(scan(duration)) except (BleakError, OSError, ValueError) as exc: @@ -908,6 +1275,7 @@ def ble_gatt_dump( ] = 45.0, ) -> None: """Enumerate GATT metadata only: no characteristic reads, subscriptions or writes.""" + _configure_ble_process_lease() console.print( "Connecting for service discovery only; no characteristic values will be read or written." ) @@ -963,28 +1331,40 @@ def ble_wifi_configure( ) raise typer.Exit(code=2) - console.print( - "Two local macOS dialogs will request the Wi-Fi name and hidden password. " - "The password is never printed, logged, or written to the result file; " - "the K1 may echo the SSID in the ignored sensitive status result." - ) + ssid = "" + password = "" try: - ssid, password = prompt_wifi_credentials() + with _cli_wifi_mutation_lease(device) as canonical_device: + console.print( + "Two local macOS dialogs will request the Wi-Fi name and hidden password. " + "The password is never printed, logged, or written to the result file; " + "the K1 may echo the SSID in the ignored sensitive status result." + ) + ssid, password = prompt_wifi_credentials() + console.print( + "Credentials accepted locally. Starting the single reviewed BLE write." + ) + result = asyncio.run( + provision_wifi_once( + canonical_device, + ssid, + password, + timeout_seconds=timeout, + write_mode=write_mode, + ) + ) except CredentialDialogError as exc: console.print(f"[red]Credential entry failed:[/red] {exc}") raise typer.Exit(code=2) from exc - - console.print("Credentials accepted locally. Starting the single reviewed BLE write.") - try: - result = asyncio.run( - provision_wifi_once( - device, - ssid, - password, - timeout_seconds=timeout, - write_mode=write_mode, - ) + except ( + _CliWifiProvisioningBlocked, + ApplicationControlProcessLeaseError, + ) as exc: + console.print( + "[red]Wi-Fi provisioning blocked before credential or device access:[/red] " + f"{exc}" ) + raise typer.Exit(code=2) from exc except (BleakError, OSError, TimeoutError, ValueError) as exc: console.print(f"[red]Wi-Fi provisioning failed:[/red] {type(exc).__name__}: {exc}") console.print("No automatic retry was attempted.") diff --git a/src/k1link/device_plugins/xgrids_k1/connection_supervisor.py b/src/k1link/device_plugins/xgrids_k1/connection_supervisor.py new file mode 100644 index 0000000..0aeab42 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/connection_supervisor.py @@ -0,0 +1,2181 @@ +from __future__ import annotations + +import asyncio +import ipaddress +import math +import threading +import time +from collections.abc import Awaitable, Callable +from contextlib import suppress +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Literal, Protocol + +from k1link.device_plugins.xgrids_k1.host_diagnostics import ( + HostFailureDiagnostic, + host_diagnostics_for_reasons, +) + +ConnectionMode = Literal["bridge", "quick-connect", "direct-connect"] +RouteClass = Literal["direct", "default", "tunnel", "unavailable", "unknown"] +TcpState = Literal["unknown", "reachable", "unreachable"] +IdentityState = Literal["unverified", "verified", "stale", "mismatch"] +PlaneState = Literal["idle", "healthy", "stalled", "lost"] +LeaseState = Literal["absent", "configured-unverified", "reachable", "lost"] +DeviceNetworkState = Literal["unconfigured", "applied"] +DeviceNetworkSource = Literal[ + "ble-post-write-status", + "ble-read-only-status", + "durable-semantic-topology", +] +AllowedAction = Literal[ + "select-connection-intent", + "inspect-host-network", + "probe-endpoint", + "verify-control-device-info", + "start-acquisition", + "stop-acquisition", + "acknowledge-data-loss", +] + +DEFAULT_CONTROL_PROOF_TTL_SECONDS = 90.0 +# A production monitor pass may legitimately take up to 11.5 seconds before +# its one-second interval: two bounded route-plus-association samples and one +# bounded TCP probe. The fallback silence lease therefore outlives two whole +# healthy passes (2 * 12.5 seconds), including one delayed/superseded pass. +# Explicit negative samples and proven route/fingerprint changes remain the +# primary, immediate loss signals; this TTL only bounds complete observer +# silence. +DEFAULT_TRANSPORT_OBSERVATION_TTL_SECONDS = 30.0 + + +class ConnectionSupervisorError(RuntimeError): + """The read-only connection supervisor contract was violated.""" + + +class ConnectionSupervisorClosed(ConnectionSupervisorError): + """The supervisor or its monitor has already been closed.""" + + +class ConnectionMonitorProbeSuperseded(ConnectionSupervisorError): + """An operator-owned lifecycle transition superseded a read-only probe.""" + + +class HostPathProbe(Protocol): + def __call__(self, target: EndpointTarget) -> Awaitable[HostPathProbeResult]: ... + + +class TcpReachabilityProbe(Protocol): + def __call__( + self, + target: EndpointTarget, + ) -> Awaitable[bool | TcpReachabilityProbeResult]: ... + + +class EndpointTargetProvider(Protocol): + def __call__(self) -> EndpointTarget | None: ... + + +@dataclass(frozen=True, slots=True) +class TcpReachabilityProbeResult: + """One secret-free result for an exact, read-only TCP connect attempt.""" + + reachable: bool + reason_code: str | None = None + + def __post_init__(self) -> None: + if not isinstance(self.reachable, bool): + raise ConnectionSupervisorError("TCP reachability must be boolean") + if not self.reachable and not _nonblank(self.reason_code): + raise ConnectionSupervisorError("unreachable TCP result requires a reason code") + if self.reachable and self.reason_code is not None: + raise ConnectionSupervisorError("reachable TCP result cannot carry a failure reason") + + +@dataclass(frozen=True, slots=True) +class EndpointTarget: + ipv4: str + port: int = 1883 + + def __post_init__(self) -> None: + try: + address = ipaddress.ip_address(self.ipv4) + except ValueError as exc: + raise ConnectionSupervisorError("endpoint target must be a valid IPv4 address") from exc + if not isinstance(address, ipaddress.IPv4Address): + raise ConnectionSupervisorError("endpoint target must be IPv4") + if not address.is_private: + raise ConnectionSupervisorError("endpoint target must stay on a private network") + if isinstance(self.port, bool) or not 1 <= self.port <= 65_535: + raise ConnectionSupervisorError("endpoint port must be within 1..65535") + + def as_dict(self) -> dict[str, object]: + return {"ipv4": self.ipv4, "port": self.port} + + +@dataclass(frozen=True, slots=True) +class HostPathProbeResult: + available: bool + fingerprint: str | None + interface: str | None + source_ipv4: str | None + route_class: RouteClass + reason_code: str | None = None + # ``route`` is the conservative default for all existing probes. The + # facade marks only the narrower case where the kernel route was present + # but its bounded association-identity observer could not prove identity. + # That narrower class may be either unavailable (a helper timeout) or an + # available raw route with privacy-limited association evidence. The + # read-only monitor may retain an already verified binding across that + # uncertainty without delaying a real route/interface/source transition. + observation_failure_class: Literal["route", "association-observer"] = "route" + # Secret-free hash of the kernel-selected route before the association + # identity is folded in. It lets the monitor prove that a technical + # association-observer failure did not coincide with a real route change. + kernel_route_fingerprint: str | None = None + + def __post_init__(self) -> None: + if self.observation_failure_class not in {"route", "association-observer"}: + raise ConnectionSupervisorError("host path failure class is invalid") + if self.kernel_route_fingerprint is not None and not _nonblank( + self.kernel_route_fingerprint + ): + raise ConnectionSupervisorError("kernel route fingerprint must be nonblank") + if self.available and not _nonblank(self.fingerprint): + raise ConnectionSupervisorError( + "an available host path requires a non-secret network fingerprint" + ) + if self.available and self.route_class == "unavailable": + raise ConnectionSupervisorError("an available host path cannot be unavailable") + if not self.available and self.route_class != "unavailable": + raise ConnectionSupervisorError("an unavailable host path must use unavailable route") + if self.source_ipv4 is not None: + try: + address = ipaddress.ip_address(self.source_ipv4) + except ValueError as exc: + raise ConnectionSupervisorError("host source address must be valid IPv4") from exc + if not isinstance(address, ipaddress.IPv4Address): + raise ConnectionSupervisorError("host source address must be IPv4") + + +@dataclass(frozen=True, slots=True) +class ConnectionIntent: + intent_id: str + requested_mode: ConnectionMode + expected_device_id: str | None + requested_at: str + + def as_dict(self) -> dict[str, object]: + return { + "intent_id": self.intent_id, + "requested_mode": self.requested_mode, + "expected_device_id": self.expected_device_id, + "requested_at": self.requested_at, + } + + +@dataclass(frozen=True, slots=True) +class HostPathObservation: + epoch: int + available: bool + fingerprint: str | None + interface: str | None + source_ipv4: str | None + route_class: RouteClass + reason_code: str | None + observed_at: str + kernel_route_fingerprint: str | None = None + + def as_dict(self) -> dict[str, object]: + return { + "epoch": self.epoch, + "available": self.available, + "fingerprint": self.fingerprint, + "interface": self.interface, + "source_ipv4": self.source_ipv4, + "route_class": self.route_class, + "reason_code": self.reason_code, + "observed_at": self.observed_at, + "kernel_route_fingerprint": self.kernel_route_fingerprint, + } + + +@dataclass(frozen=True, slots=True) +class DeviceNetworkObservation: + """Last semantically proven topology on the K1 itself. + + This is deliberately independent from the Mac route, TCP reachability and + MQTT identity. A BLE status can prove that the device entered AP/STA mode + even when the host failed to follow it onto that network. + """ + + state: DeviceNetworkState + intent_id: str | None + transport_ref: str | None + connection_mode: ConnectionMode | None + target: EndpointTarget | None + source: DeviceNetworkSource | None + observed_at: str | None + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "intent_id": self.intent_id, + "transport_ref": self.transport_ref, + "connection_mode": self.connection_mode, + "target": self.target.as_dict() if self.target is not None else None, + "source": self.source, + "observed_at": self.observed_at, + } + + +@dataclass(frozen=True, slots=True) +class EndpointObservation: + target: EndpointTarget | None + tcp_state: TcpState + intent_id: str | None + host_path_epoch: int | None + reason_code: str | None + observed_at: str | None + + def as_dict(self) -> dict[str, object]: + return { + "target": self.target.as_dict() if self.target is not None else None, + "tcp_state": self.tcp_state, + "intent_id": self.intent_id, + "host_path_epoch": self.host_path_epoch, + "reason_code": self.reason_code, + "observed_at": self.observed_at, + } + + +@dataclass(frozen=True, slots=True) +class DeviceIdentityObservation: + state: IdentityState + intent_id: str | None + logical_device_id: str | None + compatibility_profile_id: str | None + connection_mode: ConnectionMode | None + source: Literal["mqtt-device-info"] | None + host_path_epoch: int | None + observed_at: str | None + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "intent_id": self.intent_id, + "logical_device_id": self.logical_device_id, + "compatibility_profile_id": self.compatibility_profile_id, + "connection_mode": self.connection_mode, + "source": self.source, + "host_path_epoch": self.host_path_epoch, + "observed_at": self.observed_at, + } + + +@dataclass(frozen=True, slots=True) +class PlaneObservation: + state: PlaneState + session_id: str | None + host_path_epoch: int | None + reason_code: str | None + observed_at: str | None + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "session_id": self.session_id, + "host_path_epoch": self.host_path_epoch, + "reason_code": self.reason_code, + "observed_at": self.observed_at, + } + + +@dataclass(frozen=True, slots=True) +class ConnectionLease: + state: LeaseState + generation: int + intent_id: str | None + host_path_epoch: int | None + connection_mode: ConnectionMode | None + target: EndpointTarget | None + logical_device_id: str | None + reason_code: str | None + observed_at: str | None + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "generation": self.generation, + "intent_id": self.intent_id, + "host_path_epoch": self.host_path_epoch, + "connection_mode": self.connection_mode, + "target": self.target.as_dict() if self.target is not None else None, + "logical_device_id": self.logical_device_id, + "reason_code": self.reason_code, + "observed_at": self.observed_at, + } + + +@dataclass(frozen=True, slots=True) +class ConnectionAuthority: + network_mutation_allowed: bool + control_allowed: bool + acquisition_start_allowed: bool + data_ingest_authoritative: bool + physical_motion_allowed: Literal[False] + reason_codes: tuple[str, ...] + + def as_dict(self) -> dict[str, object]: + return { + "network_mutation_allowed": self.network_mutation_allowed, + "control_allowed": self.control_allowed, + "acquisition_start_allowed": self.acquisition_start_allowed, + "data_ingest_authoritative": self.data_ingest_authoritative, + "physical_motion_allowed": self.physical_motion_allowed, + "reason_codes": list(self.reason_codes), + } + + +@dataclass(frozen=True, slots=True) +class LastKnownConnection: + connection_mode: ConnectionMode + target: EndpointTarget + logical_device_id: str + compatibility_profile_id: str + verified_at: str + + def as_dict(self) -> dict[str, object]: + return { + "connection_mode": self.connection_mode, + "target": self.target.as_dict(), + "logical_device_id": self.logical_device_id, + "compatibility_profile_id": self.compatibility_profile_id, + "verified_at": self.verified_at, + } + + +@dataclass(frozen=True, slots=True) +class VerifiedControlEvidence: + """Proof emitted only after MQTT control and exact DeviceInfo both succeeded.""" + + intent_id: str + transport_ref: str + host_path_epoch: int + target: EndpointTarget + connection_mode: ConnectionMode + logical_device_id: str + compatibility_profile_id: str + control_session_id: str + control_proof_revision: int = 1 + control_proof_source: str = "mqtt-device-info" + + def __post_init__(self) -> None: + for name, value in ( + ("intent_id", self.intent_id), + ("transport_ref", self.transport_ref), + ("logical_device_id", self.logical_device_id), + ("compatibility_profile_id", self.compatibility_profile_id), + ("control_session_id", self.control_session_id), + ("control_proof_source", self.control_proof_source), + ): + if not _nonblank(value): + raise ConnectionSupervisorError(f"{name} must be nonblank") + if self.host_path_epoch < 1: + raise ConnectionSupervisorError("control evidence requires a positive host epoch") + if ( + isinstance(self.control_proof_revision, bool) + or not isinstance(self.control_proof_revision, int) + or self.control_proof_revision < 1 + ): + raise ConnectionSupervisorError("control proof revision must be positive") + + +@dataclass(frozen=True, slots=True) +class ConnectionSupervisorSnapshot: + revision: int + closed: bool + intent: ConnectionIntent | None + device_network: DeviceNetworkObservation + host_path: HostPathObservation + endpoint: EndpointObservation + device_identity: DeviceIdentityObservation + control_plane: PlaneObservation + data_plane: PlaneObservation + lease: ConnectionLease + authority: ConnectionAuthority + last_known: LastKnownConnection | None + diagnostics: tuple[HostFailureDiagnostic, ...] + allowed_actions: tuple[AllowedAction, ...] + host_path_negative_streak: int + endpoint_negative_streak: int + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": "missioncore.k1-connection-supervisor/v1", + "revision": self.revision, + "closed": self.closed, + "intent": self.intent.as_dict() if self.intent is not None else None, + "observed": { + "device_network": self.device_network.as_dict(), + "host_path": self.host_path.as_dict(), + "endpoint": self.endpoint.as_dict(), + "device_identity": self.device_identity.as_dict(), + "control_plane": self.control_plane.as_dict(), + "data_plane": self.data_plane.as_dict(), + }, + "lease": self.lease.as_dict(), + "authority": self.authority.as_dict(), + "last_known": self.last_known.as_dict() if self.last_known is not None else None, + "diagnostics": [diagnostic.as_dict() for diagnostic in self.diagnostics], + "allowed_actions": list(self.allowed_actions), + "loss_confirmation": { + "host_path_negative_streak": self.host_path_negative_streak, + "endpoint_negative_streak": self.endpoint_negative_streak, + }, + } + + +class ConnectionSupervisor: + """Thread-safe, read-only connection evidence reducer for one K1 runtime. + + This class cannot scan BLE, provision Wi-Fi, reconnect MQTT, or publish a + command. Callers inject read-only observations. A reachable TCP port is only + a transport fact; control authority requires a current-epoch + :class:`VerifiedControlEvidence` containing exact DeviceInfo identity. + """ + + def __init__( + self, + *, + clock: Callable[[], datetime] | None = None, + monotonic_clock: Callable[[], float] | None = None, + suspend_aware_clock: Callable[[], float] | None = None, + observation_ttl_seconds: float = DEFAULT_TRANSPORT_OBSERVATION_TTL_SECONDS, + control_proof_ttl_seconds: float = DEFAULT_CONTROL_PROOF_TTL_SECONDS, + ) -> None: + if ( + not isinstance(observation_ttl_seconds, (int, float)) + or isinstance(observation_ttl_seconds, bool) + or not 0.25 <= float(observation_ttl_seconds) <= 300.0 + ): + raise ConnectionSupervisorError("observation TTL must be within 0.25..300 seconds") + if ( + not isinstance(control_proof_ttl_seconds, (int, float)) + or isinstance(control_proof_ttl_seconds, bool) + or not 1.0 <= float(control_proof_ttl_seconds) <= 300.0 + ): + raise ConnectionSupervisorError("control proof TTL must be within 1..300 seconds") + self._lock = threading.RLock() + self._clock = clock or (lambda: datetime.now(UTC)) + self._monotonic_clock = monotonic_clock or time.monotonic + # ``time.monotonic`` on macOS is backed by mach_absolute_time and does + # not reliably advance while the machine sleeps. The wall clock does; + # using both means NTP cannot extend authority, while suspend/wake + # cannot preserve a pre-sleep route lease either. + self._suspend_aware_clock = suspend_aware_clock or time.time + self._observation_ttl_seconds = float(observation_ttl_seconds) + self._control_proof_ttl_seconds = float(control_proof_ttl_seconds) + self._revision = 0 + self._closed = False + self._intent: ConnectionIntent | None = None + self._device_network = self._empty_device_network() + self._host_path = HostPathObservation( + epoch=0, + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-path-not-observed", + observed_at=self._now(), + ) + self._endpoint = self._empty_endpoint() + self._device_identity = self._empty_identity() + self._control_plane = self._empty_plane() + self._data_plane = self._empty_plane() + self._data_plane_intent_id: str | None = None + self._lease_generation = 0 + self._lease = self._empty_lease() + self._last_known: LastKnownConnection | None = None + self._host_path_observed_monotonic: float | None = None + self._endpoint_observed_monotonic: float | None = None + self._host_path_observed_suspend_aware: float | None = None + self._endpoint_observed_suspend_aware: float | None = None + self._host_path_negative_streak = 0 + self._endpoint_negative_streak = 0 + self._control_proof_session_id: str | None = None + self._control_proof_revision = 0 + self._control_proof_source: str | None = None + self._control_proof_observed_monotonic: float | None = None + self._control_proof_observed_suspend_aware: float | None = None + + def set_intent( + self, + *, + intent_id: str, + requested_mode: ConnectionMode, + expected_device_id: str | None = None, + ) -> ConnectionSupervisorSnapshot: + if not _nonblank(intent_id): + raise ConnectionSupervisorError("intent_id must be nonblank") + if expected_device_id is not None and not _nonblank(expected_device_id): + raise ConnectionSupervisorError("expected_device_id must be nonblank when present") + with self._lock: + self._require_open_locked() + previous_intent = self._intent + intent_changed = ( + previous_intent is None + or previous_intent.intent_id != intent_id + or previous_intent.requested_mode != requested_mode + or previous_intent.expected_device_id != expected_device_id + ) + self._intent = ConnectionIntent( + intent_id=intent_id, + requested_mode=requested_mode, + expected_device_id=expected_device_id, + requested_at=self._now(), + ) + # An explicit new network intent invalidates every authority derived + # from the old route. The old target remains last-known context, + # but it cannot silently become control-authoritative again merely + # because its TCP port is still open. + if intent_changed: + self._host_path_negative_streak = 0 + self._endpoint_negative_streak = 0 + had_control = self._control_plane.state == "healthy" + self._endpoint = self._empty_endpoint( + host_path_epoch=(self._host_path.epoch or None), + intent_id=intent_id, + ) + self._device_identity = self._empty_identity() + self._control_plane = ( + self._lost_plane( + self._host_path.epoch, + "connection-intent-changed", + previous=self._control_plane, + ) + if had_control + else self._empty_plane(host_path_epoch=(self._host_path.epoch or None)) + ) + self._data_plane = self._terminate_data_session( + self._host_path.epoch, + "connection-intent-changed", + ) + self._revoke_lease_locked("connection-intent-changed") + self._revision += 1 + return self._snapshot_locked() + + def observe_device_network_applied( + self, + *, + intent_id: str, + transport_ref: str, + connection_mode: ConnectionMode, + target: EndpointTarget, + source: DeviceNetworkSource, + ) -> bool: + """Commit one exact post-write or read-only BLE topology observation. + + The observation says only what the K1 reported about its own network. + It grants no host/control authority by itself and is rejected when it + belongs to a superseded operator intent. + """ + + if not _nonblank(intent_id) or not _nonblank(transport_ref): + raise ConnectionSupervisorError( + "device network evidence requires exact intent and transport identity" + ) + with self._lock: + self._require_open_locked() + self._expire_stale_observations_locked() + if ( + self._intent is None + or self._intent.intent_id != intent_id + or self._intent.requested_mode != connection_mode + ): + return False + previous = self._device_network + topology_changed = bool( + previous.state != "applied" + or previous.intent_id != intent_id + or previous.transport_ref != transport_ref + or previous.connection_mode != connection_mode + or previous.target != target + ) + self._device_network = DeviceNetworkObservation( + state="applied", + intent_id=intent_id, + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + source=source, + observed_at=self._now(), + ) + if topology_changed: + self._endpoint_negative_streak = 0 + self._endpoint = self._empty_endpoint( + host_path_epoch=(self._host_path.epoch or None), + intent_id=intent_id, + ) + self._device_identity = self._empty_identity() + self._control_plane = self._empty_plane( + host_path_epoch=(self._host_path.epoch or None) + ) + self._data_plane = self._empty_plane( + host_path_epoch=(self._host_path.epoch or None) + ) + self._data_plane_intent_id = None + self._revoke_lease_locked("device-network-applied-awaiting-host-observation") + self._revision += 1 + return True + + def observe_host_path(self, result: HostPathProbeResult) -> int: + """Record one route observation and return its authoritative epoch.""" + + with self._lock: + self._require_open_locked() + previous = self._host_path + first_observation = previous.epoch == 0 + continuity_changed = ( + first_observation + or previous.available != result.available + or previous.fingerprint != result.fingerprint + or previous.interface != result.interface + or previous.source_ipv4 != result.source_ipv4 + or previous.route_class != result.route_class + or previous.kernel_route_fingerprint + != result.kernel_route_fingerprint + ) + epoch = previous.epoch + 1 if continuity_changed else previous.epoch + self._host_path = HostPathObservation( + epoch=epoch, + available=result.available, + fingerprint=result.fingerprint, + interface=result.interface, + source_ipv4=result.source_ipv4, + route_class=result.route_class, + reason_code=result.reason_code, + observed_at=self._now(), + kernel_route_fingerprint=result.kernel_route_fingerprint, + ) + self._host_path_observed_monotonic = self._monotonic_now() + self._host_path_observed_suspend_aware = self._suspend_aware_now() + self._host_path_negative_streak = ( + 0 if result.available else self._host_path_negative_streak + 1 + ) + if continuity_changed: + self._endpoint_negative_streak = 0 + self._endpoint = self._empty_endpoint( + host_path_epoch=epoch, + intent_id=(self._intent.intent_id if self._intent is not None else None), + ) + if self._device_identity.state == "verified": + self._device_identity = DeviceIdentityObservation( + state="stale", + intent_id=self._device_identity.intent_id, + logical_device_id=self._device_identity.logical_device_id, + compatibility_profile_id=(self._device_identity.compatibility_profile_id), + connection_mode=self._device_identity.connection_mode, + source=self._device_identity.source, + host_path_epoch=self._device_identity.host_path_epoch, + observed_at=self._device_identity.observed_at, + ) + had_control_continuity = bool( + self._control_plane.state == "healthy" + or self._lease.state == "reachable" + or self._last_known is not None + ) + self._control_plane = ( + self._lost_plane( + epoch, + "host-path-epoch-changed", + previous=self._control_plane, + ) + if had_control_continuity + else self._empty_plane(host_path_epoch=epoch) + ) + # The producer owns independent evidence continuity. Keep its + # original epoch so surviving packets remain evidence-only. + self._revoke_lease_locked( + "host-path-observed-awaiting-endpoint" + if first_observation and result.available + else result.reason_code + if first_observation and not result.available and result.reason_code + else "host-path-epoch-changed" + ) + elif not result.available: + self._revoke_lease_locked(result.reason_code or "host-path-unavailable") + self._revision += 1 + return epoch + + def observe_host_path_if_current( + self, + *, + expected_intent_id: str, + expected_target: EndpointTarget, + expected_host_path_observation: HostPathObservation | None = None, + expected_host_path_epoch: int | None = None, + result: HostPathProbeResult, + ) -> int | None: + """Atomically publish a monitor path only for its exact applied target.""" + + if not _nonblank(expected_intent_id): + raise ConnectionSupervisorError("monitor host path requires an exact intent") + if expected_host_path_epoch is not None and expected_host_path_epoch < 0: + raise ConnectionSupervisorError("expected host path epoch must be nonnegative") + with self._lock: + if not self._monitor_context_is_current_locked( + expected_intent_id=expected_intent_id, + expected_target=expected_target, + ): + return None + if ( + expected_host_path_epoch is not None + and self._host_path.epoch != expected_host_path_epoch + ): + return None + if ( + expected_host_path_observation is not None + and self._host_path is not expected_host_path_observation + ): + return None + # The lock is re-entrant. Keeping it held across the established + # reducer makes the context and optional observation-identity + # comparison plus publication one CAS. + return self.observe_host_path(result) + + def record_monitor_failure_if_current( + self, + *, + expected_intent_id: str, + expected_target: EndpointTarget, + reason_code: str, + ) -> bool: + """Atomically fail closed only when the probed intent/target still owns state.""" + + if not _nonblank(reason_code): + raise ConnectionSupervisorError("monitor failure requires a reason code") + return ( + self.observe_host_path_if_current( + expected_intent_id=expected_intent_id, + expected_target=expected_target, + result=HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code=reason_code, + ), + ) + is not None + ) + + def observe_endpoint( + self, + *, + target: EndpointTarget, + intent_id: str, + host_path_epoch: int, + reachable: bool, + reason_code: str | None = None, + ) -> bool: + """Record one TCP probe; stale async results are rejected by epoch.""" + + with self._lock: + self._require_open_locked() + if ( + self._intent is None + or intent_id != self._intent.intent_id + or not self._host_path.available + or host_path_epoch != self._host_path.epoch + ): + return False + target_changed = self._endpoint.target not in (None, target) + if target_changed: + self._endpoint_negative_streak = 0 + self._device_identity = self._empty_identity() + self._control_plane = self._empty_plane(host_path_epoch=host_path_epoch) + self._data_plane = self._empty_plane(host_path_epoch=host_path_epoch) + self._data_plane_intent_id = None + self._revoke_lease_locked("endpoint-target-changed") + self._endpoint = EndpointObservation( + target=target, + tcp_state="reachable" if reachable else "unreachable", + intent_id=intent_id, + host_path_epoch=host_path_epoch, + reason_code=reason_code, + observed_at=self._now(), + ) + self._endpoint_observed_monotonic = self._monotonic_now() + self._endpoint_observed_suspend_aware = self._suspend_aware_now() + self._endpoint_negative_streak = ( + 0 if reachable else self._endpoint_negative_streak + 1 + ) + if reachable: + device_network_matches = bool( + self._device_network.state == "applied" + and self._device_network.intent_id == intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target == target + ) + if self._lease.state != "reachable" and device_network_matches: + self._lease = ConnectionLease( + state="configured-unverified", + generation=self._lease_generation, + intent_id=intent_id, + host_path_epoch=host_path_epoch, + connection_mode=( + self._intent.requested_mode if self._intent is not None else None + ), + target=target, + logical_device_id=None, + reason_code="device-identity-unverified", + observed_at=self._now(), + ) + else: + self._control_plane = self._lost_plane( + host_path_epoch, + reason_code or "endpoint-unreachable", + previous=self._control_plane, + ) + # TCP loss proves nothing about the independently observed + # data producer; only its exact session may change that plane. + self._revoke_lease_locked(reason_code or "endpoint-unreachable") + self._revision += 1 + return True + + def observe_endpoint_if_current( + self, + *, + target: EndpointTarget, + intent_id: str, + host_path_epoch: int, + reachable: bool, + reason_code: str | None = None, + ) -> bool: + """Atomically publish a monitor endpoint only for its exact applied target.""" + + if not _nonblank(intent_id): + raise ConnectionSupervisorError("monitor endpoint requires an exact intent") + with self._lock: + if not self._monitor_context_is_current_locked( + expected_intent_id=intent_id, + expected_target=target, + ): + return False + return self.observe_endpoint( + target=target, + intent_id=intent_id, + host_path_epoch=host_path_epoch, + reachable=reachable, + reason_code=reason_code, + ) + + def observe_control_evidence(self, evidence: VerifiedControlEvidence) -> bool: + """Promote only current-epoch TCP + exact DeviceInfo/control evidence.""" + + with self._lock: + self._require_open_locked() + self._expire_stale_observations_locked() + if ( + self._intent is None + or evidence.intent_id != self._intent.intent_id + or evidence.connection_mode != self._intent.requested_mode + or not self._host_path.available + or evidence.host_path_epoch != self._host_path.epoch + or self._endpoint.host_path_epoch != evidence.host_path_epoch + or self._endpoint.target != evidence.target + or self._endpoint.tcp_state != "reachable" + or self._device_network.state != "applied" + or self._device_network.intent_id != evidence.intent_id + or self._device_network.transport_ref != evidence.transport_ref + or self._device_network.connection_mode != evidence.connection_mode + or self._device_network.target != evidence.target + ): + return False + if ( + self._control_proof_session_id == evidence.control_session_id + and evidence.control_proof_revision <= self._control_proof_revision + ): + # A cached DeviceInfo/session snapshot must not resurrect a + # proof that already expired or extend a healthy proof merely + # because the API snapshot was polled again. + return bool( + self._control_plane.state == "healthy" + and self._control_plane.session_id == evidence.control_session_id + and self._device_identity.state == "verified" + and self._lease.state == "reachable" + ) + expected_device_id = ( + self._intent.expected_device_id if self._intent is not None else None + ) + observed_at = self._now() + if expected_device_id is not None and evidence.logical_device_id != expected_device_id: + self._device_identity = DeviceIdentityObservation( + state="mismatch", + intent_id=evidence.intent_id, + logical_device_id=evidence.logical_device_id, + compatibility_profile_id=evidence.compatibility_profile_id, + connection_mode=evidence.connection_mode, + source="mqtt-device-info", + host_path_epoch=evidence.host_path_epoch, + observed_at=observed_at, + ) + self._control_plane = self._lost_plane( + evidence.host_path_epoch, + "device-identity-mismatch", + previous=self._control_plane, + ) + self._revoke_lease_locked("device-identity-mismatch") + self._revision += 1 + return True + self._device_identity = DeviceIdentityObservation( + state="verified", + intent_id=evidence.intent_id, + logical_device_id=evidence.logical_device_id, + compatibility_profile_id=evidence.compatibility_profile_id, + connection_mode=evidence.connection_mode, + source="mqtt-device-info", + host_path_epoch=evidence.host_path_epoch, + observed_at=observed_at, + ) + self._control_plane = PlaneObservation( + state="healthy", + session_id=evidence.control_session_id, + host_path_epoch=evidence.host_path_epoch, + reason_code=None, + observed_at=observed_at, + ) + self._record_control_proof_locked( + control_session_id=evidence.control_session_id, + revision=evidence.control_proof_revision, + source=evidence.control_proof_source, + ) + if self._data_plane.state == "idle": + self._data_plane = self._empty_plane(host_path_epoch=evidence.host_path_epoch) + lease_changed = ( + self._lease.state != "reachable" + or self._lease.host_path_epoch != evidence.host_path_epoch + or self._lease.target != evidence.target + or self._lease.logical_device_id != evidence.logical_device_id + or self._lease.connection_mode != evidence.connection_mode + ) + if lease_changed: + self._lease_generation += 1 + self._lease = ConnectionLease( + state="reachable", + generation=self._lease_generation, + intent_id=evidence.intent_id, + host_path_epoch=evidence.host_path_epoch, + connection_mode=evidence.connection_mode, + target=evidence.target, + logical_device_id=evidence.logical_device_id, + reason_code=None, + observed_at=observed_at, + ) + self._last_known = LastKnownConnection( + connection_mode=evidence.connection_mode, + target=evidence.target, + logical_device_id=evidence.logical_device_id, + compatibility_profile_id=evidence.compatibility_profile_id, + verified_at=observed_at, + ) + self._revision += 1 + return True + + def refresh_control_evidence( + self, + *, + intent_id: str, + host_path_epoch: int, + control_session_id: str, + control_proof_revision: int, + control_proof_source: str, + ) -> bool: + """Advance freshness only for new evidence on the exact live session. + + UI polling cannot keep authority alive: the caller must present a + strictly increasing revision minted by the MQTT transport on receipt + of remote evidence. This reducer neither publishes a challenge nor + reconnects the control socket. + """ + + if not _nonblank(intent_id) or not _nonblank(control_session_id): + raise ConnectionSupervisorError( + "control proof refresh requires exact intent and session identity" + ) + if not _nonblank(control_proof_source): + raise ConnectionSupervisorError("control proof refresh requires a source") + if ( + isinstance(control_proof_revision, bool) + or not isinstance(control_proof_revision, int) + or control_proof_revision < 1 + ): + raise ConnectionSupervisorError("control proof revision must be positive") + with self._lock: + self._require_open_locked() + self._expire_stale_observations_locked(include_control=False) + if ( + self._intent is None + or self._intent.intent_id != intent_id + or self._host_path.epoch != host_path_epoch + or self._control_plane.state != "healthy" + or self._control_plane.session_id != control_session_id + or self._control_plane.host_path_epoch != host_path_epoch + or self._lease.state != "reachable" + or self._lease.intent_id != intent_id + or self._lease.host_path_epoch != host_path_epoch + or self._control_proof_session_id != control_session_id + or control_proof_revision <= self._control_proof_revision + ): + return False + self._record_control_proof_locked( + control_session_id=control_session_id, + revision=control_proof_revision, + source=control_proof_source, + ) + self._revision += 1 + return True + + def observe_control_loss( + self, + *, + intent_id: str, + host_path_epoch: int, + control_session_id: str, + reason_code: str, + ) -> bool: + if not _nonblank(reason_code): + raise ConnectionSupervisorError("control loss requires a reason code") + if not _nonblank(intent_id) or not _nonblank(control_session_id): + raise ConnectionSupervisorError( + "control loss requires exact intent and control session identity" + ) + with self._lock: + self._require_open_locked() + self._expire_stale_observations_locked() + if ( + self._intent is None + or intent_id != self._intent.intent_id + or host_path_epoch != self._host_path.epoch + or control_session_id != self._control_plane.session_id + ): + return False + self._control_plane = self._lost_plane( + host_path_epoch, + reason_code, + previous=self._control_plane, + ) + self._revoke_lease_locked(reason_code) + self._revision += 1 + return True + + def observe_data_plane( + self, + *, + intent_id: str, + host_path_epoch: int, + state: PlaneState, + session_id: str | None = None, + reason_code: str | None = None, + ) -> bool: + """Record data evidence without deriving it from control continuity. + + A data session is bound to the host epoch on which it first became + healthy. The same exact session may continue to report evidence after + route, TCP, or control loss, but it keeps that original epoch and is + therefore non-authoritative for a replacement control binding. A new + data session still requires the current reachable lease. + """ + + with self._lock: + self._require_open_locked() + self._expire_stale_observations_locked() + if self._intent is None or intent_id != self._intent.intent_id: + return False + exact_data_session = bool( + self._data_plane.state != "idle" + and self._data_plane_intent_id == intent_id + and self._data_plane.session_id == session_id + and self._data_plane.host_path_epoch == host_path_epoch + ) + current_control_binding = bool( + host_path_epoch == self._host_path.epoch and self._lease.state == "reachable" + ) + if state == "idle" and host_path_epoch != self._host_path.epoch: + return False + if state == "healthy" and not (exact_data_session or current_control_binding): + return False + if state in {"stalled", "lost"} and not exact_data_session: + return False + if state in {"healthy", "stalled", "lost"} and not _nonblank(session_id): + raise ConnectionSupervisorError(f"{state} data plane requires a session id") + if state in {"stalled", "lost"} and not _nonblank(reason_code): + raise ConnectionSupervisorError(f"{state} data plane requires a reason code") + self._data_plane = PlaneObservation( + state=state, + session_id=session_id, + host_path_epoch=host_path_epoch, + reason_code=reason_code, + observed_at=self._now(), + ) + self._data_plane_intent_id = None if state == "idle" else intent_id + self._revision += 1 + return True + + def record_monitor_failure(self, reason_code: str) -> None: + """Fail closed when the read-only monitoring cycle itself cannot run.""" + + if not _nonblank(reason_code): + raise ConnectionSupervisorError("monitor failure requires a reason code") + self.observe_host_path( + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code=reason_code, + ) + ) + + def preferred_target(self) -> EndpointTarget | None: + with self._lock: + self._expire_stale_observations_locked() + if ( + self._intent is not None + and self._device_network.state == "applied" + and self._device_network.intent_id == self._intent.intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target is not None + ): + return self._device_network.target + if self._last_known is not None: + return self._last_known.target + return None + + def snapshot(self) -> ConnectionSupervisorSnapshot: + with self._lock: + self._expire_stale_observations_locked() + return self._snapshot_locked() + + def association_timeout_retention_candidate( + self, + *, + expected_target: EndpointTarget, + ) -> ConnectionSupervisorSnapshot | None: + """Return the exact current route candidate without running TTL reduction. + + A command-bound kernel/CoreWLAN sample is itself the route refresh. If + its bounded association observer crosses the silence TTL, expiring the + previous observation *after* that fresh sample but before publishing it + would manufacture a route loss. This narrow accessor therefore only + exposes the immutable current snapshot when the applied intent/target + still match. The caller must compare every raw route field and publish + through :meth:`observe_host_path_if_current`; it grants no authority and + cannot commit an observation on its own. + """ + + with self._lock: + intent = self._intent + if intent is None or not self._monitor_context_is_current_locked( + expected_intent_id=intent.intent_id, + expected_target=expected_target, + ): + return None + return self._snapshot_locked() + + def close(self) -> ConnectionSupervisorSnapshot: + with self._lock: + if not self._closed: + self._closed = True + self._control_plane = self._lost_plane( + self._host_path.epoch, + "supervisor-closed", + previous=self._control_plane, + ) + self._data_plane = self._terminate_data_session( + self._host_path.epoch, + "supervisor-closed", + ) + self._revoke_lease_locked("supervisor-closed") + self._revision += 1 + return self._snapshot_locked() + + def _snapshot_locked(self) -> ConnectionSupervisorSnapshot: + authority = self._authority_locked() + diagnostics = host_diagnostics_for_reasons( + self._host_path.reason_code, + self._endpoint.reason_code, + self._control_plane.reason_code, + self._lease.reason_code, + ) + return ConnectionSupervisorSnapshot( + revision=self._revision, + closed=self._closed, + intent=self._intent, + device_network=self._device_network, + host_path=self._host_path, + endpoint=self._endpoint, + device_identity=self._device_identity, + control_plane=self._control_plane, + data_plane=self._data_plane, + lease=self._lease, + authority=authority, + last_known=self._last_known, + diagnostics=diagnostics, + allowed_actions=self._allowed_actions_locked(authority), + host_path_negative_streak=self._host_path_negative_streak, + endpoint_negative_streak=self._endpoint_negative_streak, + ) + + def _monitor_context_is_current_locked( + self, + *, + expected_intent_id: str, + expected_target: EndpointTarget, + ) -> bool: + """Match one monitor observation to the exact current applied topology.""" + + return bool( + not self._closed + and self._intent is not None + and self._intent.intent_id == expected_intent_id + and self._device_network.state == "applied" + and self._device_network.intent_id == expected_intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target == expected_target + ) + + def _authority_locked(self) -> ConnectionAuthority: + reasons = ["read-only-supervisor", "physical-motion-out-of-scope"] + epochs_match = ( + self._lease.host_path_epoch + == self._host_path.epoch + == self._endpoint.host_path_epoch + == self._device_identity.host_path_epoch + == self._control_plane.host_path_epoch + ) + intent_matches = bool( + self._intent is not None + and self._device_network.state == "applied" + and self._device_network.intent_id == self._intent.intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target == self._endpoint.target + and self._endpoint.intent_id == self._intent.intent_id + and self._device_identity.intent_id == self._intent.intent_id + and self._lease.intent_id == self._intent.intent_id + ) + control_allowed = bool( + not self._closed + and self._host_path.available + and self._host_path.route_class == "direct" + and self._endpoint.tcp_state == "reachable" + and self._device_identity.state == "verified" + and self._control_plane.state == "healthy" + and self._lease.state == "reachable" + and epochs_match + and intent_matches + ) + if self._closed: + reasons.append("supervisor-closed") + elif not self._host_path.available: + reasons.append("host-path-unavailable") + elif self._host_path.route_class != "direct": + reasons.append("host-route-not-direct") + elif self._endpoint.tcp_state != "reachable": + reasons.append("endpoint-not-reachable") + elif self._device_identity.state != "verified": + reasons.append(f"device-identity-{self._device_identity.state}") + elif self._control_plane.state != "healthy": + reasons.append("control-plane-not-healthy") + elif self._lease.state != "reachable" or not epochs_match or not intent_matches: + reasons.append("connection-lease-not-reachable") + data_degraded = self._data_plane.state in {"stalled", "lost"} + if data_degraded: + reasons.append(f"data-plane-{self._data_plane.state}") + return ConnectionAuthority( + network_mutation_allowed=False, + control_allowed=control_allowed, + acquisition_start_allowed=control_allowed and not data_degraded, + data_ingest_authoritative=( + control_allowed + and self._data_plane.state == "healthy" + and self._data_plane.host_path_epoch == self._host_path.epoch + ), + physical_motion_allowed=False, + reason_codes=tuple(reasons), + ) + + def _allowed_actions_locked( + self, + authority: ConnectionAuthority, + ) -> tuple[AllowedAction, ...]: + if self._closed: + return () + actions: list[AllowedAction] = ["select-connection-intent"] + current_device_network = bool( + self._intent is not None + and self._device_network.state == "applied" + and self._device_network.intent_id == self._intent.intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target is not None + ) + if not current_device_network: + return tuple(actions) + if not self._host_path.available: + actions.append("inspect-host-network") + return tuple(actions) + if self._endpoint.target is None or self._endpoint.tcp_state != "reachable": + actions.append("probe-endpoint") + return tuple(actions) + if not authority.control_allowed: + actions.append("verify-control-device-info") + if self._data_plane.state == "lost": + actions.append("acknowledge-data-loss") + elif self._data_plane.state in {"healthy", "stalled"}: + actions.append("stop-acquisition") + elif authority.acquisition_start_allowed: + actions.append("start-acquisition") + return tuple(actions) + + def _revoke_lease_locked(self, reason_code: str) -> None: + current_network = bool( + self._intent is not None + and self._device_network.state == "applied" + and self._device_network.intent_id == self._intent.intent_id + and self._device_network.connection_mode == self._intent.requested_mode + and self._device_network.target is not None + ) + state: LeaseState = ( + "configured-unverified" + if current_network + else "lost" + if self._last_known is not None + else "absent" + ) + connection_mode = ( + self._device_network.connection_mode + if current_network + else self._last_known.connection_mode + if self._last_known is not None + else None + ) + target = ( + self._device_network.target + if current_network + else self._last_known.target + if self._last_known is not None + else None + ) + next_intent_id = self._intent.intent_id if self._intent is not None else None + next_host_epoch = self._host_path.epoch if self._host_path.epoch > 0 else None + semantic_change = bool( + self._lease.state != state + or self._lease.intent_id != next_intent_id + or self._lease.host_path_epoch != next_host_epoch + or self._lease.connection_mode != connection_mode + or self._lease.target != target + or self._lease.reason_code != reason_code + ) + if semantic_change and self._lease.state in {"reachable", "configured-unverified"}: + self._lease_generation += 1 + self._lease = ConnectionLease( + state=state, + generation=self._lease_generation, + intent_id=next_intent_id, + host_path_epoch=next_host_epoch, + connection_mode=connection_mode, + target=target, + logical_device_id=( + self._last_known.logical_device_id if self._last_known is not None else None + ), + reason_code=reason_code, + observed_at=self._now(), + ) + + def _record_control_proof_locked( + self, + *, + control_session_id: str, + revision: int, + source: str, + ) -> None: + self._control_proof_session_id = control_session_id + self._control_proof_revision = revision + self._control_proof_source = source + self._control_proof_observed_suspend_aware = self._suspend_aware_now() + self._control_proof_observed_monotonic = self._monotonic_now() + + def _expire_stale_observations_locked(self, *, include_control: bool = True) -> None: + """Fail closed when the monitor stops producing fresh route/TCP facts. + + Wall-clock timestamps are evidence for humans. Authority expiry uses a + process-local monotonic clock so NTP changes cannot extend a lease. + Route, endpoint, and control expiry do not fabricate data expiry; the + exact data session reports that transition independently. + """ + + if self._closed: + return + now = self._monotonic_now() + suspend_aware_now = self._suspend_aware_now() + host_stale = bool( + self._host_path.available + and self._observation_expired( + monotonic_observed=self._host_path_observed_monotonic, + suspend_aware_observed=self._host_path_observed_suspend_aware, + monotonic_now=now, + suspend_aware_now=suspend_aware_now, + ) + ) + if host_stale: + next_epoch = self._host_path.epoch + 1 + self._host_path = HostPathObservation( + epoch=next_epoch, + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-path-observation-stale", + observed_at=self._now(), + ) + self._host_path_observed_monotonic = now + self._host_path_observed_suspend_aware = suspend_aware_now + self._endpoint_observed_monotonic = None + self._endpoint_observed_suspend_aware = None + self._endpoint = self._empty_endpoint( + host_path_epoch=next_epoch, + intent_id=(self._intent.intent_id if self._intent is not None else None), + ) + if self._device_identity.state == "verified": + self._device_identity = DeviceIdentityObservation( + state="stale", + intent_id=self._device_identity.intent_id, + logical_device_id=self._device_identity.logical_device_id, + compatibility_profile_id=self._device_identity.compatibility_profile_id, + connection_mode=self._device_identity.connection_mode, + source=self._device_identity.source, + host_path_epoch=self._device_identity.host_path_epoch, + observed_at=self._device_identity.observed_at, + ) + self._control_plane = self._lost_plane( + next_epoch, + "host-path-observation-stale", + previous=self._control_plane, + ) + self._revoke_lease_locked("host-path-observation-stale") + self._revision += 1 + return + + endpoint_stale = bool( + self._endpoint.tcp_state == "reachable" + and self._observation_expired( + monotonic_observed=self._endpoint_observed_monotonic, + suspend_aware_observed=self._endpoint_observed_suspend_aware, + monotonic_now=now, + suspend_aware_now=suspend_aware_now, + ) + ) + if endpoint_stale: + self._endpoint = EndpointObservation( + target=self._endpoint.target, + tcp_state="unreachable", + intent_id=self._endpoint.intent_id, + host_path_epoch=self._endpoint.host_path_epoch, + reason_code="endpoint-observation-stale", + observed_at=self._now(), + ) + self._endpoint_observed_monotonic = now + self._endpoint_observed_suspend_aware = suspend_aware_now + self._control_plane = self._lost_plane( + self._host_path.epoch, + "endpoint-observation-stale", + previous=self._control_plane, + ) + self._revoke_lease_locked("endpoint-observation-stale") + self._revision += 1 + return + + control_stale = bool( + include_control + and self._control_plane.state == "healthy" + and self._control_plane.session_id is not None + and self._control_plane.session_id == self._control_proof_session_id + and self._observation_expired( + monotonic_observed=self._control_proof_observed_monotonic, + suspend_aware_observed=self._control_proof_observed_suspend_aware, + monotonic_now=now, + suspend_aware_now=suspend_aware_now, + ttl_seconds=self._control_proof_ttl_seconds, + ) + ) + if control_stale: + if self._device_identity.state == "verified": + self._device_identity = DeviceIdentityObservation( + state="stale", + intent_id=self._device_identity.intent_id, + logical_device_id=self._device_identity.logical_device_id, + compatibility_profile_id=(self._device_identity.compatibility_profile_id), + connection_mode=self._device_identity.connection_mode, + source=self._device_identity.source, + host_path_epoch=self._device_identity.host_path_epoch, + observed_at=self._device_identity.observed_at, + ) + self._control_plane = self._lost_plane( + self._host_path.epoch, + "control-proof-observation-stale", + previous=self._control_plane, + ) + self._revoke_lease_locked("control-proof-observation-stale") + self._revision += 1 + + def _empty_device_network(self) -> DeviceNetworkObservation: + return DeviceNetworkObservation( + state="unconfigured", + intent_id=None, + transport_ref=None, + connection_mode=None, + target=None, + source=None, + observed_at=None, + ) + + def _empty_endpoint( + self, + *, + host_path_epoch: int | None = None, + intent_id: str | None = None, + ) -> EndpointObservation: + return EndpointObservation( + target=None, + tcp_state="unknown", + intent_id=intent_id, + host_path_epoch=host_path_epoch, + reason_code=None, + observed_at=None, + ) + + def _empty_identity(self) -> DeviceIdentityObservation: + return DeviceIdentityObservation( + state="unverified", + intent_id=None, + logical_device_id=None, + compatibility_profile_id=None, + connection_mode=None, + source=None, + host_path_epoch=None, + observed_at=None, + ) + + def _empty_plane(self, *, host_path_epoch: int | None = None) -> PlaneObservation: + return PlaneObservation( + state="idle", + session_id=None, + host_path_epoch=host_path_epoch, + reason_code=None, + observed_at=None, + ) + + def _lost_plane( + self, + host_path_epoch: int, + reason_code: str, + *, + previous: PlaneObservation, + ) -> PlaneObservation: + return PlaneObservation( + state="lost", + session_id=previous.session_id, + host_path_epoch=host_path_epoch, + reason_code=reason_code, + observed_at=self._now(), + ) + + def _terminate_data_session( + self, + host_path_epoch: int, + reason_code: str, + ) -> PlaneObservation: + """End data only at an explicit intent or supervisor lifecycle boundary.""" + + if self._data_plane.state in {"healthy", "stalled", "lost"}: + return self._lost_plane( + host_path_epoch, + reason_code, + previous=self._data_plane, + ) + return self._empty_plane(host_path_epoch=host_path_epoch) + + def _empty_lease(self) -> ConnectionLease: + return ConnectionLease( + state="absent", + generation=self._lease_generation, + intent_id=None, + host_path_epoch=None, + connection_mode=None, + target=None, + logical_device_id=None, + reason_code="connection-not-observed", + observed_at=None, + ) + + def _require_open_locked(self) -> None: + if self._closed: + raise ConnectionSupervisorClosed("connection supervisor is closed") + + def _now(self) -> str: + value = self._clock() + if value.tzinfo is None or value.utcoffset() is None: + raise ConnectionSupervisorError("supervisor clock must return an aware datetime") + return value.astimezone(UTC).isoformat().replace("+00:00", "Z") + + def _monotonic_now(self) -> float: + value = self._monotonic_clock() + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + ): + raise ConnectionSupervisorError("monotonic clock must return a number") + return float(value) + + def _suspend_aware_now(self) -> float: + value = self._suspend_aware_clock() + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + ): + raise ConnectionSupervisorError("suspend-aware clock must return a number") + return float(value) + + def _observation_expired( + self, + *, + monotonic_observed: float | None, + suspend_aware_observed: float | None, + monotonic_now: float, + suspend_aware_now: float, + ttl_seconds: float | None = None, + ) -> bool: + monotonic_elapsed = ( + monotonic_now - monotonic_observed if monotonic_observed is not None else float("inf") + ) + suspend_elapsed = ( + suspend_aware_now - suspend_aware_observed + if suspend_aware_observed is not None + else float("inf") + ) + ttl = self._observation_ttl_seconds if ttl_seconds is None else ttl_seconds + return bool( + monotonic_elapsed < 0 + or suspend_elapsed < 0 + or monotonic_elapsed > ttl + or suspend_elapsed > ttl + ) + + +class ReadOnlyConnectionMonitor: + """One-loop lifecycle for injected route and TCP observations only. + + The monitor never owns BLE, MQTT, Wi-Fi association, reconnection, or + command publication. It serializes probes so a delayed TCP result cannot + outlive its host-path epoch. + """ + + def __init__( + self, + supervisor: ConnectionSupervisor, + *, + host_path_probe: HostPathProbe, + tcp_probe: TcpReachabilityProbe, + target_provider: EndpointTargetProvider, + interval_seconds: float = 1.0, + ) -> None: + if ( + not isinstance(interval_seconds, (int, float)) + or isinstance(interval_seconds, bool) + or not 0.05 <= float(interval_seconds) <= 60.0 + ): + raise ConnectionSupervisorError("monitor interval must be within 0.05..60 seconds") + self._supervisor = supervisor + self._host_path_probe = host_path_probe + self._tcp_probe = tcp_probe + self._target_provider = target_provider + self._interval_seconds = float(interval_seconds) + self._owner_loop: asyncio.AbstractEventLoop | None = None + self._poll_lock: asyncio.Lock | None = None + self._task: asyncio.Task[None] | None = None + self._closed = False + self._association_observer_failure_key: ( + tuple[str, EndpointTarget, int, str] | None + ) = None + self._association_observer_failure_host_path: HostPathObservation | None = None + self._association_observer_failure_count = 0 + + def _defer_association_observer_failure( + self, + result: HostPathProbeResult, + *, + intent_id: str, + target: EndpointTarget, + baseline: ConnectionSupervisorSnapshot, + ) -> bool: + """Debounce only a technical association observer failure. + + The kernel route was available when the facade created this typed + result. One or two slow helper starts therefore cannot manufacture a + new host epoch and stale an otherwise exact DeviceInfo/control proof. + A changed interface/source is still a concrete path transition and is + published immediately. The third consecutive failure for the exact + intent and target is admitted fail-closed with its original reason. + """ + + if ( + result.available + or result.observation_failure_class != "association-observer" + ): + self._reset_association_observer_failures() + return False + previous_path = baseline.host_path + if ( + previous_path.kernel_route_fingerprint is None + or result.kernel_route_fingerprint is None + or previous_path.kernel_route_fingerprint + != result.kernel_route_fingerprint + or previous_path.interface != result.interface + or previous_path.source_ipv4 != result.source_ipv4 + ): + self._reset_association_observer_failures() + return False + assert previous_path.kernel_route_fingerprint is not None + key = ( + intent_id, + target, + previous_path.epoch, + previous_path.kernel_route_fingerprint, + ) + if ( + self._association_observer_failure_key != key + or self._association_observer_failure_host_path is not previous_path + ): + self._association_observer_failure_key = key + self._association_observer_failure_host_path = previous_path + self._association_observer_failure_count = 1 + else: + self._association_observer_failure_count += 1 + return self._association_observer_failure_count < 3 + + def _retain_verified_path_for_unproven_association( + self, + result: HostPathProbeResult, + *, + target: EndpointTarget, + baseline: ConnectionSupervisorSnapshot, + ) -> HostPathProbeResult | None: + """Refresh one exact verified path while association identity is unknown. + + CoreWLAN association inspection is an additional continuity signal, + not a substitute for the kernel route or the live control endpoint. + A helper timeout or privacy-limited observation therefore cannot by + itself manufacture a new host epoch after DeviceInfo has already bound + an exact healthy control session. Retention is deliberately narrow: + the raw kernel route tuple must be byte-for-byte unchanged and every + verified control fact must still refer to this exact intent/target. + + Returning the previous proven fingerprint as a fresh positive sample + advances the host observation TTL. The caller must still run the TCP + probe and a second host sample before refreshing endpoint liveness. + Proven association changes, endpoint/control loss and intent/target + transitions continue through their normal immediate fail-closed paths. + """ + + unproven_association = bool( + result.observation_failure_class == "association-observer" + or ( + result.available + and result.reason_code == "association-identity-unavailable" + ) + ) + if not unproven_association: + return None + previous_path = baseline.host_path + intent = baseline.intent + exact_verified_control = bool( + intent is not None + and previous_path.available + and previous_path.fingerprint is not None + and previous_path.kernel_route_fingerprint is not None + and baseline.device_network.state == "applied" + and baseline.device_network.intent_id == intent.intent_id + and baseline.device_network.target == target + and baseline.endpoint.target == target + and baseline.endpoint.intent_id == intent.intent_id + and baseline.endpoint.host_path_epoch == previous_path.epoch + and baseline.endpoint.tcp_state == "reachable" + and baseline.device_identity.state == "verified" + and baseline.device_identity.intent_id == intent.intent_id + and baseline.device_identity.host_path_epoch == previous_path.epoch + and baseline.control_plane.state == "healthy" + and baseline.control_plane.host_path_epoch == previous_path.epoch + and baseline.control_plane.session_id is not None + and baseline.lease.state == "reachable" + and baseline.lease.intent_id == intent.intent_id + and baseline.lease.host_path_epoch == previous_path.epoch + and baseline.lease.target == target + and baseline.authority.control_allowed + ) + if not exact_verified_control: + return None + if ( + result.kernel_route_fingerprint is None + or result.kernel_route_fingerprint + != previous_path.kernel_route_fingerprint + or result.interface != previous_path.interface + or result.source_ipv4 != previous_path.source_ipv4 + ): + return None + self._reset_association_observer_failures() + return HostPathProbeResult( + available=True, + fingerprint=previous_path.fingerprint, + interface=previous_path.interface, + source_ipv4=previous_path.source_ipv4, + route_class=previous_path.route_class, + reason_code=None, + kernel_route_fingerprint=previous_path.kernel_route_fingerprint, + ) + + def _reset_association_observer_failures(self) -> None: + self._association_observer_failure_key = None + self._association_observer_failure_host_path = None + self._association_observer_failure_count = 0 + + async def poll_once(self) -> ConnectionSupervisorSnapshot: + self._bind_loop() + if self._closed: + raise ConnectionSupervisorClosed("connection monitor is closed") + assert self._poll_lock is not None + async with self._poll_lock: + # A second explicit poll may already be queued on the serialization + # lock when close is requested. Re-check after admission so it + # cannot start fresh host/TCP contact against a closed supervisor. + if self._closed: + raise ConnectionSupervisorClosed("connection monitor is closed") + initial_snapshot = self._supervisor.snapshot() + intent = initial_snapshot.intent + if intent is None: + self._reset_association_observer_failures() + return initial_snapshot + intent_id = intent.intent_id + configured_target = initial_snapshot.device_network.target + try: + target = self._target_provider() + if target is not None and not isinstance(target, EndpointTarget): + raise TypeError("target provider returned an invalid target") + except ConnectionMonitorProbeSuperseded: + self._reset_association_observer_failures() + return self._supervisor.snapshot() + except Exception: + self._reset_association_observer_failures() + if configured_target is not None: + self._supervisor.record_monitor_failure_if_current( + expected_intent_id=intent_id, + expected_target=configured_target, + reason_code="target-provider-error", + ) + return self._supervisor.snapshot() + if target is None: + self._reset_association_observer_failures() + if configured_target is not None: + self._supervisor.observe_host_path_if_current( + expected_intent_id=intent_id, + expected_target=configured_target, + result=HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="endpoint-target-unconfigured", + ), + ) + return self._supervisor.snapshot() + try: + path = await self._host_path_probe(target) + if not isinstance(path, HostPathProbeResult): + raise TypeError("host path probe returned an invalid result") + except ConnectionMonitorProbeSuperseded: + self._reset_association_observer_failures() + return self._supervisor.snapshot() + except Exception: + self._reset_association_observer_failures() + if self._closed: + return self._supervisor.snapshot() + if not self._target_is_current( + target, + expected_intent_id=intent_id, + ): + return self._supervisor.snapshot() + self._supervisor.record_monitor_failure_if_current( + expected_intent_id=intent_id, + expected_target=target, + reason_code="host-path-probe-error", + ) + return self._supervisor.snapshot() + if self._closed: + return self._supervisor.snapshot() + # A target change while a route probe was in flight makes the + # result stale. Drop it before it can rotate the host epoch for the + # newly selected target. + current_snapshot = self._supervisor.snapshot() + current_intent = current_snapshot.intent + if ( + current_intent is None + or current_intent.intent_id != intent_id + or not self._target_is_current( + target, + expected_intent_id=intent_id, + ) + ): + self._reset_association_observer_failures() + return self._supervisor.snapshot() + retained_path = self._retain_verified_path_for_unproven_association( + path, + target=target, + baseline=current_snapshot, + ) + if retained_path is not None: + path = retained_path + elif self._defer_association_observer_failure( + path, + intent_id=intent_id, + target=target, + baseline=current_snapshot, + ): + return self._supervisor.snapshot() + epoch = self._supervisor.observe_host_path_if_current( + expected_intent_id=intent_id, + expected_target=target, + expected_host_path_observation=current_snapshot.host_path, + result=path, + ) + if epoch is None: + return self._supervisor.snapshot() + if not path.available: + return self._supervisor.snapshot() + try: + tcp_result = await self._tcp_probe(target) + if isinstance(tcp_result, TcpReachabilityProbeResult): + reachable = tcp_result.reachable + reason_code = tcp_result.reason_code + elif isinstance(tcp_result, bool): + reachable = tcp_result + reason_code = None if reachable else "tcp-endpoint-unreachable" + else: + raise TypeError("TCP probe returned an invalid result") + except ConnectionMonitorProbeSuperseded: + return self._supervisor.snapshot() + except Exception: + reachable = False + reason_code = "tcp-probe-error" + if self._closed: + return self._supervisor.snapshot() + # The host may have changed Wi-Fi while the TCP connect was in + # flight. Re-read the route fingerprint before admitting the TCP + # result; otherwise an old-route success can be attached to a new + # network that happens to expose the same private address. + try: + final_path = await self._host_path_probe(target) + if not isinstance(final_path, HostPathProbeResult): + raise TypeError("host path probe returned an invalid result") + except ConnectionMonitorProbeSuperseded: + self._reset_association_observer_failures() + return self._supervisor.snapshot() + except Exception: + self._reset_association_observer_failures() + if self._closed: + return self._supervisor.snapshot() + if not self._target_is_current( + target, + expected_intent_id=intent_id, + ): + return self._supervisor.snapshot() + self._supervisor.record_monitor_failure_if_current( + expected_intent_id=intent_id, + expected_target=target, + reason_code="host-path-recheck-error", + ) + return self._supervisor.snapshot() + if self._closed: + return self._supervisor.snapshot() + # The second host probe is another asynchronous boundary. A target + # or intent transition while it was in flight supersedes both its + # route result and the preceding TCP result. Guard before *any* + # reducer write, including the changed-path branch below. + final_baseline = self._supervisor.snapshot() + current_intent = final_baseline.intent + if ( + current_intent is None + or current_intent.intent_id != intent_id + or not self._target_is_current( + target, + expected_intent_id=intent_id, + ) + ): + self._reset_association_observer_failures() + return self._supervisor.snapshot() + retained_final_path = self._retain_verified_path_for_unproven_association( + final_path, + target=target, + baseline=final_baseline, + ) + if retained_final_path is not None: + final_path = retained_final_path + elif self._defer_association_observer_failure( + final_path, + intent_id=intent_id, + target=target, + baseline=final_baseline, + ): + return self._supervisor.snapshot() + if final_path != path: + self._supervisor.observe_host_path_if_current( + expected_intent_id=intent_id, + expected_target=target, + expected_host_path_observation=final_baseline.host_path, + result=final_path, + ) + return self._supervisor.snapshot() + self._supervisor.observe_endpoint_if_current( + target=target, + intent_id=intent_id, + host_path_epoch=epoch, + reachable=reachable, + reason_code=reason_code, + ) + return self._supervisor.snapshot() + + async def start(self) -> None: + loop = self._bind_loop() + if self._closed: + raise ConnectionSupervisorClosed("connection monitor is closed") + if self._task is not None and not self._task.done(): + return + self._task = loop.create_task(self._run(), name="k1-read-only-connection-monitor") + + async def close(self) -> None: + loop = self._bind_loop() + self.request_close() + task = self._task + self._task = None + if task is not None and task is not asyncio.current_task(loop=loop): + task.cancel() + with suppress(asyncio.CancelledError): + await task + + def request_close(self) -> None: + """Synchronously revoke authority and cancel the owner-loop task. + + Service shutdown is synchronous, while the monitor is asyncio-owned. + This method makes close deterministic without moving the monitor to a + second event loop. + """ + + if self._closed: + return + self._closed = True + self._supervisor.close() + task = self._task + owner_loop = self._owner_loop + if task is None or task.done(): + return + if owner_loop is not None and owner_loop.is_running(): + owner_loop.call_soon_threadsafe(task.cancel) + else: + task.cancel() + + async def __aenter__(self) -> ReadOnlyConnectionMonitor: + await self.start() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: object | None, + ) -> None: + await self.close() + + async def _run(self) -> None: + failure_streak = 0 + while not self._closed: + delay_seconds = self._interval_seconds + poll_context = self._supervisor.snapshot() + try: + await self.poll_once() + failure_streak = 0 + except asyncio.CancelledError: + raise + except ConnectionSupervisorClosed: + return + except Exception: + if self._closed: + return + if ( + poll_context.intent is not None + and poll_context.device_network.target is not None + ): + self._supervisor.record_monitor_failure_if_current( + expected_intent_id=poll_context.intent.intent_id, + expected_target=poll_context.device_network.target, + reason_code="connection-monitor-loop-failed", + ) + failure_streak += 1 + # An unexpected bug must revoke authority, but it must not + # silently kill the only observer forever. Retry the + # read-only monitor with bounded exponential backoff; no BLE, + # Wi-Fi, MQTT reconnect, or physical command is performed. + delay_seconds = min( + self._interval_seconds * (2 ** min(failure_streak - 1, 5)), + max(30.0, self._interval_seconds), + ) + await asyncio.sleep(delay_seconds) + + def _target_is_current( + self, + expected: EndpointTarget, + *, + expected_intent_id: str, + ) -> bool: + try: + current = self._target_provider() + except ConnectionMonitorProbeSuperseded: + return False + except Exception: + self._supervisor.record_monitor_failure_if_current( + expected_intent_id=expected_intent_id, + expected_target=expected, + reason_code="target-provider-error", + ) + return False + return current == expected + + def _bind_loop(self) -> asyncio.AbstractEventLoop: + loop = asyncio.get_running_loop() + if self._owner_loop is None: + self._owner_loop = loop + self._poll_lock = asyncio.Lock() + elif self._owner_loop is not loop: + raise ConnectionSupervisorError( + "connection monitor cannot move between asyncio event loops" + ) + return loop + + +def _nonblank(value: object) -> bool: + return isinstance(value, str) and bool(value.strip()) diff --git a/src/k1link/device_plugins/xgrids_k1/device_identity_pin_store.py b/src/k1link/device_plugins/xgrids_k1/device_identity_pin_store.py new file mode 100644 index 0000000..6c57dd1 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/device_identity_pin_store.py @@ -0,0 +1,641 @@ +from __future__ import annotations + +import fcntl +import hmac +import json +import os +import re +import stat +import tempfile +import threading +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import IO, Literal, cast + +from k1link.sessions.store import resolve_missioncore_data_dir + +DEVICE_IDENTITY_PIN_SCHEMA = "missioncore.xgrids-k1-device-identity-pins/v1" +DEVICE_IDENTITY_PIN_FILENAME = "device-identity-pins.json" +DEVICE_IDENTITY_PIN_LOCK_FILENAME = ".device-identity-pins.lock" +DEVICE_IDENTITY_PIN_MAX_BYTES = 64 * 1024 +DEVICE_IDENTITY_PIN_MAX_COUNT = 256 +DEVICE_IDENTITY_PIN_MAX_REVISION = (1 << 63) - 1 + +DeviceIdentityPinStoreStatus = Literal["empty", "available", "corrupt"] + +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_MAX_VENDOR_DEVICE_ID_BYTES = 4 * 1024 + + +class DeviceIdentityPinStoreError(RuntimeError): + """Base error for the durable BLE-to-K1 identity binding.""" + + reason_code = "device-identity-pin-store-error" + + +class DeviceIdentityPinStoreCorrupt(DeviceIdentityPinStoreError): + """The private identity store cannot be trusted and fails closed.""" + + reason_code = "device-identity-pin-store-corrupt" + + +class DeviceIdentityPinMismatch(DeviceIdentityPinStoreError): + """Live DeviceInfo identity does not match the first-contact pin.""" + + reason_code = "device-identity-pin-mismatch" + + def __init__( + self, + *, + transport_ref: str, + expected_vendor_device_id: str, + observed_vendor_device_id: str, + expected_compatibility_profile_id: str, + observed_compatibility_profile_id: str, + ) -> None: + super().__init__( + "live K1 identity/profile does not match the durable BLE transport pin" + ) + self.transport_ref = transport_ref + self.expected_vendor_device_id = expected_vendor_device_id + self.observed_vendor_device_id = observed_vendor_device_id + self.expected_compatibility_profile_id = expected_compatibility_profile_id + self.observed_compatibility_profile_id = observed_compatibility_profile_id + + +@dataclass(frozen=True, slots=True) +class DeviceIdentityPin: + """Immutable first-contact binding for one CoreBluetooth transport.""" + + transport_ref: str + vendor_device_id: str + compatibility_profile_id: str + + def as_dict(self) -> dict[str, str]: + return { + "transport_ref": self.transport_ref, + "vendor_device_id": self.vendor_device_id, + "compatibility_profile_id": self.compatibility_profile_id, + } + + +@dataclass(frozen=True, slots=True) +class DeviceIdentityPinDecision: + pin: DeviceIdentityPin + created: bool + revision: int + + +@dataclass(frozen=True, slots=True) +class DeviceIdentityPinStoreSnapshot: + status: DeviceIdentityPinStoreStatus + revision: int | None + pins: tuple[DeviceIdentityPin, ...] + reason_code: str | None + + def for_transport(self, transport_ref: str) -> DeviceIdentityPin | None: + return next( + (pin for pin in self.pins if pin.transport_ref == transport_ref), + None, + ) + + def as_public_dict(self) -> dict[str, object]: + """Expose store health without publishing durable device identifiers.""" + + return { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "status": self.status, + "revision": self.revision, + "pin_count": len(self.pins), + "reason_code": self.reason_code, + } + + +class DeviceIdentityPinStore: + """Private, atomic first-contact K1 identity pins. + + A pin binds one exact BLE ``transport_ref`` to the logical vendor + ``device_id`` proved by DeviceInfo and the reviewed compatibility profile. + An IP address is intentionally absent: TCP reachability can never substitute + for this identity. The stable flock serializes the complete + reload/check/publish transaction across Mission Core processes. + """ + + def __init__(self, repository_root: Path) -> None: + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = data_dir / "xgrids-k1" / DEVICE_IDENTITY_PIN_FILENAME + self._lock_path = data_dir / "xgrids-k1" / DEVICE_IDENTITY_PIN_LOCK_FILENAME + self._data_dir = data_dir + self._thread_lock = threading.RLock() + self._revision = 0 + self._pins: dict[str, DeviceIdentityPin] = {} + self._corrupt = False + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> DeviceIdentityPinStoreSnapshot: + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return DeviceIdentityPinStoreSnapshot( + status="corrupt", + revision=None, + pins=(), + reason_code=DeviceIdentityPinStoreCorrupt.reason_code, + ) + pins = tuple(self._pins[key] for key in sorted(self._pins)) + if not pins: + return DeviceIdentityPinStoreSnapshot( + status="empty", + revision=None, + pins=(), + reason_code=None, + ) + return DeviceIdentityPinStoreSnapshot( + status="available", + revision=self._revision, + pins=pins, + reason_code=None, + ) + + def pin_or_match( + self, + *, + transport_ref: str, + vendor_device_id: str, + compatibility_profile_id: str, + ) -> DeviceIdentityPinDecision: + """Create the first pin or verify an exact existing pin. + + Matching an existing pin is read-only and does not bump the revision or + rewrite the file. Any identity or profile mismatch is a typed, + fail-closed error and preserves the original bytes. + """ + + _validate_identifier(transport_ref, field_name="transport_ref") + _validate_vendor_device_id(vendor_device_id) + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin store is corrupt; live K1 identity was not adopted" + ) + + current = self._pins.get(transport_ref) + if current is not None: + vendor_matches = hmac.compare_digest( + current.vendor_device_id, + vendor_device_id, + ) + profile_matches = hmac.compare_digest( + current.compatibility_profile_id, + compatibility_profile_id, + ) + if not vendor_matches or not profile_matches: + raise DeviceIdentityPinMismatch( + transport_ref=transport_ref, + expected_vendor_device_id=current.vendor_device_id, + observed_vendor_device_id=vendor_device_id, + expected_compatibility_profile_id=( + current.compatibility_profile_id + ), + observed_compatibility_profile_id=compatibility_profile_id, + ) + return DeviceIdentityPinDecision( + pin=current, + created=False, + revision=self._revision, + ) + + if len(self._pins) >= DEVICE_IDENTITY_PIN_MAX_COUNT: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin store reached its bounded pin count" + ) + if self._revision >= DEVICE_IDENTITY_PIN_MAX_REVISION: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin store revision is exhausted" + ) + + pin = DeviceIdentityPin( + transport_ref=transport_ref, + vendor_device_id=vendor_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + next_pins = dict(self._pins) + next_pins[transport_ref] = pin + revision = self._revision + 1 + self._persist_locked(revision=revision, pins=next_pins) + return DeviceIdentityPinDecision( + pin=pin, + created=True, + revision=revision, + ) + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + data_dir_created = _ensure_private_directory(self._data_dir, parents=True) + if data_dir_created: + _fsync_directory(self._data_dir.parent) + store_dir_created = _ensure_private_directory(self.path.parent, parents=False) + if store_dir_created: + _fsync_directory(self._data_dir) + + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self._lock_path, flags, 0o600) + except OSError as exc: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin lock cannot be opened safely" + ) from exc + stream: IO[bytes] | None = None + try: + try: + _validate_private_open_file( + descriptor, + self._lock_path, + label="device identity pin lock", + require_empty=True, + ) + except ValueError as exc: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin lock is not a stable private file" + ) from exc + stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + try: + try: + _validate_private_open_file( + stream.fileno(), + self._lock_path, + label="device identity pin lock", + require_empty=True, + ) + except ValueError as exc: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin lock changed while being acquired" + ) from exc + _fsync_directory(self.path.parent) + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + if stream is not None: + stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _persist_locked( + self, + *, + revision: int, + pins: Mapping[str, DeviceIdentityPin], + ) -> None: + ordered = tuple(pins[key] for key in sorted(pins)) + payload: dict[str, object] = { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "revision": revision, + "pins": [pin.as_dict() for pin in ordered], + } + _write_private_json_atomic( + self.path, + payload, + data_dir=self._data_dir, + ) + self._revision = revision + self._pins = dict(pins) + self._corrupt = False + + def _reload_locked(self) -> None: + try: + payload = _read_private_json(self.path) + except FileNotFoundError: + self._revision = 0 + self._pins = {} + self._corrupt = False + return + except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): + self._revision = 0 + self._pins = {} + self._corrupt = True + return + try: + revision, pins = _document_from_mapping(payload) + except (TypeError, ValueError): + self._revision = 0 + self._pins = {} + self._corrupt = True + return + self._revision = revision + self._pins = {pin.transport_ref: pin for pin in pins} + self._corrupt = False + + +def _read_private_json(path: Path) -> object: + try: + initial = path.lstat() + except FileNotFoundError: + raise + except OSError as exc: + raise ValueError("device identity pin file cannot be inspected safely") from exc + _validate_private_metadata( + initial, + label="device identity pin file", + require_empty=False, + ) + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise ValueError("device identity pin file cannot be opened safely") from exc + try: + metadata = _validate_private_open_file( + descriptor, + path, + label="device identity pin file", + require_empty=False, + ) + if (initial.st_dev, initial.st_ino) != (metadata.st_dev, metadata.st_ino): + raise ValueError("device identity pin file changed while opening") + if metadata.st_size > DEVICE_IDENTITY_PIN_MAX_BYTES: + raise ValueError("device identity pin file exceeds the bounded size") + chunks: list[bytes] = [] + remaining = DEVICE_IDENTITY_PIN_MAX_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 64 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + raw = b"".join(chunks) + if len(raw) > DEVICE_IDENTITY_PIN_MAX_BYTES: + raise ValueError("device identity pin file exceeds the bounded size") + finally: + os.close(descriptor) + return json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object) + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = ( + json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + if len(serialized) > DEVICE_IDENTITY_PIN_MAX_BYTES: + raise ValueError("device identity pin payload exceeds the bounded size") + + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(path.parent, parents=False) + previous_identity = _existing_private_file_identity(path) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + _require_unchanged_existing_path(path, previous_identity) + os.replace(temp_path, path) + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _existing_private_file_identity(path: Path) -> tuple[int, int] | None: + try: + metadata = path.lstat() + except FileNotFoundError: + return None + _validate_private_metadata( + metadata, + label="device identity pin file", + require_empty=False, + ) + return metadata.st_dev, metadata.st_ino + + +def _require_unchanged_existing_path( + path: Path, + expected: tuple[int, int] | None, +) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + if expected is None: + return + raise ValueError("device identity pin file disappeared during publication") from None + _validate_private_metadata( + metadata, + label="device identity pin file", + require_empty=False, + ) + observed = metadata.st_dev, metadata.st_ino + if expected is None or observed != expected: + raise ValueError("device identity pin file changed during publication") + + +def _validate_private_open_file( + descriptor: int, + path: Path, + *, + label: str, + require_empty: bool, +) -> os.stat_result: + metadata = os.fstat(descriptor) + _validate_private_metadata(metadata, label=label, require_empty=require_empty) + try: + path_metadata = path.lstat() + except OSError as exc: + raise ValueError(f"{label} path cannot be verified") from exc + if (path_metadata.st_dev, path_metadata.st_ino) != ( + metadata.st_dev, + metadata.st_ino, + ): + raise ValueError(f"{label} path does not reference the opened inode") + _validate_private_metadata(path_metadata, label=label, require_empty=False) + return metadata + + +def _validate_private_metadata( + metadata: os.stat_result, + *, + label: str, + require_empty: bool, +) -> None: + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} is not a regular file") + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError(f"{label} is not private") + if metadata.st_nlink != 1: + raise ValueError(f"{label} has an unsafe hard link") + if require_empty and metadata.st_size != 0: + raise ValueError(f"{label} must remain empty") + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + return True + except OSError as exc: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin directory is unavailable" + ) from exc + if not stat.S_ISDIR(metadata.st_mode): + raise DeviceIdentityPinStoreCorrupt( + "device identity pin directory is not a regular private directory" + ) + if stat.S_IMODE(metadata.st_mode) != 0o700: + raise DeviceIdentityPinStoreCorrupt( + "device identity pin directory permissions are not private" + ) + return False + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("device identity pin store contains duplicate fields") + document[key] = value + return document + + +def _document_from_mapping(value: object) -> tuple[int, tuple[DeviceIdentityPin, ...]]: + document = _exact_mapping( + value, + {"schema_version", "revision", "pins"}, + label="device identity pin document", + ) + if document["schema_version"] != DEVICE_IDENTITY_PIN_SCHEMA: + raise ValueError("unsupported device identity pin schema") + revision = _positive_revision(document["revision"]) + raw_pins = document["pins"] + if not isinstance(raw_pins, list) or not 1 <= len(raw_pins) <= DEVICE_IDENTITY_PIN_MAX_COUNT: + raise ValueError("device identity pin list has an invalid bounded size") + + pins: list[DeviceIdentityPin] = [] + seen_transport_refs: set[str] = set() + for raw_pin in raw_pins: + pin_document = _exact_mapping( + raw_pin, + {"transport_ref", "vendor_device_id", "compatibility_profile_id"}, + label="device identity pin", + ) + transport_ref = _required_string( + pin_document["transport_ref"], + field_name="transport_ref", + ) + vendor_device_id = _required_string( + pin_document["vendor_device_id"], + field_name="vendor_device_id", + ) + compatibility_profile_id = _required_string( + pin_document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ) + _validate_identifier(transport_ref, field_name="transport_ref") + _validate_vendor_device_id(vendor_device_id) + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + if transport_ref in seen_transport_refs: + raise ValueError("device identity pin transport_ref is duplicated") + seen_transport_refs.add(transport_ref) + pins.append( + DeviceIdentityPin( + transport_ref=transport_ref, + vendor_device_id=vendor_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + ) + + if pins != sorted(pins, key=lambda pin: pin.transport_ref): + raise ValueError("device identity pins are not in canonical order") + if revision != len(pins): + raise ValueError("device identity pin revision does not match immutable pin count") + return revision, tuple(pins) + + +def _exact_mapping(value: object, keys: set[str], *, label: str) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"{label} does not match the secret-free schema") + return cast(Mapping[str, object], value) + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _validate_vendor_device_id(value: str) -> None: + try: + encoded = value.encode("ascii") + except UnicodeEncodeError as exc: + raise ValueError("vendor_device_id must use printable ASCII") from exc + if not encoded or len(encoded) > _MAX_VENDOR_DEVICE_ID_BYTES: + raise ValueError("vendor_device_id is outside the bounded identity schema") + if any(byte <= 0x20 or byte > 0x7E for byte in encoded): + raise ValueError("vendor_device_id must use printable ASCII without spaces") + + +def _positive_revision(value: object) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or value > DEVICE_IDENTITY_PIN_MAX_REVISION + ): + raise ValueError("device identity pin revision must be a bounded positive integer") + return value diff --git a/src/k1link/device_plugins/xgrids_k1/facade.py b/src/k1link/device_plugins/xgrids_k1/facade.py index 2813e06..c47d8c8 100644 --- a/src/k1link/device_plugins/xgrids_k1/facade.py +++ b/src/k1link/device_plugins/xgrids_k1/facade.py @@ -1,25 +1,34 @@ from __future__ import annotations import asyncio +import errno import hashlib import hmac import importlib.util import json import logging +import os import secrets import socket +import stat import subprocess import sys import threading import time import unicodedata -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Collection, Iterator, Mapping, Sequence +from concurrent.futures import Future as ConcurrentFuture +from concurrent.futures import TimeoutError as ConcurrentFutureTimeout +from contextlib import AbstractContextManager, ExitStack, contextmanager, suppress +from dataclasses import dataclass +from dataclasses import replace as dataclass_replace from datetime import UTC, datetime from functools import wraps from pathlib import Path from typing import Any, Concatenate, Literal, Protocol, Self, cast +from uuid import uuid4 -from bleak.exc import BleakError +from bleak.exc import BleakDeviceNotFoundError, BleakError from missioncore_plugin_sdk.v0alpha2 import ( RuntimeActionInvocation, RuntimePluginDescriptor, @@ -36,13 +45,56 @@ from pydantic import ( from k1link.artifacts import write_json_atomic from k1link.compute.live_perception import LivePerceptionIngress +from k1link.data_plane import DecodedDataPlaneView, DecodedPointCloudView +from k1link.device_plugins.xgrids_k1.active_acquisition_recovery_checkpoint import ( + ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA, + ActiveAcquisitionRecoveryCheckpoint, + ActiveAcquisitionRecoveryCheckpointError, + ActiveAcquisitionRecoveryCheckpointStore, + ActiveAcquisitionRecoveryConnection, + ActiveAcquisitionRecoveryFirstPublishedPclProof, + ActiveAcquisitionRecoveryIdentity, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ActiveAcquisitionRecoveryStartOriginProof, + ActiveAcquisitionRecoveryStatusProof, + ActiveAcquisitionRecoveryTransportBinding, + active_acquisition_project_name_sha256, +) +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + ApplicationControlProcessLease, + ApplicationControlProcessLeaseReleaseAmbiguous, + ApplicationControlProcessLeaseUnavailable, +) +from k1link.device_plugins.xgrids_k1.ble.ap_activation import ( + FRAME_LENGTH as QUICK_CONNECT_FRAME_LENGTH, +) from k1link.device_plugins.xgrids_k1.ble.ap_activation import ( device_ap_activation_session, + is_ap_ready_status, +) +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleRuntimeBusy, + BleRuntimeOwnerLoopConflict, + BleRuntimeProcessLeaseBorrowToken, + bind_ble_runtime_owner_loop, + ble_runtime_snapshot, + borrow_ble_runtime_process_lease, + configure_ble_runtime_process_lease, + defer_until_ble_runtime_idle, + invalidate_ble_runtime_owner_loop, ) from k1link.device_plugins.xgrids_k1.ble.scanner import ( - BLE_DISCOVERY_CANDIDATE_LEASE_TTL_SECONDS as BLE_DISCOVERY_LEASE_TTL_SECONDS, -) -from k1link.device_plugins.xgrids_k1.ble.scanner import ( + BLE_SCAN_HARD_TIMEOUT_GRACE_SECONDS, + CapturedDiscoveredDevice, + capture_discovered_device, + connected_device_capture, + connected_device_recovery_name, + connected_device_recovery_snapshot, + invalidate_connected_device_session, + invalidate_runtime_handles, + pin_connected_device_handle, scan, ) from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( @@ -50,28 +102,87 @@ from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( provision_wifi_once, read_wifi_status_once, ) +from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( + FRAME_LENGTH as BRIDGE_FRAME_LENGTH, +) from k1link.device_plugins.xgrids_k1.calibration_snapshot import ( DeviceCalibrationSnapshotReader, unavailable_device_calibration_snapshot, ) from k1link.device_plugins.xgrids_k1.camera import ( CAMERA_EXCLUSIVE_GROUP, + CAMERA_PRODUCER_STALL_MILLISECONDS, CAMERA_SOURCE_LABELS, CAMERA_SOURCE_PATHS, CameraSourceId, CommittedCameraSegment, XgridsK1CameraGateway, build_xgrids_k1_camera_router, + classify_camera_recording_health, +) +from k1link.device_plugins.xgrids_k1.connection_supervisor import ( + ConnectionMonitorProbeSuperseded, + ConnectionSupervisor, + ConnectionSupervisorSnapshot, + EndpointTarget, + HostPathProbeResult, + ReadOnlyConnectionMonitor, + TcpReachabilityProbeResult, + VerifiedControlEvidence, +) +from k1link.device_plugins.xgrids_k1.device_identity_pin_store import ( + DeviceIdentityPinStore, + DeviceIdentityPinStoreCorrupt, + DeviceIdentityPinStoreError, ) from k1link.device_plugins.xgrids_k1.firmware_credential import ( K1_FW302_CREDENTIAL_PROVIDER_ID, ) +from k1link.device_plugins.xgrids_k1.host_diagnostics import ( + HostDiagnosticBoundary, + host_diagnostic_for_exception, + host_diagnostic_for_reason, +) from k1link.device_plugins.xgrids_k1.live_perception_shadow import ( + build_live_perception_result_receiver, build_live_perception_shadow_router, ensure_live_shadow_token, ) from k1link.device_plugins.xgrids_k1.mqtt import validate_private_ipv4 -from k1link.device_plugins.xgrids_k1.mqtt.capture import seal_capture_clock +from k1link.device_plugins.xgrids_k1.mqtt.capture import RecoveryDecision, seal_capture_clock +from k1link.device_plugins.xgrids_k1.network_mutation_ledger import ( + NetworkMutationBlocked, + NetworkMutationLedger, + NetworkMutationLedgerSnapshot, + NetworkMutationRecord, + NetworkStatusEvidence, + PreviousConnectionEvidence, +) +from k1link.device_plugins.xgrids_k1.network_provisioning_idempotency_journal import ( + NetworkProvisioningIdempotencyBlocked, + NetworkProvisioningIdempotencyCorrupt, + NetworkProvisioningIdempotencyError, + NetworkProvisioningIdempotencyJournal, + NetworkProvisioningLegacyAdoptionProvenance, + NetworkProvisioningTerminalMetadata, + derive_request_binding_sha256, +) +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + LedgerPhysicalCommandCoordinator, + PhysicalCommandIntentContext, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PhysicalCommandLedger, + PhysicalCommandLedgerError, + PhysicalCommandReconciliation, + PhysicalCommandRecord, + PhysicalCommandStatusEvidence, + PhysicalCommandTransitionError, + physical_transport_ref_comparison_key, +) +from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( + ApplicationAcceptanceError, +) from k1link.device_plugins.xgrids_k1.protocol.application_authority import ( MacOSKeychainApplicationAuthorityLoader, ) @@ -87,8 +198,12 @@ from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( WriteDisabledOneShotPublisher, ) from k1link.device_plugins.xgrids_k1.protocol.application_session import ( + ApplicationConnectionBinding, + ApplicationConnectionBindingLost, + ApplicationStartCheckpointSettlementError, InteractiveApplicationControlSession, OperatorPresenceConfirmation, + StartPreparationCheckpointPhase, ) from k1link.device_plugins.xgrids_k1.protocol.modeling import observe_modeling_report from k1link.device_plugins.xgrids_k1.protocol.modeling_safety import ( @@ -98,21 +213,37 @@ from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_mes from k1link.device_plugins.xgrids_k1.quick_connect_profile import ( quick_connect_host_profile_id, ) +from k1link.device_plugins.xgrids_k1.semantic_topology_store import ( + SemanticTopologyRecord, + SemanticTopologyStore, + SemanticTopologyStoreCorrupt, + SemanticTopologyStoreError, +) from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage from k1link.device_plugins.xgrids_k1.viewer.runtime import ( VisualizationRuntime, new_live_session_dir, ) from k1link.host_network import ( + HostWifiAssociationIdentityProbe, + HostWifiAssociationIdentityResult, HostWifiProfileError, + associate_with_ephemeral_wifi_credentials_once, associate_with_wifi_profile_once, + bind_route_fingerprint_to_wifi_association, ensure_wifi_profile_from_credential_source, ) -from k1link.sessions import ActiveSessionLease, resolve_missioncore_evidence_dir +from k1link.sessions import ( + ActiveSessionLease, + recover_stale_active_session_marker, + resolve_missioncore_evidence_dir, +) from k1link.viewer.metrics import BridgeMetrics from k1link.viewer.rerun_bridge import RerunSceneSettings +from k1link.web.camera_archive import CAMERA_ARCHIVE_SCHEMA from k1link.web.device_lifecycle import ( TERMINAL_ACQUISITION_STATES, + TERMINAL_OPERATION_STATUSES, AcquisitionRecord, OperationJournal, new_acquisition_id, @@ -127,14 +258,59 @@ from k1link.web.plugin_runtime import ( ) XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1" -XGRIDS_K1_PLUGIN_VERSION = "0.6.0" +XGRIDS_K1_PLUGIN_VERSION = "0.7.5" XGRIDS_K1_MODEL_ID = "xgrids.lixelkity-k1" XGRIDS_K1_COMPATIBILITY_PROFILE_ID = "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2" DEFAULT_ACQUISITION_CAMERA_SOURCE: CameraSourceId = "sensor.camera.right" CONTROL_MQTT_PORT = 1883 CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS = 1.5 +CONTROL_ENDPOINT_ADMISSION_TIMEOUT_SECONDS = 8.0 +CONTROL_ENDPOINT_ADMISSION_INTERVAL_SECONDS = 0.5 +HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS = 2.0 +# ``xcrun swift`` plus a temporarily busy CoreWLAN inspection exceeded two +# seconds on the field host immediately after association. Three seconds +# keeps one host contact bounded at five seconds including its route subprocess. +# Operator transitions take the lifecycle gate before waiting on that contact, +# so the next TCP/recheck contact is superseded and the eight-second quiescence +# window need not cover the whole monitor pass. A complete healthy pass remains +# bounded at 2 * (2 + 3) + 1.5 = 11.5 seconds. The supervisor's 30-second +# fallback silence lease covers two such passes plus their one-second intervals, +# without weakening the sub-second association cache or skipping the final +# route/association continuity recheck. +CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0 +# Command checkpoints must not inherit either the monitor's three-second +# contact budget or the host helper's generic 30-second budget. One command +# may legitimately wait about three seconds for the service-lifetime +# association probe and then need another CoreWLAN helper run. Eight seconds +# covers that observed 3.0 + 3.2 second path while keeping every +# workspace/prepare/START guard bounded well below the transport silence TTL. +# Including the preceding raw-route subprocess, one command host sample has a +# reviewed ten-second upper bound (2 + 8); this is separate from the monitor's +# unchanged five-second contact bound (2 + 3). +# A technical observer timeout is still handled below only by the exact +# kernel-route/TCP/control-proof retention gate. +COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS = 8.0 +CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS = ( + HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS + CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS +) +CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS = ( + 2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS +) +CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0 +CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"}) +LIVE_DATA_PLANE_STALL_SECONDS = 5.0 +LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0 logger = logging.getLogger(__name__) + +def _capture_network_intent_device( + macos_uuid: str, +) -> CapturedDiscoveredDevice | None: + """Capture the exact latest-scan object for a mutating connect intent.""" + + return capture_discovered_device(macos_uuid) + + ConnectionMode = Literal["bridge", "quick-connect", "direct-connect"] ConnectionTopology = Literal["direct-lan", "device-ap", "controller-hotspot"] CONNECTION_TOPOLOGY_BY_MODE: dict[ConnectionMode, ConnectionTopology] = { @@ -160,6 +336,33 @@ class NetworkWriteReconciliationRequired(RuntimeError): self.reason_code = "network-write-reconciliation-required" +class ProvisioningAlreadyRunning(RuntimeError): + """Another admitted network mutation still owns the provisioning gate.""" + + def __init__(self) -> None: + super().__init__("другая операция настройки Wi-Fi уже выполняется") + self.reason_code = "provisioning-already-running" + + +class NetworkProvisioningConflict(RuntimeError): + """A mutable local precondition rejected a new network-write intent.""" + + def __init__(self, message: str, *, reason_code: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + +class SnapshotRuntimeConflict(RuntimeError): + """A browser action belongs to another local service lifetime.""" + + def __init__(self) -> None: + super().__init__( + "Открытая вкладка устарела после перезапуска локального сервиса. " + "Обновите страницу перед следующим действием." + ) + self.reason_code = "snapshot-runtime-conflict" + + class LocalAcquisitionLifecycleError(RuntimeError): """A local producer lifecycle invariant failed before scanner authority.""" @@ -168,17 +371,453 @@ class LocalAcquisitionLifecycleError(RuntimeError): self.reason_code = reason_code +class BleDiscoveryUnavailable(RuntimeError): + """A process-owned CoreBluetooth discovery lease is unavailable.""" + + def __init__(self, message: str, *, reason_code: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + +class ConnectionVerificationError(RuntimeError): + """A safe, stable failure classification for read-only connection checks.""" + + def __init__( + self, + message: str, + *, + reason_code: str, + operation_stage: str | None = None, + ) -> None: + super().__init__(message) + self.reason_code = reason_code + self.operation_stage = operation_stage + + +class ConfiguredEndpointProbeError(RuntimeError): + """A safe failure classification for a pure host-route/TCP probe.""" + + def __init__(self, message: str, *, reason_code: str) -> None: + super().__init__(message) + self.reason_code = reason_code + + ACTION_STATE_READ = "state.read" ACTION_DISCOVERY_SCAN = "discovery.scan" ACTION_DEVICE_INSPECT = "device.inspect" ACTION_SENSOR_CATALOG_READ = "sensor.catalog.read" ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ = "calibration.device-snapshot.read" ACTION_NETWORK_PROVISION = "network.provision" +ACTION_CONNECTION_MODE_SELECT = "connection.mode.select" +ACTION_CONNECTION_RECONFIGURE_PREPARE = "connection.reconfigure.prepare" ACTION_CONNECTION_VERIFY = "connection.verify" +ACTION_CONFIGURED_ENDPOINT_PROBE = "connection.endpoint-probe" +ACTION_CONNECTION_CONTROL_BOOTSTRAP = "connection.control-bootstrap" ACTION_ACQUISITION_PREPARE = "acquisition.prepare" ACTION_ACQUISITION_START = "acquisition.start" ACTION_ACQUISITION_STOP = "acquisition.stop" ACTION_ACQUISITION_ABORT = "acquisition.abort" +ACTION_ACQUISITION_FORCE_FINISH = "acquisition.force-finish-local" + +_LocalStopRetirementDisposition = Literal[ + "not-applicable", + "not-dispatched", + "outcome-unknown", + "standby-confirmed", +] + +# A durable read-only Verify may first spend up to thirty seconds waiting for +# the exact saved CoreBluetooth UUID to advertise, then perform one bounded +# GATT status read and the MQTT DeviceInfo/DeviceStatus identity barrier. The +# browser keeps this explicit request alive for 150 seconds; keep the durable +# journal deadline inside that caller bound while no longer expiring a valid +# LixelGo-parity discovery attempt at the old forty-second boundary. +CONNECTION_VERIFY_HARD_TIMEOUT_SECONDS = 125.0 +CONNECTION_VERIFY_EXACT_UUID_SCAN_TIMEOUT_SECONDS = 30.0 +CONTROL_DEVICE_INFO_BOOTSTRAP_TIMEOUT_SECONDS = 30.0 +CONTROL_BOOTSTRAP_CONTINUATION_TIMEOUT_SECONDS = 60.0 +PHYSICAL_RECONCILIATION_PROOF_TIMEOUT_SECONDS = 5.0 +# DeviceStatus proof collection and its durable ledger publication consume the +# proof budget above. Adopting that already-committed SCANNING fact in the +# local control worker is a separate bounded transition and must not inherit a +# nearly exhausted proof deadline. +PHYSICAL_RECONCILIATION_CONTROL_ADOPTION_TIMEOUT_SECONDS = 5.0 +RESTART_RECEIVER_SOURCE_READY_TIMEOUT_SECONDS = 12.0 +CONTROL_LOCAL_RETIREMENT_TIMEOUT_SECONDS = 3.0 +ACTIVE_STREAM_RECOVERY_ATTEMPT_TIMEOUT_SECONDS = 55.0 +ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS = CAMERA_PRODUCER_STALL_MILLISECONDS +CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS = 1.0 +CAMERA_POST_RECOVERY_RESTART_MAX_ATTEMPTS = 3 +CAMERA_POST_RECOVERY_RESTART_BACKOFF_BASE_SECONDS = 0.1 +CAMERA_POST_RECOVERY_RESTART_BACKOFF_MAX_SECONDS = 0.4 +CAMERA_POST_PCL_RETRYABLE_STARTUP_ERROR_CODES = frozenset( + { + "ffmpeg-pipes-unavailable", + "ffmpeg-start-failed", + "invalid-fmp4", + } +) +CONNECTION_VERIFY_BLE_OPERATION_STAGES = frozenset( + { + "resolution", + "exact-uuid-scan", + "connect", + "gatt-contract", + "status-read", + } +) +CONNECTION_SCENARIO_RESET_DRAIN_TIMEOUT_SECONDS = 330.0 +_SNAPSHOT_RUNTIME_START_LOCK = threading.Lock() +_LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS = 0 + + +def _camera_post_pcl_startup_failure_is_retryable( + camera: Mapping[str, Any], + *, + evidence_session_id: str, +) -> bool: + """Classify only local camera failures before the first media commit.""" + + recording = camera.get("recording") + error = camera.get("error") + code = error.get("code") if isinstance(error, Mapping) else None + if not ( + camera.get("phase") == "error" + and camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == evidence_session_id + and code in CAMERA_POST_PCL_RETRYABLE_STARTUP_ERROR_CODES + ): + return False + if code != "invalid-fmp4": + return recording.get("active_epoch") is None + # `_mark_producer_failure` publishes the error before `_producer_ended` + # detaches/finalizes the epoch. An init box may already be committed, and + # there is a second gap where the producer is detached but its summary is + # not yet available. Only the acquisition-session lifetime media count is + # stable across both windows and across a fresh local producer epoch. + committed_media_count = recording.get("committed_media_segment_count") + return bool( + isinstance(committed_media_count, int) + and not isinstance(committed_media_count, bool) + and committed_media_count == 0 + ) + + +def _camera_source_end_archive_seal_pending( + camera: Mapping[str, Any], + *, + evidence_session_id: str, +) -> bool: + """Recognize the exact reader-EOF -> canonical-summary publication gap. + + ``_producer_ended`` first detaches the producer and publishes the typed + source-ended error, then closes/fsyncs the archive and appends its summary. + A recovered PCL in that bounded window must remain owned by backend camera + recovery; treating it as a never-started camera would allocate a new epoch + before the old evidence is durably sealed. + """ + + recording = camera.get("recording") + error = camera.get("error") + error_code = error.get("code") if isinstance(error, Mapping) else None + generation = camera.get("generation") + completed_epochs = ( + recording.get("completed_epochs") if isinstance(recording, Mapping) else None + ) + last_summary = ( + recording.get("last_summary") if isinstance(recording, Mapping) else None + ) + prior_summary = bool( + isinstance(last_summary, Mapping) + and last_summary.get("schema_version") == CAMERA_ARCHIVE_SCHEMA + and last_summary.get("source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(last_summary.get("codec_epoch"), int) + and not isinstance(last_summary.get("codec_epoch"), bool) + and isinstance(generation, int) + and cast(int, last_summary["codec_epoch"]) < generation + ) + return bool( + camera.get("phase") == "error" + and camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and error_code == "camera-source-ended" + and isinstance(generation, int) + and not isinstance(generation, bool) + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == evidence_session_id + and recording.get("active_epoch") is None + and recording.get("producer_alive") is False + and recording.get("current_epoch") is None + and isinstance(recording.get("committed_media_segment_count"), int) + and not isinstance(recording.get("committed_media_segment_count"), bool) + and isinstance(completed_epochs, int) + and not isinstance(completed_epochs, bool) + and completed_epochs >= 0 + and ( + (completed_epochs == 0 and last_summary is None) + or (completed_epochs > 0 and prior_summary) + ) + ) + + +@dataclass(frozen=True, slots=True) +class _ActiveStreamRecoveryLineage: + """Immutable admission fence for the sole automatic K1 rebind path.""" + + recovery_generation: int + snapshot_runtime_id: str + acquisition_id: str + device_id: str + device_session_id: str + evidence_session_id: str + runtime_producer_generation: int + start_operation_id: str + physical_operation_id: str + physical_revision: int + intent_id: str + transport_ref: str + connection_mode: ConnectionMode + target_ipv4: str + target_port: int + + +@dataclass(frozen=True, slots=True) +class _PostRecoveryCameraRestartKey: + """Exact recovered-PCL and camera progress CAS claimed by one worker.""" + + recovery_generation: int + acquisition_id: str + evidence_session_id: str + runtime_producer_generation: int + camera_generation: int + camera_active_epoch: int | None + camera_media_segment_count: int + + +@dataclass(frozen=True, slots=True) +class _PostRecoveryCameraRestartAttempt: + """Process-local identity preventing an old worker clearing a newer claim.""" + + key: _PostRecoveryCameraRestartKey + nonce: int + failure_batch: int + + +@dataclass(frozen=True, slots=True) +class _PostRecoveryCameraRestartRetry: + """One bounded retry cooldown, independent of duplicate PCL cadence.""" + + key: _PostRecoveryCameraRestartKey + failure_batch: int + not_before_monotonic: float + + +@dataclass(frozen=True, slots=True) +class _PostRecoveryCameraRestartAdmission: + """One exact successor epoch admitted by the recovered PCL owner.""" + + key: _PostRecoveryCameraRestartKey + camera_generation: int + active_epoch: int + + +@dataclass(frozen=True, slots=True) +class _PreparedStopDispatchLineage: + """Exact local owner for the durable STOP PREPARE -> dispatch handoff. + + The physical ledger replaces its current successful START record with the + successor STOP at PREPARE. This frozen proof keeps the fully validated + parent active audit available only until that exact STOP either reaches + the transport dispatch fence or becomes stale/terminal. + """ + + snapshot_runtime_id: str + stop_operation_id: str + acquisition_id: str + parent_physical_operation_id: str + local_start_operation_id: str | None + parent_record_revision: int + parent_reconciliation_id: str | None + parent_identity_sha256: str + parent_reconciliations_sha256: str + compatibility_profile_id: str + control_binding_sha256: str + control_session_generation: int + minimum_control_state_revision: int + device_id: str + device_session_id: str + evidence_session_id: str | None + runtime_source_mode: str + runtime_producer_generation: int | None + intent_id: str + transport_ref: str + connection_mode: ConnectionMode + target_ipv4: str + target_port: int + host_path_epoch: int + control_session_id: str + control_producer_generation: int + + +@dataclass(frozen=True, slots=True) +class _PreparedStopRecoveryOwner: + """Exact retained receiver owner for read-only PREPARED STOP classification. + + Unlike the short dispatch barrier, this owner becomes authoritative only + after the asynchronous STOP worker has terminally failed before publish + and the operation journal records side-effect ``none``. It may then carry + the same local receiver/evidence lease through a fresh read-only binding, + but it never authorizes a START/STOP replay or a network mutation. + """ + + lineage: _PreparedStopDispatchLineage + acquisition: AcquisitionRecord + session_lease: ActiveSessionLease + out_dir: Path + start_operation_id: str | None + runtime_producer_generation: int + checkpoint_trust_token: _ActiveAcquisitionCheckpointTrustToken | None = None + + +@dataclass(frozen=True, slots=True) +class _ActiveAcquisitionCheckpointTrustToken: + """Immutable local CAS returned by checkpoint/ledger lineage validation.""" + + trust: Literal["trusted"] + checkpoint_revision: int + checkpoint_state: Literal["prepared", "active"] + transport_revision: int + acquisition_id: str + root_start_operation_id: str + current_evidence_session_id: str + checkpoint_head_operation_id: str + checkpoint_head_revision: int + checkpoint_head_proof_sha256: str + physical_head_operation_id: str + physical_head_revision: int + checkpoint: ActiveAcquisitionRecoveryCheckpoint + physical_record: PhysicalCommandRecord + + +@dataclass(frozen=True, slots=True) +class _ClassifiedStopRebindPending: + """Frozen same-process SCANNING proof awaiting its first published PCL.""" + + token: _ActiveAcquisitionCheckpointTrustToken + checkpoint_gap_revision: int + acquisition_id: str + root_start_operation_id: str + classified_stop_operation_id: str + classified_stop_revision: int + reconciliation_id: str + evidence_session_id: str + runtime_producer_generation: int + status_proof: ActiveAcquisitionRecoveryStatusProof + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof + transition_id: str + + +@dataclass(frozen=True, slots=True) +class _RestartStopOnlyCheckpointLineage: + """Checkpoint fields retained only to cease one restart STOP safely.""" + + checkpoint_revision: int + acquisition_id: str + original_start_operation_id: str + prior_evidence_session_id: str + cessation_evidence_session_id: str + + +@dataclass(frozen=True, slots=True) +class _RestartRehydrationPending: + """Exact commandless restart receiver awaiting its first published PCL. + + Every store request input is frozen at explicit Verify. A duplicate frame + can therefore retry only the identical transition after a lost response; + it cannot manufacture a later timestamp, proof id, binding, or authority. + """ + + phase: Literal["awaiting-suback", "awaiting-first-pcl", "blocked"] + token: _ActiveAcquisitionCheckpointTrustToken + reconciliation_id: str + gap_started_at_utc: str + old_evidence_session_id: str + new_evidence_session_id: str + out_dir: Path + runtime_producer_generation: int + status_proof: ActiveAcquisitionRecoveryStatusProof + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof + origin_proof: ActiveAcquisitionRecoveryStartOriginProof | None + transition_id: str + first_pcl_proof: ActiveAcquisitionRecoveryFirstPublishedPclProof | None = None + reason_code: str | None = None + + +@dataclass(frozen=True, slots=True) +class _PreAdmittedGracefulStop: + """Journal identity accepted before a durable promotion can delay STOP. + + This token grants no scanner authority and does not mean that physical + STOP was prepared or dispatched. It only binds the public request to the + exact operation/deadline that was made visible before lifecycle waiting. + The lifecycle owner must re-read and validate the same journal row before + it can prepare the physical successor. + """ + + operation_id: str + acquisition_id: str + request_fingerprint: str + created: bool + accepted_at: datetime + deadline_at: datetime | None + + +def _sha256_audit_document(value: Mapping[str, object]) -> str: + """Hash one secret-free audit mapping with the ledger's canonical shape.""" + + return hashlib.sha256( + json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + + +def _active_stream_reconciliation_id( + lineage: _ActiveStreamRecoveryLineage, + *, + attempt: int, +) -> str: + """Return one globally scoped, exact-retry-stable read-only proof id.""" + + if not isinstance(attempt, int) or isinstance(attempt, bool) or attempt < 1: + raise ValueError("active stream recovery attempt must be a positive integer") + lineage_sha256 = hashlib.sha256( + ( + f"{lineage.acquisition_id}\0{lineage.physical_operation_id}\0" + f"{lineage.recovery_generation}\0{attempt}" + ).encode() + ).hexdigest() + # Fixed width also leaves room for the coordinator's derived + # ``.device-info`` verification id inside the 160-character ledger schema. + return f"auto-rebind:sha256-{lineage_sha256}" + + +def _allocate_snapshot_runtime_started_monotonic_ns() -> str: + """Allocate a process-comparable runtime order independent of wall clock.""" + + global _LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS + + candidate = time.monotonic_ns() + with _SNAPSHOT_RUNTIME_START_LOCK: + if candidate <= _LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS: + candidate = _LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS + 1 + _LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS = candidate + return str(candidate) def _serialized_acquisition_access[**P, R]( @@ -197,7 +836,7 @@ def _serialized_acquisition_access[**P, R]( *args: P.args, **kwargs: P.kwargs, ) -> R: - with service._acquisition_lifecycle_gate: # noqa: SLF001 + with service._acquisition_lifecycle_access(): # noqa: SLF001 return method(service, *args, **kwargs) return cast( @@ -206,6 +845,37 @@ def _serialized_acquisition_access[**P, R]( ) +def _serialized_k1_transition_access[**P, R]( + method: Callable[Concatenate[XgridsK1CompatibilityService, P], R], +) -> Callable[Concatenate[XgridsK1CompatibilityService, P], R]: + """Fail closed when network and control admission overlap locally.""" + + @wraps(method) + def serialized( + service: XgridsK1CompatibilityService, + *args: P.args, + **kwargs: P.kwargs, + ) -> R: + if not service._k1_lifecycle_transition_gate.acquire(blocking=False): # noqa: SLF001 + raise ApplicationControlProcessLeaseUnavailable( + "another local K1 network/control transition is already running" + ) + try: + if not service._wait_for_connection_monitor_quiescence(): # noqa: SLF001 + raise ApplicationControlProcessLeaseUnavailable( + "read-only connection monitor did not quiesce before " + "the K1 lifecycle transition" + ) + return method(service, *args, **kwargs) + finally: + service._k1_lifecycle_transition_gate.release() # noqa: SLF001 + + return cast( + "Callable[Concatenate[XgridsK1CompatibilityService, P], R]", + serialized, + ) + + ACTION_ACQUISITION_STATE_READ = "acquisition.state.read" ACTION_STREAM_START_LIVE = "stream.start-live" ACTION_STREAM_START_REPLAY = "stream.start-replay" @@ -219,6 +889,11 @@ ACTION_APPLICATION_CONTROL_SHADOW_DISARM = "application-control.shadow-disarm" ACTION_APPLICATION_CONTROL_SESSION_OPEN = "application-control.session.open" ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER = "application-control.workspace.enter" ACTION_APPLICATION_CONTROL_SESSION_CLOSE = "application-control.session.close" +ACTION_PHYSICAL_COMMAND_RECONCILE = "physical-command.reconcile" +ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE = "physical-command.retire-unavailable" +ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION = ( + "physical-command.reopen-retired-reconciliation" +) RequestedStreamId = Literal[ "spatial.point-cloud.live", @@ -248,8 +923,38 @@ class CompatibilityProfileLoader(Protocol): def load_compatibility_profile(self, path: Path) -> dict[str, Any]: ... +class HostWifiAssociationProbe(Protocol): + """Service-lifetime source of opaque host Wi-Fi continuity evidence.""" + + def observe( + self, + interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: ... + + +@dataclass(frozen=True, slots=True) +class _CorrelatedEndpointObservation: + """One route/association A -> TCP -> route/association B observation.""" + + path: HostPathProbeResult + reachable: bool + reason_code: str | None + + +@dataclass(frozen=True, slots=True) +class _ConfiguredEndpointHostObservation: + """One kernel-route A -> TCP -> kernel-route B diagnostic observation.""" + + path: HostPathProbeResult + reachable: bool + reason_code: str | None + + class BleScanRequest(StrictRequest): duration_seconds: float = Field(default=6.0, ge=1.0, le=60.0) + operation_id: str | None = Field(default=None, min_length=1, max_length=128) class CompatibilityAttestationRequest(StrictRequest): @@ -267,13 +972,28 @@ class ConnectRequest(StrictRequest): connection_mode: ConnectionMode = "bridge" compatibility_attestation: CompatibilityAttestationRequest operation_id: str | None = Field(default=None, min_length=1, max_length=128) - idempotency_key: str | None = Field(default=None, min_length=1, max_length=160) + idempotency_key: str = Field(min_length=16, max_length=160) + expected_mode_revision: int = Field(ge=0) + expected_discovery_generation: int = Field(ge=0) + expected_reconfiguration_revision: int | None = Field(default=None, ge=0) + expected_reconfiguration_intent_id: str | None = Field( + default=None, + min_length=1, + max_length=128, + ) + # Bridge changes the K1 network. It may change the controller's network as + # a separate host-side effect only when the caller opts in explicitly. + # Quick Connect owns its reviewed AP handoff, while Direct Connect must + # never reinterpret this Bridge-only permission. + allow_host_wifi_switch: bool = Field(default=False, strict=True) @model_validator(mode="after") def validate_connection_topology(self) -> Self: expected = CONNECTION_TOPOLOGY_BY_MODE[self.connection_mode] if self.compatibility_attestation.topology != expected: raise ValueError(f"connection_mode={self.connection_mode} requires topology={expected}") + if self.allow_host_wifi_switch and self.connection_mode != "bridge": + raise ValueError("allow_host_wifi_switch is available only for Bridge") if self.connection_mode == "quick-connect": if self.ssid is not None or self.password is not None: raise ValueError( @@ -290,27 +1010,188 @@ class ConnectRequest(StrictRequest): class ConnectionVerifyRequest(StrictRequest): - """Refresh one lease or adopt a scanned K1 already on the direct LAN.""" + """Read one server-bound K1 topology without changing the device.""" + # ``{}`` is retained for older local clients. It never selects a device: + # the service resolves that form only from its current/durable exact + # target. New clients send the complete server-issued target contract. device_id: str | None = Field(default=None, min_length=1, max_length=128) + source: Literal["fresh-scan", "retained-current-process", "durable-configured-state"] = ( + "fresh-scan" + ) compatibility_attestation: CompatibilityAttestationRequest | None = None + operation_id: str | None = Field(default=None, min_length=1, max_length=128) + expected_discovery_generation: int | None = Field(default=None, ge=0) + expected_reconfiguration_revision: int | None = Field(default=None, ge=0) + expected_reconfiguration_intent_id: str | None = Field( + default=None, + min_length=1, + max_length=128, + ) @model_validator(mode="after") - def validate_adoption_request(self) -> Self: - has_device = self.device_id is not None - has_attestation = self.compatibility_attestation is not None - if has_device != has_attestation: - raise ValueError( - "device_id and compatibility_attestation must be provided together" - ) + def validate_exact_or_legacy_server_target(self) -> Self: + if (self.device_id is None) != (self.compatibility_attestation is None): + raise ValueError("device_id and compatibility_attestation must be supplied together") + if self.device_id is None and self.source != "fresh-scan": + raise ValueError("legacy server-target verification cannot select a source") if ( - self.compatibility_attestation is not None - and self.compatibility_attestation.topology != "direct-lan" + self.device_id is not None + and self.source == "fresh-scan" + and self.expected_discovery_generation is None ): - raise ValueError("connection.verify adoption requires topology=direct-lan") + raise ValueError("fresh-scan verification requires expected_discovery_generation") + if self.source != "fresh-scan" and self.expected_discovery_generation is not None: + raise ValueError( + "expected_discovery_generation belongs only to fresh-scan verification" + ) return self +class ConfiguredEndpointProbeRequest(StrictRequest): + """Probe the server-selected configured endpoint without touching BLE.""" + + operation_id: str | None = Field(default=None, min_length=1, max_length=128) + + +class DesiredConnectionModeRequest(StrictRequest): + """Select a mode, optionally retiring the complete local scenario.""" + + connection_mode: ConnectionMode + expected_revision: int = Field(ge=0) + reset_scenario: Literal[True] | None = None + reset_id: str | None = Field(default=None, min_length=16, max_length=128) + + @model_validator(mode="after") + def validate_reset_identity(self) -> Self: + if (self.reset_scenario is True) != (self.reset_id is not None): + raise ValueError("reset_id is required only for reset_scenario") + return self + + +class PrepareConnectionReconfigurationRequest(StrictRequest): + """Prepare or cancel one local-only Bridge reconfiguration dialogue.""" + + intent: Literal["select-device", "change-network", "cancel"] + expected_reconfiguration_revision: int = Field(ge=0) + expected_reconfiguration_intent_id: str | None = Field( + default=None, + min_length=1, + max_length=128, + ) + expected_desired_mode_revision: int = Field(ge=0) + expected_active_binding_key: str | None = Field( + default=None, + min_length=64, + max_length=64, + pattern=r"^[0-9a-f]{64}$", + ) + + +@dataclass(frozen=True, slots=True) +class _ConfiguredEndpointProbeTarget: + target: EndpointTarget + connection_mode: ConnectionMode + source: Literal["current-supervisor", "durable-semantic-topology"] + transport_ref: str + intent_id: str | None + semantic_revision: int | None + + +@dataclass(frozen=True, slots=True) +class _ConnectionVerificationTarget: + """One server-selected, revision-fenced target for a read-only BLE read.""" + + transport_ref: str + connection_mode: ConnectionMode + transport_source: Literal["fresh-scan", "retained-current-process", "durable-state"] + authority_source: Literal[ + "fresh-request", "current-session", "unresolved-ledger", "semantic-topology" + ] + discovery_generation: int | None + device_session_id: str | None + selected_device_id: str | None + selected_connection_mode: ConnectionMode | None + network_operation_id: str | None + network_revision: int | None + idempotency_operation_id: str | None + idempotency_revision: int | None + semantic_predecessor_revision: int + + +@dataclass(frozen=True, slots=True) +class _ControlBootstrapContinuation: + """Exact, read-only continuation of one durably applied network intent.""" + + operation_id: str + parent_operation_id: str + snapshot_runtime_id: str + parent_intent_id: str + transport_ref: str + connection_mode: ConnectionMode + target: EndpointTarget + device_session_id: str + desired_mode_revision: int + semantic_topology_revision: int + network_ledger_revision: int + physical_record_operation_id: str | None + physical_record_created_at_utc: str | None + physical_reopening_id: str | None + + +@dataclass(frozen=True, slots=True) +class _ProvisionalFreshBridgeTopology: + """Fresh Bridge 7f02 evidence awaiting the exact DeviceInfo identity pin. + + ``reconfiguration_intent_id`` is present for an explicit select-device + dialogue and absent for an ordinary cold, read-only-first selection. The + process-local intent/session/discovery tuple is always exact so cleanup of + a losing Verify cannot erase a newer winner. + """ + + reconfiguration_intent_id: str | None + reconfiguration_revision: int + supervisor_intent_id: str + discovery_generation: int + device_session_id: str + transport_ref: str + connection_mode: Literal["bridge"] + ipv4: str + firmware_version: str + observed_at_utc: str + semantic_predecessor_revision: int + + +@dataclass(frozen=True, slots=True) +class _ScenarioResetRetiredTransportAuthority: + """One exact retired transport re-admitted for a new network intent. + + The public scenario-reset marker is presentation/replay state and is not + mutation authority. This private tuple binds the reset to the immutable + retirement and physical-record revision that it actually committed. Any + later retirement, reopen or physical audit transition invalidates it. + """ + + reset_id: str + reset_revision: int + desired_mode: ConnectionMode + retired_transport_ref: str + retirement_id: str + retirement_original_attempt_sha256: str + retirement_retired_at_utc: str + physical_record_operation_id: str + physical_record_created_at_utc: str + physical_record_revision: int + minimum_discovery_generation: int + physical_reopening_id: str | None = None + physical_reopened_record_revision: int | None = None + physical_reopening_discovery_generation: int | None = None + consumed_operation_id: str | None = None + consumed_idempotency_key_sha256: str | None = None + consumed_request_binding_sha256: str | None = None + consumed_request_fingerprint: str | None = None + + class LiveRequest(StrictRequest): project_name: str = Field(min_length=1, max_length=96) host: str | None = Field(default=None, max_length=15) @@ -328,10 +1209,25 @@ class LiveRequest(StrictRequest): class OperationContextRequest(StrictRequest): operation_id: str | None = Field(default=None, min_length=1, max_length=128) - idempotency_key: str | None = Field(default=None, min_length=1, max_length=160) + idempotency_key: str = Field(min_length=16, max_length=160) deadline_seconds: float | None = Field(default=None, ge=1.0, le=86_400.0) +class OptionalApplicationControlCasRequest(StrictRequest): + expected_control_session_generation: int | None = Field(default=None, ge=1) + expected_control_state_revision: int | None = Field(default=None, ge=1) + + @model_validator(mode="after") + def validate_complete_control_checkpoint(self) -> Self: + generation_present = self.expected_control_session_generation is not None + revision_present = self.expected_control_state_revision is not None + if generation_present != revision_present: + raise ValueError( + "control session generation and state revision must be supplied together" + ) + return self + + class OperatorPresenceRequest(StrictRequest): operator_present: Literal[True] owner_controlled_device: Literal[True] @@ -352,7 +1248,7 @@ class OperatorPresenceRequest(StrictRequest): ) -class PrepareAcquisitionRequest(OperationContextRequest): +class PrepareAcquisitionRequest(OperationContextRequest, OptionalApplicationControlCasRequest): project_name: str = Field(min_length=1, max_length=96) mount_type: Literal["handheld"] = "handheld" gnss_mode: Literal["none"] = "none" @@ -368,23 +1264,32 @@ class PrepareAcquisitionRequest(OperationContextRequest): return normalize_project_name(value) -class StartAcquisitionRequest(OperationContextRequest): +class StartAcquisitionRequest(OperationContextRequest, OptionalApplicationControlCasRequest): acquisition_id: str | None = Field(default=None, min_length=1, max_length=128) expected_state_revision: int | None = Field(default=None, ge=1) physical_acceptance: OperatorPresenceRequest | None = None -class StopAcquisitionRequest(OperationContextRequest): +class StopAcquisitionRequest(OperationContextRequest, OptionalApplicationControlCasRequest): acquisition_id: str | None = Field(default=None, min_length=1, max_length=128) mode: Literal["graceful", "capture-only"] = "capture-only" operator_confirmed: bool = False physical_acceptance: OperatorPresenceRequest | None = None -class AbortAcquisitionRequest(OperationContextRequest): +class AbortAcquisitionRequest(OperationContextRequest, OptionalApplicationControlCasRequest): acquisition_id: str | None = Field(default=None, min_length=1, max_length=128) +class ForceFinishAcquisitionRequest(OperationContextRequest): + """Seal only host-owned producers after an unrecoverable link loss.""" + + acquisition_id: str = Field(min_length=1, max_length=128) + expected_state_revision: int = Field(ge=1) + expected_recovery_generation: int = Field(ge=1) + operator_confirmed: Literal[True] + + class ReplayRequest(StrictRequest): path: str = Field(min_length=1, max_length=4096) speed: float = Field(default=1.0, ge=0.0, le=100.0) @@ -435,6 +1340,46 @@ class OpenApplicationControlSessionRequest(OperatorPresenceRequest): class EnterApplicationWorkspaceRequest(StrictRequest): operator_confirmed: Literal[True] + expected_session_generation: int = Field(ge=1) + expected_state_revision: int = Field(ge=1) + + +class CloseApplicationControlSessionRequest(StrictRequest): + expected_session_generation: int = Field(ge=1) + expected_state_revision: int = Field(ge=1) + + +class ReconcilePhysicalCommandRequest(StrictRequest): + """Classify an ambiguous physical edge without replaying START or STOP.""" + + reconciliation_id: str = Field(min_length=1, max_length=128) + expected_session_generation: int = Field(ge=1) + expected_state_revision: int = Field(ge=1) + + +class RetireUnavailablePhysicalCommandRequest(StrictRequest): + """Explicitly release one unreachable physical target without device I/O.""" + + retirement_id: str = Field(min_length=1, max_length=128) + expected_operation_id: str = Field(min_length=1, max_length=128) + expected_revision: int = Field(ge=1) + expected_transport_ref: str = Field(min_length=1, max_length=128) + operator_confirmed: Literal[True] + reason: Literal["device-permanently-unavailable-or-replaced"] + + +class ReopenRetiredPhysicalCommandReconciliationRequest(StrictRequest): + """Re-admit one freshly observed retired UUID for explicit Verify only.""" + + reopening_id: str = Field(min_length=1, max_length=128) + expected_revision: int = Field(ge=1) + expected_retirement_id: str = Field(min_length=1, max_length=128) + expected_transport_ref: str = Field(min_length=1, max_length=128) + expected_discovery_generation: int = Field(ge=1) + expected_desired_mode: ConnectionMode + expected_desired_mode_revision: int = Field(ge=0) + operator_confirmed: Literal[True] + reason: Literal["device-returned-for-explicit-reconciliation"] class XgridsK1CompatibilityService: @@ -446,31 +1391,240 @@ class XgridsK1CompatibilityService: *, application_authority_loader: ApplicationAuthorityLoader | None = None, calibration_snapshot_reader: DeviceCalibrationSnapshotReader | None = None, + host_wifi_association_probe: HostWifiAssociationProbe | None = None, ) -> None: self.repository_root = repository_root.resolve() + # Every CoreBluetooth entrypoint, including read-only calibration and + # pre-control recovery, must resolve to the same process-wide OS lock. + # Configuration is deliberately part of service construction so an + # unconfigured low-level caller fails closed instead of bypassing the + # network/control lifecycle owner. + configure_ble_runtime_process_lease(self.repository_root) self.evidence_root = resolve_missioncore_evidence_dir(self.repository_root) + self._host_wifi_association_probe = ( + host_wifi_association_probe + or HostWifiAssociationIdentityProbe( + self.repository_root / "plugins" / "xgrids-k1" / "macos" / "associate_wifi.swift" + ) + ) self._lock = threading.Lock() + self._snapshot_runtime_started_at_utc = _utc_now_iso() + self._snapshot_runtime_started_monotonic_ns = ( + _allocate_snapshot_runtime_started_monotonic_ns() + ) + self._snapshot_runtime_id = f"snapshot-runtime-{secrets.token_hex(16)}" + self._snapshot_revision = 0 + self._runtime_event_loop: asyncio.AbstractEventLoop | None = None self._acquisition_lifecycle_gate = threading.RLock() + # ``threading.RLock`` does not promise waiter fairness. State polling + # can therefore repeatedly barge ahead of an explicit operator reset + # even after the reset has advertised ``waiting-for-local-lifecycle``. + # This admission condition gives the sole reset writer priority while + # preserving re-entrant ``self.state()`` calls on the owning thread. + self._acquisition_lifecycle_admission = threading.Condition() + self._acquisition_lifecycle_reader_depth = threading.local() + self._acquisition_lifecycle_writer_token: object | None = None + self._acquisition_lifecycle_writer_thread_id: int | None = None + self._k1_lifecycle_transition_gate = threading.Lock() + self._connection_reconfiguration_gate = threading.Lock() + self._k1_command_dispatch_gate = threading.Lock() + # Linearizes an explicit physical STOP intent ahead of any camera-only + # watchdog candidate commit. It is never held across FFmpeg shutdown, + # archive fsync, Popen, MQTT validation or device I/O. + self._camera_restart_commit_gate = threading.Lock() + self._camera_stop_priority_counts: dict[str, int] = {} + self._k1_process_lease_gate = threading.Lock() + self._connection_monitor_contact_gate = threading.Lock() + self._service_close_requested = threading.Event() self._provisioning_gate = threading.Lock() self._calibration_gate = threading.Lock() self._provisioning_active = False self._fingerprint_key = secrets.token_bytes(32) self._devices: list[dict[str, Any]] = [] self._ble_discovery_generation = 0 + self._physical_retirement_reopen_generation_floors: dict[str, int] = {} self._ble_device_last_seen_monotonic: dict[str, float] = {} + self._ble_device_last_seen_suspend_aware: dict[str, float] = {} self._selected_device_id: str | None = None self._k1_ip: str | None = None self._connection_mode: ConnectionMode | None = None + # The operator's draft is intentionally distinct from the factual + # topology. Selecting a dropdown never changes K1, the Mac network, or + # MQTT ownership; only a successful explicit connect commits + # ``_connection_mode``. + self._desired_connection_mode: ConnectionMode = "bridge" + self._desired_connection_mode_revision = 0 + self._connection_scenario_reset: dict[str, object] | None = None + # OperationRecord.sequence is local to one operation, so it cannot be + # used as a chronological reset cutoff. Remember the exact network + # operations which existed when the reset committed; a later operation + # with a fresh id is then visible immediately, regardless of how many + # lifecycle transitions either record contains. + self._connection_scenario_reset_network_operation_ids: frozenset[str] = ( + frozenset() + ) + self._connection_scenario_reset_retired_transport_authority: ( + _ScenarioResetRetiredTransportAuthority | None + ) = None + self._scenario_reset_successful_scan_generation: int | None = None + self._scenario_reset_successful_scan_reset_id: str | None = None + self._connection_scenario_reset_pending: dict[str, object] | None = None + # Operator-only CAS for the explicit select-device/change-network + # dialogue. Unlike the supervisor revision, this value is not moved + # by the one-second route monitor. + self._connection_reconfiguration_revision = 0 + self._connection_reconfiguration_intent_id: str | None = None + self._connection_reconfiguration_intent: ( + Literal["select-device", "change-network"] | None + ) = None + self._connection_reconfiguration_required_transport_ref: str | None = None + self._connection_reconfiguration_required_connection_mode: ConnectionMode | None = None + self._connection_reconfiguration_minimum_discovery_generation: int | None = None + self._connection_reconfiguration_prepared_at: str | None = None self._device_ids_by_transport_ref: dict[str, str] = {} + self._vendor_device_ids_by_transport_ref: dict[str, str] = {} + self._device_identity_pin_store: DeviceIdentityPinStore | None = None + self._device_identity_pin_store_reason: str | None = None + try: + identity_pin_store = DeviceIdentityPinStore(self.repository_root) + identity_pin_snapshot = identity_pin_store.snapshot() + self._device_identity_pin_store = identity_pin_store + if identity_pin_snapshot.status == "corrupt": + self._device_identity_pin_store_reason = ( + identity_pin_snapshot.reason_code or DeviceIdentityPinStoreCorrupt.reason_code + ) + else: + self._vendor_device_ids_by_transport_ref = { + pin.transport_ref: pin.vendor_device_id for pin in identity_pin_snapshot.pins + } + except (DeviceIdentityPinStoreError, OSError, ValueError) as exc: + self._device_identity_pin_store_reason = getattr( + exc, + "reason_code", + DeviceIdentityPinStoreCorrupt.reason_code, + ) + self._semantic_topology_store: SemanticTopologyStore | None = None + self._semantic_topology_store_reason: str | None = None + try: + semantic_topology_store = SemanticTopologyStore(self.repository_root) + semantic_snapshot = semantic_topology_store.snapshot() + self._semantic_topology_store = semantic_topology_store + if semantic_snapshot.status == "corrupt": + self._semantic_topology_store_reason = ( + semantic_snapshot.reason_code or SemanticTopologyStoreCorrupt.reason_code + ) + except (SemanticTopologyStoreError, OSError, ValueError) as exc: + self._semantic_topology_store_reason = getattr( + exc, + "reason_code", + SemanticTopologyStoreCorrupt.reason_code, + ) self._device_id: str | None = None self._device_session_id: str | None = None self._device_session_opened_at: str | None = None + self._device_session_discovery_generation: int | None = None self._connection_lease_generation = 0 - # This is deliberately process-owned and cannot be cleared by a - # browser refresh, a new idempotency key, or another discovery scan. - # It is raised only when a BLE write may have left device state - # unknown; a reviewed read-only status observation clears it. - self._network_write_reconciliation: dict[str, Any] | None = None + self._network_mutation_ledger = NetworkMutationLedger(self.repository_root) + self._network_provisioning_idempotency_journal: ( + NetworkProvisioningIdempotencyJournal | None + ) = None + self._network_provisioning_idempotency_reason: str | None = None + try: + idempotency_journal = NetworkProvisioningIdempotencyJournal(self.repository_root) + idempotency_snapshot = idempotency_journal.snapshot() + if idempotency_snapshot.status == "corrupt": + self._network_provisioning_idempotency_reason = ( + idempotency_snapshot.reason_code + or NetworkProvisioningIdempotencyCorrupt.reason_code + ) + else: + self._network_provisioning_idempotency_journal = idempotency_journal + except (NetworkProvisioningIdempotencyError, OSError, ValueError) as exc: + self._network_provisioning_idempotency_reason = getattr( + exc, + "reason_code", + NetworkProvisioningIdempotencyCorrupt.reason_code, + ) + try: + ledger_snapshot = _recover_network_provisioning_after_process_restart( + self.repository_root, + network_ledger=self._network_mutation_ledger, + idempotency_journal=self._network_provisioning_idempotency_journal, + ) + except (NetworkProvisioningIdempotencyError, OSError, ValueError) as exc: + self._network_provisioning_idempotency_journal = None + self._network_provisioning_idempotency_reason = getattr( + exc, + "reason_code", + NetworkProvisioningIdempotencyCorrupt.reason_code, + ) + ledger_snapshot = self._network_mutation_ledger.snapshot() + self._network_write_reconciliation = _network_mutation_reconciliation_projection( + ledger_snapshot + ) + self._physical_command_ledger = PhysicalCommandLedger(self.repository_root) + self._physical_command_coordinator = LedgerPhysicalCommandCoordinator( + self._physical_command_ledger + ) + self._active_acquisition_checkpoint: ( + ActiveAcquisitionRecoveryCheckpointStore | None + ) = None + self._active_acquisition_checkpoint_reason: str | None = None + try: + active_acquisition_checkpoint = ActiveAcquisitionRecoveryCheckpointStore( + self.repository_root + ) + checkpoint_snapshot = active_acquisition_checkpoint.snapshot() + self._active_acquisition_checkpoint = active_acquisition_checkpoint + if checkpoint_snapshot.status == "corrupt": + self._active_acquisition_checkpoint_reason = ( + checkpoint_snapshot.reason_code + or "active-acquisition-recovery-checkpoint-corrupt" + ) + except ( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ) as exc: + # Loading the plugin must not take away the existing explicit + # Verify -> STOP-only safety path. An unavailable checkpoint grants + # no acquisition/resume authority and is never repaired here. + self._active_acquisition_checkpoint_reason = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-unavailable", + ) + ) + self._active_acquisition_checkpoint_trust: Literal[ + "trusted", "corrupt", "unavailable" + ] = ( + "trusted" + if self._active_acquisition_checkpoint_reason is None + else "corrupt" + if self._active_acquisition_checkpoint_reason + == "active-acquisition-recovery-checkpoint-corrupt" + else "unavailable" + ) + try: + _recover_physical_command_after_process_restart( + self.repository_root, + physical_ledger=self._physical_command_ledger, + checkpoint_store=self._active_acquisition_checkpoint, + ) + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + self._active_acquisition_checkpoint_trust = "unavailable" + self._active_acquisition_checkpoint_reason = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-unavailable", + ) + ) + checkpoint_trust_token = self._validate_active_acquisition_checkpoint_lineage() + self._settle_durable_prepared_stop_standby_checkpoint( + checkpoint_trust_token + ) self._connection_verification: dict[str, Any] = { "status": "not-probed", "lease_state": "disconnected", @@ -479,10 +1633,79 @@ class XgridsK1CompatibilityService: "network_reachability": "unknown", "observed_at": None, } + self._configured_endpoint_probe: dict[str, Any] = { + "schema_version": "missioncore.xgrids-k1-configured-endpoint-probe/v1", + "status": "not-probed", + "target_source": None, + "connection_mode": None, + "endpoint": None, + "transport_ref": None, + "intent_id": None, + "semantic_revision": None, + "host_route_available": None, + "host_route_class": None, + "tcp_reachable": None, + "identity_validation": "not-performed", + "control_authority_granted": False, + "ble_operation_performed": False, + "network_mutation_performed": False, + "automatic_retry": False, + "observed_at": None, + "reason_code": None, + } + self._connection_supervisor = ConnectionSupervisor() + self._connection_monitor = self._new_connection_monitor() + self._connection_monitor_started = False + # A single failed route/TCP sample is not sufficient to destroy an + # operator session. The monitor must publish the same loss class in + # two distinct supervisor revisions. A terminal failure emitted by + # the canonical MQTT owner is already conclusive and bypasses this + # debounce; neither path performs a reconnect or a BLE write. + self._connection_loss_signature: tuple[str, str, str, str, str, int] | None = None + self._connection_loss_evidence_token: tuple[object, ...] | None = None + self._connection_loss_confirmation_count = 0 + self._last_live_data_monotonic: float | None = None + self._last_live_data_suspend_aware: float | None = None + self._last_live_data_session_id: str | None = None + self._live_perception_camera_binding: ( + tuple[ + str, + CameraSourceId, + int, + ] + | None + ) = None + # One immutable claim per acquisition/runtime generation. The claim is + # created only by a successfully published point-cloud frame and keeps + # the 10 Hz publisher from spawning duplicate FFmpeg activation workers. + self._camera_activation_lineage: tuple[str, str, int] | None = None + self._camera_activation_retry_lineage: tuple[str, str, int] | None = None + self._camera_activation_retry_not_before_monotonic = 0.0 + # The first recovered PCL is a distinct edge from initial camera + # admission. In-flight work coalesces duplicate 10 Hz callbacks, while + # the immutable claim is published only after an exact successor epoch + # exists. A terminal marker bounds failures without pretending success. + self._post_recovery_camera_restart_claim: ( + _PostRecoveryCameraRestartAdmission | None + ) = None + self._post_recovery_camera_restart_inflight: ( + _PostRecoveryCameraRestartAttempt | None + ) = None + self._post_recovery_camera_restart_retry: ( + _PostRecoveryCameraRestartRetry | None + ) = None + self._post_recovery_camera_restart_terminal: ( + _PostRecoveryCameraRestartKey | None + ) = None + self._post_recovery_camera_restart_nonce = 0 self._compatibility_attestation: dict[str, Any] | None = None self._operation_phase: str | None = None self._operation_message: str | None = None self._operations = OperationJournal() + self._control_bootstrap_continuation: _ControlBootstrapContinuation | None = None + self._control_bootstrap_continuation_task: asyncio.Task[None] | None = None + self._control_bootstrap_continuation_loop: asyncio.AbstractEventLoop | None = None + self._control_bootstrap_continuation_closing = False self._acquisition: AcquisitionRecord | None = None self._acquisition_project_name: str | None = None self._acquisition_mount_type: Literal["handheld"] | None = None @@ -491,6 +1714,50 @@ class XgridsK1CompatibilityService: self._acquisition_start_operation_id: str | None = None self._acquisition_stop_operation_id: str | None = None self._acquisition_session_lease: ActiveSessionLease | None = None + self._classified_stop_rebind_pending: ( + _ClassifiedStopRebindPending | None + ) = None + self._classified_stop_rebind_inflight: ( + _ClassifiedStopRebindPending | None + ) = None + self._restart_rehydration_pending: _RestartRehydrationPending | None = None + self._restart_stop_only_prepared_pending: ( + _RestartRehydrationPending | None + ) = None + # Durable checkpoint lineage is authority, not presentation data. A + # STOP-only shell keeps its exact checkpoint CAS process-locally so + # public ``AcquisitionRecord.result`` never exposes operation/evidence + # identifiers merely to make the later STOP settlement possible. + self._restart_stop_only_checkpoint_owner: ( + tuple[AcquisitionRecord, _RestartStopOnlyCheckpointLineage] | None + ) = None + self._active_stream_recovery_generation = 0 + self._active_stream_recovery_lineage: _ActiveStreamRecoveryLineage | None = None + self._active_stream_recovery_future: ConcurrentFuture[RecoveryDecision] | None = None + self._active_stream_recovery_state: Literal[ + "inactive", + "reconnecting", + "blocked", + "recovered", + "standby", + "fault", + "force-finishing", + "force-finished", + ] = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code: str | None = None + self._active_stream_recovery_started_at_utc: str | None = None + self._active_stream_recovery_started_monotonic: float | None = None + self._active_stream_recovery_terminal_outcome: Literal["standby", "fault"] | None = None + self._prepared_stop_dispatch_lineage: _PreparedStopDispatchLineage | None = None + self._prepared_stop_recovery_owner: _PreparedStopRecoveryOwner | None = None + self._operator_manual_acquisition_process_lease_id: str | None = None + self._application_control_process_lease: ApplicationControlProcessLease | None = None + self._application_control_process_lease_holders: set[str] = set() + self._application_control_process_lease_quarantine: ( + tuple[ApplicationControlProcessLease, str] | None + ) = None + self._pending_local_control_retirement = False # The host-owned visual runtime receives the vendor normalizer # explicitly. There is no implicit K1 decoder in the visual layer. self._modeling_control_safety = LiveModelingControlSafety() @@ -515,18 +1782,1295 @@ class XgridsK1CompatibilityService: self._application_control_session = InteractiveApplicationControlSession( authority_loader, transport_factory=self._application_control_transport, - scanning_observer=self._activate_default_acquisition_camera, + scanning_observer=self._observe_application_scanning_checkpoint, + connection_path_validator=self._validate_application_connection_path, + connection_binding_validator=self._validate_application_connection_binding, + connection_dispatch_lease=self._acquire_application_dispatch_lease, + physical_command_coordinator=self._physical_command_coordinator, ) self.runtime = VisualizationRuntime( normalizer=normalize_k1_message, message_observer=self._observe_runtime_message, + published_envelope_observer=self._observe_published_runtime_envelope, + recovery_promotion_checkpoint=( + self._confirm_active_acquisition_checkpoint_rebind + ), ) self.camera_preview = XgridsK1CameraGateway( self.repository_root, XGRIDS_K1_PLUGIN_ID, committed_segment_observer=self._observe_committed_camera_segment, + process_fence_descriptor_factory=(self._duplicate_camera_process_fence_descriptor), + producer_stall_observer=self._observe_camera_producer_stall, ) + @contextmanager + def _acquisition_lifecycle_access(self) -> Iterator[None]: + """Admit one re-entrant reader behind an explicit reset writer. + + The admission is held for the whole serialized method, not merely + until the RLock is acquired. That prevents an outer acquisition + method which later calls ``self.state()`` from blocking on its own + reset writer. Readers already admitted when the writer registers are + finite; all later polling waits on the condition instead of barging on + the implementation-defined RLock queue. + """ + + depth = int(getattr(self._acquisition_lifecycle_reader_depth, "value", 0)) + if depth == 0: + with self._acquisition_lifecycle_admission: + current_thread_id = threading.get_ident() + while ( + self._acquisition_lifecycle_writer_token is not None + and self._acquisition_lifecycle_writer_thread_id != current_thread_id + ): + self._acquisition_lifecycle_admission.wait() + self._acquisition_lifecycle_reader_depth.value = depth + 1 + try: + with self._acquisition_lifecycle_gate: + yield + finally: + remaining_depth = int(self._acquisition_lifecycle_reader_depth.value) - 1 + if remaining_depth: + self._acquisition_lifecycle_reader_depth.value = remaining_depth + else: + del self._acquisition_lifecycle_reader_depth.value + + def _try_register_acquisition_lifecycle_writer(self, token: object) -> bool: + """Try to block new readers for one fully fenced reset attempt.""" + + with self._acquisition_lifecycle_admission: + if self._acquisition_lifecycle_writer_token is not None: + return False + self._acquisition_lifecycle_writer_token = token + self._acquisition_lifecycle_writer_thread_id = threading.get_ident() + return True + + def _unregister_acquisition_lifecycle_writer(self, token: object) -> bool: + """Release this thread's writer turn and wake every queued reader.""" + + with self._acquisition_lifecycle_admission: + if ( + self._acquisition_lifecycle_writer_token is not token + or self._acquisition_lifecycle_writer_thread_id != threading.get_ident() + ): + return False + self._acquisition_lifecycle_writer_token = None + self._acquisition_lifecycle_writer_thread_id = None + self._acquisition_lifecycle_admission.notify_all() + return True + + def _new_connection_monitor(self) -> ReadOnlyConnectionMonitor: + return ReadOnlyConnectionMonitor( + self._connection_supervisor, + host_path_probe=self._monitor_host_path, + tcp_probe=self._monitor_control_endpoint, + target_provider=self._connection_monitor_target, + interval_seconds=1.0, + ) + + def _connection_monitor_binding(self) -> tuple[str, EndpointTarget] | None: + """Resolve one exact read-only target without relying on UI selection. + + ``_selected_device_id`` and ``_device_session_id`` are process-local + BLE/UI projection. They may be retired after the network transaction + and its MQTT bootstrap have already durably succeeded. Treating that + cleanup as if the configured endpoint disappeared makes the monitor + publish ``endpoint-target-unconfigured`` against an otherwise exact + supervisor intent. + + A live process-local binding remains the preferred source. If both + ephemeral fields are absent, the monitor may continue only when the + current supervisor device-network fact and the durable BLE-proved + semantic topology agree on transport, mode and IPv4. Partial or + conflicting ephemeral state, a pending supervisor intent, corrupt or + changed durable evidence, and physical retirement all fail closed. + This fallback authorizes only host-route and TCP observations; it does + not recreate a BLE session, MQTT control, or mutation authority. + """ + + with self._lock: + selected_device_id = self._selected_device_id + device_session_id = self._device_session_id + network_transition_active = "network" in self._application_control_process_lease_holders + if self._k1_lifecycle_transition_gate.locked() or network_transition_active: + raise ConnectionMonitorProbeSuperseded( + "operator lifecycle transition superseded the read-only monitor" + ) + + supervisor = self._connection_supervisor.snapshot() + if supervisor.closed: + raise ConnectionMonitorProbeSuperseded( + "closed connection supervision superseded the read-only monitor" + ) + intent = supervisor.intent + device_network = supervisor.device_network + transport_ref = device_network.transport_ref + target = device_network.target + if ( + intent is None + or device_network.state != "applied" + or device_network.intent_id != intent.intent_id + or device_network.connection_mode != intent.requested_mode + or not isinstance(transport_ref, str) + or not transport_ref + or target is None + ): + return None + if ( + physical_transport_ref_comparison_key(transport_ref) + in self._retired_physical_transport_refs() + ): + return None + + ephemeral_binding_present = bool( + selected_device_id is not None or device_session_id is not None + ) + if ephemeral_binding_present: + if ( + selected_device_id is None + or device_session_id is None + or physical_transport_ref_comparison_key(selected_device_id) + != physical_transport_ref_comparison_key(transport_ref) + ): + return None + if device_network.source != "durable-semantic-topology": + return transport_ref, target + + store = self._semantic_topology_store + if store is None: + return None + try: + semantic = store.snapshot() + except (SemanticTopologyStoreError, OSError, ValueError): + return None + record = semantic.record + if ( + semantic.status != "available" + or record is None + or physical_transport_ref_comparison_key(record.transport_ref) + != physical_transport_ref_comparison_key(transport_ref) + or record.connection_mode != device_network.connection_mode + or record.ipv4 != target.ipv4 + or record.compatibility_profile_id != XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ): + return None + return transport_ref, target + + def _connection_monitor_target(self) -> EndpointTarget | None: + binding = self._connection_monitor_binding() + return binding[1] if binding is not None else None + + def _wait_for_connection_monitor_quiescence(self) -> bool: + """Wait until an already-admitted read-only physical contact has exited.""" + + acquired = self._connection_monitor_contact_gate.acquire( + timeout=CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS + ) + if acquired: + self._connection_monitor_contact_gate.release() + return acquired + + async def _await_connection_monitor_quiescence(self) -> bool: + """Wait for monitor quiescence without blocking the service event loop.""" + + return await asyncio.to_thread(self._wait_for_connection_monitor_quiescence) + + def _require_monitor_target_current_and_not_retired( + self, + target: EndpointTarget, + ) -> str: + """Bind one monitor contact to the current non-retired transport.""" + + binding = self._connection_monitor_binding() + if binding is None or binding[1] != target: + # Preserve the stronger physical-retirement error for direct + # monitor-probe callers while the target provider itself remains + # a quiet no-contact ``None`` boundary. + supervisor = self._connection_supervisor.snapshot() + transport_ref = supervisor.device_network.transport_ref + if transport_ref is not None and supervisor.device_network.target == target: + self._require_transport_not_physically_retired( + transport_ref, + verification=True, + ) + raise ConnectionLeaseUnavailable( + "monitor target changed before the read-only probe", + reason_code="connection-monitor-target-changed", + ) + transport_ref = binding[0] + self._require_transport_not_physically_retired( + transport_ref, + verification=True, + ) + return transport_ref + + def _semantic_topology_public_snapshot(self) -> dict[str, object]: + store = self._semantic_topology_store + if store is None: + with self._lock: + reason_code = self._semantic_topology_store_reason + return { + "schema_version": "missioncore.xgrids-k1-semantic-topology/v1", + "status": "corrupt", + "configured_offline_evidence": False, + "live_connection_authority": False, + "reason_code": (reason_code or SemanticTopologyStoreCorrupt.reason_code), + "record": None, + } + try: + snapshot = store.snapshot() + except (SemanticTopologyStoreError, OSError, ValueError) as exc: + reason_code = getattr( + exc, + "reason_code", + SemanticTopologyStoreCorrupt.reason_code, + ) + with self._lock: + self._semantic_topology_store_reason = reason_code + return { + "schema_version": "missioncore.xgrids-k1-semantic-topology/v1", + "status": "corrupt", + "configured_offline_evidence": False, + "live_connection_authority": False, + "reason_code": reason_code, + "record": None, + } + with self._lock: + self._semantic_topology_store_reason = snapshot.reason_code + return snapshot.as_dict() + + def _durable_terminal_replay_binding( + self, + *, + operation_id: str, + transport_ref: str, + connection_mode: ConnectionMode, + ) -> dict[str, object] | None: + """Rebuild an exact, non-live parent tuple from matching durable proof.""" + + topology_store = self._semantic_topology_store + if topology_store is None: + return None + try: + ledger = self._network_mutation_ledger.snapshot() + topology = topology_store.snapshot() + except (OSError, ValueError, SemanticTopologyStoreError): + return None + ledger_record = ledger.record + topology_record = topology.record + if not ( + ledger.status == "resolved" + and ledger_record is not None + and ledger_record.operation_id == operation_id + and ledger_record.transport_ref == transport_ref + and ledger_record.intended_mode == connection_mode + and ledger_record.stage == "resolved" + and ledger_record.write_confirmed is True + and ledger_record.resolution == "target-observed" + and ledger_record.last_observation is not None + and topology.status == "available" + and topology_record is not None + and topology_record.transport_ref == transport_ref + and topology_record.connection_mode == connection_mode + and topology_record.source in {"ble-post-write-status", "ble-read-only-status"} + and ledger_record.last_observation.ipv4 == topology_record.ipv4 + and topology_record.compatibility_profile_id == XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ): + return None + return { + "replay_binding_available": True, + "snapshot_runtime_id": self._snapshot_runtime_id, + "parent_intent_id": operation_id, + "transport_ref": transport_ref, + "connection_mode": connection_mode, + "target_ipv4": topology_record.ipv4, + "target_port": CONTROL_MQTT_PORT, + "network_ledger_revision": ledger_record.revision, + "semantic_topology_revision": topology_record.revision, + } + + def _device_identity_pin_public_snapshot(self) -> dict[str, object]: + store = self._device_identity_pin_store + if store is None: + with self._lock: + reason_code = self._device_identity_pin_store_reason + return { + "schema_version": ("missioncore.xgrids-k1-device-identity-pins/v1"), + "status": "corrupt", + "revision": None, + "pin_count": 0, + "reason_code": (reason_code or DeviceIdentityPinStoreCorrupt.reason_code), + } + try: + snapshot = store.snapshot() + except (DeviceIdentityPinStoreError, OSError, ValueError) as exc: + reason_code = getattr( + exc, + "reason_code", + DeviceIdentityPinStoreCorrupt.reason_code, + ) + with self._lock: + self._device_identity_pin_store_reason = reason_code + return { + "schema_version": ("missioncore.xgrids-k1-device-identity-pins/v1"), + "status": "corrupt", + "revision": None, + "pin_count": 0, + "reason_code": reason_code, + } + with self._lock: + self._device_identity_pin_store_reason = snapshot.reason_code + if snapshot.status == "available": + self._vendor_device_ids_by_transport_ref = { + pin.transport_ref: pin.vendor_device_id for pin in snapshot.pins + } + return snapshot.as_public_dict() + + def _expected_vendor_device_id(self, transport_ref: str) -> str | None: + store = self._device_identity_pin_store + if store is None: + raise DeviceIdentityPinStoreCorrupt("device identity pin store is unavailable") + snapshot = store.snapshot() + if snapshot.status == "corrupt": + raise DeviceIdentityPinStoreCorrupt("device identity pin store is corrupt") + pin = snapshot.for_transport(transport_ref) + with self._lock: + self._device_identity_pin_store_reason = None + if pin is not None: + self._vendor_device_ids_by_transport_ref[transport_ref] = pin.vendor_device_id + return pin.vendor_device_id if pin is not None else None + + def _pin_or_match_device_identity( + self, + *, + transport_ref: str, + logical_device_id: str, + compatibility_profile_id: str, + ) -> None: + store = self._device_identity_pin_store + if store is None: + raise DeviceIdentityPinStoreCorrupt("device identity pin store is unavailable") + store.pin_or_match( + transport_ref=transport_ref, + vendor_device_id=logical_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + with self._lock: + self._device_identity_pin_store_reason = None + self._vendor_device_ids_by_transport_ref[transport_ref] = logical_device_id + + def _commit_applied_device_topology( + self, + *, + intent_id: str, + transport_ref: str, + connection_mode: ConnectionMode, + ipv4: str, + compatibility_attestation: CompatibilityAttestationRequest, + captured_device: CapturedDiscoveredDevice | None, + source: Literal["ble-post-write-status", "ble-read-only-status"], + observed_at_utc: str, + ) -> str | None: + """Commit what the K1 reported, independently from the Mac route. + + Once 7f02 proves a topology, the previous route is no longer allowed to + remain the active facade state. Host association, TCP and DeviceInfo + are later evidence planes and may all fail without erasing this fact. + """ + + target = EndpointTarget(validate_private_ipv4(ipv4), CONTROL_MQTT_PORT) + self._persist_semantic_topology( + transport_ref=transport_ref, + connection_mode=connection_mode, + ipv4=target.ipv4, + firmware_version=compatibility_attestation.firmware_version, + source=source, + observed_at_utc=observed_at_utc, + ) + accepted = self._connection_supervisor.observe_device_network_applied( + intent_id=intent_id, + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + source=source, + ) + if not accepted: + raise RuntimeError("доказанная топология K1 относится к устаревшему intent") + + return self._project_applied_device_topology( + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + compatibility_attestation=_attestation_snapshot(compatibility_attestation), + captured_device=captured_device, + source=source, + observed_at_utc=observed_at_utc, + force_new_session=True, + ) + + def _project_applied_device_topology( + self, + *, + transport_ref: str, + connection_mode: ConnectionMode, + target: EndpointTarget, + compatibility_attestation: Mapping[str, Any], + captured_device: CapturedDiscoveredDevice | None, + source: Literal[ + "ble-post-write-status", + "ble-read-only-status", + "durable-semantic-topology", + ], + observed_at_utc: str, + force_new_session: bool, + retained_prepared_stop_owner: _PreparedStopRecoveryOwner | None = None, + ) -> str | None: + """Project one already-persisted/supervised K1 topology into the facade. + + This is intentionally a projection, not another source of truth. The + durable semantic store and the connection supervisor must already have + accepted the exact topology before this method is called. Host routing, + TCP and DeviceInfo are deliberately left unverified. + """ + + with self._lock: + if retained_prepared_stop_owner is not None and ( + physical_transport_ref_comparison_key(transport_ref) + != physical_transport_ref_comparison_key( + retained_prepared_stop_owner.lineage.transport_ref + ) + or connection_mode + != retained_prepared_stop_owner.lineage.connection_mode + or target.ipv4 != retained_prepared_stop_owner.lineage.target_ipv4 + or target.port != retained_prepared_stop_owner.lineage.target_port + ): + raise RuntimeError( + "retained PREPARED STOP recovery cannot rotate its live receiver target" + ) + topology_changed = bool( + self._selected_device_id != transport_ref + or self._connection_mode != connection_mode + or self._k1_ip != target.ipv4 + ) + rotate_session = bool( + force_new_session or topology_changed or self._device_session_id is None + ) + next_device_session_id = ( + new_device_session_id() if rotate_session else self._device_session_id + ) + assert next_device_session_id is not None + carry_retained_acquisition = bool( + rotate_session + and retained_prepared_stop_owner is not None + and self._acquisition is retained_prepared_stop_owner.acquisition + and self._acquisition_session_lease + is retained_prepared_stop_owner.session_lease + and self._acquisition_out_dir == retained_prepared_stop_owner.out_dir + and self._acquisition_stop_operation_id + == retained_prepared_stop_owner.lineage.stop_operation_id + and self._acquisition_start_operation_id + == retained_prepared_stop_owner.start_operation_id + ) + if retained_prepared_stop_owner is not None and not carry_retained_acquisition: + raise RuntimeError( + "retained PREPARED STOP acquisition changed before topology rotation" + ) + retired_ingress_session_id = ( + self._acquisition_out_dir.name + if rotate_session + and not carry_retained_acquisition + and self._acquisition_out_dir is not None + else None + ) + if rotate_session and not carry_retained_acquisition: + self._acquisition = None + self._restart_stop_only_checkpoint_owner = None + self._acquisition_project_name = None + self._acquisition_mount_type = None + self._acquisition_gnss_mode = None + self._acquisition_out_dir = None + self._acquisition_start_operation_id = None + self._acquisition_stop_operation_id = None + self._selected_device_id = transport_ref + self._k1_ip = target.ipv4 + self._connection_mode = connection_mode + self._device_id = self._device_ids_by_transport_ref.setdefault( + transport_ref, + new_device_id(), + ) + self._device_session_id = next_device_session_id + self._device_session_opened_at = observed_at_utc + if rotate_session: + self._device_session_discovery_generation = self._ble_discovery_generation + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + if rotate_session: + self._connection_lease_generation += 1 + self._device_calibration = unavailable_device_calibration_snapshot( + XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ) + self._compatibility_attestation = dict(compatibility_attestation) + self._connection_verification = { + "status": "device-network-applied", + "lease_state": "configured-unverified", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": "not-performed", + "network_reachability": "unknown", + "reason_code": "device_topology_applied_host_unverified", + "write_performed": source == "ble-post-write-status", + "observed_at": observed_at_utc, + } + self._operation_message = ( + "Сохранённая сеть K1 принята для проверки; маршрут и DeviceInfo " + "ещё не подтверждены." + if source == "durable-semantic-topology" + else "K1 подтвердил новую сеть; маршрут этого компьютера и DeviceInfo " + "ещё не проверены." + ) + if captured_device is not None: + pin_connected_device_handle( + captured_device, + device_session_id=next_device_session_id, + ) + return retired_ingress_session_id + + def _persist_semantic_topology( + self, + *, + transport_ref: str, + connection_mode: ConnectionMode, + ipv4: str, + firmware_version: str, + source: Literal["ble-post-write-status", "ble-read-only-status"], + observed_at_utc: str, + predecessor_revision: int | None = None, + ) -> None: + """Persist exact K1 status evidence before any authority is promoted.""" + + store = self._semantic_topology_store + if store is None: + raise SemanticTopologyStoreCorrupt( + "semantic topology store is unavailable; mutation remains fenced" + ) + store.commit( + transport_ref=transport_ref, + connection_mode=connection_mode, + ipv4=validate_private_ipv4(ipv4), + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version=firmware_version, + source=source, + observed_at_utc=observed_at_utc, + predecessor_revision=predecessor_revision, + ) + with self._lock: + self._semantic_topology_store_reason = None + + def _require_semantic_topology_store_writable(self) -> None: + """Fail before any K1 write when its result could not be persisted.""" + + store = self._semantic_topology_store + if store is None: + raise SemanticTopologyStoreCorrupt( + "semantic topology store is unavailable; network mutation is blocked" + ) + snapshot = store.snapshot() + if snapshot.status == "corrupt": + with self._lock: + self._semantic_topology_store_reason = ( + snapshot.reason_code or SemanticTopologyStoreCorrupt.reason_code + ) + raise SemanticTopologyStoreCorrupt( + "semantic topology store is corrupt; network mutation is blocked" + ) + + def _current_firmware_version(self) -> str: + with self._lock: + attestation = ( + dict(self._compatibility_attestation) + if self._compatibility_attestation is not None + else None + ) + firmware_version = ( + attestation.get("firmware_version") if isinstance(attestation, Mapping) else None + ) + if firmware_version != "3.0.2": + raise RuntimeError( + "точный compatibility-профиль K1 отсутствует; read-only topology не принята" + ) + return firmware_version + + def _update_applied_topology_reachability( + self, + *, + connection_mode: ConnectionMode, + ipv4: str, + host_route_class: str, + host_route_ready: bool, + endpoint_reachable: bool, + write_performed: bool, + ) -> None: + """Project host/TCP facts without promoting them to control authority.""" + + with self._lock: + if self._connection_mode != connection_mode or self._k1_ip != ipv4: + raise RuntimeError("применённая топология изменилась во время проверки хоста") + verification_status = ( + "tcp-reachable-device-info-unverified" + if endpoint_reachable + else "endpoint-unreachable" + if host_route_ready + else "host-route-mismatch" + ) + self._connection_verification = { + "status": verification_status, + "lease_state": "configured-unverified", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": ( + "provisioning-status+mqtt-tcp-connect" if host_route_ready else "host-route" + ), + "network_reachability": ("reachable" if endpoint_reachable else "unreachable"), + **( + {} + if endpoint_reachable + else { + "reason_code": ( + "connection_lease_endpoint_unreachable_after_provision" + if host_route_ready + else "connection_lease_host_route_mismatch" + ), + "host_route_class": host_route_class, + } + ), + "write_performed": write_performed, + "observed_at": _utc_now_iso(), + } + self._operation_message = ( + ( + "Сетевые настройки K1 сохранены; подключение к устройству ещё не подтверждено." + if endpoint_reachable + else "K1 подтвердил сеть, но управляющий канал пока недоступен. " + "Новая команда Wi-Fi не отправлялась." + ) + if host_route_ready + else "K1 подтвердил сеть, но этот компьютер находится на другом маршруте." + ) + + def _sample_host_path( + self, + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + """Bind one kernel route sample to this service's Wi-Fi association.""" + + path = _inspect_host_path(target) + association = self._host_wifi_association_probe.observe( + path.interface, + timeout_seconds=association_timeout_seconds, + ) + if not path.available: + return path + association_reason = association["reason_code"] + retained_unproven_path = self._retain_path_across_unproven_association_observation( + target=target, + path=path, + association=association, + ) + if retained_unproven_path is not None: + return retained_unproven_path + if ( + fail_closed_unproven_association + and not association["continuity_proven"] + # Missing BSSID is a documented macOS privacy condition. Its + # process-scoped fallback token still detects route/interface + # transitions. The monitor marks that result as unproven below so + # it cannot replace a verified association fingerprint merely + # because macOS changed what identity fields it disclosed. + and association_reason != "association-identity-unavailable" + ): + return HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code=(association_reason or "host-wifi-association-observation-unavailable"), + observation_failure_class=( + "association-observer" + if association_reason in CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS + else "route" + ), + kernel_route_fingerprint=path.fingerprint, + ) + assert path.fingerprint is not None + return HostPathProbeResult( + available=True, + fingerprint=bind_route_fingerprint_to_wifi_association( + path.fingerprint, + association, + ), + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class=path.route_class, + reason_code=( + path.reason_code + or (association["reason_code"] if not association["continuity_proven"] else None) + ), + observation_failure_class=( + "route" if association["continuity_proven"] else "association-observer" + ), + kernel_route_fingerprint=path.fingerprint, + ) + + def _retain_path_across_unproven_association_observation( + self, + *, + target: str, + path: HostPathProbeResult, + association: HostWifiAssociationIdentityResult, + ) -> HostPathProbeResult | None: + """Keep one proven route fingerprint when CoreWLAN is only unproven. + + The association helper is an additional continuity observer. Its + timeout or privacy-limited identity is not evidence that the kernel + route changed, and its process-local fallback token must never + manufacture a new host epoch. + Retention is deliberately narrow: the applied intent and MQTT target, + the last reachable endpoint, and every raw kernel route field must be + exact. This preserves neither DeviceInfo nor command authority by + itself; configured-unverified callers still need DeviceInfo, while an + already-open command session separately validates its retained MQTT + control proof before this path guard runs. + + A privacy-limited observation has the stricter gate below: exact + DeviceInfo, healthy control session, reachable lease and command + authority must all still match the same intent/mode/target/epoch. The + legacy timeout bridge may retain only its exact reachable route but + cannot manufacture any missing command authority. + + A proven association change, a different route/interface/source, a + stale or unreachable endpoint, or a different intent/target bypasses + this bridge and continues through the normal fail-closed reducer. + """ + + if ( + association["reason_code"] + not in { + "host-wifi-operation-timeout", + "association-identity-unavailable", + } + or association["continuity_proven"] + or not path.available + or path.route_class != "direct" + or path.fingerprint is None + ): + return None + endpoint_target = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT) + snapshot = self._connection_supervisor.association_timeout_retention_candidate( + expected_target=endpoint_target, + ) + if snapshot is None: + return None + intent = snapshot.intent + previous = snapshot.host_path + exact_reachable_route = bool( + intent is not None + and snapshot.device_network.state == "applied" + and snapshot.device_network.intent_id == intent.intent_id + and snapshot.device_network.connection_mode == intent.requested_mode + and snapshot.device_network.target == endpoint_target + and previous.available + and previous.fingerprint is not None + and previous.kernel_route_fingerprint is not None + and previous.route_class == "direct" + and previous.kernel_route_fingerprint == path.fingerprint + and previous.interface == path.interface + and previous.source_ipv4 == path.source_ipv4 + and snapshot.endpoint.target == endpoint_target + and snapshot.endpoint.intent_id == intent.intent_id + and snapshot.endpoint.host_path_epoch == previous.epoch + and snapshot.endpoint.tcp_state == "reachable" + ) + if not exact_reachable_route: + return None + if association["reason_code"] == "association-identity-unavailable": + exact_verified_control = bool( + intent is not None + and snapshot.device_identity.state == "verified" + and snapshot.device_identity.intent_id == intent.intent_id + and snapshot.device_identity.connection_mode == intent.requested_mode + and snapshot.device_identity.host_path_epoch == previous.epoch + and snapshot.control_plane.state == "healthy" + and isinstance(snapshot.control_plane.session_id, str) + and bool(snapshot.control_plane.session_id.strip()) + and snapshot.control_plane.host_path_epoch == previous.epoch + and snapshot.lease.state == "reachable" + and snapshot.lease.intent_id == intent.intent_id + and snapshot.lease.host_path_epoch == previous.epoch + and snapshot.lease.connection_mode == intent.requested_mode + and snapshot.lease.target == endpoint_target + and snapshot.authority.control_allowed + ) + if not exact_verified_control: + return None + return HostPathProbeResult( + available=True, + fingerprint=previous.fingerprint, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class=path.route_class, + # Do not project an observer timeout as a route failure. The + # result still carries its narrow failure class for the monitor's + # continuity logic, while public route diagnostics remain true to + # the retained positive kernel/TCP evidence. + reason_code=None, + observation_failure_class="association-observer", + kernel_route_fingerprint=path.fingerprint, + ) + + async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult: + if not self._connection_monitor_contact_gate.acquire(blocking=False): + raise ConnectionMonitorProbeSuperseded( + "another read-only monitor contact is still active" + ) + process_lease_acquired = False + try: + self._connection_monitor_target() + self._acquire_k1_lifecycle_process_lease("monitor") + process_lease_acquired = True + self._require_monitor_target_current_and_not_retired(target) + result = await _run_blocking_operation_without_abandonment( + self._sample_host_path, + target.ipv4, + association_timeout_seconds=(CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS), + fail_closed_unproven_association=True, + ) + self._require_monitor_target_current_and_not_retired(target) + return result + except ( + ApplicationControlProcessLeaseUnavailable, + ConnectionLeaseUnavailable, + ) as exc: + raise ConnectionMonitorProbeSuperseded(str(exc)) from exc + finally: + if process_lease_acquired: + self._release_k1_lifecycle_process_lease("monitor") + self._connection_monitor_contact_gate.release() + + async def _monitor_control_endpoint( + self, + target: EndpointTarget, + ) -> TcpReachabilityProbeResult: + if not self._connection_monitor_contact_gate.acquire(blocking=False): + raise ConnectionMonitorProbeSuperseded( + "another read-only monitor contact is still active" + ) + process_lease_acquired = False + try: + self._connection_monitor_target() + self._acquire_k1_lifecycle_process_lease("monitor") + process_lease_acquired = True + self._require_monitor_target_current_and_not_retired(target) + result = await _run_blocking_operation_without_abandonment( + _probe_control_endpoint_socket, + target.ipv4, + ) + self._require_monitor_target_current_and_not_retired(target) + return result + except ( + ApplicationControlProcessLeaseUnavailable, + ConnectionLeaseUnavailable, + ) as exc: + raise ConnectionMonitorProbeSuperseded(str(exc)) from exc + finally: + if process_lease_acquired: + self._release_k1_lifecycle_process_lease("monitor") + self._connection_monitor_contact_gate.release() + + async def _start_connection_monitor(self) -> None: + try: + await self._connection_monitor.start() + except Exception: + # This watcher never mutates the K1. If its own lifecycle fails, + # revoke authority and keep the main plugin runtime alive so the UI + # can present a diagnosable, explicitly unavailable connection. + logger.exception( + "K1 read-only connection monitor failed to start", + extra={ + "event_code": "k1_connection_monitor_start_failed", + "automatic_retry": False, + }, + ) + self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed") + + def _observe_connection_transport( + self, + target: str, + *, + path: HostPathProbeResult, + reachable: bool | None, + reason_code: str | None = None, + ) -> None: + supervisor = self._connection_supervisor.snapshot() + intent = supervisor.intent + endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT) + if ( + intent is None + or supervisor.device_network.state != "applied" + or supervisor.device_network.intent_id != intent.intent_id + or supervisor.device_network.connection_mode != intent.requested_mode + or supervisor.device_network.target != endpoint + ): + return + epoch = self._connection_supervisor.observe_host_path(path) + if path.available and reachable is not None: + self._connection_supervisor.observe_endpoint( + target=endpoint, + intent_id=intent.intent_id, + host_path_epoch=epoch, + reachable=reachable, + reason_code=(None if reachable else reason_code or "tcp-endpoint-unreachable"), + ) + + def _probe_control_endpoint( + self, + target: str, + *, + association_timeout_seconds: float = 30.0, + ) -> _CorrelatedEndpointObservation: + """Correlate TCP with exact route and Wi-Fi association continuity.""" + + first_path = self._sample_host_path( + target, + association_timeout_seconds=association_timeout_seconds, + ) + self._observe_connection_transport( + target, + path=first_path, + reachable=None, + ) + if not first_path.available: + return _CorrelatedEndpointObservation( + path=first_path, + reachable=False, + reason_code=first_path.reason_code or "host-path-unavailable", + ) + + tcp_reachable = _control_endpoint_reachable(target) + final_path = self._sample_host_path( + target, + association_timeout_seconds=association_timeout_seconds, + ) + fingerprint_unchanged = bool( + final_path.available + and first_path.fingerprint is not None + and final_path.fingerprint == first_path.fingerprint + ) + reachable = tcp_reachable and fingerprint_unchanged + reason_code = ( + None + if reachable + else final_path.reason_code or "host-path-unavailable" + if not final_path.available + else "host-path-changed-during-tcp-probe" + if not fingerprint_unchanged + else "tcp-endpoint-unreachable" + ) + self._observe_connection_transport( + target, + path=final_path, + reachable=reachable if final_path.available else None, + reason_code=reason_code, + ) + return _CorrelatedEndpointObservation( + path=final_path, + reachable=reachable, + reason_code=reason_code, + ) + + def _admit_read_only_device_network( + self, + *, + transport_ref: str, + connection_mode: ConnectionMode, + ipv4: str, + firmware_version: str, + observed_at_utc: str, + semantic_predecessor_revision: int | None = None, + force_new_intent: bool = False, + persist_semantic_topology: bool = True, + ) -> str: + """Bind an exact BLE status read to a fresh non-mutating intent.""" + + target = EndpointTarget(validate_private_ipv4(ipv4), CONTROL_MQTT_PORT) + # Ordinarily the durable fact precedes in-memory projection. A fresh + # Bridge selection (explicit reconfiguration or ordinary cold + # read-only-first) may defer persistence until its MQTT DeviceInfo + # identity has matched the immutable transport pin. Exact lifecycle + # ownership blocks START/adoption in that interval. + if persist_semantic_topology: + self._persist_semantic_topology( + transport_ref=transport_ref, + connection_mode=connection_mode, + ipv4=target.ipv4, + firmware_version=firmware_version, + source="ble-read-only-status", + observed_at_utc=observed_at_utc, + predecessor_revision=semantic_predecessor_revision, + ) + supervisor = self._connection_supervisor.snapshot() + current = bool( + supervisor.intent is not None + and supervisor.intent.requested_mode == connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.transport_ref == transport_ref + and supervisor.device_network.connection_mode == connection_mode + and supervisor.device_network.target == target + ) + if current and not force_new_intent: + assert supervisor.intent is not None + return supervisor.intent.intent_id + expected_vendor_device_id = self._expected_vendor_device_id(transport_ref) + intent_id = f"read-only-network-{secrets.token_hex(12)}" + self._connection_supervisor.set_intent( + intent_id=intent_id, + requested_mode=connection_mode, + expected_device_id=expected_vendor_device_id, + ) + if not self._connection_supervisor.observe_device_network_applied( + intent_id=intent_id, + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + source="ble-read-only-status", + ): + raise RuntimeError("read-only статус K1 потерял актуальность до admission") + return intent_id + + def _apply_read_only_device_topology( + self, + *, + transport_ref: str, + connection_mode: ConnectionMode, + ipv4: str, + firmware_version: str, + observed_at_utc: str, + captured_device: CapturedDiscoveredDevice | None = None, + semantic_predecessor_revision: int | None = None, + compatibility_attestation: Mapping[str, Any] | None = None, + force_new_intent: bool = False, + force_new_session: bool = False, + persist_semantic_topology: bool = True, + retained_prepared_stop_owner: _PreparedStopRecoveryOwner | None = None, + ) -> str: + """Persist, supervise and project one exact non-mutating BLE status. + + The normal order is durable topology first, supervisor intent second, + facade projection third. Fresh Bridge selection deliberately uses a + provisional projection until DeviceInfo identity is pinned; unresolved + network-write reconciliation retains the durable-first order. + """ + + if compatibility_attestation is None: + with self._lock: + attestation_snapshot = ( + dict(self._compatibility_attestation) + if self._compatibility_attestation is not None + else None + ) + if attestation_snapshot is None: + raise RuntimeError( + "точный compatibility-профиль K1 отсутствует; topology не спроецирована" + ) + else: + attestation_snapshot = dict(compatibility_attestation) + if ( + attestation_snapshot.get("firmware_version") != firmware_version + or attestation_snapshot.get("topology") != CONNECTION_TOPOLOGY_BY_MODE[connection_mode] + or attestation_snapshot.get("verification") != "live-device-info" + ): + raise RuntimeError("topology и точный compatibility-профиль K1 не совпадают") + target = EndpointTarget(validate_private_ipv4(ipv4), CONTROL_MQTT_PORT) + intent_id = self._admit_read_only_device_network( + transport_ref=transport_ref, + connection_mode=connection_mode, + ipv4=target.ipv4, + firmware_version=firmware_version, + observed_at_utc=observed_at_utc, + semantic_predecessor_revision=semantic_predecessor_revision, + force_new_intent=force_new_intent, + persist_semantic_topology=persist_semantic_topology, + ) + retired_ingress_session_id = self._project_applied_device_topology( + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + compatibility_attestation=attestation_snapshot, + captured_device=captured_device, + source="ble-read-only-status", + observed_at_utc=observed_at_utc, + force_new_session=force_new_session, + retained_prepared_stop_owner=retained_prepared_stop_owner, + ) + if retired_ingress_session_id is not None: + self.live_perception_ingress.end_session(retired_ingress_session_id) + return intent_id + + def _apply_durable_semantic_topology( + self, + *, + record: SemanticTopologyRecord, + compatibility_attestation: Mapping[str, Any], + observed_at_utc: str, + ) -> str: + """Project a persisted device-reported topology for live LAN verification. + + The durable record grants no control authority by itself. It only pins + the exact transport, mode and endpoint that the host may probe after a + restart. MQTT DeviceInfo in the subsequently opened control session is + still required before the supervisor can grant control or START. + """ + + attestation_snapshot = dict(compatibility_attestation) + if ( + record.compatibility_profile_id != XGRIDS_K1_COMPATIBILITY_PROFILE_ID + or attestation_snapshot.get("firmware_version") != record.firmware_version + or attestation_snapshot.get("topology") + != CONNECTION_TOPOLOGY_BY_MODE[record.connection_mode] + or attestation_snapshot.get("verification") != "live-device-info" + ): + raise RuntimeError("durable topology и точный compatibility-профиль K1 не совпадают") + target = EndpointTarget(validate_private_ipv4(record.ipv4), CONTROL_MQTT_PORT) + intent_id = f"durable-network-{secrets.token_hex(12)}" + self._connection_supervisor.set_intent( + intent_id=intent_id, + requested_mode=record.connection_mode, + expected_device_id=self._expected_vendor_device_id(record.transport_ref), + ) + if not self._connection_supervisor.observe_device_network_applied( + intent_id=intent_id, + transport_ref=record.transport_ref, + connection_mode=record.connection_mode, + target=target, + source="durable-semantic-topology", + ): + raise RuntimeError("durable topology K1 потеряла актуальность до admission") + retired_ingress_session_id = self._project_applied_device_topology( + transport_ref=record.transport_ref, + connection_mode=record.connection_mode, + target=target, + compatibility_attestation=attestation_snapshot, + captured_device=None, + source="durable-semantic-topology", + observed_at_utc=observed_at_utc, + force_new_session=True, + ) + if retired_ingress_session_id is not None: + self.live_perception_ingress.end_session(retired_ingress_session_id) + return intent_id + + def _require_supervised_device_network( + self, + *, + transport_ref: str, + connection_mode: ConnectionMode, + ipv4: str, + ) -> None: + target = EndpointTarget(validate_private_ipv4(ipv4), CONTROL_MQTT_PORT) + supervisor = self._connection_supervisor.snapshot() + if not ( + supervisor.intent is not None + and supervisor.intent.requested_mode == connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.transport_ref == transport_ref + and supervisor.device_network.connection_mode == connection_mode + and supervisor.device_network.target == target + ): + raise ConnectionLeaseUnavailable( + "топология K1 не имеет актуального BLE-доказательства для этого intent", + reason_code="connection_device_network_proof_missing", + ) + + def bind_runtime_event_loop( + self, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + """Bind retained CoreBluetooth handles to one persistent runtime loop.""" + + runtime_loop = loop or asyncio.get_running_loop() + start_connection_monitor = False + with self._lock: + current = self._runtime_event_loop + if current is runtime_loop: + bind_ble_runtime_owner_loop(runtime_loop) + else: + if current is not None and not current.is_closed(): + raise BleRuntimeOwnerLoopConflict() + if current is not None: + # A BLEDevice retained by CoreBluetooth also retains the loop + # which discovered it. Never carry that lease across a closed + # loop even in an embedded/test runtime. + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + invalidate_ble_runtime_owner_loop(current) + invalidate_runtime_handles() + self._connection_monitor = self._new_connection_monitor() + self._connection_monitor_started = False + bind_ble_runtime_owner_loop(runtime_loop) + self._runtime_event_loop = runtime_loop + if not self._connection_monitor_started: + self._connection_monitor_started = True + start_connection_monitor = True + if start_connection_monitor: + runtime_loop.create_task( + self._start_connection_monitor(), + name="k1-connection-monitor-start", + ) + + def _read_wifi_status_on_runtime_loop( + self, + device_id: str, + *, + timeout_seconds: float, + rediscover: bool, + ) -> Mapping[str, Any]: + """Schedule BLE status I/O on the loop that owns the retained handle.""" + + with self._lock: + runtime_loop = self._runtime_event_loop + if runtime_loop is None or not runtime_loop.is_running(): + raise RuntimeError( + "BLE runtime loop недоступен; повторите действие через plugin runtime" + ) + try: + caller_loop = asyncio.get_running_loop() + except RuntimeError: + caller_loop = None + if caller_loop is runtime_loop: + raise RuntimeError("синхронная BLE-граница не может блокировать runtime event loop") + future = asyncio.run_coroutine_threadsafe( + read_wifi_status_once( + device_id, + timeout_seconds=timeout_seconds, + rediscover=rediscover, + ), + runtime_loop, + ) + try: + return future.result(timeout=timeout_seconds + 10.0) + except TimeoutError as exc: + future.cancel() + raise RuntimeError( + "BLE status-read не завершился на runtime event loop вовремя" + ) from exc + def _application_control_transport(self, host: str) -> ReviewedApplicationMqttTransport: with self._lock: connection_mode = self._connection_mode @@ -535,102 +3079,4113 @@ class XgridsK1CompatibilityService: allow_device_ap=connection_mode == "quick-connect", ) - def _fresh_ble_devices_locked( + def _validate_application_connection_binding( + self, + binding: ApplicationConnectionBinding, + ) -> None: + """Authorize one command only for the exact live DeviceInfo route epoch.""" + + self._validate_application_connection_path(binding) + + # The session validates its retained MQTT transport before invoking + # this callback. Project a strictly newer remote proof revision into + # the supervisor before testing its TTL-bound authority; otherwise a + # healthy heartbeat could be ignored merely because no browser polled + # state between two operator commands. + self._reconcile_connection_supervisor( + self._application_control_session.snapshot(), + self.runtime.snapshot(), + ) + supervisor = self._connection_supervisor.snapshot() + target = EndpointTarget(binding.target_ipv4, binding.target_port) + binding_is_current = bool( + not supervisor.closed + and supervisor.intent is not None + and supervisor.intent.intent_id == binding.intent_id + and supervisor.intent.requested_mode == binding.connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == binding.intent_id + and supervisor.device_network.transport_ref == binding.transport_ref + and supervisor.device_network.connection_mode == binding.connection_mode + and supervisor.device_network.target == target + and supervisor.host_path.available + and supervisor.host_path.epoch == binding.host_path_epoch + and supervisor.endpoint.target == target + and supervisor.endpoint.intent_id == binding.intent_id + and supervisor.endpoint.host_path_epoch == binding.host_path_epoch + and supervisor.endpoint.tcp_state == "reachable" + and supervisor.device_identity.state == "verified" + and supervisor.device_identity.intent_id == binding.intent_id + and supervisor.device_identity.connection_mode == binding.connection_mode + and supervisor.device_identity.host_path_epoch == binding.host_path_epoch + and supervisor.control_plane.state == "healthy" + and supervisor.control_plane.host_path_epoch == binding.host_path_epoch + and supervisor.lease.state == "reachable" + and supervisor.lease.intent_id == binding.intent_id + and supervisor.lease.host_path_epoch == binding.host_path_epoch + and supervisor.lease.connection_mode == binding.connection_mode + and supervisor.lease.target == target + and supervisor.authority.control_allowed + ) + if not binding_is_current: + raise ApplicationConnectionBindingLost( + "маршрут K1 изменился или потерял актуальную MQTT/DeviceInfo authority" + ) + logical_device_id = supervisor.device_identity.logical_device_id + compatibility_profile_id = supervisor.device_identity.compatibility_profile_id + if logical_device_id is None or compatibility_profile_id is None: + raise ApplicationConnectionBindingLost( + "DeviceInfo identity для текущего маршрута K1 отсутствует" + ) + try: + self._pin_or_match_device_identity( + transport_ref=binding.transport_ref, + logical_device_id=logical_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + except DeviceIdentityPinStoreError as exc: + with self._lock: + self._device_identity_pin_store_reason = getattr( + exc, + "reason_code", + DeviceIdentityPinStoreError.reason_code, + ) + raise ApplicationConnectionBindingLost( + "долговечная привязка BLE transport к DeviceInfo identity не подтверждена" + ) from exc + + def _validate_application_connection_path( + self, + binding: ApplicationConnectionBinding, + ) -> None: + """Re-sample and authorize the exact host route before every publish. + + DeviceInfo is intentionally absent from this pre-bootstrap check. The + retained route epoch, however, is never trusted by itself: every MQTT + envelope gets a fresh kernel/CoreWLAN path sample. Any Wi-Fi, VPN, + source-address or interface change rotates the supervisor epoch and + invalidates the bound socket before the envelope is handed to MQTT. + """ + + with self._lock: + if self._provisioning_active: + raise ApplicationConnectionBindingLost( + "настройка сети K1 уже началась; прежний control path отозван" + ) + target = EndpointTarget(binding.target_ipv4, binding.target_port) + path = self._sample_host_path( + binding.target_ipv4, + association_timeout_seconds=COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS, + ) + observed_epoch = self._connection_supervisor.observe_host_path_if_current( + expected_intent_id=binding.intent_id, + expected_target=target, + expected_host_path_epoch=binding.host_path_epoch, + result=path, + ) + if observed_epoch is None: + raise ApplicationConnectionBindingLost( + "маршрут K1 изменился во время контрольной проверки" + ) + supervisor = self._connection_supervisor.snapshot() + path_is_current = bool( + not supervisor.closed + and path.available + and path.route_class == "direct" + and observed_epoch == binding.host_path_epoch + and supervisor.host_path.epoch == binding.host_path_epoch + and supervisor.intent is not None + and supervisor.intent.intent_id == binding.intent_id + and supervisor.intent.requested_mode == binding.connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == binding.intent_id + and supervisor.device_network.transport_ref == binding.transport_ref + and supervisor.device_network.connection_mode == binding.connection_mode + and supervisor.device_network.target == target + and supervisor.endpoint.target == target + and supervisor.endpoint.intent_id == binding.intent_id + and supervisor.endpoint.host_path_epoch == binding.host_path_epoch + and supervisor.endpoint.tcp_state == "reachable" + ) + if not path_is_current: + raise ApplicationConnectionBindingLost( + "маршрут K1 изменился или не имеет свежего прямого host/TCP доказательства" + ) + + def _acquire_k1_lifecycle_process_lease( + self, + holder: Literal["control", "network", "camera", "monitor"], + ) -> None: + """Share one OS lease across every K1 network/control owner in this service.""" + + with self._k1_process_lease_gate: + with self._lock: + lease = self._application_control_process_lease + holders = self._application_control_process_lease_holders + quarantine = self._application_control_process_lease_quarantine + if quarantine is not None: + raise ApplicationControlProcessLeaseUnavailable( + "K1 lifecycle process lease release is quarantined; " + "restart Mission Core before another device lifecycle" + ) + if holder in holders: + raise ApplicationControlProcessLeaseUnavailable( + f"this Mission Core service already owns K1 {holder} lifecycle" + ) + monitor_conflicts_with_network = bool( + (holder == "monitor" and "network" in holders) + or (holder == "network" and "monitor" in holders) + ) + if monitor_conflicts_with_network: + raise ApplicationControlProcessLeaseUnavailable( + "read-only K1 monitor contact cannot overlap a network mutation" + ) + if lease is not None: + if lease.release_state != "owned": + raise ApplicationControlProcessLeaseUnavailable( + "K1 lifecycle process lease no longer proves OS ownership; " + "restart Mission Core before another device lifecycle" + ) + ble_snapshot = ble_runtime_snapshot() + if ( + ble_snapshot["active_operation_kind"] is not None + or ble_snapshot["poisoned"] + ): + raise ApplicationControlProcessLeaseUnavailable( + "K1 lifecycle remains quarantined by native BLE cleanup" + ) + holders.add(holder) + return + candidate = ApplicationControlProcessLease.acquire(self.repository_root) + with self._lock: + if self._application_control_process_lease is not None: + candidate.release() + raise ApplicationControlProcessLeaseUnavailable( + "K1 lifecycle ownership changed during process admission" + ) + self._application_control_process_lease = candidate + self._application_control_process_lease_holders.add(holder) + + def _release_k1_lifecycle_process_lease( + self, + holder: Literal["control", "network", "camera", "monitor"], + ) -> None: + with self._k1_process_lease_gate: + with self._lock: + holders = self._application_control_process_lease_holders + if holder not in holders: + return + if holders != {holder}: + holders.remove(holder) + return + lease = self._application_control_process_lease + if lease is None: + raise ApplicationControlProcessLeaseUnavailable( + "K1 lifecycle holder lost its process lease" + ) + # The last holder is also the only process-visible proof that + # the cross-process fence still exists. Keep both it and the + # exact lease pointer published until the underlying release + # succeeds. A pre-side-effect release failure can then be + # retried without falsely advertising an unfenced K1 target. + # `_k1_process_lease_gate` serializes every holder mutation + # across this call, so the identity check below is a strict + # assertion rather than a best-effort cleanup. + try: + release_outcome = lease.release() + except ApplicationControlProcessLeaseReleaseAmbiguous as exc: + with self._lock: + if ( + self._application_control_process_lease is not lease + or self._application_control_process_lease_holders != {holder} + ): + raise RuntimeError( + "K1 lifecycle process lease owner changed during release" + ) from exc + self._application_control_process_lease_holders.remove(holder) + self._application_control_process_lease = None + self._application_control_process_lease_quarantine = ( + lease, + exc.reason_code, + ) + raise + else: + with self._lock: + if ( + self._application_control_process_lease is not lease + or self._application_control_process_lease_holders != {holder} + ): + raise RuntimeError( + "K1 lifecycle process lease owner changed during release" + ) + self._application_control_process_lease_holders.remove(holder) + self._application_control_process_lease = None + if ( + release_outcome.unlock_error_code is not None + or release_outcome.close_error_code is not None + ): + logger.warning( + "K1 lifecycle process lease released with syscall diagnostics", + extra={ + "event_code": "k1_process_lease_release_diagnostic", + "unlock_error_code": release_outcome.unlock_error_code, + "close_error_code": release_outcome.close_error_code, + "restart_required": False, + }, + ) + + def _release_network_process_lease_after_ble_cleanup(self) -> None: + """Keep the cross-process fence until CoreBluetooth is proven idle.""" + + defer_until_ble_runtime_idle(lambda: self._release_k1_lifecycle_process_lease("network")) + + def _borrow_network_ble_process_lease( + self, + ) -> AbstractContextManager[BleRuntimeProcessLeaseBorrowToken]: + """Prove the high-level network owner to the low-level BLE boundary.""" + + with self._lock: + lease = self._application_control_process_lease + network_owned = "network" in self._application_control_process_lease_holders + if lease is None or not network_owned: + raise ApplicationControlProcessLeaseUnavailable( + "K1 network lifecycle lease disappeared before BLE admission" + ) + return borrow_ble_runtime_process_lease(lease) + + def _duplicate_network_process_fence_descriptor(self) -> int: + """Give one host-network child the exact active lifecycle flock.""" + + with self._k1_process_lease_gate: + with self._lock: + lease = self._application_control_process_lease + network_owned = "network" in self._application_control_process_lease_holders + if lease is None or not network_owned: + raise ApplicationControlProcessLeaseUnavailable( + "K1 network lifecycle lease disappeared before helper admission" + ) + return lease.duplicate_descriptor_for_child() + + def _acquire_application_control_process_lease(self) -> None: + self._acquire_k1_lifecycle_process_lease("control") + + def _release_application_control_process_lease(self) -> None: + self._release_k1_lifecycle_process_lease("control") + + def _ensure_camera_preview_process_lease(self) -> bool: + """Retain cross-process ownership for one selected RTSP authority.""" + + with self._lock: + if "camera" in self._application_control_process_lease_holders: + return False + self._acquire_k1_lifecycle_process_lease("camera") + return True + + def _release_camera_preview_process_lease(self) -> None: + with self._lock: + owned = "camera" in self._application_control_process_lease_holders + if owned: + self._release_k1_lifecycle_process_lease("camera") + + def _duplicate_camera_process_fence_descriptor(self) -> int: + """Give FFmpeg one inherited duplicate of the exact camera flock.""" + + with self._k1_process_lease_gate: + with self._lock: + lease = self._application_control_process_lease + camera_owned = "camera" in self._application_control_process_lease_holders + if lease is None or not camera_owned: + raise ApplicationControlProcessLeaseUnavailable( + "K1 camera lifecycle lease disappeared before FFmpeg admission" + ) + return lease.duplicate_descriptor_for_child() + + def _stop_camera_preview_and_release_process_lease(self) -> dict[str, Any]: + """Prove FFmpeg teardown before releasing the selected-target fence.""" + + snapshot = self.camera_preview.stop_current() + self._release_camera_preview_process_lease() + return snapshot + + def _acquire_application_dispatch_lease( + self, + binding: ApplicationConnectionBinding, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + ) -> Callable[[], None]: + """Hold the local network/control fence across one MQTT publish call.""" + + if dispatch_admission_deadline_reached is None: + acquired = self._k1_command_dispatch_gate.acquire(blocking=False) + else: + acquired = False + # An explicit STOP already owns a hard public deadline. A short + # process-local lifecycle holder is contention, not proof that the + # K1 route disappeared, so wait reversibly until that deadline + # instead of classifying STOP as definitely not dispatched on the + # first failed try_lock. The final deadline/permit/ledger admission + # remains atomic in the physical coordinator after this lease. + while not dispatch_admission_deadline_reached(): + if self._k1_command_dispatch_gate.acquire(timeout=0.05): + acquired = True + break + if not acquired: + raise ApplicationConnectionBindingLost( + "локальная lifecycle-операция не освободила K1 dispatch до дедлайна" + ) + try: + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationConnectionBindingLost( + "дедлайн K1 dispatch истёк до проверки маршрута" + ) + self._validate_application_connection_path(binding) + except BaseException: + self._k1_command_dispatch_gate.release() + raise + return self._k1_command_dispatch_gate.release + + def _reconcile_application_control_process_lease( + self, + application_control_session: Mapping[str, Any], + ) -> None: + """Release only after the MQTT worker and its socket have retired.""" + + if application_control_session.get("can_open") is True: + self._release_application_control_process_lease() + + def _retire_application_control_for_network_change( self, *, - now_monotonic: float | None = None, - ) -> list[dict[str, Any]]: - """Return only candidates leased by the current in-memory scan generation.""" + allow_terminal_failure: bool = False, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> None: + self._application_control_session.retire_for_network_change( + allow_terminal_failure=allow_terminal_failure, + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + with self._lock: + self._pending_local_control_retirement = False + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) - now = time.monotonic() if now_monotonic is None else now_monotonic - fresh: list[dict[str, Any]] = [] - for item in self._devices: - device_id = str(item.get("device_id") or "") - last_seen = self._ble_device_last_seen_monotonic.get(device_id) - if last_seen is None: - continue - age_seconds = now - last_seen - if 0.0 <= age_seconds <= BLE_DISCOVERY_LEASE_TTL_SECONDS: - fresh.append(dict(item)) - return fresh + def _retry_pending_local_control_retirement(self) -> bool: + """Finish local socket retirement after its worker actually exits.""" + + with self._lock: + pending = self._pending_local_control_retirement + if not pending: + return True + try: + self._retire_application_control_for_network_change() + except (AttributeError, RuntimeError): + return False + return True + + def _retire_prepared_acquisition_on_terminal_control_failure( + self, + snapshot: Mapping[str, Any], + ) -> bool: + """Make a no-START PREPARE terminal before control-loss teardown.""" + + state = str(snapshot.get("state") or "unknown") + failure = snapshot.get("failure") + with self._lock: + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + runtime = self.runtime.snapshot() + if ( + state not in {"failed", "closed"} + or acquisition_lease is not None + or runtime.get("source_mode") != "idle" + ): + return False + if state == "failed" and not ( + isinstance(failure, Mapping) and failure.get("modeling_command_attempted") is False + ): + return False + with self._lock: + if ( + acquisition is not None + and acquisition.state == "prepared" + and self._acquisition is acquisition + and self._acquisition_session_lease is None + and acquisition_start_operation_id is None + and self._acquisition_start_operation_id is None + ): + # PREPARE has no receiver or physical command edge. A terminal + # control failure that explicitly proves no modeling attempt + # can therefore retire both local halves on ordinary polling, + # without asking the operator to refresh or press ABORT first. + acquisition.transition( + "aborted", + message_code="acquisition.prestart_control_failed", + result={ + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "prestart-control-terminal-failure", + }, + ) + return True + return False + + def _retire_terminal_prestart_control_failure(self) -> bool: + """Automatically clear only proven local, pre-START terminal state.""" + + snapshot = self._application_control_session.snapshot() + self._retire_prepared_acquisition_on_terminal_control_failure(snapshot) + state = str(snapshot.get("state") or "unknown") + failure = snapshot.get("failure") + with self._lock: + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + runtime = self.runtime.snapshot() + if ( + state not in {"failed", "closed"} + or acquisition_lease is not None + or (acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES) + or runtime.get("source_mode") != "idle" + ): + return False + if state == "failed" and not ( + isinstance(failure, Mapping) and failure.get("modeling_command_attempted") is False + ): + return False + try: + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + except (AttributeError, RuntimeError): + return False + return True + + def _retire_orphaned_prestart_control_owner(self) -> bool: + """Retire a structurally orphaned local pre-START socket on polling. + + Merely losing one host/TCP sample is not enough. The exact verified + control binding must no longer match the facade/supervisor intent (or + the facade binding must already be gone). The lifecycle gate excludes + a Verify/bootstrap that has just reached ``connection-ready``. + """ + + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + return False + try: + # State reduction must never block the UI poll behind a host/TCP + # sample. Defer this optional cleanup; the held transition gate + # prevents a new monitor contact from being admitted meanwhile. + if not self._connection_monitor_contact_gate.acquire(blocking=False): + return False + self._connection_monitor_contact_gate.release() + with self._lock: + if self._provisioning_active: + return False + selected_transport_ref = self._selected_device_id + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + if ( + acquisition_lease is not None + or acquisition_start_operation_id is not None + or ( + acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES + ) + or self.runtime.snapshot().get("source_mode") != "idle" + ): + return False + physical = self._physical_command_coordinator.snapshot() + if ( + physical.get("status") not in {"empty", "resolved"} + or physical.get("requires_reconciliation") is True + or _physical_command_reports_active(physical) + ): + return False + + control = dict(self._application_control_session.snapshot()) + if control.get("state") not in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + return False + verified = control.get("verified_control") + supervisor = self._connection_supervisor.snapshot() + structurally_current = bool( + isinstance(verified, Mapping) + and selected_transport_ref is not None + and supervisor.intent is not None + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.transport_ref == selected_transport_ref + and verified.get("intent_id") == supervisor.intent.intent_id + and verified.get("transport_ref") == selected_transport_ref + and verified.get("connection_mode") == supervisor.intent.requested_mode + and supervisor.device_network.connection_mode == supervisor.intent.requested_mode + and supervisor.device_network.target is not None + and verified.get("target_ipv4") == supervisor.device_network.target.ipv4 + and verified.get("target_port") == supervisor.device_network.target.port + ) + control_loss_reason = supervisor.control_plane.reason_code + proven_terminal_or_stale_control = bool( + not supervisor.authority.control_allowed + and supervisor.control_plane.state == "lost" + and ( + control_loss_reason + in { + "control-proof-observation-stale", + "control-proof-stale", + "control-proof-invalid", + "mqtt_network_loop_failed", + "application-connection-binding-lost", + "host-path-observation-stale", + "endpoint-observation-stale", + } + # A newly available but different route fingerprint is a + # conclusive epoch transition. An unavailable sample can + # carry the same supervisor loss reason when a bounded + # host inspector times out; keep the socket locally owned + # until the existing three-sample loss debounce confirms + # that negative path. + or ( + control_loss_reason == "host-path-epoch-changed" + and supervisor.host_path.available + ) + or supervisor.host_path_negative_streak >= 3 + or supervisor.endpoint_negative_streak >= 3 + ) + ) + if structurally_current and not proven_terminal_or_stale_control: + # One transient route/TCP sample may revoke the projection but + # is not enough to close a structurally current socket. TTL + # expiry, exact control loss, or the supervisor's debounced + # negative streak is conclusive local orphan evidence. + return False + + generation = control.get("session_generation") + revision = control.get("state_revision") + self._application_control_session.close_prestart( + expected_session_generation=(generation if isinstance(generation, int) else None), + expected_state_revision=(revision if isinstance(revision, int) else None), + ) + self._application_control_session.close() + terminal = self._application_control_session.snapshot() + if terminal.get("state") not in {"idle", "completed", "closed", "failed"}: + with self._lock: + self._pending_local_control_retirement = True + return False + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + logger.info( + "Orphaned K1 pre-START control owner retired during state reduction", + extra={ + "event_code": "k1_orphaned_prestart_control_retired", + "device_write_performed": False, + "automatic_retry": False, + }, + ) + return True + except (AttributeError, RuntimeError): + return False + finally: + self._k1_lifecycle_transition_gate.release() + + @staticmethod + def _host_timezone_name() -> str: + """Resolve a safe host timezone for the read-only DeviceInfo dialogue.""" + + tzinfo = datetime.now().astimezone().tzinfo + key = getattr(tzinfo, "key", None) + return key if isinstance(key, str) and key else "UTC" + + async def _await_local_control_terminal( + self, + *, + timeout_seconds: float = CONTROL_LOCAL_RETIREMENT_TIMEOUT_SECONDS, + ) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + while True: + snapshot = dict(self._application_control_session.snapshot()) + if snapshot.get("state") in {"idle", "completed", "closed", "failed"}: + return snapshot + if time.monotonic() >= deadline: + raise ConnectionVerificationError( + "локальный control worker не завершился вовремя", + reason_code="control-local-retirement-timeout", + ) + await asyncio.sleep(0.05) + + async def _retire_prestart_control_for_mode_transition(self) -> None: + """Close a pre-START socket locally before one explicit mode switch. + + This boundary never publishes START/STOP and never touches BLE or the + host Wi-Fi adapter. It only cancels the worker waiting at a pre-START + checkpoint, waits for its socket to retire, then resets local state. + """ + + snapshot = dict(self._application_control_session.snapshot()) + if snapshot.get("state") not in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + return + generation = snapshot.get("session_generation") + revision = snapshot.get("state_revision") + self._application_control_session.close_prestart( + expected_session_generation=(generation if isinstance(generation, int) else None), + expected_state_revision=(revision if isinstance(revision, int) else None), + ) + try: + await self._await_local_control_terminal() + except ConnectionVerificationError: + # ``close`` is still local-only and joins the process worker with + # its own bounded deadline. It cannot publish a device STOP. + await asyncio.to_thread(self._application_control_session.close) + await self._await_local_control_terminal(timeout_seconds=1.0) + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + + def _control_bootstrap_continuation_current_reason( + self, + continuation: _ControlBootstrapContinuation, + ) -> str | None: + """Return a stable reason when a background proof lost its exact parent.""" + + with self._lock: + if self._control_bootstrap_continuation_closing: + return "control-bootstrap-service-closing" + if self._snapshot_runtime_id != continuation.snapshot_runtime_id: + return "control-bootstrap-runtime-superseded" + if self._control_bootstrap_continuation != continuation: + return "control-bootstrap-owner-superseded" + if ( + self._selected_device_id != continuation.transport_ref + or self._connection_mode != continuation.connection_mode + or self._k1_ip != continuation.target.ipv4 + or self._device_session_id != continuation.device_session_id + or self._desired_connection_mode != continuation.connection_mode + or self._desired_connection_mode_revision != continuation.desired_mode_revision + ): + return "control-bootstrap-local-binding-superseded" + + network_operation = self._operations.get(continuation.parent_operation_id) + network_result = network_operation.result or {} + if not ( + network_operation.action == ACTION_NETWORK_PROVISION + and network_operation.status == "succeeded" + and network_result.get("phase") == "network_applied" + ): + return "control-bootstrap-parent-operation-superseded" + + ledger = self._network_mutation_ledger.snapshot() + ledger_record = ledger.record + if not ( + ledger.status == "resolved" + and ledger_record is not None + and ledger_record.operation_id == continuation.parent_operation_id + and ledger_record.transport_ref == continuation.transport_ref + and ledger_record.intended_mode == continuation.connection_mode + and ledger_record.stage == "resolved" + and ledger_record.resolution == "target-observed" + and ledger_record.revision == continuation.network_ledger_revision + ): + return "control-bootstrap-network-ledger-superseded" + + topology_store = self._semantic_topology_store + if topology_store is None: + return "control-bootstrap-topology-unavailable" + topology = topology_store.snapshot() + topology_record = topology.record + if not ( + topology.status == "available" + and topology_record is not None + and topology_record.revision == continuation.semantic_topology_revision + and topology_record.transport_ref == continuation.transport_ref + and topology_record.connection_mode == continuation.connection_mode + and topology_record.ipv4 == continuation.target.ipv4 + and topology_record.source == "ble-post-write-status" + ): + return "control-bootstrap-topology-superseded" + + supervisor = self._connection_supervisor.snapshot() + if not ( + not supervisor.closed + and supervisor.intent is not None + and supervisor.intent.intent_id == continuation.parent_intent_id + and supervisor.intent.requested_mode == continuation.connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == continuation.parent_intent_id + and supervisor.device_network.transport_ref == continuation.transport_ref + and supervisor.device_network.connection_mode == continuation.connection_mode + and supervisor.device_network.target == continuation.target + ): + return "control-bootstrap-parent-intent-superseded" + return None + + def _control_bootstrap_continuation_ready_reason( + self, + continuation: _ControlBootstrapContinuation, + ) -> str | None: + """Fence the sole success commit to the exact live control proof.""" + + reason = self._control_bootstrap_continuation_current_reason(continuation) + if reason is not None: + return reason + supervisor = self._connection_supervisor.snapshot() + endpoint = supervisor.endpoint + lease = supervisor.lease + if not ( + supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.intent_id == continuation.parent_intent_id + and supervisor.host_path.available + and supervisor.host_path.epoch >= 1 + and endpoint.target == continuation.target + and endpoint.intent_id == continuation.parent_intent_id + and endpoint.tcp_state == "reachable" + and endpoint.host_path_epoch == supervisor.host_path.epoch + and lease.state == "reachable" + and lease.intent_id == continuation.parent_intent_id + and lease.target == continuation.target + and lease.connection_mode == continuation.connection_mode + and lease.host_path_epoch == supervisor.host_path.epoch + and supervisor.control_plane.state == "healthy" + and supervisor.control_plane.host_path_epoch == supervisor.host_path.epoch + ): + return "control-bootstrap-final-control-proof-superseded" + + control = self._application_control_session.snapshot() + verified = control.get("verified_control") + if not ( + control.get("state") in {"connection-ready", "workspace-ready", "project-ready"} + and isinstance(verified, Mapping) + and verified.get("intent_id") == continuation.parent_intent_id + and verified.get("transport_ref") == continuation.transport_ref + and verified.get("connection_mode") == continuation.connection_mode + and verified.get("target_ipv4") == continuation.target.ipv4 + and verified.get("target_port") == continuation.target.port + and verified.get("host_path_epoch") == supervisor.host_path.epoch + and verified.get("control_session_id") == supervisor.control_plane.session_id + and verified.get("control_proof_fresh") is True + ): + return "control-bootstrap-final-session-proof-superseded" + return None + + def _control_bootstrap_continuation_physical_reason( + self, + continuation: _ControlBootstrapContinuation, + reconciliation: Mapping[str, object], + ) -> str | None: + """Fence the hidden settlement to its exact reopened physical row.""" + + expects_settlement = bool( + continuation.physical_record_operation_id is not None + and continuation.physical_record_created_at_utc is not None + and continuation.physical_reopening_id is not None + ) + if not expects_settlement: + return ( + "control-bootstrap-unexpected-physical-reconciliation" + if reconciliation.get("performed") is True + else None + ) + resolution = reconciliation.get("resolution") + if not ( + reconciliation.get("performed") is True + and resolution + in {"physical-standby-observed", "physical-active-observed"} + and reconciliation.get("device_write_performed") is False + and reconciliation.get("automatic_retry") is False + ): + return "control-bootstrap-physical-reconciliation-missing" + + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + reconciliations = ( + record.get("reconciliations") if isinstance(record, Mapping) else None + ) + latest = ( + reconciliations[-1] + if isinstance(reconciliations, list) and reconciliations + else None + ) + reopens = ( + record.get("operator_reconciliation_reopens") + if isinstance(record, Mapping) + else None + ) + exact_reopen = bool( + isinstance(reopens, list) + and len( + [ + item + for item in reopens + if isinstance(item, Mapping) + and item.get("reopening_id") + == continuation.physical_reopening_id + and item.get("reason") + == "reset-network-intent-read-only-settlement" + ] + ) + == 1 + ) + if not ( + physical.get("status") == "resolved" + and physical.get("requires_reconciliation") is False + and isinstance(record, Mapping) + and record.get("operation_id") + == continuation.physical_record_operation_id + and record.get("created_at_utc") + == continuation.physical_record_created_at_utc + and exact_reopen + and isinstance(latest, Mapping) + and latest.get("reconciliation_id") + == f"{continuation.operation_id}.physical" + and latest.get("resolution") == resolution + ): + return "control-bootstrap-physical-reconciliation-superseded" + if resolution == "physical-active-observed": + if not ( + physical.get("physical_active") is True + and physical.get("resolved_active_recovery_required") is True + ): + return "control-bootstrap-physical-active-proof-missing" + elif physical.get("resolved_active_recovery_required") is True: + return "control-bootstrap-physical-standby-proof-missing" + return None + + def _control_bootstrap_continuation_scanning_reason( + self, + continuation: _ControlBootstrapContinuation, + reconciliation: Mapping[str, object], + ) -> str | None: + """Fence SCANNING success to STOP-only control and physical proof.""" + + reason = self._control_bootstrap_continuation_current_reason(continuation) + if reason is not None: + return reason + physical_reason = self._control_bootstrap_continuation_physical_reason( + continuation, + reconciliation, + ) + if physical_reason is not None: + return physical_reason + if reconciliation.get("resolution") != "physical-active-observed": + return "control-bootstrap-physical-active-proof-missing" + + supervisor = self._connection_supervisor.snapshot() + endpoint = supervisor.endpoint + if not ( + supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.intent_id == continuation.parent_intent_id + and supervisor.host_path.available + and endpoint.target == continuation.target + and endpoint.intent_id == continuation.parent_intent_id + and endpoint.tcp_state == "reachable" + and endpoint.host_path_epoch == supervisor.host_path.epoch + and supervisor.control_plane.state == "healthy" + and supervisor.control_plane.host_path_epoch == supervisor.host_path.epoch + ): + return "control-bootstrap-final-control-proof-superseded" + + control = self._application_control_session.snapshot() + verified = control.get("verified_control") + if not ( + control.get("state") == "scanning" + and control.get("can_start") is False + and control.get("can_stop") is True + and isinstance(verified, Mapping) + and verified.get("intent_id") == continuation.parent_intent_id + and verified.get("transport_ref") == continuation.transport_ref + and verified.get("connection_mode") == continuation.connection_mode + and verified.get("target_ipv4") == continuation.target.ipv4 + and verified.get("target_port") == continuation.target.port + and verified.get("host_path_epoch") == supervisor.host_path.epoch + and verified.get("control_session_id") + == supervisor.control_plane.session_id + and verified.get("control_proof_fresh") is True + ): + return "control-bootstrap-final-stop-only-proof-superseded" + return None + + def _schedule_control_bootstrap_continuation( + self, + *, + parent_operation_id: str, + connection_mode: ConnectionMode, + transport_ref: str, + target: EndpointTarget, + device_session_id: str, + ) -> None: + """Own exactly one post-Apply read-only bootstrap on the service loop.""" + + loop = asyncio.get_running_loop() + with self._lock: + existing_task = self._control_bootstrap_continuation_task + if existing_task is not None and not existing_task.done(): + raise RuntimeError("a control bootstrap continuation is already owned") + if self._control_bootstrap_continuation_closing: + raise RuntimeError("service is closing") + snapshot_runtime_id = self._snapshot_runtime_id + desired_mode_revision = self._desired_connection_mode_revision + device_id = self._device_id + reset_authority = ( + self._connection_scenario_reset_retired_transport_authority + ) + + supervisor = self._connection_supervisor.snapshot() + ledger = self._network_mutation_ledger.snapshot() + topology_store = self._semantic_topology_store + topology = topology_store.snapshot() if topology_store is not None else None + if not ( + supervisor.intent is not None + and supervisor.intent.intent_id == parent_operation_id + and supervisor.intent.requested_mode == connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == parent_operation_id + and supervisor.device_network.transport_ref == transport_ref + and supervisor.device_network.connection_mode == connection_mode + and supervisor.device_network.target == target + and ledger.status == "resolved" + and ledger.record is not None + and ledger.record.operation_id == parent_operation_id + and ledger.record.transport_ref == transport_ref + and ledger.record.intended_mode == connection_mode + and ledger.record.resolution == "target-observed" + and topology is not None + and topology.status == "available" + and topology.record is not None + and topology.record.transport_ref == transport_ref + and topology.record.connection_mode == connection_mode + and topology.record.ipv4 == target.ipv4 + and topology.record.source == "ble-post-write-status" + ): + raise RuntimeError("durable network proof changed before control continuation") + assert ledger.record is not None + assert topology is not None and topology.record is not None + reset_owned_physical_settlement = bool( + reset_authority is not None + and reset_authority.consumed_operation_id == parent_operation_id + and reset_authority.physical_reopening_id is not None + and physical_transport_ref_comparison_key( + reset_authority.retired_transport_ref + ) + == physical_transport_ref_comparison_key(transport_ref) + ) + + bootstrap_operation_id = f"op-{uuid4()}" + operation, created = self._operations.begin( + ACTION_CONNECTION_CONTROL_BOOTSTRAP, + operation_id=bootstrap_operation_id, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=CONTROL_BOOTSTRAP_CONTINUATION_TIMEOUT_SECONDS, + cancellable=True, + context={ + "ownership": "service-owned-apply-continuation", + "snapshot_runtime_id": snapshot_runtime_id, + "parent_operation_id": parent_operation_id, + "parent_intent_id": parent_operation_id, + "connection_mode": connection_mode, + "transport_ref": transport_ref, + "target_ipv4": target.ipv4, + "target_port": target.port, + "device_session_id": device_session_id, + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + ) + if not created: + raise RuntimeError("control bootstrap operation id collision") + continuation = _ControlBootstrapContinuation( + operation_id=operation.operation_id, + parent_operation_id=parent_operation_id, + snapshot_runtime_id=snapshot_runtime_id, + parent_intent_id=parent_operation_id, + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + device_session_id=device_session_id, + desired_mode_revision=desired_mode_revision, + semantic_topology_revision=topology.record.revision, + network_ledger_revision=ledger.record.revision, + physical_record_operation_id=( + reset_authority.physical_record_operation_id + if reset_owned_physical_settlement and reset_authority is not None + else None + ), + physical_record_created_at_utc=( + reset_authority.physical_record_created_at_utc + if reset_owned_physical_settlement and reset_authority is not None + else None + ), + physical_reopening_id=( + reset_authority.physical_reopening_id + if reset_owned_physical_settlement and reset_authority is not None + else None + ), + ) + task = loop.create_task( + self._run_control_bootstrap_continuation(continuation), + name=f"k1-control-bootstrap-{parent_operation_id}", + ) + with self._lock: + self._control_bootstrap_continuation = continuation + self._control_bootstrap_continuation_task = task + self._control_bootstrap_continuation_loop = loop + + def consume_outcome(completed: asyncio.Task[None]) -> None: + try: + completed.result() + except asyncio.CancelledError as exc: + self._operations.transition_if_pending( + continuation.operation_id, + "cancelled", + stage_code="control-bootstrap-cancelled", + message_code="connection.control_bootstrap.cancelled", + error=_operation_error( + exc, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + except Exception: + # The subordinate operation already contains the stable, + # secret-free failure. Never log transport exception text. + logger.warning( + "K1 read-only control bootstrap continuation failed", + extra={ + "event_code": "k1_control_bootstrap_continuation_failed", + "operation_id": continuation.operation_id, + "parent_operation_id": continuation.parent_operation_id, + "automatic_retry": False, + }, + ) + finally: + with self._lock: + if self._control_bootstrap_continuation_task is completed: + self._control_bootstrap_continuation = None + self._control_bootstrap_continuation_task = None + self._control_bootstrap_continuation_loop = None + + task.add_done_callback(consume_outcome) + + async def _run_control_bootstrap_continuation( + self, + continuation: _ControlBootstrapContinuation, + ) -> None: + """Observe route/TCP/DeviceInfo once, fenced to the exact Apply parent.""" + + transition_gate_acquired = False + network_process_lease_acquired = False + control_owner_before = self._application_control_session + control_before = dict(control_owner_before.snapshot()) + bootstrap_control_established = False + + def control_generation_changed() -> bool: + current_owner = self._application_control_session + current = dict(current_owner.snapshot()) + return bool( + current_owner is not control_owner_before + or current.get("session_generation") != control_before.get("session_generation") + ) + + try: + async with asyncio.timeout(CONTROL_BOOTSTRAP_CONTINUATION_TIMEOUT_SECONDS): + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise ConnectionVerificationError( + "read-only control bootstrap lost lifecycle ownership", + reason_code="control-bootstrap-lifecycle-busy", + ) + transition_gate_acquired = True + reason = self._control_bootstrap_continuation_current_reason(continuation) + if reason is not None: + raise ConnectionVerificationError( + "read-only control bootstrap parent was superseded", + reason_code=reason, + ) + # Connect closes its BLE borrow before returning, but native + # CoreBluetooth cleanup may retain the same service-owned + # network flock briefly. Join only that bounded handoff; do + # not contend with or steal another lifecycle owner. + lease_handoff_deadline = asyncio.get_running_loop().time() + 5.0 + while True: + with self._lock: + previous_network_owner_active = ( + "network" in self._application_control_process_lease_holders + ) + if not previous_network_owner_active: + break + if asyncio.get_running_loop().time() >= lease_handoff_deadline: + raise ConnectionVerificationError( + "network lifecycle cleanup did not release its owner", + reason_code="control-bootstrap-network-owner-busy", + ) + await asyncio.sleep(0.01) + reason = self._control_bootstrap_continuation_current_reason(continuation) + if reason is not None: + raise ConnectionVerificationError( + "read-only control bootstrap parent was superseded", + reason_code=reason, + ) + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + self._operations.transition_if_pending( + continuation.operation_id, + "running", + stage_code="host-route-and-control-endpoint", + message_code="connection.control_bootstrap.host_probe", + result={ + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + ) + + endpoint_observation = await _await_control_endpoint_reachable( + continuation.target.ipv4, + endpoint_probe=self._probe_control_endpoint, + ) + reason = self._control_bootstrap_continuation_current_reason(continuation) + if reason is not None: + raise ConnectionVerificationError( + "read-only endpoint observation was superseded", + reason_code=reason, + ) + if not isinstance(endpoint_observation, _CorrelatedEndpointObservation): + raise ConnectionVerificationError( + "read-only endpoint observation was not correlated", + reason_code="control-bootstrap-endpoint-observation-invalid", + ) + path = endpoint_observation.path + host_route_class = ( + "device-ap" + if continuation.connection_mode == "quick-connect" and path.available + else "direct-or-routed" + if path.available and path.route_class == "direct" + else path.route_class + ) + host_route_ready = bool( + path.available and host_route_class in {"direct-or-routed", "device-ap"} + ) + endpoint_reachable = bool(endpoint_observation.reachable) + self._update_applied_topology_reachability( + connection_mode=continuation.connection_mode, + ipv4=continuation.target.ipv4, + host_route_class=host_route_class, + host_route_ready=host_route_ready, + endpoint_reachable=endpoint_reachable, + write_performed=True, + ) + if not endpoint_reachable: + raise ConnectionVerificationError( + "K1 network was applied but its control endpoint is not ready", + reason_code=( + endpoint_observation.reason_code + or "control-bootstrap-endpoint-unreachable" + ), + ) + + await self._bootstrap_prestart_control_ready_owned( + parent_operation_id=continuation.parent_operation_id, + connection_mode=continuation.connection_mode, + inspection_only=True, + ) + bootstrap_control_established = True + # DeviceInfo must still belong to the exact applied parent + # before the subordinate operation consumes any physical + # reconciliation proof. + reason = self._control_bootstrap_continuation_ready_reason(continuation) + if reason is not None: + raise ConnectionVerificationError( + "read-only DeviceInfo proof was superseded", + reason_code=reason, + ) + physical_reconciliation = ( + await self._reconcile_physical_command_after_verify_owned( + verify_operation_id=continuation.operation_id, + allow_receiver_rehydrate=False, + ) + ) + if ( + physical_reconciliation.get("resolution") + != "physical-active-observed" + ): + inspected = self._application_control_session.snapshot() + if ( + inspected.get("state") == "connection-ready" + and inspected.get("inspection_only") is True + and inspected.get("inspection_promotion_allowed") is not True + ): + generation = inspected.get("session_generation") + revision = inspected.get("state_revision") + if not isinstance(generation, int) or not isinstance( + revision, + int, + ): + raise ConnectionVerificationError( + "read-only continuation lost its inspection checkpoint", + reason_code=( + "control-bootstrap-inspection-checkpoint-invalid" + ), + ) + self._application_control_session.release_inspection_for_operator_dialogue( + expected_session_generation=generation, + expected_state_revision=revision, + ) + if ( + physical_reconciliation.get("resolution") + == "physical-active-observed" + ): + reason = self._control_bootstrap_continuation_scanning_reason( + continuation, + physical_reconciliation, + ) + else: + reason = self._control_bootstrap_continuation_ready_reason( + continuation + ) + if reason is None: + reason = ( + self._control_bootstrap_continuation_physical_reason( + continuation, + physical_reconciliation, + ) + ) + if reason is not None: + raise ConnectionVerificationError( + "read-only physical settlement was superseded", + reason_code=reason, + ) + self._operations.transition_if_pending( + continuation.operation_id, + "succeeded", + stage_code=( + "physical-reconciliation-confirmed" + if physical_reconciliation.get("performed") is True + else "device-info-confirmed" + ), + message_code="connection.control_bootstrap.completed", + result={ + "connection_mode": continuation.connection_mode, + "control_verified": True, + "physical_reconciliation": physical_reconciliation, + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + ) + except BaseException as exc: + logger.warning( + "K1 read-only control bootstrap failed before project admission", + extra={ + "event_code": "k1_control_bootstrap_failure_detail", + "operation_id": continuation.operation_id, + "parent_operation_id": continuation.parent_operation_id, + "error_type": type(exc).__name__, + "reason_code": str( + getattr(exc, "reason_code", "control-bootstrap-failed") + ), + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + ) + if bootstrap_control_established or control_generation_changed(): + try: + await self._retire_control_generation_opened_by_verify() + except BaseException as cleanup_exc: + exc.add_note( + "control-bootstrap-owned local cleanup failed: " + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + cancelled = isinstance(exc, asyncio.CancelledError) + failure = ( + exc + if isinstance(exc, Exception) + else ConnectionVerificationError( + "read-only control bootstrap was cancelled", + reason_code="control-bootstrap-continuation-cancelled", + ) + ) + self._operations.transition_if_pending( + continuation.operation_id, + "cancelled" if cancelled else "failed", + stage_code=( + "control-bootstrap-cancelled" if cancelled else "control-bootstrap-failed" + ), + message_code=( + "connection.control_bootstrap.cancelled" + if cancelled + else "connection.control_bootstrap.failed" + ), + error=_operation_error( + failure, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + raise + finally: + if network_process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + + def _request_control_bootstrap_continuation_close(self) -> None: + """Cancel the one retained continuation on its owner loop.""" + + with self._lock: + self._control_bootstrap_continuation_closing = True + task = self._control_bootstrap_continuation_task + owner_loop = self._control_bootstrap_continuation_loop + continuation = self._control_bootstrap_continuation + if task is None or task.done(): + self._operations.transition_if_pending( + continuation.operation_id if continuation is not None else None, + "cancelled", + stage_code="control-bootstrap-cancelled", + message_code="connection.control_bootstrap.cancelled", + error={ + "category": "connection", + "code": "control-bootstrap-service-closed", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + return + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if owner_loop is running_loop: + task.cancel() + elif owner_loop is not None and owner_loop.is_running(): + try: + owner_loop.call_soon_threadsafe(task.cancel) + except RuntimeError: + self._operations.transition_if_pending( + continuation.operation_id if continuation is not None else None, + "cancelled", + stage_code="control-bootstrap-cancelled", + message_code="connection.control_bootstrap.cancelled", + error={ + "category": "connection", + "code": "control-bootstrap-owner-loop-closed", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + else: + self._operations.transition_if_pending( + continuation.operation_id if continuation is not None else None, + "cancelled", + stage_code="control-bootstrap-cancelled", + message_code="connection.control_bootstrap.cancelled", + error={ + "category": "connection", + "code": "control-bootstrap-owner-loop-unavailable", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + + def _supersede_control_bootstrap_continuation(self) -> None: + """Fence and cancel one old read-only continuation without shutdown. + + Scenario reset is not service shutdown: a later explicit connection + may schedule a new bootstrap. Clear the retained owner first so every + late boundary check observes supersession, then cancel the old task on + its owning loop. This method performs no device or host-network I/O. + """ + + with self._lock: + continuation = self._control_bootstrap_continuation + task = self._control_bootstrap_continuation_task + owner_loop = self._control_bootstrap_continuation_loop + self._control_bootstrap_continuation = None + self._control_bootstrap_continuation_task = None + self._control_bootstrap_continuation_loop = None + self._operations.transition_if_pending( + continuation.operation_id if continuation is not None else None, + "cancelled", + stage_code="control-bootstrap-superseded", + message_code="connection.control_bootstrap.superseded", + error={ + "category": "connection", + "code": "connection-scenario-reset-by-operator", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + if task is None or task.done(): + return + try: + running_loop = asyncio.get_running_loop() + except RuntimeError: + running_loop = None + if owner_loop is running_loop: + task.cancel() + elif owner_loop is not None and owner_loop.is_running(): + # The loop can close between the liveness check and scheduling. + # Clearing the retained owner above is already the authoritative + # generation fence in that race. + with suppress(RuntimeError): + owner_loop.call_soon_threadsafe(task.cancel) + + async def _bootstrap_prestart_control_ready_owned( + self, + *, + parent_operation_id: str | None, + connection_mode: ConnectionMode, + inspection_only: bool = False, + force_new_binding: bool = False, + ) -> None: + """Reach MQTT DeviceInfo authority without crossing a physical edge. + + The caller owns the lifecycle transition gate but must have released + both the BLE borrow and the command-dispatch gate. The resulting + worker remains at ``connection-ready`` until an explicit later + workspace/project/START intent. + """ + + with self._lock: + device_id = self._device_id + device_session_id = self._device_session_id + with self._lock: + continuation = self._control_bootstrap_continuation + owned_operation = bool( + continuation is not None + and continuation.parent_operation_id == parent_operation_id + and continuation.connection_mode == connection_mode + ) + if owned_operation: + assert continuation is not None + operation = self._operations.get(continuation.operation_id) + if operation.status not in {"accepted", "running"}: + return + else: + operation, created = self._operations.begin( + ACTION_CONNECTION_CONTROL_BOOTSTRAP, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=CONTROL_DEVICE_INFO_BOOTSTRAP_TIMEOUT_SECONDS, + context={ + "connection_mode": connection_mode, + "parent_operation_id": parent_operation_id, + "automatic_retry": False, + }, + ) + if not created: + return + self._operations.transition( + operation.operation_id, + "running", + stage_code="mqtt-device-info", + message_code="connection.control_bootstrap.running", + ) + process_lease_acquired = False + opened_here = False + try: + existing = dict(self._application_control_session.snapshot()) + if force_new_binding and existing.get("state") != "idle": + # Active-stream recovery deliberately crosses a host-path + # epoch. The old socket is therefore evidence for the frozen + # START lineage, but it can never authorize the new route. + # Retire only that local owner before constructing the fresh + # inspection binding below; do not validate/reuse it and do + # not publish START, STOP, BLE, or network mutations. + if existing.get("state") not in {"completed", "closed", "failed"}: + await asyncio.to_thread(self._application_control_session.close) + existing = await self._await_local_control_terminal() + if existing.get("state") != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + existing = dict(self._application_control_session.snapshot()) + if existing.get("state") != "idle": + raise ConnectionVerificationError( + "прежний локальный control owner не освобождён", + reason_code="control-local-retirement-incomplete", + ) + if existing.get("state") in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + # Reuse the existing DeviceInfo owner only after its retained + # MQTT proof and exact host path have refreshed at this + # command boundary. Reconcile-after-refresh avoids retiring + # an unchanged session solely because monitor silence crossed + # the fallback TTL immediately before this bootstrap call. + self._application_control_session.validate_connection_binding() + existing = dict(self._application_control_session.snapshot()) + self._reconcile_connection_supervisor(existing, self.runtime.snapshot()) + supervisor = self._connection_supervisor.snapshot() + if ( + supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.requested_mode == connection_mode + ): + if not owned_operation: + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="device-info-confirmed", + message_code="connection.control_bootstrap.completed", + result={ + "connection_mode": connection_mode, + "control_verified": True, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + return + await self._retire_prestart_control_for_mode_transition() + elif existing.get("state") != "idle": + if existing.get("state") not in {"completed", "closed", "failed"}: + await asyncio.to_thread(self._application_control_session.close) + existing = await self._await_local_control_terminal() + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + + supervisor = self._connection_supervisor.snapshot() + endpoint = supervisor.device_network.target + if ( + supervisor.intent is None + or supervisor.intent.requested_mode != connection_mode + or supervisor.device_network.state != "applied" + or supervisor.device_network.intent_id != supervisor.intent.intent_id + or endpoint is None + or supervisor.device_network.transport_ref is None + or supervisor.endpoint.target != endpoint + or supervisor.endpoint.tcp_state != "reachable" + or supervisor.endpoint.host_path_epoch != supervisor.host_path.epoch + or not supervisor.host_path.available + or supervisor.host_path.epoch < 1 + ): + raise ConnectionVerificationError( + "K1 настроен, но управляющий endpoint ещё не подтверждён", + reason_code="control-bootstrap-endpoint-unverified", + ) + + self._application_control.disarm() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + self._acquire_application_control_process_lease() + process_lease_acquired = True + binding = ApplicationConnectionBinding( + intent_id=supervisor.intent.intent_id, + transport_ref=str(supervisor.device_network.transport_ref), + host_path_epoch=supervisor.host_path.epoch, + target_ipv4=endpoint.ipv4, + target_port=endpoint.port, + connection_mode=supervisor.intent.requested_mode, + ) + # ``open`` performs only MQTT bootstrap + correlated DeviceInfo. + # Physical acceptance remains mandatory at request_start/stop. + if inspection_only: + # A failed asynchronous STOP can leave only its process-local + # observer owner attached while the durable ledger is still + # exactly PREPARED. Detach that stale owner before the fresh + # Verify observer is installed, otherwise an early READY / + # SCANNING status can be misrouted into the old STOP record and + # poison the read-only DeviceInfo bootstrap. The coordinator + # detaches only its exact process-local unresolved owner; the + # durable command row remains unchanged for fresh DeviceInfo- + # bound classification. + self._physical_command_coordinator.prepare_read_only_bootstrap() + opened_here = True + self._application_control_session.open( + host=endpoint.ipv4, + timezone_name=self._host_timezone_name(), + connection_binding=binding, + inspection_only=inspection_only, + ) + deadline = time.monotonic() + CONTROL_DEVICE_INFO_BOOTSTRAP_TIMEOUT_SECONDS + while True: + control = dict(self._application_control_session.snapshot()) + state = str(control.get("state") or "unknown") + if state == "connection-ready": + self._reconcile_connection_supervisor(control, self.runtime.snapshot()) + ready = self._connection_supervisor.snapshot() + if ( + ready.authority.control_allowed + and ready.intent is not None + and ready.intent.requested_mode == connection_mode + ): + break + raise ConnectionVerificationError( + "K1 ответил DeviceInfo, но точная identity текущего маршрута не принята", + reason_code="control-bootstrap-device-identity-unverified", + ) + if state in {"failed", "closed", "completed", "idle"}: + failure = control.get("failure") + reported = failure.get("reason_code") if isinstance(failure, Mapping) else None + raise ConnectionVerificationError( + "K1 не завершил read-only проверку управляющего канала", + reason_code=( + str(reported) + if isinstance(reported, str) and reported + else "control-bootstrap-terminal-failure" + ), + ) + if time.monotonic() >= deadline: + raise ConnectionVerificationError( + "K1 не подтвердил DeviceInfo за отведённое время", + reason_code="control-bootstrap-timeout", + ) + await asyncio.sleep(0.05) + + if not owned_operation: + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="device-info-confirmed", + message_code="connection.control_bootstrap.completed", + result={ + "connection_mode": connection_mode, + "control_verified": True, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + except BaseException as exc: + control = dict(self._application_control_session.snapshot()) + if opened_here: + try: + control_state = str(control.get("state") or "unknown") + if control_state in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + generation = control.get("session_generation") + revision = control.get("state_revision") + self._application_control_session.close_prestart( + expected_session_generation=( + generation if isinstance(generation, int) else None + ), + expected_state_revision=( + revision if isinstance(revision, int) else None + ), + ) + elif control_state not in {"idle", "completed", "closed", "failed"}: + await asyncio.to_thread(self._application_control_session.close) + terminal = await self._await_local_control_terminal(timeout_seconds=3.0) + if terminal.get("state") != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + except BaseException as cleanup_exc: + exc.add_note( + "local control bootstrap cleanup failed: " + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + elif control.get("state") not in { + "idle", + "completed", + "closed", + "failed", + "connection-ready", + "workspace-ready", + "project-ready", + }: + try: + await asyncio.to_thread(self._application_control_session.close) + except BaseException as cleanup_exc: + exc.add_note( + "local control bootstrap cleanup failed: " + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + terminal = dict(self._application_control_session.snapshot()) + self._reconcile_application_control_process_lease(terminal) + if not isinstance(exc, Exception): + cancelled = isinstance(exc, asyncio.CancelledError) + self._operations.transition_if_pending( + operation.operation_id, + "cancelled" if cancelled else "failed", + stage_code=( + "device-info-cancelled" if cancelled else "device-info-interrupted" + ), + message_code=( + "connection.control_bootstrap.cancelled" + if cancelled + else "connection.control_bootstrap.interrupted" + ), + error=_operation_error( + exc, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + # Task/process cancellation must remain a control-flow signal. + # The caller has already sealed any applied network mutation + # and will expose it without replaying the physical write. + raise + failure = ( + exc + if isinstance(exc, ConnectionVerificationError) + else ConnectionVerificationError( + "K1 не подтвердил готовность управляющего канала", + reason_code=str(getattr(exc, "reason_code", "control-bootstrap-failed")), + ) + ) + self._operations.transition_if_pending( + operation.operation_id, + "failed", + stage_code="device-info-failed", + message_code="connection.control_bootstrap.failed", + error=_operation_error( + failure, + category="connection", + side_effect_status="none", + safe_to_retry=True, + ), + ) + if failure is exc: + raise + raise failure from exc + finally: + if process_lease_acquired: + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + + def _retire_ephemeral_device_binding_for_new_intent(self) -> None: + """Atomically retire the old in-memory device session before GATT. + + The durable semantic topology and audit journals remain intact. A + failed new network attempt must never resurrect the previous selected + session or its pinned CoreBluetooth object. + """ + + with self._lock: + selected_device_id = self._selected_device_id + device_session_id = self._device_session_id + self._selected_device_id = None + self._k1_ip = None + self._connection_mode = None + self._device_session_id = None + self._device_session_opened_at = None + self._device_session_discovery_generation = None + self._compatibility_attestation = None + self._device_calibration = unavailable_device_calibration_snapshot( + XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ) + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + if selected_device_id is not None and device_session_id is not None: + invalidate_connected_device_session( + selected_device_id, + device_session_id=device_session_id, + ) + + def _revoke_retired_physical_target_locally( + self, + transport_ref: str, + *, + retirement_id: str, + ) -> None: + """Revoke only matching process-local authority after durable retirement.""" + + supervisor = self._connection_supervisor.snapshot() + supervisor_transport_ref = supervisor.device_network.transport_ref + retired_transport_key = physical_transport_ref_comparison_key(transport_ref) + local_retirement_intent_id = f"physical-retirement-{retirement_id}" + if ( + supervisor_transport_ref is not None + and physical_transport_ref_comparison_key(supervisor_transport_ref) + == retired_transport_key + and ( + supervisor.intent is None + or supervisor.intent.intent_id != local_retirement_intent_id + ) + ): + with self._lock: + desired_mode = self._desired_connection_mode + self._connection_supervisor.set_intent( + intent_id=local_retirement_intent_id, + requested_mode=desired_mode, + ) + with self._lock: + selected_device_id = self._selected_device_id + selected_matches = bool( + selected_device_id is not None + and physical_transport_ref_comparison_key(selected_device_id) + == retired_transport_key + ) + if selected_matches: + self._retire_ephemeral_device_binding_for_new_intent() + + def _require_network_provisioning_idempotency_journal( + self, + ) -> NetworkProvisioningIdempotencyJournal: + journal = self._network_provisioning_idempotency_journal + if journal is None: + raise NetworkProvisioningIdempotencyCorrupt( + "durable network idempotency authority is unavailable" + ) + snapshot = journal.snapshot() + if snapshot.status == "corrupt": + raise NetworkProvisioningIdempotencyCorrupt( + "durable network idempotency authority is corrupt" + ) + return journal + + def _quarantine_corrupt_network_audit_under_exclusive_lease( + self, + ) -> NetworkProvisioningIdempotencyJournal: + """Replace corrupt network audit files without interpreting them. + + The caller already owns the process-wide network lifecycle lease. Each + corrupt artifact is privately renamed with a timestamp and random + non-semantic token; a fresh empty authority is then created. Physical command + evidence is deliberately outside this recovery boundary. + """ + + journal = self._network_provisioning_idempotency_journal + candidate_journal = journal or NetworkProvisioningIdempotencyJournal(self.repository_root) + candidates = ( + (self._network_mutation_ledger, self._network_mutation_ledger.path), + (candidate_journal, candidate_journal.path), + ) + quarantined: list[str] = [] + for store, path in candidates: + # The high-level lifecycle lease excludes Mission Core peers; the + # store lock additionally keeps this rename stable against its own + # snapshot/persist paths in the current process. + with store._lock, store._process_lock_locked(): # noqa: SLF001 + store._reload_locked() # noqa: SLF001 + if not store._corrupt or not os.path.lexists(path): # noqa: SLF001 + continue + try: + metadata = path.lstat() + if not stat.S_ISLNK(metadata.st_mode): + descriptor = os.open( + path, + os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0), + ) + try: + opened = os.fstat(descriptor) + if (opened.st_dev, opened.st_ino) != ( + metadata.st_dev, + metadata.st_ino, + ): + raise OSError("network audit changed while opening") + # Bounded validation read: never hash or retain an + # attacker-sized payload and never follow symlinks. + os.read(descriptor, 64 * 1024) + finally: + os.close(descriptor) + current = path.lstat() + if (current.st_dev, current.st_ino) != ( + metadata.st_dev, + metadata.st_ino, + ): + raise OSError("network audit changed before quarantine") + quarantine = path.with_name( + f"{path.name}.corrupt-{time.time_ns()}-{secrets.token_hex(8)}" + ) + if os.path.lexists(quarantine): + raise OSError("network audit quarantine destination collision") + os.rename(path, quarantine) + moved_metadata = quarantine.lstat() + if (moved_metadata.st_dev, moved_metadata.st_ino) != ( + metadata.st_dev, + metadata.st_ino, + ): + raise OSError("network audit changed during quarantine") + if not stat.S_ISLNK(metadata.st_mode): + os.chmod(quarantine, 0o600) + directory_descriptor = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory_descriptor) + finally: + os.close(directory_descriptor) + except OSError as exc: + raise NetworkProvisioningIdempotencyCorrupt( + "corrupt network audit could not be quarantined safely" + ) from exc + quarantined.append(quarantine.name) + + if quarantined: + self._network_mutation_ledger = NetworkMutationLedger(self.repository_root) + journal = NetworkProvisioningIdempotencyJournal(self.repository_root) + with self._lock: + self._network_provisioning_idempotency_journal = journal + self._network_provisioning_idempotency_reason = None + logger.error( + "Corrupt K1 network audit was quarantined before a fresh explicit intent", + extra={ + "event_code": "k1_network_audit_quarantined", + "quarantined_files": quarantined, + "automatic_retry": False, + }, + ) + admitted_journal = journal or candidate_journal + if self._network_provisioning_idempotency_journal is None: + with self._lock: + self._network_provisioning_idempotency_journal = admitted_journal + self._network_provisioning_idempotency_reason = None + return admitted_journal + + def _network_provisioning_idempotency_public_snapshot(self) -> dict[str, object]: + journal = self._network_provisioning_idempotency_journal + if journal is None: + return { + "status": "corrupt", + "revision": None, + "reason_code": ( + self._network_provisioning_idempotency_reason + or NetworkProvisioningIdempotencyCorrupt.reason_code + ), + "active_operation_id": None, + "active_action": None, + "active_stage": None, + "terminal_record_count": 0, + "mutation_allowed": False, + } + snapshot = journal.snapshot() + active = snapshot.active_record + return { + "status": snapshot.status, + "revision": snapshot.revision, + "reason_code": snapshot.reason_code, + "active_operation_id": active.operation_id if active is not None else None, + "active_action": active.action if active is not None else None, + "active_stage": active.stage if active is not None else None, + "terminal_record_count": sum(record.stage == "terminal" for record in snapshot.records), + # A non-terminal record is recovered/terminalized only after the + # next explicit request owns the lifecycle lease. It is not a + # browser-side permanent block. Corruption still fails closed. + "mutation_allowed": snapshot.status != "corrupt", + } + + def _fresh_ble_devices_locked( + self, + ) -> list[dict[str, Any]]: + """Return candidates owned by the latest still-current explicit scan. + + The low-level scanner retains the exact CoreBluetooth objects for the + current scan generation until an explicit invalidation. Wall-clock + age must not erase the rows while an operator is reading or completing + the form. A later exact capture and live GATT open remain mandatory + before any candidate can authorize a device or network mutation. + """ + + return [dict(item) for item in self._devices] + + def _connection_reconfiguration_snapshot_locked( + self, + *, + fresh_devices: Sequence[Mapping[str, Any]], + ) -> dict[str, object]: + intent = self._connection_reconfiguration_intent + minimum_generation = self._connection_reconfiguration_minimum_discovery_generation + fresh_scan_completed = bool( + intent is not None + and minimum_generation is not None + and self._ble_discovery_generation >= minimum_generation + and fresh_devices + ) + required_transport_ref = self._connection_reconfiguration_required_transport_ref + return { + "schema_version": "missioncore.xgrids-k1-connection-reconfiguration/v1", + "revision": self._connection_reconfiguration_revision, + "intent_id": self._connection_reconfiguration_intent_id, + "intent": intent, + "status": ( + "idle" + if intent is None + else "fresh-scan-completed" + if fresh_scan_completed + else "awaiting-fresh-scan" + ), + "required_transport_ref": required_transport_ref, + "required_connection_mode": (self._connection_reconfiguration_required_connection_mode), + "minimum_discovery_generation": minimum_generation, + "fresh_discovery_generation": ( + self._ble_discovery_generation if fresh_scan_completed else None + ), + "required_transport_observed": ( + any( + isinstance(required_transport_ref, str) + and str(device.get("device_id") or "").strip() + and physical_transport_ref_comparison_key(str(device.get("device_id") or "")) + == physical_transport_ref_comparison_key(required_transport_ref) + for device in fresh_devices + ) + if intent == "change-network" and fresh_scan_completed + else None + ), + "prepared_at": self._connection_reconfiguration_prepared_at, + "automatic_retry": False, + } + + def _clear_connection_reconfiguration_locked(self) -> None: + """Consume/cancel a pending local dialogue and invalidate stale forms.""" + + self._connection_reconfiguration_revision += 1 + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._scenario_reset_successful_scan_generation = None + self._scenario_reset_successful_scan_reset_id = None + self._connection_reconfiguration_intent_id = None + self._connection_reconfiguration_intent = None + self._connection_reconfiguration_required_transport_ref = None + self._connection_reconfiguration_required_connection_mode = None + self._connection_reconfiguration_minimum_discovery_generation = None + self._connection_reconfiguration_prepared_at = None + + def _consume_connection_reconfiguration( + self, + *, + intent_id: str | None, + revision: int | None, + ) -> bool: + """Consume only the exact winning dialogue after a proven successor.""" + + if intent_id is None or revision is None: + return False + with self._lock: + if ( + self._connection_reconfiguration_intent_id != intent_id + or self._connection_reconfiguration_revision != revision + ): + return False + self._clear_connection_reconfiguration_locked() + return True + + def _reset_provisional_fresh_bridge_topology( + self, + provisional: _ProvisionalFreshBridgeTopology, + ) -> None: + """Drop only the exact untrusted B projection after a losing Verify.""" + + with self._lock: + exact_dialogue = bool( + ( + provisional.reconfiguration_intent_id is not None + and self._connection_reconfiguration_intent == "select-device" + and self._connection_reconfiguration_intent_id + == provisional.reconfiguration_intent_id + ) + or ( + provisional.reconfiguration_intent_id is None + and self._connection_reconfiguration_intent is None + and self._connection_reconfiguration_intent_id is None + ) + ) and ( + self._connection_reconfiguration_revision == provisional.reconfiguration_revision + ) + exact_projection = bool( + self._selected_device_id == provisional.transport_ref + and self._connection_mode == provisional.connection_mode + and self._k1_ip == provisional.ipv4 + and self._device_session_id == provisional.device_session_id + and self._device_session_discovery_generation == provisional.discovery_generation + and self._ble_discovery_generation == provisional.discovery_generation + ) + supervisor = self._connection_supervisor.snapshot() + exact_supervisor = bool( + supervisor.intent is not None + and supervisor.intent.intent_id == provisional.supervisor_intent_id + and supervisor.intent.requested_mode == provisional.connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == provisional.supervisor_intent_id + and supervisor.device_network.transport_ref == provisional.transport_ref + and supervisor.device_network.connection_mode == provisional.connection_mode + and supervisor.device_network.target + == EndpointTarget(provisional.ipv4, CONTROL_MQTT_PORT) + ) + if not (exact_dialogue and exact_projection and exact_supervisor): + return + self._retire_ephemeral_device_binding_for_new_intent() + self._connection_supervisor.set_intent( + intent_id=( + provisional.reconfiguration_intent_id + or f"fresh-verify-rejected-{provisional.device_session_id}" + ), + requested_mode="bridge", + expected_device_id=None, + ) + with self._lock: + self._connection_verification = { + "status": "not-probed", + "lease_state": "disconnected", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": "not-performed", + "network_reachability": "unknown", + "observed_at": None, + "reason_code": "connection-reconfiguration-provisional-rejected", + } + + def _commit_provisional_fresh_bridge_topology( + self, + provisional: _ProvisionalFreshBridgeTopology, + ) -> None: + """CAS-persist a fresh Bridge target only after live DeviceInfo pin proof.""" + + with self._lock: + exact_dialogue = bool( + ( + provisional.reconfiguration_intent_id is not None + and self._connection_reconfiguration_intent == "select-device" + and self._connection_reconfiguration_intent_id + == provisional.reconfiguration_intent_id + ) + or ( + provisional.reconfiguration_intent_id is None + and self._connection_reconfiguration_intent is None + and self._connection_reconfiguration_intent_id is None + ) + ) and ( + self._connection_reconfiguration_revision == provisional.reconfiguration_revision + ) + exact_projection = bool( + self._selected_device_id == provisional.transport_ref + and self._connection_mode == provisional.connection_mode + and self._k1_ip == provisional.ipv4 + and self._device_session_id == provisional.device_session_id + and self._device_session_discovery_generation == provisional.discovery_generation + and self._ble_discovery_generation == provisional.discovery_generation + ) + if not (exact_dialogue and exact_projection): + raise NetworkProvisioningConflict( + "выбранное устройство изменилось до фиксации DeviceInfo", + reason_code="connection-reconfiguration-revision-conflict", + ) + + supervisor = self._connection_supervisor.snapshot() + target = EndpointTarget(provisional.ipv4, CONTROL_MQTT_PORT) + logical_device_id = supervisor.device_identity.logical_device_id + compatibility_profile_id = supervisor.device_identity.compatibility_profile_id + if not ( + supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.intent_id == provisional.supervisor_intent_id + and supervisor.intent.requested_mode == provisional.connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == provisional.supervisor_intent_id + and supervisor.device_network.transport_ref == provisional.transport_ref + and supervisor.device_network.target == target + and supervisor.lease.target == target + and isinstance(logical_device_id, str) + and isinstance(compatibility_profile_id, str) + ): + raise ConnectionVerificationError( + "DeviceInfo не подтвердил точную identity выбранного K1", + reason_code="control-bootstrap-device-identity-unverified", + ) + pin_store = self._device_identity_pin_store + if pin_store is None: + raise DeviceIdentityPinStoreCorrupt("device identity pin store is unavailable") + pin_snapshot = pin_store.snapshot() + pin = pin_snapshot.for_transport(provisional.transport_ref) + if not ( + pin_snapshot.status == "available" + and pin is not None + and pin.vendor_device_id == logical_device_id + and pin.compatibility_profile_id == compatibility_profile_id + ): + raise ConnectionVerificationError( + "DeviceInfo identity не закреплена за выбранным BLE transport", + reason_code="device-identity-pin-mismatch", + ) + self._persist_semantic_topology( + transport_ref=provisional.transport_ref, + connection_mode=provisional.connection_mode, + ipv4=provisional.ipv4, + firmware_version=provisional.firmware_version, + source="ble-read-only-status", + observed_at_utc=provisional.observed_at_utc, + predecessor_revision=provisional.semantic_predecessor_revision, + ) + + def _connection_reconfiguration_safety_reasons( + self, + *, + intent: Literal["select-device", "change-network"], + transition_gate_owned: bool, + active_binding: Mapping[str, object] | None = None, + ) -> list[str]: + """Return one factual Bridge-only pre-START handoff admission policy.""" + + with self._lock: + provisioning_active = self._provisioning_active + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + acquisition_stop_operation_id = self._acquisition_stop_operation_id + desired_mode = self._desired_connection_mode + configured_mode = self._connection_mode + pending_required_mode = self._connection_reconfiguration_required_connection_mode + lifecycle_holders = set(self._application_control_process_lease_holders) + runtime = self.runtime.snapshot() + physical = self._physical_command_coordinator.snapshot() + control = self._application_control_session.snapshot() + ble_runtime = ble_runtime_snapshot() + semantic = self._semantic_topology_public_snapshot() + identity = self._device_identity_pin_public_snapshot() + idempotency = self._network_provisioning_idempotency_public_snapshot() + network_ledger = self._network_mutation_ledger.snapshot() + + reasons: list[str] = [] + if not transition_gate_owned: + gate_available = self._k1_lifecycle_transition_gate.acquire(blocking=False) + if gate_available: + self._k1_lifecycle_transition_gate.release() + else: + reasons.append("connection-reconfiguration-lifecycle-busy") + effective_mode = configured_mode or pending_required_mode or desired_mode + if desired_mode != "bridge" or effective_mode != "bridge": + reasons.append("connection-reconfiguration-bridge-only") + if provisioning_active: + reasons.append("network-provision-operation-active") + locally_prepared = bool( + acquisition is not None + and acquisition.state == "prepared" + and acquisition_lease is None + and acquisition_start_operation_id is None + and runtime.get("source_mode") == "idle" + ) + if ( + acquisition is not None + and acquisition.state not in TERMINAL_ACQUISITION_STATES + and not locally_prepared + ): + reasons.append("acquisition-active") + if acquisition_lease is not None: + reasons.append("acquisition-cleanup-pending") + if acquisition_start_operation_id is not None: + reasons.append("acquisition-start-operation-active") + if acquisition_stop_operation_id is not None: + reasons.append("acquisition-stop-operation-active") + if runtime.get("source_mode") != "idle" or runtime.get("phase") in { + "starting_live", + "stopping", + "error", + }: + reasons.append("local-runtime-active") + + physical_status = str(physical.get("status") or "unknown") + if physical_status == "corrupt": + reasons.append("physical-command-ledger-corrupt") + elif physical.get("requires_reconciliation") is True or physical_status == "unresolved": + reasons.append("physical-command-reconciliation-required") + elif physical.get("resolved_active_recovery_required") is True: + # Includes the pending-reopen fence where the prior active proof + # is deliberately non-authoritative until fresh Verify. It must + # not become a network/select-device mutation window merely + # because ``physical_active`` is temporarily false. + reasons.append("physical-command-reconciliation-required") + elif _physical_command_prepared_stop_ready_successor_pending(physical): + reasons.append("physical-command-successor-binding-pinned") + elif physical_status not in {"empty", "resolved"}: + reasons.append("physical-command-ledger-unavailable") + if _physical_command_reports_active(physical): + reasons.append("physical-device-already-active") + + if ble_runtime.get("poisoned") is True: + reasons.append("ble-runtime-restart-required") + elif ble_runtime.get("cleanup_pending") is True: + reasons.append("ble-runtime-cleanup-pending") + elif ble_runtime.get("active_operation_kind") is not None: + reasons.append("ble-runtime-busy") + if "network" in lifecycle_holders: + reasons.append("k1-lifecycle-process-lease-network-owned") + if "camera" in lifecycle_holders: + reasons.append("camera-preview-active") + + control_state = str(control.get("state") or "unknown") + if control_state not in { + "idle", + "completed", + "closed", + "failed", + "connection-ready", + "workspace-ready", + "project-ready", + }: + reasons.append("control-session-state-unsafe") + if semantic.get("status") == "corrupt": + reasons.append("semantic-topology-store-corrupt") + if identity.get("status") == "corrupt": + reasons.append("device-identity-pin-store-corrupt") + if idempotency.get("status") == "corrupt": + reasons.append("network-provisioning-idempotency-corrupt") + elif idempotency.get("active_operation_id") is not None: + reasons.append("network-provisioning-idempotency-reconciliation-required") + if network_ledger.status == "unresolved": + reasons.append("network-write-reconciliation-required") + if intent == "change-network" and not ( + isinstance(active_binding, Mapping) + and isinstance(active_binding.get("transport_ref"), str) + and bool(str(active_binding.get("transport_ref") or "").strip()) + and active_binding.get("connection_mode") == "bridge" + ): + reasons.append("connection-reconfiguration-current-device-unavailable") + return list(dict.fromkeys(reasons)) + + def _physical_operator_retirement_projection( + self, + *, + allow_owned_network_holder: bool = False, + ) -> dict[str, object]: + """Project one exact local-only retirement checkpoint. + + This method performs no BLE, Wi-Fi, MQTT, START or STOP operation. Its + optimistic projection is advisory for the UI; the action re-samples + every fact under the lifecycle gate/process lease and the durable + ledger performs the final operation/revision/transport CAS. + """ + + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + with self._lock: + provisioning_active = self._provisioning_active + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + acquisition_stop_operation_id = self._acquisition_stop_operation_id + reconfiguration_active = self._connection_reconfiguration_intent is not None + pending_local_control_retirement = self._pending_local_control_retirement + lifecycle_holders = set(self._application_control_process_lease_holders) + runtime = self.runtime.snapshot() + control = self._application_control_session.snapshot() + native = ble_runtime_snapshot() + + reasons: list[str] = [] + retireable_ambiguous = bool( + physical.get("status") == "unresolved" + and physical.get("requires_reconciliation") is True + and isinstance(record, Mapping) + and record.get("stage") in {"dispatching", "observing"} + and record.get("resolution") is None + ) + retireable_proven_active = bool( + physical.get("status") == "resolved" + and physical.get("resolved_active_recovery_required") is True + and _physical_command_reports_active(physical) + and isinstance(record, Mapping) + and record.get("stage") == "resolved" + ) + retireable_pending_reopen = bool( + physical.get("status") == "resolved" + and physical.get("resolved_active_recovery_required") is True + and physical.get("reopened_physical_state_recovery_required") is True + and physical.get("physical_active") is False + and isinstance(record, Mapping) + and record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "not-dispatched" + ) + if physical.get("status") == "corrupt": + reasons.append("physical-command-ledger-corrupt") + elif not ( + ( + retireable_ambiguous + or retireable_proven_active + or retireable_pending_reopen + ) + and isinstance(record, Mapping) + and isinstance(record.get("operation_id"), str) + and isinstance(record.get("revision"), int) + and isinstance(record.get("connection"), Mapping) + and isinstance(record["connection"].get("transport_ref"), str) + ): + reasons.append("physical-command-retirement-not-required") + if provisioning_active: + reasons.append("network-provision-operation-active") + if reconfiguration_active: + reasons.append("connection-reconfiguration-active") + if pending_local_control_retirement: + reasons.append("control-local-retirement-pending") + if self._calibration_gate.locked(): + reasons.append("device-calibration-read-active") + if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: + reasons.append("acquisition-active") + if acquisition_lease is not None: + reasons.append("acquisition-cleanup-pending") + if acquisition_start_operation_id is not None: + reasons.append("acquisition-start-operation-active") + if acquisition_stop_operation_id is not None: + reasons.append("acquisition-stop-operation-active") + if runtime.get("source_mode") != "idle" or runtime.get("phase") in { + "starting_live", + "stopping", + "error", + }: + reasons.append("local-runtime-active") + if str(control.get("state") or "unknown") not in { + "idle", + "completed", + "closed", + "failed", + }: + reasons.append("control-session-state-unsafe") + active_operation_id = physical.get("active_operation_id") + if active_operation_id is not None: + reasons.append("physical-command-retirement-operation-conflict") + if native.get("poisoned") is True: + reasons.append("ble-runtime-restart-required") + elif native.get("cleanup_pending") is True: + reasons.append("ble-runtime-cleanup-pending") + elif native.get("active_operation_kind") is not None: + reasons.append("ble-runtime-busy") + disallowed_holders = set(lifecycle_holders) + disallowed_holders.discard("monitor") + if allow_owned_network_holder: + disallowed_holders.discard("network") + if disallowed_holders: + reasons.append("k1-lifecycle-process-lease-active") + + connection = record.get("connection") if isinstance(record, Mapping) else None + return { + "allowed": not reasons, + "reason_codes": list(dict.fromkeys(reasons)), + "expected_operation_id": ( + record.get("operation_id") if isinstance(record, Mapping) else None + ), + "expected_revision": (record.get("revision") if isinstance(record, Mapping) else None), + "expected_transport_ref": ( + connection.get("transport_ref") if isinstance(connection, Mapping) else None + ), + "physical_outcome": "unknown", + "device_io_performed": False, + "automatic_retry": False, + } + + @staticmethod + def _active_physical_retirement_documents( + record: Mapping[str, object], + ) -> list[Mapping[str, object]]: + retirements = record.get("operator_retirements") + reopens = record.get("operator_reconciliation_reopens") + if not isinstance(retirements, list): + return [] + reopen_documents = reopens if isinstance(reopens, list) else [] + reopened_ids = { + str(item.get("retirement_id")) + for item in reopen_documents + if isinstance(item, Mapping) and isinstance(item.get("retirement_id"), str) + } + return [ + item + for item in retirements + if isinstance(item, Mapping) + and isinstance(item.get("retirement_id"), str) + and item.get("retirement_id") not in reopened_ids + ] + + def _physical_operator_reconciliation_reopen_projection( + self, + *, + allow_owned_network_holder: bool = False, + ) -> dict[str, object]: + """Project one exact fresh UUID checkpoint for local retirement reopen.""" + + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + with self._lock: + provisioning_active = self._provisioning_active + desired_connection_mode = self._desired_connection_mode + desired_connection_mode_revision = self._desired_connection_mode_revision + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + acquisition_stop_operation_id = self._acquisition_stop_operation_id + pending_local_control_retirement = self._pending_local_control_retirement + lifecycle_holders = set(self._application_control_process_lease_holders) + fresh_devices = self._fresh_ble_devices_locked() + discovery_generation = self._ble_discovery_generation + generation_floors = dict(self._physical_retirement_reopen_generation_floors) + runtime = self.runtime.snapshot() + control = self._application_control_session.snapshot() + native = ble_runtime_snapshot() + supervisor = self._connection_supervisor.snapshot() + network_ledger = self._network_mutation_ledger.snapshot() + idempotency = self._network_provisioning_idempotency_public_snapshot() + semantic_topology = self._semantic_topology_public_snapshot() + identity_pins = self._device_identity_pin_public_snapshot() + + active_retirements = ( + self._active_physical_retirement_documents(record) + if isinstance(record, Mapping) + else [] + ) + current_operation_id = record.get("operation_id") if isinstance(record, Mapping) else None + retirement = next( + ( + item + for item in reversed(active_retirements) + if isinstance( + original_attempt := item.get("original_attempt"), + Mapping, + ) + and original_attempt.get("operation_id") == current_operation_id + ), + None, + ) + retired_transport_ref = ( + retirement.get("retired_transport_ref") if isinstance(retirement, Mapping) else None + ) + connection = record.get("connection") if isinstance(record, Mapping) else None + recovery_connection_mode = ( + connection.get("connection_mode") if isinstance(connection, Mapping) else None + ) + retirement_id = ( + str(retirement.get("retirement_id")) + if isinstance(retirement, Mapping) and isinstance(retirement.get("retirement_id"), str) + else None + ) + minimum_generation = generation_floors.get(retirement_id or "", 0) + if retirement_id is not None and retirement_id not in generation_floors: + # Another already-running service process may observe this durable + # retirement after it retained a candidate from its current scan + # generation for the same UUID. Establish the local floor where + # this process first saw the retirement; only a later explicit scan + # can then supply presence evidence for a reopen. + with self._lock: + minimum_generation = self._physical_retirement_reopen_generation_floors.setdefault( + retirement_id, + self._ble_discovery_generation, + ) + matching_candidates = [ + item + for item in fresh_devices + if discovery_generation > minimum_generation + and isinstance(item.get("device_id"), str) + and isinstance(retired_transport_ref, str) + and physical_transport_ref_comparison_key(str(item["device_id"])) + == physical_transport_ref_comparison_key(retired_transport_ref) + ] + candidate_transport_ref = ( + str(matching_candidates[0]["device_id"]) if len(matching_candidates) == 1 else None + ) + + reasons: list[str] = [] + if physical.get("status") == "corrupt": + reasons.append("physical-command-ledger-corrupt") + elif not ( + physical.get("status") == "resolved" + and isinstance(record, Mapping) + and record.get("stage") == "resolved" + and record.get("resolution") == "operator-retired-outcome-unknown" + and isinstance(record.get("revision"), int) + and isinstance(retirement, Mapping) + and isinstance(retirement.get("retirement_id"), str) + and isinstance(retired_transport_ref, str) + ): + reasons.append("physical-command-reconciliation-reopen-not-required") + if ( + isinstance(retirement, Mapping) + and isinstance(recovery_connection_mode, str) + and recovery_connection_mode != desired_connection_mode + ): + reasons.append("physical-command-reconciliation-reopen-mode-mismatch") + post_reopen_shared_reasons = _shared_fresh_observation_reasons( + supervisor_closed=supervisor.closed, + provisioning_active=provisioning_active, + acquisition_active=bool( + acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES + ), + acquisition_cleanup_pending=acquisition_lease is not None, + runtime_active=bool( + runtime.get("source_mode") != "idle" + or runtime.get("phase") in {"starting_live", "stopping", "error"} + ), + ble_runtime=native, + lifecycle_process_lease_holders=tuple(sorted(lifecycle_holders)), + application_control_session=control, + ) + post_reopen_durable_reasons = _durable_connection_health_reasons( + network_provisioning_idempotency=idempotency, + network_provisioning_idempotency_available=( + self._network_provisioning_idempotency_journal is not None + ), + semantic_topology_store=semantic_topology, + device_identity_pin_store=identity_pins, + ) + reasons.extend( + _fresh_observation_reasons( + shared_reasons=post_reopen_shared_reasons, + durable_health_reasons=post_reopen_durable_reasons, + application_control_session=control, + network_ledger_status=network_ledger.status, + network_ledger_operation_id=( + network_ledger.record.operation_id + if network_ledger.record is not None + else None + ), + network_ledger_stage=( + network_ledger.record.stage if network_ledger.record is not None else None + ), + network_ledger_resolution=( + network_ledger.record.resolution if network_ledger.record is not None else None + ), + network_ledger_transport_ref=( + network_ledger.record.transport_ref + if network_ledger.record is not None + else None + ), + network_ledger_connection_mode=( + network_ledger.record.intended_mode + if network_ledger.record is not None + else None + ), + network_provisioning_active_operation_id=( + str(idempotency["active_operation_id"]) + if isinstance(idempotency.get("active_operation_id"), str) + and str(idempotency["active_operation_id"]).strip() + else None + ), + network_provisioning_active_action=( + str(idempotency["active_action"]) + if isinstance(idempotency.get("active_action"), str) + and str(idempotency["active_action"]).strip() + else None + ), + network_provisioning_active_stage=( + str(idempotency["active_stage"]) + if isinstance(idempotency.get("active_stage"), str) + and str(idempotency["active_stage"]).strip() + else None + ), + eligible_fresh_transport_refs=( + [candidate_transport_ref] if candidate_transport_ref is not None else [] + ), + physical_recovery_required=True, + physical_recovery_transport_ref=( + str(retired_transport_ref) if isinstance(retired_transport_ref, str) else None + ), + physical_recovery_mode=( + cast(ConnectionMode, recovery_connection_mode) + if recovery_connection_mode in {"bridge", "quick-connect", "direct-connect"} + else None + ), + ) + ) + if not matching_candidates: + reasons.append("physical-command-reconciliation-reopen-target-not-observed") + elif len(matching_candidates) != 1: + reasons.append("physical-command-reconciliation-reopen-candidate-ambiguous") + elif matching_candidates[0].get("connectable") is False: + reasons.append("physical-command-reconciliation-reopen-target-not-connectable") + if provisioning_active: + reasons.append("network-provision-operation-active") + if pending_local_control_retirement: + reasons.append("control-local-retirement-pending") + if self._calibration_gate.locked(): + reasons.append("device-calibration-read-active") + if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: + reasons.append("acquisition-active") + if acquisition_lease is not None: + reasons.append("acquisition-cleanup-pending") + if acquisition_start_operation_id is not None: + reasons.append("acquisition-start-operation-active") + if acquisition_stop_operation_id is not None: + reasons.append("acquisition-stop-operation-active") + if runtime.get("source_mode") != "idle" or runtime.get("phase") in { + "starting_live", + "stopping", + "error", + }: + reasons.append("local-runtime-active") + if str(control.get("state") or "unknown") not in { + "idle", + "completed", + "closed", + "failed", + }: + reasons.append("control-session-state-unsafe") + if physical.get("active_operation_id") is not None: + reasons.append("physical-command-reconciliation-reopen-operation-conflict") + if native.get("poisoned") is True: + reasons.append("ble-runtime-restart-required") + elif native.get("cleanup_pending") is True: + reasons.append("ble-runtime-cleanup-pending") + elif native.get("active_operation_kind") is not None: + reasons.append("ble-runtime-busy") + disallowed_holders = set(lifecycle_holders) + disallowed_holders.discard("monitor") + if allow_owned_network_holder: + disallowed_holders.discard("network") + if disallowed_holders: + reasons.append("k1-lifecycle-process-lease-active") + + return { + "allowed": not reasons, + "reason_codes": list(dict.fromkeys(reasons)), + "expected_revision": (record.get("revision") if isinstance(record, Mapping) else None), + "expected_retirement_id": (retirement_id), + "expected_transport_ref": candidate_transport_ref, + "expected_discovery_generation": ( + discovery_generation if candidate_transport_ref is not None else None + ), + "expected_desired_mode": desired_connection_mode, + "expected_desired_mode_revision": desired_connection_mode_revision, + "device_io_performed": False, + "automatic_retry": False, + } + + def _retired_physical_transport_refs(self) -> frozenset[str]: + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + if not isinstance(record, Mapping): + return frozenset() + retirements = self._active_physical_retirement_documents(record) + return frozenset( + physical_transport_ref_comparison_key(str(item.get("retired_transport_ref"))) + for item in retirements + if isinstance(item, Mapping) + and isinstance(item.get("retired_transport_ref"), str) + and item.get("retired_transport_ref") + ) + + @classmethod + def _scenario_reset_retired_authority_from_commit( + cls, + *, + reset_id: str, + reset_revision: int, + desired_mode: ConnectionMode, + minimum_discovery_generation: int, + physical_preflight: Mapping[str, object], + physical_retirement: Mapping[str, object], + ) -> _ScenarioResetRetiredTransportAuthority | None: + """Bind a reset only to the exact active retirement it committed.""" + + if physical_retirement.get("disposition") != "operator-retired-outcome-unknown": + return None + record = physical_retirement.get("record") + if not isinstance(record, Mapping): + return None + preflight_operation_id = physical_preflight.get("operation_id") + preflight_transport_ref = physical_preflight.get("transport_ref") + if not ( + isinstance(preflight_operation_id, str) + and isinstance(preflight_transport_ref, str) + ): + return None + matching_retirements = [ + item + for item in cls._active_physical_retirement_documents(record) + if item.get("reason") == "connection-scenario-reset-by-operator" + and isinstance(item.get("retired_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["retired_transport_ref"]) + ) + == physical_transport_ref_comparison_key(preflight_transport_ref) + and isinstance(item.get("original_attempt"), Mapping) + and cast(Mapping[str, object], item["original_attempt"]).get("operation_id") + == preflight_operation_id + ] + if len(matching_retirements) != 1: + return None + retirement = matching_retirements[0] + retirement_id = retirement.get("retirement_id") + original_attempt_sha256 = retirement.get("original_attempt_sha256") + retired_at_utc = retirement.get("retired_at_utc") + retired_transport_ref = retirement.get("retired_transport_ref") + record_operation_id = record.get("operation_id") + record_created_at_utc = record.get("created_at_utc") + record_revision = record.get("revision") + if not ( + isinstance(retirement_id, str) + and isinstance(original_attempt_sha256, str) + and isinstance(retired_at_utc, str) + and isinstance(retired_transport_ref, str) + and isinstance(record_operation_id, str) + and isinstance(record_created_at_utc, str) + and isinstance(record_revision, int) + ): + return None + return _ScenarioResetRetiredTransportAuthority( + reset_id=reset_id, + reset_revision=reset_revision, + desired_mode=desired_mode, + retired_transport_ref=retired_transport_ref, + retirement_id=retirement_id, + retirement_original_attempt_sha256=original_attempt_sha256, + retirement_retired_at_utc=retired_at_utc, + physical_record_operation_id=record_operation_id, + physical_record_created_at_utc=record_created_at_utc, + physical_record_revision=record_revision, + minimum_discovery_generation=minimum_discovery_generation, + ) + + def _scenario_reset_retired_authority_matches_locked( + self, + *, + authority: _ScenarioResetRetiredTransportAuthority, + physical: Mapping[str, object], + request: ConnectRequest | None = None, + consumed_network_record_trusted: bool = False, + ) -> bool: + """Check the private reset/scan/retirement CAS without side effects.""" + + reset = self._connection_scenario_reset + successful_generation = self._scenario_reset_successful_scan_generation + settled_generation = ( + reset.get("settled_by_discovery_generation") + if isinstance(reset, Mapping) + else None + ) + if not ( + self._connection_scenario_reset_pending is None + and isinstance(reset, Mapping) + and reset.get("reset_id") == authority.reset_id + and reset.get("revision") == authority.reset_revision + and reset.get("desired_mode") == authority.desired_mode + and reset.get("active") is False + and reset.get("local_session_closed") is True + and reset.get("device_command_performed") is False + and reset.get("network_write_performed") is False + and reset.get("automatic_scan") is False + and isinstance(settled_generation, int) + and self._desired_connection_mode_revision == authority.reset_revision + and self._desired_connection_mode == authority.desired_mode + ): + return False + + consumed = authority.consumed_request_fingerprint is not None + if consumed: + if not ( + request is not None + and authority.physical_reopening_id is not None + and isinstance(authority.physical_reopened_record_revision, int) + and isinstance( + authority.physical_reopening_discovery_generation, + int, + ) + and authority.consumed_operation_id is not None + and authority.consumed_request_binding_sha256 is not None + and consumed_network_record_trusted + and physical_transport_ref_comparison_key(request.device_id) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + and request.connection_mode == authority.desired_mode + and request.expected_mode_revision == authority.reset_revision + and request.expected_discovery_generation + == authority.physical_reopening_discovery_generation + and authority.consumed_idempotency_key_sha256 + == hashlib.sha256(request.idempotency_key.encode("utf-8")).hexdigest() + and authority.consumed_request_binding_sha256 + == self._network_provision_request_binding_sha256(request) + and authority.consumed_request_fingerprint + == self._network_provision_request_fingerprint(request) + ): + return False + else: + candidate_refs = [ + str(item.get("device_id")) + for item in self._fresh_ble_devices_locked() + if isinstance(item.get("device_id"), str) + and item.get("connectable") is not False + and physical_transport_ref_comparison_key(str(item["device_id"])) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + ] + if not ( + isinstance(successful_generation, int) + and successful_generation == self._ble_discovery_generation + and successful_generation >= settled_generation + and successful_generation >= authority.minimum_discovery_generation + and self._scenario_reset_successful_scan_reset_id + == authority.reset_id + and len(candidate_refs) == 1 + and ( + request is None + or ( + physical_transport_ref_comparison_key(request.device_id) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + and request.connection_mode == authority.desired_mode + and request.expected_mode_revision + == authority.reset_revision + and request.expected_discovery_generation + == successful_generation + and ( + request.operation_id is None + or request.operation_id + not in self._connection_scenario_reset_network_operation_ids + ) + ) + ) + ): + return False + + record = physical.get("record") + if not ( + isinstance(record, Mapping) + and record.get("operation_id") == authority.physical_record_operation_id + and record.get("created_at_utc") == authority.physical_record_created_at_utc + ): + return False + if consumed: + reopens = record.get("operator_reconciliation_reopens") + active_retirements = self._active_physical_retirement_documents(record) + if not ( + isinstance(reopens, list) + and isinstance(authority.physical_reopened_record_revision, int) + and isinstance(record.get("revision"), int) + and cast(int, record["revision"]) + >= authority.physical_reopened_record_revision + and not any( + isinstance(item.get("retired_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["retired_transport_ref"]) + ) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + for item in active_retirements + ) + ): + return False + matching_reopens = [ + item + for item in reopens + if isinstance(item, Mapping) + and item.get("reopening_id") == authority.physical_reopening_id + and item.get("retirement_id") == authority.retirement_id + and item.get("retired_record_revision") + == authority.physical_record_revision + and item.get("reopened_transport_ref") + == authority.retired_transport_ref + and item.get("discovery_generation") + == authority.physical_reopening_discovery_generation + and item.get("reason") + == "reset-network-intent-read-only-settlement" + ] + return len(matching_reopens) == 1 + if not ( + physical.get("status") == "resolved" + and record.get("resolution") == "operator-retired-outcome-unknown" + and record.get("revision") == authority.physical_record_revision + ): + return False + matching_retirements = [ + item + for item in self._active_physical_retirement_documents(record) + if item.get("retirement_id") == authority.retirement_id + and item.get("original_attempt_sha256") + == authority.retirement_original_attempt_sha256 + and item.get("retired_at_utc") == authority.retirement_retired_at_utc + and isinstance(item.get("retired_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["retired_transport_ref"]) + ) + == physical_transport_ref_comparison_key(authority.retired_transport_ref) + ] + return len(matching_retirements) == 1 + + def _network_provision_request_binding_sha256( + self, + request: ConnectRequest, + ) -> str: + """Derive the durable exact-request binding without retaining secrets.""" + + quick_connect = request.connection_mode == "quick-connect" + password = "" if request.password is None else request.password.get_secret_value() + canonical_request = b"" + try: + canonical_request = json.dumps( + { + "schema_version": ( + "missioncore.xgrids-k1-network-provision-request/v1" + ), + "device_id": request.device_id, + "ssid": request.ssid, + "password": password if not quick_connect else None, + "connection_mode": request.connection_mode, + "expected_mode_revision": request.expected_mode_revision, + "expected_discovery_generation": ( + request.expected_discovery_generation + ), + "expected_reconfiguration_revision": ( + request.expected_reconfiguration_revision + ), + "expected_reconfiguration_intent_id": ( + request.expected_reconfiguration_intent_id + ), + "compatibility_attestation": ( + request.compatibility_attestation.model_dump(mode="json") + ), + **( + {"allow_host_wifi_switch": True} + if request.allow_host_wifi_switch + else {} + ), + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return derive_request_binding_sha256( + request.idempotency_key, + action=ACTION_NETWORK_PROVISION, + canonical_request=canonical_request, + ) + finally: + password = "" + canonical_request = b"" + + def _network_provision_request_fingerprint( + self, + request: ConnectRequest, + ) -> str: + """Hash the exact request body while keeping credentials out of audit.""" + + quick_connect = request.connection_mode == "quick-connect" + password = "" if request.password is None else request.password.get_secret_value() + try: + return self._request_fingerprint( + ACTION_NETWORK_PROVISION, + { + "device_id": request.device_id, + "ssid": request.ssid, + "password": password if not quick_connect else None, + "connection_mode": request.connection_mode, + "expected_mode_revision": request.expected_mode_revision, + "expected_discovery_generation": ( + request.expected_discovery_generation + ), + "expected_reconfiguration_revision": ( + request.expected_reconfiguration_revision + ), + "expected_reconfiguration_intent_id": ( + request.expected_reconfiguration_intent_id + ), + "compatibility_attestation": ( + request.compatibility_attestation.model_dump(mode="json") + ), + **( + {"allow_host_wifi_switch": True} + if request.allow_host_wifi_switch + else {} + ), + }, + ) + finally: + password = "" + + def _durable_reset_reopen_not_dispatched_replay_matches( + self, + request: ConnectRequest, + ) -> bool: + """Recognize one exact crash-safe replay after the hidden local reopen. + + A process can stop after the reset-owned physical reopen fsync but + before network DISPATCHING. The next process has no private reset + marker, so the only safe exception is the conjunction of three + durable facts: the exact request is terminal with side effect NONE, + its network ledger row is resolved NOT-DISPATCHED, and the current + physical head carries the deterministic hidden reopen for that same + operation/request binding. This exception reaches only terminal + journal replay; it never authorizes BLE capture or another write. + """ + + journal = self._network_provisioning_idempotency_journal + if journal is None: + return False + journal_snapshot = journal.snapshot() + network_snapshot = self._network_mutation_ledger.snapshot() + physical = self._physical_command_coordinator.snapshot() + physical_record = physical.get("record") + if ( + journal_snapshot.status == "corrupt" + or network_snapshot.status != "resolved" + or network_snapshot.record is None + or network_snapshot.record.stage != "resolved" + or network_snapshot.record.resolution != "not-dispatched" + or physical.get("requires_reconciliation") is not True + or not isinstance(physical_record, Mapping) + or physical_record.get("stage") != "observing" + or physical_record.get("resolution") is not None + ): + return False + request_key_sha256 = hashlib.sha256( + request.idempotency_key.encode("utf-8") + ).hexdigest() + request_binding_sha256 = self._network_provision_request_binding_sha256( + request + ) + matching_journal_records = [ + item + for item in journal_snapshot.records + if item.action == ACTION_NETWORK_PROVISION + and item.idempotency_key_sha256 == request_key_sha256 + and item.request_binding_sha256 == request_binding_sha256 + and item.stage == "terminal" + and item.terminal is not None + and item.terminal.side_effect_status == "none" + and item.terminal.safe_to_retry is True + ] + if len(matching_journal_records) != 1: + return False + terminal_record = matching_journal_records[0] + network_record = network_snapshot.record + if not ( + network_record.operation_id == terminal_record.operation_id + and physical_transport_ref_comparison_key(network_record.transport_ref) + == physical_transport_ref_comparison_key(request.device_id) + and network_record.intended_mode == request.connection_mode + ): + return False + expected_reopening_id = ( + "reset-network-intent-" + + hashlib.sha256( + ( + terminal_record.operation_id + + ":" + + request_binding_sha256 + ).encode("utf-8") + ).hexdigest()[:48] + ) + reopens = physical_record.get("operator_reconciliation_reopens") + matching_reopens = [ + item + for item in reopens + if isinstance(item, Mapping) + and item.get("reopening_id") == expected_reopening_id + and isinstance(item.get("reopened_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["reopened_transport_ref"]) + ) + == physical_transport_ref_comparison_key(request.device_id) + and item.get("discovery_generation") + == request.expected_discovery_generation + and item.get("reason") + == "reset-network-intent-read-only-settlement" + ] if isinstance(reopens, list) else [] + return len(matching_reopens) == 1 + + def _hidden_reset_reopen_requires_reconciliation(self) -> bool: + """Return whether a crash-retained hidden reopen fences new Apply.""" + + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + if not ( + physical.get("requires_reconciliation") is True + and isinstance(record, Mapping) + and record.get("stage") == "observing" + and record.get("resolution") is None + ): + return False + reopens = record.get("operator_reconciliation_reopens") + return bool( + isinstance(reopens, list) + and any( + isinstance(item, Mapping) + and item.get("reason") + == "reset-network-intent-read-only-settlement" + for item in reopens + ) + ) + + def _scenario_reset_consumed_network_record_is_trusted( + self, + *, + authority: _ScenarioResetRetiredTransportAuthority, + request: ConnectRequest, + ) -> bool: + """Require the durable request row before admitting consumed replay.""" + + journal = self._network_provisioning_idempotency_journal + if ( + journal is None + or authority.consumed_operation_id is None + or authority.consumed_idempotency_key_sha256 is None + or authority.consumed_request_binding_sha256 is None + ): + return False + snapshot = journal.snapshot() + if snapshot.status == "corrupt": + return False + request_key_sha256 = hashlib.sha256( + request.idempotency_key.encode("utf-8") + ).hexdigest() + request_binding_sha256 = self._network_provision_request_binding_sha256(request) + matching = [ + record + for record in snapshot.records + if record.operation_id == authority.consumed_operation_id + and record.idempotency_key_sha256 + == authority.consumed_idempotency_key_sha256 + == request_key_sha256 + and record.action == ACTION_NETWORK_PROVISION + and record.request_binding_sha256 + == authority.consumed_request_binding_sha256 + == request_binding_sha256 + and record.stage in {"unresolved", "terminal"} + ] + return len(matching) == 1 + + def _scenario_reset_consumed_request_replay_matches( + self, + request: ConnectRequest, + ) -> bool: + """Recognize the exact in-process consumed request before hidden fence.""" + + physical = self._physical_command_coordinator.snapshot() + with self._lock: + authority = self._connection_scenario_reset_retired_transport_authority + if ( + authority is None + or authority.consumed_request_fingerprint is None + or not self._scenario_reset_consumed_network_record_is_trusted( + authority=authority, + request=request, + ) + ): + return False + with self._lock: + return bool( + self._connection_scenario_reset_retired_transport_authority + == authority + and self._scenario_reset_retired_authority_matches_locked( + authority=authority, + physical=physical, + request=request, + consumed_network_record_trusted=True, + ) + ) + + def _consumed_reset_authority_has_safe_reconfiguration_successor_locked( + self, + *, + authority: _ScenarioResetRetiredTransportAuthority, + physical: Mapping[str, object], + request: ConnectRequest, + ) -> bool: + """Let one later explicit Change Network supersede a spent exception. + + The old changed-key/body request remains denied. Supersession needs a + separately committed reconfiguration generation, its fresh Scan, and + a durably settled standby physical state. SCANNING therefore cannot + use this path until an explicit STOP has itself reached standby. + """ + + record = physical.get("record") + return bool( + authority.consumed_request_fingerprint is not None + and physical.get("status") == "resolved" + and physical.get("requires_reconciliation") is not True + and physical.get("resolved_active_recovery_required") is not True + and not _physical_command_reports_active(physical) + and isinstance(record, Mapping) + and ( + physical.get("reconciled_physical_state") == "standby" + or record.get("resolution") + in { + "physical-standby-observed", + "stop-standby-observed", + } + ) + and not any( + isinstance(item.get("retired_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["retired_transport_ref"]) + ) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + for item in self._active_physical_retirement_documents(record) + ) + and self._connection_reconfiguration_intent == "change-network" + and request.expected_reconfiguration_intent_id + == self._connection_reconfiguration_intent_id + and request.expected_reconfiguration_revision + == self._connection_reconfiguration_revision + and self._connection_reconfiguration_required_connection_mode + == "bridge" + and isinstance( + self._connection_reconfiguration_minimum_discovery_generation, + int, + ) + and request.expected_discovery_generation + == self._ble_discovery_generation + and request.expected_discovery_generation + >= self._connection_reconfiguration_minimum_discovery_generation + and request.connection_mode == "bridge" + and isinstance( + self._connection_reconfiguration_required_transport_ref, + str, + ) + and physical_transport_ref_comparison_key(request.device_id) + == physical_transport_ref_comparison_key( + self._connection_reconfiguration_required_transport_ref + ) + == physical_transport_ref_comparison_key( + authority.retired_transport_ref + ) + ) + + def _current_scenario_reset_retired_reprovision_transport_ref( + self, + physical: Mapping[str, object], + ) -> str | None: + """Return one policy candidate; request admission still rechecks it.""" + + with self._lock: + authority = self._connection_scenario_reset_retired_transport_authority + if ( + authority is None + or authority.consumed_request_fingerprint is not None + or not self._scenario_reset_retired_authority_matches_locked( + authority=authority, + physical=physical, + ) + ): + return None + return authority.retired_transport_ref + + def _require_network_provision_transport_allowed( + self, + request: ConnectRequest, + ) -> None: + """Allow a retired UUID only for its exact reset-owned fresh Apply.""" + + physical = self._physical_command_coordinator.snapshot() + record = physical.get("record") + active_retirements = ( + self._active_physical_retirement_documents(record) + if isinstance(record, Mapping) + else [] + ) + request_key = physical_transport_ref_comparison_key(request.device_id) + request_is_actively_retired = any( + isinstance(item.get("retired_transport_ref"), str) + and physical_transport_ref_comparison_key( + cast(str, item["retired_transport_ref"]) + ) + == request_key + for item in active_retirements + ) + with self._lock: + authority = self._connection_scenario_reset_retired_transport_authority + safe_reconfiguration_successor = bool( + authority is not None + and self._consumed_reset_authority_has_safe_reconfiguration_successor_locked( + authority=authority, + physical=physical, + request=request, + ) + ) + if safe_reconfiguration_successor: + return + request_matches_authority_ref = bool( + authority is not None + and physical_transport_ref_comparison_key(authority.retired_transport_ref) + == request_key + ) + if not request_is_actively_retired and not request_matches_authority_ref: + return + consumed_network_record_trusted = bool( + authority is not None + and authority.consumed_request_fingerprint is not None + and self._scenario_reset_consumed_network_record_is_trusted( + authority=authority, + request=request, + ) + ) + with self._lock: + allowed = bool( + authority is not None + and self._connection_scenario_reset_retired_transport_authority + == authority + and request_matches_authority_ref + and self._scenario_reset_retired_authority_matches_locked( + authority=authority, + physical=physical, + request=request, + consumed_network_record_trusted=( + consumed_network_record_trusted + ), + ) + ) + if not allowed: + raise NetworkProvisioningConflict( + "Этот K1 был явно выведен из текущего контура; " + "выполните новый сброс сценария и Bluetooth-поиск", + reason_code="physical-command-target-retired", + ) + + def _require_transport_not_physically_retired( + self, + transport_ref: str, + *, + verification: bool, + ) -> None: + if ( + physical_transport_ref_comparison_key(transport_ref) + not in self._retired_physical_transport_refs() + ): + return + error_type = ConnectionVerificationError if verification else NetworkProvisioningConflict + raise error_type( + "Этот K1 был явно выведен из текущего контура; выберите другое устройство", + reason_code="physical-command-target-retired", + ) + + def _record_physical_retirement_reopen_generation_floor( + self, + retirement_id: str, + ) -> None: + """Require a scan admitted after this process observed retirement.""" + + with self._lock: + current = self._ble_discovery_generation + previous = self._physical_retirement_reopen_generation_floors.get( + retirement_id, + 0, + ) + self._physical_retirement_reopen_generation_floors[retirement_id] = max( + previous, + current, + ) + + @staticmethod + def _active_connection_binding_projection( + *, + supervisor: ConnectionSupervisorSnapshot, + application_control_session: Mapping[str, object], + active_connection_mode: ConnectionMode | None, + ) -> tuple[dict[str, Any] | None, str | None]: + """Build the one authoritative binding accepted by reconfiguration. + + Retained selection/configuration is deliberately insufficient. The + public key and the change-network pin are derived from the same exact + live control proof, so policy and mutation admission cannot disagree. + """ + + verified = application_control_session.get("verified_control") + intent = supervisor.intent + device_network = supervisor.device_network + lease_target = supervisor.lease.target + if not ( + active_connection_mode is not None + and intent is not None + and device_network.state == "applied" + and device_network.intent_id == intent.intent_id + and isinstance(device_network.transport_ref, str) + and bool(device_network.transport_ref.strip()) + and device_network.connection_mode == active_connection_mode + and lease_target is not None + and isinstance(verified, Mapping) + and isinstance(verified.get("control_session_id"), str) + and verified.get("intent_id") == intent.intent_id + and verified.get("transport_ref") == device_network.transport_ref + and verified.get("connection_mode") == active_connection_mode + and verified.get("target_ipv4") == lease_target.ipv4 + and verified.get("target_port") == lease_target.port + ): + return None, None + + binding_material = { + "intent_id": intent.intent_id, + "transport_ref": device_network.transport_ref, + "connection_mode": active_connection_mode, + "target_ipv4": lease_target.ipv4, + "target_port": lease_target.port, + "host_path_epoch": supervisor.lease.host_path_epoch, + "control_session_id": verified.get("control_session_id"), + } + binding_key = hashlib.sha256( + json.dumps( + binding_material, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + return ( + { + "binding_key": binding_key, + **binding_material, + "logical_device_id": supervisor.device_identity.logical_device_id, + "compatibility_profile_id": (supervisor.device_identity.compatibility_profile_id), + "control_proof_source": verified.get("control_proof_source"), + "control_proof_revision": verified.get("control_proof_revision"), + }, + binding_key, + ) + + def require_snapshot_runtime_id(self, expected_snapshot_runtime_id: str) -> None: + """Reject an action issued by a browser bound to an older service. + + The runtime identifier is immutable for this service lifetime. This + check is intentionally side-effect free and is called by the plugin + envelope before event-loop binding, journaling, leases, or device I/O. + """ + + with self._lock: + current_snapshot_runtime_id = self._snapshot_runtime_id + if expected_snapshot_runtime_id != current_snapshot_runtime_id: + raise SnapshotRuntimeConflict() @_serialized_acquisition_access def state(self) -> dict[str, Any]: + self._retry_pending_local_control_retirement() application_control = self._application_control.snapshot().as_dict() - application_control_session = self._application_control_session.snapshot() + application_control_session = _application_control_session_public_snapshot( + self._application_control_session.snapshot() + ) runtime = self.runtime.snapshot() camera_preview = self.camera_preview.snapshot() - self._reconcile_acquisition(runtime, camera_preview, application_control_session) + self._reconcile_application_control_process_lease(application_control_session) + self._reconcile_terminal_control_transport(application_control_session) + self._reconcile_connection_supervisor(application_control_session, runtime) + if self._retire_orphaned_prestart_control_owner(): + application_control_session = _application_control_session_public_snapshot( + self._application_control_session.snapshot() + ) + self._reconcile_application_control_process_lease(application_control_session) + # Preserve the terminal pre-START proof long enough to retire a local + # PREPARE before the connection-loss reducer resets the control owner. + self._retire_prepared_acquisition_on_terminal_control_failure(application_control_session) + # Freeze the terminal control/physical evidence before local socket + # retirement resets the ephemeral owner to ``idle``. In particular, + # a correlated STOP response followed by fresh SCAN_STOPPING proves + # that STOP was accepted but does *not* prove READY. That distinction + # must survive local teardown so acquisition/operation reduction never + # degrades it to a generic runtime failure or makes STOP replayable. + terminal_control_proof = application_control_session + physical_command_proof = self._physical_command_coordinator.snapshot() + with self._lock: + prepared_stop_acquisition = self._acquisition + prepared_stop_acquisition_id = ( + prepared_stop_acquisition.acquisition_id + if prepared_stop_acquisition is not None + else None + ) + prepared_stop_operation_id = self._acquisition_stop_operation_id + # Settle an exact PREPARED/no-publish STOP while the failed worker is + # still the current control owner. Proven-loss teardown below may + # synchronously retire that owner to ``idle``; doing this afterwards + # would discard the only terminal worker proof and could leave the + # operation running forever. The helper re-samples both facts under + # the publish dispatch gate, so a concurrent mark_dispatching/publish + # can never be mislabeled as not-dispatched. + prepared_stop_worker_outcome = self._settle_prepared_stop_worker_failure( + acquisition_id=prepared_stop_acquisition_id, + stop_operation_id=prepared_stop_operation_id, + ) + self._retire_ephemeral_connection_binding_on_proven_loss(application_control_session) + # Proven-loss retirement may synchronously reset a terminal control + # owner to idle. Continue the same atomic snapshot from that factual + # state instead of returning the stale pre-retirement failure row. + application_control_session = _application_control_session_public_snapshot( + self._application_control_session.snapshot() + ) + self._reconcile_acquisition( + runtime, + camera_preview, + application_control_session, + terminal_control_proof=terminal_control_proof, + physical_command_proof=physical_command_proof, + prepared_stop_worker_outcome=prepared_stop_worker_outcome, + ) + # Acquisition reconciliation may locally retire an expired STOP + # control owner. Publish that factual post-reduction state in the same + # snapshot so passive BLE/read-only recovery is available immediately. + application_control_session = _application_control_session_public_snapshot( + self._application_control_session.snapshot() + ) + self._reconcile_application_control_process_lease(application_control_session) + if self._retire_terminal_prestart_control_failure(): + application_control_session = _application_control_session_public_snapshot( + self._application_control_session.snapshot() + ) + self._reconcile_application_control_process_lease(application_control_session) runtime = self.runtime.snapshot() camera_preview = self.camera_preview.snapshot() + connection_recovery = self._active_stream_recovery_projection( + runtime, + camera=camera_preview, + ) + supervisor_snapshot = self._connection_supervisor.snapshot() metrics = runtime["metrics"] + ledger_snapshot = self._network_mutation_ledger.snapshot() + idempotency_snapshot = self._network_provisioning_idempotency_public_snapshot() + durable_reconciliation = _network_mutation_reconciliation_projection(ledger_snapshot) + semantic_topology_store = self._semantic_topology_public_snapshot() + device_identity_pin_store = self._device_identity_pin_public_snapshot() + physical_command = self._physical_command_coordinator.snapshot() + active_acquisition_checkpoint = ( + self._active_acquisition_checkpoint_public_snapshot() + ) + operator_retirement = self._physical_operator_retirement_projection() + physical_command["operator_retirement"] = operator_retirement + physical_command["operator_retirement_allowed"] = operator_retirement["allowed"] + physical_command["operator_retirement_reason_codes"] = operator_retirement["reason_codes"] + physical_command["operator_reconciliation_reopen"] = ( + self._physical_operator_reconciliation_reopen_projection() + ) + ble_runtime = ble_runtime_snapshot() with self._lock: operation_phase = self._operation_phase operation_message = self._operation_message devices = self._fresh_ble_devices_locked() - discovery_stale = bool(self._devices) and not devices + ble_discovery_generation = self._ble_discovery_generation selected_device_id = self._selected_device_id k1_ip = self._k1_ip connection_mode = self._connection_mode + desired_connection_mode = self._desired_connection_mode + desired_connection_mode_revision = self._desired_connection_mode_revision + connection_scenario_reset = ( + dict(self._connection_scenario_reset) + if self._connection_scenario_reset is not None + else None + ) + connection_scenario_reset_network_operation_ids = ( + self._connection_scenario_reset_network_operation_ids + ) + connection_scenario_reset_pending = ( + dict(self._connection_scenario_reset_pending) + if self._connection_scenario_reset_pending is not None + else None + ) device_id = self._device_id device_session_id = self._device_session_id device_session_opened_at = self._device_session_opened_at - connection_lease_generation = self._connection_lease_generation + self._network_write_reconciliation = durable_reconciliation network_write_reconciliation = ( - dict(self._network_write_reconciliation) - if self._network_write_reconciliation is not None - else None + dict(durable_reconciliation) if durable_reconciliation is not None else None ) - connection_verification = dict(self._connection_verification) + connection_verification = _legacy_connection_projection( + supervisor_snapshot, + self._connection_verification, + ) + configured_endpoint_probe = dict(self._configured_endpoint_probe) compatibility_attestation = ( dict(self._compatibility_attestation) if self._compatibility_attestation is not None else None ) device_calibration = dict(self._device_calibration) - acquisition = self._acquisition.as_dict() if self._acquisition is not None else None + acquisition_owner = self._acquisition + restart_pending = self._restart_rehydration_pending + restart_cleanup_pending = bool( + acquisition_owner is not None + and restart_pending is not None + and restart_pending.phase == "blocked" + and restart_pending.reason_code + == "restart-receiver-cleanup-pending" + and acquisition_owner.acquisition_id + == restart_pending.token.acquisition_id + and self._acquisition_out_dir == restart_pending.out_dir + and self._acquisition_session_lease is not None + and self._acquisition_session_lease.session_root + == restart_pending.out_dir + and runtime.get("producer_generation") + == restart_pending.runtime_producer_generation + ) + acquisition = ( + acquisition_owner.as_dict() + if acquisition_owner is not None + else None + ) if acquisition is not None: acquisition["project_name"] = self._acquisition_project_name acquisition["mount_type"] = self._acquisition_mount_type acquisition["gnss_mode"] = self._acquisition_gnss_mode acquisition["cleanup_pending"] = ( self._acquisition_session_lease is not None - and acquisition["state"] in TERMINAL_ACQUISITION_STATES + and ( + acquisition["state"] in TERMINAL_ACQUISITION_STATES + or restart_cleanup_pending + ) ) + provisioning_active = self._provisioning_active + acquisition_active = bool( + self._acquisition is not None + and self._acquisition.state not in TERMINAL_ACQUISITION_STATES + ) + acquisition_cleanup_pending = self._acquisition_session_lease is not None + acquisition_start_operation_id = self._acquisition_start_operation_id + lifecycle_process_lease_holders = tuple( + sorted(self._application_control_process_lease_holders) + ) + process_lease_quarantine = self._application_control_process_lease_quarantine + connection_reconfiguration = self._connection_reconfiguration_snapshot_locked( + fresh_devices=devices, + ) - runtime_active = runtime["source_mode"] != "idle" or runtime["phase"] in { - "starting_live", - "stopping", - "error", + retired_reprovision_transport_ref = ( + self._current_scenario_reset_retired_reprovision_transport_ref( + physical_command + ) + ) + connected_recovery = connected_device_recovery_snapshot( + selected_device_id, + device_session_id=device_session_id, + ) + current_device_recovery = ( + { + "transport_ref": selected_device_id, + "connection_mode": connection_mode, + # Retention is recovery context, not live presence or mutation + # authority. Keep the legacy key fail-closed while exposing the + # facts separately to newer clients. + "handle_available": False, + "handle_retained": connected_recovery.get("status") == "retained", + "advertised_now": any( + str(item.get("device_id")) == selected_device_id for item in devices + ), + "gatt_validated_recently": bool( + connected_recovery.get("gatt_validated_recently", False) + ), + "observed_at": device_session_opened_at, + } + if selected_device_id is not None + and device_session_id is not None + and connection_mode is not None + else None + ) + + released_terminal_acquisition = bool( + runtime.get("source_mode") == "idle" + and acquisition is not None + and acquisition.get("state") in {"failed", "interrupted"} + and acquisition.get("cleanup_pending") is False + and application_control_session.get("state") + in { + "idle", + "connection-ready", + "active-recovery-requested", + "scanning", + "completed", + "closed", + } + ) + # A sealed terminal receiver is historical evidence, not a live + # runtime owner. Do not let its retained error phase poison BLE + # discovery or the global connection banner after local cleanup and a + # fresh passive control inspection. + runtime_active = bool( + not released_terminal_acquisition + and ( + runtime["source_mode"] != "idle" + or runtime["phase"] + in { + "starting_live", + "stopping", + "error", + } + ) + ) + authority = supervisor_snapshot.authority + active_connection_mode: ConnectionMode | None = ( + connection_mode + if authority.control_allowed + and connection_mode is not None + and supervisor_snapshot.intent is not None + and supervisor_snapshot.intent.requested_mode == connection_mode + and supervisor_snapshot.device_network.connection_mode == connection_mode + else None + ) + active_binding, active_binding_key = self._active_connection_binding_projection( + supervisor=supervisor_snapshot, + application_control_session=application_control_session, + active_connection_mode=active_connection_mode, + ) + connection_policy = _connection_policy_projection( + supervisor=supervisor_snapshot, + ledger=ledger_snapshot, + network_provisioning_idempotency=idempotency_snapshot, + network_provisioning_idempotency_available=( + self._network_provisioning_idempotency_journal is not None + ), + semantic_topology_store=semantic_topology_store, + device_identity_pin_store=device_identity_pin_store, + physical_command=physical_command, + ble_runtime=ble_runtime, + lifecycle_process_lease_holders=lifecycle_process_lease_holders, + fresh_devices=devices, + current_device_recovery=current_device_recovery, + provisioning_active=provisioning_active, + acquisition_active=acquisition_active, + acquisition_state=(str(acquisition.get("state")) if acquisition is not None else None), + acquisition_cleanup_pending=acquisition_cleanup_pending, + runtime_active=runtime_active, + application_control_session=application_control_session, + desired_connection_mode=desired_connection_mode, + active_connection_mode=active_connection_mode, + retired_reprovision_transport_ref=( + retired_reprovision_transport_ref + ), + ) + checkpoint_trust = getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + if checkpoint_trust != "trusted": + start_action = connection_policy["actions"].get("start-acquisition") + if isinstance(start_action, dict): + reasons = list(start_action.get("reason_codes", [])) + reasons.append( + "active-acquisition-recovery-checkpoint-" + + checkpoint_trust + ) + start_action["reason_codes"] = list(dict.fromkeys(reasons)) + start_action["allowed"] = False + connection_policy["allowed_actions"] = [ + action + for action in connection_policy["allowed_actions"] + if action != "start-acquisition" + ] + process_lease_quarantined = process_lease_quarantine is not None + if process_lease_quarantined: + quarantine_reason = "application-control-process-lease-release-ambiguous" + for action in connection_policy["actions"].values(): + if not isinstance(action, dict): + continue + reasons = list(action.get("reason_codes", [])) + reasons.append(quarantine_reason) + action["reason_codes"] = list(dict.fromkeys(reasons)) + action["allowed"] = False + connection_policy["allowed_actions"] = [] + connection_policy["recommended_action"] = "restart-mission-core" + facts = connection_policy.get("facts") + if isinstance(facts, dict): + facts["process_lease_quarantined"] = True + facts["process_lease_restart_required"] = True + reconfiguration_pending = connection_reconfiguration["intent"] is not None + reconfiguration_active_reason = "connection-reconfiguration-active" + if reconfiguration_pending: + # A prepared operator dialogue owns the next exact fresh target. + # Old retained/durable topology remains rollback evidence but must + # not be adopted automatically while selection is in progress. + for action_name in ( + "recover-current-device-network", + "observe-current-device-network", + "observe-configured-device-network", + "inspect-configured-endpoint", + "start-acquisition", + ): + action = connection_policy["actions"].get(action_name) + if not isinstance(action, dict): + continue + reasons = list(action.get("reason_codes", [])) + reasons.append(reconfiguration_active_reason) + action["reason_codes"] = list(dict.fromkeys(reasons)) + action["allowed"] = False + + required_ref = connection_reconfiguration.get("required_transport_ref") + required_mode = connection_reconfiguration.get("required_connection_mode") + if connection_reconfiguration["intent"] == "change-network": + for action_name in ( + "provision-fresh-device", + "observe-fresh-device-network", + ): + action = connection_policy["actions"].get(action_name) + if not isinstance(action, dict): + continue + action["required_transport_ref"] = required_ref + action["required_connection_mode"] = required_mode + if connection_reconfiguration.get("required_transport_observed") is not True: + reasons = list(action.get("reason_codes", [])) + reasons.append("connection-reconfiguration-required-device-not-observed") + action["reason_codes"] = list(dict.fromkeys(reasons)) + action["allowed"] = False + + select_device_reasons = self._connection_reconfiguration_safety_reasons( + intent="select-device", + transition_gate_owned=False, + active_binding=active_binding, + ) + change_network_reasons = self._connection_reconfiguration_safety_reasons( + intent="change-network", + transition_gate_owned=False, + active_binding=active_binding, + ) + + def reconfiguration_decision( + reasons: Sequence[str], + *, + target_source: str, + required_transport_ref: object = None, + required_connection_mode: object = None, + ) -> dict[str, object]: + unique_reasons = list(dict.fromkeys(str(reason) for reason in reasons)) + return { + "allowed": not unique_reasons, + "reason_codes": unique_reasons, + "target_source": target_source, + "required_transport_ref": required_transport_ref, + "required_connection_mode": required_connection_mode, + "requires_live_gatt_validation": False, + "device_write_performed": False, + "automatic_retry": False, + } + + connection_policy["actions"]["prepare-select-device"] = reconfiguration_decision( + select_device_reasons, + target_source="local-prestart-handoff", + required_connection_mode="bridge", + ) + connection_policy["actions"]["prepare-change-network"] = reconfiguration_decision( + change_network_reasons, + target_source="local-prestart-handoff", + required_transport_ref=( + active_binding.get("transport_ref") if isinstance(active_binding, Mapping) else None + ), + required_connection_mode="bridge", + ) + cancel_reasons = ( + [] if reconfiguration_pending else ["connection-reconfiguration-not-active"] + ) + if "connection-reconfiguration-lifecycle-busy" in select_device_reasons: + cancel_reasons.append("connection-reconfiguration-lifecycle-busy") + connection_policy["actions"]["cancel-reconfiguration"] = reconfiguration_decision( + cancel_reasons, + target_source="local-reconfiguration-intent", + ) + connection_policy["actions"]["retire-unavailable-physical-target"] = { + "allowed": operator_retirement["allowed"], + "reason_codes": list(cast(Sequence[str], operator_retirement["reason_codes"])), + "target_source": "durable-physical-command", + "required_transport_ref": operator_retirement["expected_transport_ref"], + "requires_live_gatt_validation": False, + "physical_command_allowed": False, + "physical_outcome": "unknown", + "device_write_performed": False, + "automatic_retry": False, } - connection_reachability = connection_verification.get("network_reachability") - if connection_reachability == "unreachable": - device_connectivity = "offline" - elif connection_reachability == "degraded": - device_connectivity = "degraded" - elif k1_ip is not None: + connection_policy["allowed_actions"] = [ + action_name + for action_name, action in connection_policy["actions"].items() + if action.get("allowed") is True + ] + if reconfiguration_pending: + if connection_reconfiguration["status"] == "awaiting-fresh-scan": + connection_policy["recommended_action"] = ( + "scan-ble" + if connection_policy["actions"]["scan-ble"].get("allowed") is True + else "cancel-reconfiguration" + ) + elif connection_reconfiguration["intent"] == "change-network": + connection_policy["recommended_action"] = ( + "provision-fresh-device" + if connection_reconfiguration.get("required_transport_observed") is True + else "scan-ble" + ) + else: + connection_policy["recommended_action"] = "select-fresh-device" + elif operator_retirement["allowed"] is True: + connection_policy["recommended_action"] = "retire-unavailable-physical-target" + current_device_network = bool( + supervisor_snapshot.intent is not None + and supervisor_snapshot.device_network.state == "applied" + and supervisor_snapshot.device_network.intent_id == supervisor_snapshot.intent.intent_id + and supervisor_snapshot.device_network.connection_mode + == supervisor_snapshot.intent.requested_mode + ) + if authority.control_allowed: device_connectivity = "connected" + elif current_device_network: + device_connectivity = ( + "degraded" if supervisor_snapshot.endpoint.tcp_state == "reachable" else "offline" + ) + elif supervisor_snapshot.last_known is not None or k1_ip is not None: + device_connectivity = "offline" else: device_connectivity = "unknown" - if operation_phase is not None: + if connection_recovery["state"] in {"reconnecting", "blocked"}: + phase = "reconnecting" + message = ( + "Восстанавливаем соединение с прежним K1. " + "Физические команды и настройки сети не повторяются." + if connection_recovery["state"] == "reconnecting" + else "Автоматическая сверка остановлена; локальный приём можно завершить." + ) + elif operation_phase is not None: phase = operation_phase - message = operation_message - elif runtime_active: + message = operation_message or "" + elif runtime_active and ( + runtime.get("source_mode") != "live" or authority.data_ingest_authoritative + ): phase = runtime["phase"] - message = runtime["message"] - elif k1_ip is not None and device_connectivity != "offline": + control_phase = application_control_session.get("state") + if control_phase == "scanning": + message = "K1 подтвердил режим сканирования; приём и запись активны." + elif control_phase in { + "stop-requested", + "stopping", + "awaiting-standby-confirmation", + }: + message = "STOP отправлен один раз; ожидаем подтверждённый READY от K1." + else: + message = runtime["message"] + elif authority.control_allowed: phase = "connected" message = runtime["message"] + elif ( + acquisition is not None + and acquisition.get("state") == "completed" + and isinstance(acquisition.get("result"), Mapping) + and acquisition["result"].get("device_state") == "ready" + and acquisition["result"].get("device_stop") == "protocol-confirmed" + ): + phase = "idle" + message = "Приём и запись завершены. K1 остановлен и готов к новому запуску." elif connection_verification.get("reason_code") == ("connection_lease_host_route_mismatch"): phase = "device_selected" message = ( "K1 получил адрес, но этот компьютер подключён к другой сети. " "Подключите компьютер к той же локальной сети и повторите запуск." ) + elif current_device_network: + phase = "device_selected" + message = ( + "Сетевые настройки сохранены, но подключение к K1 не подтверждено." + if supervisor_snapshot.endpoint.tcp_state == "reachable" + else "Старый сетевой адрес K1 недоступен. Новая команда Wi-Fi " + "не отправлялась; свежий Bluetooth-кандидат можно настроить заново." + ) elif selected_device_id is not None: phase = "device_selected" - message = "Устройство выбрано. Теперь введите название и пароль Wi-Fi." + selected_advertised_now = any( + str(item.get("device_id")) == selected_device_id for item in devices + ) + message = ( + "Устройство выбрано. Теперь введите название и пароль Wi-Fi." + if selected_advertised_now + else "Текущая BLE-реклама выбранного K1 больше не подтверждена. " + "Для новой сетевой команды выполните поиск ещё раз." + ) elif devices: likely_count = sum(bool(item.get("likely_k1")) for item in devices) phase = "idle" @@ -638,33 +7193,270 @@ class XgridsK1CompatibilityService: f"Найдено BLE-устройств: {len(devices)}. " f"Совместимых профилей: {likely_count}. Выберите нужное устройство." ) - elif discovery_stale: - phase = "idle" - message = "Результаты Bluetooth-поиска устарели. Выполните поиск ещё раз." else: phase = "idle" message = runtime["message"] - operation_documents = self._operations.snapshot() + journal_operation_documents = self._operations.snapshot() + connection_attempt = _connection_attempt_projection(journal_operation_documents) + if connection_scenario_reset is not None and connection_attempt is not None: + attempt_id = connection_attempt.get("attempt_id") + if ( + isinstance(attempt_id, str) + and attempt_id in connection_scenario_reset_network_operation_ids + ): + connection_attempt = None + # DeviceInfo bootstrap is a subordinate implementation operation. + # Preserve its stages in the joined connection attempt without + # displacing the user's Connect/Verify action in the public journal. + operation_documents = [ + item + for item in journal_operation_documents + if item.get("action") != ACTION_CONNECTION_CONTROL_BOOTSTRAP + ] + if connection_attempt is not None: + supervisor_document = supervisor_snapshot.as_dict() + connection_attempt["diagnostic_bundle"] = { + "schema_version": "missioncore.xgrids-k1-connection-diagnostic/v1", + "redacted": True, + "generated_at_utc": _utc_now_iso(), + "snapshot_runtime_id": self._snapshot_runtime_id, + "attempt": { + key: value + for key, value in connection_attempt.items() + if key != "diagnostic_bundle" + }, + "network_mutation_ledger": _network_mutation_ledger_public_snapshot( + ledger_snapshot + ), + "connection_supervisor": { + "revision": supervisor_document.get("revision"), + "closed": supervisor_document.get("closed"), + "lease": supervisor_document.get("lease"), + "device_network": supervisor_document.get("device_network"), + "endpoint": supervisor_document.get("endpoint"), + "authority": supervisor_document.get("authority"), + }, + "automatic_retry": False, + } + connection_ready = active_connection_mode is not None + ready_to_start = bool( + connection_ready + and desired_connection_mode == active_connection_mode + and connection_policy["actions"]["start-acquisition"]["allowed"] is True + ) + mode_change_state = ( + "connecting" + if provisioning_active + else "switch-selected" + if desired_connection_mode != connection_mode + else "ready" + if connection_ready + else "awaiting-control" + if connection_mode is not None + else "disconnected" + ) + mode_selection_reasons = _connection_mode_selection_reason_codes( + acquisition_state=(str(acquisition.get("state")) if acquisition is not None else None), + acquisition_lease_retained=acquisition_cleanup_pending, + acquisition_start_operation_id=acquisition_start_operation_id, + runtime=runtime, + physical_command=physical_command, + control_state=str(application_control_session.get("state") or "unknown"), + desired_connection_mode=desired_connection_mode, + ) + if reconfiguration_pending: + mode_selection_reasons.insert(0, "connection-reconfiguration-active") + transition_gate_available = self._k1_lifecycle_transition_gate.acquire(blocking=False) + if transition_gate_available: + self._k1_lifecycle_transition_gate.release() + elif "connection-mode-selection-lifecycle-busy" not in mode_selection_reasons: + mode_selection_reasons.insert(0, "connection-mode-selection-lifecycle-busy") + if provisioning_active and ( + "connection-mode-selection-lifecycle-busy" not in mode_selection_reasons + ): + mode_selection_reasons.insert(0, "connection-mode-selection-lifecycle-busy") + mode_selection = { + "allowed": not mode_selection_reasons, + "reason_codes": list(dict.fromkeys(mode_selection_reasons)), + "automatic_retry": False, + } + lifecycle_allowed_actions = list(connection_policy["allowed_actions"]) + if mode_selection["allowed"] is True: + lifecycle_allowed_actions.append("select-connection-mode") + if connection_mode is not None and not connection_ready: + lifecycle_allowed_actions.append("verify-control-read-only") + if connection_attempt is not None: + # The journal records what the attempt proved when it completed, + # while this projection tells the operator what is safe *now*. + # A later host/AP loss or mode draft must never leave a historical + # successful attempt advertising START after current authority was + # revoked. + attempt_status = str(connection_attempt.get("status") or "unknown") + attempt_control_ready_now = bool( + connection_attempt.get("control_state") == "ready" + and _network_attempt_matches_live_control_authority( + operation_documents=journal_operation_documents, + attempt_id=connection_attempt.get("attempt_id"), + snapshot_runtime_id=self._snapshot_runtime_id, + supervisor=supervisor_snapshot, + active_binding=active_binding, + ) + ) + if connection_attempt.get("control_state") == "ready" and not attempt_control_ready_now: + connection_attempt["control_state"] = "control_not_ready" + start_action = connection_policy["actions"]["start-acquisition"] + start_reason_codes = { + str(reason) + for reason in start_action.get("reason_codes", []) + if isinstance(reason, str) + } + ready_after_current_network_lease_release = bool( + connection_ready + and desired_connection_mode == active_connection_mode + and start_reason_codes == {"k1-lifecycle-process-lease-network-owned"} + ) + preferred_read_only_observation = next( + ( + action_name + for action_name in ( + "observe-current-device-network", + "observe-configured-device-network", + "observe-fresh-device-network", + ) + if action_name in lifecycle_allowed_actions + ), + None, + ) + if attempt_status in {"accepted", "running"}: + current_safe_next_action = "wait-for-current-attempt" + elif ( + isinstance( + connection_attempt.get("physical_reconciliation"), + Mapping, + ) + and connection_attempt["physical_reconciliation"].get("resolution") + == "physical-active-observed" + and "stop-acquisition" in lifecycle_allowed_actions + ): + current_safe_next_action = "stop-acquisition" + elif attempt_control_ready_now and ( + ready_to_start or ready_after_current_network_lease_release + ): + current_safe_next_action = "start-acquisition" + elif "stop-local-receiver" in lifecycle_allowed_actions: + current_safe_next_action = "stop-local-receiver" + elif "retire-unavailable-physical-target" in lifecycle_allowed_actions: + current_safe_next_action = "retire-unavailable-physical-target" + elif ( + ledger_snapshot.status == "unresolved" + and connection_attempt.get("phase") == "network_outcome_unknown" + and preferred_read_only_observation is not None + ) or "verify-control-read-only" in lifecycle_allowed_actions: + current_safe_next_action = "verify-control-read-only" + elif "scan-ble" in lifecycle_allowed_actions: + current_safe_next_action = "scan-select-connect" + else: + # A terminal journal row is historical evidence, not a live + # operation. Never tell the operator to wait on work that has + # already completed, and never advertise an action the current + # policy denies. + current_safe_next_action = "manual-recovery-required" + connection_attempt["safe_next_action"] = current_safe_next_action + diagnostic_bundle = connection_attempt.get("diagnostic_bundle") + if isinstance(diagnostic_bundle, dict): + diagnostic_attempt = diagnostic_bundle.get("attempt") + if isinstance(diagnostic_attempt, dict): + diagnostic_attempt["control_state"] = connection_attempt.get("control_state") + diagnostic_attempt["safe_next_action"] = current_safe_next_action + connection_lifecycle = { + "schema_version": "missioncore.xgrids-k1-connection-lifecycle/v1", + "revision": (supervisor_snapshot.revision + desired_connection_mode_revision), + "desired_mode": desired_connection_mode, + "configured_mode": connection_mode, + "active_mode": active_connection_mode, + "mode_change": { + "state": mode_change_state, + "from": active_connection_mode or connection_mode, + "to": desired_connection_mode, + }, + "mode_selection": mode_selection, + "active_binding_key": active_binding_key, + "active_binding": active_binding, + "connection_ready": connection_ready, + "ready_to_start": ready_to_start, + "operation": connection_attempt, + "allowed_actions": list(dict.fromkeys(lifecycle_allowed_actions)), + "automatic_retry": False, + } active_profile_id = ( XGRIDS_K1_COMPATIBILITY_PROFILE_ID if compatibility_attestation is not None else None ) + published_pcl_frames = metrics.get("pcl_frames") if isinstance(metrics, Mapping) else None + first_authoritative_pcl_published = bool( + isinstance(published_pcl_frames, int) + and not isinstance(published_pcl_frames, bool) + and published_pcl_frames > 0 + ) + camera_recording = camera_preview.get("recording") + camera_current_epoch = ( + camera_recording.get("current_epoch") + if isinstance(camera_recording, Mapping) + else None + ) + camera_activation_complete = bool( + acquisition_active + and camera_preview.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and camera_preview.get("phase") == "streaming" + and isinstance(camera_recording, Mapping) + and camera_recording.get("active") is True + and camera_recording.get("session") + == ( + self._acquisition_out_dir.name + if self._acquisition_out_dir is not None + else None + ) + and camera_recording.get("active_epoch") is not None + and camera_recording.get("media_ready") is True + and isinstance(camera_current_epoch, Mapping) + and camera_current_epoch.get("generation") + == camera_recording.get("active_epoch") + and camera_current_epoch.get("init_committed") is True + and camera_current_epoch.get("first_media_committed") is True + ) + camera_activation_admitted = bool( + not acquisition_active or camera_activation_complete + ) + camera_preview["activation_admission"] = { + "state": ( + "admitted" + if camera_activation_complete + else "activating" + if acquisition_active and first_authoritative_pcl_published + else "waiting-for-first-authoritative-pcl" + if acquisition_active + else "inactive" + ), + "basis": "post-rerun-publish-pcl-frame", + "runtime_producer_generation": runtime.get("producer_generation"), + "device_command_sent": False, + } if device_calibration.get("status") != "available": device_calibration = unavailable_device_calibration_snapshot(active_profile_id) - active_control = application_control_session["state"] not in { - "idle", - "completed", - "closed", - "failed", - } - return { + active_control = authority.control_allowed + snapshot: dict[str, Any] = { "contract_version": "missioncore.device-plugin-state/v1alpha2", "phase": phase, "message": message, "devices": devices, + "ble_discovery_generation": ble_discovery_generation, "selected_device_id": selected_device_id, "k1_ip": k1_ip, "connection_mode": connection_mode, + "configured_connection_mode": connection_mode, + "active_connection_mode": active_connection_mode, + "desired_connection_mode": desired_connection_mode, + "desired_connection_mode_revision": desired_connection_mode_revision, "compatibility": { "profile_id": active_profile_id, "decision": "limited" if active_profile_id is not None else "unknown", @@ -695,6 +7487,22 @@ class XgridsK1CompatibilityService: "modeling_control_safety": self._modeling_control_safety.snapshot().as_dict(), "application_control_execution": application_control, "application_control_session": application_control_session, + "physical_command": physical_command, + "active_acquisition_recovery_checkpoint": ( + active_acquisition_checkpoint + ), + "ble_runtime": ble_runtime, + "k1_lifecycle_process_lease": { + "held_by_current_service": bool(lifecycle_process_lease_holders), + "holders": list(lifecycle_process_lease_holders), + "process_lease_quarantined": process_lease_quarantined, + "reason_code": ( + process_lease_quarantine[1] + if process_lease_quarantine is not None + else None + ), + "restart_required": process_lease_quarantined, + }, "device_ref": ( { "device_id": device_id, @@ -719,25 +7527,45 @@ class XgridsK1CompatibilityService: ), "connection_verification": { **connection_verification, - "lease_generation": connection_lease_generation, + # The supervisor is the single authority for lease epochs. + "lease_generation": supervisor_snapshot.lease.generation, }, + "configured_endpoint_probe": configured_endpoint_probe, + "connection_supervisor": supervisor_snapshot.as_dict(), + "connection_policy": connection_policy, + "connection_lifecycle": connection_lifecycle, + "connection_scenario_reset": connection_scenario_reset, + "connection_scenario_reset_pending": connection_scenario_reset_pending, + "connection_reconfiguration": connection_reconfiguration, + "semantic_topology_store": semantic_topology_store, + "device_identity_pin_store": device_identity_pin_store, "network_write_reconciliation": network_write_reconciliation, + "network_mutation_ledger": _network_mutation_ledger_public_snapshot(ledger_snapshot), + "network_provisioning_idempotency": idempotency_snapshot, + "current_device_recovery": current_device_recovery, "sensor_catalog": _sensor_catalog( active_profile_id, device_session_id, camera_preview, + activation_admitted=camera_activation_admitted, ), "device_calibration": device_calibration, "camera_preview": camera_preview, + "connection_recovery": connection_recovery, "live_perception_shadow": self.live_perception_ingress.snapshot(), "acquisition": acquisition, "operations": operation_documents, "last_operation": operation_documents[-1] if operation_documents else None, + "connection_attempt": connection_attempt, "foxglove_ws_url": runtime["foxglove_ws_url"], "foxglove_viewer_url": runtime["foxglove_viewer_url"], "rerun_grpc_url": runtime["rerun_grpc_url"], "viewer_settings": runtime["viewer_settings"], "source_mode": runtime["source_mode"], + # Public runtime lineage fence. Recovery controls must compare this + # value with connection_recovery.runtime_producer_generation before + # issuing the explicit local-only force-finish action. + "producer_generation": runtime["producer_generation"], "metrics": { "pipeline_ms": metrics["mqtt_to_publish_ms"], "end_to_end_ms": metrics["mqtt_to_publish_ms"], @@ -754,41 +7582,1723 @@ class XgridsK1CompatibilityService: **metrics, }, } - - async def scan_ble(self, duration_seconds: float) -> dict[str, Any]: + # State calls are serialized by _acquisition_lifecycle_gate. Stamp the + # completed atomic snapshot last so REST and WebSocket consumers share + # one process-scoped monotonic ordering contract. with self._lock: - if self._provisioning_active: - raise RuntimeError("нельзя запускать BLE-поиск во время настройки Wi-Fi") - self._ble_discovery_generation += 1 - scan_generation = self._ble_discovery_generation - # Discovery results are an in-memory lease, not a durable catalog. - # A new explicit scan invalidates the previous generation before I/O. - self._devices = [] - self._ble_device_last_seen_monotonic = {} - operation, _ = self._operations.begin( - ACTION_DISCOVERY_SCAN, - deadline_seconds=duration_seconds + 10.0, + self._snapshot_revision += 1 + snapshot.update( + { + "snapshot_runtime_started_at_utc": (self._snapshot_runtime_started_at_utc), + "snapshot_runtime_started_monotonic_ns": ( + self._snapshot_runtime_started_monotonic_ns + ), + "snapshot_runtime_id": self._snapshot_runtime_id, + "snapshot_revision": self._snapshot_revision, + } + ) + return snapshot + + def _preflight_network_scenario_reset(self) -> dict[str, object]: + """Validate both durable network journals before local teardown.""" + + ledger_snapshot = self._network_mutation_ledger.snapshot() + if ledger_snapshot.status == "corrupt": + raise NetworkProvisioningConflict( + "журнал настройки сети повреждён; безопасный сброс недоступен", + reason_code="connection-scenario-reset-network-ledger-corrupt", + ) + ledger_record = ledger_snapshot.record + journal = self._network_provisioning_idempotency_journal + journal_snapshot = journal.snapshot() if journal is not None else None + if journal_snapshot is not None and journal_snapshot.status == "corrupt": + raise NetworkProvisioningConflict( + "журнал сетевых операций повреждён; безопасный сброс недоступен", + reason_code="connection-scenario-reset-idempotency-corrupt", + ) + journal_record = journal_snapshot.active_record if journal_snapshot is not None else None + if ledger_record is not None and ledger_record.unresolved: + if journal is None or journal_record is None: + raise NetworkProvisioningConflict( + "активная сетевая операция не имеет доверенного журнала", + reason_code="connection-scenario-reset-idempotency-unavailable", + ) + if journal_record.operation_id != ledger_record.operation_id: + raise NetworkProvisioningConflict( + "сетевые журналы описывают разные активные операции", + reason_code="connection-scenario-reset-network-audit-mismatch", + ) + return { + "ledger_operation_id": ( + ledger_record.operation_id if ledger_record is not None else None + ), + "ledger_revision": (ledger_record.revision if ledger_record is not None else None), + "ledger_unresolved": bool(ledger_record is not None and ledger_record.unresolved), + "journal_operation_id": ( + journal_record.operation_id if journal_record is not None else None + ), + "journal_revision": (journal_record.revision if journal_record is not None else None), + } + + def _retire_network_scenario_for_operator_reset( + self, + preflight: Mapping[str, object], + ) -> dict[str, object]: + """Supersede one old network intent without replaying its write. + + A dispatched write remains recorded with an unknown device-side + outcome. The reset changes only which local scenario owns the next + explicit action; it never claims that credentials were or were not + applied by the K1. + """ + + ledger_snapshot = self._network_mutation_ledger.snapshot() + ledger_record = ledger_snapshot.record + journal = self._network_provisioning_idempotency_journal + journal_snapshot = journal.snapshot() if journal is not None else None + if ledger_snapshot.status == "corrupt" or ( + journal_snapshot is not None and journal_snapshot.status == "corrupt" + ): + raise NetworkProvisioningConflict( + "сетевая история изменилась после проверки сброса", + reason_code="connection-scenario-reset-network-audit-changed", + ) + journal_record = journal_snapshot.active_record if journal_snapshot is not None else None + if ( + bool(preflight.get("ledger_unresolved")) + and ledger_record is not None + and ledger_record.unresolved + and ( + ledger_record.operation_id != preflight.get("ledger_operation_id") + or ledger_record.revision != preflight.get("ledger_revision") + or journal_record is None + or journal_record.operation_id != preflight.get("journal_operation_id") + or journal_record.revision != preflight.get("journal_revision") + ) + ): + raise NetworkProvisioningConflict( + "сетевая история изменилась после проверки сброса", + reason_code="connection-scenario-reset-network-audit-changed", + ) + + ledger_resolution: str | None = None + network_side_effect_status = "none" + if ledger_record is not None and ledger_record.unresolved: + ledger_resolution = ( + "not-dispatched" if ledger_record.stage == "prepared" else "superseded" + ) + network_side_effect_status = ( + "none" if ledger_resolution == "not-dispatched" else "unknown" + ) + self._network_mutation_ledger.resolve( + ledger_record.operation_id, + expected_revision=ledger_record.revision, + resolution=cast(Any, ledger_resolution), + ) + + if journal is not None and journal_record is not None: + side_effect_status = "none" if journal_record.stage == "prepared" else "unknown" + safe_to_retry = side_effect_status == "none" + journal.complete( + journal_record.operation_id, + expected_revision=journal_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="cancelled", + outcome_code="connection-scenario-reset", + error_code="connection-scenario-reset-by-operator", + side_effect_status=cast(Any, side_effect_status), + retryable=safe_to_retry, + safe_to_retry=safe_to_retry, + ), + ) + self._operations.transition_if_pending( + journal_record.operation_id, + "cancelled", + stage_code="connection-scenario-reset", + message_code="connection.mode.scenario_reset", + ) + self._network_write_reconciliation = _network_mutation_reconciliation_projection( + self._network_mutation_ledger.snapshot() ) - self._operations.transition( - operation.operation_id, - "running", - stage_code="ble-discovery", - message_code="discovery.scan.running", - ) - self._set_operation("scanning", "Сканируем все устройства Bluetooth (BLE)…") + return { + "ledger_resolution": ledger_resolution, + "side_effect_status": network_side_effect_status, + } + + def _cleanup_terminal_acquisition_sources_for_scenario_reset( + self, + *, + reason_code: str, + preserve_stop_operation_id: str | None = None, + ) -> bool: + """Seal retained terminal producers without changing device state. + + A failed/interrupted acquisition can be terminal while its evidence + lease, receiver, camera or recovery continuation is still live. The + generic active force-finish helper intentionally does not own this + case. Scenario reset uses this bounded local-only seam before it + retires the old connection lineage; it never publishes STOP, opens + BLE, changes host Wi-Fi or mutates the K1 network. + """ + + runtime_snapshot = self.runtime.snapshot() + camera_snapshot = self.camera_preview.snapshot() + with self._lock: + acquisition = self._acquisition + if acquisition is None or acquisition.state not in TERMINAL_ACQUISITION_STATES: + return False + recording = camera_snapshot.get("recording") + terminal_sources_retained = bool( + self._acquisition_session_lease is not None + or self._operator_manual_acquisition_process_lease_id is not None + or "camera" in self._application_control_process_lease_holders + or self._active_stream_recovery_future is not None + or self._active_stream_recovery_lineage is not None + or runtime_snapshot.get("source_mode") != "idle" + or camera_snapshot.get("phase") != "idle" + or (isinstance(recording, Mapping) and recording.get("active") is True) + ) + if not terminal_sources_retained: + return False + future = self._active_stream_recovery_future + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_state = "force-finishing" + self._active_stream_recovery_reason_code = reason_code + self._active_stream_recovery_lineage = None + self._active_stream_recovery_future = None + self._active_stream_recovery_terminal_outcome = None + + if future is not None: + future.cancel() + + cleanup_error: Exception | None = None try: - result = await scan(duration_seconds) + self._stop_acquisition_sources( + camera_status=( + "complete" + if acquisition.state == "completed" + else "failed" + if acquisition.state == "failed" + else "interrupted" + ), + camera_failure_code=(None if acquisition.state == "completed" else reason_code), + ) + except Exception as exc: + cleanup_error = exc + try: + self._stop_camera_preview_and_release_process_lease() + except Exception as exc: + if cleanup_error is not None: + exc.add_note( + "terminal acquisition cleanup also failed: " + f"{type(cleanup_error).__name__}: {cleanup_error}" + ) + cleanup_error = exc + + self._terminalize_pending_start_for_local_stop_retirement( + acquisition_id=acquisition.acquisition_id, + stop_operation_id=preserve_stop_operation_id, + ) + self._cancel_pending_acquisition_operations( + exclude_operation_id="", + reason_code="superseded-by-connection-scenario-reset", + preserve_operation_ids=( + (preserve_stop_operation_id,) + if preserve_stop_operation_id is not None + else () + ), + ) + with self._lock: + self._active_stream_recovery_state = ( + "blocked" if cleanup_error is not None else "force-finished" + ) + if cleanup_error is not None: + self._active_stream_recovery_reason_code = ( + "connection-scenario-reset-local-cleanup-failed" + ) + if cleanup_error is not None: + raise cleanup_error + return True + + def _cleanup_residual_sources_for_connection_scenario_reset( + self, + *, + reason_code: str, + ) -> None: + """Seal source owners that are not attached to an acquisition row. + + Replay and standalone camera preview can exist without an acquisition, + so neither terminal cleanup nor active force-finish owns them. This + local-only tail also advances the recovery generation before blocking + cleanup, making every late callback from the retired scenario stale. + It never contacts the K1, opens BLE, changes a network or publishes a + physical command. + """ + + with self._lock: + future = self._active_stream_recovery_future + camera_lease_retained = ( + "camera" in self._application_control_process_lease_holders + ) + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_state = "force-finishing" + self._active_stream_recovery_reason_code = reason_code + self._active_stream_recovery_lineage = None + self._active_stream_recovery_future = None + self._active_stream_recovery_terminal_outcome = None + + if future is not None: + future.cancel() + + cleanup_error: Exception | None = None + runtime_snapshot = self.runtime.snapshot() + try: + if ( + runtime_snapshot.get("source_mode") != "idle" + or runtime_snapshot.get("phase") != "idle" + ): + self.runtime.stop() + self._reset_live_data_plane_observation() + except Exception as exc: + cleanup_error = exc + + camera_snapshot = self.camera_preview.snapshot() + recording = camera_snapshot.get("recording") + try: + if ( + camera_lease_retained + or camera_snapshot.get("active_source_id") is not None + or camera_snapshot.get("phase") != "idle" + or camera_snapshot.get("error") is not None + or (isinstance(recording, Mapping) and recording.get("active") is True) + ): + self._stop_camera_preview_and_release_process_lease() + except Exception as exc: + if cleanup_error is not None: + exc.add_note( + "local runtime cleanup also failed: " + f"{type(cleanup_error).__name__}: {cleanup_error}" + ) + cleanup_error = exc + + try: + self._application_control.disarm() + except Exception as exc: + if cleanup_error is not None: + exc.add_note( + "local source cleanup also failed: " + f"{type(cleanup_error).__name__}: {cleanup_error}" + ) + cleanup_error = exc + + with self._lock: + self._live_perception_camera_binding = None + self._active_stream_recovery_state = ( + "blocked" if cleanup_error is not None else "force-finished" + ) + if cleanup_error is not None: + self._active_stream_recovery_reason_code = ( + "connection-scenario-reset-local-cleanup-failed" + ) + if cleanup_error is not None: + raise cleanup_error + + def _acquire_connection_scenario_reset_boundary( + self, + request: DesiredConnectionModeRequest, + *, + deadline: float, + writer_token: object, + ) -> bool: + """Join old owners without retaining a gate that they need to exit. + + Network provisioning deliberately releases and later reacquires the + lifecycle transition gate while it retains the process-wide ``network`` + holder. A reset therefore reserves reconfiguration, transition and + process ownership without blocking first. Only after that exact + prefix is owned does it publish writer priority and wait in bounded + slices for the acquisition gate. New state readers cannot barge ahead, + while an older network owner can still reacquire transition and exit. + """ + + assert request.reset_id is not None + while True: + if self._service_close_requested.is_set(): + raise NetworkProvisioningConflict( + "локальный сервис K1 завершает работу", + reason_code="connection-scenario-reset-service-closing", + ) + with self._lock: + pending_reset = self._connection_scenario_reset_pending + prior_reset = self._connection_scenario_reset + duplicate_reset = bool( + prior_reset is not None + and prior_reset.get("reset_id") == request.reset_id + and prior_reset.get("revision") == request.expected_revision + 1 + and prior_reset.get("desired_mode") == request.connection_mode + and self._desired_connection_mode_revision == request.expected_revision + 1 + and self._desired_connection_mode == request.connection_mode + ) + reset_is_current = bool( + pending_reset is not None + and pending_reset.get("reset_id") == request.reset_id + and pending_reset.get("desired_mode") == request.connection_mode + and pending_reset.get("expected_revision") == request.expected_revision + ) + if duplicate_reset: + # The original owner may have committed while this exact + # response-loss retry was waiting. Replay dominates the now + # cleared pending slot and must not reserve or retire again. + return False + if not reset_is_current: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + if time.monotonic() >= deadline: + raise NetworkProvisioningConflict( + "предыдущая lifecycle-операция K1 не завершилась вовремя", + reason_code="connection-scenario-reset-lifecycle-timeout", + ) + + reconfiguration_acquired = self._connection_reconfiguration_gate.acquire( + blocking=False + ) + lifecycle_acquired = False + process_acquired = False + writer_registered = False + acquisition_acquired = False + ownership_transferred = False + try: + if reconfiguration_acquired: + lifecycle_acquired = self._k1_lifecycle_transition_gate.acquire(blocking=False) + if lifecycle_acquired: + try: + self._acquire_k1_lifecycle_process_lease("network") + process_acquired = True + except ApplicationControlProcessLeaseUnavailable: + process_acquired = False + if process_acquired: + writer_registered = self._try_register_acquisition_lifecycle_writer( + writer_token + ) + if not writer_registered: + self._release_k1_lifecycle_process_lease("network") + process_acquired = False + self._k1_lifecycle_transition_gate.release() + lifecycle_acquired = False + self._connection_reconfiguration_gate.release() + reconfiguration_acquired = False + remaining = deadline - time.monotonic() + if remaining > 0: + with self._acquisition_lifecycle_admission: + if self._acquisition_lifecycle_writer_token is not None: + self._acquisition_lifecycle_admission.wait( + timeout=min(0.05, remaining) + ) + continue + while True: + if self._service_close_requested.is_set(): + raise NetworkProvisioningConflict( + "локальный сервис K1 завершает работу", + reason_code="connection-scenario-reset-service-closing", + ) + with self._lock: + current_pending = self._connection_scenario_reset_pending + current_reset = self._connection_scenario_reset + duplicate_reset = bool( + current_reset is not None + and current_reset.get("reset_id") == request.reset_id + and current_reset.get("revision") + == request.expected_revision + 1 + and current_reset.get("desired_mode") + == request.connection_mode + and self._desired_connection_mode_revision + == request.expected_revision + 1 + and self._desired_connection_mode == request.connection_mode + ) + reset_is_current = bool( + current_pending is not None + and current_pending.get("reset_id") == request.reset_id + and current_pending.get("desired_mode") + == request.connection_mode + and current_pending.get("expected_revision") + == request.expected_revision + ) + if duplicate_reset: + return False + if not reset_is_current: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise NetworkProvisioningConflict( + "предыдущая lifecycle-операция K1 не завершилась вовремя", + reason_code="connection-scenario-reset-lifecycle-timeout", + ) + acquisition_acquired = self._acquisition_lifecycle_gate.acquire( + timeout=min(0.05, remaining) + ) + if acquisition_acquired: + if self._service_close_requested.is_set(): + raise NetworkProvisioningConflict( + "локальный сервис K1 завершает работу", + reason_code="connection-scenario-reset-service-closing", + ) + with self._lock: + final_pending = self._connection_scenario_reset_pending + final_reset = self._connection_scenario_reset + duplicate_reset = bool( + final_reset is not None + and final_reset.get("reset_id") == request.reset_id + and final_reset.get("revision") + == request.expected_revision + 1 + and final_reset.get("desired_mode") + == request.connection_mode + and self._desired_connection_mode_revision + == request.expected_revision + 1 + and self._desired_connection_mode + == request.connection_mode + ) + reset_is_current = bool( + final_pending is not None + and final_pending.get("reset_id") == request.reset_id + and final_pending.get("desired_mode") + == request.connection_mode + and final_pending.get("expected_revision") + == request.expected_revision + ) + if duplicate_reset: + return False + if not reset_is_current: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + if time.monotonic() >= deadline: + raise NetworkProvisioningConflict( + "предыдущая lifecycle-операция K1 не завершилась вовремя", + reason_code="connection-scenario-reset-lifecycle-timeout", + ) + # The caller releases the four exact boundaries. + # Writer admission remains registered through its + # post-finally response snapshot and is released by + # ``select_connection_mode`` on every exit path. + ownership_transferred = True + return True + finally: + if not ownership_transferred: + if acquisition_acquired: + self._acquisition_lifecycle_gate.release() + if writer_registered: + self._unregister_acquisition_lifecycle_writer(writer_token) + if process_acquired: + self._release_k1_lifecycle_process_lease("network") + if lifecycle_acquired: + self._k1_lifecycle_transition_gate.release() + if reconfiguration_acquired: + self._connection_reconfiguration_gate.release() + + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(min(0.05, remaining)) + + def select_connection_mode( + self, + request: DesiredConnectionModeRequest, + ) -> dict[str, Any]: + """Queue reset intent ahead of old lifecycle work, then serialize it.""" + + reset_id = request.reset_id if request.reset_scenario is True else None + reset_deadline = ( + time.monotonic() + CONNECTION_SCENARIO_RESET_DRAIN_TIMEOUT_SECONDS + if reset_id is not None + else None + ) + reset_writer_token = object() if reset_id is not None else None + if reset_id is not None: + with self._lock: + acquisition = self._acquisition + acquisition_id = ( + acquisition.acquisition_id if acquisition is not None else None + ) + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "сброс сценария недоступен, пока read-only классификация STOP " + "применяется к приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + if self._service_close_requested.is_set(): + raise NetworkProvisioningConflict( + "локальный сервис K1 завершает работу", + reason_code="connection-scenario-reset-service-closing", + ) + if reset_id is not None: + with self._lock: + current_pending = self._connection_scenario_reset_pending + committed_reset = self._connection_scenario_reset + if ( + committed_reset is not None + and committed_reset.get("reset_id") == reset_id + and ( + committed_reset.get("desired_mode") != request.connection_mode + or committed_reset.get("request_revision") != request.expected_revision + ) + ): + raise NetworkProvisioningConflict( + "reset_id уже привязан к другому сценарию подключения", + reason_code="connection-scenario-reset-id-conflict", + ) + if ( + current_pending is not None + and current_pending.get("reset_id") == reset_id + and ( + current_pending.get("desired_mode") != request.connection_mode + or current_pending.get("expected_revision") != request.expected_revision + ) + ): + raise NetworkProvisioningConflict( + "reset_id уже привязан к другому сценарию подключения", + reason_code="connection-scenario-reset-id-conflict", + ) + if not ( + current_pending is not None + and current_pending.get("status") == "retiring-local-session" + ): + self._connection_scenario_reset_pending = { + "reset_id": reset_id, + "desired_mode": request.connection_mode, + "expected_revision": request.expected_revision, + "status": "waiting-for-local-lifecycle", + "device_command_performed": False, + "network_write_performed": False, + "automatic_scan": False, + } + if reset_deadline is None: + return self._select_connection_mode_owned( + request, + reset_deadline=None, + reset_writer_token=None, + ) + try: + try: + self._select_connection_mode_owned( + request, + reset_deadline=reset_deadline, + reset_writer_token=reset_writer_token, + ) + finally: + if reset_id is not None: + with self._lock: + if ( + self._connection_scenario_reset_pending is not None + and self._connection_scenario_reset_pending.get("reset_id") == reset_id + ): + self._connection_scenario_reset_pending = None + # `_select_connection_mode_owned()` constructs its snapshot before + # the outer retirement marker is cleared. Publish one post-finally + # local snapshot while this exact writer still has priority. + return self.state() + finally: + if reset_writer_token is not None: + self._unregister_acquisition_lifecycle_writer(reset_writer_token) + + def _select_connection_mode_owned( + self, + request: DesiredConnectionModeRequest, + *, + reset_deadline: float | None, + reset_writer_token: object | None, + ) -> dict[str, Any]: + """Commit one mode draft or one explicit local scenario reset. + + ``reset_scenario`` is the operator's single reset intent. It may seal + the local receiver, camera and control socket and may retire old audit + ownership, but it never contacts the K1, switches host Wi-Fi, repeats + a network write, publishes START/STOP, or launches a BLE scan. + """ + + if request.reset_scenario is True: + with self._lock: + pending_reset = self._connection_scenario_reset_pending + prior_reset = self._connection_scenario_reset + reset_superseded = bool( + pending_reset is None + or pending_reset.get("reset_id") != request.reset_id + or pending_reset.get("desired_mode") != request.connection_mode + or pending_reset.get("expected_revision") != request.expected_revision + ) + duplicate_reset = bool( + prior_reset is not None + and prior_reset.get("reset_id") == request.reset_id + and prior_reset.get("revision") == request.expected_revision + 1 + and prior_reset.get("desired_mode") == request.connection_mode + and self._desired_connection_mode_revision == request.expected_revision + 1 + and self._desired_connection_mode == request.connection_mode + ) + if duplicate_reset: + # A timed-out HTTP response may be retried with the old CAS. + # Converge on the already committed reset instead of retiring + # whatever explicit new scenario may since have started. + return self.state() + if reset_superseded: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + + # An ordinary local mode draft joins reconfiguration -> lifecycle with + # non-blocking acquisition, giving it or an admitted Connect one exact + # winner. An explicit reset uses the stronger all-or-nothing boundary + # above: it never sleeps while retaining a partial gate set that an old + # network owner needs in order to finish and release its process lease. + reset_wait = request.reset_scenario is True and reset_deadline is not None + acquisition_gate_acquired = False + network_process_lease_acquired = False + if reset_wait: + assert reset_deadline is not None + assert reset_writer_token is not None + boundary_acquired = self._acquire_connection_scenario_reset_boundary( + request, + deadline=reset_deadline, + writer_token=reset_writer_token, + ) + if not boundary_acquired: + return self.state() + acquisition_gate_acquired = True + reconfiguration_gate_acquired = True + lifecycle_gate_acquired = True + network_process_lease_acquired = True + else: + reconfiguration_gate_acquired = self._connection_reconfiguration_gate.acquire( + blocking=False + ) + if not reconfiguration_gate_acquired: + raise NetworkProvisioningConflict( + "способ подключения нельзя изменить во время lifecycle-перехода K1", + reason_code="connection-mode-selection-lifecycle-busy", + ) + if not reset_wait: + lifecycle_gate_acquired = self._k1_lifecycle_transition_gate.acquire(blocking=False) + if not lifecycle_gate_acquired: + self._connection_reconfiguration_gate.release() + raise NetworkProvisioningConflict( + "способ подключения нельзя изменить во время lifecycle-перехода K1", + reason_code="connection-mode-selection-lifecycle-busy", + ) + try: + if not self._wait_for_connection_monitor_quiescence(): + raise NetworkProvisioningConflict( + "фоновая проверка подключения не завершилась вовремя", + reason_code="connection-mode-selection-lifecycle-busy", + ) + with self._lock: + if request.reset_scenario is True: + pending_reset = self._connection_scenario_reset_pending + if pending_reset is None or pending_reset.get("reset_id") != request.reset_id: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + pending_reset["status"] = "retiring-local-session" + if ( + request.reset_scenario is not True + and self._connection_reconfiguration_intent is not None + ): + raise NetworkProvisioningConflict( + "сначала завершите или отмените переподключение устройства", + reason_code="connection-reconfiguration-active", + ) + if request.expected_revision != self._desired_connection_mode_revision: + raise NetworkProvisioningConflict( + "выбранный способ подключения уже изменился; обновите состояние", + reason_code="connection-mode-draft-revision-conflict", + ) + changes_mode = request.connection_mode != self._desired_connection_mode + desired_connection_mode = self._desired_connection_mode + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + if request.reset_scenario is True: + assert reset_deadline is not None + network_preflight = self._preflight_network_scenario_reset() + physical_retirement_id = ( + "scenario-reset-" + f"{self._snapshot_runtime_id}-" + f"{request.expected_revision}-" + f"{request.connection_mode}" + ) + try: + physical_preflight = ( + self._physical_command_coordinator.preview_connection_scenario_retirement( + retirement_id=physical_retirement_id, + ) + ) + except RuntimeError as exc: + raise NetworkProvisioningConflict( + "физическая история K1 не допускает безопасный сброс", + reason_code="connection-scenario-reset-physical-audit-blocked", + ) from exc + with self._lock: + pending_reset = self._connection_scenario_reset_pending + if pending_reset is None or pending_reset.get("reset_id") != request.reset_id: + raise NetworkProvisioningConflict( + "выбран более новый сценарий подключения", + reason_code="connection-scenario-reset-superseded", + ) + if request.expected_revision != self._desired_connection_mode_revision: + raise NetworkProvisioningConflict( + "выбранный способ подключения уже изменился; обновите состояние", + reason_code="connection-mode-draft-revision-conflict", + ) + runtime_before_reset = self.runtime.snapshot() + control_before_reset = self._application_control_session.snapshot() + local_physical_activity_unresolved = bool( + ( + acquisition is not None + and acquisition.state + in { + "awaiting_external_start", + "starting", + "acquiring", + "awaiting_external_stop", + "stopping", + "finalizing", + } + ) + or runtime_before_reset.get("source_mode") == "live" + or control_before_reset.get("state") + in { + "start-requested", + "initializing", + "scanning", + "stop-requested", + "stopping", + "awaiting-standby-confirmation", + } + ) + scenario_control_retirement = ( + self._fence_local_control_for_acquisition_retirement( + acquisition_id=( + acquisition.acquisition_id if acquisition is not None else None + ), + ) + ) + self._supersede_control_bootstrap_continuation() + self._cleanup_terminal_acquisition_sources_for_scenario_reset( + reason_code="connection-scenario-reset-by-operator", + preserve_stop_operation_id=scenario_control_retirement[1], + ) + self._force_finish_active_acquisition_locally( + reason_code="connection-scenario-reset-by-operator", + require_recovery=False, + fenced_control_retirement=scenario_control_retirement, + ) + control = dict(self._application_control_session.snapshot()) + control_state = str(control.get("state") or "unknown") + if control_state not in {"idle", "completed", "closed", "failed"}: + self._application_control_session.close() + control = dict(self._application_control_session.snapshot()) + control_state = str(control.get("state") or "unknown") + if control_state not in {"idle", "completed", "closed", "failed"}: + raise NetworkProvisioningConflict( + "локальный control worker ещё завершает предыдущий сценарий", + reason_code="connection-scenario-reset-control-retirement-pending", + ) + if control_state != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + self._cleanup_residual_sources_for_connection_scenario_reset( + reason_code="connection-scenario-reset-by-operator", + ) + + network_retirement = self._retire_network_scenario_for_operator_reset( + network_preflight + ) + physical_retirement = self._physical_command_coordinator.retire_connection_scenario( + retirement_id=physical_retirement_id, + expected_revision=cast( + int | None, + physical_preflight.get("record_revision"), + ), + ) + physical_disposition = str(physical_retirement.get("disposition") or "unknown") + retired_physical_operation_id = physical_preflight.get("operation_id") + with self._lock: + if ( + isinstance(retired_physical_operation_id, str) + and self._acquisition_stop_operation_id + == retired_physical_operation_id + ): + self._acquisition_stop_operation_id = None + dispatch_lineage = self._prepared_stop_dispatch_lineage + if ( + dispatch_lineage is not None + and dispatch_lineage.stop_operation_id + == retired_physical_operation_id + ): + self._prepared_stop_dispatch_lineage = None + recovery_owner = self._prepared_stop_recovery_owner + if ( + recovery_owner is not None + and recovery_owner.lineage.stop_operation_id + == retired_physical_operation_id + ): + self._prepared_stop_recovery_owner = None + self._classified_stop_rebind_pending = None + self._classified_stop_rebind_inflight = None + previous_device_may_continue_scanning = ( + physical_disposition == ("operator-retired-outcome-unknown") + or ( + local_physical_activity_unresolved + and scenario_control_retirement[0] != "standby-confirmed" + ) + ) + reset_operation_documents = self._operations.snapshot(limit=128) + reset_operation_sequence = max( + ( + cast(int, item.get("sequence")) + # Preserve the legacy public marker value, which was + # calculated from OperationJournal.snapshot()'s default + # twenty-record window. Identity cutoff below uses the + # complete bounded journal independently. + for item in reset_operation_documents[-20:] + if isinstance(item.get("sequence"), int) + ), + default=0, + ) + reset_network_operation_ids = frozenset( + cast(str, item.get("operation_id")) + for item in reset_operation_documents + if item.get("action") == ACTION_NETWORK_PROVISION + and isinstance(item.get("operation_id"), str) + ) + reset_revision = request.expected_revision + 1 + reset_intent_id = ( + "connection-scenario-reset-" + + hashlib.sha256(cast(str, request.reset_id).encode("utf-8")).hexdigest()[:24] + ) + # Finish every fallible local tail before publishing the + # committed replay marker. An exception here leaves the + # request revision unchanged, so the same reset_id can safely + # resume the idempotent durable retirement on retry. + self._connection_supervisor.set_intent( + intent_id=reset_intent_id, + requested_mode=request.connection_mode, + ) + self._retire_ephemeral_device_binding_for_new_intent() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + reset_supervisor_lease_generation = ( + self._connection_supervisor.snapshot().lease.generation + ) + with self._lock: + if request.expected_revision != self._desired_connection_mode_revision: + raise NetworkProvisioningConflict( + "выбранный способ подключения уже изменился; обновите состояние", + reason_code="connection-mode-draft-revision-conflict", + ) + self._desired_connection_mode = request.connection_mode + self._desired_connection_mode_revision = reset_revision + self._clear_connection_reconfiguration_locked() + # The generic new-intent helper retains the provisional + # logical id for pre-write diagnostics. A committed + # operator scenario reset instead publishes no old device + # binding at all; the per-transport identity cache remains + # available for a later explicit selection. + self._device_id = None + self._operation_phase = None + self._operation_message = None + self._connection_verification = { + "status": "not-probed", + "lease_state": "disconnected", + "lease_generation": reset_supervisor_lease_generation, + "endpoint_validation": "not-performed", + "network_reachability": "unknown", + "observed_at": None, + } + self._configured_endpoint_probe = { + "schema_version": ( + "missioncore.xgrids-k1-configured-endpoint-probe/v1" + ), + "status": "not-probed", + "target_source": None, + "connection_mode": None, + "endpoint": None, + "transport_ref": None, + "intent_id": None, + "semantic_revision": None, + "host_route_available": None, + "host_route_class": None, + "tcp_reachable": None, + "identity_validation": "not-performed", + "control_authority_granted": False, + "ble_operation_performed": False, + "network_mutation_performed": False, + "automatic_retry": False, + "observed_at": None, + "reason_code": None, + } + self._active_stream_recovery_lineage = None + self._active_stream_recovery_future = None + self._active_stream_recovery_state = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = None + self._active_stream_recovery_started_at_utc = None + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None + self._connection_scenario_reset_network_operation_ids = ( + reset_network_operation_ids + ) + self._connection_scenario_reset_retired_transport_authority = ( + self._scenario_reset_retired_authority_from_commit( + reset_id=cast(str, request.reset_id), + reset_revision=reset_revision, + desired_mode=request.connection_mode, + minimum_discovery_generation=( + self._ble_discovery_generation + 1 + ), + physical_preflight=physical_preflight, + physical_retirement=physical_retirement, + ) + ) + self._connection_scenario_reset = { + "reset_id": request.reset_id, + "request_revision": request.expected_revision, + "revision": reset_revision, + "desired_mode": request.connection_mode, + "active": True, + "settled_by_discovery_generation": None, + "local_session_closed": True, + "previous_device_may_continue_scanning": ( + previous_device_may_continue_scanning + ), + "physical_disposition": physical_disposition, + "network_disposition": network_retirement["ledger_resolution"], + "device_command_performed": False, + "network_write_performed": False, + "automatic_scan": False, + "operation_sequence": reset_operation_sequence, + } + elif changes_mode: + runtime = self.runtime.snapshot() + control = self._application_control_session.snapshot() + physical = self._physical_command_coordinator.snapshot() + acquisition_state = acquisition.state if acquisition is not None else None + selection_reasons = _connection_mode_selection_reason_codes( + acquisition_state=acquisition_state, + acquisition_lease_retained=acquisition_lease is not None, + acquisition_start_operation_id=acquisition_start_operation_id, + runtime=runtime, + physical_command=physical, + control_state=str(control.get("state") or "unknown"), + desired_connection_mode=desired_connection_mode, + requested_connection_mode=request.connection_mode, + ) + if "connection-mode-selection-physical-state-unsafe" in selection_reasons: + raise NetworkProvisioningConflict( + "способ подключения нельзя изменить во время запуска или записи K1", + reason_code="connection-mode-selection-physical-state-unsafe", + ) + if "connection-mode-selection-control-state-unsafe" in selection_reasons: + raise NetworkProvisioningConflict( + "способ подключения нельзя изменить после начала START-диалога K1", + reason_code="connection-mode-selection-control-state-unsafe", + ) + with self._lock: + # START and Connect also own the lifecycle gate while they + # move these facts. Repeating the local identity check + # prevents a future writer that does not use `_lock` from + # turning this safe draft commit into a stale overwrite. + if request.expected_revision != self._desired_connection_mode_revision: + raise NetworkProvisioningConflict( + "выбранный способ подключения уже изменился; обновите состояние", + reason_code="connection-mode-draft-revision-conflict", + ) + self._desired_connection_mode = request.connection_mode + self._desired_connection_mode_revision += 1 + finally: + if network_process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + self._k1_lifecycle_transition_gate.release() + self._connection_reconfiguration_gate.release() + if acquisition_gate_acquired: + self._acquisition_lifecycle_gate.release() + return self.state() + + async def prepare_connection_reconfiguration( + self, + request: PrepareConnectionReconfigurationRequest, + ) -> dict[str, Any]: + """Prepare/cancel one explicit local-only Bridge reconfiguration. + + This method never opens BLE, changes Wi-Fi, reconnects MQTT, or + publishes START/STOP. It only retires a safe pre-START local owner, + invalidates stale candidate authority and records the next operator + intent under a stable process-local CAS. + """ + + network_process_lease_acquired = False + reconfiguration_gate_acquired = False + transition_gate_acquired = False + if not self._connection_reconfiguration_gate.acquire(blocking=False): + raise NetworkProvisioningConflict( + "другой диалог переподключения уже изменяет локальное состояние", + reason_code="connection-reconfiguration-lifecycle-busy", + ) + reconfiguration_gate_acquired = True + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + self._connection_reconfiguration_gate.release() + raise NetworkProvisioningConflict( + "другая lifecycle-операция устройства ещё выполняется", + reason_code="connection-reconfiguration-lifecycle-busy", + ) + transition_gate_acquired = True + try: + if not await self._await_connection_monitor_quiescence(): + raise NetworkProvisioningConflict( + "фоновая проверка подключения не завершилась вовремя", + reason_code="connection-reconfiguration-lifecycle-busy", + ) + current_state = self.state() + lifecycle = current_state["connection_lifecycle"] + active_binding_key = lifecycle.get("active_binding_key") + active_binding = lifecycle.get("active_binding") + with self._lock: + current_revision = self._connection_reconfiguration_revision + current_intent_id = self._connection_reconfiguration_intent_id + current_intent = self._connection_reconfiguration_intent + desired_mode_revision = self._desired_connection_mode_revision + if ( + request.expected_reconfiguration_revision != current_revision + or request.expected_reconfiguration_intent_id != current_intent_id + ): + raise NetworkProvisioningConflict( + "диалог переподключения уже изменился в другой вкладке", + reason_code="connection-reconfiguration-revision-conflict", + ) + if request.expected_desired_mode_revision != desired_mode_revision: + raise NetworkProvisioningConflict( + "способ подключения изменился до переподключения", + reason_code="connection-mode-draft-revision-conflict", + ) + if request.expected_active_binding_key != active_binding_key: + raise NetworkProvisioningConflict( + "активное подключение изменилось до переподключения", + reason_code="connection-reconfiguration-binding-conflict", + ) + + if request.intent == "cancel": + if current_intent is None: + raise NetworkProvisioningConflict( + "активного диалога переподключения уже нет", + reason_code="connection-reconfiguration-not-active", + ) + with self._lock: + if ( + self._connection_reconfiguration_revision != current_revision + or self._connection_reconfiguration_intent_id != current_intent_id + ): + raise NetworkProvisioningConflict( + "диалог переподключения уже изменился в другой вкладке", + reason_code="connection-reconfiguration-revision-conflict", + ) + self._clear_connection_reconfiguration_locked() + self._k1_lifecycle_transition_gate.release() + transition_gate_acquired = False + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + return self.state() + + required_transport_ref: str | None = None + if request.intent == "change-network": + candidate_transport_ref = ( + active_binding.get("transport_ref") + if isinstance(active_binding, Mapping) + else None + ) + candidate_connection_mode = ( + active_binding.get("connection_mode") + if isinstance(active_binding, Mapping) + else None + ) + if not ( + isinstance(candidate_transport_ref, str) + and candidate_transport_ref.strip() + and candidate_connection_mode == "bridge" + ): + # Validate the exact authoritative target before acquiring + # the process lease or retiring PREPARE/control. A stale + # configured selection must have zero collateral effect. + raise NetworkProvisioningConflict( + "текущее Bridge-устройство недоступно для изменения сети", + reason_code=("connection-reconfiguration-current-device-unavailable"), + ) + required_transport_ref = candidate_transport_ref + + safety_reasons = self._connection_reconfiguration_safety_reasons( + intent=request.intent, + transition_gate_owned=True, + active_binding=active_binding, + ) + if safety_reasons: + reason_code = safety_reasons[0] + raise NetworkProvisioningConflict( + "переподключение недоступно до завершения текущего состояния устройства", + reason_code=reason_code, + ) + + try: + # Probe and hold the cross-process lifecycle fence before + # retiring any local owner. Another Mission Core process must + # make Prepare fail without collateral teardown, not defer the + # surprise to the following Scan. + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + except ApplicationControlProcessLeaseUnavailable as exc: + raise NetworkProvisioningConflict( + "другой процесс уже управляет lifecycle устройства", + reason_code="connection-reconfiguration-process-lease-busy", + ) from exc + + runtime_idle = self.runtime.snapshot().get("source_mode") == "idle" + with self._lock: + acquisition = self._acquisition + locally_prepared = bool( + acquisition is not None + and acquisition.state == "prepared" + and self._acquisition_session_lease is None + and self._acquisition_start_operation_id is None + and runtime_idle + ) + if locally_prepared: + assert acquisition is not None + with self._lock: + if ( + self._acquisition is not acquisition + or acquisition.state != "prepared" + or self._acquisition_session_lease is not None + or self._acquisition_start_operation_id is not None + ): + raise NetworkProvisioningConflict( + "подготовленный приём изменился во время переподключения", + reason_code="connection-reconfiguration-acquisition-changed", + ) + acquisition.transition( + "aborted", + message_code="acquisition.reconfiguration_aborted", + result={ + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "superseded-by-connection-reconfiguration", + }, + ) + + control = dict(self._application_control_session.snapshot()) + control_state = str(control.get("state") or "unknown") + if control_state in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + await self._retire_prestart_control_for_mode_transition() + elif control_state != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + else: + self._reconcile_application_control_process_lease(control) + + winning_intent_id = f"connection-reconfigure-{uuid4()}" + expected_vendor_device_id = ( + self._expected_vendor_device_id(required_transport_ref) + if request.intent == "change-network" and required_transport_ref is not None + else None + ) + self._connection_supervisor.set_intent( + intent_id=winning_intent_id, + requested_mode="bridge", + expected_device_id=expected_vendor_device_id, + ) + self._retire_ephemeral_device_binding_for_new_intent() + with self._lock: + # Local generation invalidation is the candidate-level fence; + # revision + random intent id are the semantic fence carried by + # subsequent Verify/Connect requests. + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._connection_reconfiguration_revision += 1 + self._connection_reconfiguration_intent_id = winning_intent_id + self._connection_reconfiguration_intent = request.intent + self._connection_reconfiguration_required_transport_ref = ( + required_transport_ref if request.intent == "change-network" else None + ) + self._connection_reconfiguration_required_connection_mode = "bridge" + self._connection_reconfiguration_minimum_discovery_generation = ( + self._ble_discovery_generation + 1 + ) + self._connection_reconfiguration_prepared_at = _utc_now_iso() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + finally: + if network_process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + return self.state() + + def _require_connection_mode_draft_for_connect( + self, + request: ConnectRequest, + ) -> None: + """Fence Connect against a stale dropdown without performing I/O.""" + + with self._lock: + desired = self._desired_connection_mode + revision = self._desired_connection_mode_revision + discovery_generation = self._ble_discovery_generation + if request.expected_mode_revision != revision: + raise NetworkProvisioningConflict( + "способ подключения изменился до начала операции; обновите состояние", + reason_code="connection-mode-draft-revision-conflict", + ) + if request.connection_mode != desired: + raise NetworkProvisioningConflict( + "запрошенное подключение не совпадает с выбранным способом", + reason_code="connection-mode-draft-mismatch", + ) + if request.expected_discovery_generation != discovery_generation: + raise NetworkProvisioningConflict( + "результаты Bluetooth-поиска изменились; выберите K1 заново", + reason_code="network-provision-discovery-generation-conflict", + ) + + def _require_current_connection_reconfiguration_target( + self, + *, + expected_revision: int | None, + expected_intent_id: str | None, + device_id: str | None, + connection_mode: ConnectionMode | None, + source: str, + discovery_generation: int | None, + ) -> None: + """Fence Verify/Connect against a cancelled or superseded dialogue.""" + + with self._lock: + current_revision = self._connection_reconfiguration_revision + current_intent_id = self._connection_reconfiguration_intent_id + current_intent = self._connection_reconfiguration_intent + required_transport_ref = self._connection_reconfiguration_required_transport_ref + required_connection_mode = self._connection_reconfiguration_required_connection_mode + minimum_generation = self._connection_reconfiguration_minimum_discovery_generation + current_discovery_generation = self._ble_discovery_generation + + if current_intent is None: + if expected_intent_id is not None or ( + expected_revision is not None and expected_revision != current_revision + ): + raise NetworkProvisioningConflict( + "диалог переподключения уже завершён или отменён", + reason_code="connection-reconfiguration-revision-conflict", + ) + return + if expected_revision != current_revision or expected_intent_id != current_intent_id: + raise NetworkProvisioningConflict( + "диалог переподключения уже изменился в другой вкладке", + reason_code="connection-reconfiguration-revision-conflict", + ) + if source != "fresh-scan": + raise NetworkProvisioningConflict( + "переподключение требует нового явного Bluetooth-поиска", + reason_code="connection-reconfiguration-fresh-scan-required", + ) + if ( + discovery_generation is None + or minimum_generation is None + or discovery_generation != current_discovery_generation + or discovery_generation < minimum_generation + ): + raise NetworkProvisioningConflict( + "результаты Bluetooth-поиска не принадлежат текущему переподключению", + reason_code="connection-reconfiguration-discovery-conflict", + ) + if connection_mode != required_connection_mode: + raise NetworkProvisioningConflict( + "переподключение зафиксировано в режиме Bridge", + reason_code="connection-reconfiguration-target-mismatch", + ) + if current_intent == "change-network" and not ( + isinstance(device_id, str) + and isinstance(required_transport_ref, str) + and physical_transport_ref_comparison_key(device_id) + == physical_transport_ref_comparison_key(required_transport_ref) + ): + raise NetworkProvisioningConflict( + "изменение сети привязано к исходному выбранному устройству", + reason_code="connection-reconfiguration-target-mismatch", + ) + + def _require_physical_command_network_mutation_allowed(self) -> None: + """Fail closed before Connect can cross any GATT/Wi-Fi side effect.""" + + physical = self._physical_command_coordinator.snapshot() + status = str(physical.get("status") or "unknown") + if status == "corrupt": + raise NetworkProvisioningConflict( + "журнал физических команд K1 повреждён; смена сети запрещена", + reason_code="physical-command-ledger-corrupt", + ) + if status not in {"empty", "resolved", "unresolved"}: + raise NetworkProvisioningConflict( + "журнал физических команд K1 недоступен; смена сети запрещена", + reason_code="physical-command-ledger-unavailable", + ) + if _physical_command_reports_active(physical): + raise NetworkProvisioningConflict( + "K1 подтверждён в активном сканировании; сначала выполните явный STOP", + reason_code="physical-device-already-active", + ) + if ( + physical.get("requires_reconciliation") is True + or physical.get("resolved_active_recovery_required") is True + or _physical_command_prepared_stop_ready_successor_pending(physical) + or status == "unresolved" + ): + raise NetworkProvisioningConflict( + "сначала выполните read-only проверку незавершённой команды K1", + reason_code="physical-command-reconciliation-required", + ) + + def _require_desired_mode_matches_active(self) -> None: + with self._lock: + desired = self._desired_connection_mode + configured = self._connection_mode + # The legacy explicit-host acquisition path has no plugin-managed + # network topology to switch. It remains available for replay and + # compatibility callers; once Connect has configured a topology, + # however, START authority is exact and must match the local draft. + if configured is None: + return + supervisor = self._connection_supervisor.snapshot() + if supervisor.intent is None: + return + active = ( + configured + if configured is not None + and supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.requested_mode == configured + else None + ) + if active is None or desired != active: + raise NetworkProvisioningConflict( + "выбран другой способ подключения; сначала завершите переключение K1", + reason_code="connection-mode-switch-pending", + ) + + async def _prepare_pending_mode_scan(self) -> None: + """Commit a selected pre-START mode switch using local teardown only.""" + + with self._lock: + desired = self._desired_connection_mode + configured = self._connection_mode + if configured is None or desired == configured: + return + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise BleDiscoveryUnavailable( + "другая lifecycle-операция K1 ещё выполняется", + reason_code="connection-mode-switch-lifecycle-busy", + ) + try: + if not await self._await_connection_monitor_quiescence(): + raise BleDiscoveryUnavailable( + "фоновая проверка подключения не завершилась вовремя", + reason_code="connection-mode-switch-lifecycle-busy", + ) + # Re-sample every physical/local fact only after becoming the + # single lifecycle winner. START uses the same gate, so a prepared + # local receiver cannot become a physical command while this + # mode-switch commit is deciding whether it is safe to retire. + with self._lock: + desired = self._desired_connection_mode + configured = self._connection_mode + acquisition = self._acquisition + acquisition_lease = self._acquisition_session_lease + acquisition_start_operation_id = self._acquisition_start_operation_id + if configured is None or desired == configured: + return + runtime = self.runtime.snapshot() + control = self._application_control_session.snapshot() + physical = self._physical_command_coordinator.snapshot() + physical_start_active = _physical_command_reports_active(physical) + locally_prepared_acquisition = bool( + acquisition is not None + and acquisition.state == "prepared" + and acquisition_lease is None + and acquisition_start_operation_id is None + and runtime.get("source_mode") == "idle" + ) + if ( + acquisition_lease is not None + or ( + acquisition is not None + and acquisition.state not in TERMINAL_ACQUISITION_STATES + and not locally_prepared_acquisition + ) + or runtime.get("source_mode") != "idle" + or physical.get("requires_reconciliation") is True + or physical.get("resolved_active_recovery_required") is True + or _physical_command_prepared_stop_ready_successor_pending(physical) + or physical.get("status") == "corrupt" + or physical_start_active + ): + raise BleDiscoveryUnavailable( + "переключение способа связи недоступно до завершения текущей записи", + reason_code="connection-mode-switch-physical-state-unsafe", + ) + control_state = str(control.get("state") or "unknown") + if control_state not in { + "idle", + "completed", + "closed", + "failed", + "connection-ready", + "workspace-ready", + "project-ready", + }: + raise BleDiscoveryUnavailable( + "переключение способа связи недоступно после начала управляющего диалога", + reason_code="connection-mode-switch-control-state-unsafe", + ) + if control_state in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + await self._retire_prestart_control_for_mode_transition() + elif control_state != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + if locally_prepared_acquisition: + # PREPARE has allocated only local metadata/evidence paths. No + # receiver, physical START, or device mutation exists, so the + # explicit Scan commit can retire it without publishing STOP. + assert acquisition is not None + with self._lock: + if ( + self._acquisition is not acquisition + or acquisition.state != "prepared" + or self._acquisition_session_lease is not None + or self._acquisition_start_operation_id is not None + ): + raise BleDiscoveryUnavailable( + "состояние подготовленного приёма изменилось во время переключения", + reason_code="connection-mode-switch-acquisition-changed", + ) + acquisition.transition( + "aborted", + message_code="acquisition.mode_switch_aborted", + result={ + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "superseded-by-connection-mode-switch", + }, + ) + self._connection_supervisor.set_intent( + intent_id=(f"mode-switch-scan-{self._desired_connection_mode_revision}-{uuid4()}"), + requested_mode=desired, + ) + self._retire_ephemeral_device_binding_for_new_intent() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + finally: + self._k1_lifecycle_transition_gate.release() + + async def scan_ble(self, request: BleScanRequest | float) -> dict[str, Any]: + scenario_reset_at_entry: tuple[str, int, ConnectionMode] | None = None + with self._lock: + scenario_reset_pending = self._connection_scenario_reset_pending is not None + scenario_reset = self._connection_scenario_reset + scenario_reset_id = ( + scenario_reset.get("reset_id") + if isinstance(scenario_reset, Mapping) + else None + ) + scenario_reset_revision = ( + scenario_reset.get("revision") + if isinstance(scenario_reset, Mapping) + else None + ) + scenario_reset_mode = ( + scenario_reset.get("desired_mode") + if isinstance(scenario_reset, Mapping) + else None + ) + if ( + scenario_reset is not None + and isinstance(scenario_reset_id, str) + and isinstance(scenario_reset_revision, int) + and scenario_reset_mode + in {"bridge", "quick-connect", "direct-connect"} + and scenario_reset_revision == self._desired_connection_mode_revision + and scenario_reset_mode == self._desired_connection_mode + ): + # Capture before any await or lifecycle admission. A Scan + # click that predates a later reset can never settle the reset + # merely because its CoreBluetooth admission was delayed. + scenario_reset_at_entry = ( + scenario_reset_id, + scenario_reset_revision, + scenario_reset_mode, + ) + if scenario_reset_pending: + raise BleDiscoveryUnavailable( + "сначала завершается явная смена сценария подключения", + reason_code="connection-scenario-reset-pending", + ) + await self._prepare_pending_mode_scan() + if isinstance(request, BleScanRequest): + duration_seconds = request.duration_seconds + operation_id = request.operation_id + else: + # The public action contract enforces the operator range. Keeping + # this service-port compatibility path positive-only lets focused + # tests exercise deadlines without weakening the HTTP schema. + duration_seconds = float(request) + if duration_seconds <= 0: + raise ValueError("duration_seconds must be positive") + operation_id = None + request_fingerprint = self._request_fingerprint( + ACTION_DISCOVERY_SCAN, + {"duration_seconds": duration_seconds}, + ) + operation, created = self._operations.begin( + ACTION_DISCOVERY_SCAN, + operation_id=operation_id, + deadline_seconds=(duration_seconds + BLE_SCAN_HARD_TIMEOUT_GRACE_SECONDS), + request_fingerprint=request_fingerprint, + ) + if not created: + return await asyncio.to_thread(self.state) + + scan_generation: int | None = None + scenario_reset_at_admission: tuple[str, int, ConnectionMode] | None = None + network_process_lease_acquired = False + transition_gate_acquired = False + + def admit_scan() -> None: + nonlocal scan_generation, scenario_reset_at_admission + with self._lock: + if self._provisioning_active: + raise BleDiscoveryUnavailable( + "нельзя запускать BLE-поиск во время настройки Wi-Fi", + reason_code="ble-discovery-blocked-by-provisioning", + ) + self._ble_discovery_generation += 1 + scan_generation = self._ble_discovery_generation + # Only an admitted scan invalidates the previous in-memory + # candidate lease. A busy contender must not erase the + # result owned by the BLE operation already in flight. + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._scenario_reset_successful_scan_generation = None + self._scenario_reset_successful_scan_reset_id = None + self._operation_phase = "scanning" + self._operation_message = "Сканируем все устройства Bluetooth (BLE)…" + current_reset = self._connection_scenario_reset + if scenario_reset_at_entry is not None: + entry_reset_id, entry_reset_revision, entry_reset_mode = ( + scenario_reset_at_entry + ) + if ( + current_reset is not None + and current_reset.get("reset_id") == entry_reset_id + and current_reset.get("revision") == entry_reset_revision + and current_reset.get("desired_mode") == entry_reset_mode + and entry_reset_revision == self._desired_connection_mode_revision + and entry_reset_mode == self._desired_connection_mode + ): + # The admitted generation owns only this exact marker; + # a later reset cannot be settled by an older callback. + # A repeat Scan after first settlement remains bound to + # the same reset and supersedes its earlier candidates. + scenario_reset_at_admission = scenario_reset_at_entry + self._operations.transition( + operation.operation_id, + "running", + stage_code="ble-discovery", + message_code="discovery.scan.running", + ) + + try: + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + with self._lock: + provisioning_active = self._provisioning_active + raise BleDiscoveryUnavailable( + ( + "нельзя запускать BLE-поиск во время настройки Wi-Fi" + if provisioning_active + else "другая lifecycle-операция K1 ещё выполняется" + ), + reason_code=( + "ble-discovery-blocked-by-provisioning" + if provisioning_active + else "ble-discovery-lifecycle-busy" + ), + ) + transition_gate_acquired = True + if not await self._await_connection_monitor_quiescence(): + raise BleDiscoveryUnavailable( + "фоновая проверка подключения не завершилась вовремя", + reason_code="ble-discovery-lifecycle-busy", + ) + control_snapshot = self._application_control_session.snapshot() + if control_snapshot.get("state") == "failed": + # BLE discovery is an explicit operator action and performs no + # K1 write. Retire only the dead local MQTT worker so a power- + # cycled scanner can advertise again; the unresolved physical + # command ledger remains intact for later read-only proof. + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + # The OS lifecycle lease excludes every other Mission Core process; + # the low-level arbiter then serializes native CoreBluetooth inside + # this process. On timeout the OS lease follows detached cleanup. + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + self._k1_lifecycle_transition_gate.release() + transition_gate_acquired = False + with self._borrow_network_ble_process_lease(): + result = await scan(duration_seconds, on_admitted=admit_scan) devices = [ { "device_id": item["macos_uuid"], "name": item["local_name"] or item["name"], "rssi": item["rssi"], "address": None, - "connectable": True, + # An advertisement proves only current visibility. GATT + # connect/status-read is the separate connectability gate. + "connectable": None, "likely_k1": item["k1_name_candidate"], } for item in result["devices"] ] + observed_suspend_aware = time.time() observed_monotonic = time.monotonic() with self._lock: if self._ble_discovery_generation == scan_generation: @@ -796,9 +9306,13 @@ class XgridsK1CompatibilityService: self._ble_device_last_seen_monotonic = { str(item["device_id"]): observed_monotonic for item in devices } + self._ble_device_last_seen_suspend_aware = { + str(item["device_id"]): observed_suspend_aware for item in devices + } self._operation_message = ( f"Поиск завершён. Найдено BLE-устройств: {len(devices)}." ) + self._operation_phase = None self._operations.transition( operation.operation_id, "succeeded", @@ -806,158 +9320,1158 @@ class XgridsK1CompatibilityService: message_code="discovery.scan.completed", result={"candidate_count": len(devices)}, ) - except Exception as exc: with self._lock: - if self._ble_discovery_generation == scan_generation: + if ( + scan_generation is not None + and self._ble_discovery_generation == scan_generation + and scenario_reset_at_admission is not None + ): + admitted_reset_id, admitted_reset_revision, admitted_reset_mode = ( + scenario_reset_at_admission + ) + current_reset = self._connection_scenario_reset + pending_reset = self._connection_scenario_reset_pending + different_pending_reset = bool( + pending_reset is not None + and ( + pending_reset.get("reset_id") != admitted_reset_id + or pending_reset.get("desired_mode") != admitted_reset_mode + or pending_reset.get("expected_revision") + != admitted_reset_revision - 1 + ) + ) + if ( + not different_pending_reset + and current_reset is not None + and current_reset.get("reset_id") == admitted_reset_id + and current_reset.get("revision") == admitted_reset_revision + and current_reset.get("desired_mode") == admitted_reset_mode + and self._desired_connection_mode_revision + == admitted_reset_revision + and self._desired_connection_mode == admitted_reset_mode + ): + self._scenario_reset_successful_scan_generation = ( + scan_generation + ) + self._scenario_reset_successful_scan_reset_id = ( + admitted_reset_id + ) + # A successful explicit Scan, including an empty one, + # ends only the presentation boundary it was admitted + # under. Keep the marker itself for exact reset-id + # replay and operation-sequence suppression. A repeat + # successful Scan advances private candidate authority + # while preserving the first public settlement edge. + if current_reset.get("active") is not False: + current_reset["active"] = False + current_reset["settled_by_discovery_generation"] = ( + scan_generation + ) + except asyncio.CancelledError: + with self._lock: + if ( + scan_generation is not None + and self._ble_discovery_generation == scan_generation + ): self._devices = [] self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._operation_phase = None + self._operations.transition( + operation.operation_id, + "cancelled", + stage_code="cancelled", + message_code="discovery.scan.cancelled", + error={ + "category": "transport", + "code": "ble-discovery-cancelled", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + raise + except Exception as exc: + if isinstance(exc, ApplicationControlProcessLeaseUnavailable): + with self._lock: + provisioning_active = self._provisioning_active + exc = BleDiscoveryUnavailable( + ( + "нельзя запускать BLE-поиск во время настройки Wi-Fi" + if provisioning_active + else "другая lifecycle-операция K1 уже владеет Bluetooth" + ), + reason_code=( + "ble-discovery-blocked-by-provisioning" + if provisioning_active + else "ble-discovery-lifecycle-busy" + ), + ) + is_busy = isinstance( + exc, + ( + BleRuntimeBusy, + BleDiscoveryUnavailable, + ApplicationControlProcessLeaseUnavailable, + ), + ) + with self._lock: + if ( + scan_generation is not None + and self._ble_discovery_generation == scan_generation + ): + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._operation_phase = None self._operations.transition( operation.operation_id, "failed", - stage_code="failed", - message_code="discovery.scan.failed", - error=_operation_error(exc, category="transport", side_effect_status="none"), + stage_code="busy" if is_busy else "failed", + message_code=( + "discovery.scan.already_running" if is_busy else "discovery.scan.failed" + ), + error=_operation_error( + exc, + category="conflict" if is_busy else "transport", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + host_boundary="corebluetooth", + ), ) - raise + raise exc finally: - with self._lock: - if self._ble_discovery_generation == scan_generation: - self._operation_phase = None - return self.state() + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + if network_process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + return await asyncio.to_thread(self.state) async def connect(self, request: ConnectRequest) -> dict[str, Any]: with self._lock: - scanned_devices = self._fresh_ble_devices_locked() - discovery_generation = self._ble_discovery_generation - network_write_reconciliation = ( - dict(self._network_write_reconciliation) - if self._network_write_reconciliation is not None - else None + scenario_reset_pending = self._connection_scenario_reset_pending is not None + if scenario_reset_pending: + raise NetworkProvisioningConflict( + "сначала завершается явная смена сценария подключения", + reason_code="connection-scenario-reset-pending", ) - known_ids = {str(item["device_id"]) for item in scanned_devices} - if request.device_id not in known_ids: - raise ValueError("сначала найдите и выберите устройство через Bluetooth") - if network_write_reconciliation is not None: - logger.warning( - "K1 network mutation blocked pending read-only reconciliation", - extra={ - "event_code": "k1_network_write_reconciliation_required", - "operation_id": network_write_reconciliation.get("operation_id"), - "operation_stage": network_write_reconciliation.get("operation_stage"), - "connection_mode": network_write_reconciliation.get("connection_mode"), - "reason_code": "network-write-reconciliation-required", - "device_write_attempted": True, - "device_write_confirmed": network_write_reconciliation.get( - "device_write_confirmed", - False, - ), - "automatic_retry": False, - }, - ) - raise NetworkWriteReconciliationRequired( - "предыдущая BLE-запись завершилась до подтверждения актуального " - "состояния K1; новая запись заблокирована. Выполните свежий " - "Bluetooth-поиск и read-only проверку существующего Bridge-подключения" - ) - quick_connect = request.connection_mode == "quick-connect" - selected_device = next( - item for item in scanned_devices if item["device_id"] == request.device_id + durable_reset_replay = ( + self._durable_reset_reopen_not_dispatched_replay_matches(request) ) - selected_device_name = str(selected_device.get("name") or "").strip() - control_snapshot = self._application_control_session.snapshot() + if not durable_reset_replay: + consumed_reset_replay = ( + self._scenario_reset_consumed_request_replay_matches(request) + ) + self._require_network_provision_transport_allowed(request) + if ( + self._hidden_reset_reopen_requires_reconciliation() + and not consumed_reset_replay + ): + raise NetworkProvisioningConflict( + "Сетевой intent не был отправлен; сначала завершите read-only " + "сверку физического состояния K1", + reason_code="physical-command-reconciliation-required", + ) + # A proven loss may have cleared the device binding while the local + # MQTT worker was still unwinding. Retry only that local retirement; + # never reconnect or repeat a device/network command automatically. + self._retry_pending_local_control_retirement() + reconfiguration_gate_acquired = False + if not self._connection_reconfiguration_gate.acquire(blocking=False): + raise NetworkProvisioningConflict( + "другой диалог переподключения уже изменяет локальное состояние", + reason_code="connection-reconfiguration-lifecycle-busy", + ) + reconfiguration_gate_acquired = True + # Reject a stale dropdown before process ownership, journal writes, + # CoreBluetooth, CoreWLAN, TCP, or MQTT are touched. try: - self._application_control_session.retire_for_network_change() - except RuntimeError as exc: - failure = control_snapshot.get("failure") - logger.warning( - "K1 network change blocked by retained control ownership", - extra={ - "event_code": "k1_network_change_blocked_by_control_session", - "reason_code": ( - str(failure.get("reason_code")) - if isinstance(failure, dict) and failure.get("reason_code") - else None - ), - "failed_phase": ( - str(failure.get("failed_phase")) - if isinstance(failure, dict) and failure.get("failed_phase") - else str(control_snapshot.get("state") or "unknown") - ), - "automatic_retry": False, - }, - ) - raise RuntimeError( - "сначала завершите текущую control-сессию K1; " - "после START новый сетевой путь допустим только при " - "подтверждённом STOP" - ) from exc - if quick_connect and not selected_device_name: - raise ValueError( - "выбранный BLE-кандидат не сообщил имя точки доступа; " - "Quick Connect остановлен без команды устройству" - ) - quick_connect_profile_id = ( - quick_connect_host_profile_id(selected_device_name) if quick_connect else None - ) - host_wifi_helper_path = ( - self.repository_root / "plugins" / "xgrids-k1" / "macos" / "associate_wifi.swift" - ) + if not durable_reset_replay: + self._require_network_provision_transport_allowed(request) + self._require_current_connection_reconfiguration_target( + expected_revision=request.expected_reconfiguration_revision, + expected_intent_id=request.expected_reconfiguration_intent_id, + device_id=request.device_id, + connection_mode=request.connection_mode, + source="fresh-scan", + discovery_generation=request.expected_discovery_generation, + ) + self._require_connection_mode_draft_for_connect(request) + except BaseException: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + raise + quick_connect = request.connection_mode == "quick-connect" # Bridge and Direct Connect unwrap once at the BLE service boundary. # Quick Connect carries no browser/API credential. It first sends the # reviewed fixed AP-enable command, then the host-network adapter uses # the selected device's advertised name as its exact SSID and resolves # a device-scoped profile inside the OS credential store. password = "" if request.password is None else request.password.get_secret_value() - request_fingerprint = self._request_fingerprint( - ACTION_NETWORK_PROVISION, - { - "device_id": request.device_id, - "ssid": request.ssid, - "password": password if not quick_connect else None, - "host_wifi_profile": (quick_connect_profile_id if quick_connect else None), - "connection_mode": request.connection_mode, - "compatibility_attestation": request.compatibility_attestation.model_dump( - mode="json" - ), - }, - ) - operation, created = self._operations.begin( - ACTION_NETWORK_PROVISION, - operation_id=request.operation_id, - idempotency_key=request.idempotency_key, - device_id=self._device_id, - device_session_id=self._device_session_id, - deadline_seconds=240.0, - request_fingerprint=request_fingerprint, - ) - if not created: - password = "" - return self.state() - - if not self._provisioning_gate.acquire(blocking=False): - password = "" - self._operations.transition( - operation.operation_id, - "failed", - stage_code="busy", - message_code="network.provision.already_running", - error={ - "category": "conflict", - "code": "provisioning-already-running", - "retryable": True, - "safe_to_retry": True, - "side_effect_status": "none", + request_binding_sha256 = self._network_provision_request_binding_sha256(request) + request_fingerprint = self._network_provision_request_fingerprint(request) + idempotency_journal: NetworkProvisioningIdempotencyJournal | None = None + requested_operation_id = request.operation_id or f"op-{uuid4()}" + network_process_lease_acquired = False + admission_transition_gate_acquired = False + idempotency_record = None + try: + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + with self._lock: + provisioning_active = self._provisioning_active + if provisioning_active: + raise ApplicationControlProcessLeaseUnavailable( + "another network provisioning operation is already running" + ) + raise NetworkProvisioningConflict( + "другая локальная операция уже меняет network/control lifecycle K1", + reason_code="network-provision-lifecycle-busy", + ) + admission_transition_gate_acquired = True + if not await self._await_connection_monitor_quiescence(): + raise NetworkProvisioningConflict( + "фоновая проверка подключения не завершилась вовремя", + reason_code="network-provision-lifecycle-busy", + ) + # Own the process-wide K1 lifecycle before publishing PREPARED. + # Otherwise a concurrently starting backend could mistake a live + # request for a dead process and terminalize it as not-dispatched. + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + self._k1_lifecycle_transition_gate.release() + admission_transition_gate_acquired = False + # Another backend may have committed operator retirement after + # the browser's initial admission check but before this process + # won the shared lifecycle lease. Recheck before quarantine, + # idempotency/operation journals, BLE, Wi-Fi, TCP or MQTT. + durable_reset_replay = ( + self._durable_reset_reopen_not_dispatched_replay_matches(request) + ) + if not durable_reset_replay: + consumed_reset_replay = ( + self._scenario_reset_consumed_request_replay_matches(request) + ) + self._require_network_provision_transport_allowed(request) + if ( + self._hidden_reset_reopen_requires_reconciliation() + and not consumed_reset_replay + ): + raise NetworkProvisioningConflict( + "Сетевой intent не был отправлен; сначала завершите " + "read-only сверку физического состояния K1", + reason_code="physical-command-reconciliation-required", + ) + idempotency_journal = self._quarantine_corrupt_network_audit_under_exclusive_lease() + recovered_ledger_snapshot = _recover_network_provisioning_under_exclusive_process_lease( + network_ledger=self._network_mutation_ledger, + idempotency_journal=idempotency_journal, + ) + with self._lock: + self._network_write_reconciliation = _network_mutation_reconciliation_projection( + recovered_ledger_snapshot + ) + idempotency_admission = idempotency_journal.begin( + idempotency_key=request.idempotency_key, + action=ACTION_NETWORK_PROVISION, + operation_id=requested_operation_id, + request_binding_sha256=request_binding_sha256, + ) + idempotency_record = idempotency_admission.record + operation, created = self._operations.begin( + ACTION_NETWORK_PROVISION, + operation_id=idempotency_record.operation_id, + idempotency_key=request.idempotency_key, + device_id=self._device_id, + device_session_id=self._device_session_id, + deadline_seconds=240.0, + request_fingerprint=request_fingerprint, + context={ + "snapshot_runtime_id": self._snapshot_runtime_id, + "transport_ref": request.device_id, + "connection_mode": request.connection_mode, + "allow_host_wifi_switch": request.allow_host_wifi_switch, + "automatic_retry": False, }, ) - raise RuntimeError("другая операция настройки Wi-Fi уже выполняется") + except ApplicationControlProcessLeaseUnavailable as exc: + password = "" + if admission_transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + admission_transition_gate_acquired = False + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + # Contention can happen before this request has opened the durable + # journal (same-service second click or another backend process). + # It is still one user-facing single-flight conflict, never a raw + # process-lease implementation error. + raise ProvisioningAlreadyRunning() from exc + except NetworkProvisioningIdempotencyBlocked as exc: + password = "" + if admission_transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + admission_transition_gate_acquired = False + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + if network_process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + raise ProvisioningAlreadyRunning() from exc + except BaseException as exc: + password = "" + if admission_transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + admission_transition_gate_acquired = False + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + if ( + idempotency_journal is not None + and idempotency_record is not None + and idempotency_record.stage == "prepared" + ): + try: + idempotency_journal.complete( + idempotency_record.operation_id, + expected_revision=idempotency_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="network.provision.admission_failed", + error_code="operation-journal-admission-failed", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + except BaseException as terminalize_exc: + exc.add_note( + "durable network intent could not be terminalized after " + "in-memory operation admission failed: " + f"{type(terminalize_exc).__name__}: {terminalize_exc}" + ) + if network_process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + raise + assert idempotency_journal is not None + if idempotency_admission.disposition == "terminal-replay": + terminal = idempotency_record.terminal + assert terminal is not None + if created: + if terminal.outcome == "succeeded": + replay_binding = self._durable_terminal_replay_binding( + operation_id=operation.operation_id, + transport_ref=request.device_id, + connection_mode=request.connection_mode, + ) + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="durable-terminal-replay", + message_code="network.provision.completed", + result={ + "phase": "network_applied", + "control_state": "unknown", + "durable_replay": True, + "original_operation_id": idempotency_record.operation_id, + "outcome_code": terminal.outcome_code, + "side_effect_status": terminal.side_effect_status, + "automatic_retry": False, + **( + replay_binding + if replay_binding is not None + else {"replay_binding_available": False} + ), + }, + ) + else: + self._operations.transition( + operation.operation_id, + "cancelled" if terminal.outcome == "cancelled" else "failed", + stage_code="durable-terminal-replay", + message_code="network.provision.failed", + error={ + "category": "device", + "code": terminal.error_code, + "retryable": terminal.retryable, + "safe_to_retry": terminal.safe_to_retry, + "side_effect_status": ( + "none" if terminal.side_effect_status == "none" else "confirmed" + ), + "durable_replay": True, + }, + ) + password = "" + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + return await asyncio.to_thread(self.state) + if not created: + password = "" + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + raise NetworkProvisioningIdempotencyBlocked( + "in-memory operation state conflicts with a newly admitted durable request", + record=idempotency_record, + ) + + if not self._provisioning_gate.acquire(blocking=False): + assert idempotency_journal is not None + password = "" + try: + idempotency_record = idempotency_journal.complete( + operation.operation_id, + expected_revision=idempotency_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="network.provision.busy", + error_code="provisioning-already-running", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + self._operations.transition( + operation.operation_id, + "failed", + stage_code="busy", + message_code="network.provision.already_running", + error={ + "category": "conflict", + "code": "provisioning-already-running", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + finally: + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + raise ProvisioningAlreadyRunning() session_dir: Path | None = None + transition_gate_acquired = False + command_dispatch_gate_acquired = False device_write_attempted = False device_write_confirmed = False device_state_reconciled = False + ledger_dispatch_recorded = False + ledger_resolved = False + network_ledger_record: NetworkMutationRecord | None = None + applied_topology_committed = False retired_ingress_session_id: str | None = None - operation_stage = "device-ap-activation" if quick_connect else "ble-provisioning-write" + operation_stage = "network-provision-precondition" + selected_device_name = "" + quick_connect_profile_id: str | None = None + captured_device: CapturedDiscoveredDevice | None = None + same_device_recovery = False + recovery_handle_source: Literal["retained", "fresh"] | None = None + control_bootstrap_request: tuple[str, ConnectionMode, str, EndpointTarget, str] | None = ( + None + ) + host_wifi_helper_path = ( + self.repository_root / "plugins" / "xgrids-k1" / "macos" / "associate_wifi.swift" + ) + network_ble_borrow_stack = ExitStack() try: + network_ble_borrow_stack.enter_context(self._borrow_network_ble_process_lease()) + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise NetworkProvisioningConflict( + "другая локальная операция уже меняет network/control lifecycle K1", + reason_code="network-provision-lifecycle-busy", + ) + transition_gate_acquired = True + if not await self._await_connection_monitor_quiescence(): + raise NetworkProvisioningConflict( + "фоновая проверка подключения не завершилась вовремя", + reason_code="network-provision-lifecycle-busy", + ) + # Initial validation intentionally happens before all ownership and + # journal work for fast rejection. This second exact check closes + # the legal race where Mode Select committed after that validation + # but before Connect acquired the lifecycle transition gate. + self._require_current_connection_reconfiguration_target( + expected_revision=request.expected_reconfiguration_revision, + expected_intent_id=request.expected_reconfiguration_intent_id, + device_id=request.device_id, + connection_mode=request.connection_mode, + source="fresh-scan", + discovery_generation=request.expected_discovery_generation, + ) + self._require_connection_mode_draft_for_connect(request) + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + self._require_physical_command_network_mutation_allowed() + with self._lock: + # Only a candidate from the still-current explicit scan may + # admit a new selection. The row itself is not connection + # authority: exact handle capture and live GATT validation are + # still required before any device or network mutation. + scanned_devices = self._fresh_ble_devices_locked() + discovery_generation = self._ble_discovery_generation + current_selected_device_id = self._selected_device_id + current_connection_mode = self._connection_mode + current_k1_ip = self._k1_ip + current_device_session_id = self._device_session_id + current_device_session_discovery_generation = ( + self._device_session_discovery_generation + ) + current_acquisition = self._acquisition + current_acquisition_lease = self._acquisition_session_lease + ledger_snapshot = self._network_mutation_ledger.snapshot() + if ( + ledger_snapshot.status == "unresolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.stage == "prepared" + ): + self._network_mutation_ledger.resolve( + ledger_snapshot.record.operation_id, + expected_revision=ledger_snapshot.record.revision, + resolution="not-dispatched", + ) + ledger_snapshot = self._network_mutation_ledger.snapshot() + known_ids = {str(item["device_id"]) for item in scanned_devices} + + previous_connection = ( + PreviousConnectionEvidence( + transport_ref=current_selected_device_id, + mode=current_connection_mode, + ipv4=current_k1_ip, + device_session_id=current_device_session_id, + ) + if current_connection_mode is not None and current_selected_device_id is not None + else None + ) + + def persist_write_dispatch( + baseline_status: Mapping[str, Any], + resolved_write_mode: Literal["with_response", "without_response"], + ) -> None: + nonlocal idempotency_record, ledger_dispatch_recorded + nonlocal ledger_resolved, network_ledger_record + + def settle_prepared_network_edge() -> None: + """Seal the callback-local no-write edge after a failure.""" + + nonlocal ledger_resolved, network_ledger_record + snapshot = self._network_mutation_ledger.snapshot() + current = snapshot.record + if not ( + snapshot.status == "unresolved" + and current is not None + and current.operation_id == operation.operation_id + and current.stage == "prepared" + ): + return + network_ledger_record = self._network_mutation_ledger.resolve( + operation.operation_id, + expected_revision=current.revision, + resolution="not-dispatched", + ) + ledger_resolved = True + + def consume_exact_reopen( + *, + reset_authority: _ScenarioResetRetiredTransportAuthority, + reopening_id: str, + reopened_record: Mapping[str, object], + ) -> None: + """Bind one committed reopen to this exact durable request.""" + + reopened_revision = reopened_record.get("revision") + reopens = reopened_record.get("operator_reconciliation_reopens") + matching_reopens = ( + [ + item + for item in reopens + if isinstance(item, Mapping) + and item.get("reopening_id") == reopening_id + and item.get("retirement_id") + == reset_authority.retirement_id + and item.get("retired_record_revision") + == reset_authority.physical_record_revision + and item.get("reopened_transport_ref") + == reset_authority.retired_transport_ref + and item.get("discovery_generation") + == request.expected_discovery_generation + and item.get("reason") + == "reset-network-intent-read-only-settlement" + ] + if isinstance(reopens, list) + else [] + ) + if not ( + isinstance(reopened_revision, int) + and reopened_record.get("operation_id") + == reset_authority.physical_record_operation_id + and reopened_record.get("created_at_utc") + == reset_authority.physical_record_created_at_utc + and len(matching_reopens) == 1 + ): + raise RuntimeError( + "physical reset-network reopen omitted its exact audit" + ) + with self._lock: + if ( + self._connection_scenario_reset_retired_transport_authority + != reset_authority + ): + raise NetworkProvisioningConflict( + "Сценарий подключения изменился до durable dispatch", + reason_code="connection-scenario-reset-superseded", + ) + self._connection_scenario_reset_retired_transport_authority = ( + dataclass_replace( + reset_authority, + physical_reopening_id=reopening_id, + physical_reopened_record_revision=reopened_revision, + physical_reopening_discovery_generation=( + request.expected_discovery_generation + ), + consumed_operation_id=operation.operation_id, + consumed_idempotency_key_sha256=hashlib.sha256( + request.idempotency_key.encode("utf-8") + ).hexdigest(), + consumed_request_binding_sha256=( + request_binding_sha256 + ), + consumed_request_fingerprint=request_fingerprint, + ) + ) + + with self._lock: + scenario_reset_pending = self._connection_scenario_reset_pending is not None + if scenario_reset_pending: + # This callback is the durable last boundary immediately + # before the helper performs its one GATT write. A reset + # published before it wins with a proven no-dispatch + # outcome; a reset published after it must retain the + # ledger's truthful unknown physical edge. + raise NetworkProvisioningConflict( + "сетевой запрос отменён явной сменой сценария подключения", + reason_code="connection-scenario-reset-pending", + ) + # Live 7f02 was read, but no durable edge or GATT write has + # happened yet. Recheck the exact reset/scan/retirement CAS at + # this final callback boundary so no baseline-read TOCTOU can + # authorize a stale physical or network mutation. + self._require_network_provision_transport_allowed(request) + physical_at_dispatch = ( + self._physical_command_coordinator.snapshot() + ) + with self._lock: + reset_authority = ( + self._connection_scenario_reset_retired_transport_authority + ) + safe_reconfiguration_successor = bool( + reset_authority is not None + and self._consumed_reset_authority_has_safe_reconfiguration_successor_locked( # noqa: E501 + authority=reset_authority, + physical=physical_at_dispatch, + request=request, + ) + ) + reset_owned_retired_target = bool( + reset_authority is not None + and not safe_reconfiguration_successor + and physical_transport_ref_comparison_key( + reset_authority.retired_transport_ref + ) + == physical_transport_ref_comparison_key(request.device_id) + ) + assert idempotency_record is not None + # PREPARED is the durable proof that a failure in the local + # physical reopen still precedes both the network dispatch + # marker and the helper's sole GATT write. + try: + network_ledger_record = self._network_mutation_ledger.prepare( + operation_id=operation.operation_id, + transport_ref=request.device_id, + intended_mode=request.connection_mode, + write_mode=resolved_write_mode, + baseline_status=_network_status_evidence(baseline_status), + previous_connection=previous_connection, + ) + except BaseException: + settle_prepared_network_edge() + raise + if reset_owned_retired_target: + assert reset_authority is not None + if reset_authority.consumed_request_fingerprint is not None: + raise NetworkProvisioningConflict( + "Точный сетевой intent уже пересёк durable dispatch boundary", + reason_code="network-provision-idempotent-retry-dispatch-blocked", + ) + reopening_id = ( + "reset-network-intent-" + + hashlib.sha256( + ( + operation.operation_id + + ":" + + request_binding_sha256 + ).encode("utf-8") + ).hexdigest()[:48] + ) + try: + reopened_record = ( + self._physical_command_coordinator.reopen_retired_reconciliation( + reopening_id=reopening_id, + expected_revision=( + reset_authority.physical_record_revision + ), + expected_retirement_id=reset_authority.retirement_id, + expected_transport_ref=( + reset_authority.retired_transport_ref + ), + expected_discovery_generation=( + request.expected_discovery_generation + ), + reason=( + "reset-network-intent-read-only-settlement" + ), + ) + ) + except BaseException: + # A store may fsync the reopen and lose only its return + # path. Re-read and bind that exact immutable audit for + # terminal replay classification, but never continue to + # the network dispatch edge or GATT write. + physical_after_error = ( + self._physical_command_coordinator.snapshot() + ) + record_after_error = physical_after_error.get("record") + if isinstance(record_after_error, Mapping): + with suppress( + RuntimeError, + NetworkProvisioningConflict, + ): + consume_exact_reopen( + reset_authority=reset_authority, + reopening_id=reopening_id, + reopened_record=record_after_error, + ) + settle_prepared_network_edge() + raise + try: + consume_exact_reopen( + reset_authority=reset_authority, + reopening_id=reopening_id, + reopened_record=reopened_record, + ) + except BaseException: + settle_prepared_network_edge() + raise + # This is the durable ambiguity edge. It is intentionally + # adjacent to, and ordered before, the network-ledger dispatch + # marker and the helper's GATT write call. + try: + idempotency_record = idempotency_journal.mark_unresolved( + operation.operation_id, + expected_revision=idempotency_record.revision, + ) + except BaseException: + # If the independent journal committed and only its return + # path failed, retain that exact row for outer + # terminalization. The network ledger is still PREPARED, + # so the helper has not performed its GATT write. + journal_after_error = idempotency_journal.snapshot() + matching_after_error = next( + ( + item + for item in journal_after_error.records + if item.operation_id == operation.operation_id + and item.idempotency_key_sha256 + == hashlib.sha256( + request.idempotency_key.encode("utf-8") + ).hexdigest() + and item.request_binding_sha256 + == request_binding_sha256 + ), + None, + ) + if matching_after_error is not None: + idempotency_record = matching_after_error + settle_prepared_network_edge() + raise + prepared_record = network_ledger_record + with self._lock: + current_reconfiguration_exact = bool( + ( + request.expected_reconfiguration_intent_id is None + and self._connection_reconfiguration_intent_id is None + and self._connection_reconfiguration_intent is None + and ( + request.expected_reconfiguration_revision is None + or self._connection_reconfiguration_revision + == request.expected_reconfiguration_revision + ) + ) + or ( + request.expected_reconfiguration_intent_id is not None + and request.expected_reconfiguration_revision is not None + and self._connection_reconfiguration_intent_id + == request.expected_reconfiguration_intent_id + and self._connection_reconfiguration_revision + == request.expected_reconfiguration_revision + ) + ) + if not current_reconfiguration_exact: + settle_prepared_network_edge() + raise NetworkProvisioningConflict( + "Диалог переподключения изменился до durable dispatch", + reason_code=( + "connection-reconfiguration-revision-conflict" + ), + ) + try: + network_ledger_record = ( + self._network_mutation_ledger.mark_dispatching( + operation.operation_id, + expected_revision=prepared_record.revision, + ) + ) + except BaseException as dispatch_error: + # Atomic replace may have committed while only the + # return path failed. A mere reload does not prove the + # parent directory fsync completed, so the store must + # verify the exact predecessor and re-fsync the + # identical record before this callback may return. + dispatch_after_error = self._network_mutation_ledger.snapshot() + current_dispatch = dispatch_after_error.record + still_exact_prepared = bool( + dispatch_after_error.status == "unresolved" + and current_dispatch is not None + and current_dispatch.operation_id + == prepared_record.operation_id + and current_dispatch.transport_ref + == prepared_record.transport_ref + and current_dispatch.intended_mode + == prepared_record.intended_mode + and current_dispatch.write_mode + == prepared_record.write_mode + and current_dispatch.baseline_status + == prepared_record.baseline_status + and current_dispatch.previous_connection + == prepared_record.previous_connection + and current_dispatch.stage == "prepared" + and current_dispatch.revision + == prepared_record.revision + and current_dispatch.write_confirmed is None + and current_dispatch.last_observation is None + and current_dispatch.resolution is None + ) + if still_exact_prepared: + settle_prepared_network_edge() + raise + try: + network_ledger_record = ( + self._network_mutation_ledger.confirm_dispatching_after_uncertain_return( + operation.operation_id, + expected_prepared=prepared_record, + ) + ) + except BaseException as confirmation_error: + raise confirmation_error from dispatch_error + # From the durable-confirmed marker to callback return, + # only non-throwing in-memory CAS updates remain. + ledger_dispatch_recorded = True + if ( + request.expected_reconfiguration_intent_id is not None + and request.expected_reconfiguration_revision is not None + ): + self._clear_connection_reconfiguration_locked() + if ( + not reset_owned_retired_target + and reset_authority is not None + and self._connection_scenario_reset_retired_transport_authority + == reset_authority + ): + # Any successor dispatch burns the one reset-owned + # retired-target exception, even when it selected B. + self._connection_scenario_reset_retired_transport_authority = None + + selected_device = next( + (item for item in scanned_devices if str(item["device_id"]) == request.device_id), + None, + ) + if selected_device is not None: + captured_device = _capture_network_intent_device(request.device_id) + + same_device_recovery_identity = bool( + current_selected_device_id == request.device_id + and current_device_session_id is not None + ) + if same_device_recovery_identity: + assert current_device_session_id is not None + recovery_snapshot = connected_device_recovery_snapshot( + request.device_id, + device_session_id=current_device_session_id, + ) + # Fresh physical evidence is authoritative. The retained token + # is only an exact UUID/session recovery identity. Its native + # CoreBluetooth object is deliberately resolved later, inside + # the consuming BLE operation, so slow host preflight cannot + # expire it and concurrent BLE work cannot race it. + if captured_device is not None: + same_device_recovery = True + recovery_handle_source = "fresh" + + if request.device_id not in known_ids or captured_device is None: + raise NetworkProvisioningConflict( + "сначала выполните Bluetooth-поиск и выберите доступный K1; " + "точный BLE-сеанс этого результата больше недоступен", + reason_code="network-provision-candidate-not-fresh", + ) + self._operations.transition( + operation.operation_id, + "running", + stage_code="scan-selection-admitted", + message_code="network.provision.scan_selection_admitted", + ) + if ( + current_device_session_id is not None + and current_device_session_discovery_generation is not None + and discovery_generation <= current_device_session_discovery_generation + ): + raise NetworkProvisioningConflict( + "для нового подключения K1 выполните новый Bluetooth-поиск " + "и заново выберите устройство", + reason_code="network-provision-candidate-not-fresh", + ) + + # The exact selected scan object is the last harmless admission + # boundary. Once the lifecycle and dispatch gates are held and no + # physical acquisition is active, this explicit operator intent + # owns the connection lifecycle. All later audit/store/credential + # or transport failures must leave a clean unselected session. + runtime_state = self.runtime.snapshot() + control_snapshot = self._application_control_session.snapshot() + control_state = str(control_snapshot.get("state") or "unknown") + if ( + current_acquisition is not None + and current_acquisition.state not in TERMINAL_ACQUISITION_STATES + ): + raise NetworkProvisioningConflict( + "смена сети K1 недоступна во время активной acquisition-сессии", + reason_code="network-provision-recovery-acquisition-active", + ) + if current_acquisition_lease is not None: + raise NetworkProvisioningConflict( + "смена сети K1 ожидает завершения очистки evidence-сессии", + reason_code="network-provision-recovery-cleanup-pending", + ) + runtime_phase = str(runtime_state.get("phase") or "unknown") + runtime_source_mode = str(runtime_state.get("source_mode") or "unknown") + if runtime_phase in {"starting_live", "live", "replay", "stopping"} or ( + runtime_source_mode != "idle" + and runtime_phase not in {"idle", "completed", "stopped", "error"} + ): + raise NetworkProvisioningConflict( + "смена сети K1 недоступна при активном локальном источнике", + reason_code="network-provision-recovery-runtime-active", + ) + if control_state in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + await self._retire_prestart_control_for_mode_transition() + control_snapshot = self._application_control_session.snapshot() + control_state = str(control_snapshot.get("state") or "unknown") + control_failure = control_snapshot.get("failure") + control_admissible = control_state in {"idle", "completed", "closed"} or bool( + control_state == "failed" + and isinstance(control_failure, Mapping) + and ( + control_failure.get("safe_to_retry") is True + or control_failure.get("network_change_admissible") is True + ) + ) + if same_device_recovery and not control_admissible: + raise NetworkProvisioningConflict( + "смена сети K1 требует завершённой control-сессии " + "или подтверждённого допуска после её сетевой ошибки", + reason_code="network-provision-recovery-control-not-admissible", + ) + + if not self._k1_command_dispatch_gate.acquire(blocking=False): + raise NetworkProvisioningConflict( + "control-команда K1 уже находится в точном publish-окне; " + "смена сети остановлена до любого BLE write", + reason_code="network-provision-control-dispatch-active", + ) + command_dispatch_gate_acquired = True + with self._lock: + candidate_changed = bool( + self._ble_discovery_generation != discovery_generation + or request.device_id + not in {str(item["device_id"]) for item in self._fresh_ble_devices_locked()} + ) + recovery_context_changed = bool( + same_device_recovery + and ( + self._selected_device_id != current_selected_device_id + or self._connection_mode != current_connection_mode + or self._device_session_id != current_device_session_id + ) + ) + if candidate_changed or recovery_context_changed: + raise NetworkProvisioningConflict( + ( + "контекст выбранного K1 изменился до сетевой команды" + if recovery_context_changed + else "результаты Bluetooth-поиска изменились; выполните поиск ещё раз" + ), + reason_code="network-provision-candidate-changed", + ) + active_acquisition = self._acquisition + if ( + active_acquisition is not None + and active_acquisition.state not in TERMINAL_ACQUISITION_STATES + ): + raise NetworkProvisioningConflict( + "нельзя менять устройство или его сеть во время " + "активной acquisition-сессии", + reason_code="network-provision-lifecycle-busy", + ) + if self._acquisition_session_lease is not None: + raise NetworkProvisioningConflict( + "предыдущая evidence-сессия ещё не завершила локальную очистку", + reason_code="network-provision-lifecycle-busy", + ) + self._provisioning_active = True + + try: + self._retire_application_control_for_network_change() + except RuntimeError as exc: + failure = control_snapshot.get("failure") + logger.warning( + "K1 network change blocked by retained control ownership", + extra={ + "event_code": "k1_network_change_blocked_by_control_session", + "reason_code": ( + str(failure.get("reason_code")) + if isinstance(failure, dict) and failure.get("reason_code") + else None + ), + "failed_phase": ( + str(failure.get("failed_phase")) + if isinstance(failure, dict) and failure.get("failed_phase") + else str(control_snapshot.get("state") or "unknown") + ), + "automatic_retry": False, + }, + ) + raise NetworkProvisioningConflict( + "сначала завершите текущую control-сессию K1; " + "после START новый сетевой путь допустим только при " + "подтверждённом STOP", + reason_code="network-provision-lifecycle-busy", + ) from exc + + self._retire_ephemeral_device_binding_for_new_intent() + + try: + self._network_mutation_ledger.require_mutation_allowed() + except NetworkMutationBlocked as exc: + raise NetworkProvisioningConflict( + "журнал сетевой команды K1 повреждён или не был закрыт " + "эксклюзивным lifecycle recovery", + reason_code=getattr( + exc, + "reason_code", + "network-provision-ledger-recovery-failed", + ), + ) from exc + # A device write is inadmissible when its exact semantic result + # could not be durably published afterward. This preflight is + # intentionally before control retirement and before GATT I/O. + self._require_semantic_topology_store_writable() + selected_device_name = ( + str(selected_device.get("name") or "").strip() + if selected_device is not None + else "" + ) + if ( + not selected_device_name + and recovery_handle_source == "retained" + and current_device_session_id is not None + ): + # This is identity context from the exact process token, not + # advertisement/presence evidence. It is used only to select + # the already-enrolled host Quick Connect profile. + selected_device_name = ( + connected_device_recovery_name( + request.device_id, + device_session_id=current_device_session_id, + ) + or "" + ).strip() + if quick_connect and not selected_device_name: + raise NetworkProvisioningConflict( + "выбранный BLE-кандидат не сообщил имя точки доступа; " + "Quick Connect остановлен без команды устройству", + reason_code="network-provision-candidate-name-unavailable", + ) + quick_connect_profile_id = ( + quick_connect_host_profile_id(selected_device_name) if quick_connect else None + ) + operation_stage = "device-ap-activation" if quick_connect else "ble-provisioning-write" + + # Resolve the durable identity expectation while the previous + # control session is still intact. A corrupt/unavailable pin store + # is a fail-closed precondition, not a reason to retire ownership. + expected_vendor_device_id = self._expected_vendor_device_id(request.device_id) + # Reserve and fsync the evidence destination before retiring a + # healthy control path. ENOSPC/EACCES must be a zero-handoff, + # zero-device-write failure, not collateral lifecycle damage. + session_dir = _new_operation_session_dir( + self.evidence_root, + "viewer_k1_ap_association" if quick_connect else "viewer_wifi_provisioning", + ) + session_dir.mkdir(parents=True, exist_ok=False) + write_json_atomic( + session_dir / "operation-reservation.redacted.json", + { + "schema": "missioncore.xgrids-k1-operation-reservation/v1", + "operation_id": operation.operation_id, + "action": ACTION_NETWORK_PROVISION, + "connection_mode": request.connection_mode, + "transport_ref_sha256": hashlib.sha256( + request.device_id.encode("utf-8") + ).hexdigest(), + "device_write_performed": False, + "reserved_at": _utc_now_iso(), + }, + ) + if recovery_handle_source == "retained": + assert current_device_session_id is not None + recovery_snapshot = connected_device_recovery_snapshot( + request.device_id, + device_session_id=current_device_session_id, + ) + if recovery_snapshot["status"] != "retained": + raise NetworkProvisioningConflict( + "сохранённый BLE recovery-token больше не принадлежит этой device-сессии", + reason_code="network-provision-recovery-handle-invalidated", + ) + + self._connection_supervisor.set_intent( + intent_id=operation.operation_id, + requested_mode=request.connection_mode, + expected_device_id=expected_vendor_device_id, + ) + if quick_connect: assert quick_connect_profile_id is not None operation_stage = "host-wifi-profile-preflight" @@ -971,74 +10485,24 @@ class XgridsK1CompatibilityService: stage_code=operation_stage, message_code="network.provision.host_wifi_profile_preflight", ) - profile_preflight = await asyncio.to_thread( + profile_preflight = await _run_blocking_operation_without_abandonment( ensure_wifi_profile_from_credential_source, host_wifi_helper_path, quick_connect_profile_id, selected_device_name, K1_FW302_CREDENTIAL_PROVIDER_ID, - timeout_seconds=30.0, + timeout_seconds=120.0, + process_fence_descriptor_factory=( + self._duplicate_network_process_fence_descriptor + ), ) if not profile_preflight["available"]: raise HostWifiProfileError("credential-source-unavailable") - with self._lock: - if ( - self._ble_discovery_generation != discovery_generation - or request.device_id - not in { - str(item["device_id"]) - for item in self._fresh_ble_devices_locked() - } - ): - raise ValueError( - "результаты Bluetooth-поиска изменились или устарели; " - "выполните поиск ещё раз" - ) - active_acquisition = self._acquisition - if ( - active_acquisition is not None - and active_acquisition.state not in TERMINAL_ACQUISITION_STATES - ): - raise RuntimeError( - "нельзя менять устройство или его сеть во время активной acquisition-сессии" - ) - if self._acquisition_session_lease is not None: - raise RuntimeError( - "предыдущая evidence-сессия ещё не завершила локальную очистку" - ) - self._provisioning_active = True - # Once a new network operation is admitted, the previous route - # and device session can no longer be represented as current. - # This is especially important when Quick Connect may move the - # host off the prior LAN before CoreWLAN reports an outcome. - self._selected_device_id = None - self._k1_ip = None - self._connection_mode = None - self._compatibility_attestation = None - self._device_calibration = unavailable_device_calibration_snapshot(None) - self._device_session_id = None - self._device_session_opened_at = None - self._connection_verification = { - "status": "not-probed", - "lease_state": "disconnected", - "lease_generation": self._connection_lease_generation, - "endpoint_validation": "not-performed", - "network_reachability": "unknown", - "observed_at": None, - } - # A new admitted network route starts a new device context. - # Preserve immutable evidence and operation history, but never - # project the previous terminal acquisition into this route. - if self._acquisition_out_dir is not None: - retired_ingress_session_id = self._acquisition_out_dir.name - self._acquisition = None - self._acquisition_project_name = None - self._acquisition_mount_type = None - self._acquisition_gnss_mode = None - self._acquisition_out_dir = None - self._acquisition_start_operation_id = None - self._acquisition_stop_operation_id = None + # PREPARING and DISPATCHING never erase the previous committed + # connection. It remains last-known context until the requested + # topology is semantically observed and atomically committed. + # This rule is symmetric for every mode transition. # A connection-mode change can replace both the device session and # its address. @@ -1047,11 +10511,12 @@ class XgridsK1CompatibilityService: # of the newly selected Bridge route. stop() is idempotent once # the source thread has already exited and clears phase/source. self.runtime.stop() + self._reset_live_data_plane_observation() if retired_ingress_session_id is not None: self.live_perception_ingress.end_session(retired_ingress_session_id) # Revoke preview/producer state only after the active-acquisition # guard; a rejected network write must never stop evidence capture. - self.camera_preview.stop_current() + self._stop_camera_preview_and_release_process_lease() operation_message = ( "Включаем точку доступа выбранного K1 и подключаем к ней управляющее устройство." @@ -1059,11 +10524,6 @@ class XgridsK1CompatibilityService: else "Передаём устройству настройки Wi-Fi одним подтверждённым запросом." ) self._set_operation("provisioning", operation_message) - session_dir = _new_operation_session_dir( - self.evidence_root, - "viewer_k1_ap_association" if quick_connect else "viewer_wifi_provisioning", - ) - session_dir.mkdir(parents=True, exist_ok=False) self._operations.transition( operation.operation_id, "running", @@ -1076,8 +10536,26 @@ class XgridsK1CompatibilityService: async with device_ap_activation_session( request.device_id, timeout_seconds=15.0, - write_mode="auto", + # The accepted macOS / K1 FW 3.0.2 transport is one + # write-with-response frame. Do not reinterpret the live + # characteristic metadata into another write mode here. + write_mode="with_response", + captured_device=captured_device, + recovery_device_session_id=( + current_device_session_id if recovery_handle_source == "retained" else None + ), + on_write_dispatch=persist_write_dispatch, ) as activation: + if recovery_handle_source == "retained": + assert current_device_session_id is not None + captured_device = connected_device_capture( + request.device_id, + device_session_id=current_device_session_id, + ) + if captured_device is None: + raise RuntimeError( + "BLE recovery token was not GATT-validated before AP admission" + ) device_write_attempted = bool(activation["write_performed"]) device_write_confirmed = bool( activation["write_performed"] @@ -1086,10 +10564,97 @@ class XgridsK1CompatibilityService: or activation["ready_observed"] ) ) - # Only an actual post-write 7f02 observation reconciles - # device state. A GATT acknowledgement alone confirms the - # transport write, not the network state K1 retained. - device_state_reconciled = bool(activation.get("observations")) + if device_write_attempted: + self._operations.transition( + operation.operation_id, + "running", + stage_code="ble-write-dispatched", + message_code="network.provision.ble_write_dispatched", + result={ + "side_effect_status": "unknown", + "automatic_retry": False, + }, + ) + # Only the reviewed AP-ready state proves that this exact + # Quick Connect mutation applied. Any other observation is + # diagnostic and keeps the durable ambiguity barrier. + activation_observations = activation.get("observations") or [] + activation_final_status = ( + activation_observations[-1].get("status") + if activation_observations + and isinstance(activation_observations[-1], Mapping) + else None + ) + assert network_ledger_record is not None + activation_baseline_evidence = network_ledger_record.baseline_status + applied_target = _post_dispatch_network_target( + request.connection_mode, + activation_final_status, + transport_ref=request.device_id, + baseline_status=activation_baseline_evidence, + previous_connection=previous_connection, + ) + device_state_reconciled = applied_target is not None + activation_evidence = _optional_network_status_evidence(activation_final_status) + if ledger_dispatch_recorded: + assert network_ledger_record is not None + network_ledger_record = self._network_mutation_ledger.mark_observing( + operation.operation_id, + expected_revision=network_ledger_record.revision, + write_confirmed=device_write_confirmed, + observation=activation_evidence, + ) + self._operations.transition( + operation.operation_id, + "running", + stage_code="status-observing", + message_code="network.provision.status_observing", + result={ + "side_effect_status": ( + "confirmed" if device_write_confirmed else "unknown" + ), + "automatic_retry": False, + }, + ) + if device_state_reconciled: + assert applied_target is not None + retired_ingress_session_id = self._commit_applied_device_topology( + intent_id=operation.operation_id, + transport_ref=request.device_id, + connection_mode=request.connection_mode, + ipv4=applied_target, + compatibility_attestation=(request.compatibility_attestation), + captured_device=captured_device, + source="ble-post-write-status", + observed_at_utc=_status_observed_at_utc( + activation, + activation_observations[-1] if activation_observations else None, + ), + ) + applied_topology_committed = True + if ledger_dispatch_recorded and activation_evidence is not None: + assert network_ledger_record is not None + network_ledger_record = self._network_mutation_ledger.resolve( + operation.operation_id, + expected_revision=network_ledger_record.revision, + resolution="target-observed", + observation=activation_evidence, + ) + ledger_resolved = True + with self._lock: + self._network_write_reconciliation = None + self._operations.transition( + operation.operation_id, + "running", + stage_code="device-topology-applied", + message_code="network.provision.device_topology_applied", + result={ + "side_effect_status": "applied", + "automatic_retry": False, + }, + ) + if retired_ingress_session_id is not None: + self.live_perception_ingress.end_session(retired_ingress_session_id) write_json_atomic( session_dir / "ap-activation.redacted.json", activation, @@ -1099,6 +10664,14 @@ class XgridsK1CompatibilityService: "K1 не подтвердил готовность точки доступа; " "системное подключение Wi-Fi не запускалось" ) + if applied_target is None: + raise NetworkProvisioningConflict( + "Текущий 7f02-статус точки K1 неотличим от baseline или " + "предыдущей топологии; системное подключение Wi-Fi не запускалось", + reason_code=( + "network-provision-target-not-distinguishable-from-baseline" + ), + ) operation_stage = "host-wifi-association" self._operations.transition( operation.operation_id, @@ -1110,13 +10683,19 @@ class XgridsK1CompatibilityService: # CoreWLAN discovery/association handoff, as the reviewed # LixelGO Quick Connect flow does. try: - association = await asyncio.to_thread( + association = await _run_blocking_operation_without_abandonment( associate_with_wifi_profile_once, host_wifi_helper_path, quick_connect_profile_id, selected_device_name, - scan_timeout_seconds=15.0, + # Historical physical associations took up to + # 18.142 s after AP-ready. The K1 control flag does + # not prove that the RF beacon is already visible. + scan_timeout_seconds=30.0, timeout_seconds=180.0, + process_fence_descriptor_factory=( + self._duplicate_network_process_fence_descriptor + ), ) except HostWifiProfileError as exc: write_json_atomic( @@ -1133,7 +10712,7 @@ class XgridsK1CompatibilityService: ) raise completed_at = _utc_now_iso() - ipv4: str | None = AP_FALLBACK_IPV4 + ipv4: str | None = applied_target connection_manifest: dict[str, Any] = { "schema_version": 1, "started_at_utc": activation["started_at_utc"], @@ -1174,14 +10753,76 @@ class XgridsK1CompatibilityService: request.ssid, password, timeout_seconds=45.0, - write_mode="auto", + # Physical Bridge acceptance on macOS / K1 FW 3.0.2 used + # exactly one write-with-response frame. Payload and retry + # policy remain owned by the reviewed BLE helper. + write_mode="with_response", + captured_device=captured_device, + recovery_device_session_id=( + current_device_session_id if recovery_handle_source == "retained" else None + ), + on_write_dispatch=persist_write_dispatch, ) + if recovery_handle_source == "retained": + assert current_device_session_id is not None + captured_device = connected_device_capture( + request.device_id, + device_session_id=current_device_session_id, + ) + if captured_device is None: + raise RuntimeError( + "BLE recovery token was not GATT-validated before topology admission" + ) device_write_attempted = True - # A successful return may still describe a disconnect before - # any post-write status was read. Keep the ambiguity fence in - # that case even when write-with-response was acknowledged. + self._operations.transition( + operation.operation_id, + "running", + stage_code="ble-write-dispatched", + message_code="network.provision.ble_write_dispatched", + result={ + "side_effect_status": "unknown", + "automatic_retry": False, + }, + ) + # A successful transport return is not device-topology proof. + # Bridge/Direct is applied only after a valid non-AP private + # address appears in a post-write 7f02 observation that is + # semantically distinguishable from the live baseline. observations = result.get("observations") or [] - device_state_reconciled = bool(observations) + assert network_ledger_record is not None + bridge_baseline_evidence = network_ledger_record.baseline_status + applied_observation = next( + ( + observation + for observation in reversed(observations) + if isinstance(observation, Mapping) + and ( + _post_dispatch_network_target( + request.connection_mode, + observation.get("status"), + transport_ref=request.device_id, + baseline_status=bridge_baseline_evidence, + previous_connection=previous_connection, + expected_network_name=request.ssid, + ) + ) + is not None + ), + None, + ) + applied_target = ( + _post_dispatch_network_target( + request.connection_mode, + applied_observation.get("status"), + transport_ref=request.device_id, + baseline_status=bridge_baseline_evidence, + previous_connection=previous_connection, + expected_network_name=request.ssid, + ) + if isinstance(applied_observation, Mapping) + else None + ) + device_state_reconciled = applied_target is not None device_write_confirmed = bool( result.get("write_mode") == "with_response" or ( @@ -1189,8 +10830,68 @@ class XgridsK1CompatibilityService: and observations[-1].get("status") != result.get("baseline_status") ) ) + bridge_observation = _last_network_status_evidence(observations) + if ledger_dispatch_recorded: + assert network_ledger_record is not None + network_ledger_record = self._network_mutation_ledger.mark_observing( + operation.operation_id, + expected_revision=network_ledger_record.revision, + write_confirmed=device_write_confirmed, + observation=bridge_observation, + ) + self._operations.transition( + operation.operation_id, + "running", + stage_code="status-observing", + message_code="network.provision.status_observing", + result={ + "side_effect_status": ( + "confirmed" if device_write_confirmed else "unknown" + ), + "automatic_retry": False, + }, + ) + if device_state_reconciled: + assert applied_target is not None + retired_ingress_session_id = self._commit_applied_device_topology( + intent_id=operation.operation_id, + transport_ref=request.device_id, + connection_mode=request.connection_mode, + ipv4=applied_target, + compatibility_attestation=request.compatibility_attestation, + captured_device=captured_device, + source="ble-post-write-status", + observed_at_utc=_status_observed_at_utc( + result, + applied_observation, + ), + ) + applied_topology_committed = True + if ledger_dispatch_recorded and bridge_observation is not None: + assert network_ledger_record is not None + network_ledger_record = self._network_mutation_ledger.resolve( + operation.operation_id, + expected_revision=network_ledger_record.revision, + resolution="target-observed", + observation=bridge_observation, + ) + ledger_resolved = True + with self._lock: + self._network_write_reconciliation = None + self._operations.transition( + operation.operation_id, + "running", + stage_code="device-topology-applied", + message_code="network.provision.device_topology_applied", + result={ + "side_effect_status": "applied", + "automatic_retry": False, + }, + ) + if retired_ingress_session_id is not None: + self.live_perception_ingress.end_session(retired_ingress_session_id) write_json_atomic(session_dir / "provisioning.sensitive.json", result) - ipv4 = _provisioned_ipv4(result) + ipv4 = applied_target connection_manifest = { "schema_version": 1, "started_at_utc": result["started_at_utc"], @@ -1220,74 +10921,138 @@ class XgridsK1CompatibilityService: "Устройство сообщило IPv4-адрес, который уже принадлежит этому компьютеру; " "адрес K1 не принят и автоматического повтора не было" ) - host_route_class = "device-ap" if quick_connect else _host_route_class(ipv4) - host_route_ready = host_route_class not in {"tunnel", "default-route"} + # Route/TCP/DeviceInfo are observation planes, not part of the + # exactly-once K1 network mutation. Keep them out of the Apply + # response path. The one exception is Bridge's explicit host + # Wi-Fi opt-in: a bounded route classification is required to + # decide whether that separately authorized host mutation is + # necessary at all. + host_route_class: str | None = None + host_association: Mapping[str, Any] | None = None + host_wifi_association_outcome: str | None = None + if request.connection_mode == "bridge" and request.allow_host_wifi_switch: + host_route_class = _host_route_class(ipv4) + if ( + request.connection_mode == "bridge" + and request.allow_host_wifi_switch + and (host_route_class != "direct-or-routed") + ): + if request.ssid is None: + raise RuntimeError("SSID отсутствует перед host network handoff") + operation_stage = "host-bridge-wifi-association" + self._operations.transition( + operation.operation_id, + "running", + stage_code=operation_stage, + message_code="network.provision.host_bridge_wifi_association", + result={ + "side_effect_status": "applied", + "automatic_retry": False, + }, + ) + host_association = await _run_blocking_operation_without_abandonment( + associate_with_ephemeral_wifi_credentials_once, + host_wifi_helper_path, + request.ssid, + password, + scan_timeout_seconds=30.0, + timeout_seconds=180.0, + process_fence_descriptor_factory=( + self._duplicate_network_process_fence_descriptor + ), + ) + if host_association is None: + raise RuntimeError("host Wi-Fi association не вернул результат") + host_wifi_association_outcome = str(host_association.get("outcome") or "unknown") + # Association is the last authorized network mutation. Its + # resulting route is deliberately left for the supervised + # read-only continuation rather than delaying Apply again. + host_route_class = None + write_json_atomic( + session_dir / "host-bridge-wifi-association.redacted.json", + { + "schema_version": 1, + "completed_at_utc": _utc_now_iso(), + "adapter": host_association.get("adapter"), + "outcome": host_association.get("outcome"), + "already_associated": host_association.get("already_associated"), + "profile_enrolled": False, + "scan_attempt_count": host_association.get("scan_attempt_count"), + "scan_elapsed_ms": host_association.get("scan_elapsed_ms"), + "credential_source": "operation-memory", + "automatic_retry": False, + "device_write_performed": False, + }, + ) + elif request.connection_mode == "bridge" and not request.allow_host_wifi_switch: + host_wifi_association_outcome = "not-authorized" + self._operations.transition( + operation.operation_id, + "running", + stage_code="host-wifi-switch-not-authorized", + message_code="network.provision.host_wifi_switch_not_authorized", + result={ + "phase": "network_applied", + "control_state": "unknown", + "side_effect_status": "applied", + "host_wifi_switch_authorized": False, + "automatic_retry": False, + }, + ) + host_route_ready: bool | None = None + endpoint_reachable: bool | None = None connection_manifest.update( { "host_route_class": host_route_class, "host_route_ready": host_route_ready, + "control_endpoint": f"mqtt://{ipv4}:{CONTROL_MQTT_PORT}", + "control_endpoint_reachable": endpoint_reachable, + "host_wifi_switch_authorized": ( + request.allow_host_wifi_switch + if request.connection_mode == "bridge" + else None + ), + "host_wifi_association_performed": host_association is not None, + "host_wifi_association_outcome": host_wifi_association_outcome, } ) write_json_atomic(session_dir / "manifest.redacted.json", connection_manifest) - with self._lock: - self._network_write_reconciliation = None - self._selected_device_id = request.device_id - self._k1_ip = ipv4 - self._connection_mode = request.connection_mode - self._device_id = self._device_ids_by_transport_ref.setdefault( - request.device_id, - new_device_id(), + if not applied_topology_committed: + raise RuntimeError( + "K1 сообщил целевую сеть, но её доказательство не было атомарно зафиксировано" ) - self._device_session_id = new_device_session_id() - self._device_session_opened_at = _utc_now_iso() - self._connection_lease_generation += 1 - self._compatibility_attestation = _attestation_snapshot( - request.compatibility_attestation - ) - self._device_calibration = unavailable_device_calibration_snapshot( - XGRIDS_K1_COMPATIBILITY_PROFILE_ID - ) - self._connection_verification = { - "status": ("configured" if host_route_ready else "host-route-mismatch"), - "lease_state": "configured" if host_route_ready else "disconnected", - "lease_generation": self._connection_lease_generation, - "endpoint_validation": ( - "provisioning-status" if host_route_ready else "host-route" - ), - "network_reachability": ("unknown" if host_route_ready else "unreachable"), - **( - {} - if host_route_ready - else { - "reason_code": "connection_lease_host_route_mismatch", - "host_route_class": host_route_class, - } - ), - "write_performed": True, - "observed_at": _utc_now_iso(), - } - self._operation_message = ( - { - "bridge": "K1 подключён к общей сети и сообщил локальный адрес.", - "direct-connect": ( - "K1 подключён к хотспоту контроллера и сообщил локальный адрес." - ), - "quick-connect": ( - "Mission Core подключён к точке доступа K1; адрес K1 подтверждён." - ), - }[request.connection_mode] - if host_route_ready - else ( - "K1 получил адрес, но этот компьютер подключён к другой сети. " - "Подключите компьютер к той же локальной сети." - ) + if not ledger_resolved: + raise RuntimeError( + "состояние K1 применено, но durable network edge требует read-only сверки" ) + idempotency_record = idempotency_journal.complete( + operation.operation_id, + expected_revision=idempotency_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="succeeded", + outcome_code="network.provision.completed", + error_code=None, + side_effect_status="applied", + retryable=False, + safe_to_retry=False, + ), + ) self._operations.transition( operation.operation_id, "succeeded", - stage_code="connected", + stage_code="network-configured", message_code="network.provision.completed", result={ + "phase": "network_applied", + # Network durability is sealed before the subordinate + # read-only DeviceInfo bootstrap. Its initial state is + # therefore unknown, never implicitly ready. + "control_state": "unknown", + "snapshot_runtime_id": self._snapshot_runtime_id, + "parent_intent_id": operation.operation_id, + "transport_ref": request.device_id, + "target_ipv4": ipv4, + "target_port": CONTROL_MQTT_PORT, "device_id": self._device_id, "device_session_id": self._device_session_id, "lan_address_observed": True, @@ -1298,10 +11063,41 @@ class XgridsK1CompatibilityService: ), "host_route_class": host_route_class, "host_route_ready": host_route_ready, + "control_endpoint_reachable": endpoint_reachable, + "host_wifi_switch_authorized": ( + request.allow_host_wifi_switch + if request.connection_mode == "bridge" + else None + ), + "host_wifi_association_performed": host_association is not None, + "host_wifi_association_outcome": host_wifi_association_outcome, + "side_effect_status": "applied", + "automatic_retry": False, }, evidence_refs=(f"evidence-session-{session_dir.name}",), ) - except Exception as exc: + # The physical network transaction is now durably terminal and + # must never be replayed merely because route/TCP/DeviceInfo fail. + # Capture the exact continuation tuple before releasing lifecycle + # ownership; it will be scheduled after every mutation gate has + # been released and cannot delay this Apply response. + network_ble_borrow_stack.close() + if command_dispatch_gate_acquired: + self._k1_command_dispatch_gate.release() + command_dispatch_gate_acquired = False + with self._lock: + self._provisioning_active = False + continuation_device_session_id = self._device_session_id + if continuation_device_session_id is None: + raise RuntimeError("applied topology has no exact device session") + control_bootstrap_request = ( + operation.operation_id, + request.connection_mode, + request.device_id, + EndpointTarget(ipv4, CONTROL_MQTT_PORT), + continuation_device_session_id, + ) + except (asyncio.CancelledError, Exception) as exc: annotated_stage = getattr(exc, "operation_stage", None) if isinstance(annotated_stage, str) and annotated_stage: operation_stage = annotated_stage @@ -1313,21 +11109,39 @@ class XgridsK1CompatibilityService: ) if device_write_confirmed: device_write_attempted = True - if device_write_attempted and not device_state_reconciled: + if applied_topology_committed: + self._connection_supervisor.record_monitor_failure( + f"{operation_stage}-failed-after-device-network-applied" + ) with self._lock: - self._network_write_reconciliation = { - "status": "device-state-unknown-after-write", - "operation_id": operation.operation_id, - "transport_ref": request.device_id, - "connection_mode": request.connection_mode, - "operation_stage": operation_stage, - "reason_code": getattr(exc, "reason_code", None) - or type(exc).__name__, - "device_write_confirmed": device_write_confirmed, - "required_action": "explicit-read-only-ble-status-observation", - "scope": "process-runtime", + self._connection_verification = { + "status": "device-network-applied-host-failed", + "lease_state": "configured-unverified", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": operation_stage, + "network_reachability": "unreachable", + "reason_code": (f"{operation_stage}-failed-after-device-network-applied"), + "write_performed": True, "observed_at": _utc_now_iso(), } + if ledger_dispatch_recorded and not ledger_resolved: + ledger_after_failure = self._network_mutation_ledger.snapshot() + if ( + ledger_after_failure.record is not None + and ledger_after_failure.record.stage == "dispatching" + and device_write_confirmed + ): + self._network_mutation_ledger.mark_observing( + operation.operation_id, + expected_revision=ledger_after_failure.record.revision, + write_confirmed=True, + ) + with self._lock: + self._network_write_reconciliation = ( + _network_mutation_reconciliation_projection( + self._network_mutation_ledger.snapshot() + ) + ) side_effect_status: Literal["none", "confirmed", "unknown"] = ( "confirmed" if device_write_confirmed and device_state_reconciled @@ -1340,7 +11154,64 @@ class XgridsK1CompatibilityService: category="transport" if quick_connect else "device", side_effect_status=side_effect_status, safe_to_retry=not device_write_attempted, + host_boundary="corebluetooth" if isinstance(exc, BleakError) else None, ) + if idempotency_record.stage == "prepared": + cancelled = isinstance(exc, asyncio.CancelledError) + idempotency_record = idempotency_journal.complete( + operation.operation_id, + expected_revision=idempotency_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="cancelled" if cancelled else "failed", + outcome_code=( + "network.provision.cancelled" + if cancelled + else "network.provision.failed-before-dispatch" + ), + error_code=( + "request-cancelled" + if cancelled + else "network-provision-precondition-failed" + ), + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + elif idempotency_record.stage == "unresolved" and ledger_resolved: + settled_ledger = self._network_mutation_ledger.snapshot().record + definitely_not_dispatched = bool( + settled_ledger is not None + and settled_ledger.operation_id == operation.operation_id + and settled_ledger.stage == "resolved" + and settled_ledger.resolution == "not-dispatched" + ) + if definitely_not_dispatched or device_state_reconciled: + idempotency_record = idempotency_journal.complete( + operation.operation_id, + expected_revision=idempotency_record.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code=( + "network.provision.failed-before-dispatch" + if definitely_not_dispatched + else "network.provision.failed-after-reconciliation" + ), + error_code=( + "network-provision-precondition-failed" + if definitely_not_dispatched + else "post-write-operation-failed" + ), + side_effect_status=( + "none" if definitely_not_dispatched else "reconciled" + ), + retryable=definitely_not_dispatched, + safe_to_retry=definitely_not_dispatched, + ), + ) + # An unresolved record without exact target-observed network + # evidence remains unresolved by design. Retrying the same key or + # presenting a new one cannot cross another GATT side-effect edge. self._operations.transition_if_pending( operation.operation_id, "failed", @@ -1353,7 +11224,7 @@ class XgridsK1CompatibilityService: ) # Never log the request, SSID, password, BLE payload, or exception # text here: lower transport errors may embed sensitive frame data. - # The structured record is sufficient to correlate the HTTP 502 + # The structured record is sufficient to correlate the HTTP error # with its operation stage and retry/side-effect safety contract. logger.error( "K1 network provisioning failed", @@ -1371,8 +11242,18 @@ class XgridsK1CompatibilityService: "device_write_confirmed": device_write_confirmed, "ble_att_error_code": operation_error.get("ble_att_error_code"), "ble_att_error_name": operation_error.get("ble_att_error_name"), + "resolved_write_mode": operation_error.get("resolved_write_mode"), + "write_characteristic_properties": operation_error.get( + "write_characteristic_properties" + ), + "max_write_without_response_size": operation_error.get( + "max_write_without_response_size" + ), + "frame_length": operation_error.get("frame_length"), "helper_stage": operation_error.get("helper_stage"), "helper_elapsed_ms": operation_error.get("helper_elapsed_ms"), + "scan_attempt_count": operation_error.get("scan_attempt_count"), + "scan_elapsed_ms": operation_error.get("scan_elapsed_ms"), "automatic_retry": False, }, ) @@ -1382,24 +11263,2967 @@ class XgridsK1CompatibilityService: with self._lock: self._provisioning_active = False self._operation_phase = None + network_ble_borrow_stack.close() + if command_dispatch_gate_acquired: + self._k1_command_dispatch_gate.release() + if network_process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() self._provisioning_gate.release() - return self.state() + if control_bootstrap_request is not None: + ( + parent_operation_id, + connection_mode, + transport_ref, + target, + device_session_id, + ) = control_bootstrap_request + try: + self._schedule_control_bootstrap_continuation( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + transport_ref=transport_ref, + target=target, + device_session_id=device_session_id, + ) + except Exception: + # Scheduling is subordinate to the already durable network + # result. With no exact owned child, the projection offers an + # explicit read-only Verify and never replays Apply. + logger.warning( + "K1 control bootstrap continuation was not scheduled", + extra={ + "event_code": "k1_control_bootstrap_continuation_not_scheduled", + "parent_operation_id": parent_operation_id, + "automatic_retry": False, + }, + ) + else: + # Give an immediately-completing local/test bootstrap one turn + # without joining it. A blocked route or DeviceInfo probe does + # not delay the response beyond this scheduling checkpoint. + await asyncio.sleep(0) + return await asyncio.to_thread(self.state) def inspect_device(self) -> dict[str, Any]: """Return profile-scoped facts without probing or mutating the device.""" return self.state() + @staticmethod + def _active_acquisition_checkpoint_transition_id( + kind: str, + *parts: object, + ) -> str: + material = "\0".join(str(part) for part in parts).encode("utf-8") + return f"checkpoint-{kind}:sha256-{hashlib.sha256(material).hexdigest()}" + + def _set_active_acquisition_checkpoint_reason( + self, + reason_code: str | None, + ) -> None: + with self._lock: + if ( + reason_code is None + and getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + != "trusted" + ): + return + self._active_acquisition_checkpoint_reason = reason_code + + def _mark_active_acquisition_checkpoint_untrusted( + self, + *, + trust: Literal["corrupt", "unavailable"], + reason_code: str, + ) -> None: + """Revoke local recovery admission without rewriting durable evidence.""" + + with self._lock: + current = getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + if current == "corrupt": + return + self._active_acquisition_checkpoint_trust = trust + self._active_acquisition_checkpoint_reason = reason_code + + @staticmethod + def _checkpoint_binding_matches_physical_connection( + binding: ActiveAcquisitionRecoveryTransportBinding, + record: PhysicalCommandRecord, + ) -> bool: + connection = record.connection + return bool( + connection.intent_id == binding.intent_id + and connection.transport_ref == binding.transport_ref + and connection.connection_mode == binding.connection_mode + and connection.target_ipv4 == binding.target_ipv4 + and connection.target_port == binding.target_port + and connection.host_path_epoch == binding.host_path_epoch + and connection.control_session_id == binding.control_session_id + and connection.producer_generation == binding.producer_generation + ) + + def _active_checkpoint_allows_physical_stop_successor( + self, + *, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, + ) -> bool: + """Validate an exact STOP successor while its ACTIVE proof stays parent.""" + + persisted = checkpoint.current_active_physical_proof + status = checkpoint.current_active_status_proof + reconciliation = record.reconciliations[-1] if record.reconciliations else None + unresolved_stop = bool( + record.stage in {"prepared", "dispatching", "observing"} + and record.resolution is None + ) + classified_terminal_stop = bool( + record.stage == "resolved" + and record.resolution == "not-dispatched" + and reconciliation is not None + and reconciliation.kind == "prepared-stop-classification" + and reconciliation.resolution + in {"physical-active-observed", "physical-standby-observed"} + and reconciliation.original_attempt.operation_id == record.operation_id + and reconciliation.original_attempt.acquisition_id == record.acquisition_id + and reconciliation.original_attempt.action == "stop" + and reconciliation.original_attempt.publish_call_returned is None + and reconciliation.original_attempt.packet_id is None + and not reconciliation.original_attempt.qos2_completed + and reconciliation.original_attempt.application_response is None + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and not reconciliation.observation.mqtt_retained + ) + if not ( + persisted is not None + and status is not None + and record.action == "stop" + and (unresolved_stop or classified_terminal_stop) + and record.acquisition_id == checkpoint.acquisition_id + and record.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and record.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and record.compatibility_profile_id == checkpoint.compatibility_profile_id + and record.baseline_status.source == "live-control-session" + and record.baseline_status.session_state == "scanning" + and record.baseline_status.project_bound + and record.baseline_status.init_ready + and not record.baseline_status.mqtt_retained + and record.baseline_status.project_id_sha256 + == checkpoint.active_project_id_sha256 + # A later explicit STOP may use a newer non-retained SCANNING + # message on the same exact binding. The immutable ledger edge, + # not equality with the checkpoint's earlier status payload, is + # the durable fence here. + and status.binding == checkpoint.current_binding + and status.session_state == "scanning" + and status.project_id_sha256 == checkpoint.active_project_id_sha256 + ): + return False + try: + ancestry = self._physical_command_ledger.prove_classified_stop_ancestry( + head_operation_id=record.operation_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + acquisition_id=checkpoint.acquisition_id, + ) + except PhysicalCommandLedgerError: + return False + direct_start_successor = bool( + record.parent_operation_id == checkpoint.original_start_operation_id + and ancestry is None + and self._checkpoint_binding_matches_physical_connection( + checkpoint.current_binding, + record, + ) + ) + classified_stop_successor = bool( + record.parent_operation_id != checkpoint.original_start_operation_id + and ancestry is not None + and ancestry.ledger_revision == record.revision + and ancestry.head_operation_id == record.operation_id + and ancestry.head_parent_operation_id == record.parent_operation_id + and ancestry.root_start_operation_id + == checkpoint.original_start_operation_id + and ancestry.acquisition_id == checkpoint.acquisition_id + # A read-only SCANNING classification rotates only the volatile + # control generation. The next explicit STOP must still target + # the checkpoint's exact device transport and endpoint. + and record.connection.transport_ref + == checkpoint.connection.transport_ref + and record.connection.connection_mode + == checkpoint.connection.connection_mode + and record.connection.target_ipv4 == checkpoint.connection.target_ipv4 + and record.connection.target_port == checkpoint.connection.target_port + ) + return direct_start_successor or classified_stop_successor + + def _active_checkpoint_allows_restart_start_descendant( + self, + *, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, + ) -> bool: + """Accept one exact read-only SCANNING descendant around gap fsync. + + Explicit Verify durably appends the physical reconciliation before it + records the local receiver gap. A crash or storage failure between + those publications must remain retryable without treating the newer + physical revision as unrelated. If a gap is already open, its failed + binding still has to be the checkpoint's exact current binding. + """ + + persisted = checkpoint.current_active_physical_proof + status = checkpoint.current_active_status_proof + gap_open = bool( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ) + if not ( + persisted is not None + and status is not None + and ( + not gap_open + or checkpoint.last_gap_failed_binding == checkpoint.current_binding + ) + and record.operation_id == persisted.operation_id + and record.operation_id == checkpoint.original_start_operation_id + and record.action == "start" + and record.stage == "resolved" + and record.revision > persisted.ledger_revision + and record.resolution + in {"start-active-observed", "physical-active-observed"} + and record.acquisition_id == checkpoint.acquisition_id + and record.payload_sha256 == checkpoint.start_payload_sha256 + and record.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and record.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and record.compatibility_profile_id == checkpoint.compatibility_profile_id + and record.reconciliations + ): + return False + latest = record.reconciliations[-1] + observation = latest.observation + return bool( + latest.kind in {"ambiguous-outcome", "resolved-active-rebind"} + and latest.resolution == "physical-active-observed" + and latest.original_attempt.operation_id + == checkpoint.original_start_operation_id + and latest.original_attempt.acquisition_id == checkpoint.acquisition_id + and latest.original_attempt.payload_sha256 + == checkpoint.start_payload_sha256 + and latest.verified_binding.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and latest.verified_binding.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and latest.verified_binding.connection.transport_ref + == checkpoint.connection.transport_ref + and latest.verified_binding.connection.connection_mode + == checkpoint.connection.connection_mode + and latest.verified_binding.connection.target_ipv4 + == checkpoint.connection.target_ipv4 + and latest.verified_binding.connection.target_port + == checkpoint.connection.target_port + and observation.source == "explicit-read-only-reconciliation" + and observation.session_state == "scanning" + and observation.project_bound + and observation.init_ready + and not observation.mqtt_retained + and observation.project_id_sha256 == checkpoint.active_project_id_sha256 + ) + + def _active_checkpoint_allows_reset_reopened_start_descendant( + self, + *, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, + ) -> bool: + """Admit one append-only reset/reopen descendant of a proven START. + + A scenario reset is allowed to retain the immutable active checkpoint + while locally retiring and later reopening the same physical UUID for + read-only settlement. Repeated resets can leave the ledger at an + ``observing`` checkpoint even though the original successful START and + its active rebinds remain fully proven. This admission is deliberately + limited to an initial terminal START proof carried byte-for-byte by a + validated reconciliation/retirement audit; it grants only the token + needed to consume a fresh READY/SCANNING observation. + """ + + persisted = checkpoint.current_active_physical_proof + status = checkpoint.current_active_status_proof + if not ( + persisted is not None + and status is not None + and persisted.action == "start" + and persisted.resolution == "start-active-observed" + and persisted.reconciliation_kind is None + and persisted.reconciliation_resolution is None + and record.operation_id == persisted.operation_id + and record.operation_id == checkpoint.original_start_operation_id + and record.acquisition_id == checkpoint.acquisition_id + and record.action == "start" + and record.stage == "observing" + and record.resolution is None + and record.revision > persisted.ledger_revision + and record.payload_sha256 == checkpoint.start_payload_sha256 + and record.reopened_resolved_start_requires_reconciliation + and record.reconciled_physical_state == "active" + ): + return False + + attempts = [ + *(item.original_attempt for item in record.reconciliations), + *(item.original_attempt for item in record.operator_retirements), + ] + original = next( + ( + attempt + for attempt in attempts + if attempt.revision == persisted.ledger_revision + and attempt.operation_id == persisted.operation_id + and attempt.acquisition_id == checkpoint.acquisition_id + and attempt.action == "start" + and attempt.stage == "resolved" + and attempt.resolution == "start-active-observed" + and attempt.payload_sha256 == checkpoint.start_payload_sha256 + and attempt.compatibility_profile_id + == checkpoint.compatibility_profile_id + and attempt.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and attempt.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and attempt.last_status is not None + and attempt.last_status.source == status.source + and attempt.last_status.session_state == status.session_state + and attempt.last_status.session_state_code + == status.session_state_code + and attempt.last_status.project_id_sha256 + == status.project_id_sha256 + and attempt.last_status.init_ready == status.init_ready + and attempt.last_status.status_message_sha256 + == status.status_message_sha256 + and attempt.last_status.mqtt_retained is False + and attempt.last_status.observed_at_utc == status.observed_at_utc + and self._active_acquisition_checkpoint_transition_id( + "physical-proof", + attempt.operation_id, + attempt.revision, + "normal", + ) + == persisted.proof_id + ), + None, + ) + if original is None or not self._checkpoint_binding_matches_physical_connection( + status.binding, + record, + ): + return False + + latest_active = next( + ( + item + for item in reversed(record.reconciliations) + if item.original_attempt.operation_id == record.operation_id + and item.kind == "resolved-active-rebind" + ), + None, + ) + active_status = ( + latest_active.observation + if latest_active is not None + else original.last_status + ) + return bool( + active_status is not None + and active_status.session_state == "scanning" + and active_status.project_bound + and active_status.init_ready + and not active_status.mqtt_retained + and active_status.project_id_sha256 + == checkpoint.active_project_id_sha256 + ) + + def _checkpoint_trust_token( + self, + *, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, + ) -> _ActiveAcquisitionCheckpointTrustToken: + if checkpoint.state not in {"prepared", "active"}: + raise ActiveAcquisitionRecoveryCheckpointError( + "ceased checkpoint cannot issue a recovery trust token" + ) + persisted = checkpoint.current_active_physical_proof + if checkpoint.state == "active": + assert persisted is not None + checkpoint_head_operation_id = persisted.operation_id + checkpoint_head_revision = persisted.ledger_revision + checkpoint_head_document = persisted.as_dict() + else: + checkpoint_head_operation_id = checkpoint.original_start_operation_id + checkpoint_head_revision = record.revision + checkpoint_head_document = { + "operation_id": checkpoint.original_start_operation_id, + "payload_sha256": checkpoint.start_payload_sha256, + "binding": checkpoint.prepared_binding.as_dict(), + } + return _ActiveAcquisitionCheckpointTrustToken( + trust="trusted", + checkpoint_revision=checkpoint.revision, + checkpoint_state=cast( + Literal["prepared", "active"], + checkpoint.state, + ), + transport_revision=checkpoint.transport_revision, + acquisition_id=checkpoint.acquisition_id, + root_start_operation_id=checkpoint.original_start_operation_id, + current_evidence_session_id=checkpoint.current_evidence_session_id, + checkpoint_head_operation_id=checkpoint_head_operation_id, + checkpoint_head_revision=checkpoint_head_revision, + checkpoint_head_proof_sha256=_sha256_audit_document( + checkpoint_head_document + ), + physical_head_operation_id=record.operation_id, + physical_head_revision=record.revision, + checkpoint=checkpoint, + physical_record=record, + ) + + def _physical_record_descends_from_checkpoint_token( + self, + token: _ActiveAcquisitionCheckpointTrustToken, + current: PhysicalCommandRecord, + ) -> bool: + prior = token.physical_record + in_main_prefix = bool( + current.schema_version == prior.schema_version + and current.operation_id == prior.operation_id + and current.parent_operation_id == prior.parent_operation_id + and current.acquisition_id == prior.acquisition_id + and current.action == prior.action + and current.identity == prior.identity + and current.connection == prior.connection + and current.compatibility_profile_id == prior.compatibility_profile_id + and current.payload_sha256 == prior.payload_sha256 + and current.baseline_status == prior.baseline_status + and current.created_at_utc == prior.created_at_utc + and current.revision >= prior.revision + and current.reconciliations[: len(prior.reconciliations)] + == prior.reconciliations + and current.operator_confirmations[: len(prior.operator_confirmations)] + == prior.operator_confirmations + and current.operator_retirements[: len(prior.operator_retirements)] + == prior.operator_retirements + and current.operator_reconciliation_reopens[ + : len(prior.operator_reconciliation_reopens) + ] + == prior.operator_reconciliation_reopens + ) + if in_main_prefix: + return True + return self._physical_command_ledger.prove_immediate_archived_reconciliation_successor( + expected_current=current, + predecessor=prior, + ) + + @staticmethod + def _active_reconciliation_matches_checkpoint_token( + token: _ActiveAcquisitionCheckpointTrustToken, + current: PhysicalCommandRecord, + ) -> bool: + """Bind a post-token SCANNING proof to the checkpoint's exact K1/project.""" + + if token.checkpoint_state != "active" or not current.reconciliations: + return False + checkpoint = token.checkpoint + reconciliation = current.reconciliations[-1] + verified = reconciliation.verified_binding + connection = verified.connection + observation = reconciliation.observation + return bool( + reconciliation.resolution == "physical-active-observed" + and observation.source == "explicit-read-only-reconciliation" + and observation.session_state == "scanning" + and observation.project_bound + and observation.init_ready + and not observation.mqtt_retained + and observation.project_id_sha256 + == checkpoint.active_project_id_sha256 + and verified.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and verified.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and connection.transport_ref == checkpoint.connection.transport_ref + and connection.connection_mode == checkpoint.connection.connection_mode + and connection.target_ipv4 == checkpoint.connection.target_ipv4 + and connection.target_port == checkpoint.connection.target_port + and observation.control_session_id == connection.control_session_id + and observation.host_path_epoch == connection.host_path_epoch + and observation.producer_generation == connection.producer_generation + ) + + def _active_acquisition_checkpoint_trust_token_is_current( + self, + token: _ActiveAcquisitionCheckpointTrustToken | None, + *, + reconciled_record: Mapping[str, Any] | None = None, + ) -> bool: + """CAS-check the exact checkpoint and expected ledger descendant.""" + + if ( + token is None + or token.trust != "trusted" + or getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + != "trusted" + or self._active_acquisition_checkpoint is None + ): + return False + try: + checkpoint_snapshot = self._active_acquisition_checkpoint.snapshot() + ledger_snapshot = self._physical_command_ledger.snapshot() + current = ledger_snapshot.record + if not ( + checkpoint_snapshot.status == token.checkpoint_state + and checkpoint_snapshot.checkpoint == token.checkpoint + and token.checkpoint_revision == token.checkpoint.revision + and token.transport_revision == token.checkpoint.transport_revision + and token.acquisition_id == token.checkpoint.acquisition_id + and token.root_start_operation_id + == token.checkpoint.original_start_operation_id + and token.current_evidence_session_id + == token.checkpoint.current_evidence_session_id + and ledger_snapshot.status != "corrupt" + and current is not None + and self._physical_record_descends_from_checkpoint_token( + token, + current, + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint trust token changed before recovery commit" + ) + if reconciled_record is not None and ( + reconciled_record.get("operation_id") != current.operation_id + or reconciled_record.get("revision") != current.revision + or dict(reconciled_record) != current.as_dict() + or not self._active_reconciliation_matches_checkpoint_token( + token, + current, + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "physical reconciliation changed after trust validation" + ) + return True + except ( + ActiveAcquisitionRecoveryCheckpointError, + PhysicalCommandLedgerError, + OSError, + ValueError, + ) as exc: + self._mark_active_acquisition_checkpoint_untrusted( + trust="unavailable", + reason_code=str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-token-superseded", + ) + ), + ) + return False + + def _validate_active_acquisition_checkpoint_lineage( + self, + ) -> _ActiveAcquisitionCheckpointTrustToken | None: + """Recompute the current unceased checkpoint/ledger lineage locally. + + The checkpoint stores a bounded S1/S2 certificate, while the physical + ledger and immutable archive retain its proof material. This method is + deliberately read-only: a mismatch revokes local START/resume admission + for the process lifetime and never repairs either durable document. + """ + + if getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) != "trusted": + return None + store = self._active_acquisition_checkpoint + if store is None: + self._mark_active_acquisition_checkpoint_untrusted( + trust="unavailable", + reason_code="active-acquisition-recovery-checkpoint-unavailable", + ) + return None + try: + checkpoint_snapshot = store.snapshot() + if checkpoint_snapshot.status == "corrupt": + self._mark_active_acquisition_checkpoint_untrusted( + trust="corrupt", + reason_code=( + checkpoint_snapshot.reason_code + or "active-acquisition-recovery-checkpoint-corrupt" + ), + ) + return None + checkpoint = checkpoint_snapshot.checkpoint + if checkpoint is None or checkpoint.state == "ceased": + return None + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if ledger_snapshot.status == "corrupt" or record is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint lineage lacks an exact physical record" + ) + if not _physical_start_checkpoint_immutables_match( + checkpoint, + record, + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint immutable START lineage changed" + ) + if checkpoint.state == "prepared": + # DISPATCHING/OBSERVING remains an ambiguous original START for + # a future explicit Verify. RESOLVED/no-dispatch should already + # have been settled under the startup process lease. + if record.stage == "resolved" and record.resolution == "not-dispatched": + raise ActiveAcquisitionRecoveryCheckpointError( + "prepared checkpoint was not settled after no-dispatch" + ) + return self._checkpoint_trust_token( + checkpoint=checkpoint, + record=record, + ) + + persisted = checkpoint.current_active_physical_proof + if persisted is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "active checkpoint lacks its physical head" + ) + exact_checkpoint_head = bool( + record.operation_id == persisted.operation_id + and record.revision == persisted.ledger_revision + ) + if exact_checkpoint_head: + reconciliation = ( + record.reconciliations[-1] + if persisted.reconciliation_kind is not None + and record.reconciliations + else None + ) + recomputed = self._checkpoint_physical_proof( + record=record, + binding=persisted.binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + if recomputed != persisted: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint physical ancestry certificate changed" + ) + elif not ( + self._active_checkpoint_allows_physical_stop_successor( + checkpoint=checkpoint, + record=record, + ) + or self._active_checkpoint_allows_restart_start_descendant( + checkpoint=checkpoint, + record=record, + ) + or self._active_checkpoint_allows_reset_reopened_start_descendant( + checkpoint=checkpoint, + record=record, + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "active checkpoint physical head is no longer current" + ) + # Fence the read-only computation against a concurrent checkpoint + # transition. A changed revision is retried only by a later call. + after = store.snapshot() + if ( + after.status != checkpoint_snapshot.status + or after.checkpoint is None + or after.checkpoint != checkpoint + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint changed during ancestry validation" + ) + return self._checkpoint_trust_token( + checkpoint=checkpoint, + record=record, + ) + except ( + ActiveAcquisitionRecoveryCheckpointError, + PhysicalCommandLedgerError, + OSError, + ValueError, + ) as exc: + self._mark_active_acquisition_checkpoint_untrusted( + trust="unavailable", + reason_code=str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-lineage-unavailable", + ) + ), + ) + return None + + def _require_active_acquisition_checkpoint_store( + self, + ) -> ActiveAcquisitionRecoveryCheckpointStore: + if getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) != "trusted": + raise ActiveAcquisitionRecoveryCheckpointError( + "active acquisition recovery checkpoint is not locally trusted" + ) + store = self._active_acquisition_checkpoint + if store is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "active acquisition recovery checkpoint is unavailable" + ) + snapshot = store.snapshot() + if snapshot.status == "corrupt": + raise ActiveAcquisitionRecoveryCheckpointError( + "active acquisition recovery checkpoint is corrupt" + ) + return store + + def _active_acquisition_checkpoint_public_snapshot(self) -> dict[str, object]: + """Return bounded, secret-free checkpoint facts and no authority.""" + + store = self._active_acquisition_checkpoint + with self._lock: + local_reason = self._active_acquisition_checkpoint_reason + local_trust = getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + pending = self._restart_rehydration_pending + restart_phase = pending.phase if pending is not None else "inactive" + restart_reason = pending.reason_code if pending is not None else None + if store is None: + return { + "status": "unavailable", + "revision": None, + "state": None, + "transport_revision": None, + "gap_open": False, + "restart_resume_authority": False, + "restart_rehydration_phase": restart_phase, + "restart_rehydration_reason_code": restart_reason, + "reason_code": ( + local_reason + or "active-acquisition-recovery-checkpoint-unavailable" + ), + } + try: + snapshot = store.snapshot() + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-unavailable", + ) + ) + self._set_active_acquisition_checkpoint_reason(reason) + return { + "status": "unavailable", + "revision": None, + "state": None, + "transport_revision": None, + "gap_open": False, + "restart_resume_authority": False, + "restart_rehydration_phase": restart_phase, + "restart_rehydration_reason_code": restart_reason, + "reason_code": reason, + } + checkpoint = snapshot.checkpoint + return { + "status": local_trust if local_trust != "trusted" else snapshot.status, + "revision": checkpoint.revision if checkpoint is not None else None, + "state": checkpoint.state if checkpoint is not None else None, + "transport_revision": ( + checkpoint.transport_revision if checkpoint is not None else None + ), + "gap_open": bool( + checkpoint is not None + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ), + "restart_resume_authority": False, + "restart_rehydration_phase": restart_phase, + "restart_rehydration_reason_code": restart_reason, + "reason_code": local_reason or snapshot.reason_code, + } + + def _active_acquisition_checkpoint_start_operation_id( + self, + *, + acquisition_id: str, + evidence_session_id: str, + local_start_operation_id: str | None, + ) -> str | None: + """Return the checkpoint-owned original START without granting authority. + + The operation journal pointer is cleared after the START operation is + completed. The active checkpoint deliberately retains that immutable + physical lineage for later STOP and same-process recovery. A present + local pointer must agree exactly; absence alone is not a mismatch. + """ + + store = self._active_acquisition_checkpoint + if getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) != "trusted": + return None + if store is None: + return local_start_operation_id + try: + snapshot = store.snapshot() + checkpoint = snapshot.checkpoint + if snapshot.status == "empty" and checkpoint is None: + # Legacy same-process tests/installs may still have a live + # operation-journal pointer but no checkpoint predecessor. + # This narrow fallback cannot outlive START completion and + # does not grant restart/resume authority. + return local_start_operation_id + if not ( + snapshot.status in {"prepared", "active"} + and checkpoint is not None + and checkpoint.acquisition_id == acquisition_id + and checkpoint.current_evidence_session_id == evidence_session_id + and checkpoint.current_binding.runtime_instance_id + == self._snapshot_runtime_id + and ( + local_start_operation_id is None + or checkpoint.original_start_operation_id + == local_start_operation_id + ) + ): + return None + return checkpoint.original_start_operation_id + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError): + return None + + def _trusted_recovery_only_checkpoint_lineage( + self, + acquisition: AcquisitionRecord | None, + ) -> _RestartStopOnlyCheckpointLineage | None: + """Return the process-local checkpoint CAS for one STOP-only shell.""" + + if ( + acquisition is None + or acquisition.control_mode != "plugin-commanded" + or acquisition.message_code + != "acquisition.physical_active_recovered_stop_only" + or not isinstance(acquisition.result, Mapping) + or acquisition.result.get("recovery_only") is not True + or getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + != "trusted" + or self._active_acquisition_checkpoint is None + ): + return None + with self._lock: + owner = self._restart_stop_only_checkpoint_owner + prepared_pending = self._restart_stop_only_prepared_pending + if owner is None or owner[0] is not acquisition: + return None + lineage = owner[1] + if lineage.acquisition_id != acquisition.acquisition_id: + return None + try: + snapshot = self._active_acquisition_checkpoint.snapshot() + checkpoint = snapshot.checkpoint + if ( + snapshot.status == "prepared" + and checkpoint is not None + and prepared_pending is not None + and checkpoint == prepared_pending.token.checkpoint + and checkpoint.acquisition_id == acquisition.acquisition_id + and checkpoint.revision == lineage.checkpoint_revision + and checkpoint.original_start_operation_id + == lineage.original_start_operation_id + and checkpoint.current_evidence_session_id + == lineage.prior_evidence_session_id + and prepared_pending.new_evidence_session_id + == lineage.cessation_evidence_session_id + and prepared_pending.origin_proof is not None + ): + return lineage + if not ( + snapshot.status == "active" + and checkpoint is not None + and checkpoint.acquisition_id == acquisition.acquisition_id + and checkpoint.revision == lineage.checkpoint_revision + and checkpoint.original_start_operation_id + == lineage.original_start_operation_id + and checkpoint.current_evidence_session_id + == lineage.prior_evidence_session_id + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + and checkpoint.last_gap_failed_binding + == checkpoint.current_binding + and bool(lineage.cessation_evidence_session_id) + ): + return None + return lineage + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError): + return None + + @staticmethod + def _checkpoint_verified_control( + control: Mapping[str, Any], + ) -> Mapping[str, Any]: + verified = control.get("verified_control") + if not isinstance(verified, Mapping): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint transition requires verified MQTT control" + ) + required_strings = ( + "logical_device_id", + "compatibility_profile_id", + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "control_session_id", + ) + if any( + not isinstance(verified.get(field), str) or not verified.get(field) + for field in required_strings + ) or any( + not isinstance(verified.get(field), int) + or isinstance(verified.get(field), bool) + or int(verified[field]) < 1 + for field in ("target_port", "host_path_epoch", "producer_generation") + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint control proof is incomplete" + ) + return cast(Mapping[str, Any], verified) + + def _checkpoint_transport_binding( + self, + *, + identity_vendor_sha256: str, + identity_serial_sha256: str, + compatibility_profile_id: str, + connection: Mapping[str, Any], + verified_control: Mapping[str, Any], + ) -> ActiveAcquisitionRecoveryTransportBinding: + exact_fields = ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + if any( + connection.get(field) != verified_control.get(field) + for field in exact_fields + ) or verified_control.get("compatibility_profile_id") != compatibility_profile_id: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint control and physical bindings do not match" + ) + return ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=self._snapshot_runtime_id, + intent_id=str(connection["intent_id"]), + transport_ref=str(connection["transport_ref"]), + connection_mode=cast(Any, connection["connection_mode"]), + target_ipv4=str(connection["target_ipv4"]), + target_port=int(connection["target_port"]), + host_path_epoch=int(connection["host_path_epoch"]), + control_session_id=str(connection["control_session_id"]), + # This is the MQTT DeviceInfo-verified control generation. It is + # intentionally unrelated to the viewer/runtime producer epoch. + producer_generation=int(connection["producer_generation"]), + logical_device_id=str(verified_control["logical_device_id"]), + compatibility_profile_id=compatibility_profile_id, + vendor_device_id_sha256=identity_vendor_sha256, + device_serial_sha256=identity_serial_sha256, + ) + + @staticmethod + def _checkpoint_status_proof( + *, + status: PhysicalCommandStatusEvidence, + binding: ActiveAcquisitionRecoveryTransportBinding, + evidence_session_id: str, + ) -> ActiveAcquisitionRecoveryStatusProof: + if status.session_state not in {"ready", "scanning", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointError( + "physical status is not a checkpoint terminal proof" + ) + return ActiveAcquisitionRecoveryStatusProof( + source=cast(Any, status.source), + session_state=cast(Any, status.session_state), + session_state_code=status.session_state_code, + binding=binding, + evidence_session_id=evidence_session_id, + project_id_sha256=status.project_id_sha256, + init_ready=status.init_ready, + status_message_sha256=status.status_message_sha256, + mqtt_retained=cast(Any, status.mqtt_retained), + observed_at_utc=status.observed_at_utc, + ) + + def _checkpoint_physical_proof( + self, + *, + record: PhysicalCommandRecord, + binding: ActiveAcquisitionRecoveryTransportBinding, + checkpoint: ActiveAcquisitionRecoveryCheckpoint | None, + reconciliation: PhysicalCommandReconciliation | None = None, + ) -> ActiveAcquisitionRecoveryPhysicalLineageProof: + if record.stage != "resolved" or record.resolution is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint physical lineage is not terminal" + ) + if record.schema_version != ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint physical lineage schema changed" + ) + if reconciliation is not None and ( + not record.reconciliations or record.reconciliations[-1] != reconciliation + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint reconciliation is not the physical ledger head" + ) + original_start_operation_id = ( + record.operation_id + if record.action == "start" + else checkpoint.original_start_operation_id + if checkpoint is not None + else record.parent_operation_id + ) + if not isinstance(original_start_operation_id, str): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint STOP lacks original START lineage" + ) + if checkpoint is not None and ( + checkpoint.original_start_operation_id != original_start_operation_id + or checkpoint.acquisition_id != record.acquisition_id + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint and physical START lineage changed" + ) + original_start_payload_sha256 = ( + record.payload_sha256 + if record.action == "start" + else checkpoint.start_payload_sha256 + if checkpoint is not None + else None + ) + if original_start_payload_sha256 is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint STOP lacks original START payload" + ) + status = ( + reconciliation.observation + if reconciliation is not None + else record.last_status + ) + proof_material = ( + record.operation_id, + record.revision, + reconciliation.reconciliation_id if reconciliation is not None else "normal", + ) + try: + ancestor_chain = ( + self._physical_command_ledger.prove_classified_stop_ancestry( + head_operation_id=record.operation_id, + expected_start_operation_id=original_start_operation_id, + acquisition_id=record.acquisition_id, + ) + if record.action == "stop" + else None + ) + except PhysicalCommandLedgerError as exc: + raise ActiveAcquisitionRecoveryCheckpointError( + "checkpoint STOP ancestry cannot be proven from the physical ledger" + ) from exc + return ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=record.revision, + proof_id=self._active_acquisition_checkpoint_transition_id( + "physical-proof", + *proof_material, + ), + operation_id=record.operation_id, + original_start_operation_id=original_start_operation_id, + parent_operation_id=record.parent_operation_id, + acquisition_id=record.acquisition_id, + action=cast(Any, record.action), + resolution=cast(Any, record.resolution), + payload_sha256=record.payload_sha256, + original_start_payload_sha256=original_start_payload_sha256, + reconciliation_kind=( + cast(Any, reconciliation.kind) if reconciliation is not None else None + ), + reconciliation_resolution=( + cast(Any, reconciliation.resolution) + if reconciliation is not None + else None + ), + status_message_sha256=( + status.status_message_sha256 if status is not None else None + ), + observed_session_state=( + cast(Any, status.session_state) if status is not None else None + ), + binding=binding, + composite_complete=record.resolution + in {"start-active-observed", "stop-standby-observed"}, + edge_terminal=True, + late_start_excluded=True, + stop_fence=( + "definitely-not-dispatched" + if record.action == "stop" and record.resolution == "not-dispatched" + else "none" + ), + observed_at_utc=( + status.observed_at_utc if status is not None else record.updated_at_utc + ), + ancestor_chain=ancestor_chain, + reconciliation_original_attempt_sha256=( + reconciliation.original_attempt_sha256 + if reconciliation is not None + else None + ), + ) + + def _checkpoint_reconciled_start_origin_proof( + self, + *, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, + reconciliation: PhysicalCommandReconciliation, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, + ) -> ActiveAcquisitionRecoveryStartOriginProof: + """Project the immutable crash-era START edge without replay authority.""" + + attempt = reconciliation.original_attempt + active_observation = bool( + reconciliation.resolution == "physical-active-observed" + and reconciliation.observation.session_state == "scanning" + and reconciliation.observation.project_bound + and reconciliation.observation.init_ready + ) + standby_observation = bool( + reconciliation.resolution == "physical-standby-observed" + and reconciliation.observation.session_state in {"ready", "scan_over"} + and not reconciliation.observation.project_bound + and not reconciliation.observation.init_ready + ) + if not ( + checkpoint.state == "prepared" + and record.operation_id == checkpoint.original_start_operation_id + and record.acquisition_id == checkpoint.acquisition_id + and record.action == "start" + and record.payload_sha256 == checkpoint.start_payload_sha256 + and attempt.operation_id == checkpoint.original_start_operation_id + and attempt.acquisition_id == checkpoint.acquisition_id + and attempt.action == "start" + and attempt.payload_sha256 == checkpoint.start_payload_sha256 + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and (active_observation or standby_observation) + and not reconciliation.observation.mqtt_retained + and physical_proof.operation_id == record.operation_id + and physical_proof.ledger_revision == record.revision + and physical_proof.reconciliation_original_attempt_sha256 + == reconciliation.original_attempt_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "prepared restart lacks one exact reconciled START edge" + ) + if ( + attempt.identity.vendor_device_id_sha256 + != checkpoint.identity.vendor_device_id_sha256 + or attempt.identity.device_serial_sha256 + != checkpoint.identity.device_serial_sha256 + or attempt.compatibility_profile_id != checkpoint.compatibility_profile_id + or attempt.connection.transport_ref != checkpoint.connection.transport_ref + or attempt.connection.connection_mode != checkpoint.connection.connection_mode + or attempt.connection.target_ipv4 != checkpoint.connection.target_ipv4 + or attempt.connection.target_port != checkpoint.connection.target_port + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "reconciled START origin changed checkpoint identity or target" + ) + baseline = self._checkpoint_status_proof( + status=attempt.baseline_status, + binding=checkpoint.prepared_binding, + evidence_session_id=checkpoint.original_evidence_session_id, + ) + composite_origin = bool( + attempt.stage == "resolved" + and attempt.resolution == "start-active-observed" + ) + ambiguous_origin = bool( + attempt.stage in {"dispatching", "observing"} + and attempt.resolution is None + ) + if composite_origin: + if not ( + (reconciliation.kind == "resolved-active-rebind" and active_observation) + or ( + reconciliation.kind == "resolved-active-cessation" + and standby_observation + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "resolved START origin has the wrong physical classification" + ) + origin_kind: Literal["composite-resolved", "ambiguous-reconciled"] = ( + "composite-resolved" + ) + project_strength: Literal[ + "exact-vendor-project-id", "edge-correlated-vendor-project-id" + ] = "exact-vendor-project-id" + original_project_id_sha256 = ( + attempt.last_status.project_id_sha256 + if attempt.last_status is not None + else None + ) + if original_project_id_sha256 is None or ( + active_observation + and reconciliation.observation.project_id_sha256 + != original_project_id_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "resolved START origin changed or lacks its exact vendor project" + ) + elif ambiguous_origin: + if not ( + ( + reconciliation.kind + in {"ambiguous-outcome", "resolved-active-rebind"} + and active_observation + ) + or ( + reconciliation.kind + in {"ambiguous-outcome", "resolved-active-cessation"} + and standby_observation + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "ambiguous START origin has the wrong physical classification" + ) + if reconciliation.kind == "resolved-active-rebind": + prior_active_project_id_sha256 = next( + ( + item.observation.project_id_sha256 + for item in reversed(record.reconciliations[:-1]) + if item.original_attempt_sha256 + == reconciliation.original_attempt_sha256 + and item.resolution == "physical-active-observed" + ), + None, + ) + if ( + prior_active_project_id_sha256 is None + or reconciliation.observation.project_id_sha256 + != prior_active_project_id_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "ambiguous START rebind changed or lacks its edge-correlated project" + ) + origin_kind = "ambiguous-reconciled" + project_strength = "edge-correlated-vendor-project-id" + original_project_id_sha256 = None + else: + raise ActiveAcquisitionRecoveryCheckpointError( + "prepared restart reconciliation is not a START origin" + ) + return ActiveAcquisitionRecoveryStartOriginProof( + schema_version=ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA, + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=record.revision, + physical_proof_id=physical_proof.proof_id, + reconciliation_id=reconciliation.reconciliation_id, + origin_kind=origin_kind, + original_attempt_revision=attempt.revision, + original_attempt_stage=cast(Any, attempt.stage), + original_attempt_resolution=cast(Any, attempt.resolution), + original_attempt_sha256=reconciliation.original_attempt_sha256, + operation_id=attempt.operation_id, + parent_operation_id=attempt.parent_operation_id, + acquisition_id=attempt.acquisition_id, + payload_sha256=attempt.payload_sha256, + baseline_status_proof=baseline, + original_project_id_sha256=original_project_id_sha256, + automatic_replay_allowed=False, + project_evidence_strength=project_strength, + reconciled_active_project_id_sha256=( + reconciliation.observation.project_id_sha256 + if active_observation + else None + ), + ) + + def _prepared_restart_reconciliation_is_current( + self, + token: _ActiveAcquisitionCheckpointTrustToken, + *, + record: PhysicalCommandRecord, + reconciliation: PhysicalCommandReconciliation, + ) -> bool: + """CAS a PREPARED checkpoint against its post-crash read-only head.""" + + store = self._active_acquisition_checkpoint + if token.checkpoint_state != "prepared" or store is None: + return False + try: + checkpoint_snapshot = store.snapshot() + ledger_snapshot = self._physical_command_ledger.snapshot() + active_reconciliation = bool( + reconciliation.resolution == "physical-active-observed" + and reconciliation.kind + in {"ambiguous-outcome", "resolved-active-rebind"} + ) + standby_reconciliation = bool( + reconciliation.resolution == "physical-standby-observed" + and reconciliation.kind + in {"ambiguous-outcome", "resolved-active-cessation"} + ) + return bool( + checkpoint_snapshot.status == "prepared" + and checkpoint_snapshot.checkpoint == token.checkpoint + and ledger_snapshot.status == "resolved" + and ledger_snapshot.record == record + and record.reconciliations + and record.reconciliations[-1] == reconciliation + and self._physical_record_descends_from_checkpoint_token( + token, + record, + ) + and (active_reconciliation or standby_reconciliation) + ) + except ( + ActiveAcquisitionRecoveryCheckpointError, + PhysicalCommandLedgerError, + OSError, + ValueError, + ): + return False + + def _settle_restart_checkpoint_after_verified_standby( + self, + *, + token: _ActiveAcquisitionCheckpointTrustToken | None, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + ) -> bool: + """Persist a fresh restart READY/SCAN_OVER without issuing STOP. + + The physical ledger already proves that the scanner is not running. + This method projects only that immutable read-only fact into the + checkpoint; it never creates a runtime, evidence writer, or command. + """ + + store = self._active_acquisition_checkpoint + if token is None or store is None: + return False + status_proof: ActiveAcquisitionRecoveryStatusProof | None = None + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof | None = None + origin_proof: ActiveAcquisitionRecoveryStartOriginProof | None = None + try: + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.as_dict() == dict(reconciled_record) + and record.reconciliations + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "restart standby settlement lost its exact physical head" + ) + reconciliation = record.reconciliations[-1] + status = reconciliation.observation + if not ( + reconciliation.reconciliation_id == reconciliation_id + and reconciliation.resolution == "physical-standby-observed" + and reconciliation.kind + in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-cessation", + } + and status.source == "explicit-read-only-reconciliation" + and status.session_state in {"ready", "scan_over"} + and not status.project_bound + and not status.init_ready + and not status.mqtt_retained + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "restart standby settlement lacks fresh READY/SCAN_OVER" + ) + if token.checkpoint_state == "prepared": + current = self._prepared_restart_reconciliation_is_current( + token, + record=record, + reconciliation=reconciliation, + ) + else: + checkpoint_snapshot = store.snapshot() + current = bool( + checkpoint_snapshot.status == "active" + and checkpoint_snapshot.checkpoint == token.checkpoint + and self._physical_record_descends_from_checkpoint_token( + token, + record, + ) + ) + if not current: + raise ActiveAcquisitionRecoveryCheckpointError( + "restart standby checkpoint CAS changed" + ) + + checkpoint = token.checkpoint + identity = reconciliation.verified_binding.identity + connection = reconciliation.verified_binding.connection + # Terminal checkpoint cessation grants no control or replay + # authority. Bind it to the durable DeviceInfo-verified + # reconciliation itself so a process exit after the physical + # ledger fsync cannot strand the ACTIVE checkpoint merely because + # that ephemeral inspection worker is gone. + verified_control = { + "logical_device_id": checkpoint.identity.logical_device_id, + "compatibility_profile_id": record.compatibility_profile_id, + **connection.as_dict(), + } + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=verified_control, + ) + evidence_session_id = ( + "restart-standby:sha256-" + + hashlib.sha256( + ( + f"{checkpoint.acquisition_id}\0{self._snapshot_runtime_id}\0" + f"{reconciliation_id}" + ).encode() + ).hexdigest() + ) + status_proof = self._checkpoint_status_proof( + status=status, + binding=binding, + evidence_session_id=evidence_session_id, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + gap_started_at_utc = ( + checkpoint.last_gap_started_at_utc + if checkpoint.state == "active" + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + else status.observed_at_utc + ) + if checkpoint.state == "active": + store.cease_active_reconciled_standby( + transition_id=self._active_acquisition_checkpoint_transition_id( + "cease-active-reconciled-standby", + checkpoint.acquisition_id, + reconciliation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + gap_started_at_utc=gap_started_at_utc, + cessation_status_proof=status_proof, + cessation_physical_proof=physical_proof, + ) + elif checkpoint.state == "prepared" and ( + reconciliation.kind == "resolved-active-cessation" + ): + origin_proof = self._checkpoint_reconciled_start_origin_proof( + checkpoint=checkpoint, + record=record, + reconciliation=reconciliation, + physical_proof=physical_proof, + ) + original_project_id_sha256 = ( + reconciliation.original_attempt.last_status.project_id_sha256 + if reconciliation.original_attempt.last_status is not None + else None + ) + if original_project_id_sha256 is None: + raise ActiveAcquisitionRecoveryCheckpointError( + "resolved START standby lacks its original vendor project" + ) + store.cease_prepared_resolved_start_standby( + transition_id=self._active_acquisition_checkpoint_transition_id( + "cease-prepared-resolved-start-standby", + checkpoint.acquisition_id, + reconciliation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + gap_started_at_utc=gap_started_at_utc, + origin_proof=origin_proof, + cessation_status_proof=status_proof, + cessation_physical_proof=physical_proof, + reconciliation_id=reconciliation.reconciliation_id, + reconciliation_original_attempt_sha256=( + reconciliation.original_attempt_sha256 + ), + reconciliation_original_project_id_sha256=( + original_project_id_sha256 + ), + ) + else: + store.cease( + transition_id=self._active_acquisition_checkpoint_transition_id( + "restart-standby-cease", + checkpoint.acquisition_id, + reconciliation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + physical_proof=physical_proof, + status_proof=status_proof, + ) + self._set_active_acquisition_checkpoint_reason(None) + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + committed_after_error = False + with suppress( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ): + committed = store.snapshot().checkpoint + committed_after_error = bool( + committed is not None + and status_proof is not None + and physical_proof is not None + and committed.state == "ceased" + and committed.acquisition_id == token.acquisition_id + and committed.original_start_operation_id + == token.root_start_operation_id + and committed.current_binding == status_proof.binding + and committed.current_evidence_session_id + == status_proof.evidence_session_id + and committed.cessation_status_proof == status_proof + and committed.cessation_physical_proof == physical_proof + and ( + origin_proof is None + or committed.reconciled_start_origin_proof == origin_proof + ) + ) + if committed_after_error: + self._set_active_acquisition_checkpoint_reason(None) + return True + self._set_active_acquisition_checkpoint_reason( + str( + getattr( + exc, + "reason_code", + "restart-standby-checkpoint-settlement-failed", + ) + ) + ) + return False + + def _settle_durable_prepared_stop_standby_checkpoint( + self, + token: _ActiveAcquisitionCheckpointTrustToken | None, + ) -> bool: + """Finish the exact fsync-after-ledger crash gap without device I/O.""" + + if token is None or token.checkpoint_state != "active": + return False + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.action == "stop" + and record.stage == "resolved" + and record.resolution == "not-dispatched" + and reconciliation is not None + and reconciliation.kind == "prepared-stop-classification" + and reconciliation.resolution == "physical-standby-observed" + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and reconciliation.observation.session_state in {"ready", "scan_over"} + and not reconciliation.observation.project_bound + and not reconciliation.observation.init_ready + and not reconciliation.observation.mqtt_retained + ): + return False + return self._settle_restart_checkpoint_after_verified_standby( + token=token, + reconciliation_id=reconciliation.reconciliation_id, + reconciled_record=record.as_dict(), + ) + + def _observe_start_preparation_checkpoint( + self, + phase: StartPreparationCheckpointPhase, + context: PhysicalCommandIntentContext, + envelope: Any, + ) -> None: + """Persist PREPARED before worker release, or cease proven no-dispatch.""" + + store = self._active_acquisition_checkpoint + if store is None: + if phase == "resolved-not-dispatched": + return + raise ActiveAcquisitionRecoveryCheckpointError( + "START is blocked because its recovery checkpoint is unavailable" + ) + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + project_name = self._acquisition_project_name + mount_type = self._acquisition_mount_type + gnss_mode = self._acquisition_gnss_mode + start_operation_id = self._acquisition_start_operation_id + if phase == "prepared": + if not ( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and acquisition.acquisition_id == context.acquisition_id + and start_operation_id == context.operation_id + and out_dir is not None + and isinstance(project_name, str) + and mount_type == "handheld" + and gnss_mode == "none" + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "START checkpoint lost its exact local acquisition" + ) + checkpoint_snapshot = store.snapshot() + if checkpoint_snapshot.status == "corrupt" or ( + checkpoint_snapshot.checkpoint is not None + and checkpoint_snapshot.checkpoint.state != "ceased" + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "an unceased or corrupt acquisition checkpoint blocks START" + ) + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "unresolved" + and record is not None + and record.stage == "prepared" + and record.action == "start" + and record.operation_id == context.operation_id + and record.acquisition_id == context.acquisition_id + and record.payload_sha256 == envelope.payload_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "physical PREPARED START does not match checkpoint input" + ) + control = self._application_control_session.snapshot() + verified_control = self._checkpoint_verified_control(control) + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=record.identity.vendor_device_id_sha256, + identity_serial_sha256=record.identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=record.connection.as_dict(), + verified_control=verified_control, + ) + store.prepare( + transition_id=self._active_acquisition_checkpoint_transition_id( + "prepare", + context.operation_id, + envelope.payload_sha256, + ), + predecessor_revision=( + checkpoint_snapshot.checkpoint.revision + if checkpoint_snapshot.checkpoint is not None + else 0 + ), + acquisition_id=context.acquisition_id, + original_start_operation_id=context.operation_id, + start_payload_sha256=envelope.payload_sha256, + identity=ActiveAcquisitionRecoveryIdentity( + logical_device_id=str(verified_control["logical_device_id"]), + vendor_device_id_sha256=( + record.identity.vendor_device_id_sha256 + ), + device_serial_sha256=record.identity.device_serial_sha256, + ), + connection=ActiveAcquisitionRecoveryConnection( + transport_ref=record.connection.transport_ref, + connection_mode=record.connection.connection_mode, + target_ipv4=record.connection.target_ipv4, + target_port=record.connection.target_port, + ), + compatibility_profile_id=record.compatibility_profile_id, + project_name=project_name, + project_name_wire_sha256=( + active_acquisition_project_name_sha256(project_name) + ), + original_evidence_session_id=out_dir.name, + duration_seconds=acquisition.duration_seconds, + requested_streams=acquisition.requested_streams, + evidence_policy=acquisition.evidence_policy, + mount_type=mount_type, + gnss_mode=gnss_mode, + prepared_binding=binding, + ) + self._set_active_acquisition_checkpoint_reason(None) + return + + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.operation_id == context.operation_id + and record.acquisition_id == context.acquisition_id + and record.action == "start" + and record.payload_sha256 == envelope.payload_sha256 + and record.resolution == "not-dispatched" + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "failed START lacks durable no-dispatch resolution" + ) + if checkpoint is None: + if checkpoint_snapshot.status == "empty": + return + raise ActiveAcquisitionRecoveryCheckpointError( + "failed START checkpoint state is unavailable or corrupt" + ) + if checkpoint.state == "ceased": + # A callback failure before checkpoint PREPARED was replaced leaves + # the valid ceased predecessor intact. The exact new physical row + # is already sealed no-dispatch by the session; there is therefore + # no new checkpoint transition to settle here. + return + if not ( + checkpoint.state == "prepared" + and checkpoint.acquisition_id == context.acquisition_id + and checkpoint.original_start_operation_id == context.operation_id + and checkpoint.start_payload_sha256 == envelope.payload_sha256 + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "failed START does not own the current prepared checkpoint" + ) + store.cease( + transition_id=self._active_acquisition_checkpoint_transition_id( + "cease-undispatched-start", + record.operation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + physical_proof=self._checkpoint_physical_proof( + record=record, + binding=checkpoint.prepared_binding, + checkpoint=checkpoint, + ), + status_proof=None, + ) + self._set_active_acquisition_checkpoint_reason(None) + + def _observe_application_scanning_checkpoint(self) -> None: + """Activate only after composite START and fresh initialized SCANNING.""" + + try: + store = self._require_active_acquisition_checkpoint_store() + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + start_operation_id = self._acquisition_start_operation_id + if not ( + checkpoint is not None + and acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and checkpoint.acquisition_id == acquisition.acquisition_id + and checkpoint.original_start_operation_id == start_operation_id + and out_dir is not None + and checkpoint.current_evidence_session_id == out_dir.name + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "SCANNING does not own one exact acquisition checkpoint" + ) + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.operation_id == start_operation_id + and record.acquisition_id == acquisition.acquisition_id + and record.action == "start" + and record.resolution == "start-active-observed" + and record.last_status is not None + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "SCANNING lacks composite durable START proof" + ) + verified_control = self._checkpoint_verified_control( + self._application_control_session.snapshot() + ) + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=record.identity.vendor_device_id_sha256, + identity_serial_sha256=record.identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=record.connection.as_dict(), + verified_control=verified_control, + ) + status_proof = self._checkpoint_status_proof( + status=record.last_status, + binding=binding, + evidence_session_id=out_dir.name, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + ) + if checkpoint.state == "active": + if not ( + checkpoint.current_binding == binding + and checkpoint.current_evidence_session_id == out_dir.name + and checkpoint.activation_status_proof == status_proof + and checkpoint.activation_physical_proof == physical_proof + and checkpoint.current_active_status_proof == status_proof + and checkpoint.current_active_physical_proof == physical_proof + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "SCANNING active checkpoint postcondition changed" + ) + self._set_active_acquisition_checkpoint_reason(None) + return + if checkpoint.state != "prepared": + raise ActiveAcquisitionRecoveryCheckpointError( + "SCANNING checkpoint is neither PREPARED nor exact ACTIVE" + ) + store.activate( + transition_id=self._active_acquisition_checkpoint_transition_id( + "activate", + record.operation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + status_proof=status_proof, + physical_proof=physical_proof, + ) + self._set_active_acquisition_checkpoint_reason(None) + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason_code = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-activation-failed", + ) + ) + self._set_active_acquisition_checkpoint_reason(reason_code) + logger.exception( + "active acquisition checkpoint activation failed closed", + extra={ + "event_code": "active_acquisition_checkpoint_activation_failed", + "reason_code": reason_code, + "device_command_retried": False, + }, + ) + raise + + def _active_acquisition_checkpoint_matches_current( + self, + *, + acquisition_id: str, + start_operation_id: str | None, + evidence_session_id: str, + verified_control: Mapping[str, Any] | None, + ) -> bool: + store = self._active_acquisition_checkpoint + if ( + store is None + or getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) + != "trusted" + or verified_control is None + or start_operation_id is None + ): + return False + try: + snapshot = store.snapshot() + checkpoint = snapshot.checkpoint + record = self._physical_command_ledger.snapshot().record + if not ( + snapshot.status == "active" + and checkpoint is not None + and record is not None + and checkpoint.acquisition_id == acquisition_id + and checkpoint.original_start_operation_id == start_operation_id + and checkpoint.current_evidence_session_id == evidence_session_id + and checkpoint.current_binding.runtime_instance_id + == self._snapshot_runtime_id + and not ( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ) + ): + return False + reconciliation = ( + record.reconciliations[-1] if record.reconciliations else None + ) + use_reconciliation = bool( + reconciliation is not None + and reconciliation.kind + in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + } + and reconciliation.resolution == "physical-active-observed" + ) + identity = ( + reconciliation.verified_binding.identity + if use_reconciliation and reconciliation is not None + else record.identity + ) + connection = ( + reconciliation.verified_binding.connection + if use_reconciliation and reconciliation is not None + else record.connection + ) + current_binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=verified_control, + ) + return checkpoint.current_binding == current_binding + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError): + return False + def _observe_runtime_message( self, message: StreamMessage, metrics: BridgeMetrics, ) -> bool: + runtime_snapshot = self.runtime.snapshot() + current_generation = runtime_snapshot.get("producer_generation") + if message.source == "live_mqtt" and ( + runtime_snapshot.get("source_mode") != "live" + or message.producer_generation is None + or message.producer_generation != current_generation + ): + # A stopped or replaced producer may finish an in-flight callback. + # Consume it without letting that packet revive data-plane health or + # contaminate the next session's evidence. + return True + if ( + message.producer_generation is not None + and message.producer_generation != current_generation + ): + return True self._observe_live_perception_mqtt(message) if self._modeling_control_safety.observe(message, metrics): return True return observe_modeling_report(message, metrics) + @staticmethod + def _restart_rehydration_checkpoint_committed( + *, + pending: _RestartRehydrationPending, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + ) -> bool: + """Recognize one exact durable promotion after a lost response.""" + + first_pcl = pending.first_pcl_proof + receipt_kind = ( + "activate-reconciled" + if pending.token.checkpoint_state == "prepared" + else "rebind-active" + ) + transition_id_sha256 = hashlib.sha256( + pending.transition_id.encode() + ).hexdigest() + receipt_present = any( + receipt.transition_id_sha256 == transition_id_sha256 + and receipt.kind == receipt_kind + and receipt.committed_revision == checkpoint.revision + for receipt in checkpoint.transition_receipts + ) + common = bool( + first_pcl is not None + and receipt_present + and checkpoint.state == "active" + and checkpoint.acquisition_id == pending.token.acquisition_id + and checkpoint.original_start_operation_id + == pending.token.root_start_operation_id + and checkpoint.current_evidence_session_id + == pending.new_evidence_session_id + and checkpoint.current_binding == pending.status_proof.binding + and checkpoint.current_active_status_proof == pending.status_proof + and checkpoint.current_active_physical_proof == pending.physical_proof + and checkpoint.physical_lineage_head_revision + == pending.physical_proof.ledger_revision + and checkpoint.last_gap_started_at_utc == pending.gap_started_at_utc + and checkpoint.last_gap_recovered_at_utc + == pending.status_proof.observed_at_utc + ) + if not common: + return False + if pending.token.checkpoint_state == "prepared": + return bool( + pending.origin_proof is not None + and checkpoint.reconciled_start_origin_proof == pending.origin_proof + and checkpoint.first_published_pcl_proof == first_pcl + and checkpoint.activation_status_proof == pending.status_proof + and checkpoint.activation_physical_proof == pending.physical_proof + ) + return bool( + pending.origin_proof is None + and checkpoint.last_rebind_status_proof == pending.status_proof + and checkpoint.last_rebind_physical_proof == pending.physical_proof + ) + + @staticmethod + def _classified_stop_rebind_checkpoint_committed( + *, + pending: _ClassifiedStopRebindPending, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + ) -> bool: + """Recognize the exact durable rebind after a lost store response.""" + + transition_id_sha256 = hashlib.sha256( + pending.transition_id.encode() + ).hexdigest() + return bool( + checkpoint.state == "active" + and checkpoint.revision == pending.checkpoint_gap_revision + 1 + and checkpoint.acquisition_id == pending.acquisition_id + and checkpoint.original_start_operation_id + == pending.root_start_operation_id + and checkpoint.current_evidence_session_id + == pending.evidence_session_id + and checkpoint.current_binding == pending.status_proof.binding + and checkpoint.current_active_status_proof == pending.status_proof + and checkpoint.current_active_physical_proof == pending.physical_proof + and checkpoint.last_rebind_status_proof == pending.status_proof + and checkpoint.last_rebind_physical_proof == pending.physical_proof + and checkpoint.last_gap_failed_binding + == pending.token.checkpoint.current_binding + and checkpoint.last_gap_recovered_at_utc + == pending.status_proof.observed_at_utc + and any( + receipt.transition_id_sha256 == transition_id_sha256 + and receipt.kind == "rebind-active" + and receipt.committed_revision == checkpoint.revision + for receipt in checkpoint.transition_receipts + ) + ) + + def _classified_stop_rebind_current_record( + self, + pending: _ClassifiedStopRebindPending, + *, + producer_generation: int, + ) -> PhysicalCommandRecord | None: + """Revalidate the frozen local/control/ledger claim without mutation.""" + + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + lease = self._acquisition_session_lease + owner = self._prepared_stop_recovery_owner + stop_operation_id = self._acquisition_stop_operation_id + start_operation_id = self._acquisition_start_operation_id + runtime = dict(self.runtime.snapshot()) + if not ( + acquisition is not None + and acquisition.acquisition_id == pending.acquisition_id + and acquisition.control_mode == "plugin-commanded" + and acquisition.state + in {"starting", "awaiting_external_start", "acquiring"} + and out_dir is not None + and out_dir.name == pending.evidence_session_id + and lease is not None + and owner is not None + and owner.acquisition is acquisition + and owner.session_lease is lease + and owner.out_dir == out_dir + and owner.lineage.stop_operation_id + == pending.classified_stop_operation_id + and owner.checkpoint_trust_token == pending.token + and stop_operation_id is None + and start_operation_id in {None, pending.root_start_operation_id} + and runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") == producer_generation + ): + return None + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + physical = dict(self._physical_command_coordinator.snapshot()) + projected_record = physical.get("record") + control = dict(self._application_control_session.snapshot()) + verified_control = control.get("verified_control") + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and isinstance(projected_record, Mapping) + and dict(projected_record) == record.as_dict() + and record.operation_id == pending.classified_stop_operation_id + and record.revision == pending.classified_stop_revision + and record.acquisition_id == pending.acquisition_id + and reconciliation is not None + and reconciliation.reconciliation_id == pending.reconciliation_id + and isinstance(verified_control, Mapping) + and control.get("state") == "scanning" + and control.get("can_stop") is True + and self._matching_classified_prepared_stop_active( + physical, + acquisition_id=pending.acquisition_id, + stop_operation_id=pending.classified_stop_operation_id, + start_operation_id=pending.root_start_operation_id, + verified_control=verified_control, + ) + ): + return None + try: + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=( + reconciliation.verified_binding.identity.vendor_device_id_sha256 + ), + identity_serial_sha256=( + reconciliation.verified_binding.identity.device_serial_sha256 + ), + compatibility_profile_id=record.compatibility_profile_id, + connection=reconciliation.verified_binding.connection.as_dict(), + verified_control=verified_control, + ) + except (ActiveAcquisitionRecoveryCheckpointError, ValueError): + return None + if binding != pending.status_proof.binding: + return None + return record + + def _promote_classified_stop_rebind_from_pcl( + self, + *, + envelope: DecodedPointCloudView, + producer_generation: int, + ) -> bool | None: + """Close a classified-STOP gap only after one exact published PCL.""" + + with self._lock: + pending = self._classified_stop_rebind_pending + if pending is None: + return None + if ( + not envelope.context.live + or envelope.point_count < 1 + or producer_generation != pending.runtime_producer_generation + ): + return False + + # Keep lifecycle ownership through the durable checkpoint replacement. + # STOP may still publish its priority while fsync is in flight because + # the camera gate is held only for the two short admission checks. It + # then waits for lifecycle ownership, and the post-fsync check below + # makes that priority win before any camera authority is granted. + with self._acquisition_lifecycle_access(): + record = self._classified_stop_rebind_current_record( + pending, + producer_generation=producer_generation, + ) + with self._lock: + if ( + record is None + or self._camera_stop_priority_counts.get( + pending.acquisition_id, + 0, + ) + > 0 + or self._classified_stop_rebind_pending != pending + or self._classified_stop_rebind_inflight is not None + ): + return False + self._classified_stop_rebind_inflight = pending + assert record is not None + try: + store = self._require_active_acquisition_checkpoint_store() + committed: ActiveAcquisitionRecoveryCheckpoint | None = None + try: + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + committed = ( + checkpoint + if checkpoint is not None + and self._classified_stop_rebind_checkpoint_committed( + pending=pending, + checkpoint=checkpoint, + ) + else None + ) + if committed is None and not ( + checkpoint_snapshot.status == "active" + and checkpoint is not None + and checkpoint.revision + == pending.checkpoint_gap_revision + and checkpoint.acquisition_id == pending.acquisition_id + and checkpoint.original_start_operation_id + == pending.root_start_operation_id + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + and checkpoint.last_gap_failed_binding + == pending.token.checkpoint.current_binding + ): + return False + if committed is None: + assert checkpoint is not None + committed = store.rebind_active( + transition_id=pending.transition_id, + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + status_proof=pending.status_proof, + physical_proof=pending.physical_proof, + ) + except ( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ) as exc: + # The replacement may have reached disk before the store + # call lost its response. Recognize only this + # transition's exact durable receipt. + with suppress( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ): + after_error = store.snapshot().checkpoint + if after_error is not None and ( + self._classified_stop_rebind_checkpoint_committed( + pending=pending, + checkpoint=after_error, + ) + ): + committed = after_error + if committed is None: + self._set_active_acquisition_checkpoint_reason( + str( + getattr( + exc, + "reason_code", + "classified-stop-checkpoint-rebind-failed", + ) + ) + ) + return False + + # STOP wins every race after fsync. Revalidate local, runtime, + # control and ledger truth before publishing a refreshed token + # or allowing this frame to reach camera activation. + current_record = self._classified_stop_rebind_current_record( + pending, + producer_generation=producer_generation, + ) + after = store.snapshot() + if not ( + current_record == record + and committed is not None + and after.status == "active" + and after.checkpoint == committed + and self._classified_stop_rebind_checkpoint_committed( + pending=pending, + checkpoint=committed, + ) + ): + return False + assert current_record is not None + refreshed_token = self._checkpoint_trust_token( + checkpoint=committed, + record=current_record, + ) + with self._lock: + if not ( + self._camera_stop_priority_counts.get( + pending.acquisition_id, + 0, + ) + == 0 + and self._classified_stop_rebind_pending == pending + and self._classified_stop_rebind_inflight == pending + ): + return False + current_owner = self._prepared_stop_recovery_owner + if not ( + current_owner is not None + and current_owner.acquisition is self._acquisition + and current_owner.lineage.stop_operation_id + == pending.classified_stop_operation_id + ): + return False + self._prepared_stop_recovery_owner = dataclass_replace( + current_owner, + checkpoint_trust_token=refreshed_token, + ) + self._classified_stop_rebind_pending = None + lineage = self._active_stream_recovery_lineage + if ( + lineage is not None + and lineage.acquisition_id == pending.acquisition_id + and lineage.physical_operation_id + == pending.classified_stop_operation_id + and lineage.runtime_producer_generation + == pending.runtime_producer_generation + ): + self._active_stream_recovery_state = "recovered" + self._active_stream_recovery_reason_code = None + self._set_active_acquisition_checkpoint_reason(None) + return True + finally: + # Every exception/return after reservation releases only this + # immutable claim. A newer PCL claim or STOP cleanup cannot be + # cleared by a stale worker. + with self._lock: + if self._classified_stop_rebind_inflight == pending: + self._classified_stop_rebind_inflight = None + + def _promote_restart_rehydration_from_pcl( + self, + *, + envelope: DecodedPointCloudView, + producer_generation: int, + published_pcl_frames: int, + ) -> bool | None: + """Commit one post-restart transport only after Rerun published PCL. + + ``None`` means no restart owner exists. ``False`` consumes the frame + without camera authority. ``True`` means the exact checkpoint fsync is + complete and ordinary post-PCL camera admission may continue. + """ + + with self._lock: + pending = self._restart_rehydration_pending + if pending is None: + return None + if pending.phase != "awaiting-first-pcl": + return False + if pending.runtime_producer_generation != producer_generation: + return False + + with self._acquisition_lifecycle_access(): + with self._lock: + current_pending = self._restart_rehydration_pending + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + lease = self._acquisition_session_lease + stop_priority = self._camera_stop_priority_counts.get( + pending.token.acquisition_id, + 0, + ) + if ( + current_pending != pending + or current_pending.phase != "awaiting-first-pcl" + or stop_priority > 0 + ): + return False + runtime = dict(self.runtime.snapshot()) + control = dict(self._application_control_session.snapshot()) + if not ( + acquisition is not None + and acquisition.acquisition_id == pending.token.acquisition_id + and acquisition.control_mode == "plugin-commanded" + and acquisition.state + in {"starting", "awaiting_external_start", "acquiring"} + and out_dir == pending.out_dir + and lease is not None + and lease.session_root == pending.out_dir + and runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") == producer_generation + and control.get("state") == "scanning" + and control.get("can_stop") is True + ): + return False + + first_pcl = pending.first_pcl_proof + if first_pcl is None: + published_at_utc = _utc_now_millisecond_iso() + first_pcl = ActiveAcquisitionRecoveryFirstPublishedPclProof( + schema_version=( + ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA + ), + proof_id=self._active_acquisition_checkpoint_transition_id( + "restart-first-pcl-proof", + pending.token.acquisition_id, + pending.reconciliation_id, + pending.new_evidence_session_id, + producer_generation, + envelope.context.sequence, + ), + runtime_instance_id=self._snapshot_runtime_id, + evidence_session_id=pending.new_evidence_session_id, + viewer_producer_generation=producer_generation, + frame_sequence=envelope.context.sequence, + captured_at_epoch_ns=envelope.context.captured_at_epoch_ns, + point_count=envelope.point_count, + published_pcl_frames=published_pcl_frames, + published_at_utc=published_at_utc, + ) + frozen = dataclass_replace(pending, first_pcl_proof=first_pcl) + with self._lock: + if self._restart_rehydration_pending != pending: + return False + self._restart_rehydration_pending = frozen + pending = frozen + + if not self._k1_command_dispatch_gate.acquire(blocking=False): + return False + try: + with self._lock: + if ( + self._restart_rehydration_pending != pending + or self._camera_stop_priority_counts.get( + pending.token.acquisition_id, + 0, + ) + > 0 + ): + return False + ledger_snapshot = self._physical_command_ledger.snapshot() + store = self._active_acquisition_checkpoint + if store is None or not ( + ledger_snapshot.status == "resolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.operation_id + == pending.physical_proof.operation_id + and ledger_snapshot.record.revision + == pending.physical_proof.ledger_revision + and ledger_snapshot.record.reconciliations + and ledger_snapshot.record.reconciliations[-1].reconciliation_id + == pending.reconciliation_id + ): + return False + assert pending.first_pcl_proof is not None + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + committed = ( + checkpoint + if checkpoint is not None + and self._restart_rehydration_checkpoint_committed( + pending=pending, + checkpoint=checkpoint, + ) + else None + ) + if committed is None and not ( + checkpoint is not None + and checkpoint.revision == pending.token.checkpoint_revision + and checkpoint.acquisition_id == pending.token.acquisition_id + and checkpoint.original_start_operation_id + == pending.token.root_start_operation_id + and checkpoint.current_evidence_session_id + == pending.old_evidence_session_id + ): + return False + if committed is not None: + pass + elif checkpoint is not None and checkpoint.state == "prepared": + if pending.origin_proof is None: + return False + committed = store.activate_reconciled( + transition_id=pending.transition_id, + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + gap_started_at_utc=pending.gap_started_at_utc, + origin_proof=pending.origin_proof, + status_proof=pending.status_proof, + physical_proof=pending.physical_proof, + first_pcl_proof=pending.first_pcl_proof, + ) + elif checkpoint is not None and checkpoint.state == "active" and ( + checkpoint.last_gap_started_at_utc + == pending.gap_started_at_utc + and checkpoint.last_gap_recovered_at_utc is None + ): + committed = store.rebind_active( + transition_id=pending.transition_id, + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + status_proof=pending.status_proof, + physical_proof=pending.physical_proof, + ) + else: + return False + assert committed is not None + after = store.snapshot() + runtime_after = dict(self.runtime.snapshot()) + ledger_after = self._physical_command_ledger.snapshot() + with self._lock: + local_current = bool( + self._restart_rehydration_pending == pending + and self._acquisition is acquisition + and self._acquisition_out_dir == pending.out_dir + and self._acquisition_session_lease is lease + and self._camera_stop_priority_counts.get( + pending.token.acquisition_id, + 0, + ) + == 0 + ) + if not ( + after.status == "active" + and after.checkpoint == committed + and committed.current_binding == pending.status_proof.binding + and committed.current_evidence_session_id + == pending.new_evidence_session_id + and committed.last_gap_recovered_at_utc is not None + and ledger_after.record == ledger_snapshot.record + and runtime_after.get("phase") == "live" + and runtime_after.get("source_ready") is True + and runtime_after.get("producer_generation") + == producer_generation + and local_current + ): + return False + with self._lock: + if self._restart_rehydration_pending == pending: + self._restart_rehydration_pending = None + recovery_lineage = self._active_stream_recovery_lineage + if ( + recovery_lineage is not None + and recovery_lineage.acquisition_id + == pending.token.acquisition_id + and recovery_lineage.start_operation_id + == pending.token.root_start_operation_id + and recovery_lineage.evidence_session_id + == pending.new_evidence_session_id + and recovery_lineage.runtime_producer_generation + == producer_generation + ): + self._active_stream_recovery_state = "recovered" + self._active_stream_recovery_reason_code = None + self._active_stream_recovery_started_monotonic = None + if self._acquisition is acquisition and acquisition.state != "acquiring": + acquisition.transition( + "acquiring", + message_code="acquisition.restart.first_pcl_confirmed", + result={ + "read_only_recovery": True, + "device_state": "scanning", + "receiver_rehydration": "active", + "automatic_replay_allowed": False, + }, + ) + self._set_active_acquisition_checkpoint_reason(None) + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason_code = str( + getattr( + exc, + "reason_code", + "restart-first-pcl-checkpoint-promotion-failed", + ) + ) + committed_after_error = False + with suppress( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ): + after_error = ( + self._active_acquisition_checkpoint.snapshot().checkpoint + if self._active_acquisition_checkpoint is not None + else None + ) + committed_after_error = bool( + after_error is not None + and self._restart_rehydration_checkpoint_committed( + pending=pending, + checkpoint=after_error, + ) + ) + with self._lock: + if self._restart_rehydration_pending == pending: + next_phase: Literal["awaiting-first-pcl", "blocked"] = ( + "awaiting-first-pcl" + if committed_after_error + else "blocked" + ) + self._restart_rehydration_pending = dataclass_replace( + pending, + phase=next_phase, + reason_code=reason_code, + ) + self._active_stream_recovery_state = ( + "reconnecting" + if committed_after_error + else "blocked" + ) + self._active_stream_recovery_reason_code = reason_code + self._set_active_acquisition_checkpoint_reason(reason_code) + return False + finally: + self._k1_command_dispatch_gate.release() + + def _observe_published_runtime_envelope( + self, + envelope: DecodedDataPlaneView, + producer_generation: int, + ) -> None: + """Admit the acquisition camera only after authoritative PCL publish. + + This callback runs on the Rerun publisher thread *after* + ``RerunBridge.process`` has published the decoded frame and incremented + ``pcl_frames``. It performs local snapshots and enqueues at most one + generation-fenced worker; FFmpeg startup never blocks point-cloud + publication and no K1 camera/START/STOP command is emitted here. + """ + + if ( + not isinstance(envelope, DecodedPointCloudView) + or not envelope.context.live + or envelope.point_count < 1 + ): + return + runtime = self.runtime.snapshot() + metrics = runtime.get("metrics") + point_frames = metrics.get("pcl_frames") if isinstance(metrics, Mapping) else None + if not ( + runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") == producer_generation + and isinstance(point_frames, int) + and not isinstance(point_frames, bool) + and point_frames > 0 + ): + return + + classified_stop_promotion = self._promote_classified_stop_rebind_from_pcl( + envelope=envelope, + producer_generation=producer_generation, + ) + if classified_stop_promotion is False: + # The frame is already durable/visible, but a stale generation, + # STOP-priority owner or failed checkpoint CAS grants no camera + # authority. A later exact frame may retry the frozen transition. + return + + restart_promotion = self._promote_restart_rehydration_from_pcl( + envelope=envelope, + producer_generation=producer_generation, + published_pcl_frames=point_frames, + ) + if restart_promotion is False: + # The real PCL remains captured/published, but until its exact + # checkpoint fsync succeeds it grants neither camera nor ordinary + # active-acquisition projection. A later PCL retries verbatim. + return + + with self._lock: + session_id = ( + self._acquisition_out_dir.name + if self._acquisition_out_dir is not None + else self._device_session_id + ) + supervisor = self._connection_supervisor.snapshot() + exact_data_session = bool( + not supervisor.closed + and supervisor.data_plane.state in {"healthy", "stalled", "lost"} + and supervisor.data_plane.session_id == session_id + and supervisor.data_plane.host_path_epoch is not None + ) + current_reachable_lease = bool( + supervisor.authority.control_allowed + and supervisor.lease.state == "reachable" + and supervisor.lease.host_path_epoch == supervisor.host_path.epoch + and supervisor.host_path.epoch > 0 + ) + data_host_epoch = ( + supervisor.host_path.epoch + if current_reachable_lease + else supervisor.data_plane.host_path_epoch + if exact_data_session + else None + ) + if ( + session_id is not None + and supervisor.intent is not None + and (exact_data_session or current_reachable_lease) + and data_host_epoch is not None + ): + accepted = True + if ( + supervisor.data_plane.state != "healthy" + or supervisor.data_plane.session_id != session_id + or supervisor.data_plane.host_path_epoch != data_host_epoch + ): + accepted = self._connection_supervisor.observe_data_plane( + intent_id=supervisor.intent.intent_id, + host_path_epoch=data_host_epoch, + state="healthy", + session_id=session_id, + ) + if accepted: + with self._lock: + self._last_live_data_monotonic = time.monotonic() + self._last_live_data_suspend_aware = time.time() + self._last_live_data_session_id = session_id + + physical = self._physical_command_coordinator.snapshot() + control = self._application_control_session.snapshot() + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + start_operation_id = self._acquisition_start_operation_id + if ( + acquisition is None + or acquisition.state not in {"starting", "awaiting_external_start", "acquiring"} + or out_dir is None + ): + return + acquisition_id = acquisition.acquisition_id + control_mode = acquisition.control_mode + + if control_mode == "plugin-commanded": + start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=acquisition_id, + evidence_session_id=out_dir.name, + local_start_operation_id=start_operation_id, + ) + ) + verified_control_value = control.get("verified_control") + verified_control = ( + cast(Mapping[str, Any], verified_control_value) + if isinstance(verified_control_value, Mapping) + else None + ) + if control_mode == "plugin-commanded" and not ( + control.get("state") == "scanning" + and control.get("can_stop") is True + and self._matching_start_active_confirmed( + physical, + acquisition_id=acquisition_id, + start_operation_id=start_operation_id, + verified_control=verified_control, + ) + and self._active_acquisition_checkpoint_matches_current( + acquisition_id=acquisition_id, + start_operation_id=start_operation_id, + evidence_session_id=out_dir.name, + verified_control=verified_control, + ) + ): + # A PCL received during SCAN_STARTING is real data, but does not yet + # authorize a plugin-owned camera. The next published PCL retries + # this local admission after the durable SCANNING proof arrives. + return + + lineage = (acquisition_id, out_dir.name, producer_generation) + now = time.monotonic() + camera = self.camera_preview.snapshot() + with self._lock: + active_recovery_lineage = self._active_stream_recovery_lineage + runtime_recovery = runtime.get("connection_recovery") + if ( + control_mode == "plugin-commanded" + and active_recovery_lineage is not None + and active_recovery_lineage.acquisition_id == acquisition_id + and active_recovery_lineage.evidence_session_id == out_dir.name + and active_recovery_lineage.runtime_producer_generation + == producer_generation + and isinstance(runtime_recovery, Mapping) + and runtime_recovery.get("state") == "recovered" + and self._enqueue_post_recovery_camera_restart( + active_recovery_lineage, + runtime=runtime, + camera=camera, + ) + ): + return + with self._lock: + current = self._acquisition + if ( + self._camera_activation_lineage == lineage + and _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=out_dir.name, + ) + ): + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = lineage + self._camera_activation_retry_not_before_monotonic = ( + now + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS + ) + if ( + current is None + or current.acquisition_id != acquisition_id + or current.state not in {"starting", "awaiting_external_start", "acquiring"} + or self._acquisition_out_dir != out_dir + or self._camera_activation_lineage == lineage + or ( + self._camera_activation_retry_lineage == lineage + and now < self._camera_activation_retry_not_before_monotonic + ) + ): + return + self._camera_activation_lineage = lineage + + def activate() -> None: + activated = False + try: + activated = self._activate_default_acquisition_camera( + expected_acquisition_id=acquisition_id, + expected_evidence_session_id=out_dir.name, + expected_runtime_generation=producer_generation, + ) + except Exception: + # The gateway records its own exact local error. Never convert + # camera startup into a publisher-thread failure or retry storm; + # acquisition reconciliation/recovery owns the resulting state. + logger.exception( + "K1 camera activation failed after first authoritative PCL", + extra={ + "event_code": "k1_camera_post_pcl_activation_failed", + "acquisition_id": acquisition_id, + "evidence_session_id": out_dir.name, + "runtime_producer_generation": producer_generation, + }, + ) + finally: + with self._lock: + if self._camera_activation_lineage == lineage and activated: + if self._camera_activation_retry_lineage == lineage: + self._camera_activation_retry_lineage = None + self._camera_activation_retry_not_before_monotonic = 0.0 + elif self._camera_activation_lineage == lineage: + # Selection/FFmpeg/RTSP startup is local and retryable. + # Do not permanently latch a failed asynchronous claim, + # but also do not let the 10 Hz PCL publisher create a + # worker storm while the source is still unavailable. + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = lineage + self._camera_activation_retry_not_before_monotonic = ( + time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS + ) + + worker = threading.Thread( + target=activate, + name=f"k1-camera-post-pcl-{producer_generation}", + daemon=True, + ) + try: + worker.start() + except RuntimeError: + with self._lock: + if self._camera_activation_lineage == lineage: + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = lineage + self._camera_activation_retry_not_before_monotonic = ( + time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS + ) + raise + + def _reset_live_data_plane_observation(self) -> None: + """Make a stopped/replaced producer incapable of authorizing new data.""" + + with self._lock: + self._last_live_data_monotonic = None + self._last_live_data_suspend_aware = None + self._last_live_data_session_id = None + supervisor = self._connection_supervisor.snapshot() + if supervisor.intent is not None and supervisor.host_path.epoch > 0: + self._connection_supervisor.observe_data_plane( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + state="idle", + ) + def _observe_live_perception_mqtt(self, message: StreamMessage) -> None: if message.source != "live_mqtt": return @@ -1426,6 +14250,14 @@ class XgridsK1CompatibilityService: self, segment: CommittedCameraSegment, ) -> None: + with self._lock: + binding = self._live_perception_camera_binding + ingress_session_id = self.live_perception_ingress.snapshot().get("session_id") + if binding != (ingress_session_id, segment.source_id, segment.generation): + # Archive commitment remains authoritative evidence, but a late + # callback from producer generation N must never enter the live AI + # ingress after that session has moved to generation N+1. + return self.live_perception_ingress.publish( modality="camera-init" if segment.kind == "init" else "camera-frame", source_id=segment.source_id, @@ -1435,159 +14267,3756 @@ class XgridsK1CompatibilityService: payload=segment.payload, ) - def verify_connection( - self, - request: ConnectionVerifyRequest | None = None, - ) -> dict[str, Any]: - """Refresh a lease or adopt an externally configured direct-LAN K1.""" + def _configured_endpoint_target(self) -> _ConfiguredEndpointProbeTarget: + """Resolve only an exact current target or durable BLE-proved topology.""" - request = request or ConnectionVerifyRequest() - if request.device_id is None: - self._refresh_live_lan_address(rediscover=True) - else: - assert request.compatibility_attestation is not None - self._adopt_existing_lan_connection( - request.device_id, - request.compatibility_attestation, + supervisor = self._connection_supervisor.snapshot() + intent = supervisor.intent + device_network = supervisor.device_network + if ( + not supervisor.closed + and intent is not None + and device_network.state == "applied" + and device_network.intent_id == intent.intent_id + and device_network.connection_mode == intent.requested_mode + and device_network.target is not None + and isinstance(device_network.transport_ref, str) + and device_network.transport_ref + ): + self._require_transport_not_physically_retired( + device_network.transport_ref, + verification=True, ) + return _ConfiguredEndpointProbeTarget( + target=device_network.target, + connection_mode=device_network.connection_mode, + source="current-supervisor", + transport_ref=device_network.transport_ref, + intent_id=intent.intent_id, + semantic_revision=None, + ) + if supervisor.closed: + raise ConfiguredEndpointProbeError( + "Read-only монитор подключения уже закрыт", + reason_code="connection-supervisor-closed", + ) + + store = self._semantic_topology_store + if store is None: + raise ConfiguredEndpointProbeError( + "Сохранённая топология K1 недоступна", + reason_code="configured-endpoint-topology-corrupt", + ) + try: + snapshot = store.snapshot() + except (SemanticTopologyStoreError, OSError, ValueError) as exc: + raise ConfiguredEndpointProbeError( + "Сохранённая топология K1 недоступна", + reason_code="configured-endpoint-topology-corrupt", + ) from exc + record = snapshot.record + if snapshot.status == "corrupt": + raise ConfiguredEndpointProbeError( + "Сохранённая топология K1 повреждена", + reason_code="configured-endpoint-topology-corrupt", + ) + if snapshot.status != "available" or record is None: + raise ConfiguredEndpointProbeError( + "Нет сохранённого endpoint K1 для проверки", + reason_code="configured-endpoint-unavailable", + ) + self._require_transport_not_physically_retired( + record.transport_ref, + verification=True, + ) + return _ConfiguredEndpointProbeTarget( + target=EndpointTarget(record.ipv4, CONTROL_MQTT_PORT), + connection_mode=record.connection_mode, + source="durable-semantic-topology", + transport_ref=record.transport_ref, + intent_id=None, + semantic_revision=record.revision, + ) + + def _require_same_configured_endpoint_target( + self, + expected: _ConfiguredEndpointProbeTarget, + ) -> None: + """Reject a result if its trusted target changed during the socket probe.""" + + current = self._configured_endpoint_target() + if current != expected: + raise ConfiguredEndpointProbeError( + "Сохранённый endpoint изменился во время проверки", + reason_code="configured-endpoint-changed-during-probe", + ) + + async def probe_configured_endpoint( + self, + request: ConfiguredEndpointProbeRequest | None = None, + ) -> dict[str, Any]: + """Probe host route and TCP 1883 without entering any BLE code path.""" + + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise ConfiguredEndpointProbeError( + "Другая локальная операция меняет network/control lifecycle K1", + reason_code="configured-endpoint-probe-lifecycle-busy", + ) + process_lease_acquired = False + try: + if not await self._await_connection_monitor_quiescence(): + raise ConfiguredEndpointProbeError( + "Фоновая проверка подключения не завершилась вовремя", + reason_code="configured-endpoint-probe-lifecycle-busy", + ) + # A host-only TCP probe still contacts the physical target. Share + # the same cross-process lifecycle lease as retirement so a stale + # backend cannot probe an endpoint after another backend retires it. + self._acquire_k1_lifecycle_process_lease("network") + process_lease_acquired = True + return await _run_blocking_operation_without_abandonment( + self._probe_configured_endpoint_owned, + request or ConfiguredEndpointProbeRequest(), + ) + finally: + if process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + self._k1_lifecycle_transition_gate.release() + + def _probe_configured_endpoint_owned( + self, + request: ConfiguredEndpointProbeRequest, + ) -> dict[str, Any]: + """Execute one explicit, bounded and non-authoritative host-only probe.""" + + with self._lock: + device_id = self._device_id + device_session_id = self._device_session_id + request_fingerprint = self._request_fingerprint( + ACTION_CONFIGURED_ENDPOINT_PROBE, + {}, + ) + operation, created = self._operations.begin( + ACTION_CONFIGURED_ENDPOINT_PROBE, + operation_id=request.operation_id, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=10.0, + request_fingerprint=request_fingerprint, + ) + if not created: + return self.state() + self._operations.transition( + operation.operation_id, + "running", + stage_code="host-route-and-tcp-probe", + message_code="connection.endpoint-probe.running", + ) + try: + target = self._configured_endpoint_target() + observation = _probe_configured_endpoint_host_only( + target.target.ipv4, + path_probe=self._sample_host_path, + ) + self._require_same_configured_endpoint_target(target) + except Exception as exc: + failure = _configured_endpoint_probe_error(exc) + self._operations.transition( + operation.operation_id, + "failed", + stage_code="host-route-and-tcp-probe-failed", + message_code="connection.endpoint-probe.failed", + error=_operation_error( + failure, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + if failure is exc: + raise + raise failure from exc + + status = ( + "reachable" + if observation.reachable + else "endpoint-unreachable" + if observation.path.available + else "host-route-unavailable" + ) + result = { + "schema_version": "missioncore.xgrids-k1-configured-endpoint-probe/v1", + "status": status, + "target_source": target.source, + "connection_mode": target.connection_mode, + "endpoint": target.target.ipv4, + "transport_ref": target.transport_ref, + "intent_id": target.intent_id, + "semantic_revision": target.semantic_revision, + "host_route_available": observation.path.available, + "host_route_class": observation.path.route_class, + "tcp_reachable": observation.reachable, + "identity_validation": "not-performed", + "control_authority_granted": False, + "ble_operation_performed": False, + "network_mutation_performed": False, + "automatic_retry": False, + "observed_at": _utc_now_iso(), + "reason_code": observation.reason_code, + } + with self._lock: + self._configured_endpoint_probe = dict(result) + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="host-route-and-tcp-probe-completed", + message_code="connection.endpoint-probe.completed", + result=dict(result), + ) return self.state() - def _adopt_existing_lan_connection( + async def verify_connection( + self, + request: ConnectionVerifyRequest, + ) -> dict[str, Any]: + """Serialize every topology observation with network/control ownership.""" + + request = self._resolve_legacy_connection_verify_request(request) + if request.device_id is None: + raise ConnectionVerificationError( + "Read-only проверка K1 не получила точную серверную цель", + reason_code="connection-verify-connection-missing", + ) + self._require_transport_not_physically_retired( + request.device_id, + verification=True, + ) + + reconfiguration_gate_acquired = False + if not self._connection_reconfiguration_gate.acquire(blocking=False): + raise ConnectionVerificationError( + "другой диалог переподключения уже изменяет локальное состояние", + reason_code="connection-reconfiguration-lifecycle-busy", + ) + reconfiguration_gate_acquired = True + requested_mode = ( + { + "direct-lan": "bridge", + "device-ap": "quick-connect", + "controller-hotspot": "direct-connect", + }[request.compatibility_attestation.topology] + if request.compatibility_attestation is not None + else None + ) + try: + self._require_current_connection_reconfiguration_target( + expected_revision=request.expected_reconfiguration_revision, + expected_intent_id=request.expected_reconfiguration_intent_id, + device_id=request.device_id, + connection_mode=cast(ConnectionMode | None, requested_mode), + source=request.source, + discovery_generation=request.expected_discovery_generation, + ) + except BaseException: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + raise + transition_gate_acquired = False + command_dispatch_gate_acquired = False + network_process_lease_acquired = False + + def release_owned_verify_gates() -> None: + nonlocal command_dispatch_gate_acquired + nonlocal network_process_lease_acquired + nonlocal transition_gate_acquired + nonlocal reconfiguration_gate_acquired + if command_dispatch_gate_acquired: + self._k1_command_dispatch_gate.release() + command_dispatch_gate_acquired = False + if network_process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + network_process_lease_acquired = False + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + transition_gate_acquired = False + if reconfiguration_gate_acquired: + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + + try: + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise ConnectionVerificationError( + "другая локальная операция уже меняет network/control lifecycle K1", + reason_code="connection-verify-lifecycle-busy", + ) + transition_gate_acquired = True + if not await self._await_connection_monitor_quiescence(): + raise ConnectionVerificationError( + "фоновая проверка подключения не завершилась вовремя", + reason_code="connection-verify-lifecycle-busy", + ) + self._require_current_connection_reconfiguration_target( + expected_revision=request.expected_reconfiguration_revision, + expected_intent_id=request.expected_reconfiguration_intent_id, + device_id=request.device_id, + connection_mode=cast(ConnectionMode | None, requested_mode), + source=request.source, + discovery_generation=request.expected_discovery_generation, + ) + self._connection_reconfiguration_gate.release() + reconfiguration_gate_acquired = False + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + if not self._k1_command_dispatch_gate.acquire(blocking=False): + raise ConnectionVerificationError( + "control-команда K1 уже находится в точном publish-окне", + reason_code="connection-verify-control-dispatch-active", + ) + command_dispatch_gate_acquired = True + with self._borrow_network_ble_process_lease(): + ( + verify_operation_id, + verify_created, + provisional_topology, + ) = await self._verify_connection_owned(request) + if not verify_created: + release_owned_verify_gates() + return await asyncio.to_thread(self.state) + # DeviceInfo uses the same exact route but its MQTT publish needs + # the dispatch gate released. The lifecycle transition gate stays + # owned across the read-only handoff. + self._k1_command_dispatch_gate.release() + command_dispatch_gate_acquired = False + with self._lock: + connection_mode = self._connection_mode + if connection_mode is None: + raise ConnectionVerificationError( + "read-only проверка не создала точную topology K1", + reason_code="connection-verify-connection-missing", + ) + control_owner_before = self._application_control_session + control_before = dict(control_owner_before.snapshot()) + control_opened_by_verify = False + + def verify_control_generation_changed() -> bool: + current_owner = self._application_control_session + current = dict(current_owner.snapshot()) + return bool( + current_owner is not control_owner_before + or current.get("session_generation") != control_before.get("session_generation") + ) + + physical_reconciliation: dict[str, Any] = { + "performed": False, + "resolution": None, + "observed_session_state": None, + "device_write_performed": False, + "automatic_retry": False, + } + verified_binding: dict[str, Any] | None = None + control_stage = "device-info" + try: + await self._bootstrap_prestart_control_ready_owned( + parent_operation_id=verify_operation_id, + connection_mode=connection_mode, + inspection_only=True, + ) + control_after = dict(self._application_control_session.snapshot()) + control_opened_by_verify = bool( + self._application_control_session is not control_owner_before + or control_after.get("session_generation") + != control_before.get("session_generation") + ) + control_stage = "physical-reconciliation" + physical_reconciliation = await self._reconcile_physical_command_after_verify_owned( + verify_operation_id=verify_operation_id, + ) + verified_session = dict(self._application_control_session.snapshot()) + if ( + verified_session.get("state") == "connection-ready" + and verified_session.get("inspection_only") is True + and verified_session.get("inspection_promotion_allowed") is not True + ): + generation = verified_session.get("session_generation") + revision = verified_session.get("state_revision") + if not isinstance(generation, int) or not isinstance(revision, int): + raise ConnectionVerificationError( + "Verify не получил точный checkpoint inspection-сессии", + reason_code="connection-verify-control-checkpoint-invalid", + ) + self._application_control_session.release_inspection_for_operator_dialogue( + expected_session_generation=generation, + expected_state_revision=revision, + ) + if provisional_topology is not None: + control_stage = "semantic-topology-commit" + self._commit_provisional_fresh_bridge_topology(provisional_topology) + control_stage = "final-control-binding" + ready = self._connection_supervisor.snapshot() + if not ( + ready.authority.control_allowed + and ready.intent is not None + and ready.intent.requested_mode == connection_mode + and ready.device_network.state == "applied" + and ready.device_network.intent_id == ready.intent.intent_id + and isinstance(ready.device_network.transport_ref, str) + and ready.device_network.transport_ref + and ready.device_network.connection_mode == connection_mode + and ready.device_network.target is not None + and ready.endpoint.target == ready.device_network.target + and ready.endpoint.intent_id == ready.intent.intent_id + and ready.endpoint.tcp_state == "reachable" + and ready.endpoint.host_path_epoch == ready.host_path.epoch + and ready.host_path.available + ): + raise ConnectionVerificationError( + "Verify lost its exact control binding before commit", + reason_code="connection-verify-final-binding-superseded", + ) + assert ready.intent is not None + assert ready.device_network.transport_ref is not None + assert ready.device_network.target is not None + verified_binding = { + "snapshot_runtime_id": self._snapshot_runtime_id, + "intent_id": ready.intent.intent_id, + "transport_ref": ready.device_network.transport_ref, + "connection_mode": ready.intent.requested_mode, + "target_ipv4": ready.device_network.target.ipv4, + "target_port": ready.device_network.target.port, + "host_path_epoch": ready.host_path.epoch, + } + except asyncio.CancelledError as exc: + if control_opened_by_verify or verify_control_generation_changed(): + try: + await self._retire_control_generation_opened_by_verify() + except BaseException as cleanup_exc: + exc.add_note( + "verify-owned local control cleanup failed: " + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + if provisional_topology is not None: + self._reset_provisional_fresh_bridge_topology(provisional_topology) + self._operations.transition_if_pending( + verify_operation_id, + "cancelled", + stage_code=f"{control_stage}-cancelled", + message_code="connection.verify.cancelled", + error={ + "category": "connection", + "code": "connection-verify-cancelled", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + raise + except Exception as exc: + if control_opened_by_verify or verify_control_generation_changed(): + try: + await self._retire_control_generation_opened_by_verify() + except BaseException as cleanup_exc: + exc.add_note( + "verify-owned local control cleanup failed: " + f"{type(cleanup_exc).__name__}: {cleanup_exc}" + ) + if provisional_topology is not None: + self._reset_provisional_fresh_bridge_topology(provisional_topology) + failure = ( + exc + if isinstance(exc, ConnectionVerificationError) + else ConnectionVerificationError( + "K1 не подтвердил готовность управляющего канала", + reason_code=str(getattr(exc, "reason_code", "control-bootstrap-failed")), + ) + ) + self._operations.transition_if_pending( + verify_operation_id, + "failed", + stage_code=f"{control_stage}-failed", + message_code="connection.verify.failed", + error=_operation_error( + failure, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + if failure is exc: + raise + raise failure from exc + if verified_binding is None: + raise RuntimeError("successful Verify omitted its exact control binding") + self._operations.transition( + verify_operation_id, + "succeeded", + stage_code=( + "physical-reconciliation-confirmed" + if physical_reconciliation["performed"] + else "device-info-confirmed" + ), + message_code="connection.verify.completed", + result={ + "write_performed": False, + "control_verified": True, + "lease_generation": ready.lease.generation, + "lease_state": ready.lease.state, + "verified_binding": verified_binding, + "physical_reconciliation": physical_reconciliation, + }, + ) + with self._lock: + consume_select_device = self._connection_reconfiguration_intent == "select-device" + if consume_select_device: + self._consume_connection_reconfiguration( + intent_id=request.expected_reconfiguration_intent_id, + revision=request.expected_reconfiguration_revision, + ) + # Construct the response only after the high-level network owner + # and lifecycle transition fence are released. Otherwise a + # successful SCANNING reconciliation falsely projects STOP as + # blocked by the Verify operation which has already completed. + release_owned_verify_gates() + return await asyncio.to_thread(self.state) + finally: + release_owned_verify_gates() + + async def _reconcile_physical_command_after_verify_owned( + self, + *, + verify_operation_id: str, + allow_receiver_rehydrate: bool = True, + ) -> dict[str, Any]: + """Consume exact read-only recovery proof before bootstrap returns. + + A restarted process may inherit an ambiguous START/STOP ledger edge. + DeviceInfo alone cannot resolve it. Explicit Verify or the exact + service-owned continuation of a reset network Apply therefore waits a + bounded interval for a later non-retained DeviceStatus on the same + control generation, then commits only the local durable classification. + This helper never publishes, touches BLE, changes Wi-Fi, or retries the + old physical command. A reset-owned Apply continuation disables old + receiver/camera rehydration: SCANNING becomes an explicit STOP-only + shell, never a resurrection of the retired local acquisition. + """ + + # Recompute any persisted S1/S2 certificate from the current ledger + # and immutable archive before read-only reconciliation can advance the + # physical head. Failure revokes only checkpoint START/resume admission; + # explicit Verify remains the path to a bounded STOP-only projection. + checkpoint_trust_token = ( + self._validate_active_acquisition_checkpoint_lineage() + ) + + deadline = time.monotonic() + PHYSICAL_RECONCILIATION_PROOF_TIMEOUT_SECONDS + while True: + physical = dict(self._physical_command_coordinator.snapshot()) + if physical.get("status") == "corrupt": + raise ConnectionVerificationError( + "Durable START/STOP ledger K1 повреждён; автоматическая сверка запрещена", + reason_code="physical-command-ledger-corrupt", + ) + resolved_scan_over_recovery_required = ( + physical.get("resolved_scan_over_recovery_required") is True + ) + unresolved_recovery_required = bool( + physical.get("requires_reconciliation") is True + and not resolved_scan_over_recovery_required + ) + resolved_active_recovery_required = ( + physical.get("resolved_active_recovery_required") is True + ) + if resolved_active_recovery_required: + # A reset may reopen a previously proven successful START as + # an observing checkpoint. The durable command is still not + # ambiguous: extend its resolved-active lineage instead of + # trying to append another ambiguous-outcome for the same + # operation. + unresolved_recovery_required = False + if not ( + unresolved_recovery_required + or resolved_active_recovery_required + or resolved_scan_over_recovery_required + ): + return { + "performed": False, + "resolution": None, + "observed_session_state": physical.get("observed_session_state"), + "device_write_performed": False, + "automatic_retry": False, + } + + control = dict(self._application_control_session.snapshot()) + if control.get("state") != "connection-ready": + raise ConnectionVerificationError( + "Управляющая сессия изменилась до read-only сверки START/STOP", + reason_code="physical-command-reconciliation-control-changed", + ) + try: + self._application_control_session.validate_physical_reconciliation_binding() + except Exception as exc: + raise ConnectionVerificationError( + "Точный маршрут управляющей сессии изменился до read-only сверки", + reason_code="physical-command-reconciliation-binding-changed", + ) from exc + + if physical.get("reconciliation_ready") is True: + observed_state = physical.get("observed_session_state") + if observed_state not in {"ready", "scan_over", "scanning"}: + raise ConnectionVerificationError( + "K1 не подтвердил каноническое READY или SCANNING состояние", + reason_code="physical-command-reconciliation-state-unsafe", + ) + # Fence the local ledger commit against a route/session change + # after the proof snapshot. The coordinator rechecks that the + # DeviceInfo and DeviceStatus belong to its exact runtime. + durable_reconciliation_committed = False + reconciled_record: Mapping[str, Any] | None = None + try: + self._application_control_session.validate_physical_reconciliation_binding() + reconciliation_id = f"{verify_operation_id}.physical" + retained_owner: _PreparedStopRecoveryOwner | None = None + if unresolved_recovery_required: + with self._lock: + retained_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + if ( + allow_receiver_rehydrate + and retained_owner is not None + and self._prepared_stop_recovery_owner_is_current( + retained_owner, + ) + ): + def reconcile_retained_prepared_stop( + reconciliation_id: str = reconciliation_id, + ) -> Mapping[str, Any]: + return self._physical_command_coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ) + + reconciled_record = self._classify_retained_prepared_stop_owned( + owner=retained_owner, + reconciliation_id=reconciliation_id, + reconcile=reconcile_retained_prepared_stop, + ) + else: + reconciled_record = ( + self._physical_command_coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ) + ) + else: + reconciled_record = ( + self._physical_command_coordinator.reconcile_resolved_active( + reconciliation_id=reconciliation_id, + ) + ) + durable_reconciliation_committed = True + if reconciled_record is not None and not ( + allow_receiver_rehydrate + and retained_owner is not None + and reconciled_record.get("operation_id") + == retained_owner.lineage.stop_operation_id + and reconciled_record.get("acquisition_id") + == retained_owner.lineage.acquisition_id + ): + self._settle_classified_prepared_stop_operation( + reconciled_record=reconciled_record, + reconciliation_id=reconciliation_id, + ) + if observed_state == "scanning": + session_generation = control.get("session_generation") + state_revision = control.get("state_revision") + if ( + not isinstance(session_generation, int) + or isinstance(session_generation, bool) + or not isinstance(state_revision, int) + or isinstance(state_revision, bool) + ): + raise RuntimeError("active recovery lacks an exact control checkpoint") + self._application_control_session.adopt_reconciled_scanning( + reconciliation_id=reconciliation_id, + expected_session_generation=session_generation, + expected_state_revision=state_revision, + ) + adoption_deadline = ( + time.monotonic() + + PHYSICAL_RECONCILIATION_CONTROL_ADOPTION_TIMEOUT_SECONDS + ) + while True: + adopted = dict(self._application_control_session.snapshot()) + adopted_state = str(adopted.get("state") or "unknown") + if adopted_state == "scanning": + break + if adopted_state in { + "idle", + "completed", + "closed", + "failed", + }: + raise RuntimeError( + "active recovery control worker terminated before adoption" + ) + if time.monotonic() >= adoption_deadline: + logger.error( + "K1 local control worker did not adopt reconciled SCANNING", + extra={ + "event_code": ( + "k1_physical_reconciliation_adoption_timeout" + ), + "reason_code": ( + "physical-command-reconciliation-control-adoption-timeout" + ), + "device_write_performed": False, + "automatic_retry": False, + }, + ) + raise ConnectionVerificationError( + "Локальный управляющий канал K1 не принял " + "подтверждённое SCANNING состояние", + reason_code=( + "physical-command-reconciliation-control-adoption-timeout" + ), + ) + await asyncio.sleep(0.01) + if reconciled_record is None: + raise RuntimeError( + "active recovery omitted its durable physical record" + ) + with self._lock: + retained_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + if ( + allow_receiver_rehydrate + and retained_owner is not None + and reconciled_record.get("operation_id") + == retained_owner.lineage.stop_operation_id + and reconciled_record.get("acquisition_id") + == retained_owner.lineage.acquisition_id + ): + self._adopt_classified_prepared_stop_scanning_in_place( + owner=retained_owner, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + checkpoint_trust_token=checkpoint_trust_token, + ) + else: + restart_outcome = None + if allow_receiver_rehydrate: + restart_outcome = ( + await self._rehydrate_active_acquisition_after_restart( + token=checkpoint_trust_token, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + ) + ) + if restart_outcome is None: + checkpoint_lineage = ( + self._restart_stop_only_checkpoint_lineage( + token=checkpoint_trust_token, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + ) + ) + self._materialize_recovered_physical_stop_acquisition( + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + checkpoint_lineage=checkpoint_lineage, + ) + elif reconciled_record is not None: + settled = self._settle_restart_checkpoint_after_verified_standby( + token=checkpoint_trust_token, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + ) + if checkpoint_trust_token is not None and not settled: + raise ActiveAcquisitionRecoveryCheckpointError( + "restart standby checkpoint settlement failed closed" + ) + except Exception as exc: + latest = dict(self._physical_command_coordinator.snapshot()) + if ( + not durable_reconciliation_committed + and latest.get("requires_reconciliation") is not True + and latest.get("resolved_active_recovery_required") is not True + and latest.get("resolved_scan_over_recovery_required") is not True + ): + return { + "performed": False, + "resolution": None, + "observed_session_state": latest.get("observed_session_state"), + "device_write_performed": False, + "automatic_retry": False, + } + raise ConnectionVerificationError( + "Read-only сверка физического состояния K1 не была применена", + reason_code=str( + getattr( + exc, + "reason_code", + "physical-command-reconciliation-failed", + ) + ), + ) from exc + resolution = ( + "physical-standby-observed" + if observed_state in {"ready", "scan_over"} + else "physical-active-observed" + ) + return { + "performed": True, + "resolution": resolution, + "observed_session_state": observed_state, + "device_write_performed": False, + "automatic_retry": False, + } + + if time.monotonic() >= deadline: + raise ConnectionVerificationError( + "K1 не передал свежий non-retained DeviceStatus для безопасной сверки", + reason_code="physical-command-reconciliation-proof-timeout", + ) + await asyncio.sleep(0.05) + + def _settle_classified_prepared_stop_operation( + self, + *, + reconciled_record: Mapping[str, Any], + reconciliation_id: str, + ) -> None: + """Seal the old facade STOP after PREPARED is proven not dispatched. + + The read-only classification is authoritative only for the exact + current STOP row and its newest reconciliation. It proves no physical + edge happened, so the original operation must become terminal and can + never be replayed under its old idempotency key. A later explicit + recovery STOP, when SCANNING was observed, owns a fresh operation id. + """ + + reconciliations = reconciled_record.get("reconciliations") + latest = ( + reconciliations[-1] + if isinstance(reconciliations, list) and reconciliations + else None + ) + operation_id = reconciled_record.get("operation_id") + original_attempt = latest.get("original_attempt") if isinstance(latest, Mapping) else None + if not ( + reconciled_record.get("action") == "stop" + and reconciled_record.get("stage") == "resolved" + and reconciled_record.get("resolution") == "not-dispatched" + and isinstance(operation_id, str) + and isinstance(latest, Mapping) + and latest.get("reconciliation_id") == reconciliation_id + and latest.get("kind") == "prepared-stop-classification" + and isinstance(original_attempt, Mapping) + and original_attempt.get("operation_id") == operation_id + ): + return + try: + operation = self._operations.get(operation_id) + except KeyError: + return + if operation.action != ACTION_ACQUISITION_STOP: + return + self._operations.transition_if_pending( + operation_id, + "failed", + stage_code="physical-stop-classified-not-dispatched", + message_code="acquisition.stop.physical_command_not_dispatched", + error={ + "category": "device", + "code": "physical-stop-not-dispatched", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "none", + "physical_command_sent": False, + "automatic_replay_allowed": False, + }, + ) + + def _classify_retained_prepared_stop_owned( + self, + *, + owner: _PreparedStopRecoveryOwner, + reconciliation_id: str, + reconcile: Callable[[], Mapping[str, Any]], + ) -> Mapping[str, Any]: + """Commit S0 classification and reserve its local projection atomically. + + The durable ledger transition is synchronous. Holding the acquisition + lifecycle gate across that commit and the local CAS prevents a state + poll from combining the new resolved S0 with the old awaiting-STOP + acquisition and releasing its receiver/evidence owner in between. + """ + + with self._acquisition_lifecycle_access(): + if not self._prepared_stop_recovery_owner_is_current(owner): + raise RuntimeError( + "retained acquisition changed before PREPARED STOP classification" + ) + reconciled = reconcile() + if not ( + isinstance(reconciled, Mapping) + and reconciled.get("operation_id") == owner.lineage.stop_operation_id + and reconciled.get("acquisition_id") == owner.lineage.acquisition_id + and reconciled.get("action") == "stop" + and reconciled.get("stage") == "resolved" + and reconciled.get("resolution") == "not-dispatched" + ): + raise RuntimeError("PREPARED STOP classification changed immutable owner") + self._settle_classified_prepared_stop_operation( + reconciled_record=reconciled, + reconciliation_id=reconciliation_id, + ) + # The coordinator has already committed the exact observation and + # verified binding into the immutable reconciliation audit. Do + # not combine that durable fact with a later process-local control + # snapshot: the inspection worker may rotate immediately after + # the commit, and losing that ephemeral object must not strand the + # still-owned receiver between durable truth and local projection. + reconciliations = reconciled.get("reconciliations") + latest = ( + reconciliations[-1] + if isinstance(reconciliations, list) and reconciliations + else None + ) + if not ( + isinstance(latest, Mapping) + and latest.get("reconciliation_id") == reconciliation_id + and latest.get("kind") == "prepared-stop-classification" + ): + raise RuntimeError("PREPARED STOP classification audit is not current") + resolution = latest.get("resolution") + physical = { + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": ( + resolution == "physical-active-observed" + ), + "reconciled_physical_state": ( + "active" + if resolution == "physical-active-observed" + else "standby" + if resolution == "physical-standby-observed" + else None + ), + "record": reconciled, + } + if self._matching_classified_prepared_stop_active( + physical, + acquisition_id=owner.lineage.acquisition_id, + stop_operation_id=owner.lineage.stop_operation_id, + start_operation_id=owner.start_operation_id, + ): + # Reserve the classified-active state before the async control + # worker transitions to SCANNING. The exact owner remains + # provisional and the later adoption only rotates its fresh + # local binding/camera fence. + with self._lock: + acquisition = self._acquisition + if acquisition is not owner.acquisition: + raise RuntimeError("retained acquisition owner changed") + acquisition.message_code = ( + "acquisition.recovery.scanning_adoption_pending" + ) + acquisition.state_revision += 1 + acquisition.updated_at = datetime.now(UTC) + return reconciled + if self._matching_classified_prepared_stop_standby( + physical, + acquisition_id=owner.lineage.acquisition_id, + stop_operation_id=owner.lineage.stop_operation_id, + start_operation_id=owner.start_operation_id, + ): + with self._lock: + acquisition = self._acquisition + if acquisition is not owner.acquisition: + raise RuntimeError("retained acquisition owner changed") + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "finalizing", + message_code="acquisition.recovery.device_standby_observed", + ) + self._active_stream_recovery_terminal_outcome = "standby" + return reconciled + raise RuntimeError("PREPARED STOP classification is not canonical") + + def _cleanup_inert_terminal_camera_residual_for_recovered_stop( + self, + *, + runtime: Mapping[str, Any], + camera: Mapping[str, Any], + ingress: Mapping[str, Any], + ) -> bool: + """Retire only a proven-dead local camera owner before STOP recovery. + + A terminal camera producer failure may already have removed the + acquisition/evidence owner while leaving the selected-source error and + its process lease published. That residue must not block rehydrating + the physical START identity needed for an explicit STOP. Conversely, + no live producer, receiver, operation, or recovery owner may be + destroyed merely because Verify observed physical SCANNING. + + The acquisition lifecycle gate serializes this whole proof/cleanup + with camera activation and acquisition mutations. Metadata is cleared + only after ``stop_current`` proves the gateway idle; a cleanup failure + therefore leaves the camera process fence retained and Verify fails + closed. + """ + + recording = camera.get("recording") + terminal_camera = bool( + ( + camera.get("phase") == "error" + and camera.get("active_source_id") + == DEFAULT_ACQUISITION_CAMERA_SOURCE + ) + or ( + camera.get("phase") == "idle" + and camera.get("active_source_id") is None + ) + ) + with self._lock: + metadata_checkpoint = ( + self._live_perception_camera_binding, + self._camera_activation_lineage, + self._camera_activation_retry_lineage, + self._camera_activation_retry_not_before_monotonic, + ) + cleanup_allowed = bool( + self._acquisition is None + and self._acquisition_session_lease is None + and self._acquisition_out_dir is None + and self._acquisition_start_operation_id is None + and self._acquisition_stop_operation_id is None + and self._prepared_stop_dispatch_lineage is None + and getattr(self, "_prepared_stop_recovery_owner", None) is None + and self._operator_manual_acquisition_process_lease_id is None + and self._active_stream_recovery_lineage is None + and self._active_stream_recovery_future is None + and self._active_stream_recovery_state == "inactive" + and runtime.get("source_mode") == "idle" + and runtime.get("phase") == "idle" + and ingress.get("active") is False + and terminal_camera + and isinstance(recording, Mapping) + and recording.get("active") is False + and recording.get("session") is None + and recording.get("active_epoch") is None + and recording.get("producer_alive") is False + and "camera" in self._application_control_process_lease_holders + ) + if not cleanup_allowed: + return False + + self.camera_preview.stop_current() + stopped_camera = self.camera_preview.snapshot() + stopped_recording = stopped_camera.get("recording") + stopped_runtime = self.runtime.snapshot() + stopped_ingress = self.live_perception_ingress.snapshot() + if not ( + stopped_camera.get("phase") == "idle" + and stopped_camera.get("active_source_id") is None + and isinstance(stopped_recording, Mapping) + and stopped_recording.get("active") is False + and stopped_recording.get("session") is None + and stopped_recording.get("active_epoch") is None + and stopped_recording.get("producer_alive") is False + and stopped_runtime.get("source_mode") == "idle" + and stopped_runtime.get("phase") == "idle" + and stopped_ingress.get("active") is False + ): + raise RuntimeError("terminal camera residual did not stop cleanly") + + with self._lock: + if not ( + self._acquisition is None + and self._acquisition_session_lease is None + and self._acquisition_out_dir is None + and self._acquisition_start_operation_id is None + and self._acquisition_stop_operation_id is None + and self._prepared_stop_dispatch_lineage is None + and getattr(self, "_prepared_stop_recovery_owner", None) is None + and self._operator_manual_acquisition_process_lease_id is None + and self._active_stream_recovery_lineage is None + and self._active_stream_recovery_future is None + and self._active_stream_recovery_state == "inactive" + and "camera" in self._application_control_process_lease_holders + and metadata_checkpoint + == ( + self._live_perception_camera_binding, + self._camera_activation_lineage, + self._camera_activation_retry_lineage, + self._camera_activation_retry_not_before_monotonic, + ) + ): + raise RuntimeError("terminal camera residual changed during local cleanup") + self._live_perception_camera_binding = None + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = None + self._camera_activation_retry_not_before_monotonic = 0.0 + + self._release_camera_preview_process_lease() + return True + + def _restart_stop_only_checkpoint_lineage( + self, + *, + token: _ActiveAcquisitionCheckpointTrustToken | None, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + ) -> _RestartStopOnlyCheckpointLineage | None: + """Open one durable restart gap before creating a no-evidence STOP shell.""" + + if not self._active_acquisition_checkpoint_trust_token_is_current( + token, + reconciled_record=reconciled_record, + ): + return None + assert token is not None + if token.checkpoint_state != "active": + return None + store = self._active_acquisition_checkpoint + if store is None: + return None + try: + snapshot = store.snapshot() + checkpoint = snapshot.checkpoint + if not ( + snapshot.status == "active" + and checkpoint == token.checkpoint + and checkpoint is not None + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "restart STOP-only checkpoint changed before gap commit" + ) + gap_open = bool( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ) + if not gap_open: + checkpoint = store.record_gap( + transition_id=self._active_acquisition_checkpoint_transition_id( + "restart-stop-only-gap", + checkpoint.acquisition_id, + reconciliation_id, + checkpoint.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + failed_binding=checkpoint.current_binding, + gap_started_at_utc=_utc_now_millisecond_iso(), + ) + if not ( + checkpoint.state == "active" + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + and checkpoint.last_gap_failed_binding + == checkpoint.current_binding + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "restart STOP-only checkpoint lacks one exact open gap" + ) + crosses_runtime = ( + checkpoint.current_binding.runtime_instance_id + != self._snapshot_runtime_id + ) + cessation_evidence_session_id = ( + "restart-stop-only:sha256-" + + hashlib.sha256( + ( + f"{checkpoint.acquisition_id}\0{self._snapshot_runtime_id}\0" + f"{reconciliation_id}" + ).encode() + ).hexdigest() + if crosses_runtime + else checkpoint.current_evidence_session_id + ) + return _RestartStopOnlyCheckpointLineage( + checkpoint_revision=checkpoint.revision, + acquisition_id=checkpoint.acquisition_id, + original_start_operation_id=( + checkpoint.original_start_operation_id + ), + prior_evidence_session_id=( + checkpoint.current_evidence_session_id + ), + cessation_evidence_session_id=cessation_evidence_session_id, + ) + except ( + ActiveAcquisitionRecoveryCheckpointError, + OSError, + ValueError, + ) as exc: + self._mark_active_acquisition_checkpoint_untrusted( + trust="unavailable", + reason_code=str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-gap-failed", + ) + ), + ) + return None + + async def _rehydrate_active_acquisition_after_restart( + self, + *, + token: _ActiveAcquisitionCheckpointTrustToken | None, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + ) -> Literal["rehydrated", "stop-only"] | None: + """Start only a new local receiver for one explicitly verified scan. + + The scanner is already SCANNING. This method never calls a START, + STOP, BLE, network-mutation, workspace, or project command. Its only + externally contacting action is the new capture receiver, after the + durable checkpoint/physical lineage has been proven exactly. + """ + + if token is None or self._active_acquisition_checkpoint is None: + return None + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.as_dict() == dict(reconciled_record) + and record.reconciliations + ): + return None + reconciliation = record.reconciliations[-1] + if reconciliation.reconciliation_id != reconciliation_id: + return None + checkpoint = token.checkpoint + if token.checkpoint_state == "active": + if not self._active_acquisition_checkpoint_trust_token_is_current( + token, + reconciled_record=reconciled_record, + ): + return None + origin_proof = None + elif token.checkpoint_state == "prepared": + if not self._prepared_restart_reconciliation_is_current( + token, + record=record, + reconciliation=reconciliation, + ): + return None + origin_proof = cast(ActiveAcquisitionRecoveryStartOriginProof | None, None) + else: + return None + + control = dict(self._application_control_session.snapshot()) + if control.get("state") != "scanning" or control.get("can_stop") is not True: + return None + verified_control = self._checkpoint_verified_control(control) + identity = reconciliation.verified_binding.identity + connection = reconciliation.verified_binding.connection + if not ( + verified_control.get("logical_device_id") + == checkpoint.identity.logical_device_id + and identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and record.compatibility_profile_id == checkpoint.compatibility_profile_id + and connection.transport_ref == checkpoint.connection.transport_ref + and connection.connection_mode == checkpoint.connection.connection_mode + and connection.target_ipv4 == checkpoint.connection.target_ipv4 + and connection.target_port == checkpoint.connection.target_port + ): + return None + + runtime_before = dict(self.runtime.snapshot()) + ingress_before = self.live_perception_ingress.snapshot() + camera_before = self.camera_preview.snapshot() + recording_before = camera_before.get("recording") + with self._lock: + local_idle = bool( + ( + self._acquisition is None + or self._acquisition.state in TERMINAL_ACQUISITION_STATES + ) + and self._acquisition_session_lease is None + and self._acquisition_out_dir is None + and self._prepared_stop_dispatch_lineage is None + and self._prepared_stop_recovery_owner is None + and self._restart_rehydration_pending is None + and self._restart_stop_only_prepared_pending is None + and self._active_stream_recovery_lineage is None + and self._active_stream_recovery_future is None + and self._active_stream_recovery_state == "inactive" + ) + if not ( + local_idle + and runtime_before.get("source_mode") == "idle" + and runtime_before.get("phase") == "idle" + and ingress_before.get("active") is False + and camera_before.get("phase") == "idle" + and camera_before.get("active_source_id") is None + and isinstance(recording_before, Mapping) + and recording_before.get("active") is False + ): + return None + + out_dir = new_live_session_dir(self.evidence_root) + forbidden_names = { + checkpoint.original_evidence_session_id, + checkpoint.current_evidence_session_id, + } + while out_dir.name in forbidden_names or out_dir.exists(): + out_dir = out_dir.with_name(f"{out_dir.name}_restart") + recover_stale_active_session_marker(out_dir.parent) + predicted_generation = runtime_before.get("producer_generation") + if not isinstance(predicted_generation, int) or isinstance( + predicted_generation, + bool, + ): + return None + predicted_generation += 1 + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=verified_control, + ) + status_proof = self._checkpoint_status_proof( + status=reconciliation.observation, + binding=binding, + evidence_session_id=out_dir.name, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + if token.checkpoint_state == "prepared": + origin_proof = self._checkpoint_reconciled_start_origin_proof( + checkpoint=checkpoint, + record=record, + reconciliation=reconciliation, + physical_proof=physical_proof, + ) + + gap_started_at_utc = ( + checkpoint.last_gap_started_at_utc + if checkpoint.state == "active" + and checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + else _utc_now_millisecond_iso() + ) + store = self._active_acquisition_checkpoint + if checkpoint.state == "active" and not ( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ): + checkpoint = store.record_gap( + transition_id=self._active_acquisition_checkpoint_transition_id( + "restart-rehydrate-gap", + checkpoint.acquisition_id, + reconciliation_id, + checkpoint.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + failed_binding=checkpoint.current_binding, + gap_started_at_utc=gap_started_at_utc, + ) + token = self._checkpoint_trust_token(checkpoint=checkpoint, record=record) + + transition_id = self._active_acquisition_checkpoint_transition_id( + ( + "restart-activate-reconciled" + if checkpoint.state == "prepared" + else "restart-rebind" + ), + checkpoint.acquisition_id, + reconciliation_id, + out_dir.name, + checkpoint.revision, + ) + pending = _RestartRehydrationPending( + phase="awaiting-suback", + token=token, + reconciliation_id=reconciliation_id, + gap_started_at_utc=gap_started_at_utc, + old_evidence_session_id=checkpoint.current_evidence_session_id, + new_evidence_session_id=out_dir.name, + out_dir=out_dir, + runtime_producer_generation=predicted_generation, + status_proof=status_proof, + physical_proof=physical_proof, + origin_proof=origin_proof, + transition_id=transition_id, + ) + device_id, device_session_id = self._ensure_device_context() + supervisor = self._connection_supervisor.snapshot() + with self._lock: + exact_topology = bool( + self._selected_device_id == connection.transport_ref + and self._device_id == device_id + and self._device_session_id == device_session_id + and self._connection_mode == connection.connection_mode + and self._k1_ip == connection.target_ipv4 + ) + if not ( + exact_topology + and supervisor.intent is not None + and supervisor.intent.intent_id == connection.intent_id + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == connection.intent_id + and supervisor.device_network.transport_ref == connection.transport_ref + and supervisor.device_network.connection_mode == connection.connection_mode + and supervisor.device_network.target + == EndpointTarget(connection.target_ipv4, connection.target_port) + ): + return None + acquisition = AcquisitionRecord( + acquisition_id=checkpoint.acquisition_id, + device_id=device_id, + device_session_id=device_session_id, + compatibility_profile_id=checkpoint.compatibility_profile_id, + control_mode="plugin-commanded", + requested_streams=checkpoint.requested_streams, + target_host=checkpoint.connection.target_ipv4, + duration_seconds=checkpoint.duration_seconds, + evidence_policy=checkpoint.evidence_policy, + state="awaiting_external_start", + message_code="acquisition.restart.awaiting_receiver_ready", + result={ + "read_only_recovery": True, + "device_state": "scanning", + "receiver_rehydration": "awaiting-suback", + "automatic_replay_allowed": False, + }, + ) + # Acquire the process marker only after every proof/CAS that can fail + # without a live writer. From this point onward the local transaction + # either publishes the exact lease owner or releases it immediately. + lease = ActiveSessionLease.acquire(out_dir.parent, out_dir) + with self._lock: + if ( + self._restart_rehydration_pending is not None + or self._active_stream_recovery_lineage is not None + or self._active_stream_recovery_future is not None + or self._active_stream_recovery_state != "inactive" + ): + lease.release() + return None + self._active_stream_recovery_generation += 1 + recovery_lineage = _ActiveStreamRecoveryLineage( + recovery_generation=self._active_stream_recovery_generation, + snapshot_runtime_id=self._snapshot_runtime_id, + acquisition_id=checkpoint.acquisition_id, + device_id=device_id, + device_session_id=device_session_id, + evidence_session_id=out_dir.name, + runtime_producer_generation=predicted_generation, + start_operation_id=checkpoint.original_start_operation_id, + physical_operation_id=pending.physical_proof.operation_id, + physical_revision=pending.physical_proof.ledger_revision, + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + ) + self._acquisition = acquisition + self._restart_stop_only_checkpoint_owner = None + self._acquisition_project_name = checkpoint.project_name + self._acquisition_mount_type = checkpoint.mount_type + self._acquisition_gnss_mode = checkpoint.gnss_mode + self._acquisition_out_dir = out_dir + self._acquisition_start_operation_id = checkpoint.original_start_operation_id + self._acquisition_stop_operation_id = None + self._acquisition_session_lease = lease + self._restart_rehydration_pending = pending + self._active_stream_recovery_lineage = recovery_lineage + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = ( + "restart-receiver-awaiting-suback" + ) + self._active_stream_recovery_started_at_utc = gap_started_at_utc + self._active_stream_recovery_started_monotonic = time.monotonic() + self._active_stream_recovery_terminal_outcome = None + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = None + self._live_perception_camera_binding = None + + setup_error: BaseException | None = None + try: + self.live_perception_ingress.begin_session(out_dir.name) + self._modeling_control_safety.reset() + self._reset_live_data_plane_observation() + self.runtime.start_live( + checkpoint.connection.target_ipv4, + out_dir, + duration_seconds=checkpoint.duration_seconds, + project_name=checkpoint.project_name, + recover_connection=self._recover_active_stream_connection, + ) + source_deadline = ( + time.monotonic() + RESTART_RECEIVER_SOURCE_READY_TIMEOUT_SECONDS + ) + while True: + runtime = dict(self.runtime.snapshot()) + if ( + runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") == predicted_generation + ): + break + if runtime.get("phase") == "error" or time.monotonic() >= source_deadline: + raise RuntimeError( + "restart receiver did not reach MQTT SUBACK/source-ready" + ) + await asyncio.sleep(0.01) + with self._lock: + if ( + self._restart_rehydration_pending == pending + and self._acquisition is acquisition + and acquisition.state == "awaiting_external_start" + and self._camera_stop_priority_counts.get( + checkpoint.acquisition_id, + 0, + ) + == 0 + ): + next_pending = dataclass_replace( + pending, + phase="awaiting-first-pcl", + ) + self._restart_rehydration_pending = next_pending + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_reason_code = ( + "restart-receiver-awaiting-first-pcl" + ) + acquisition.message_code = ( + "acquisition.restart.awaiting_first_pcl" + ) + acquisition.result = { + "read_only_recovery": True, + "device_state": "scanning", + "receiver_rehydration": "awaiting-first-pcl", + "automatic_replay_allowed": False, + } + return "rehydrated" + except BaseException as exc: + setup_error = exc + return self._cleanup_failed_restart_rehydration( + token=token, + checkpoint=checkpoint, + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + setup_error=setup_error, + ) + + @_serialized_acquisition_access + def _cleanup_failed_restart_rehydration( + self, + *, + token: _ActiveAcquisitionCheckpointTrustToken, + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + pending: _RestartRehydrationPending, + acquisition: AcquisitionRecord, + out_dir: Path, + lease: ActiveSessionLease, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + setup_error: BaseException, + ) -> Literal["rehydrated", "stop-only"]: + """Retire a failed new writer, linearized ahead of or behind STOP.""" + + checkpoint_lineage = ( + self._restart_stop_only_checkpoint_lineage( + token=token, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + ) + if checkpoint.state == "active" + else _RestartStopOnlyCheckpointLineage( + checkpoint_revision=checkpoint.revision, + acquisition_id=checkpoint.acquisition_id, + original_start_operation_id=checkpoint.original_start_operation_id, + prior_evidence_session_id=checkpoint.current_evidence_session_id, + cessation_evidence_session_id=pending.new_evidence_session_id, + ) + ) + with self._lock: + current_pending = self._restart_rehydration_pending + stop_won = bool( + self._acquisition is acquisition + and ( + acquisition.state == "awaiting_external_stop" + or self._acquisition_stop_operation_id is not None + or self._camera_stop_priority_counts.get( + checkpoint.acquisition_id, + 0, + ) + > 0 + ) + ) + exact_setup_owner = bool( + current_pending is not None + and current_pending.token == pending.token + and self._acquisition is acquisition + and self._acquisition_out_dir == out_dir + and self._acquisition_session_lease is lease + ) + if exact_setup_owner and not stop_won and current_pending is not None: + self._restart_rehydration_pending = dataclass_replace( + current_pending, + phase="blocked", + reason_code="restart-receiver-setup-failed", + ) + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = ( + "restart-receiver-setup-failed" + ) + if stop_won or not exact_setup_owner: + # STOP owns producer teardown and terminal checkpoint proof. + # Verify must never clear or replace that acquisition owner. + return "rehydrated" + try: + self.runtime.stop() + except Exception: + self._mark_restart_rehydration_cleanup_pending( + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + ) + self._set_active_acquisition_checkpoint_reason( + "restart-receiver-cleanup-pending" + ) + return "rehydrated" + stopped_runtime = dict(self.runtime.snapshot()) + if not ( + stopped_runtime.get("phase") == "idle" + and stopped_runtime.get("source_mode") == "idle" + ): + self._mark_restart_rehydration_cleanup_pending( + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + ) + self._set_active_acquisition_checkpoint_reason( + "restart-receiver-cleanup-pending" + ) + return "rehydrated" + + # STOP publishes its priority while waiting on the lifecycle gate. The + # short commit gate below makes the rest of local teardown atomic with + # that publication: STOP-before-gate retains the exact owner/lease; + # cleanup-before-gate installs the STOP-only shell before STOP enters. + with self._camera_restart_commit_gate: + with self._lock: + current_pending = self._restart_rehydration_pending + exact_cleanup_owner = bool( + current_pending is not None + and current_pending.token == pending.token + and self._acquisition is acquisition + and self._acquisition_out_dir == out_dir + and self._acquisition_session_lease is lease + and acquisition.state == "awaiting_external_start" + and self._acquisition_stop_operation_id is None + ) + stop_won = bool( + self._camera_stop_priority_counts.get( + checkpoint.acquisition_id, + 0, + ) + ) + if stop_won or not exact_cleanup_owner: + return "rehydrated" + self._reset_live_data_plane_observation() + try: + self.live_perception_ingress.end_session(out_dir.name) + except Exception: + self._mark_restart_rehydration_cleanup_pending( + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + ) + self._set_active_acquisition_checkpoint_reason( + "restart-receiver-cleanup-pending" + ) + return "rehydrated" + if self.live_perception_ingress.snapshot().get("active") is not False: + self._mark_restart_rehydration_cleanup_pending( + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + ) + self._set_active_acquisition_checkpoint_reason( + "restart-receiver-cleanup-pending" + ) + return "rehydrated" + try: + lease.release() + except Exception: + self._mark_restart_rehydration_cleanup_pending( + pending=pending, + acquisition=acquisition, + out_dir=out_dir, + lease=lease, + ) + self._set_active_acquisition_checkpoint_reason( + "restart-receiver-cleanup-pending" + ) + return "rehydrated" + with self._lock: + self._acquisition = None + self._restart_stop_only_checkpoint_owner = None + self._acquisition_project_name = None + self._acquisition_mount_type = None + self._acquisition_gnss_mode = None + self._acquisition_out_dir = None + self._acquisition_start_operation_id = None + self._acquisition_stop_operation_id = None + self._acquisition_session_lease = None + self._restart_rehydration_pending = None + lineage = self._active_stream_recovery_lineage + if ( + lineage is not None + and lineage.acquisition_id == pending.token.acquisition_id + and lineage.evidence_session_id == pending.new_evidence_session_id + and lineage.runtime_producer_generation + == pending.runtime_producer_generation + ): + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_lineage = None + self._active_stream_recovery_state = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = None + self._active_stream_recovery_started_at_utc = None + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None + if checkpoint.state == "prepared": + self._restart_stop_only_prepared_pending = pending + self._materialize_recovered_physical_stop_acquisition( + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + checkpoint_lineage=checkpoint_lineage, + ) + logger.warning( + "K1 restart receiver failed; preserved evidence and STOP-only recovery", + extra={ + "event_code": "k1_restart_rehydration_setup_failed", + "reason_code": "restart-receiver-setup-failed", + "automatic_command_retry": False, + }, + exc_info=setup_error, + ) + return "stop-only" + + def _mark_restart_rehydration_cleanup_pending( + self, + *, + pending: _RestartRehydrationPending, + acquisition: AcquisitionRecord, + out_dir: Path, + lease: ActiveSessionLease, + ) -> None: + """Keep one failed local restart owner visible without granting writes.""" + + with self._lock: + current = self._restart_rehydration_pending + lineage = self._active_stream_recovery_lineage + if not ( + current is not None + and current.token == pending.token + and self._acquisition is acquisition + and self._acquisition_out_dir == out_dir + and self._acquisition_session_lease is lease + ): + return + self._restart_rehydration_pending = dataclass_replace( + current, + phase="blocked", + reason_code="restart-receiver-cleanup-pending", + ) + if ( + lineage is not None + and lineage.acquisition_id == pending.token.acquisition_id + and lineage.evidence_session_id == pending.new_evidence_session_id + and lineage.runtime_producer_generation + == pending.runtime_producer_generation + ): + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = ( + "restart-receiver-cleanup-pending" + ) + + def _discard_retained_owner_before_stop_only_materialization( + self, + *, + owner: _PreparedStopRecoveryOwner, + ) -> None: + """End only local producers when checkpoint CAS forbids retained resume.""" + + if not self._prepared_stop_recovery_owner_is_current( + owner, + allow_resolved_classification=True, + ): + raise RuntimeError("retained recovery owner changed before STOP-only downgrade") + acquisition = owner.acquisition + with self._lock: + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "failed", + message_code="acquisition.recovery.checkpoint_untrusted_stop_only", + result={ + "receiver_stopped": False, + "device_state": "scanning", + "recovery_only": True, + "automatic_replay_allowed": False, + }, + ) + self._stop_acquisition_sources( + camera_status="interrupted", + camera_failure_code="checkpoint-untrusted-stop-only", + ) + with self._lock: + self._acquisition_out_dir = None + self._acquisition_start_operation_id = None + self._acquisition_stop_operation_id = None + self._prepared_stop_dispatch_lineage = None + self._prepared_stop_recovery_owner = None + self._classified_stop_rebind_pending = None + self._classified_stop_rebind_inflight = None + + @_serialized_acquisition_access + def _materialize_recovered_physical_stop_acquisition( + self, + *, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + checkpoint_lineage: _RestartStopOnlyCheckpointLineage | None = None, + ) -> None: + """Rehydrate only the bounded acquisition identity required for STOP. + + Read-only topology adoption rotates the process-local device session + and deliberately clears stale acquisition state. If the same Verify + then proves canonical SCANNING, the durable physical record is the + authority for the original acquisition id. This projection restores + no receiver, stream, evidence directory, project mutation, START + capability, or replay permission; it exists only so one explicit STOP + can consume the fresh reconciled control binding. + """ + + if not isinstance(reconciliation_id, str) or not reconciliation_id: + raise RuntimeError("active recovery reconciliation id is unavailable") + if reconciled_record.get("stage") != "resolved": + raise RuntimeError("active recovery physical record is not resolved") + acquisition_id = reconciled_record.get("acquisition_id") + compatibility_profile_id = reconciled_record.get("compatibility_profile_id") + reconciliations = reconciled_record.get("reconciliations") + if ( + not isinstance(acquisition_id, str) + or not acquisition_id + or compatibility_profile_id != XGRIDS_K1_COMPATIBILITY_PROFILE_ID + or not isinstance(reconciliations, list) + ): + raise RuntimeError("active recovery physical record is incomplete") + reconciliation = next( + ( + item + for item in reversed(reconciliations) + if isinstance(item, Mapping) and item.get("reconciliation_id") == reconciliation_id + ), + None, + ) + if not isinstance(reconciliation, Mapping): + raise RuntimeError("active recovery reconciliation evidence is missing") + if reconciliation.get("resolution") != "physical-active-observed": + raise RuntimeError("active recovery did not prove physical SCANNING") + verified_binding = reconciliation.get("verified_binding") + observation = reconciliation.get("observation") + if not isinstance(verified_binding, Mapping) or not isinstance( + observation, + Mapping, + ): + raise RuntimeError("active recovery binding evidence is missing") + reconciled_connection = verified_binding.get("connection") + if not isinstance(reconciled_connection, Mapping): + raise RuntimeError("active recovery connection evidence is missing") + if not ( + observation.get("source") == "explicit-read-only-reconciliation" + and observation.get("session_state") == "scanning" + and observation.get("project_bound") is True + and observation.get("init_ready") is True + and observation.get("mqtt_retained") is False + and isinstance(observation.get("project_id_sha256"), str) + ): + raise RuntimeError("active recovery status is not canonical SCANNING") + + control = dict(self._application_control_session.snapshot()) + verified_control = control.get("verified_control") + if ( + control.get("state") != "scanning" + or control.get("can_stop") is not True + or not isinstance(verified_control, Mapping) + ): + raise RuntimeError("active recovery control is not STOP-capable") + exact_binding_fields = ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + if any( + reconciled_connection.get(field) != verified_control.get(field) + for field in exact_binding_fields + ): + raise RuntimeError("active recovery control binding changed after reconciliation") + if any( + observation.get(field) != verified_control.get(field) + for field in ( + "control_session_id", + "host_path_epoch", + "producer_generation", + ) + ): + raise RuntimeError("active recovery status generation is not current") + + runtime = self.runtime.snapshot() + camera = self.camera_preview.snapshot() + ingress = self.live_perception_ingress.snapshot() + if self._cleanup_inert_terminal_camera_residual_for_recovered_stop( + runtime=runtime, + camera=camera, + ingress=ingress, + ): + runtime = self.runtime.snapshot() + camera = self.camera_preview.snapshot() + ingress = self.live_perception_ingress.snapshot() + with self._lock: + target_host = self._k1_ip + device_id = self._device_id + device_session_id = self._device_session_id + if ( + self._selected_device_id != reconciled_connection.get("transport_ref") + or self._connection_mode != reconciled_connection.get("connection_mode") + or target_host is None + or target_host != reconciled_connection.get("target_ipv4") + or device_id is None + or device_session_id is None + ): + raise RuntimeError("active recovery facade topology is not current") + current = self._acquisition + if current is not None and current.state not in TERMINAL_ACQUISITION_STATES: + raise RuntimeError("active recovery cannot replace a live acquisition") + recording = camera.get("recording") + if ( + self._acquisition_session_lease is not None + or self._acquisition_out_dir is not None + or self._acquisition_start_operation_id is not None + or self._acquisition_stop_operation_id is not None + or self._prepared_stop_dispatch_lineage is not None + or getattr(self, "_prepared_stop_recovery_owner", None) is not None + or runtime.get("source_mode") != "idle" + or runtime.get("phase") != "idle" + or camera.get("active_source_id") is not None + or camera.get("phase") != "idle" + or not isinstance(recording, Mapping) + or recording.get("active") is not False + or recording.get("session") is not None + or recording.get("active_epoch") is not None + or recording.get("producer_alive") is not False + or self._live_perception_camera_binding is not None + or self._camera_activation_lineage is not None + or self._camera_activation_retry_lineage is not None + or "camera" in self._application_control_process_lease_holders + or self._operator_manual_acquisition_process_lease_id is not None + or self._active_stream_recovery_lineage is not None + or self._active_stream_recovery_future is not None + or self._active_stream_recovery_state != "inactive" + or ingress.get("active") is not False + ): + raise RuntimeError( + "active recovery cannot replace retained local acquisition resources" + ) + recovery_result: dict[str, object] = { + "receiver_stopped": True, + "device_state": "scanning", + "recovery_only": True, + "physical_command_operation_id": reconciled_record.get( + "operation_id" + ), + "physical_reconciliation_id": reconciliation_id, + "project_id_sha256": observation.get("project_id_sha256"), + "automatic_replay_allowed": False, + } + recovery = AcquisitionRecord( + acquisition_id=acquisition_id, + device_id=device_id, + device_session_id=device_session_id, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_mode="plugin-commanded", + requested_streams=(), + target_host=target_host, + duration_seconds=None, + evidence_policy="disabled", + state="failed", + message_code="acquisition.physical_active_recovered_stop_only", + result=recovery_result, + ) + self._acquisition = recovery + self._restart_stop_only_checkpoint_owner = ( + (recovery, checkpoint_lineage) + if checkpoint_lineage is not None + else None + ) + self._acquisition_project_name = None + self._acquisition_mount_type = None + self._acquisition_gnss_mode = None + self._acquisition_out_dir = None + self._acquisition_start_operation_id = None + self._acquisition_stop_operation_id = None + + def _adopt_classified_prepared_stop_scanning_in_place( + self, + *, + owner: _PreparedStopRecoveryOwner, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + checkpoint_trust_token: _ActiveAcquisitionCheckpointTrustToken | None = None, + ) -> bool: + """Restore the retained receiver only under one current checkpoint CAS.""" + + with self._acquisition_lifecycle_access(): + physical = dict(self._physical_command_coordinator.snapshot()) + control = dict(self._application_control_session.snapshot()) + verified_control = control.get("verified_control") + physical_record = physical.get("record") + if not ( + reconciled_record.get("operation_id") + == owner.lineage.stop_operation_id + and reconciled_record.get("acquisition_id") + == owner.lineage.acquisition_id + and isinstance(physical_record, Mapping) + and dict(physical_record) == dict(reconciled_record) + and isinstance(verified_control, Mapping) + and control.get("state") == "scanning" + and self._matching_classified_prepared_stop_active( + physical, + acquisition_id=owner.lineage.acquisition_id, + stop_operation_id=owner.lineage.stop_operation_id, + start_operation_id=owner.start_operation_id, + verified_control=verified_control, + ) + ): + raise RuntimeError("classified PREPARED STOP active proof is not current") + if not self._prepared_stop_recovery_owner_is_current( + owner, + allow_resolved_classification=True, + ): + raise RuntimeError("retained acquisition changed before SCANNING adoption") + authority_token = checkpoint_trust_token or owner.checkpoint_trust_token + checkpoint_current = self._active_acquisition_checkpoint_trust_token_is_current( + authority_token, + reconciled_record=reconciled_record, + ) + if not checkpoint_current: + if authority_token is None: + self._mark_active_acquisition_checkpoint_untrusted( + trust="unavailable", + reason_code=( + "active-acquisition-recovery-checkpoint-token-unavailable" + ), + ) + # Keep the retained-owner teardown and STOP-only projection in + # this same lifecycle critical section. A stale checkpoint can + # therefore never race from a rejected CAS into live adoption. + self._discard_retained_owner_before_stop_only_materialization( + owner=owner, + ) + self._materialize_recovered_physical_stop_acquisition( + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + ) + return False + assert authority_token is not None + self._freeze_classified_stop_rebind_until_pcl( + owner=owner, + token=authority_token, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled_record, + control=control, + ) + self.camera_preview.cancel_expected_source_end() + camera = self.camera_preview.snapshot() + recording = camera.get("recording") + camera_still_healthy = bool( + camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and camera.get("phase") != "error" + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == owner.out_dir.name + and recording.get("producer_alive") is not False + ) + with self._lock: + acquisition = self._acquisition + if acquisition is not owner.acquisition: + raise RuntimeError("retained acquisition owner changed") + fresh_device_id = self._device_id + fresh_device_session_id = self._device_session_id + fresh_target = verified_control.get("target_ipv4") + if not ( + isinstance(fresh_device_id, str) + and isinstance(fresh_device_session_id, str) + and isinstance(fresh_target, str) + and fresh_target == owner.lineage.target_ipv4 + ): + raise RuntimeError("fresh classified topology is incomplete") + acquisition.device_id = fresh_device_id + acquisition.device_session_id = fresh_device_session_id + acquisition.target_host = fresh_target + pending_start = False + if owner.start_operation_id is not None: + try: + pending_start = self._operations.get(owner.start_operation_id).status in { + "accepted", + "running", + "operator_action_required", + } + except KeyError: + pending_start = False + acquisition.transition( + "awaiting_external_start" + if pending_start + else "acquiring", + message_code=( + "acquisition.start.device_initializing" + if pending_start + else "acquisition.recovery.scanning_resumed" + ), + result={ + "read_only_recovery": True, + "device_state": "scanning", + "physical_stop_not_dispatched": True, + "physical_reconciliation_id": reconciliation_id, + "automatic_command_retry": False, + }, + ) + if self._acquisition_stop_operation_id == owner.lineage.stop_operation_id: + self._acquisition_stop_operation_id = None + self._prepared_stop_dispatch_lineage = None + # Convert the owner to the current facade session while the + # existing capture thread consumes its first recovered PCL. + # It remains commandless and is cleared by the next explicit + # STOP/terminal cleanup. + self._prepared_stop_recovery_owner = dataclass_replace( + owner, + acquisition=acquisition, + ) + if not camera_still_healthy: + activation = ( + owner.lineage.acquisition_id, + owner.out_dir.name, + owner.runtime_producer_generation, + ) + if self._camera_activation_lineage == activation: + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = activation + self._camera_activation_retry_not_before_monotonic = 0.0 + return True + + def _freeze_classified_stop_rebind_until_pcl( + self, + *, + owner: _PreparedStopRecoveryOwner, + token: _ActiveAcquisitionCheckpointTrustToken, + reconciliation_id: str, + reconciled_record: Mapping[str, Any], + control: Mapping[str, Any], + ) -> None: + """Open one exact gap; a later published PCL alone may close it.""" + + if token.checkpoint_state != "active": + raise ActiveAcquisitionRecoveryCheckpointError( + "classified STOP rebind requires an ACTIVE checkpoint" + ) + store = self._require_active_acquisition_checkpoint_store() + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + verified_control = self._checkpoint_verified_control(control) + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.as_dict() == dict(reconciled_record) + and record.operation_id == owner.lineage.stop_operation_id + and record.acquisition_id == owner.lineage.acquisition_id + and record.action == "stop" + and record.stage == "resolved" + and record.resolution == "not-dispatched" + and reconciliation is not None + and reconciliation.reconciliation_id == reconciliation_id + and reconciliation.kind == "prepared-stop-classification" + and reconciliation.resolution == "physical-active-observed" + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and reconciliation.observation.session_state == "scanning" + and reconciliation.observation.project_bound + and reconciliation.observation.init_ready + and not reconciliation.observation.mqtt_retained + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "classified STOP rebind lacks exact fresh SCANNING ancestry" + ) + snapshot = store.snapshot() + checkpoint = snapshot.checkpoint + if not ( + snapshot.status == "active" + and checkpoint is not None + and checkpoint.acquisition_id == token.acquisition_id + and checkpoint.original_start_operation_id + == token.root_start_operation_id + and checkpoint.current_evidence_session_id == owner.out_dir.name + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "classified STOP rebind checkpoint changed before gap" + ) + gap_started_at_utc = reconciliation.observation.observed_at_utc + owned_open_gap = bool( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + and checkpoint.last_gap_failed_binding == token.checkpoint.current_binding + and checkpoint.current_binding == token.checkpoint.current_binding + ) + exact_lost_response_gap = bool( + owned_open_gap + and checkpoint.revision == token.checkpoint_revision + 1 + and checkpoint.last_gap_started_at_utc == gap_started_at_utc + ) + if checkpoint == token.checkpoint and not owned_open_gap: + checkpoint = store.record_gap( + transition_id=self._active_acquisition_checkpoint_transition_id( + "classified-stop-record-gap", + token.acquisition_id, + reconciliation_id, + record.operation_id, + token.checkpoint_revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + failed_binding=checkpoint.current_binding, + gap_started_at_utc=gap_started_at_utc, + ) + elif checkpoint != token.checkpoint and not exact_lost_response_gap: + raise ActiveAcquisitionRecoveryCheckpointError( + "classified STOP rebind does not own the open checkpoint gap" + ) + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=( + reconciliation.verified_binding.identity.vendor_device_id_sha256 + ), + identity_serial_sha256=( + reconciliation.verified_binding.identity.device_serial_sha256 + ), + compatibility_profile_id=record.compatibility_profile_id, + connection=reconciliation.verified_binding.connection.as_dict(), + verified_control=verified_control, + ) + status_proof = self._checkpoint_status_proof( + status=reconciliation.observation, + binding=binding, + evidence_session_id=owner.out_dir.name, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + pending = _ClassifiedStopRebindPending( + token=token, + checkpoint_gap_revision=checkpoint.revision, + acquisition_id=owner.lineage.acquisition_id, + root_start_operation_id=token.root_start_operation_id, + classified_stop_operation_id=record.operation_id, + classified_stop_revision=record.revision, + reconciliation_id=reconciliation_id, + evidence_session_id=owner.out_dir.name, + runtime_producer_generation=owner.runtime_producer_generation, + status_proof=status_proof, + physical_proof=physical_proof, + transition_id=self._active_acquisition_checkpoint_transition_id( + "classified-stop-rebind", + token.acquisition_id, + reconciliation_id, + record.revision, + checkpoint.revision, + ), + ) + with self._lock: + existing = self._classified_stop_rebind_pending + if existing is not None and existing != pending: + raise ActiveAcquisitionRecoveryCheckpointError( + "another classified STOP rebind is already pending" + ) + self._classified_stop_rebind_pending = pending + + async def _retire_control_generation_opened_by_verify(self) -> None: + """Boundedly retire only the local control generation opened by Verify.""" + + control = dict(self._application_control_session.snapshot()) + state = str(control.get("state") or "unknown") + if state in {"connection-ready", "workspace-ready", "project-ready"}: + await self._retire_prestart_control_for_mode_transition() + return + if state in {"completed", "closed", "failed"}: + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + return + if state != "idle": + await asyncio.to_thread(self._application_control_session.close) + terminal = await self._await_local_control_terminal() + if terminal.get("state") != "idle": + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + return + self._reconcile_application_control_process_lease(control) + + def _physical_recovery_target(self) -> tuple[str, ConnectionMode] | None: + """Return the immutable BLE/mode target of a required physical recovery.""" + + physical = self._physical_command_coordinator.snapshot() + recovery_required = bool( + physical.get("requires_reconciliation") is True + or physical.get("resolved_active_recovery_required") is True + ) + record = physical.get("record") + connection = record.get("connection") if isinstance(record, Mapping) else None + transport_ref = connection.get("transport_ref") if isinstance(connection, Mapping) else None + connection_mode = ( + connection.get("connection_mode") if isinstance(connection, Mapping) else None + ) + if not recovery_required: + return None + if ( + not isinstance(transport_ref, str) + or not transport_ref.strip() + or connection_mode not in {"bridge", "quick-connect", "direct-connect"} + ): + raise ConnectionVerificationError( + "Физическая recovery-цель K1 отсутствует в durable ledger", + reason_code="physical-command-recovery-target-unavailable", + ) + return transport_ref, cast(ConnectionMode, connection_mode) + + def _resolve_legacy_connection_verify_request( + self, + request: ConnectionVerifyRequest, + ) -> ConnectionVerifyRequest: + """Resolve the legacy empty request from server-owned evidence only.""" + + if request.device_id is not None: + return request + + ledger_snapshot = self._network_mutation_ledger.snapshot() + physical_recovery_target = self._physical_recovery_target() + semantic_store = self._semantic_topology_store + semantic_snapshot = semantic_store.snapshot() if semantic_store is not None else None + semantic_record = ( + semantic_snapshot.record + if semantic_snapshot is not None and semantic_snapshot.status == "available" + else None + ) + with self._lock: + fresh_refs_by_key = { + physical_transport_ref_comparison_key(str(item.get("device_id"))): str( + item.get("device_id") + ) + for item in self._fresh_ble_devices_locked() + if isinstance(item.get("device_id"), str) and str(item.get("device_id")).strip() + } + discovery_generation = self._ble_discovery_generation + selected_device_id = self._selected_device_id + selected_connection_mode = self._connection_mode + selected_device_session_id = self._device_session_id + + if physical_recovery_target is not None: + transport_ref, connection_mode = physical_recovery_target + durable_target_available = bool( + semantic_record is not None + and physical_transport_ref_comparison_key(semantic_record.transport_ref) + == physical_transport_ref_comparison_key(transport_ref) + and semantic_record.connection_mode == connection_mode + and semantic_record.compatibility_profile_id + == XGRIDS_K1_COMPATIBILITY_PROFILE_ID + and semantic_record.firmware_version == "3.0.2" + ) + elif ledger_snapshot.status == "unresolved" and ledger_snapshot.record is not None: + transport_ref = ledger_snapshot.record.transport_ref + connection_mode = ledger_snapshot.record.intended_mode + durable_target_available = True + elif selected_device_id is not None and selected_connection_mode is not None: + transport_ref = selected_device_id + connection_mode = selected_connection_mode + durable_target_available = bool( + semantic_record is not None + and physical_transport_ref_comparison_key(semantic_record.transport_ref) + == physical_transport_ref_comparison_key(transport_ref) + and semantic_record.connection_mode == connection_mode + ) + elif semantic_record is not None: + transport_ref = semantic_record.transport_ref + connection_mode = semantic_record.connection_mode + durable_target_available = True + else: + raise ConnectionVerificationError( + "Сервер не сохранил точную цель для read-only проверки K1", + reason_code="connection-verify-connection-missing", + ) + + fresh_transport_ref = fresh_refs_by_key.get( + physical_transport_ref_comparison_key(transport_ref) + ) + if fresh_transport_ref is not None: + source: Literal[ + "fresh-scan", + "retained-current-process", + "durable-configured-state", + ] = "fresh-scan" + else: + retained = connected_device_recovery_snapshot( + transport_ref, + device_session_id=( + selected_device_session_id + if selected_device_id is not None + and physical_transport_ref_comparison_key(selected_device_id) + == physical_transport_ref_comparison_key(transport_ref) + and selected_connection_mode == connection_mode + else None + ), + ) + if retained["status"] == "retained": + source = "retained-current-process" + elif durable_target_available: + source = "durable-configured-state" + else: + raise ConnectionVerificationError( + "Для серверной цели K1 нет свежего, process-retained или durable BLE-контекста", + reason_code="connection-verify-candidate-not-fresh", + ) + + return ConnectionVerifyRequest( + # A UUID is case-insensitive for durable identity/CAS matching, but + # CoreBluetooth must receive the exact spelling from the current + # advertisement. Preserve durable evidence in its own stores and + # carry the fresh native reference only for this read-only Verify. + device_id=(fresh_transport_ref if source == "fresh-scan" else transport_ref), + source=source, + compatibility_attestation=CompatibilityAttestationRequest( + firmware_version="3.0.2", + topology=CONNECTION_TOPOLOGY_BY_MODE[connection_mode], + verification="live-device-info", + ), + operation_id=request.operation_id, + expected_discovery_generation=( + discovery_generation if source == "fresh-scan" else None + ), + expected_reconfiguration_revision=(request.expected_reconfiguration_revision), + expected_reconfiguration_intent_id=(request.expected_reconfiguration_intent_id), + ) + + async def _verify_connection_owned( + self, + request: ConnectionVerifyRequest, + ) -> tuple[str, bool, _ProvisionalFreshBridgeTopology | None]: + """Verify one server-bound topology without mutating the K1.""" + request = self._resolve_legacy_connection_verify_request(request) + if request.device_id is None or request.compatibility_attestation is None: + raise ConnectionVerificationError( + "Read-only проверка K1 не получила точную серверную цель", + reason_code="connection-verify-connection-missing", + ) + self._require_transport_not_physically_retired( + request.device_id, + verification=True, + ) + requested_mode = cast( + ConnectionMode, + { + "direct-lan": "bridge", + "device-ap": "quick-connect", + "controller-hotspot": "direct-connect", + }[request.compatibility_attestation.topology], + ) + self._require_current_connection_reconfiguration_target( + expected_revision=request.expected_reconfiguration_revision, + expected_intent_id=request.expected_reconfiguration_intent_id, + device_id=request.device_id, + connection_mode=requested_mode, + source=request.source, + discovery_generation=request.expected_discovery_generation, + ) + physical_recovery_target = self._physical_recovery_target() + if physical_recovery_target is not None: + required_transport_ref, required_mode = physical_recovery_target + if ( + physical_transport_ref_comparison_key(request.device_id) + != physical_transport_ref_comparison_key(required_transport_ref) + or requested_mode != required_mode + ): + raise ConnectionVerificationError( + "Выбран другой K1 или способ связи; physical recovery привязан " + "к исходному устройству", + reason_code="physical-command-recovery-target-mismatch", + ) + with self._lock: + device_id = self._device_id + device_session_id = self._device_session_id + request_fingerprint = self._request_fingerprint( + ACTION_CONNECTION_VERIFY, + { + "device_id": request.device_id, + "source": request.source, + "expected_discovery_generation": (request.expected_discovery_generation), + "expected_reconfiguration_revision": (request.expected_reconfiguration_revision), + "expected_reconfiguration_intent_id": (request.expected_reconfiguration_intent_id), + "compatibility_attestation": ( + request.compatibility_attestation.model_dump(mode="json") + if request.compatibility_attestation is not None + else None + ), + }, + ) + verify_operation_context: dict[str, Any] = { + "snapshot_runtime_id": self._snapshot_runtime_id, + "requested_transport_ref": request.device_id, + "requested_connection_mode": requested_mode, + "verification_source": request.source, + "automatic_retry": False, + } + network_parent = next( + ( + item + for item in reversed(self._operations.snapshot(limit=128)) + if item.get("action") == ACTION_NETWORK_PROVISION + and item.get("status") == "succeeded" + and isinstance(item.get("result"), Mapping) + and item["result"].get("phase") == "network_applied" + ), + None, + ) + supervisor_before_verify = self._connection_supervisor.snapshot() + if isinstance(network_parent, Mapping): + parent_context = network_parent.get("context") + parent_result = network_parent.get("result") + parent_operation_id = network_parent.get("operation_id") + parent_transport_ref = ( + parent_result.get("transport_ref") if isinstance(parent_result, Mapping) else None + ) + parent_target_ipv4 = ( + parent_result.get("target_ipv4") if isinstance(parent_result, Mapping) else None + ) + parent_target_port = ( + parent_result.get("target_port") if isinstance(parent_result, Mapping) else None + ) + parent_target = ( + EndpointTarget(parent_target_ipv4, parent_target_port) + if isinstance(parent_target_ipv4, str) and parent_target_port == CONTROL_MQTT_PORT + else None + ) + durable_replay_binding = ( + self._durable_terminal_replay_binding( + operation_id=parent_operation_id, + transport_ref=parent_transport_ref, + connection_mode=requested_mode, + ) + if isinstance(parent_operation_id, str) + and parent_operation_id + and isinstance(parent_transport_ref, str) + and isinstance(parent_result, Mapping) + and parent_result.get("durable_replay") is True + else None + ) + exact_parent_document = bool( + isinstance(parent_operation_id, str) + and parent_operation_id + and isinstance(parent_context, Mapping) + and parent_context.get("snapshot_runtime_id") == self._snapshot_runtime_id + and parent_context.get("connection_mode") == requested_mode + and parent_context.get("transport_ref") == parent_transport_ref + and isinstance(parent_result, Mapping) + and parent_result.get("snapshot_runtime_id") == self._snapshot_runtime_id + and parent_result.get("parent_intent_id") == parent_operation_id + and parent_result.get("connection_mode") == requested_mode + and isinstance(parent_transport_ref, str) + and physical_transport_ref_comparison_key(parent_transport_ref) + == physical_transport_ref_comparison_key(request.device_id) + and parent_target is not None + ) + exact_live_parent = bool( + exact_parent_document + and supervisor_before_verify.intent is not None + and supervisor_before_verify.intent.intent_id == parent_operation_id + and supervisor_before_verify.intent.requested_mode == requested_mode + and supervisor_before_verify.device_network.state == "applied" + and supervisor_before_verify.device_network.intent_id == parent_operation_id + and supervisor_before_verify.device_network.transport_ref == parent_transport_ref + and supervisor_before_verify.device_network.connection_mode == requested_mode + and supervisor_before_verify.device_network.target == parent_target + ) + exact_durable_replay_parent = bool( + exact_parent_document + and isinstance(parent_result, Mapping) + and parent_result.get("durable_replay") is True + and isinstance(durable_replay_binding, Mapping) + and durable_replay_binding.get("snapshot_runtime_id") == self._snapshot_runtime_id + and durable_replay_binding.get("parent_intent_id") == parent_operation_id + and durable_replay_binding.get("transport_ref") == parent_transport_ref + and durable_replay_binding.get("connection_mode") == requested_mode + and durable_replay_binding.get("target_ipv4") == parent_target_ipv4 + and durable_replay_binding.get("target_port") == parent_target_port + ) + if parent_target is not None and (exact_live_parent or exact_durable_replay_parent): + verify_operation_context.update( + { + "recovery_parent_operation_id": parent_operation_id, + "recovery_parent_intent_id": parent_operation_id, + "recovery_transport_ref": parent_transport_ref, + "recovery_connection_mode": requested_mode, + "recovery_target_ipv4": parent_target.ipv4, + "recovery_target_port": parent_target.port, + } + ) + operation, created = self._operations.begin( + ACTION_CONNECTION_VERIFY, + operation_id=request.operation_id, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=CONNECTION_VERIFY_HARD_TIMEOUT_SECONDS, + request_fingerprint=request_fingerprint, + context=verify_operation_context, + ) + if not created: + return operation.operation_id, False, None + self._operations.transition( + operation.operation_id, + "running", + stage_code=( + "read-only-durable-network-verification" + if request.source == "durable-configured-state" + else "read-only-ble-verification" + ), + message_code="connection.verify.running", + ) + try: + _, provisional_topology = await self._adopt_existing_lan_connection( + request.device_id, + request.compatibility_attestation, + verification_source=request.source, + expected_discovery_generation=(request.expected_discovery_generation), + require_live_gatt_validation=(physical_recovery_target is not None), + ) + except asyncio.CancelledError: + self._operations.transition( + operation.operation_id, + "cancelled", + stage_code="cancelled", + message_code="connection.verify.cancelled", + error={ + "category": "connection", + "code": "connection-verify-cancelled", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + raise + except Exception as exc: + failure = _connection_verification_error(exc) + raw_operation_stage = getattr(exc, "operation_stage", None) + operation_stage = ( + raw_operation_stage + if isinstance(raw_operation_stage, str) + and raw_operation_stage in CONNECTION_VERIFY_BLE_OPERATION_STAGES + else None + ) + if operation_stage is not None: + # The safe mapped exception is what reaches both the HTTP + # boundary and the durable operation journal. Preserve only a + # small stage vocabulary; never copy native CoreBluetooth + # text, UUIDs, IPs or tracebacks into operator-visible state. + failure.operation_stage = operation_stage + failure_category = ( + "device-not-found" + if isinstance(exc, BleakDeviceNotFoundError) + else "timeout" + if isinstance(exc, TimeoutError) + else "ble-runtime" + if isinstance(exc, BleakError) + else "connection" + ) + logger.info( + "K1 read-only connection verification failed safely", + extra={ + "event_code": "k1_connection_verify_failed", + "operation_id": operation.operation_id, + "operation_stage": ( + operation_stage if operation_stage is not None else "unknown" + ), + "reason_code": failure.reason_code, + "failure_category": failure_category, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + if failure.reason_code in { + "connection-verify-address-unavailable", + "connection-verify-connection-missing", + "connection-verify-route-mismatch", + "connection-verify-mqtt-unreachable", + }: + # The exact read-only attempt observed a usable BLE identity + # but no usable route for the requested topology. Publish + # that fact instead of falling back to the misleading legacy + # "not-probed" state. This grants no provisioning authority; + # it only lets the operator understand why physical + # reconciliation remains unresolved. + with self._lock: + self._connection_verification = { + "status": "unreachable", + "lease_state": "disconnected", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": "ble-wifi-status-read", + "network_reachability": "unreachable", + "host_route_class": None, + "address_source": "ble-wifi-status-read", + "connection_origin": "external-existing-network", + "admission_source": ACTION_CONNECTION_VERIFY, + "address_changed": False, + "previous_address_present": self._k1_ip is not None, + "write_performed": False, + "observed_at": _utc_now_iso(), + "reason_code": failure.reason_code, + } + self._operations.transition( + operation.operation_id, + "failed", + stage_code=( + "read-only-durable-network-verification-failed" + if request.source == "durable-configured-state" + else "read-only-ble-verification-failed" + ), + message_code="connection.verify.failed", + error=_operation_error( + failure, + category="connection", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + if failure is exc: + raise + raise failure from exc + self._operations.transition( + operation.operation_id, + "running", + stage_code="topology-confirmed-device-info-pending", + message_code="connection.verify.device_info_pending", + ) + return operation.operation_id, True, provisional_topology + + async def _adopt_existing_lan_connection( self, device_id: str, compatibility_attestation: CompatibilityAttestationRequest, - ) -> str: - """Create a process lease from BLE status without changing K1 Wi-Fi state.""" + *, + verification_source: Literal[ + "fresh-scan", "retained-current-process", "durable-configured-state" + ] = "fresh-scan", + expected_discovery_generation: int | None = None, + require_live_gatt_validation: bool = False, + ) -> tuple[str, _ProvisionalFreshBridgeTopology | None]: + """Create a process lease from live or persisted device topology evidence.""" - if compatibility_attestation.topology != "direct-lan": - raise ValueError("read-only K1 adoption supports only direct-lan topology") + requested_mode: ConnectionMode + if compatibility_attestation.topology == "direct-lan": + requested_mode = "bridge" + elif compatibility_attestation.topology == "device-ap": + requested_mode = "quick-connect" + elif compatibility_attestation.topology == "controller-hotspot": + requested_mode = "direct-connect" + else: + raise ValueError("read-only K1 adoption received an unsupported topology") with self._lock: scanned_device = next( ( dict(item) for item in self._fresh_ble_devices_locked() - if str(item.get("device_id")) == device_id + if isinstance(item.get("device_id"), str) + and physical_transport_ref_comparison_key(str(item.get("device_id"))) + == physical_transport_ref_comparison_key(device_id) ), None, ) discovery_generation = self._ble_discovery_generation + current_selected_device_id = self._selected_device_id + current_connection_mode = self._connection_mode + current_device_session_id = self._device_session_id acquisition = self._acquisition acquisition_session_lease = self._acquisition_session_lease provisioning_active = self._provisioning_active previous_target = self._k1_ip - if scanned_device is None: - raise ValueError("сначала найдите и выберите устройство через Bluetooth") - if scanned_device.get("connectable") is False: + reconfiguration_intent = self._connection_reconfiguration_intent + reconfiguration_intent_id = self._connection_reconfiguration_intent_id + reconfiguration_revision = self._connection_reconfiguration_revision + retained_prepared_stop_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + retained_stop_recovery = bool( + retained_prepared_stop_owner is not None + and self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ) + actual_transport_ref = ( + str(scanned_device["device_id"]) + if scanned_device is not None + else ( + current_selected_device_id + if isinstance(current_selected_device_id, str) + and physical_transport_ref_comparison_key(current_selected_device_id) + == physical_transport_ref_comparison_key(device_id) + else device_id + ) + ) + if retained_stop_recovery: + assert retained_prepared_stop_owner is not None + if ( + physical_transport_ref_comparison_key(actual_transport_ref) + != physical_transport_ref_comparison_key( + retained_prepared_stop_owner.lineage.transport_ref + ) + or requested_mode + != retained_prepared_stop_owner.lineage.connection_mode + ): + raise ConnectionVerificationError( + "Read-only recovery не может перенести активный receiver на другой K1 " + "или topology", + reason_code="physical-command-reconciliation-binding-changed", + ) + if ( + verification_source == "fresh-scan" + and expected_discovery_generation != discovery_generation + ): + raise ConnectionVerificationError( + "результаты Bluetooth-поиска изменились; выберите K1 заново", + reason_code="connection-verify-discovery-generation-conflict", + ) + if scanned_device is not None and scanned_device.get("connectable") is False: raise ValueError("выбранное Bluetooth-устройство сейчас недоступно для подключения") if provisioning_active: raise RuntimeError("нельзя проверять существующую сеть во время настройки Wi-Fi") - if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: + if ( + acquisition is not None + and acquisition.state not in TERMINAL_ACQUISITION_STATES + and not retained_stop_recovery + ): raise RuntimeError( "нельзя менять connection lease во время активной acquisition-сессии" ) - if acquisition_session_lease is not None: + if acquisition_session_lease is not None and not retained_stop_recovery: raise RuntimeError("предыдущая evidence-сессия ещё не остановлена и не запечатана") runtime_state = self.runtime.snapshot() - if runtime_state.get("source_mode") != "idle": + if runtime_state.get("source_mode") != "idle" and not retained_stop_recovery: raise RuntimeError("нельзя менять connection lease при активном live/replay источнике") if not self._provisioning_gate.acquire(blocking=False): raise RuntimeError("другая операция настройки или проверки сети уже выполняется") + verification_fence_active = False + provisional_topology: _ProvisionalFreshBridgeTopology | None = None try: - status_read = asyncio.run( - read_wifi_status_once( - device_id, - timeout_seconds=20.0, - rediscover=True, + ledger_snapshot = self._network_mutation_ledger.snapshot() + if ( + ledger_snapshot.status == "unresolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.stage == "prepared" + ): + self._network_mutation_ledger.resolve( + ledger_snapshot.record.operation_id, + expected_revision=ledger_snapshot.record.revision, + resolution="not-dispatched", ) + ledger_snapshot = self._network_mutation_ledger.snapshot() + idempotency_journal = self._require_network_provisioning_idempotency_journal() + idempotency_snapshot = idempotency_journal.snapshot() + active_idempotency = idempotency_snapshot.active_record + if ( + active_idempotency is not None + and active_idempotency.stage == "unresolved" + and active_idempotency.action == ACTION_NETWORK_PROVISION + and ledger_snapshot.status == "resolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.operation_id == active_idempotency.operation_id + ): + if ledger_snapshot.record.resolution == "not-dispatched": + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="network.provision.not-dispatched", + error_code="request-not-dispatched", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + elif ledger_snapshot.record.resolution == "target-observed": + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="succeeded", + outcome_code="network.provision.reconciled", + error_code=None, + side_effect_status="reconciled", + retryable=False, + safe_to_retry=False, + ), + ) + idempotency_snapshot = idempotency_journal.snapshot() + active_idempotency = idempotency_snapshot.active_record + expected_reconciliation = ( + ( + ledger_snapshot.record.operation_id, + ledger_snapshot.record.revision, + ledger_snapshot.record.transport_ref, + ledger_snapshot.record.intended_mode, + ) + if ledger_snapshot.status == "unresolved" and ledger_snapshot.record is not None + else None ) + expected_idempotency_reconciliation = ( + ( + active_idempotency.operation_id, + active_idempotency.revision, + ) + if active_idempotency is not None + and active_idempotency.stage == "unresolved" + and active_idempotency.action == ACTION_NETWORK_PROVISION + else None + ) + if expected_reconciliation is not None and ( + expected_idempotency_reconciliation is None + or expected_idempotency_reconciliation[0] != expected_reconciliation[0] + ): + raise ConnectionVerificationError( + "durable network edge и idempotency edge относятся к разным операциям; " + "BLE-чтение не выполнялось", + reason_code="connection-verify-idempotency-operation-mismatch", + ) + if expected_reconciliation is None and active_idempotency is not None: + raise ConnectionVerificationError( + "idempotency edge не имеет matching network-ledger evidence; " + "BLE-чтение не выполнялось", + reason_code="connection-verify-idempotency-ledger-mismatch", + ) + if expected_reconciliation is not None and ( + physical_transport_ref_comparison_key(expected_reconciliation[2]) + != physical_transport_ref_comparison_key(device_id) + or expected_reconciliation[3] != requested_mode + ): + raise ConnectionVerificationError( + "неразрешённая сетевая команда относится к другому K1 или topology; " + "BLE-чтение выбранного устройства не выполнялось", + reason_code="connection-verify-reconciliation-target-mismatch", + ) + + semantic_store = self._semantic_topology_store + if semantic_store is None: + raise SemanticTopologyStoreCorrupt( + "semantic topology store is unavailable; BLE-чтение не выполнялось" + ) + semantic_snapshot = semantic_store.snapshot() + if semantic_snapshot.status == "corrupt": + raise SemanticTopologyStoreCorrupt( + "semantic topology store is corrupt; BLE-чтение не выполнялось" + ) + semantic_record = semantic_snapshot.record + semantic_predecessor_revision = ( + semantic_record.revision if semantic_record is not None else 0 + ) + resolved_apply_durable_target = _resolved_apply_durable_observation_target( + ledger_snapshot, + semantic_snapshot.as_dict(), + ) + resolved_apply_durable_target_required = bool( + ledger_snapshot.status == "resolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.stage == "resolved" + and ledger_snapshot.record.resolution == "target-observed" + ) + if ( + verification_source == "durable-configured-state" + and resolved_apply_durable_target_required + and resolved_apply_durable_target is None + ): + raise ConnectionVerificationError( + "resolved Apply ledger и durable topology не подтверждают одну " + "точную цель; сеть, BLE и управление не проверялись", + reason_code="connection-verify-resolved-apply-target-mismatch", + ) + resolved_apply_durable_requested = bool( + verification_source == "durable-configured-state" + and resolved_apply_durable_target is not None + and physical_transport_ref_comparison_key(resolved_apply_durable_target[0]) + == physical_transport_ref_comparison_key(device_id) + and resolved_apply_durable_target[1] == requested_mode + and semantic_record is not None + and semantic_record.firmware_version == compatibility_attestation.firmware_version + ) + if resolved_apply_durable_requested: + assert resolved_apply_durable_target is not None + actual_transport_ref = resolved_apply_durable_target[0] + + fresh_capture = ( + capture_discovered_device(actual_transport_ref) + if scanned_device is not None and not resolved_apply_durable_requested + else None + ) + retained_snapshot = connected_device_recovery_snapshot( + actual_transport_ref, + device_session_id=current_device_session_id, + ) + retained_exact = bool( + isinstance(current_selected_device_id, str) + and physical_transport_ref_comparison_key(current_selected_device_id) + == physical_transport_ref_comparison_key(device_id) + and current_device_session_id is not None + and retained_snapshot["status"] == "retained" + ) + + authority_source: Literal[ + "fresh-request", + "current-session", + "unresolved-ledger", + "semantic-topology", + ] + if expected_reconciliation is not None: + authority_source = "unresolved-ledger" + elif resolved_apply_durable_requested: + authority_source = "semantic-topology" + elif scanned_device is not None: + authority_source = "fresh-request" + elif retained_exact and current_connection_mode == requested_mode: + authority_source = "current-session" + elif ( + semantic_record is not None + and physical_transport_ref_comparison_key(semantic_record.transport_ref) + == physical_transport_ref_comparison_key(device_id) + and semantic_record.connection_mode == requested_mode + and semantic_record.firmware_version == compatibility_attestation.firmware_version + ): + authority_source = "semantic-topology" + else: + raise ConnectionVerificationError( + "нет свежего кандидата, точного process recovery-token или " + "серверной durable topology для выбранного K1", + reason_code="connection-verify-candidate-not-fresh", + ) + + transport_source: Literal["fresh-scan", "retained-current-process", "durable-state"] + if resolved_apply_durable_requested: + transport_source = "durable-state" + elif scanned_device is not None: + transport_source = "fresh-scan" + elif retained_exact: + transport_source = "retained-current-process" + elif authority_source in {"unresolved-ledger", "semantic-topology"}: + transport_source = "durable-state" + else: # pragma: no cover - exhaustive fail-closed guard + raise ConnectionVerificationError( + "BLE transport source is unavailable", + reason_code="connection-verify-device-not-rediscovered", + ) + + expected_request_source = { + "fresh-scan": "fresh-scan", + "retained-current-process": "retained-current-process", + "durable-state": "durable-configured-state", + }[transport_source] + if verification_source != expected_request_source: + raise ConnectionVerificationError( + "requested verification source does not match the server-selected target", + reason_code="connection-verify-source-mismatch", + ) + + verification_target = _ConnectionVerificationTarget( + transport_ref=actual_transport_ref, + connection_mode=requested_mode, + transport_source=transport_source, + authority_source=authority_source, + discovery_generation=( + discovery_generation if transport_source == "fresh-scan" else None + ), + device_session_id=( + current_device_session_id + if transport_source == "retained-current-process" + else None + ), + selected_device_id=( + current_selected_device_id + if transport_source == "retained-current-process" + else None + ), + selected_connection_mode=( + current_connection_mode + if transport_source == "retained-current-process" + else None + ), + network_operation_id=( + expected_reconciliation[0] if expected_reconciliation is not None else None + ), + network_revision=( + expected_reconciliation[1] if expected_reconciliation is not None else None + ), + idempotency_operation_id=( + expected_idempotency_reconciliation[0] + if expected_idempotency_reconciliation is not None + else None + ), + idempotency_revision=( + expected_idempotency_reconciliation[1] + if expected_idempotency_reconciliation is not None + else None + ), + semantic_predecessor_revision=semantic_predecessor_revision, + ) + + # Publish the BLE verification fence before awaiting native I/O. + # A scan cannot invalidate a capture or its validation handoff in + # the post-GATT/pre-pin window. + with self._lock: + if self._provisioning_active: + raise RuntimeError("настройка Wi-Fi началась во время проверки сети") + self._provisioning_active = True + verification_fence_active = True + + if ( + verification_target.transport_source == "durable-state" + and verification_target.authority_source == "semantic-topology" + and not require_live_gatt_validation + ): + assert semantic_record is not None + endpoint_observation = await _run_blocking_operation_without_abandonment( + _probe_configured_endpoint_host_only, + semantic_record.ipv4, + path_probe=self._sample_host_path, + ) + current_semantic_snapshot = semantic_store.snapshot() + if ( + current_semantic_snapshot.status != "available" + or current_semantic_snapshot.record != semantic_record + ): + raise ConnectionVerificationError( + "durable topology изменилась во время сетевой проверки", + reason_code="connection-verify-candidate-changed", + ) + if ( + not endpoint_observation.path.available + or endpoint_observation.path.route_class != "direct" + ): + raise RuntimeError( + "Сохранённый адрес K1 находится вне прямого локального маршрута" + ) + if not endpoint_observation.reachable and endpoint_observation.reason_code not in { + "tcp-connection-refused", + "tcp-connection-timeout", + "tcp-endpoint-unreachable", + }: + raise RuntimeError( + "Сохранённый адрес K1 доступен по маршруту, но MQTT endpoint " + "1883 недоступен" + ) + if endpoint_observation.reachable: + observed_at = _utc_now_iso() + self._retire_application_control_for_network_change() + self._apply_durable_semantic_topology( + record=semantic_record, + compatibility_attestation=_attestation_snapshot( + compatibility_attestation + ), + observed_at_utc=observed_at, + ) + self._observe_connection_transport( + semantic_record.ipv4, + path=endpoint_observation.path, + reachable=True, + ) + self._update_applied_topology_reachability( + connection_mode=requested_mode, + ipv4=semantic_record.ipv4, + host_route_class="direct-or-routed", + host_route_ready=True, + endpoint_reachable=True, + write_performed=False, + ) + with self._lock: + self._connection_verification = { + "status": "adopted", + "lease_state": "configured-unverified", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": ( + "durable-semantic-topology+mqtt-tcp-connect" + ), + "network_reachability": "reachable", + "host_route_class": "direct-or-routed", + "address_source": "durable-semantic-topology", + "connection_origin": "persisted-device-reported-network", + "admission_source": ACTION_CONNECTION_VERIFY, + "address_changed": previous_target != semantic_record.ipv4, + "previous_address_present": previous_target is not None, + "write_performed": False, + "observed_at": observed_at, + } + self._operation_message = ( + "Сохранённый адрес K1 отвечает; точная идентичность будет " + "подтверждена по MQTT DeviceInfo перед START." + ) + return semantic_record.ipv4, None + + # A Bridge/Direct address is a DHCP observation, not device + # identity. When the exact persisted endpoint has a stable + # direct route but no MQTT listener, continue into the already + # fenced durable CoreBluetooth retrieval below. That path + # performs one reviewed 7f02 read for this exact UUID, writes + # no characteristic, and still requires MQTT DeviceInfo before + # granting control authority. Never broaden this fallback to + # route/association changes or to another BLE candidate. + logger.info( + "Persisted K1 DHCP endpoint is stale; reading current address over BLE", + extra={ + "event_code": "k1_durable_endpoint_stale_ble_refresh", + "reason_code": endpoint_observation.reason_code, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + + validated_captures: list[CapturedDiscoveredDevice] = [] + exact_uuid_scan_required = bool( + transport_source == "durable-state" + and require_live_gatt_validation + ) + status_read = await read_wifi_status_once( + actual_transport_ref, + timeout_seconds=20.0, + rediscover=exact_uuid_scan_required, + captured_device=(fresh_capture if transport_source == "fresh-scan" else None), + recovery_device_session_id=( + verification_target.device_session_id + if transport_source == "retained-current-process" + else None + ), + allow_known_device_retrieval=(transport_source == "durable-state"), + on_gatt_validated=validated_captures.append, + **( + { + "exact_scan_timeout_seconds": ( + CONNECTION_VERIFY_EXACT_UUID_SCAN_TIMEOUT_SECONDS + ) + } + if exact_uuid_scan_required + else {} + ), + ) + if len(validated_captures) == 1: + validated_capture: CapturedDiscoveredDevice | None = validated_captures[0] + elif transport_source == "fresh-scan" and fresh_capture is None: + # Compatibility path for a facade-owned fresh row whose native + # scanner object was not retained (notably isolated test and + # legacy callers). The exact GATT/7f02 evidence may still be + # admitted, but no future recovery token is invented. + validated_capture = None + else: + raise RuntimeError("BLE status read did not return one validated transport") if ( status_read.get("operation") != "single_reviewed_wifi_status_read" or status_read.get("write_performed") is not False ): raise RuntimeError("BLE status read не подтвердил read-only операцию") - if status_read.get("device_macos_uuid") != device_id: + status_transport_ref = status_read.get("device_macos_uuid") + if not ( + isinstance(status_transport_ref, str) + and physical_transport_ref_comparison_key(status_transport_ref) + == physical_transport_ref_comparison_key(actual_transport_ref) + ): raise RuntimeError("BLE status read вернул состояние другого устройства") - # The exact current 7f02 state has now been observed without a - # device write. Clear only the process-owned ambiguity fence; all - # route/admission checks below still have to pass independently. - with self._lock: - reconciliation = self._network_write_reconciliation - reconciliation_cleared = bool( - reconciliation is not None - and reconciliation.get("transport_ref") == device_id + if transport_source == "fresh-scan": + # The completed, identity-matched GATT read is newer and + # stronger presence evidence than the advertisement that + # admitted it. Renew the bounded candidate lease so a slow + # read cannot make the explicit re-provisioning target expire + # before the following host/endpoint diagnosis completes. + with self._lock: + if ( + self._ble_discovery_generation == verification_target.discovery_generation + and any( + str(item.get("device_id")) == actual_transport_ref + for item in self._devices + ) + ): + self._ble_device_last_seen_monotonic[actual_transport_ref] = ( + time.monotonic() + ) + self._ble_device_last_seen_suspend_aware[actual_transport_ref] = time.time() + raw_properties = status_read.get("write_characteristic_properties") + write_characteristic_properties = ( + list(raw_properties) + if isinstance(raw_properties, (list, tuple)) + and all(isinstance(item, str) and item for item in raw_properties) + else None + ) + max_write_without_response_size = status_read.get("max_write_without_response_size") + if not isinstance(max_write_without_response_size, int) or isinstance( + max_write_without_response_size, bool + ): + max_write_without_response_size = None + mtu_size = status_read.get("mtu_size") + if not isinstance(mtu_size, int) or isinstance(mtu_size, bool): + mtu_size = None + # This event records only the currently observed, non-secret GATT + # envelope. It intentionally precedes DHCP/route admission so a + # read-only diagnostic remains available when K1 reports no LAN IP. + logger.info( + "K1 BLE GATT contract observed by read-only status", + extra={ + "event_code": "k1_ble_gatt_contract_observed", + "write_characteristic_properties": write_characteristic_properties, + "max_write_without_response_size": max_write_without_response_size, + "mtu_size": mtu_size, + "bridge_frame_length": BRIDGE_FRAME_LENGTH, + "quick_connect_frame_length": QUICK_CONNECT_FRAME_LENGTH, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + observed_status = status_read.get("status") + # A read-only response is evidence only when it semantically proves + # the exact topology requested by the unresolved write. Merely + # receiving 7f02, seeing an unchanged baseline, or seeing no IPv4 + # must never reopen mutation authority. + ledger_snapshot = self._network_mutation_ledger.snapshot() + ledger_record = ledger_snapshot.record + current_idempotency_snapshot = idempotency_journal.snapshot() + current_active_idempotency = current_idempotency_snapshot.active_record + idempotency_reconciliation_unchanged = bool( + expected_idempotency_reconciliation is not None + and current_active_idempotency is not None + and current_active_idempotency.stage == "unresolved" + and current_active_idempotency.action == ACTION_NETWORK_PROVISION + and current_active_idempotency.operation_id + == expected_idempotency_reconciliation[0] + and current_active_idempotency.revision == expected_idempotency_reconciliation[1] + ) + network_reconciliation_unchanged = bool( + ledger_snapshot.status == "unresolved" + and ledger_record is not None + and expected_reconciliation is not None + and ledger_record.operation_id == expected_reconciliation[0] + and ledger_record.revision == expected_reconciliation[1] + and physical_transport_ref_comparison_key(ledger_record.transport_ref) + == physical_transport_ref_comparison_key(device_id) + and ledger_record.intended_mode == requested_mode + ) + durable_reconciliation_unchanged = bool( + network_reconciliation_unchanged and idempotency_reconciliation_unchanged + ) + if expected_reconciliation is not None and not durable_reconciliation_unchanged: + raise ConnectionVerificationError( + "durable reconciliation state изменился во время BLE-чтения; " + "результат не применён", + reason_code="connection-verify-reconciliation-revision-changed", ) - if reconciliation_cleared: - self._network_write_reconciliation = None + applied_status_target = _applied_network_target( + requested_mode, + observed_status, + ) + target = applied_status_target + if expected_reconciliation is not None: + assert ledger_record is not None + target = _post_dispatch_network_target( + ledger_record.intended_mode, + observed_status, + transport_ref=ledger_record.transport_ref, + baseline_status=ledger_record.baseline_status, + previous_connection=ledger_record.previous_connection, + ) + if applied_status_target is not None and target is None: + raise ConnectionVerificationError( + "Текущий 7f02-статус неотличим от состояния K1 до сетевой " + "записи; durable barrier сохранён", + reason_code=("connection-verify-target-not-distinguishable-from-baseline"), + ) + reconciliation_cleared = bool( + expected_reconciliation is not None and target is not None + ) + if target is None: + observed_address = ( + observed_status.get("ipv4") if isinstance(observed_status, Mapping) else None + ) + if requested_mode != "quick-connect" and ( + not isinstance(observed_address, str) or observed_address == AP_FALLBACK_IPV4 + ): + raise ConnectionVerificationError( + "K1 не сообщил адрес общей локальной сети", + reason_code="connection-verify-address-unavailable", + ) + raise ConnectionVerificationError( + ( + "K1 не подтвердил активную точку доступа Quick Connect" + if requested_mode == "quick-connect" + else "K1 не подтвердил выбранную топологию общей сети" + ), + reason_code="connection-verify-connection-missing", + ) + target = validate_private_ipv4(target) + if ( + retained_stop_recovery + and retained_prepared_stop_owner is not None + and target != retained_prepared_stop_owner.lineage.target_ipv4 + ): + raise ConnectionVerificationError( + "Read-only recovery обнаружил другой адрес живого K1 receiver; " + "локальная ownership сохранена без изменения", + reason_code="physical-command-reconciliation-binding-changed", + ) + if await _run_blocking_operation_without_abandonment( + _target_is_local_ipv4, + target, + ): + raise RuntimeError("BLE status сообщил адрес, принадлежащий этому компьютеру") + observed_at = str(status_read["observed_at_utc"]) + with self._lock: + if not self._provisioning_active: + raise RuntimeError("read-only BLE verification fence был потерян") + if verification_target.transport_source == "fresh-scan" and ( + self._ble_discovery_generation != verification_target.discovery_generation + or not any( + str(item.get("device_id")) == actual_transport_ref + for item in self._fresh_ble_devices_locked() + ) + ): + raise RuntimeError("список найденных Bluetooth-устройств изменился") + if verification_target.transport_source == "retained-current-process" and ( + self._selected_device_id != verification_target.selected_device_id + or self._connection_mode != verification_target.selected_connection_mode + or self._device_session_id != verification_target.device_session_id + ): + raise RuntimeError("process recovery context изменился во время BLE-чтения") + current_acquisition = self._acquisition + if ( + current_acquisition is not None + and current_acquisition.state not in TERMINAL_ACQUISITION_STATES + and not ( + retained_prepared_stop_owner is not None + and current_acquisition is retained_prepared_stop_owner.acquisition + and self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ) + ): + raise RuntimeError( + "acquisition-сессия началась во время проверки существующей сети" + ) + if self._acquisition_session_lease is not None and not ( + retained_prepared_stop_owner is not None + and self._acquisition_session_lease + is retained_prepared_stop_owner.session_lease + and self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ): + raise RuntimeError( + "evidence-сессия изменилась во время проверки существующей сети" + ) + if verification_target.transport_source == "retained-current-process": + retained_after_read = connected_device_recovery_snapshot( + actual_transport_ref, + device_session_id=verification_target.device_session_id, + ) + if retained_after_read["status"] != "retained": + raise RuntimeError("process recovery token изменился после GATT validation") + defer_fresh_bridge_semantic_commit = bool( + verification_source == "fresh-scan" + and requested_mode == "bridge" + and expected_reconciliation is None + and ( + (reconfiguration_intent is None and reconfiguration_intent_id is None) + or ( + reconfiguration_intent == "select-device" + and isinstance(reconfiguration_intent_id, str) + ) + ) + ) + # Exact read-only topology evidence supersedes any previous local + # control binding. Retire that binding before projecting the new + # facade session; no device command is sent by this operation. + with self._acquisition_lifecycle_access(): + if ( + retained_stop_recovery + and retained_prepared_stop_owner is not None + and not self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ): + raise RuntimeError( + "retained PREPARED STOP owner changed before topology commit" + ) + self._retire_application_control_for_network_change() + projected_intent_id = self._apply_read_only_device_topology( + transport_ref=actual_transport_ref, + connection_mode=requested_mode, + ipv4=target, + firmware_version=compatibility_attestation.firmware_version, + observed_at_utc=observed_at, + captured_device=validated_capture, + semantic_predecessor_revision=( + verification_target.semantic_predecessor_revision + ), + compatibility_attestation=_attestation_snapshot( + compatibility_attestation + ), + force_new_intent=True, + force_new_session=True, + persist_semantic_topology=(not defer_fresh_bridge_semantic_commit), + retained_prepared_stop_owner=( + retained_prepared_stop_owner if retained_stop_recovery else None + ), + ) + if defer_fresh_bridge_semantic_commit: + with self._lock: + projected_session_id = self._device_session_id + projected_discovery_generation = self._device_session_discovery_generation + projected_generation = self._ble_discovery_generation + if ( + not isinstance(projected_session_id, str) + or projected_discovery_generation != verification_target.discovery_generation + or projected_generation != verification_target.discovery_generation + ): + raise RuntimeError( + "fresh Bridge projection lost its exact discovery/session fence" + ) + provisional_topology = _ProvisionalFreshBridgeTopology( + reconfiguration_intent_id=( + reconfiguration_intent_id + if reconfiguration_intent == "select-device" + else None + ), + reconfiguration_revision=reconfiguration_revision, + supervisor_intent_id=projected_intent_id, + discovery_generation=projected_generation, + device_session_id=projected_session_id, + transport_ref=actual_transport_ref, + connection_mode="bridge", + ipv4=target, + firmware_version=compatibility_attestation.firmware_version, + observed_at_utc=observed_at, + semantic_predecessor_revision=( + verification_target.semantic_predecessor_revision + ), + ) + # Unresolved network-write reconciliation retains the historical + # durable-first semantics. Fresh Bridge selection without such an + # edge remains provisional until exact DeviceInfo/pin admission. if reconciliation_cleared: + assert ledger_record is not None + observation_evidence = _network_status_evidence( + cast(Mapping[str, Any], observed_status) + ) + resolving_record = ledger_record + if ledger_record.stage == "dispatching": + resolving_record = self._network_mutation_ledger.mark_observing( + ledger_record.operation_id, + expected_revision=resolving_record.revision, + write_confirmed=bool(ledger_record.write_confirmed), + observation=observation_evidence, + ) + self._network_mutation_ledger.resolve( + ledger_record.operation_id, + expected_revision=resolving_record.revision, + resolution="target-observed", + observation=observation_evidence, + ) + assert current_active_idempotency is not None + idempotency_journal.complete( + current_active_idempotency.operation_id, + expected_revision=current_active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="succeeded", + outcome_code="network.provision.reconciled", + error_code=None, + side_effect_status="reconciled", + retryable=False, + safe_to_retry=False, + ), + ) + with self._lock: + self._network_write_reconciliation = None logger.info( "K1 network write ambiguity reconciled by read-only status", extra={ "event_code": "k1_network_write_reconciled", - "reason_code": "read-only-ble-status-observed", + "reason_code": "intended-topology-semantically-observed", "device_write_performed": False, "automatic_retry": False, }, ) - observed_target = status_read["status"]["ipv4"] - if observed_target is None or observed_target == AP_FALLBACK_IPV4: - raise RuntimeError("K1 не сообщил актуальный DHCP-адрес общей сети") - target = validate_private_ipv4(observed_target) - if _target_is_local_ipv4(target): - raise RuntimeError("BLE status сообщил адрес, принадлежащий этому компьютеру") - host_route_class = _host_route_class(target) + host_route_class = ( + "device-ap" + if requested_mode == "quick-connect" + else await _run_blocking_operation_without_abandonment( + _host_route_class, + target, + ) + ) if host_route_class in {"default-route", "tunnel"}: + host_path = await _run_blocking_operation_without_abandonment( + self._sample_host_path, + target, + association_timeout_seconds=( + COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS + ), + ) + self._observe_connection_transport( + target, + path=host_path, + reachable=False if host_path.available else None, + reason_code="connection_lease_host_route_mismatch", + ) + self._update_applied_topology_reachability( + connection_mode=requested_mode, + ipv4=target, + host_route_class=host_route_class, + host_route_ready=False, + endpoint_reachable=False, + write_performed=False, + ) raise RuntimeError( "K1 сообщил адрес другой сети; прямой локальный маршрут отсутствует" ) - if not _control_endpoint_reachable(target): - raise RuntimeError( - "K1 сообщил адрес общей сети, но MQTT endpoint 1883 недоступен" + control_endpoint_observation = await _run_blocking_operation_without_abandonment( + self._probe_control_endpoint, + target, + association_timeout_seconds=COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS, + ) + if not control_endpoint_observation.reachable: + self._update_applied_topology_reachability( + connection_mode=requested_mode, + ipv4=target, + host_route_class=host_route_class, + host_route_ready=True, + endpoint_reachable=False, + write_performed=False, ) + raise RuntimeError("K1 сообщил адрес общей сети, но MQTT endpoint 1883 недоступен") - # Retire only local, terminal control ownership after every external - # observation has passed. No scanner command is sent by this call. - self._application_control_session.retire_for_network_change() - observed_at = str(status_read["observed_at_utc"]) + self._update_applied_topology_reachability( + connection_mode=requested_mode, + ipv4=target, + host_route_class=host_route_class, + host_route_ready=True, + endpoint_reachable=True, + write_performed=False, + ) + # Every awaited evidence plane may span host sleep. The owned scan + # fence rejects a candidate if a new discovery generation replaced + # it while the exact live GATT evidence was being reduced. with self._lock: - if self._provisioning_active: - raise RuntimeError("настройка Wi-Fi началась во время проверки сети") - if self._ble_discovery_generation != discovery_generation or not any( - str(item.get("device_id")) == device_id for item in self._devices + if not self._provisioning_active: + raise RuntimeError("потерян fence активной сетевой операции") + if verification_target.transport_source == "fresh-scan" and ( + self._ble_discovery_generation != verification_target.discovery_generation + or not any( + str(item.get("device_id")) == actual_transport_ref + for item in self._fresh_ble_devices_locked() + ) ): raise RuntimeError("список найденных Bluetooth-устройств изменился") current_acquisition = self._acquisition if ( current_acquisition is not None and current_acquisition.state not in TERMINAL_ACQUISITION_STATES + and not ( + retained_prepared_stop_owner is not None + and current_acquisition is retained_prepared_stop_owner.acquisition + and self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ) ): raise RuntimeError( "acquisition-сессия началась во время проверки существующей сети" ) - if self._acquisition_session_lease is not None: + if self._acquisition_session_lease is not None and not ( + retained_prepared_stop_owner is not None + and self._acquisition_session_lease + is retained_prepared_stop_owner.session_lease + and self._prepared_stop_recovery_owner_allows_read_only_rebind( + retained_prepared_stop_owner, + ) + ): raise RuntimeError( "evidence-сессия изменилась во время проверки существующей сети" ) - persistent_device_id = self._device_ids_by_transport_ref.get(device_id) - if persistent_device_id is None: - persistent_device_id = new_device_id() - self._device_ids_by_transport_ref[device_id] = persistent_device_id - self._selected_device_id = device_id - self._k1_ip = target - self._connection_mode = "bridge" - self._device_id = persistent_device_id - self._device_session_id = new_device_session_id() - self._device_session_opened_at = observed_at - self._connection_lease_generation += 1 - self._compatibility_attestation = _attestation_snapshot( - compatibility_attestation - ) - self._device_calibration = unavailable_device_calibration_snapshot( - XGRIDS_K1_COMPATIBILITY_PROFILE_ID - ) + if ( + self._selected_device_id != actual_transport_ref + or self._connection_mode != requested_mode + or self._k1_ip != target + ): + raise RuntimeError( + "доказанная topology K1 потеряла facade-проекцию во время проверки" + ) self._connection_verification = { "status": "adopted", - "lease_state": "reachable", + "lease_state": "configured-unverified", "lease_generation": self._connection_lease_generation, "endpoint_validation": "ble-wifi-status-read+mqtt-tcp-connect", "network_reachability": "reachable", @@ -1604,6 +18033,11 @@ class XgridsK1CompatibilityService: "K1 найден в существующей локальной сети без изменения его настроек." ) lease_generation = self._connection_lease_generation + self._observe_connection_transport( + target, + path=control_endpoint_observation.path, + reachable=True, + ) logger.info( "K1 existing direct-LAN connection adopted without provisioning", extra={ @@ -1617,11 +18051,46 @@ class XgridsK1CompatibilityService: "automatic_retry": False, }, ) - return target + return target, provisional_topology + except BaseException: + if provisional_topology is not None: + self._reset_provisional_fresh_bridge_topology(provisional_topology) + raise finally: + with self._lock: + if verification_fence_active: + self._provisioning_active = False self._provisioning_gate.release() def _refresh_live_lan_address(self, *, rediscover: bool = False) -> str: + """Refresh from a worker while keeping CoreBluetooth on its owner loop.""" + + return self._refresh_live_lan_address_from_status( + self._read_wifi_status_on_runtime_loop, + rediscover=rediscover, + ) + + async def _refresh_live_lan_address_async(self, *, rediscover: bool = False) -> str: + async def read_status( + device_id: str, + *, + timeout_seconds: float, + rediscover: bool, + ) -> Mapping[str, Any]: + return await read_wifi_status_once( + device_id, + timeout_seconds=timeout_seconds, + rediscover=rediscover, + ) + + return await self._refresh_live_lan_address_from_status_async( + read_status, + rediscover=rediscover, + ) + + def _live_lan_address_context( + self, + ) -> tuple[str, ConnectionMode, str | None]: with self._lock: selected_device_id = self._selected_device_id connection_mode = self._connection_mode @@ -1629,56 +18098,355 @@ class XgridsK1CompatibilityService: current_target = self._k1_ip if selected_device_id is None or connection_mode is None: raise ValueError("сначала выберите и подключите K1 через BLE/Wi-Fi") + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: raise RuntimeError("нельзя менять DHCP-привязку во время активной acquisition-сессии") control_state = str(self._application_control_session.snapshot()["state"]) if control_state not in {"idle", "completed", "closed", "failed"}: raise RuntimeError("нельзя менять DHCP-привязку при открытой control-сессии") - if connection_mode == "quick-connect": - if current_target != AP_FALLBACK_IPV4: - raise RuntimeError("Quick Connect потерял фиксированный адрес точки доступа K1") - return AP_FALLBACK_IPV4 + if connection_mode == "quick-connect" and current_target != AP_FALLBACK_IPV4: + raise RuntimeError("Quick Connect потерял фиксированный адрес точки доступа K1") + return selected_device_id, connection_mode, current_target - status_read = asyncio.run( - read_wifi_status_once( + def _refresh_live_lan_address_from_status( + self, + status_reader: Callable[..., Mapping[str, Any]], + *, + rediscover: bool, + ) -> str: + selected_device_id, connection_mode, current_target = self._live_lan_address_context() + if connection_mode == "quick-connect": + return self._confirm_quick_connect_endpoint( selected_device_id, - timeout_seconds=20.0, - rediscover=rediscover, + current_target, + ) + if current_target is not None: + current_target = validate_private_ipv4(current_target) + route_class = _host_route_class(current_target) + if route_class not in {"default-route", "tunnel"}: + endpoint_observation = self._probe_control_endpoint(current_target) + if endpoint_observation.reachable: + return self._confirm_current_lan_endpoint( + selected_device_id, + connection_mode, + current_target, + endpoint_observation=endpoint_observation, + ) + + status_read = status_reader( + selected_device_id, + timeout_seconds=20.0, + rediscover=rediscover, + ) + target = _live_lan_target_from_status(status_read) + address_changed = current_target != target + self._apply_read_only_device_topology( + transport_ref=selected_device_id, + connection_mode=connection_mode, + ipv4=target, + firmware_version=self._current_firmware_version(), + observed_at_utc=str(status_read["observed_at_utc"]), + force_new_session=address_changed, + ) + host_route_class = _host_route_class(target) + endpoint_observation = ( + self._probe_control_endpoint(target) + if host_route_class not in {"default-route", "tunnel"} + else _CorrelatedEndpointObservation( + path=self._sample_host_path(target), + reachable=False, + reason_code="connection_lease_host_route_mismatch", ) ) - observed_target = status_read["status"]["ipv4"] - if observed_target is None or observed_target == AP_FALLBACK_IPV4: - raise RuntimeError("K1 не сообщил актуальный DHCP-адрес через BLE status") - target = validate_private_ipv4(observed_target) - if _target_is_local_ipv4(target): - raise RuntimeError("BLE status сообщил адрес, принадлежащий этому компьютеру") + return self._commit_live_lan_address( + selected_device_id=selected_device_id, + connection_mode=connection_mode, + current_target=current_target, + status_read=status_read, + target=target, + address_changed=address_changed, + host_route_class=host_route_class, + endpoint_observation=endpoint_observation, + ) + async def _refresh_live_lan_address_from_status_async( + self, + status_reader: Callable[..., Awaitable[Mapping[str, Any]]], + *, + rediscover: bool, + ) -> str: + selected_device_id, connection_mode, current_target = self._live_lan_address_context() + if connection_mode == "quick-connect": + return await _run_blocking_operation_without_abandonment( + self._confirm_quick_connect_endpoint, + selected_device_id, + current_target, + ) + if current_target is not None: + current_target = validate_private_ipv4(current_target) + route_class = await _run_blocking_operation_without_abandonment( + _host_route_class, + current_target, + ) + if route_class not in {"default-route", "tunnel"}: + endpoint_observation = await _run_blocking_operation_without_abandonment( + self._probe_control_endpoint, + current_target, + ) + if endpoint_observation.reachable: + return self._confirm_current_lan_endpoint( + selected_device_id, + connection_mode, + current_target, + endpoint_observation=endpoint_observation, + ) + + status_read = await status_reader( + selected_device_id, + timeout_seconds=20.0, + rediscover=rediscover, + ) + target = _live_lan_target_from_status(status_read) + address_changed = current_target != target + self._apply_read_only_device_topology( + transport_ref=selected_device_id, + connection_mode=connection_mode, + ipv4=target, + firmware_version=self._current_firmware_version(), + observed_at_utc=str(status_read["observed_at_utc"]), + force_new_session=address_changed, + ) + host_route_class = await _run_blocking_operation_without_abandonment( + _host_route_class, + target, + ) + endpoint_observation = ( + await _run_blocking_operation_without_abandonment( + self._probe_control_endpoint, + target, + ) + if host_route_class not in {"default-route", "tunnel"} + else _CorrelatedEndpointObservation( + path=await _run_blocking_operation_without_abandonment( + self._sample_host_path, + target, + ), + reachable=False, + reason_code="connection_lease_host_route_mismatch", + ) + ) + return self._commit_live_lan_address( + selected_device_id=selected_device_id, + connection_mode=connection_mode, + current_target=current_target, + status_read=status_read, + target=target, + address_changed=address_changed, + host_route_class=host_route_class, + endpoint_observation=endpoint_observation, + ) + + def _confirm_quick_connect_endpoint( + self, + selected_device_id: str, + current_target: str | None, + ) -> str: + """Confirm the fixed AP route without repeating the AP-enable command.""" + + if current_target != AP_FALLBACK_IPV4: + raise ConnectionLeaseUnavailable( + "Quick Connect потерял фиксированный адрес точки доступа K1", + reason_code="quick_connect_endpoint_unreachable", + ) + self._require_supervised_device_network( + transport_ref=selected_device_id, + connection_mode="quick-connect", + ipv4=AP_FALLBACK_IPV4, + ) + endpoint_observation = self._probe_control_endpoint(AP_FALLBACK_IPV4) + if not endpoint_observation.reachable: + with self._lock: + lease_generation = self._connection_lease_generation + self._mark_connection_lease_unreachable( + reason_code="quick_connect_endpoint_unreachable", + lease_generation=lease_generation, + ) + raise ConnectionLeaseUnavailable( + "управляющий endpoint точки доступа K1 недоступен", + reason_code="quick_connect_endpoint_unreachable", + ) + + observed_at = _utc_now_iso() + with self._lock: + if ( + self._selected_device_id != selected_device_id + or self._connection_mode != "quick-connect" + or self._k1_ip != AP_FALLBACK_IPV4 + ): + raise ConnectionLeaseUnavailable( + "Quick Connect изменился во время read-only проверки", + reason_code="connection_lease_changed_during_probe", + ) + self._connection_verification = { + "status": "tcp-reachable-device-info-unverified", + "lease_state": "configured-unverified", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": "mqtt-tcp-connect", + "network_reachability": "reachable", + "address_changed": False, + "write_performed": False, + "observed_at": observed_at, + } + self._operation_message = ( + "Точка доступа K1 и управляющий канал подтверждены без новой BLE-записи." + ) + return AP_FALLBACK_IPV4 + + def _confirm_current_lan_endpoint( + self, + selected_device_id: str, + connection_mode: ConnectionMode, + current_target: str, + *, + endpoint_observation: _CorrelatedEndpointObservation, + ) -> str: + """Promote an existing Bridge/direct endpoint without requiring BLE.""" + + self._require_supervised_device_network( + transport_ref=selected_device_id, + connection_mode=connection_mode, + ipv4=current_target, + ) + if not endpoint_observation.reachable: + raise ConnectionLeaseUnavailable( + "управляющий endpoint K1 изменился во время проверки", + reason_code=endpoint_observation.reason_code + or "connection_lease_endpoint_unreachable", + ) + observed_at = _utc_now_iso() + with self._lock: + if ( + self._selected_device_id != selected_device_id + or self._connection_mode != connection_mode + or self._k1_ip != current_target + ): + raise ConnectionLeaseUnavailable( + "подключение K1 изменилось во время read-only endpoint-проверки", + reason_code="connection_lease_changed_during_probe", + ) + lease_generation = self._connection_lease_generation + self._connection_verification = { + "status": "tcp-reachable-device-info-unverified", + "lease_state": "configured-unverified", + "lease_generation": lease_generation, + "endpoint_validation": "mqtt-tcp-connect", + "network_reachability": "reachable", + "address_changed": False, + "write_performed": False, + "observed_at": observed_at, + } + self._operation_message = "Управляющий канал K1 подтверждён без новой BLE-записи." + logger.info( + "K1 current control endpoint confirmed without BLE recovery", + extra={ + "event_code": "k1_connection_endpoint_verified", + "lease_generation": lease_generation, + "lease_state": "reachable", + "endpoint_reachable": True, + "address_changed": False, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + return current_target + + def _commit_live_lan_address( + self, + *, + selected_device_id: str, + connection_mode: ConnectionMode, + current_target: str | None, + status_read: Mapping[str, Any], + target: str, + address_changed: bool, + host_route_class: str, + endpoint_observation: _CorrelatedEndpointObservation, + ) -> str: + endpoint_reachable = endpoint_observation.reachable with self._lock: if ( self._selected_device_id != selected_device_id or self._connection_mode != connection_mode ): raise RuntimeError("выбранное подключение K1 изменилось во время DHCP refresh") - address_changed = self._k1_ip != target - self._k1_ip = target - if address_changed: - self._device_session_id = new_device_session_id() - self._device_session_opened_at = _utc_now_iso() - self._connection_lease_generation += 1 - self._device_calibration = unavailable_device_calibration_snapshot( - XGRIDS_K1_COMPATIBILITY_PROFILE_ID + if self._k1_ip != target: + raise RuntimeError( + "доказанная topology K1 потеряла facade-проекцию до проверки маршрута" ) + route_ready = host_route_class not in {"default-route", "tunnel"} + verification_status = ( + "tcp-reachable-device-info-unverified" + if endpoint_reachable + else "endpoint-unreachable" + if route_ready + else "host-route-mismatch" + ) self._connection_verification = { - "status": "live-address-observed", - "lease_state": "configured", + "status": verification_status, + "lease_state": "configured-unverified", "lease_generation": self._connection_lease_generation, - "endpoint_validation": "ble-wifi-status-read", - "network_reachability": "not-probed", + "endpoint_validation": ( + "ble-wifi-status-read+mqtt-tcp-connect" if route_ready else "host-route" + ), + "network_reachability": ("reachable" if endpoint_reachable else "unreachable"), + **( + {} + if endpoint_reachable + else { + "reason_code": ( + "connection_lease_endpoint_unreachable" + if route_ready + else "connection_lease_host_route_mismatch" + ), + "host_route_class": host_route_class, + } + ), "address_changed": address_changed, "previous_address_present": current_target is not None, "write_performed": False, "observed_at": status_read["observed_at_utc"], } + self._observe_connection_transport( + target, + path=endpoint_observation.path, + reachable=(endpoint_reachable if endpoint_observation.path.available else None), + reason_code=endpoint_observation.reason_code, + ) + if host_route_class in {"default-route", "tunnel"}: + raise ConnectionLeaseUnavailable( + "K1 сообщил адрес другой сети; прямой локальный маршрут отсутствует", + reason_code="connection_lease_host_route_mismatch", + ) + if not endpoint_reachable: + raise ConnectionLeaseUnavailable( + "K1 сообщил адрес общей сети, но MQTT endpoint 1883 недоступен", + reason_code="connection_lease_endpoint_unreachable", + ) + logger.info( + "K1 control endpoint confirmed after read-only DHCP refresh", + extra={ + "event_code": "k1_connection_endpoint_verified", + "lease_generation": self._connection_lease_generation, + "lease_state": "reachable", + "endpoint_reachable": True, + "address_changed": address_changed, + "device_write_performed": False, + "automatic_retry": False, + }, + ) return target def _reuse_or_recover_control_target(self) -> tuple[str, dict[str, Any]]: @@ -1700,9 +18468,20 @@ class XgridsK1CompatibilityService: "сначала настройте сетевое подключение K1", reason_code="connection_lease_missing", ) + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) current_target = validate_private_ipv4(current_target) - if _control_endpoint_reachable(current_target): + self._require_supervised_device_network( + transport_ref=selected_device_id, + connection_mode=connection_mode, + ipv4=current_target, + ) + + current_endpoint_observation = self._probe_control_endpoint(current_target) + if current_endpoint_observation.reachable: observed_at = _utc_now_iso() with self._lock: if ( @@ -1716,8 +18495,8 @@ class XgridsK1CompatibilityService: reason_code="connection_lease_changed_during_probe", ) self._connection_verification = { - "status": "reachable", - "lease_state": "reachable", + "status": "tcp-reachable-device-info-unverified", + "lease_state": "configured-unverified", "lease_generation": lease_generation, "endpoint_validation": "mqtt-tcp-connect", "network_reachability": "reachable", @@ -1757,30 +18536,17 @@ class XgridsK1CompatibilityService: reason_code="quick_connect_endpoint_unreachable", ) - host_route_class = _host_route_class(current_target) - if host_route_class in {"default-route", "tunnel"}: - self._mark_connection_lease_unreachable( - reason_code="connection_lease_host_route_mismatch", - lease_generation=lease_generation, - endpoint_validation="host-route", - host_route_class=host_route_class, - ) - raise ConnectionLeaseUnavailable( - "Сканер подключён к другой локальной сети: его адрес получен, " - "но этот компьютер не имеет прямого маршрута к нему. " - "Подключите компьютер к той же сети, что и сканер. " - "Команды запуска и остановки не отправлялись.", - reason_code="connection_lease_host_route_mismatch", - ) - recovered_route_class = "unknown" + recovered_endpoint_observation: _CorrelatedEndpointObservation | None = None + recovered_target: str | None = None + recovered_topology_applied = False + address_changed = False + recovery_observed_at: str | None = None try: - status_read = asyncio.run( - read_wifi_status_once( - selected_device_id, - timeout_seconds=20.0, - rediscover=True, - ) + status_read = self._read_wifi_status_on_runtime_loop( + selected_device_id, + timeout_seconds=20.0, + rediscover=True, ) observed_target = status_read["status"]["ipv4"] if observed_target is None or observed_target == AP_FALLBACK_IPV4: @@ -1794,6 +18560,32 @@ class XgridsK1CompatibilityService: "K1 сообщил адрес этого компьютера вместо адреса устройства", reason_code="connection_lease_local_address_conflict", ) + recovery_observed_at = str(status_read["observed_at_utc"]) + address_changed = recovered_target != current_target + with self._lock: + if ( + self._selected_device_id != selected_device_id + or self._connection_mode != connection_mode + or self._device_session_id != device_session_id + ): + raise ConnectionLeaseUnavailable( + "подключение K1 изменилось во время восстановления", + reason_code="connection_lease_changed_during_recovery", + ) + # The exact K1 status is a durable topology fact even when the Mac + # has already left that network or the MQTT listener is down. + self._apply_read_only_device_topology( + transport_ref=selected_device_id, + connection_mode=connection_mode, + ipv4=recovered_target, + firmware_version=self._current_firmware_version(), + observed_at_utc=recovery_observed_at, + force_new_intent=True, + # Entering BLE recovery proves continuity was lost. Rotate the + # browser/device session even if DHCP returned the same IPv4. + force_new_session=True, + ) + recovered_topology_applied = True recovered_route_class = _host_route_class(recovered_target) if recovered_route_class in {"default-route", "tunnel"}: raise ConnectionLeaseUnavailable( @@ -1803,7 +18595,8 @@ class XgridsK1CompatibilityService: "Команды запуска и остановки не отправлялись.", reason_code="connection_lease_host_route_mismatch", ) - if not _control_endpoint_reachable(recovered_target): + recovered_endpoint_observation = self._probe_control_endpoint(recovered_target) + if not recovered_endpoint_observation.reachable: raise ConnectionLeaseUnavailable( "Сканер сообщил адрес общей сети, но управляющее соединение " "с этого компьютера не открывается. Убедитесь, что оба " @@ -1814,20 +18607,37 @@ class XgridsK1CompatibilityService: reason_code = getattr(exc, "reason_code", None) if not isinstance(reason_code, str) or not reason_code: reason_code = "connection_lease_ble_recovery_failed" - self._mark_connection_lease_unreachable( - reason_code=reason_code, - lease_generation=lease_generation, - endpoint_validation=( - "host-route" - if reason_code == "connection_lease_host_route_mismatch" - else "mqtt-tcp-connect" - ), - host_route_class=( - recovered_route_class - if reason_code == "connection_lease_host_route_mismatch" - else None - ), - ) + if recovered_topology_applied and recovered_target is not None: + route_ready = recovered_route_class not in { + "default-route", + "tunnel", + "unknown", + } + self._update_applied_topology_reachability( + connection_mode=connection_mode, + ipv4=recovered_target, + host_route_class=recovered_route_class, + host_route_ready=route_ready, + endpoint_reachable=False, + write_performed=False, + ) + with self._lock: + lease_generation = self._connection_lease_generation + else: + self._mark_connection_lease_unreachable( + reason_code=reason_code, + lease_generation=lease_generation, + endpoint_validation=( + "host-route" + if reason_code == "connection_lease_host_route_mismatch" + else "mqtt-tcp-connect" + ), + host_route_class=( + recovered_route_class + if reason_code == "connection_lease_host_route_mismatch" + else None + ), + ) logger.error( "K1 process-owned connection lease recovery failed", extra={ @@ -1853,40 +18663,35 @@ class XgridsK1CompatibilityService: reason_code=reason_code, ) from exc - address_changed = recovered_target != current_target - observed_at = str(status_read["observed_at_utc"]) + assert recovered_target is not None + assert recovery_observed_at is not None + assert recovered_endpoint_observation is not None with self._lock: if ( self._selected_device_id != selected_device_id or self._connection_mode != connection_mode - or self._device_session_id != device_session_id + or self._k1_ip != recovered_target ): raise ConnectionLeaseUnavailable( "подключение K1 изменилось во время восстановления", reason_code="connection_lease_changed_during_recovery", ) - self._k1_ip = recovered_target - # Any recovery proves that continuity of the previous route was - # lost, even when DHCP returns the same address. Rotate the - # device-session generation so browser/viewer state from before - # the gap cannot be merged with the recovered connection. - self._device_session_id = new_device_session_id() - self._device_session_opened_at = observed_at - self._connection_lease_generation += 1 - self._device_calibration = unavailable_device_calibration_snapshot( - XGRIDS_K1_COMPATIBILITY_PROFILE_ID - ) lease_generation = self._connection_lease_generation self._connection_verification = { "status": "recovered", - "lease_state": "reachable", + "lease_state": "configured-unverified", "lease_generation": lease_generation, "endpoint_validation": "ble-rediscovery+mqtt-tcp-connect", "network_reachability": "reachable", "address_changed": address_changed, "write_performed": False, - "observed_at": observed_at, + "observed_at": recovery_observed_at, } + self._observe_connection_transport( + recovered_target, + path=recovered_endpoint_observation.path, + reachable=True, + ) logger.info( "K1 process-owned connection lease recovered", extra={ @@ -1929,17 +18734,41 @@ class XgridsK1CompatibilityService: "observed_at": _utc_now_iso(), } + @_serialized_k1_transition_access def read_device_calibration_snapshot(self) -> dict[str, Any]: """Read and seal the two reviewed factory YAML files without device mutation.""" if not self._calibration_gate.acquire(blocking=False): raise RuntimeError("чтение заводской калибровки уже выполняется") + process_lease_acquired = False try: - self._refresh_live_lan_address() + with self._lock: + selected_device_id = self._selected_device_id + if selected_device_id is None: + raise ValueError("сначала выберите и подключите K1 через BLE/Wi-Fi") + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + self._acquire_k1_lifecycle_process_lease("network") + process_lease_acquired = True + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + with self._borrow_network_ble_process_lease(): + self._refresh_live_lan_address() with self._lock: target = self._k1_ip connection_mode = self._connection_mode attestation = self._compatibility_attestation + current_selected_device_id = self._selected_device_id + if current_selected_device_id != selected_device_id: + raise RuntimeError("выбранный K1 изменился во время подготовки чтения калибровки") + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) if target is None or connection_mode is None: raise ValueError("у плагина нет подтверждённого локального адреса K1") if attestation is None: @@ -1951,9 +18780,14 @@ class XgridsK1CompatibilityService: evidence_root=self.evidence_root, allow_device_ap=connection_mode == "quick-connect", ) + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) with self._lock: if ( - self._k1_ip != target + self._selected_device_id != selected_device_id + or self._k1_ip != target or self._connection_mode != connection_mode or self._compatibility_attestation != attestation ): @@ -1964,15 +18798,22 @@ class XgridsK1CompatibilityService: self._device_calibration = dict(snapshot) return dict(snapshot) finally: + if process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() self._calibration_gate.release() @_serialized_acquisition_access + @_serialized_k1_transition_access def open_application_control_session( self, request: OpenApplicationControlSessionRequest, ) -> dict[str, Any]: """Open one continuous canonical dialogue; no later stage is implied.""" + process_lease_acquired = False + preflight_network_process_lease_acquired = False + preflight_ble_borrow_used = False + with self._lock: if self._provisioning_active: raise RuntimeError("нельзя открывать control-сессию во время настройки Wi-Fi") @@ -1980,29 +18821,134 @@ class XgridsK1CompatibilityService: device_session_id = self._device_session_id attestation = self._compatibility_attestation acquisition = self._acquisition - operation, _ = self._operations.begin( - ACTION_APPLICATION_CONTROL_SESSION_OPEN, - device_id=device_id, - device_session_id=device_session_id, - deadline_seconds=30.0, - ) - self._operations.transition( - operation.operation_id, - "running", - stage_code="connection-lease-check", - message_code="application-control.session.open.checking_connection_lease", - ) + selected_device_id = self._selected_device_id + if selected_device_id is not None: + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + try: + # Target recovery may perform TCP and a read-only BLE status read. + # Win the shared process fence first, then recheck retirement so a + # stale backend cannot contact a target retired by another one. + self._acquire_k1_lifecycle_process_lease("network") + preflight_network_process_lease_acquired = True + if selected_device_id is not None: + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + operation, _ = self._operations.begin( + ACTION_APPLICATION_CONTROL_SESSION_OPEN, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=30.0, + ) + self._operations.transition( + operation.operation_id, + "running", + stage_code="connection-lease-check", + message_code="application-control.session.open.checking_connection_lease", + ) + except BaseException: + if preflight_network_process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + raise try: if attestation is None: raise RuntimeError("сначала подключите K1 и выберите exact-profile") if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: raise RuntimeError("control-сессия должна быть открыта до подготовки acquisition") - target, lease_result = self._reuse_or_recover_control_target() + existing_control = self._application_control_session.snapshot() + if existing_control.get("state") in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + # An already-open owner performs its own retained-control and + # exact-route validation. Refresh that evidence before the + # supervisor's silence TTL is reduced; a true binding loss is + # propagated fail-closed and never falls through into a + # duplicate worker open. + self._application_control_session.validate_connection_binding() + existing_control = self._application_control_session.snapshot() + self._reconcile_connection_supervisor( + existing_control, + self.runtime.snapshot(), + ) + existing_supervisor = self._connection_supervisor.snapshot() + if existing_supervisor.authority.control_allowed: + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="control-already-ready", + message_code="application-control.session.open.already_ready", + result={ + "connection_lease_reused": True, + "device_info_reused": True, + "device_write_performed": False, + }, + ) + self._release_k1_lifecycle_process_lease("network") + preflight_network_process_lease_acquired = False + return self.state() + with self._borrow_network_ble_process_lease(): + preflight_ble_borrow_used = True + target, lease_result = self._reuse_or_recover_control_target() + supervisor = self._connection_supervisor.snapshot() + endpoint_target = EndpointTarget( + validate_private_ipv4(target), + CONTROL_MQTT_PORT, + ) + if ( + supervisor.intent is None + or supervisor.device_network.state != "applied" + or supervisor.device_network.transport_ref is None + or supervisor.device_network.intent_id != supervisor.intent.intent_id + or supervisor.device_network.connection_mode != supervisor.intent.requested_mode + or supervisor.device_network.target != endpoint_target + or supervisor.endpoint.target != endpoint_target + or supervisor.endpoint.tcp_state != "reachable" + or supervisor.endpoint.host_path_epoch != supervisor.host_path.epoch + or supervisor.host_path.epoch < 1 + ): + raise RuntimeError( + "маршрут K1 не имеет свежего согласованного device/host/TCP доказательства" + ) + self._require_transport_not_physically_retired( + supervisor.device_network.transport_ref, + verification=True, + ) self._application_control.disarm() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) + self._acquire_application_control_process_lease() + process_lease_acquired = True + # Keep the shared holder until a borrowed CoreBluetooth operation + # has reached proven idle. The persistent control holder already + # owns the same flock, but retaining the network label as well + # protects the error path where opening the worker later fails and + # the control label is reconciled away. + self._release_network_process_lease_after_ble_cleanup() + preflight_network_process_lease_acquired = False + connection_binding = ApplicationConnectionBinding( + intent_id=supervisor.intent.intent_id, + transport_ref=supervisor.device_network.transport_ref, + host_path_epoch=supervisor.host_path.epoch, + target_ipv4=endpoint_target.ipv4, + target_port=endpoint_target.port, + connection_mode=supervisor.intent.requested_mode, + ) + # The earlier recovery probe may have raced a Wi-Fi/VPN change. + # Re-sample immediately before the worker becomes eligible to + # publish any bootstrap envelope. + self._validate_application_connection_path(connection_binding) self._application_control_session.open( host=target, timezone_name=request.timezone_name, confirmation=request.confirmation(), + connection_binding=connection_binding, ) self._operations.transition( operation.operation_id, @@ -2014,6 +18960,14 @@ class XgridsK1CompatibilityService: except Exception as exc: control_snapshot = self._application_control_session.snapshot() safe_to_retry = control_snapshot.get("can_open") is True + if process_lease_acquired: + self._reconcile_application_control_process_lease(control_snapshot) + if preflight_network_process_lease_acquired: + if preflight_ble_borrow_used: + self._release_network_process_lease_after_ble_cleanup() + else: + self._release_k1_lifecycle_process_lease("network") + preflight_network_process_lease_acquired = False self._operations.transition_if_pending( operation.operation_id, "failed", @@ -2043,20 +18997,354 @@ class XgridsK1CompatibilityService: }, ) raise + if preflight_network_process_lease_acquired: + if preflight_ble_borrow_used: + self._release_network_process_lease_after_ble_cleanup() + else: + self._release_k1_lifecycle_process_lease("network") return self.state() @_serialized_acquisition_access + @_serialized_k1_transition_access def enter_application_workspace( self, request: EnterApplicationWorkspaceRequest, ) -> dict[str, Any]: - del request - self._application_control_session.enter_workspace() + try: + # ``enter_workspace`` validates the retained MQTT proof and then + # re-samples the exact host path before releasing its checkpoint. + # Do not snapshot/reconcile the TTL-bound supervisor first: an + # unchanged route whose fallback observation has just crossed its + # lease boundary must be allowed to refresh in that command-bound + # validator before stale reduction can rotate its epoch. + self._application_control_session.enter_workspace( + expected_session_generation=request.expected_session_generation, + expected_state_revision=request.expected_state_revision, + ) + except ApplicationConnectionBindingLost: + self._application_control_session.close_prestart() + raise return self.state() @_serialized_acquisition_access - def close_application_control_session(self) -> dict[str, Any]: - self._application_control_session.close_prestart() + @_serialized_k1_transition_access + def close_application_control_session( + self, + request: CloseApplicationControlSessionRequest, + ) -> dict[str, Any]: + control_snapshot = self._application_control_session.snapshot() + if control_snapshot.get("state") == "failed": + # The refresh/reset control is local-only. It closes no physical + # edge and never clears the durable ambiguity left by START/STOP. + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + expected_session_generation=request.expected_session_generation, + expected_state_revision=request.expected_state_revision, + ) + else: + self._application_control_session.close_prestart( + expected_session_generation=request.expected_session_generation, + expected_state_revision=request.expected_state_revision, + ) + return self.state() + + @_serialized_acquisition_access + def reconcile_physical_command( + self, + request: ReconcilePhysicalCommandRequest, + ) -> dict[str, Any]: + """Resolve the durable ambiguity from the current read-only dialogue.""" + + control_snapshot = self._application_control_session.snapshot() + if ( + control_snapshot.get("session_generation") != request.expected_session_generation + or control_snapshot.get("state_revision") != request.expected_state_revision + ): + raise RuntimeError("control session checkpoint changed before reconciliation") + if control_snapshot.get("state") != "connection-ready": + raise RuntimeError( + "physical reconciliation requires a pre-START connection-ready session" + ) + physical_snapshot = control_snapshot.get("physical_command") + if ( + not isinstance(physical_snapshot, Mapping) + or physical_snapshot.get("reconciliation_ready") is not True + ): + raise RuntimeError( + "fresh DeviceInfo and non-retained DeviceStatus proofs are not ready" + ) + # Re-check route, socket generation and fresh remote proof immediately + # before committing the local ledger transition. This performs no K1 + # write and cannot replay the ambiguous physical command. + self._application_control_session.validate_connection_binding() + self._physical_command_coordinator.reconcile_unresolved( + reconciliation_id=request.reconciliation_id + ) + return self.state() + + @staticmethod + def _physical_retirement_matches_request( + physical: Mapping[str, object], + request: RetireUnavailablePhysicalCommandRequest, + ) -> bool: + record = physical.get("record") + if not isinstance(record, Mapping): + return False + retirements = record.get("operator_retirements") + if not isinstance(retirements, list): + return False + reopens = record.get("operator_reconciliation_reopens") + reopened_retirement_ids = { + str(item.get("retirement_id")) + for item in (reopens if isinstance(reopens, list) else []) + if isinstance(item, Mapping) and isinstance(item.get("retirement_id"), str) + } + return any( + isinstance(item, Mapping) + and item.get("retirement_id") not in reopened_retirement_ids + and item.get("retirement_id") == request.retirement_id + and item.get("reason") == request.reason + and item.get("retired_transport_ref") == request.expected_transport_ref + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") == request.expected_operation_id + and item["original_attempt"].get("revision") == request.expected_revision + for item in retirements + ) + + @staticmethod + def _physical_retirement_projection_matches_request( + projection: Mapping[str, object], + request: RetireUnavailablePhysicalCommandRequest, + ) -> bool: + return ( + projection.get("expected_operation_id") == request.expected_operation_id + and projection.get("expected_revision") == request.expected_revision + and projection.get("expected_transport_ref") == request.expected_transport_ref + ) + + @_serialized_acquisition_access + @_serialized_k1_transition_access + def retire_unavailable_physical_command( + self, + request: RetireUnavailablePhysicalCommandRequest, + ) -> dict[str, Any]: + """Release one unreachable target locally, preserving unknown outcome.""" + + initial_physical = self._physical_command_coordinator.snapshot() + exact_retry = self._physical_retirement_matches_request( + initial_physical, + request, + ) + if exact_retry: + self._record_physical_retirement_reopen_generation_floor(request.retirement_id) + self._revoke_retired_physical_target_locally( + request.expected_transport_ref, + retirement_id=request.retirement_id, + ) + return self.state() + if not exact_retry: + initial = self._physical_operator_retirement_projection() + if initial.get("allowed") is not True: + initial_reason_codes = initial.get("reason_codes") + raise NetworkProvisioningConflict( + "Исходный K1 нельзя вывести из контура до завершения локальных операций", + reason_code=( + next( + iter(cast(list[str], initial_reason_codes)), + "physical-command-retirement-state-unsafe", + ) + if isinstance(initial_reason_codes, list) + else "physical-command-retirement-state-unsafe" + ), + ) + if not self._physical_retirement_projection_matches_request( + initial, + request, + ): + raise NetworkProvisioningConflict( + "Физическое состояние K1 изменилось после показа подтверждения", + reason_code="physical-command-retirement-stale-checkpoint", + ) + + process_lease_acquired = False + try: + self._acquire_k1_lifecycle_process_lease("network") + process_lease_acquired = True + current_physical = self._physical_command_coordinator.snapshot() + if not self._physical_retirement_matches_request(current_physical, request): + current = self._physical_operator_retirement_projection( + allow_owned_network_holder=True, + ) + if current.get("allowed") is not True: + current_reason_codes = current.get("reason_codes") + raise NetworkProvisioningConflict( + "Состояние K1 изменилось до подтверждения вывода из контура", + reason_code=( + next( + iter(cast(list[str], current_reason_codes)), + "physical-command-retirement-state-unsafe", + ) + if isinstance(current_reason_codes, list) + else "physical-command-retirement-state-unsafe" + ), + ) + if not self._physical_retirement_projection_matches_request( + current, + request, + ): + raise NetworkProvisioningConflict( + "Состояние K1 изменилось до подтверждения вывода из контура", + reason_code="physical-command-retirement-stale-checkpoint", + ) + try: + self._physical_command_coordinator.retire_unavailable_target( + retirement_id=request.retirement_id, + expected_operation_id=request.expected_operation_id, + expected_revision=request.expected_revision, + expected_transport_ref=request.expected_transport_ref, + reason=request.reason, + ) + except PhysicalCommandTransitionError as exc: + # A different process may commit another exact ledger edge + # after the projection above. That is an expected stale + # browser checkpoint, not a plugin/runtime gateway defect. + raise NetworkProvisioningConflict( + "Состояние K1 изменилось до подтверждения вывода из контура", + reason_code="physical-command-retirement-stale-checkpoint", + ) from exc + self._record_physical_retirement_reopen_generation_floor(request.retirement_id) + self._revoke_retired_physical_target_locally( + request.expected_transport_ref, + retirement_id=request.retirement_id, + ) + finally: + if process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + return self.state() + + @staticmethod + def _physical_reconciliation_reopen_matches_request( + physical: Mapping[str, object], + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> bool: + record = physical.get("record") + if not isinstance(record, Mapping): + return False + reopens = record.get("operator_reconciliation_reopens") + if not isinstance(reopens, list): + return False + return any( + isinstance(item, Mapping) + and item.get("reopening_id") == request.reopening_id + and item.get("retirement_id") == request.expected_retirement_id + and item.get("retired_record_revision") == request.expected_revision + and item.get("reopened_transport_ref") == request.expected_transport_ref + and item.get("discovery_generation") == request.expected_discovery_generation + and item.get("reason") == request.reason + for item in reopens + ) + + @staticmethod + def _physical_reconciliation_reopen_projection_matches_request( + projection: Mapping[str, object], + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> bool: + expected = { + "expected_revision": request.expected_revision, + "expected_retirement_id": request.expected_retirement_id, + "expected_transport_ref": request.expected_transport_ref, + "expected_discovery_generation": request.expected_discovery_generation, + "expected_desired_mode": request.expected_desired_mode, + "expected_desired_mode_revision": request.expected_desired_mode_revision, + } + return all(projection.get(key) == value for key, value in expected.items()) + + @_serialized_acquisition_access + @_serialized_k1_transition_access + def reopen_retired_physical_command_reconciliation( + self, + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> dict[str, Any]: + """Locally re-admit one fresh retired UUID for later explicit Verify.""" + + initial_physical = self._physical_command_coordinator.snapshot() + if self._physical_reconciliation_reopen_matches_request( + initial_physical, + request, + ): + return self.state() + initial = self._physical_operator_reconciliation_reopen_projection() + if initial.get("allowed") is not True: + reason_codes = initial.get("reason_codes") + raise NetworkProvisioningConflict( + "Этот K1 нельзя вернуть для проверки в текущем состоянии", + reason_code=( + next( + iter(cast(list[str], reason_codes)), + "physical-command-reconciliation-reopen-not-required", + ) + if isinstance(reason_codes, list) + else "physical-command-reconciliation-reopen-not-required" + ), + ) + if not self._physical_reconciliation_reopen_projection_matches_request( + initial, + request, + ): + raise NetworkProvisioningConflict( + "Свежий Bluetooth-кандидат изменился до возврата K1 для проверки", + reason_code=("physical-command-reconciliation-reopen-stale-checkpoint"), + ) + + process_lease_acquired = False + try: + self._acquire_k1_lifecycle_process_lease("network") + process_lease_acquired = True + current_physical = self._physical_command_coordinator.snapshot() + if self._physical_reconciliation_reopen_matches_request( + current_physical, + request, + ): + pass + else: + current = self._physical_operator_reconciliation_reopen_projection( + allow_owned_network_holder=True, + ) + if current.get("allowed") is not True or not ( + self._physical_reconciliation_reopen_projection_matches_request( + current, + request, + ) + ): + reason_codes = current.get("reason_codes") + raise NetworkProvisioningConflict( + "Свежий Bluetooth-кандидат изменился до возврата K1 для проверки", + reason_code=( + next( + iter(cast(list[str], reason_codes)), + "physical-command-reconciliation-reopen-stale-checkpoint", + ) + if isinstance(reason_codes, list) + else "physical-command-reconciliation-reopen-stale-checkpoint" + ), + ) + try: + self._physical_command_coordinator.reopen_retired_reconciliation( + reopening_id=request.reopening_id, + expected_revision=request.expected_revision, + expected_retirement_id=request.expected_retirement_id, + expected_transport_ref=request.expected_transport_ref, + expected_discovery_generation=(request.expected_discovery_generation), + reason=request.reason, + ) + except PhysicalCommandTransitionError as exc: + raise NetworkProvisioningConflict( + "Свежий Bluetooth-кандидат изменился до возврата K1 для проверки", + reason_code=("physical-command-reconciliation-reopen-stale-checkpoint"), + ) from exc + finally: + if process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") return self.state() @_serialized_acquisition_access @@ -2094,9 +19382,76 @@ class XgridsK1CompatibilityService: return self.state() @_serialized_acquisition_access + @_serialized_k1_transition_access def prepare_acquisition(self, request: PrepareAcquisitionRequest) -> dict[str, Any]: + """Fence PREPARE against cross-process retirement and legacy host bypass.""" + + # Preserve the established local lifecycle errors before contending + # for the cross-process K1 fence. A retained evidence lease or an + # already-active acquisition is actionable local state; reporting a + # peer's network holder instead would hide the cleanup the operator + # actually has to finish. The same checks are repeated in the owned + # implementation before publishing the prepared record. + with self._lock: + if self._acquisition_session_lease is not None: + raise RuntimeError("предыдущая evidence-сессия ещё не остановлена и не запечатана") + current_acquisition = self._acquisition + if ( + current_acquisition is not None + and current_acquisition.state not in TERMINAL_ACQUISITION_STATES + ): + raise RuntimeError( + "существующая acquisition-сессия ещё не завершена; сначала остановите её" + ) + if self.runtime.snapshot().get("source_mode") != "idle": + raise RuntimeError( + "нельзя готовить acquisition во время активного live/replay источника; " + "сначала остановите его" + ) + + process_lease_acquired = False + try: + self._acquire_k1_lifecycle_process_lease("network") + process_lease_acquired = True + retired_transport_refs = self._retired_physical_transport_refs() + with self._lock: + selected_device_id = self._selected_device_id + current_target = self._k1_ip + if retired_transport_refs: + if selected_device_id is None: + raise NetworkProvisioningConflict( + "После вывода исходного K1 выберите другое устройство до запуска", + reason_code="physical-command-replacement-device-required", + ) + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + if request.host is not None and ( + current_target is None + or validate_private_ipv4(request.host) != validate_private_ipv4(current_target) + ): + raise NetworkProvisioningConflict( + "Явный адрес не совпадает с выбранным новым устройством", + reason_code="physical-command-explicit-host-bypass-blocked", + ) + with self._borrow_network_ble_process_lease(): + return self._prepare_acquisition_owned(request) + finally: + if process_lease_acquired: + self._release_network_process_lease_after_ble_cleanup() + + def _prepare_acquisition_owned( + self, + request: PrepareAcquisitionRequest, + ) -> dict[str, Any]: + # This exact check is inside the shared network/control transition + # fence. Mode Select either commits before this point and PREPARE + # fails, or PREPARE keeps its active topology stable until its local + # record and project checkpoint are fully published. self._application_control.disarm() - control_state = str(self._application_control_session.snapshot()["state"]) + control_snapshot = self._application_control_session.snapshot() + control_state = str(control_snapshot["state"]) if control_state == "failed": raise RuntimeError("control-сессия K1 завершилась ошибкой и требует ручной проверки") if ( @@ -2120,6 +19475,34 @@ class XgridsK1CompatibilityService: control_mode: Literal["operator-manual", "plugin-commanded"] = ( "plugin-commanded" if control_state == "workspace-ready" else "operator-manual" ) + if control_mode == "plugin-commanded": + if ( + request.expected_control_session_generation is None + or request.expected_control_state_revision is None + ): + raise ValueError( + "plugin-commanded PREPARE requires the exact control-session checkpoint" + ) + try: + self._application_control_session.validate_connection_binding() + except ApplicationConnectionBindingLost: + self._application_control_session.close_prestart() + raise + # The command-bound validator above refreshes an unchanged route + # before supervisor TTL reduction. Reconcile only from that + # freshly sampled binding; doing this in the opposite order can + # manufacture a host epoch change at the silence-lease boundary. + control_snapshot = self._application_control_session.snapshot() + else: + # Preserve the legacy explicit-host/operator-manual admission + # order; it has no retained command binding to refresh. + self._require_desired_mode_matches_active() + self._reconcile_connection_supervisor( + control_snapshot, + self.runtime.snapshot(), + ) + if control_mode == "plugin-commanded": + self._require_desired_mode_matches_active() requested_streams = _validated_requested_streams(request) if request.host is None and control_mode == "operator-manual": self._refresh_live_lan_address() @@ -2172,6 +19555,10 @@ class XgridsK1CompatibilityService: "project_name": request.project_name, "mount_type": request.mount_type, "gnss_mode": request.gnss_mode, + "expected_control_session_generation": ( + request.expected_control_session_generation + ), + "expected_control_state_revision": request.expected_control_state_revision, "compatibility_attestation": request.compatibility_attestation.model_dump( mode="json" ), @@ -2197,6 +19584,18 @@ class XgridsK1CompatibilityService: ) try: + if control_mode == "plugin-commanded": + # Advance the read-only project prompt before publishing a + # local "prepared" acquisition. The worker performs the same + # binding guard again immediately before its MQTT dispatch. + try: + self._application_control_session.open_project_prompt( + expected_session_generation=(request.expected_control_session_generation), + expected_state_revision=request.expected_control_state_revision, + ) + except ApplicationConnectionBindingLost: + self._application_control_session.close_prestart() + raise with self._lock: if self._provisioning_active: raise RuntimeError( @@ -2231,17 +19630,24 @@ class XgridsK1CompatibilityService: }, ) self._acquisition = acquisition + self._restart_stop_only_checkpoint_owner = None self._acquisition_project_name = request.project_name self._acquisition_mount_type = request.mount_type self._acquisition_gnss_mode = request.gnss_mode self._acquisition_out_dir = new_live_session_dir(self.evidence_root) self._acquisition_start_operation_id = None self._acquisition_stop_operation_id = None + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_lineage = None + self._active_stream_recovery_state = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = None + self._active_stream_recovery_started_at_utc = None + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None self._compatibility_attestation = _attestation_snapshot( request.compatibility_attestation ) - if control_mode == "plugin-commanded": - self._application_control_session.open_project_prompt() self._operations.transition( operation.operation_id, "succeeded", @@ -2264,11 +19670,48 @@ class XgridsK1CompatibilityService: def start_acquisition(self, request: StartAcquisitionRequest) -> dict[str, Any]: acquisition = self._require_acquisition(request.acquisition_id) plugin_commanded = acquisition.control_mode == "plugin-commanded" + if not plugin_commanded: + self._require_desired_mode_matches_active() if plugin_commanded: + if getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) != "trusted": + raise ActiveAcquisitionRecoveryCheckpointError( + "START is blocked because the recovery checkpoint is not locally trusted" + ) if request.physical_acceptance is None: raise ValueError("START K1 требует явного подтверждения присутствия оператора") - if self._application_control_session.snapshot()["state"] != "project-ready": + if ( + request.expected_control_session_generation is None + or request.expected_control_state_revision is None + ): + raise ValueError( + "plugin-commanded START requires the exact control-session checkpoint" + ) + control_snapshot = self._application_control_session.snapshot() + if control_snapshot["state"] != "project-ready": raise RuntimeError("канонический диалог K1 ещё не готов принять START") + # Refresh the exact retained MQTT/route binding before any + # TTL-bound supervisor snapshot or authority gate. The later + # request_start checkpoint repeats this guard immediately before + # durable physical-command preparation. + self._application_control_session.validate_connection_binding() + control_snapshot = self._application_control_session.snapshot() + self._reconcile_connection_supervisor( + control_snapshot, + self.runtime.snapshot(), + ) + authority = self._connection_supervisor.snapshot().authority + if not authority.acquisition_start_allowed: + reasons = ", ".join(authority.reason_codes) + raise ConnectionLeaseUnavailable( + "START K1 запрещён: текущий маршрут, MQTT DeviceInfo или data-plane " + f"не дают актуального допуска ({reasons})", + reason_code="connection-supervisor-start-not-authorized", + ) + self._require_desired_mode_matches_active() if ( request.expected_state_revision is not None and request.expected_state_revision != acquisition.state_revision @@ -2279,6 +19722,10 @@ class XgridsK1CompatibilityService: { "acquisition_id": acquisition.acquisition_id, "expected_state_revision": request.expected_state_revision, + "expected_control_session_generation": ( + request.expected_control_session_generation + ), + "expected_control_state_revision": request.expected_control_state_revision, "physical_acceptance": ( request.physical_acceptance.model_dump(mode="json") if request.physical_acceptance is not None @@ -2309,7 +19756,82 @@ class XgridsK1CompatibilityService: cleanup_failed = False failure_stage = "local-start-preflight" start_checkpoint_released = False + physical_parent_operation_id: str | None = None + transition_gate_acquired = False + manual_process_lease_acquired = False + manual_process_lease_retained = False try: + failure_stage = "connection-mode-final-admission" + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + raise NetworkProvisioningConflict( + "другая lifecycle-операция K1 выполняется перед START", + reason_code="acquisition-start-lifecycle-busy", + ) + transition_gate_acquired = True + if not self._wait_for_connection_monitor_quiescence(): + raise NetworkProvisioningConflict( + "фоновая проверка подключения не завершилась перед START", + reason_code="acquisition-start-lifecycle-busy", + ) + # The initial check is a cheap admission guard. This exact second + # check runs while Mode Select is fenced and immediately precedes + # all local receiver/physical-command work, so a newer dropdown + # revision can never be followed by an old-mode START publish. + self._require_desired_mode_matches_active() + # Reject a duplicate/stale START on its own lifecycle facts before + # asking for the shared process fence. The exact checks are kept + # below as well, immediately before the state transition. + runtime_state = self.runtime.snapshot() + with self._lock: + if acquisition.state != "prepared": + raise RuntimeError( + f"acquisition нельзя запустить из состояния {acquisition.state}" + ) + if runtime_state.get("source_mode") != "idle": + raise RuntimeError( + "нельзя запускать acquisition поверх активного live/replay источника" + ) + if self._acquisition_out_dir is None: + raise RuntimeError("для acquisition не выделена evidence-сессия") + if self._acquisition_project_name is None: + raise RuntimeError("для acquisition не задано название проекта") + if not plugin_commanded: + failure_stage = "operator-manual-process-lease" + self._acquire_k1_lifecycle_process_lease("network") + manual_process_lease_acquired = True + with self._lock: + if self._operator_manual_acquisition_process_lease_id is not None: + raise ApplicationControlProcessLeaseUnavailable( + "operator-manual acquisition process lease is already retained" + ) + self._operator_manual_acquisition_process_lease_id = acquisition.acquisition_id + selected_device_id = self._selected_device_id + current_target = self._k1_ip + retired_transport_refs = self._retired_physical_transport_refs() + if retired_transport_refs: + if selected_device_id is None: + raise NetworkProvisioningConflict( + "После вывода исходного K1 выберите другое устройство до запуска", + reason_code="physical-command-replacement-device-required", + ) + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) + if current_target is None or validate_private_ipv4( + acquisition.target_host + ) != validate_private_ipv4(current_target): + raise NetworkProvisioningConflict( + "Приём нельзя запустить по адресу выведенного K1", + reason_code="physical-command-explicit-host-bypass-blocked", + ) + if plugin_commanded: + failure_stage = "physical-command-admission" + self._physical_command_ledger.require_no_unresolved_attempt() + physical_record = self._physical_command_ledger.snapshot().record + physical_parent_operation_id = ( + physical_record.operation_id if physical_record is not None else None + ) runtime_state = self.runtime.snapshot() with self._lock: if acquisition.state != "prepared": @@ -2344,15 +19866,39 @@ class XgridsK1CompatibilityService: lease.release() raise RuntimeError("evidence-сессия уже удерживается активным acquisition") self._acquisition_session_lease = lease + failure_stage = "camera-pre-pcl-reset" + # A preview selected before START is not evidence for this scan and + # must not leak a WebSocket/FFmpeg owner into calibration. The + # post-publication PCL observer is the sole automatic activation + # edge for the new acquisition. + self._stop_camera_preview_and_release_process_lease() failure_stage = "live-perception-ingress" + with self._lock: + self._live_perception_camera_binding = None + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = None + self._camera_activation_retry_not_before_monotonic = 0.0 + self._post_recovery_camera_restart_claim = None + self._post_recovery_camera_restart_inflight = None + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_terminal = None self.live_perception_ingress.begin_session(out_dir.name) self._modeling_control_safety.reset() failure_stage = "local-receiver-start" + self._reset_live_data_plane_observation() + if not plugin_commanded: + # From this boundary the receiver may contact the target. The + # process fence is released only by proven-complete receiver / + # evidence cleanup; any ambiguous cleanup keeps it fail-closed. + manual_process_lease_retained = True self.runtime.start_live( acquisition.target_host, out_dir, duration_seconds=acquisition.duration_seconds, project_name=project_name, + recover_connection=( + self._recover_active_stream_connection if plugin_commanded else None + ), ) if plugin_commanded: assert request.physical_acceptance is not None @@ -2360,12 +19906,23 @@ class XgridsK1CompatibilityService: self._application_control_session.request_start( project_name=project_name, confirmation=request.physical_acceptance.confirmation(), + command_context=PhysicalCommandIntentContext( + operation_id=operation.operation_id, + parent_operation_id=physical_parent_operation_id, + acquisition_id=acquisition.acquisition_id, + ), + preparation_checkpoint_observer=( + self._observe_start_preparation_checkpoint + ), + expected_session_generation=(request.expected_control_session_generation), + expected_state_revision=request.expected_control_state_revision, ) start_checkpoint_released = True - else: - failure_stage = "camera-recording" - self._arm_camera_recording(out_dir) except Exception as exc: + checkpoint_settlement_failed = isinstance( + exc, + ApplicationStartCheckpointSettlementError, + ) if owns_start: with self._lock: if acquisition.state not in TERMINAL_ACQUISITION_STATES: @@ -2397,7 +19954,11 @@ class XgridsK1CompatibilityService: side_effect_status=( "unknown" if start_checkpoint_released or cleanup_failed else "none" ), - safe_to_retry=not start_checkpoint_released and not cleanup_failed, + safe_to_retry=( + not start_checkpoint_released + and not cleanup_failed + and not checkpoint_settlement_failed + ), ), ) reason_code = getattr(exc, "reason_code", None) @@ -2412,21 +19973,401 @@ class XgridsK1CompatibilityService: ), "failure_stage": failure_stage, "start_checkpoint_released": start_checkpoint_released, - "safe_to_retry": not start_checkpoint_released and not cleanup_failed, + "safe_to_retry": ( + not start_checkpoint_released + and not cleanup_failed + and not checkpoint_settlement_failed + ), "automatic_retry": False, }, ) raise + finally: + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + if manual_process_lease_acquired and not manual_process_lease_retained: + self._release_operator_manual_acquisition_process_lease(acquisition.acquisition_id) return self.state() - @_serialized_acquisition_access + def _stop_request_fingerprint( + self, + request: StopAcquisitionRequest, + *, + acquisition_id: str, + ) -> str: + """Bind STOP retries to the historical public request shape.""" + + return self._request_fingerprint( + ACTION_ACQUISITION_STOP, + { + "acquisition_id": acquisition_id, + "mode": request.mode, + "expected_control_session_generation": ( + request.expected_control_session_generation + ), + "expected_control_state_revision": request.expected_control_state_revision, + "physical_acceptance": ( + request.physical_acceptance.model_dump(mode="json") + if request.physical_acceptance is not None + else None + ), + }, + ) + + def _existing_plugin_graceful_stop_retry( + self, + request: StopAcquisitionRequest, + ) -> _PreAdmittedGracefulStop | None: + """Resolve a caller-addressed STOP row before mutable lifecycle state. + + Only an explicit operation id can be checked without consulting the + current acquisition projection. Its original fingerprint already + contains the acquisition id, CAS and physical-acceptance checklist, so + the current immutable journal row is sufficient to reject a changed + request and to guarantee that no new physical edge is created. + """ + + if ( + request.mode != "graceful" + or request.operator_confirmed + or request.operation_id is None + ): + return None + try: + operation = self._operations.get(request.operation_id) + except KeyError: + return None + if operation.action != ACTION_ACQUISITION_STOP: + raise ValueError("operation id is already bound to another action") + acquisition_id = operation.context.get("acquisition_id") + if not isinstance(acquisition_id, str) or not acquisition_id: + # Rows created before this explicit context field are resolved by + # the legacy lifecycle-owned path below. + return None + request_fingerprint = self._stop_request_fingerprint( + request, + acquisition_id=acquisition_id, + ) + if operation.request_fingerprint != request_fingerprint: + raise ValueError("operation id is already bound to a different request") + return _PreAdmittedGracefulStop( + operation_id=operation.operation_id, + acquisition_id=acquisition_id, + request_fingerprint=request_fingerprint, + created=False, + accepted_at=operation.accepted_at, + deadline_at=operation.deadline_at, + ) + + def _pre_admit_plugin_graceful_stop( + self, + request: StopAcquisitionRequest, + ) -> _PreAdmittedGracefulStop | None: + """Start the operator STOP deadline before lifecycle/checkpoint waiting. + + Only a new, non-confirmation, plugin-commanded graceful STOP uses this + path. Capture-only, manual and confirmation requests retain their + historical lifecycle-owned journal admission. Cheap invalid requests + are rejected before a journal row or STOP-priority claim is created. + """ + + if request.mode != "graceful" or request.operator_confirmed: + return None + with self._lock: + acquisition = self._acquisition + if acquisition is None: + raise ValueError("acquisition ещё не подготовлена") + if ( + request.acquisition_id is not None + and request.acquisition_id != acquisition.acquisition_id + ): + raise ValueError("указана неизвестная acquisition-сессия") + if acquisition.control_mode != "plugin-commanded": + return None + acquisition_id = acquisition.acquisition_id + device_id = acquisition.device_id + device_session_id = acquisition.device_session_id + if request.physical_acceptance is None: + raise ValueError("STOP K1 требует явного подтверждения присутствия оператора") + if ( + request.expected_control_session_generation is None + or request.expected_control_state_revision is None + ): + raise ValueError( + "plugin-commanded STOP requires the exact control-session checkpoint" + ) + request_fingerprint = self._stop_request_fingerprint( + request, + acquisition_id=acquisition_id, + ) + operation, created = self._operations.begin( + ACTION_ACQUISITION_STOP, + operation_id=request.operation_id, + idempotency_key=request.idempotency_key, + device_id=device_id, + device_session_id=device_session_id, + deadline_seconds=request.deadline_seconds or 60.0, + cancellable=False, + request_fingerprint=request_fingerprint, + context={"acquisition_id": acquisition_id}, + ) + return _PreAdmittedGracefulStop( + operation_id=operation.operation_id, + acquisition_id=acquisition_id, + request_fingerprint=request_fingerprint, + created=created, + accepted_at=operation.accepted_at, + deadline_at=operation.deadline_at, + ) + + def _terminalize_pre_admitted_stop_before_prepare( + self, + token: _PreAdmittedGracefulStop, + exc: BaseException, + ) -> None: + """Seal a proven pre-PREPARE failure without inventing device I/O.""" + + if not token.created: + return + try: + physical_snapshot = self._physical_command_ledger.snapshot() + except (PhysicalCommandLedgerError, OSError, ValueError): + # Without a readable ledger we cannot prove that PREPARE did not + # reach disk. Preserve the nonterminal row for reconciliation. + return + if physical_snapshot.status == "corrupt": + # A missing record in a corrupt snapshot is not proof of absence. + # Keep the accepted operation open for explicit diagnosis instead + # of manufacturing a false no-side-effect terminal result. + return + physical = physical_snapshot.record + if physical is not None and physical.operation_id == token.operation_id: + return + try: + operation = self._operations.get(token.operation_id) + except KeyError: + return + if not ( + operation.action == ACTION_ACQUISITION_STOP + and operation.request_fingerprint == token.request_fingerprint + and operation.accepted_at == token.accepted_at + and operation.deadline_at == token.deadline_at + ): + return + cause_code = getattr(exc, "reason_code", None) + deadline_expired = cause_code == "physical-command-dispatch-deadline-expired" + self._operations.transition_if_pending( + token.operation_id, + "failed", + stage_code=( + "physical-stop-dispatch-deadline-expired-before-prepare" + if deadline_expired + else "physical-stop-admission-failed-before-prepare" + ), + message_code=( + "acquisition.stop.dispatch_deadline_expired_before_prepare" + if deadline_expired + else "acquisition.stop.admission_failed_before_prepare" + ), + error={ + "category": "conflict", + "code": ( + "physical-command-dispatch-deadline-expired" + if deadline_expired + else "physical-stop-admission-failed-before-prepare" + ), + "cause_code": ( + cause_code + if isinstance(cause_code, str) and cause_code + else type(exc).__name__ + ), + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "none", + "physical_command_sent": False, + "automatic_replay_allowed": False, + }, + ) + def stop_acquisition(self, request: StopAcquisitionRequest) -> dict[str, Any]: + exact_retry = self._existing_plugin_graceful_stop_retry(request) + if exact_retry is not None: + # Do not publish STOP priority or reconsider mutable physical + # admission for an exact historical retry. The action endpoint + # still promises the canonical full state contract, so composing + # that snapshot may wait behind a currently-owned lifecycle fsync; + # it cannot create a new PREPARE or publish edge. + return self.state() + preadmitted_stop = self._pre_admit_plugin_graceful_stop(request) + priority_acquisition_id: str | None = ( + preadmitted_stop.acquisition_id if preadmitted_stop is not None else None + ) + if request.mode == "graceful" and priority_acquisition_id is None: + # Confirmation retries do not create another operation, but still + # need the same immediate camera-priority publication. + with self._lock: + current_acquisition = self._acquisition + if ( + current_acquisition is not None + and current_acquisition.control_mode == "plugin-commanded" + and ( + request.acquisition_id is None + or request.acquisition_id == current_acquisition.acquisition_id + ) + ): + priority_acquisition_id = current_acquisition.acquisition_id + priority_published = False + try: + if priority_acquisition_id is not None: + with self._camera_restart_commit_gate, self._lock: + self._camera_stop_priority_counts[priority_acquisition_id] = ( + self._camera_stop_priority_counts.get(priority_acquisition_id, 0) + 1 + ) + priority_published = True + admitted = self._post_recovery_camera_restart_claim + if ( + admitted is not None + and admitted.key.acquisition_id == priority_acquisition_id + ): + self._post_recovery_camera_restart_claim = None + inflight = self._post_recovery_camera_restart_inflight + if ( + inflight is not None + and inflight.key.acquisition_id == priority_acquisition_id + ): + self._post_recovery_camera_restart_inflight = None + retry = self._post_recovery_camera_restart_retry + if ( + retry is not None + and retry.key.acquisition_id == priority_acquisition_id + ): + self._post_recovery_camera_restart_retry = None + terminal = self._post_recovery_camera_restart_terminal + if ( + terminal is not None + and terminal.acquisition_id == priority_acquisition_id + ): + self._post_recovery_camera_restart_terminal = None + with self._acquisition_lifecycle_access(): + if preadmitted_stop is None: + return self._stop_acquisition_owned(request) + return self._stop_acquisition_owned(request, preadmitted_stop) + except BaseException as exc: + if preadmitted_stop is not None: + self._terminalize_pre_admitted_stop_before_prepare( + preadmitted_stop, + exc, + ) + raise + finally: + if priority_acquisition_id is not None and priority_published: + with self._camera_restart_commit_gate, self._lock: + remaining = self._camera_stop_priority_counts.get( + priority_acquisition_id, + 0, + ) - 1 + if remaining > 0: + self._camera_stop_priority_counts[priority_acquisition_id] = remaining + else: + self._camera_stop_priority_counts.pop(priority_acquisition_id, None) + + def _stop_acquisition_owned( + self, + request: StopAcquisitionRequest, + preadmitted_stop: _PreAdmittedGracefulStop | None = None, + ) -> dict[str, Any]: + """Run one STOP after its priority token precedes lifecycle waiting.""" + acquisition = self._require_acquisition(request.acquisition_id) plugin_commanded = acquisition.control_mode == "plugin-commanded" + request_fingerprint = self._stop_request_fingerprint( + request, + acquisition_id=acquisition.acquisition_id, + ) + preadmitted_operation = None + if preadmitted_stop is not None: + preadmitted_operation = self._operations.get( + preadmitted_stop.operation_id + ) + if not ( + preadmitted_stop.acquisition_id == acquisition.acquisition_id + and preadmitted_stop.request_fingerprint == request_fingerprint + and preadmitted_operation.action == ACTION_ACQUISITION_STOP + and preadmitted_operation.request_fingerprint + == request_fingerprint + and preadmitted_operation.accepted_at + == preadmitted_stop.accepted_at + and preadmitted_operation.deadline_at + == preadmitted_stop.deadline_at + ): + raise LocalAcquisitionLifecycleError( + "STOP admission changed before lifecycle ownership", + reason_code="physical-stop-preadmission-changed", + ) + if not preadmitted_stop.created: + # Exact retries resolve from their immutable journal identity + # before current control/physical admission is reconsidered. + # The original STOP may already have lost its socket or + # reached a terminal NONE/UNKNOWN result; neither condition + # can authorize another PREPARE or publish edge. + return self.state() + # An exact client retry must resolve from the operation journal before + # current physical admission is reconsidered. In particular, a + # recovered STOP may already have lost its control socket and become + # standby-unconfirmed; rejecting that retry on the now-idle session + # would invite a new command even though the original side effect is + # deliberately never replayed. + if ( + preadmitted_stop is None + and request.operation_id is not None + and not request.operator_confirmed + ): + try: + existing_operation = self._operations.get(request.operation_id) + except KeyError: + existing_operation = None + if existing_operation is not None: + if existing_operation.action != ACTION_ACQUISITION_STOP: + raise ValueError("operation id is already bound to another action") + if existing_operation.request_fingerprint != request_fingerprint: + raise ValueError("operation id is already bound to a different request") + return self.state() + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition.acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "STOP недоступен, пока read-only классификация применяется к приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + capture_only_stop_retirement: ( + tuple[_LocalStopRetirementDisposition, str | None] | None + ) = None + if plugin_commanded and request.mode == "capture-only": + capture_only_stop_retirement = ( + self._fence_existing_stop_before_local_action( + acquisition_id=acquisition.acquisition_id, + reject_nonterminal_prepared=True, + ) + ) + if ( + capture_only_stop_retirement is not None + and capture_only_stop_retirement[0] == "standby-confirmed" + ): + # Canonical READY is already stronger than the requested + # local-only action. Let the ordinary durable reducer finish + # this exact S0 acquisition without creating another operation + # or relabelling its pending START/STOP journal rows. + return self.state() with self._lock: acquisition_state = acquisition.state expected_stop_operation_id = self._acquisition_stop_operation_id lease_retained = self._acquisition_session_lease is not None + control_state = self._application_control_session.snapshot()["state"] + canonical_terminal_stop_recovery = bool( + plugin_commanded + and acquisition_state in TERMINAL_ACQUISITION_STATES + and control_state == "scanning" + ) terminal_manual_stop_recovery = ( request.operator_confirmed @@ -2455,9 +20396,10 @@ class XgridsK1CompatibilityService: and not ( plugin_commanded and acquisition_state in {"starting", "awaiting_external_start"} - and self._application_control_session.snapshot()["state"] == "scanning" + and control_state == "scanning" ) and not (acquisition_state in TERMINAL_ACQUISITION_STATES and lease_retained) + and not canonical_terminal_stop_recovery ): raise ValueError( "graceful stop допустим только после подтверждённого потока point cloud" @@ -2465,7 +20407,14 @@ class XgridsK1CompatibilityService: if plugin_commanded and request.mode == "graceful" and not request.operator_confirmed: if request.physical_acceptance is None: raise ValueError("STOP K1 требует явного подтверждения присутствия оператора") - if self._application_control_session.snapshot()["state"] != "scanning": + if ( + request.expected_control_session_generation is None + or request.expected_control_state_revision is None + ): + raise ValueError( + "plugin-commanded STOP requires the exact control-session checkpoint" + ) + if control_state != "scanning": raise RuntimeError("канонический диалог K1 ещё не готов принять STOP") if terminal_manual_stop_recovery: @@ -2476,23 +20425,26 @@ class XgridsK1CompatibilityService: ) return self.state() - request_fingerprint = self._request_fingerprint( - ACTION_ACQUISITION_STOP, - { - "acquisition_id": acquisition.acquisition_id, - "mode": request.mode, - }, - ) - operation, created = self._operations.begin( - ACTION_ACQUISITION_STOP, - operation_id=request.operation_id, - idempotency_key=request.idempotency_key, - device_id=acquisition.device_id, - device_session_id=acquisition.device_session_id, - deadline_seconds=request.deadline_seconds or 60.0, - cancellable=False, - request_fingerprint=request_fingerprint, - ) + if preadmitted_stop is not None: + assert preadmitted_operation is not None + operation = preadmitted_operation + created = True + if self._operations.deadline_reached(operation.operation_id): + raise LocalAcquisitionLifecycleError( + "STOP deadline elapsed before durable physical preparation", + reason_code="physical-command-dispatch-deadline-expired", + ) + else: + operation, created = self._operations.begin( + ACTION_ACQUISITION_STOP, + operation_id=request.operation_id, + idempotency_key=request.idempotency_key, + device_id=acquisition.device_id, + device_session_id=acquisition.device_session_id, + deadline_seconds=request.deadline_seconds or 60.0, + cancellable=False, + request_fingerprint=request_fingerprint, + ) if ( request.mode == "graceful" and not request.operator_confirmed @@ -2516,7 +20468,10 @@ class XgridsK1CompatibilityService: raise ValueError("acquisition уже ожидает подтверждения graceful stop") if not created and not request.operator_confirmed: return self.state() - if acquisition.state in TERMINAL_ACQUISITION_STATES: + if ( + acquisition.state in TERMINAL_ACQUISITION_STATES + and not canonical_terminal_stop_recovery + ): if lease_retained: self._operations.transition_if_pending( operation.operation_id, @@ -2567,26 +20522,179 @@ class XgridsK1CompatibilityService: if request.mode == "graceful" and not request.operator_confirmed: if plugin_commanded: assert request.physical_acceptance is not None + try: + self._physical_command_ledger.require_no_unresolved_attempt() + physical_record = self._physical_command_ledger.snapshot().record + physical_parent_operation_id = ( + physical_record.operation_id if physical_record is not None else None + ) + recovery_confirmation_required = bool( + physical_record is not None + and getattr( + physical_record, + "successor_stop_requires_operator_confirmation", + False, + ) + ) + except Exception as exc: + self._operations.transition_if_pending( + operation.operation_id, + "failed", + stage_code="physical-command-admission-failed", + message_code="acquisition.stop.physical_command_blocked", + error=_operation_error( + exc, + category="conflict", + side_effect_status="none", + ), + ) + raise self.camera_preview.expect_source_end_for_device_stop() + prepared_stop_lineage = self._capture_prepared_stop_dispatch_lineage( + operation_id=operation.operation_id, + acquisition=acquisition, + ) + if prepared_stop_lineage is None: + self.camera_preview.cancel_expected_source_end() + failure = LocalAcquisitionLifecycleError( + "STOP отклонён: точная активная physical lineage больше не совпадает", + reason_code="physical-stop-parent-proof-unavailable", + ) + self._operations.transition_if_pending( + operation.operation_id, + "failed", + stage_code="physical-command-admission-failed", + message_code="acquisition.stop.physical_command_blocked", + error=_operation_error( + failure, + category="conflict", + side_effect_status="none", + ), + ) + raise failure + with self._lock: + self._prepared_stop_dispatch_lineage = prepared_stop_lineage + self._acquisition_stop_operation_id = operation.operation_id + + def dispatch_admission_deadline_reached() -> bool: + return self._operations.deadline_reached(operation.operation_id) + try: self._application_control_session.request_stop( - confirmation=request.physical_acceptance.confirmation() + confirmation=request.physical_acceptance.confirmation(), + command_context=PhysicalCommandIntentContext( + operation_id=operation.operation_id, + parent_operation_id=physical_parent_operation_id, + acquisition_id=acquisition.acquisition_id, + operator_confirmation_id=( + request.idempotency_key if recovery_confirmation_required else None + ), + operator_confirmed_at_utc=( + _utc_now_millisecond_iso() + if recovery_confirmation_required + else None + ), + ), + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + expected_session_generation=(request.expected_control_session_generation), + expected_state_revision=request.expected_control_state_revision, ) - except Exception: + with self._lock: + # The fresh S1 now owns the durable edge. The retained + # S0 classification owner must not race or survive it. + self._prepared_stop_recovery_owner = None + self._classified_stop_rebind_pending = None + self._classified_stop_rebind_inflight = None + except Exception as exc: + deadline_expired = ( + getattr(exc, "reason_code", None) + == "physical-command-dispatch-deadline-expired" + ) + physical_after_failure = dict( + self._physical_command_coordinator.snapshot() + ) + definite_deadline_no_dispatch = bool( + deadline_expired + and self._resolved_unclassified_stop_successor_matches( + prepared_stop_lineage, + physical_after_failure, + ) + ) + if definite_deadline_no_dispatch: + # The session may have crossed durable PREPARE before + # observing the already-running public deadline. It + # synchronously resolves that exact row as + # not-dispatched; preserve the lineage/pointer so the + # normal read-only recovery reducer can retain the + # receiver and prove zero replay. Clearing them here + # would discard the only local owner for S0. + self._operations.transition_if_pending( + operation.operation_id, + "failed", + stage_code="physical-stop-classified-not-dispatched", + message_code=( + "acquisition.stop.physical_command_not_dispatched" + ), + error={ + "category": "device", + "code": "physical-stop-not-dispatched", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "none", + "physical_command_sent": False, + "automatic_replay_allowed": False, + }, + ) + with self._lock: + if acquisition.state not in { + "awaiting_external_stop", + *TERMINAL_ACQUISITION_STATES, + }: + acquisition.transition( + "awaiting_external_stop", + message_code=( + "acquisition.stop.physical_command_not_dispatched" + ), + ) + self.camera_preview.cancel_expected_source_end() + recovery_owner = self._capture_prepared_stop_recovery_owner( + physical=physical_after_failure, + ) + if recovery_owner is None: + raise LocalAcquisitionLifecycleError( + ( + "STOP recovery owner disappeared after the " + "no-dispatch commit" + ), + reason_code=( + "prepared-stop-recovery-owner-unavailable" + ), + ) from exc + raise + with self._lock: + if self._prepared_stop_dispatch_lineage == prepared_stop_lineage: + self._prepared_stop_dispatch_lineage = None + if self._acquisition_stop_operation_id == operation.operation_id: + self._acquisition_stop_operation_id = None self.camera_preview.cancel_expected_source_end() raise with self._lock: - acquisition.transition( - "awaiting_external_stop", - message_code=( - "acquisition.stop.device_stopping" + if not canonical_terminal_stop_recovery: + acquisition.transition( + "awaiting_external_stop", + message_code=( + "acquisition.stop.device_stopping" + if plugin_commanded + else "acquisition.stop.operator_action_required" + ), + operator_instructions=() if plugin_commanded - else "acquisition.stop.operator_action_required" - ), - operator_instructions=() - if plugin_commanded - else ("Дважды нажмите физическую кнопку устройства и подтвердите остановку.",), - ) + else ( + "Дважды нажмите физическую кнопку устройства и подтвердите остановку.", + ), + ) self._acquisition_stop_operation_id = operation.operation_id self._operations.transition( operation.operation_id, @@ -2601,6 +20709,11 @@ class XgridsK1CompatibilityService: ) return self.state() + fenced_stop_operation_id = ( + capture_only_stop_retirement[1] + if capture_only_stop_retirement is not None + else None + ) self._operations.transition( operation.operation_id, "running", @@ -2614,12 +20727,31 @@ class XgridsK1CompatibilityService: camera_status="complete", camera_failure_code=None, ) + self._terminalize_pending_start_for_local_stop_retirement( + acquisition_id=acquisition.acquisition_id, + stop_operation_id=fenced_stop_operation_id, + ) self._cancel_pending_acquisition_operations( exclude_operation_id=operation.operation_id, reason_code="superseded-by-stop", + preserve_operation_ids=( + (fenced_stop_operation_id,) + if fenced_stop_operation_id is not None + else () + ), ) with self._lock: if acquisition.state not in TERMINAL_ACQUISITION_STATES: + device_stop = { + "not-dispatched": "not-sent", + "outcome-unknown": "physical-outcome-unknown", + "standby-confirmed": "protocol-confirmed", + }.get( + capture_only_stop_retirement[0] + if capture_only_stop_retirement is not None + else "not-applicable", + "unknown", + ) acquisition.transition("finalizing", message_code="acquisition.finalizing") acquisition.transition( "completed", @@ -2629,7 +20761,7 @@ class XgridsK1CompatibilityService: "device_stop": ( "operator-confirmed" if request.mode == "graceful" and request.operator_confirmed - else "unknown" + else device_stop ), }, ) @@ -2649,9 +20781,18 @@ class XgridsK1CompatibilityService: with self._lock: if acquisition.state not in TERMINAL_ACQUISITION_STATES: acquisition.transition("failed", message_code="acquisition.stop.failed") + self._terminalize_pending_start_for_local_stop_retirement( + acquisition_id=acquisition.acquisition_id, + stop_operation_id=fenced_stop_operation_id, + ) self._cancel_pending_acquisition_operations( exclude_operation_id=operation.operation_id, reason_code="acquisition-stop-failed", + preserve_operation_ids=( + (fenced_stop_operation_id,) + if fenced_stop_operation_id is not None + else () + ), ) self._operations.transition_if_pending( operation.operation_id, @@ -2666,9 +20807,55 @@ class XgridsK1CompatibilityService: @_serialized_acquisition_access def abort_acquisition(self, request: AbortAcquisitionRequest) -> dict[str, Any]: acquisition = self._require_acquisition(request.acquisition_id) + unresolved_stop = ( + self._matching_unresolved_stop_stage( + self._physical_command_coordinator.snapshot(), + acquisition_id=acquisition.acquisition_id, + ) + if acquisition.control_mode == "plugin-commanded" + else None + ) + if unresolved_stop is not None: + raise LocalAcquisitionLifecycleError( + "ABORT недоступен, пока исход PREPARED/DISPATCHING STOP не установлен", + reason_code="acquisition-abort-physical-stop-unresolved", + ) + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition.acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "ABORT недоступен, пока read-only классификация STOP применяется к приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + if acquisition.control_mode == "plugin-commanded": + resolved_stop_retirement = self._fence_existing_stop_before_local_action( + acquisition_id=acquisition.acquisition_id, + reject_nonterminal_prepared=False, + resolved_standby_only=True, + ) + if resolved_stop_retirement is not None: + # Durable READY owns the terminal result. ABORT is therefore + # an idempotent observation of the already completed physical + # STOP, not a new operation that may cancel S0 or pending P. + return self.state() + if ( + acquisition.control_mode == "plugin-commanded" + and acquisition.state == "prepared" + and ( + request.expected_control_session_generation is None + or request.expected_control_state_revision is None + ) + ): + raise ValueError("plugin-commanded ABORT requires the exact control-session checkpoint") request_fingerprint = self._request_fingerprint( ACTION_ACQUISITION_ABORT, - {"acquisition_id": acquisition.acquisition_id}, + { + "acquisition_id": acquisition.acquisition_id, + "expected_control_session_generation": ( + request.expected_control_session_generation + ), + "expected_control_state_revision": request.expected_control_state_revision, + }, ) operation, created = self._operations.begin( ACTION_ACQUISITION_ABORT, @@ -2683,7 +20870,22 @@ class XgridsK1CompatibilityService: return self.state() try: if acquisition.control_mode == "plugin-commanded" and acquisition.state == "prepared": - self._application_control_session.close_prestart() + control_state = str( + self._application_control_session.snapshot().get("state") or "unknown" + ) + if control_state in { + "connection-ready", + "workspace-ready", + "project-ready", + }: + self._application_control_session.close_prestart( + expected_session_generation=(request.expected_control_session_generation), + expected_state_revision=request.expected_control_state_revision, + ) + elif control_state not in {"idle", "completed", "closed", "failed"}: + raise RuntimeError( + "control-сессия ещё выполняет переход и не допускает локальный ABORT" + ) with self._lock: should_abort = acquisition.state not in TERMINAL_ACQUISITION_STATES if should_abort: @@ -2735,6 +20937,252 @@ class XgridsK1CompatibilityService: raise return self.state() + def _force_finish_active_acquisition_locally( + self, + *, + reason_code: str, + expected_acquisition_id: str | None = None, + expected_recovery_generation: int | None = None, + require_recovery: bool = False, + fenced_control_retirement: ( + tuple[_LocalStopRetirementDisposition, str | None] | None + ) = None, + ) -> bool: + """Fence the existing STOP owner, then cancel only local producers. + + The caller owns ``_acquisition_lifecycle_gate``. Connection-mode reset + may call this helper while it also owns the K1 transition gate. The + The dispatch fence and local control close happen before producer + cleanup. A PREPARED STOP is sealed as not-dispatched only after its + worker is terminal; DISPATCHING/OBSERVING remains outcome-unknown. The + recovery generation is then invalidated before blocking producer work. + """ + + with self._lock: + acquisition = self._acquisition + lineage = self._active_stream_recovery_lineage + recovery_state = self._active_stream_recovery_state + if acquisition is None or acquisition.state in TERMINAL_ACQUISITION_STATES: + return False + if ( + expected_acquisition_id is not None + and acquisition.acquisition_id != expected_acquisition_id + ): + raise LocalAcquisitionLifecycleError( + "acquisition changed before local force-finish", + reason_code="acquisition-force-finish-lineage-changed", + ) + if require_recovery and ( + lineage is None or recovery_state not in {"reconnecting", "blocked"} + ): + raise LocalAcquisitionLifecycleError( + "local force-finish is available only during connection recovery", + reason_code="acquisition-force-finish-not-recovering", + ) + if ( + expected_recovery_generation is not None + and expected_recovery_generation != self._active_stream_recovery_generation + ): + raise LocalAcquisitionLifecycleError( + "connection recovery generation changed", + reason_code="acquisition-force-finish-generation-changed", + ) + + stop_retirement, unresolved_stop_operation_id = ( + fenced_control_retirement + if fenced_control_retirement is not None + else self._fence_local_control_for_acquisition_retirement( + acquisition_id=acquisition.acquisition_id + ) + ) + with self._lock: + if self._acquisition is not acquisition or ( + acquisition.state in TERMINAL_ACQUISITION_STATES + ): + raise LocalAcquisitionLifecycleError( + "acquisition changed during local control retirement", + reason_code="acquisition-force-finish-lineage-changed", + ) + future = self._active_stream_recovery_future + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_state = "force-finishing" + self._active_stream_recovery_reason_code = reason_code + self._active_stream_recovery_lineage = None + self._active_stream_recovery_future = None + self._active_stream_recovery_terminal_outcome = None + + if future is not None: + future.cancel() + + cleanup_error: Exception | None = None + try: + self._stop_acquisition_sources( + camera_status="interrupted", + camera_failure_code=reason_code, + ) + except Exception as exc: + if cleanup_error is not None: + exc.add_note( + "local control cleanup also failed: " + f"{type(cleanup_error).__name__}: {cleanup_error}" + ) + cleanup_error = exc + try: + self._stop_camera_preview_and_release_process_lease() + except Exception as exc: + if cleanup_error is None: + cleanup_error = exc + + self._terminalize_pending_start_for_local_stop_retirement( + acquisition_id=acquisition.acquisition_id, + stop_operation_id=unresolved_stop_operation_id, + ) + + terminal_control = self._application_control_session.snapshot() + if terminal_control.get("state") in {"completed", "closed", "failed"}: + try: + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + except (AttributeError, RuntimeError): + with self._lock: + self._pending_local_control_retirement = True + self._cancel_pending_acquisition_operations( + exclude_operation_id="", + reason_code="superseded-by-local-force-finish", + preserve_operation_ids=( + (unresolved_stop_operation_id,) + if unresolved_stop_operation_id is not None + else () + ), + ) + device_stop = { + "outcome-unknown": "physical-outcome-unknown", + "standby-confirmed": "protocol-confirmed", + }.get(stop_retirement, "not-sent") + with self._lock: + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + result: dict[str, Any] = { + "receiver_stopped": cleanup_error is None, + "camera_stopped": cleanup_error is None, + "device_state": ( + "ready" if stop_retirement == "standby-confirmed" else "unknown" + ), + "device_stop": device_stop, + "local_force_finish": True, + "reason_code": reason_code, + "automatic_replay_allowed": False, + "automatic_command_retry": False, + } + if stop_retirement in {"not-applicable", "not-dispatched"}: + result["physical_command_sent"] = False + elif stop_retirement == "standby-confirmed": + result["physical_command_sent"] = True + acquisition.transition( + "failed" if cleanup_error is not None else "interrupted", + message_code=( + "acquisition.force_finish.cleanup_failed" + if cleanup_error is not None + else "acquisition.force_finish.local_completed" + ), + result=result, + ) + self._active_stream_recovery_state = "force-finished" + if cleanup_error is not None: + raise cleanup_error + return True + + @_serialized_acquisition_access + def force_finish_acquisition_locally( + self, + request: ForceFinishAcquisitionRequest, + ) -> dict[str, Any]: + acquisition = self._require_acquisition(request.acquisition_id) + with self._lock: + if acquisition.state_revision != request.expected_state_revision: + raise LocalAcquisitionLifecycleError( + "acquisition changed before local force-finish", + reason_code="acquisition-force-finish-state-changed", + ) + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition.acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "local force-finish недоступен, пока read-only классификация STOP " + "применяется к приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + request_fingerprint = self._request_fingerprint( + ACTION_ACQUISITION_FORCE_FINISH, + { + "acquisition_id": request.acquisition_id, + "expected_state_revision": request.expected_state_revision, + "expected_recovery_generation": request.expected_recovery_generation, + "operator_confirmed": request.operator_confirmed, + }, + ) + operation, created = self._operations.begin( + ACTION_ACQUISITION_FORCE_FINISH, + operation_id=request.operation_id, + idempotency_key=request.idempotency_key, + device_id=acquisition.device_id, + device_session_id=acquisition.device_session_id, + deadline_seconds=request.deadline_seconds or 30.0, + cancellable=False, + request_fingerprint=request_fingerprint, + ) + if not created: + return self.state() + self._operations.transition( + operation.operation_id, + "running", + stage_code="cancelling-local-recovery-owner", + message_code="acquisition.force_finish.running", + ) + try: + self._force_finish_active_acquisition_locally( + reason_code="operator-local-force-finish", + expected_acquisition_id=request.acquisition_id, + expected_recovery_generation=request.expected_recovery_generation, + require_recovery=True, + ) + self._operations.transition( + operation.operation_id, + "succeeded", + stage_code="local-evidence-sealed", + message_code="acquisition.force_finish.completed", + result={ + "acquisition_id": request.acquisition_id, + "device_stop": ( + acquisition.result.get("device_stop") + if isinstance(acquisition.result, Mapping) + else "unknown" + ), + **( + {"physical_command_sent": False} + if isinstance(acquisition.result, Mapping) + and acquisition.result.get("physical_command_sent") is False + else {} + ), + }, + ) + except Exception as exc: + self._operations.transition_if_pending( + operation.operation_id, + "failed", + stage_code="local-cleanup-failed", + message_code="acquisition.force_finish.failed", + error=_operation_error( + exc, + category="stream", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + raise + return self.state() + def start_live( self, project_name: str, @@ -2750,6 +21198,7 @@ class XgridsK1CompatibilityService: host=host, duration_seconds=duration_seconds, compatibility_attestation=compatibility_attestation, + idempotency_key=f"compat-live-prepare:{uuid4()}", ) ) acquisition = prepared.get("acquisition") @@ -2758,12 +21207,18 @@ class XgridsK1CompatibilityService: ): raise RuntimeError("acquisition prepare did not return an acquisition id") return self.start_acquisition( - StartAcquisitionRequest(acquisition_id=acquisition["acquisition_id"]) + StartAcquisitionRequest( + acquisition_id=acquisition["acquisition_id"], + idempotency_key=f"compat-live-start:{uuid4()}", + ) ) @_serialized_acquisition_access + @_serialized_k1_transition_access def start_replay(self, path: str, speed: float, loop: bool) -> dict[str, Any]: with self._lock: + if self._provisioning_active: + raise RuntimeError("нельзя запускать replay во время настройки сети K1") acquisition = self._acquisition lease_retained = self._acquisition_session_lease is not None if acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES: @@ -2797,49 +21252,143 @@ class XgridsK1CompatibilityService: camera_failure_code=(None if completed else "acquisition-cleanup-retry"), ) else: - self.camera_preview.stop_current() + self._stop_camera_preview_and_release_process_lease() self.runtime.stop() + self._reset_live_data_plane_observation() return self.state() return self.stop_acquisition( StopAcquisitionRequest( acquisition_id=acquisition.acquisition_id, mode="capture-only", + idempotency_key=f"compat-live-stop:{uuid4()}", ) ) @_serialized_acquisition_access + @_serialized_k1_transition_access def select_camera_preview(self, request: CameraPreviewSelectRequest) -> dict[str, Any]: - target = self._camera_target_for_session(request.device_session_id) + camera_holder_acquired = False + try: + with self._lock: + current_acquisition = self._acquisition + current_out_dir = self._acquisition_out_dir + acquisition_active_before_select = bool( + current_acquisition is not None + and current_acquisition.state not in TERMINAL_ACQUISITION_STATES + ) + camera_before_select = self.camera_preview.snapshot() + recording_before_select = camera_before_select.get("recording") + current_epoch_before_select = ( + recording_before_select.get("current_epoch") + if isinstance(recording_before_select, Mapping) + else None + ) + automatic_camera_activation_complete = bool( + acquisition_active_before_select + and current_out_dir is not None + and camera_before_select.get("active_source_id") + == DEFAULT_ACQUISITION_CAMERA_SOURCE + and camera_before_select.get("phase") == "streaming" + and isinstance(recording_before_select, Mapping) + and recording_before_select.get("active") is True + and recording_before_select.get("session") == current_out_dir.name + and recording_before_select.get("active_epoch") is not None + and recording_before_select.get("media_ready") is True + and isinstance(current_epoch_before_select, Mapping) + and current_epoch_before_select.get("generation") + == recording_before_select.get("active_epoch") + and current_epoch_before_select.get("init_committed") is True + and current_epoch_before_select.get("first_media_committed") is True + ) + if acquisition_active_before_select and not automatic_camera_activation_complete: + raise RuntimeError( + "camera preview доступен после автоматического запуска правой камеры" + ) + camera_holder_acquired = self._ensure_camera_preview_process_lease() + target = self._camera_target_for_session(request.device_session_id) + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + acquisition_active = ( + acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES + ) + plugin_commanded = ( + acquisition is not None and acquisition.control_mode == "plugin-commanded" + ) + device_scanning = ( + not plugin_commanded + or self._application_control_session.snapshot().get("state") == "scanning" + ) + if acquisition_active and device_scanning: + if out_dir is None: + raise RuntimeError("для acquisition не выделена evidence-сессия") + self._arm_camera_recording(out_dir, require_session=True) + camera = self.camera_preview.select(request.source_id, target) + if acquisition_active and device_scanning and out_dir is not None: + self._bind_live_perception_camera(out_dir.name, camera) + return self.state() + finally: + if ( + camera_holder_acquired + and self.camera_preview.snapshot().get("active_source_id") is None + ): + self._release_camera_preview_process_lease() + + @_serialized_acquisition_access + @_serialized_k1_transition_access + def stop_camera_preview(self, request: CameraPreviewStopRequest) -> dict[str, Any]: + self._require_camera_device_session(request.device_session_id) + self.camera_preview.stop(request.generation) + self._release_camera_preview_process_lease() + return self.state() + + def close(self) -> None: + # Wake a queued scenario reset before waiting for its shared local + # lifecycle gate. Shutdown, not the reset, owns the remaining cleanup. with self._lock: acquisition = self._acquisition - out_dir = self._acquisition_out_dir - acquisition_active = ( - acquisition is not None and acquisition.state not in TERMINAL_ACQUISITION_STATES + acquisition_id = ( + acquisition.acquisition_id if acquisition is not None else None ) - plugin_commanded = ( - acquisition is not None and acquisition.control_mode == "plugin-commanded" + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "shutdown недоступен, пока read-only классификация STOP " + "применяется к приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + self._service_close_requested.set() + with self._acquisition_lifecycle_access(): + self._close_owned() + + def _close_owned(self) -> None: + with self._lock: + acquisition_before_close = self._acquisition + acquisition_id_before_close = ( + acquisition_before_close.acquisition_id + if acquisition_before_close is not None + else None + ) + stop_retirement, fenced_stop_operation_id = ( + self._fence_local_control_for_acquisition_retirement( + acquisition_id=acquisition_id_before_close, ) - device_scanning = ( - not plugin_commanded - or self._application_control_session.snapshot().get("state") == "scanning" ) - if acquisition_active and device_scanning: - if out_dir is None: - raise RuntimeError("для acquisition не выделена evidence-сессия") - self._arm_camera_recording(out_dir, require_session=True) - self.camera_preview.select(request.source_id, target) - return self.state() - - @_serialized_acquisition_access - def stop_camera_preview(self, request: CameraPreviewStopRequest) -> dict[str, Any]: - self._camera_target_for_session(request.device_session_id) - self.camera_preview.stop(request.generation) - return self.state() - - @_serialized_acquisition_access - def close(self) -> None: + # Revoke connection authority synchronously before any producer cleanup. + # The asyncio monitor task is cancelled on its owner loop without + # creating a second loop during service shutdown. + self._request_control_bootstrap_continuation_close() + self._connection_monitor.request_close() + with self._lock: + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} self._application_control.close() - self._application_control_session.close() + self._reconcile_application_control_process_lease( + self._application_control_session.snapshot() + ) with self._lock: acquisition = self._acquisition close_active_acquisition = ( @@ -2858,6 +21407,8 @@ class XgridsK1CompatibilityService: self.camera_preview.close() except Exception as exc: camera_error = exc + else: + self._release_camera_preview_process_lease() try: self.runtime.close() except Exception as exc: @@ -2890,18 +21441,39 @@ class XgridsK1CompatibilityService: ), result={ "receiver_stopped": terminal_error is None, - "device_state": "unknown", + "device_state": ( + "ready" + if stop_retirement == "standby-confirmed" + else "unknown" + ), + "device_stop": { + "not-dispatched": "not-sent", + "outcome-unknown": "physical-outcome-unknown", + "standby-confirmed": "protocol-confirmed", + }.get(stop_retirement, "not-sent"), }, ) - self._terminalize_acquisition_operations_on_shutdown(terminal_error) + self._terminalize_acquisition_operations_on_shutdown( + terminal_error, + fenced_stop_operation_id=fenced_stop_operation_id, + ) finally: with self._lock: out_dir = self._acquisition_out_dir + runtime_loop = self._runtime_event_loop + self._runtime_event_loop = None + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} if out_dir is not None: self.live_perception_ingress.end_session(out_dir.name) self.live_perception_ingress.close() if terminal_error is None: self._release_acquisition_session_lease() + if runtime_loop is not None: + invalidate_ble_runtime_owner_loop(runtime_loop) + invalidate_runtime_handles() if terminal_error is not None: raise terminal_error @@ -2931,7 +21503,8 @@ class XgridsK1CompatibilityService: ) -> None: camera = self.camera_preview.snapshot() if camera["recording"]["active"]: - self.camera_preview.start_recording(out_dir) + camera = self.camera_preview.start_recording(out_dir) + self._bind_live_perception_camera(out_dir.name, camera) return if not out_dir.is_dir(): if not require_session and camera["active_source_id"] is None: @@ -2951,51 +21524,3436 @@ class XgridsK1CompatibilityService: "evidence-сессия не появилась вовремя для записи camera stream" ) time.sleep(0.01) - self.camera_preview.start_recording(out_dir) + camera = self.camera_preview.start_recording(out_dir) + self._bind_live_perception_camera(out_dir.name, camera) - @_serialized_acquisition_access - def _activate_default_acquisition_camera(self) -> None: - """Start the canonical right-camera archive after confirmed K1 START.""" + def _bind_live_perception_camera( + self, + session_id: str, + camera: Mapping[str, Any], + ) -> None: + """Bind one exact camera producer generation to one ingress session.""" + recording = camera.get("recording") + source_id = camera.get("active_source_id") + generation = recording.get("active_epoch") if isinstance(recording, Mapping) else None + binding = ( + (session_id, cast(CameraSourceId, source_id), generation) + if source_id in CAMERA_SOURCE_PATHS + and isinstance(generation, int) + and not isinstance(generation, bool) + and generation >= 0 + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == session_id + else None + ) + with self._lock: + self._live_perception_camera_binding = binding + + def _activate_default_acquisition_camera( + self, + *, + expected_acquisition_id: str, + expected_evidence_session_id: str, + expected_runtime_generation: int, + ) -> bool: + """Start one local right-camera producer after authoritative first PCL.""" + + runtime = self.runtime.snapshot() + metrics = runtime.get("metrics") + point_frames = metrics.get("pcl_frames") if isinstance(metrics, Mapping) else None with self._lock: acquisition = self._acquisition out_dir = self._acquisition_out_dir device_session_id = self._device_session_id + start_operation_id = self._acquisition_start_operation_id if ( acquisition is None - or acquisition.state in TERMINAL_ACQUISITION_STATES - or acquisition.control_mode != "plugin-commanded" + or acquisition.acquisition_id != expected_acquisition_id + or acquisition.state not in {"starting", "awaiting_external_start", "acquiring"} + or out_dir is None + or out_dir.name != expected_evidence_session_id ): - return - if out_dir is None: - raise RuntimeError("для acquisition не выделена evidence-сессия") + return False + if not ( + runtime.get("source_mode") == "live" + and runtime.get("producer_generation") == expected_runtime_generation + and isinstance(point_frames, int) + and not isinstance(point_frames, bool) + and point_frames > 0 + ): + return False if device_session_id is None: raise RuntimeError("для acquisition нет активной device-сессии") - target = self._camera_target_for_session(device_session_id) - camera = self.camera_preview.snapshot() - if camera.get("active_source_id") != DEFAULT_ACQUISITION_CAMERA_SOURCE: - self.camera_preview.select(DEFAULT_ACQUISITION_CAMERA_SOURCE, target) - self._arm_camera_recording(out_dir, require_session=True) + if acquisition.control_mode == "plugin-commanded": + physical = self._physical_command_coordinator.snapshot() + start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + local_start_operation_id=start_operation_id, + ) + ) + control = self._application_control_session.snapshot() + verified_control_value = control.get("verified_control") + verified_control = ( + cast(Mapping[str, Any], verified_control_value) + if isinstance(verified_control_value, Mapping) + else None + ) + if not ( + control.get("state") == "scanning" + and control.get("can_stop") is True + and self._matching_start_active_confirmed( + physical, + acquisition_id=acquisition.acquisition_id, + start_operation_id=start_operation_id, + verified_control=verified_control, + ) + and self._active_acquisition_checkpoint_matches_current( + acquisition_id=acquisition.acquisition_id, + start_operation_id=start_operation_id, + evidence_session_id=out_dir.name, + verified_control=verified_control, + ) + ): + return False + + existing = self.camera_preview.snapshot() + existing_recording = existing.get("recording") + if ( + existing.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and existing.get("phase") != "error" + and isinstance(existing_recording, Mapping) + and existing_recording.get("active") is True + and existing_recording.get("session") == out_dir.name + and existing_recording.get("active_epoch") is not None + ): + def bind_existing_camera() -> bool: + self._bind_live_perception_camera(out_dir.name, existing) + return True + + return self._commit_camera_restart_if_still_active( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + start_operation_id=start_operation_id, + runtime_producer_generation=expected_runtime_generation, + commit=bind_existing_camera, + ) + + camera_holder_acquired = self._ensure_camera_preview_process_lease() + try: + target = self._camera_target_for_session(device_session_id) + camera = self.camera_preview.snapshot() + if camera.get("active_source_id") not in { + None, + DEFAULT_ACQUISITION_CAMERA_SOURCE, + }: + self.camera_preview.select(DEFAULT_ACQUISITION_CAMERA_SOURCE, target) + camera = self.camera_preview.snapshot() + recording = camera.get("recording") + partial_same_session = _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=out_dir.name, + ) + if partial_same_session: + camera = self.camera_preview.retry_recording_producer( + DEFAULT_ACQUISITION_CAMERA_SOURCE, + target, + expected_generation=cast(int, camera["generation"]), + expected_recording_session=out_dir.name, + pre_retry_fence=lambda reserve: ( + self._reserve_camera_restart_if_still_active( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + start_operation_id=start_operation_id, + runtime_producer_generation=expected_runtime_generation, + reserve=reserve, + ) + ), + commit_fence=lambda commit: self._commit_camera_restart_if_still_active( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + start_operation_id=start_operation_id, + runtime_producer_generation=expected_runtime_generation, + commit=commit, + ), + committed_before_start=lambda committed: ( + self._bind_live_perception_camera(out_dir.name, committed) + ), + ) + else: + camera = self.camera_preview.activate_recording_producer( + DEFAULT_ACQUISITION_CAMERA_SOURCE, + target, + out_dir, + commit_fence=lambda commit: self._commit_camera_restart_if_still_active( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + start_operation_id=start_operation_id, + runtime_producer_generation=expected_runtime_generation, + commit=commit, + ), + committed_before_start=lambda committed: ( + self._bind_live_perception_camera(out_dir.name, committed) + ), + ) + except BaseException: + if ( + camera_holder_acquired + and self.camera_preview.snapshot().get("active_source_id") is None + ): + self._release_camera_preview_process_lease() + raise camera = self.camera_preview.snapshot() recording = camera.get("recording") if ( camera.get("active_source_id") != DEFAULT_ACQUISITION_CAMERA_SOURCE - or not isinstance(recording, dict) + or not isinstance(recording, Mapping) or recording.get("active") is not True + or recording.get("session") != out_dir.name + or recording.get("active_epoch") is None or camera.get("phase") == "error" ): raise RuntimeError("правая камера K1 не перешла в обязательную evidence-запись") logger.info( - "K1 right camera activated after confirmed SCANNING", + "K1 right camera activated after first authoritative PCL", extra={ "event_code": "k1_default_acquisition_camera_activated", "camera_source_id": DEFAULT_ACQUISITION_CAMERA_SOURCE, "evidence_session_id": out_dir.name, - "activation_trigger": "application-control-scanning", + "runtime_producer_generation": expected_runtime_generation, + "activation_trigger": "first-authoritative-pcl-published", + "device_command_sent": False, }, ) + return True + + def _physical_start_is_composite_active( + self, + physical: Mapping[str, Any], + *, + acquisition_id: str, + start_operation_id: str, + ) -> bool: + """Accept only the immutable, fully confirmed START lineage.""" + + record = physical.get("record") + if not isinstance(record, Mapping): + return False + response = record.get("application_response") + last_status = record.get("last_status") + return bool( + physical.get("status") == "resolved" + and physical.get("requires_reconciliation") is not True + and physical.get("resolved_active_recovery_required") is True + and record.get("operation_id") == start_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "start" + and record.get("stage") == "resolved" + and record.get("resolution") in {"start-active-observed", "physical-active-observed"} + and record.get("publish_call_returned") is True + and record.get("qos2_completed") is True + and isinstance(record.get("packet_id"), int) + and not isinstance(record.get("packet_id"), bool) + and isinstance(response, Mapping) + and response.get("operation_id") == start_operation_id + and response.get("action") == "start" + and response.get("success") is True + and isinstance(last_status, Mapping) + and last_status.get("session_state") == "scanning" + and last_status.get("project_bound") is True + and last_status.get("init_ready") is True + and last_status.get("mqtt_retained") is False + ) + + @staticmethod + def _exact_control_binding_document( + verified_control: Mapping[str, Any], + ) -> dict[str, object] | None: + """Return the complete generation-bound DeviceInfo control binding.""" + + connection_mode = verified_control.get("connection_mode") + values = { + "intent_id": verified_control.get("intent_id"), + "transport_ref": verified_control.get("transport_ref"), + "connection_mode": connection_mode, + "target_ipv4": verified_control.get("target_ipv4"), + "target_port": verified_control.get("target_port"), + "host_path_epoch": verified_control.get("host_path_epoch"), + "control_session_id": verified_control.get("control_session_id"), + "producer_generation": verified_control.get("producer_generation"), + } + if not ( + all( + isinstance(values[field], str) and bool(str(values[field]).strip()) + for field in ( + "intent_id", + "transport_ref", + "target_ipv4", + "control_session_id", + ) + ) + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and all( + isinstance(values[field], int) and not isinstance(values[field], bool) + for field in ( + "target_port", + "host_path_epoch", + "producer_generation", + ) + ) + ): + return None + return values + + def _physical_active_parent_proof( + self, + physical: Mapping[str, Any], + *, + acquisition_id: str, + parent_operation_id: str, + verified_control: Mapping[str, Any], + ) -> tuple[Mapping[str, Any], Mapping[str, Any]] | None: + """Prove the exact active parent and its current command binding. + + A normal same-session STOP uses the fully confirmed START record. A + recovered session additionally requires the latest immutable + reconciliation audit: its original-attempt digest, fresh DeviceInfo + binding and non-retained canonical SCANNING observation must all agree. + """ + + record = physical.get("record") + binding = self._exact_control_binding_document(verified_control) + if not isinstance(record, Mapping) or binding is None: + return None + if not ( + physical.get("status") == "resolved" + and physical.get("requires_reconciliation") is not True + and physical.get("resolved_active_recovery_required") is True + and record.get("operation_id") == parent_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("stage") == "resolved" + and isinstance(record.get("revision"), int) + and not isinstance(record.get("revision"), bool) + ): + return None + + reconciliations = record.get("reconciliations") + reconciliation_documents = reconciliations if isinstance(reconciliations, list) else [] + parent_reconciliation_documents = [ + item + for item in reconciliation_documents + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") == parent_operation_id + ] + latest_parent_reconciliation = ( + parent_reconciliation_documents[-1] + if parent_reconciliation_documents + else None + ) + latest_active = ( + latest_parent_reconciliation + if isinstance(latest_parent_reconciliation, Mapping) + and latest_parent_reconciliation.get("kind") + in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + } + and latest_parent_reconciliation.get("resolution") + == "physical-active-observed" + else None + ) + # Never revive an older active observation after a newer immutable + # reconciliation for the same command reported standby/cessation. + if latest_parent_reconciliation is not None and latest_active is None: + return None + if latest_active is not None: + original_attempt = latest_active.get("original_attempt") + original_attempt_sha256 = latest_active.get("original_attempt_sha256") + verified_binding = latest_active.get("verified_binding") + observation = latest_active.get("observation") + reconciled_connection = ( + verified_binding.get("connection") + if isinstance(verified_binding, Mapping) + else None + ) + if not ( + isinstance(original_attempt, Mapping) + and isinstance(original_attempt_sha256, str) + and hmac.compare_digest( + original_attempt_sha256, + _sha256_audit_document(original_attempt), + ) + and original_attempt.get("operation_id") == parent_operation_id + and original_attempt.get("acquisition_id") == acquisition_id + and original_attempt.get("action") in {"start", "stop"} + and isinstance(reconciled_connection, Mapping) + and dict(reconciled_connection) == binding + and isinstance(observation, Mapping) + and observation.get("source") == "explicit-read-only-reconciliation" + and observation.get("control_session_id") == binding["control_session_id"] + and observation.get("host_path_epoch") == binding["host_path_epoch"] + and observation.get("producer_generation") == binding["producer_generation"] + and observation.get("session_state") == "scanning" + and observation.get("project_bound") is True + and observation.get("init_ready") is True + and observation.get("mqtt_retained") is False + and isinstance(observation.get("project_id_sha256"), str) + ): + return None + return latest_active, binding + + record_connection = record.get("connection") + if not ( + record.get("action") == "start" + and self._physical_start_is_composite_active( + physical, + acquisition_id=acquisition_id, + start_operation_id=parent_operation_id, + ) + and isinstance(record_connection, Mapping) + and dict(record_connection) == binding + ): + return None + return record, binding + + def _capture_prepared_stop_dispatch_lineage( + self, + *, + operation_id: str, + acquisition: AcquisitionRecord, + ) -> _PreparedStopDispatchLineage | None: + """Freeze the proven parent immediately before durable STOP PREPARE.""" + + physical = dict(self._physical_command_coordinator.snapshot()) + control = dict(self._application_control_session.snapshot()) + runtime = dict(self.runtime.snapshot()) + verified_control = control.get("verified_control") + with self._lock: + local_start_operation_id = self._acquisition_start_operation_id + out_dir = self._acquisition_out_dir + selected_device_id = self._selected_device_id + device_id = self._device_id + device_session_id = self._device_session_id + connection_mode = self._connection_mode + target_ipv4 = self._k1_ip + if not ( + isinstance(verified_control, Mapping) + and isinstance(selected_device_id, str) + and isinstance(device_id, str) + and isinstance(device_session_id, str) + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(target_ipv4, str) + and control.get("state") == "scanning" + ): + return None + record = physical.get("record") + parent_operation_id = record.get("operation_id") if isinstance(record, Mapping) else None + local_start_matches_parent_chain = bool( + isinstance(record, Mapping) + and isinstance(parent_operation_id, str) + and ( + local_start_operation_id is None + or local_start_operation_id == parent_operation_id + or ( + record.get("action") == "stop" + and self._matching_classified_prepared_stop_active( + physical, + acquisition_id=acquisition.acquisition_id, + stop_operation_id=parent_operation_id, + start_operation_id=local_start_operation_id, + verified_control=verified_control, + ) + ) + ) + ) + if not isinstance(parent_operation_id, str) or not local_start_matches_parent_chain: + return None + active_parent = self._physical_active_parent_proof( + physical, + acquisition_id=acquisition.acquisition_id, + parent_operation_id=parent_operation_id, + verified_control=verified_control, + ) + if active_parent is None: + return None + active_proof, binding = active_parent + assert isinstance(record, Mapping) + identity = record.get("identity") + reconciliations = record.get("reconciliations") + compatibility_profile_id = record.get("compatibility_profile_id") + session_generation = control.get("session_generation") + state_revision = control.get("state_revision") + runtime_generation = runtime.get("producer_generation") + if not ( + isinstance(identity, Mapping) + and isinstance(reconciliations, list) + and compatibility_profile_id == XGRIDS_K1_COMPATIBILITY_PROFILE_ID + and isinstance(session_generation, int) + and not isinstance(session_generation, bool) + and isinstance(state_revision, int) + and not isinstance(state_revision, bool) + and ( + runtime_generation is None + or ( + isinstance(runtime_generation, int) + and not isinstance(runtime_generation, bool) + ) + ) + ): + return None + return _PreparedStopDispatchLineage( + snapshot_runtime_id=self._snapshot_runtime_id, + stop_operation_id=operation_id, + acquisition_id=acquisition.acquisition_id, + parent_physical_operation_id=parent_operation_id, + local_start_operation_id=local_start_operation_id, + parent_record_revision=int(record["revision"]), + parent_reconciliation_id=( + str(active_proof["reconciliation_id"]) + if isinstance(active_proof.get("reconciliation_id"), str) + else None + ), + parent_identity_sha256=_sha256_audit_document(identity), + parent_reconciliations_sha256=_sha256_audit_document( + { + "reconciliations": [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") + == parent_operation_id + ] + } + ), + compatibility_profile_id=compatibility_profile_id, + control_binding_sha256=_sha256_audit_document(binding), + control_session_generation=session_generation, + minimum_control_state_revision=state_revision + 1, + device_id=device_id, + device_session_id=device_session_id, + evidence_session_id=out_dir.name if out_dir is not None else None, + runtime_source_mode=str(runtime.get("source_mode") or "unknown"), + runtime_producer_generation=( + int(runtime_generation) if isinstance(runtime_generation, int) else None + ), + intent_id=str(binding["intent_id"]), + transport_ref=str(binding["transport_ref"]), + connection_mode=cast(ConnectionMode, binding["connection_mode"]), + target_ipv4=str(binding["target_ipv4"]), + target_port=int(binding["target_port"]), + host_path_epoch=int(binding["host_path_epoch"]), + control_session_id=str(binding["control_session_id"]), + control_producer_generation=int(binding["producer_generation"]), + ) + + def _prepared_stop_dispatch_lineage_is_current( + self, + *, + physical: Mapping[str, Any] | None = None, + control: Mapping[str, Any] | None = None, + ) -> bool: + """Retain topology only for the exact undispatched STOP handoff.""" + + with self._lock: + lineage = getattr(self, "_prepared_stop_dispatch_lineage", None) + acquisition = self._acquisition + stop_operation_id = self._acquisition_stop_operation_id + out_dir = self._acquisition_out_dir + selected_device_id = self._selected_device_id + device_id = self._device_id + device_session_id = self._device_session_id + connection_mode = self._connection_mode + target_ipv4 = self._k1_ip + if lineage is None: + return False + try: + operation = self._operations.get(lineage.stop_operation_id) + except KeyError: + return False + runtime = self.runtime.snapshot() + physical_snapshot = dict(physical or self._physical_command_coordinator.snapshot()) + control_snapshot = dict(control or self._application_control_session.snapshot()) + verified_control = control_snapshot.get("verified_control") + binding = ( + self._exact_control_binding_document(verified_control) + if isinstance(verified_control, Mapping) + else None + ) + record = physical_snapshot.get("record") + if not isinstance(record, Mapping) or binding is None: + return False + identity = record.get("identity") + reconciliations = record.get("reconciliations") + baseline = record.get("baseline_status") + recovery_result = acquisition.result if acquisition is not None else None + terminal_recovery_only = bool( + acquisition is not None + and acquisition.state == "failed" + and acquisition.message_code == "acquisition.physical_active_recovered_stop_only" + and isinstance(recovery_result, Mapping) + and recovery_result.get("recovery_only") is True + and recovery_result.get("receiver_stopped") is True + and recovery_result.get("device_state") == "scanning" + and recovery_result.get("physical_command_operation_id") + == lineage.parent_physical_operation_id + and recovery_result.get("physical_reconciliation_id") + == lineage.parent_reconciliation_id + and self._acquisition_session_lease is None + and out_dir is None + ) + terminal_operation_statuses = { + "succeeded", + "failed", + "cancelled", + "timed_out", + "interrupted", + } + return bool( + self._snapshot_runtime_id == lineage.snapshot_runtime_id + and operation.operation_id == lineage.stop_operation_id + and operation.action == ACTION_ACQUISITION_STOP + and operation.status not in terminal_operation_statuses + and operation.cancel_requested is False + and not self._operations.deadline_reached(operation.operation_id) + and acquisition is not None + and acquisition.acquisition_id == lineage.acquisition_id + and acquisition.device_id == lineage.device_id + and acquisition.device_session_id == lineage.device_session_id + and ( + acquisition.state not in TERMINAL_ACQUISITION_STATES + or terminal_recovery_only + ) + and stop_operation_id == lineage.stop_operation_id + and selected_device_id == lineage.transport_ref + and device_id == lineage.device_id + and device_session_id == lineage.device_session_id + and connection_mode == lineage.connection_mode + and target_ipv4 == lineage.target_ipv4 + and (out_dir.name if out_dir is not None else None) == lineage.evidence_session_id + and runtime.get("source_mode") == lineage.runtime_source_mode + and runtime.get("producer_generation") == lineage.runtime_producer_generation + and control_snapshot.get("state") in {"stop-requested", "stopping"} + and control_snapshot.get("session_generation") == lineage.control_session_generation + and isinstance(control_snapshot.get("state_revision"), int) + and control_snapshot["state_revision"] >= lineage.minimum_control_state_revision + and _sha256_audit_document(binding) == lineage.control_binding_sha256 + and physical_snapshot.get("status") == "unresolved" + and physical_snapshot.get("requires_reconciliation") is True + and physical_snapshot.get("active_operation_id") == lineage.stop_operation_id + and record.get("operation_id") == lineage.stop_operation_id + and record.get("parent_operation_id") == lineage.parent_physical_operation_id + and record.get("revision") == lineage.parent_record_revision + 1 + and record.get("acquisition_id") == lineage.acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "prepared" + and record.get("resolution") is None + and record.get("publish_call_returned") is None + and record.get("packet_id") is None + and record.get("qos2_completed") is False + and record.get("application_response") is None + and record.get("last_status") is None + and isinstance(identity, Mapping) + and _sha256_audit_document(identity) == lineage.parent_identity_sha256 + and isinstance(reconciliations, list) + and _sha256_audit_document( + { + "reconciliations": [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") + == lineage.parent_physical_operation_id + ] + } + ) + == lineage.parent_reconciliations_sha256 + and record.get("compatibility_profile_id") == lineage.compatibility_profile_id + and isinstance(record.get("connection"), Mapping) + and dict(cast(Mapping[str, object], record["connection"])) == binding + and isinstance(baseline, Mapping) + and baseline.get("control_session_id") == lineage.control_session_id + and baseline.get("host_path_epoch") == lineage.host_path_epoch + and baseline.get("producer_generation") == lineage.control_producer_generation + and baseline.get("session_state") == "scanning" + and baseline.get("project_bound") is True + and baseline.get("init_ready") is True + and baseline.get("mqtt_retained") is False + ) + + def _admit_active_stream_recovery_lineage( + self, + *, + runtime: Mapping[str, Any] | None = None, + physical: Mapping[str, Any] | None = None, + allow_runtime_live: bool = False, + ) -> _ActiveStreamRecoveryLineage | None: + """Freeze the exact local/runtime/device/physical owner once per loss.""" + + runtime_snapshot = dict(runtime or self.runtime.snapshot()) + physical_snapshot = dict(physical or self._physical_command_coordinator.snapshot()) + supervisor = self._connection_supervisor.snapshot() + with self._lock: + current = self._active_stream_recovery_lineage + acquisition = self._acquisition + start_operation_id = self._acquisition_start_operation_id + out_dir = self._acquisition_out_dir + session_lease = self._acquisition_session_lease + selected_device_id = self._selected_device_id + device_id = self._device_id + device_session_id = self._device_session_id + connection_mode = self._connection_mode + target_ipv4 = self._k1_ip + recovery_generation = self._active_stream_recovery_generation + snapshot_runtime_id = self._snapshot_runtime_id + restart_pending = self._restart_rehydration_pending + + restart_pending_owner = bool( + restart_pending is not None + and restart_pending.phase == "awaiting-first-pcl" + and acquisition is not None + and acquisition.acquisition_id == restart_pending.token.acquisition_id + and out_dir == restart_pending.out_dir + and session_lease is not None + and restart_pending.new_evidence_session_id == out_dir.name + and restart_pending.runtime_producer_generation + == runtime_snapshot.get("producer_generation") + and restart_pending.physical_proof.operation_id + == ( + physical_snapshot.get("record", {}).get("operation_id") + if isinstance(physical_snapshot.get("record"), Mapping) + else None + ) + and restart_pending.physical_proof.ledger_revision + == ( + physical_snapshot.get("record", {}).get("revision") + if isinstance(physical_snapshot.get("record"), Mapping) + else None + ) + ) + restart_pending_reconciliations = ( + physical_snapshot["record"].get("reconciliations") + if restart_pending_owner + and isinstance(physical_snapshot.get("record"), Mapping) + else None + ) + restart_pending_reconciliation = ( + restart_pending_reconciliations[-1] + if isinstance(restart_pending_reconciliations, list) + and restart_pending_reconciliations + else None + ) + restart_pending_owner = bool( + restart_pending_owner + and restart_pending is not None + and isinstance(restart_pending_reconciliation, Mapping) + and restart_pending_reconciliation.get("reconciliation_id") + == restart_pending.reconciliation_id + and restart_pending_reconciliation.get("resolution") + == "physical-active-observed" + and restart_pending_reconciliation.get("observation") + == { + "source": restart_pending.status_proof.source, + "vendor_device_id_sha256": ( + restart_pending.status_proof.binding.vendor_device_id_sha256 + ), + "device_serial_sha256": ( + restart_pending.status_proof.binding.device_serial_sha256 + ), + "control_session_id": ( + restart_pending.status_proof.binding.control_session_id + ), + "host_path_epoch": restart_pending.status_proof.binding.host_path_epoch, + "producer_generation": ( + restart_pending.status_proof.binding.producer_generation + ), + "session_state": restart_pending.status_proof.session_state, + "session_state_code": restart_pending.status_proof.session_state_code, + "project_bound": True, + "project_id_sha256": restart_pending.status_proof.project_id_sha256, + "init_ready": restart_pending.status_proof.init_ready, + "status_message_sha256": ( + restart_pending.status_proof.status_message_sha256 + ), + "mqtt_retained": restart_pending.status_proof.mqtt_retained, + "observed_at_utc": restart_pending.status_proof.observed_at_utc, + } + and isinstance( + restart_pending_reconciliation.get("verified_binding"), + Mapping, + ) + and restart_pending_reconciliation["verified_binding"].get("connection") + == { + "intent_id": restart_pending.status_proof.binding.intent_id, + "transport_ref": restart_pending.status_proof.binding.transport_ref, + "connection_mode": restart_pending.status_proof.binding.connection_mode, + "target_ipv4": restart_pending.status_proof.binding.target_ipv4, + "target_port": restart_pending.status_proof.binding.target_port, + "host_path_epoch": restart_pending.status_proof.binding.host_path_epoch, + "control_session_id": ( + restart_pending.status_proof.binding.control_session_id + ), + "producer_generation": ( + restart_pending.status_proof.binding.producer_generation + ), + } + ) + if restart_pending_owner: + assert restart_pending is not None + start_operation_id = restart_pending.token.root_start_operation_id + elif ( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and out_dir is not None + ): + start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + local_start_operation_id=start_operation_id, + ) + ) + + runtime_generation = runtime_snapshot.get("producer_generation") + physical_record = physical_snapshot.get("record") + physical_connection = ( + physical_record.get("connection") if isinstance(physical_record, Mapping) else None + ) + # A restart-pending receiver descends from the newest explicit + # read-only reconciliation, while ``record.connection`` intentionally + # remains the immutable route of the original physical START. Use + # only the frozen, revision-bound reconciliation binding for this + # narrow owner; otherwise a second link loss before the first PCL can + # never be admitted after the host/control epoch changes. + physical_owner_connection: Mapping[str, object] | None = ( + restart_pending.status_proof.binding.as_dict() + if restart_pending_owner and restart_pending is not None + else physical_connection + if isinstance(physical_connection, Mapping) + else None + ) + with self._lock: + prepared_stop_owner = getattr(self, "_prepared_stop_recovery_owner", None) + intent = supervisor.intent + endpoint = supervisor.device_network.target + validated_target_ipv4 = ( + validate_private_ipv4(target_ipv4) + if isinstance(target_ipv4, str) + else None + ) + normal_start_owner = bool( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and acquisition.state in {"starting", "awaiting_external_start", "acquiring"} + and isinstance(start_operation_id, str) + and bool(start_operation_id) + and out_dir is not None + and session_lease is not None + and isinstance(selected_device_id, str) + and isinstance(device_id, str) + and isinstance(device_session_id, str) + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(target_ipv4, str) + and runtime_snapshot.get("source_mode") == "live" + and ( + runtime_snapshot.get("phase") == "reconnecting" + or (allow_runtime_live and runtime_snapshot.get("phase") == "live") + ) + and isinstance(runtime_generation, int) + and not isinstance(runtime_generation, bool) + and runtime_generation >= 1 + and intent is not None + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == intent.intent_id + and supervisor.device_network.transport_ref == selected_device_id + and supervisor.device_network.connection_mode == connection_mode + and validated_target_ipv4 is not None + and endpoint == EndpointTarget(validated_target_ipv4, CONTROL_MQTT_PORT) + and isinstance(physical_record, Mapping) + and physical_owner_connection is not None + and isinstance(physical_record.get("revision"), int) + and physical_owner_connection.get("intent_id") == intent.intent_id + and physical_owner_connection.get("transport_ref") == selected_device_id + and physical_owner_connection.get("connection_mode") == connection_mode + and physical_owner_connection.get("target_ipv4") == target_ipv4 + and physical_owner_connection.get("target_port") == CONTROL_MQTT_PORT + and ( + restart_pending_owner + or self._physical_start_is_composite_active( + physical_snapshot, + acquisition_id=acquisition.acquisition_id, + start_operation_id=start_operation_id, + ) + ) + ) + prepared_stop_owner_current = bool( + prepared_stop_owner is not None + and acquisition is prepared_stop_owner.acquisition + and acquisition.state + in {"awaiting_external_stop", "awaiting_external_start", "acquiring"} + and out_dir == prepared_stop_owner.out_dir + and session_lease is prepared_stop_owner.session_lease + and selected_device_id == prepared_stop_owner.lineage.transport_ref + and isinstance(device_id, str) + and isinstance(device_session_id, str) + and connection_mode == prepared_stop_owner.lineage.connection_mode + and target_ipv4 == prepared_stop_owner.lineage.target_ipv4 + and runtime_generation == prepared_stop_owner.runtime_producer_generation + and intent is not None + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == intent.intent_id + and supervisor.device_network.transport_ref == selected_device_id + and supervisor.device_network.connection_mode == connection_mode + and validated_target_ipv4 is not None + and endpoint == EndpointTarget(validated_target_ipv4, CONTROL_MQTT_PORT) + and self._prepared_stop_recovery_owner_is_current( + prepared_stop_owner, + allow_resolved_classification=True, + ) + ) + if not (normal_start_owner or prepared_stop_owner_current): + return None + + assert isinstance(physical_record, Mapping) + physical_operation_id = physical_record.get("operation_id") + physical_revision = physical_record.get("revision") + assert isinstance(physical_operation_id, str) + assert isinstance(physical_revision, int) + assert acquisition is not None + assert isinstance(device_id, str) + assert isinstance(device_session_id, str) + assert out_dir is not None + assert isinstance(runtime_generation, int) + assert intent is not None + assert isinstance(selected_device_id, str) + assert connection_mode in {"bridge", "quick-connect", "direct-connect"} + assert isinstance(target_ipv4, str) + if prepared_stop_owner_current: + assert prepared_stop_owner is not None + start_operation_id = ( + prepared_stop_owner.start_operation_id + or prepared_stop_owner.lineage.parent_physical_operation_id + ) + assert isinstance(start_operation_id, str) + proposed = _ActiveStreamRecoveryLineage( + recovery_generation=recovery_generation, + snapshot_runtime_id=snapshot_runtime_id, + acquisition_id=acquisition.acquisition_id, + device_id=device_id, + device_session_id=device_session_id, + evidence_session_id=out_dir.name, + runtime_producer_generation=runtime_generation, + start_operation_id=start_operation_id, + physical_operation_id=physical_operation_id, + physical_revision=physical_revision, + intent_id=intent.intent_id, + transport_ref=selected_device_id, + connection_mode=connection_mode, + target_ipv4=target_ipv4, + target_port=CONTROL_MQTT_PORT, + ) + if current is not None: + with self._lock: + prior_recovered = self._active_stream_recovery_state == "recovered" + if prior_recovered and ( + runtime_snapshot.get("phase") == "reconnecting" + or (allow_runtime_live and runtime_snapshot.get("phase") == "live") + ): + with self._lock: + if self._active_stream_recovery_lineage == current: + self._active_stream_recovery_lineage = None + current = None + if current is None: + return self._admit_active_stream_recovery_lineage( + runtime=runtime_snapshot, + physical=physical_snapshot, + allow_runtime_live=allow_runtime_live, + ) + # The coordinator appends a read-only reconciliation audit before + # MQTT has necessarily completed its SUBACK. A failed first + # resubscribe therefore sees a newer physical revision on attempt + # two. Preserve the frozen owner while allowing only a monotonic + # revision of the same operation/acquisition/binding lineage. + same_immutable_lineage = ( + dataclass_replace( + proposed, + physical_revision=current.physical_revision, + ) + == current + ) + return ( + current + if same_immutable_lineage + and proposed.physical_revision >= current.physical_revision + else None + ) + + # Checkpoint v4 uses the physical-ledger canonical millisecond wire + # format. The generic facade timestamp retains microseconds and is not + # a valid recovery-transition value. + gap_started_at_utc = _utc_now_millisecond_iso() + with self._lock: + if self._active_stream_recovery_lineage is not None: + return self._active_stream_recovery_lineage + self._active_stream_recovery_generation += 1 + proposed = dataclass_replace( + proposed, + recovery_generation=self._active_stream_recovery_generation, + ) + self._active_stream_recovery_lineage = proposed + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = "host-route-recovery-pending" + self._active_stream_recovery_started_at_utc = gap_started_at_utc + self._active_stream_recovery_started_monotonic = time.monotonic() + self._active_stream_recovery_terminal_outcome = None + if restart_pending_owner or self._record_active_acquisition_checkpoint_gap( + proposed, + gap_started_at_utc=gap_started_at_utc, + ): + return proposed + with self._lock: + if self._active_stream_recovery_lineage == proposed: + self._active_stream_recovery_lineage = None + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = ( + "active-acquisition-checkpoint-gap-failed" + ) + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None + return None + + def _record_active_acquisition_checkpoint_gap( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + gap_started_at_utc: str, + ) -> bool: + """CAS one first loss against the exact active local binding.""" + + store = self._active_acquisition_checkpoint + if store is None: + return False + try: + snapshot = store.snapshot() + checkpoint = snapshot.checkpoint + if not ( + snapshot.status == "active" + and checkpoint is not None + and checkpoint.acquisition_id == lineage.acquisition_id + and checkpoint.original_start_operation_id + == lineage.start_operation_id + and checkpoint.current_evidence_session_id + == lineage.evidence_session_id + and checkpoint.current_binding.runtime_instance_id + == lineage.snapshot_runtime_id + and checkpoint.current_binding.transport_ref + == lineage.transport_ref + and checkpoint.current_binding.connection_mode + == lineage.connection_mode + and checkpoint.current_binding.target_ipv4 + == lineage.target_ipv4 + and checkpoint.current_binding.target_port == lineage.target_port + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "recovery loss does not descend from active checkpoint" + ) + if ( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is None + ): + if checkpoint.last_gap_failed_binding != checkpoint.current_binding: + raise ActiveAcquisitionRecoveryCheckpointError( + "open checkpoint gap changed its failed binding" + ) + return True + store.record_gap( + transition_id=self._active_acquisition_checkpoint_transition_id( + "record-gap", + lineage.acquisition_id, + lineage.recovery_generation, + checkpoint.transport_revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + failed_binding=checkpoint.current_binding, + gap_started_at_utc=gap_started_at_utc, + ) + self._set_active_acquisition_checkpoint_reason(None) + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason_code = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-gap-failed", + ) + ) + self._set_active_acquisition_checkpoint_reason(reason_code) + logger.error( + "active acquisition checkpoint gap failed closed", + extra={ + "event_code": "active_acquisition_checkpoint_gap_failed", + "reason_code": reason_code, + "automatic_command_retry": False, + }, + exc_info=exc, + ) + return False + + def _confirm_active_acquisition_checkpoint_rebind( + self, + envelope: DecodedPointCloudView, + producer_generation: int, + attempt: int, + minimum_sequence: int, + ) -> bool: + """Commit checkpoint rebind after Rerun publish, before capture/live.""" + + if ( + not envelope.context.live + or envelope.point_count < 1 + or envelope.context.sequence < minimum_sequence + ): + return False + with self._lock: + lineage = self._active_stream_recovery_lineage + recovery_attempt = self._active_stream_recovery_attempt + out_dir = self._acquisition_out_dir + if not ( + lineage is not None + and lineage.runtime_producer_generation == producer_generation + and recovery_attempt == attempt + and out_dir is not None + and out_dir.name == lineage.evidence_session_id + and self._active_stream_recovery_lineage_is_current(lineage) + ): + return False + with self._lock: + restart_pending = self._restart_rehydration_pending + if ( + restart_pending is not None + and restart_pending.phase == "awaiting-first-pcl" + and restart_pending.token.acquisition_id == lineage.acquisition_id + and restart_pending.token.root_start_operation_id + == lineage.start_operation_id + and restart_pending.out_dir == out_dir + and restart_pending.runtime_producer_generation == producer_generation + ): + return self._refresh_restart_pending_after_stream_reconciliation( + pending=restart_pending, + lineage=lineage, + attempt=attempt, + ) + store = self._active_acquisition_checkpoint + if store is None: + return False + try: + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + if not ( + checkpoint_snapshot.status == "active" + and checkpoint is not None + and checkpoint.acquisition_id == lineage.acquisition_id + and checkpoint.original_start_operation_id + == lineage.start_operation_id + and checkpoint.current_evidence_session_id + == lineage.evidence_session_id + and checkpoint.current_binding.runtime_instance_id + == lineage.snapshot_runtime_id + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "recovery promotion lacks exact active checkpoint" + ) + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + expected_reconciliation_id = _active_stream_reconciliation_id( + lineage, + attempt=attempt, + ) + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.operation_id == lineage.physical_operation_id + and record.acquisition_id == lineage.acquisition_id + and reconciliation is not None + and reconciliation.reconciliation_id + == expected_reconciliation_id + and reconciliation.kind + in {"prepared-stop-classification", "resolved-active-rebind"} + and reconciliation.resolution == "physical-active-observed" + and reconciliation.observation.session_state == "scanning" + and reconciliation.observation.mqtt_retained is False + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "recovery PCL lacks the exact fresh physical reconciliation" + ) + control = self._application_control_session.snapshot() + if control.get("state") != "scanning": + raise ActiveAcquisitionRecoveryCheckpointError( + "recovery control has not adopted SCANNING" + ) + verified_control = self._checkpoint_verified_control(control) + identity = reconciliation.verified_binding.identity + connection = reconciliation.verified_binding.connection + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=verified_control, + ) + # Same-process recovery reuses the one open capture writer. The + # standalone store permits a successor evidence session for future + # cross-process work, but this facade seam deliberately does not. + status_proof = self._checkpoint_status_proof( + status=reconciliation.observation, + binding=binding, + evidence_session_id=lineage.evidence_session_id, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + if ( + checkpoint.last_gap_started_at_utc is not None + and checkpoint.last_gap_recovered_at_utc is not None + and checkpoint.current_binding == binding + and checkpoint.current_evidence_session_id + == lineage.evidence_session_id + and checkpoint.last_rebind_physical_proof == physical_proof + ): + return True + store.rebind_active( + transition_id=self._active_acquisition_checkpoint_transition_id( + "rebind", + reconciliation.reconciliation_id, + record.revision, + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + status_proof=status_proof, + physical_proof=physical_proof, + ) + self._set_active_acquisition_checkpoint_reason(None) + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason_code = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-rebind-failed", + ) + ) + self._set_active_acquisition_checkpoint_reason(reason_code) + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = reason_code + logger.error( + "active acquisition checkpoint rebind failed closed", + extra={ + "event_code": "active_acquisition_checkpoint_rebind_failed", + "reason_code": reason_code, + "automatic_command_retry": False, + }, + exc_info=exc, + ) + return False + + def _refresh_restart_pending_after_stream_reconciliation( + self, + *, + pending: _RestartRehydrationPending, + lineage: _ActiveStreamRecoveryLineage, + attempt: int, + ) -> bool: + """Rotate only the frozen read-only proof after pre-PCL link loss.""" + + store = self._active_acquisition_checkpoint + if store is None: + return False + try: + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + expected_reconciliation_id = _active_stream_reconciliation_id( + lineage, + attempt=attempt, + ) + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + if not ( + checkpoint is not None + and checkpoint_snapshot.status == pending.token.checkpoint_state + and checkpoint.revision == pending.token.checkpoint_revision + and checkpoint.acquisition_id == pending.token.acquisition_id + and checkpoint.original_start_operation_id + == pending.token.root_start_operation_id + and checkpoint.current_evidence_session_id + == pending.old_evidence_session_id + and ledger_snapshot.status == "resolved" + and record is not None + and record.operation_id == pending.physical_proof.operation_id + and record.acquisition_id == pending.token.acquisition_id + and reconciliation is not None + and reconciliation.reconciliation_id == expected_reconciliation_id + and reconciliation.kind + in {"ambiguous-outcome", "resolved-active-rebind"} + and reconciliation.resolution == "physical-active-observed" + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and reconciliation.observation.session_state == "scanning" + and reconciliation.observation.project_bound + and reconciliation.observation.init_ready + and not reconciliation.observation.mqtt_retained + ): + return False + control = self._application_control_session.snapshot() + if control.get("state") != "scanning" or control.get("can_stop") is not True: + return False + identity = reconciliation.verified_binding.identity + connection = reconciliation.verified_binding.connection + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=self._checkpoint_verified_control(control), + ) + status_proof = self._checkpoint_status_proof( + status=reconciliation.observation, + binding=binding, + evidence_session_id=pending.new_evidence_session_id, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ) + origin_proof = ( + self._checkpoint_reconciled_start_origin_proof( + checkpoint=checkpoint, + record=record, + reconciliation=reconciliation, + physical_proof=physical_proof, + ) + if checkpoint.state == "prepared" + else None + ) + refreshed = dataclass_replace( + pending, + reconciliation_id=reconciliation.reconciliation_id, + status_proof=status_proof, + physical_proof=physical_proof, + origin_proof=origin_proof, + transition_id=self._active_acquisition_checkpoint_transition_id( + ( + "restart-activate-reconciled" + if checkpoint.state == "prepared" + else "restart-rebind" + ), + checkpoint.acquisition_id, + reconciliation.reconciliation_id, + pending.new_evidence_session_id, + checkpoint.revision, + ), + first_pcl_proof=None, + reason_code=None, + ) + with self._lock: + if ( + self._restart_rehydration_pending != pending + or self._active_stream_recovery_lineage != lineage + or self._camera_stop_priority_counts.get( + pending.token.acquisition_id, + 0, + ) + > 0 + ): + return False + self._restart_rehydration_pending = refreshed + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError): + return False + + def _cease_active_acquisition_checkpoint_from_physical_head( + self, + *, + expected_acquisition_id: str, + expected_start_operation_id: str, + evidence_session_id: str, + expected_stop_operation_id: str | None = None, + require_reconciliation: bool = False, + ) -> bool: + """Cease before terminal local teardown, without inventing STOP.""" + + store = self._active_acquisition_checkpoint + if store is None: + return False + try: + checkpoint_snapshot = store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + if checkpoint is not None and checkpoint.state == "ceased": + return bool( + checkpoint.acquisition_id == expected_acquisition_id + and checkpoint.original_start_operation_id + == expected_start_operation_id + ) + if not ( + checkpoint_snapshot.status in {"prepared", "active"} + and checkpoint is not None + and checkpoint.acquisition_id == expected_acquisition_id + and checkpoint.original_start_operation_id + == expected_start_operation_id + and isinstance(evidence_session_id, str) + and bool(evidence_session_id) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "terminal proof does not own the active checkpoint" + ) + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + reconciliation = ( + record.reconciliations[-1] + if record is not None and record.reconciliations + else None + ) + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.acquisition_id == expected_acquisition_id + and ( + expected_stop_operation_id is None + or record.operation_id == expected_stop_operation_id + ) + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "terminal checkpoint proof lacks exact physical head" + ) + use_reconciliation = bool( + reconciliation is not None + and reconciliation.kind + in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-cessation", + } + and reconciliation.resolution == "physical-standby-observed" + ) + if require_reconciliation and not use_reconciliation: + raise ActiveAcquisitionRecoveryCheckpointError( + "terminal recovery lacks fresh READY/SCAN_OVER reconciliation" + ) + status = ( + reconciliation.observation + if use_reconciliation and reconciliation is not None + else record.last_status + ) + identity = ( + reconciliation.verified_binding.identity + if use_reconciliation and reconciliation is not None + else record.identity + ) + connection = ( + reconciliation.verified_binding.connection + if use_reconciliation and reconciliation is not None + else record.connection + ) + if status is None or status.session_state not in {"ready", "scan_over"}: + raise ActiveAcquisitionRecoveryCheckpointError( + "terminal checkpoint proof is not READY/SCAN_OVER" + ) + verified_control = self._checkpoint_verified_control( + self._application_control_session.snapshot() + ) + binding = self._checkpoint_transport_binding( + identity_vendor_sha256=identity.vendor_device_id_sha256, + identity_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=record.compatibility_profile_id, + connection=connection.as_dict(), + verified_control=verified_control, + ) + status_proof = self._checkpoint_status_proof( + status=status, + binding=binding, + # Same-process recovery keeps the exact open capture writer. + evidence_session_id=evidence_session_id, + ) + physical_proof = self._checkpoint_physical_proof( + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=(reconciliation if use_reconciliation else None), + ) + if checkpoint.state == "prepared": + with self._lock: + pending = ( + self._restart_rehydration_pending + or self._restart_stop_only_prepared_pending + ) + if not ( + pending is not None + and pending.token.acquisition_id == checkpoint.acquisition_id + and pending.token.root_start_operation_id + == checkpoint.original_start_operation_id + and pending.new_evidence_session_id == evidence_session_id + and pending.origin_proof is not None + ): + raise ActiveAcquisitionRecoveryCheckpointError( + "pre-PCL STOP lacks its immutable restart origin" + ) + store.cease_prepared_reconciled( + transition_id=self._active_acquisition_checkpoint_transition_id( + "cease-prepared-reconciled", + record.operation_id, + record.revision, + reconciliation.reconciliation_id + if use_reconciliation and reconciliation is not None + else "normal", + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + gap_started_at_utc=pending.gap_started_at_utc, + origin_proof=pending.origin_proof, + active_status_proof=pending.status_proof, + active_physical_proof=pending.physical_proof, + cessation_status_proof=status_proof, + cessation_physical_proof=physical_proof, + ) + with self._lock: + if self._restart_rehydration_pending == pending: + self._restart_rehydration_pending = None + if self._restart_stop_only_prepared_pending == pending: + self._restart_stop_only_prepared_pending = None + else: + store.cease( + transition_id=self._active_acquisition_checkpoint_transition_id( + "cease", + record.operation_id, + record.revision, + reconciliation.reconciliation_id + if use_reconciliation and reconciliation is not None + else "normal", + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + physical_proof=physical_proof, + status_proof=status_proof, + ) + with self._lock: + pending = self._restart_rehydration_pending + if ( + pending is not None + and pending.token.acquisition_id == checkpoint.acquisition_id + ): + self._restart_rehydration_pending = None + self._set_active_acquisition_checkpoint_reason(None) + return True + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc: + reason_code = str( + getattr( + exc, + "reason_code", + "active-acquisition-recovery-checkpoint-cease-failed", + ) + ) + self._set_active_acquisition_checkpoint_reason(reason_code) + logger.error( + "active acquisition checkpoint cessation failed closed", + extra={ + "event_code": "active_acquisition_checkpoint_cease_failed", + "reason_code": reason_code, + "physical_command_sent": False, + }, + exc_info=exc, + ) + return False + + def _active_stream_recovery_lineage_is_current( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + require_runtime_reconnecting: bool = True, + ) -> bool: + runtime = self.runtime.snapshot() + physical = self._physical_command_coordinator.snapshot() + supervisor = self._connection_supervisor.snapshot() + record = physical.get("record") + control = self._application_control_session.snapshot() + verified_control = control.get("verified_control") + classified_prepared_stop_active = self._matching_classified_prepared_stop_active( + physical, + acquisition_id=lineage.acquisition_id, + stop_operation_id=lineage.physical_operation_id, + start_operation_id=lineage.start_operation_id, + verified_control=( + verified_control if isinstance(verified_control, Mapping) else None + ), + ) + normal_start_active = bool( + isinstance(record, Mapping) + and record.get("operation_id") == lineage.physical_operation_id + and record.get("acquisition_id") == lineage.acquisition_id + and record.get("action") == "start" + and record.get("stage") == "resolved" + and physical.get("resolved_active_recovery_required") is True + ) + with self._lock: + prepared_stop_owner = getattr(self, "_prepared_stop_recovery_owner", None) + prepared_stop_terminal_none = bool( + prepared_stop_owner is not None + and prepared_stop_owner.lineage.stop_operation_id == lineage.physical_operation_id + and prepared_stop_owner.start_operation_id == lineage.start_operation_id + and self._prepared_stop_recovery_owner_is_current(prepared_stop_owner) + ) + exact_physical_owner = bool( + normal_start_active + or classified_prepared_stop_active + or prepared_stop_terminal_none + ) + with self._lock: + acquisition = self._acquisition + return bool( + self._active_stream_recovery_lineage == lineage + and self._active_stream_recovery_generation == lineage.recovery_generation + and self._snapshot_runtime_id == lineage.snapshot_runtime_id + and acquisition is not None + and acquisition.acquisition_id == lineage.acquisition_id + and acquisition.device_id == lineage.device_id + and acquisition.device_session_id == lineage.device_session_id + and acquisition.state + in { + "starting", + "awaiting_external_start", + "awaiting_external_stop", + "acquiring", + } + and self._acquisition_start_operation_id in {None, lineage.start_operation_id} + and self._acquisition_session_lease is not None + and self._acquisition_out_dir is not None + and self._acquisition_out_dir.name == lineage.evidence_session_id + and self._selected_device_id == lineage.transport_ref + and self._device_id == lineage.device_id + and self._device_session_id == lineage.device_session_id + and self._connection_mode == lineage.connection_mode + and self._k1_ip == lineage.target_ipv4 + and runtime.get("source_mode") == "live" + and runtime.get("producer_generation") == lineage.runtime_producer_generation + and (not require_runtime_reconnecting or runtime.get("phase") == "reconnecting") + and supervisor.intent is not None + and supervisor.intent.intent_id == lineage.intent_id + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == lineage.intent_id + and supervisor.device_network.transport_ref == lineage.transport_ref + and supervisor.device_network.connection_mode == lineage.connection_mode + and supervisor.device_network.target + == EndpointTarget(lineage.target_ipv4, lineage.target_port) + and exact_physical_owner + ) + + def _active_stream_recovery_admitted_for_runtime_loss(self) -> bool: + """Pure local guard used to suppress destructive one-shot teardown.""" + + with self._acquisition_lifecycle_access(): + runtime = self.runtime.snapshot() + if runtime.get("phase") != "reconnecting": + return False + existing = self._active_stream_recovery_lineage + if existing is not None: + return self._active_stream_recovery_lineage_is_current(existing) + return self._admit_active_stream_recovery_lineage(runtime=runtime) is not None + + def _request_active_stream_recovery_for_local_loss(self, reason_code: str) -> bool: + """Freeze an exact active owner and wake its existing capture thread. + + A short host-path outage can break the control monitor or FFmpeg before + Paho's 30-second keepalive reports a data-socket loss. This local-only + seam admits the same composite START lineage while the runtime is still + live, then atomically moves that exact producer to ``reconnecting`` and + queues one coalescing wake. It performs no route probe, device read, + BLE operation, network mutation, or physical command itself. + """ + + with self._acquisition_lifecycle_access(): + runtime = self.runtime.snapshot() + if runtime.get("phase") == "reconnecting": + return self._active_stream_recovery_admitted_for_runtime_loss() + if runtime.get("phase") != "live": + return False + with self._lock: + prior_lineage = self._active_stream_recovery_lineage + lineage = self._admit_active_stream_recovery_lineage( + runtime=runtime, + allow_runtime_live=True, + ) + if lineage is None: + return False + if self.runtime.request_connection_recovery( + reason_code, + expected_generation=lineage.runtime_producer_generation, + ): + return True + + # The runtime generation or stop fence changed between the local + # snapshot and wake. Revoke only the admission created here; a + # pre-existing owner belongs to the concurrent recovery path. + with self._lock: + if ( + prior_lineage is not lineage + and self._active_stream_recovery_lineage == lineage + ): + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_lineage = None + self._active_stream_recovery_state = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = ( + "runtime-recovery-wake-superseded" + ) + self._active_stream_recovery_started_at_utc = None + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None + return False + + @staticmethod + def _active_stream_camera_recovery_reason( + camera: Mapping[str, Any], + ) -> str | None: + """Classify exact committed-media facts, never Popen/epoch allocation.""" + + return classify_camera_recording_health( + camera, + stall_milliseconds=ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS, + ) + + def _enqueue_post_recovery_camera_restart( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + runtime: Mapping[str, Any], + camera: Mapping[str, Any], + ) -> bool: + """Queue one established-epoch camera CAS after recovered PCL publish. + + The runtime invokes the facade's post-publication observer only after + the checkpoint rebind, capture gap confirmation and runtime recovery + commit. This local worker is therefore downstream of all three facts; + it never grants MQTT or physical-command authority itself. + + ``True`` consumes this PCL for camera recovery. ``False`` leaves the + ordinary first-PCL activation path available (notably when the outage + happened before an acquisition camera epoch had ever been established). + """ + + runtime_recovery = runtime.get("connection_recovery") + if not ( + runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") + == lineage.runtime_producer_generation + and isinstance(runtime_recovery, Mapping) + and runtime_recovery.get("state") == "recovered" + and self._active_stream_recovery_lineage_is_current( + lineage, + require_runtime_reconnecting=False, + ) + ): + return False + + def key_matches_lineage(candidate: _PostRecoveryCameraRestartKey) -> bool: + return bool( + candidate.recovery_generation == lineage.recovery_generation + and candidate.acquisition_id == lineage.acquisition_id + and candidate.evidence_session_id == lineage.evidence_session_id + and candidate.runtime_producer_generation + == lineage.runtime_producer_generation + ) + + with self._lock: + admitted_owner = self._post_recovery_camera_restart_claim + inflight_owner = self._post_recovery_camera_restart_inflight + retry_owner = self._post_recovery_camera_restart_retry + terminal_owner = self._post_recovery_camera_restart_terminal + candidates = tuple( + candidate + for candidate in ( + admitted_owner.key if admitted_owner is not None else None, + inflight_owner.key if inflight_owner is not None else None, + retry_owner.key if retry_owner is not None else None, + terminal_owner, + ) + if candidate is not None and key_matches_lineage(candidate) + ) + for candidate in candidates: + admission = self._post_recovery_camera_successor_admission( + camera, + key=candidate, + ) + if admission is None: + continue + with self._lock: + if self._active_stream_recovery_lineage != lineage: + return False + self._post_recovery_camera_restart_claim = admission + self._post_recovery_camera_restart_inflight = None + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_terminal = None + return True + + # An exact in-flight worker remains the sole owner even while its + # reservation temporarily makes the gateway snapshot look selected or + # startup-failed. Duplicate PCLs cannot rotate or overtake its nonce. + if inflight_owner is not None and key_matches_lineage(inflight_owner.key): + return True + + baseline = self._post_recovery_camera_restart_baseline( + camera, + evidence_session_id=lineage.evidence_session_id, + ) + if baseline is None: + if _camera_source_end_archive_seal_pending( + camera, + evidence_session_id=lineage.evidence_session_id, + ): + # The exact old epoch still owns its close/fsync. Consume this + # PCL without minting a claim or spawning anything; once its + # canonical summary appears, the next PCL admits the normal + # sealed-epoch CAS below. + return True + # A failed local spawn advances the gateway generation after the + # original sealed/live epoch CAS. Keep the original claim across a + # batch cooldown; later PCLs only wake it after not-before. + if ( + retry_owner is not None + and key_matches_lineage(retry_owner.key) + and _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=lineage.evidence_session_id, + ) + ): + key = retry_owner.key + else: + # An admitted or terminal established-camera owner must never + # fall through into the first-camera activation path merely + # because its successor now exposes a non-transport fault. + return bool(candidates) + else: + camera_generation, active_epoch, media_count = baseline + key = _PostRecoveryCameraRestartKey( + recovery_generation=lineage.recovery_generation, + acquisition_id=lineage.acquisition_id, + evidence_session_id=lineage.evidence_session_id, + runtime_producer_generation=lineage.runtime_producer_generation, + camera_generation=camera_generation, + camera_active_epoch=active_epoch, + camera_media_segment_count=media_count, + ) + + camera_generation = key.camera_generation + active_epoch = key.camera_active_epoch + + with self._lock: + admitted = self._post_recovery_camera_restart_claim + if admitted is not None and admitted.key != key: + self._post_recovery_camera_restart_claim = None + if ( + self._post_recovery_camera_restart_inflight is not None + and self._post_recovery_camera_restart_inflight.key != key + ): + self._post_recovery_camera_restart_inflight = None + retry = self._post_recovery_camera_restart_retry + if retry is not None and retry.key != key: + self._post_recovery_camera_restart_retry = None + retry = None + if ( + self._post_recovery_camera_restart_terminal is not None + and self._post_recovery_camera_restart_terminal != key + ): + self._post_recovery_camera_restart_terminal = None + if ( + self._active_stream_recovery_lineage != lineage + or self._active_stream_recovery_generation != lineage.recovery_generation + ): + return False + admitted = self._post_recovery_camera_restart_claim + if admitted is not None and admitted.key == key: + return True + inflight = self._post_recovery_camera_restart_inflight + if inflight is not None and inflight.key == key: + return True + if self._post_recovery_camera_restart_terminal == key: + return True + if retry is not None and time.monotonic() < retry.not_before_monotonic: + return True + failure_batch = retry.failure_batch if retry is not None else 0 + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_nonce += 1 + attempt_claim = _PostRecoveryCameraRestartAttempt( + key=key, + nonce=self._post_recovery_camera_restart_nonce, + failure_batch=failure_batch, + ) + self._post_recovery_camera_restart_inflight = attempt_claim + if self._active_stream_recovery_state == "reconnecting": + self._active_stream_recovery_state = "recovered" + self._active_stream_recovery_reason_code = None + + def recover_camera() -> None: + decision: RecoveryDecision = "retry" + try: + decision = self._run_post_recovery_camera_restart_worker( + lineage, + attempt_claim=attempt_claim, + ) + except Exception: + # An unexpected local snapshot/adapter exception must not + # strand the in-flight owner. Rotate it into the same bounded + # cooldown without logging paths, payloads or device secrets. + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + retry_failure_batch=attempt_claim.failure_batch + 1, + ) + logger.exception( + "K1 post-recovery camera worker failed locally", + extra={ + "event_code": "k1_camera_post_recovery_worker_failed", + "camera_generation": key.camera_generation, + "camera_active_epoch": key.camera_active_epoch, + "failure_batch": attempt_claim.failure_batch + 1, + "device_command_sent": False, + }, + ) + finally: + # Compare-and-clear is intentionally unconditional. Successful + # finish/retry publication already replaced this exact token; + # a newer attempt's nonce can never be cleared here. + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + ) + logger.info( + "K1 post-recovery PCL reduced the acquisition camera owner", + extra={ + "event_code": "k1_camera_post_recovery_pcl_reduced", + "runtime_producer_generation": ( + lineage.runtime_producer_generation + ), + "camera_generation": camera_generation, + "camera_active_epoch": active_epoch, + "decision": decision, + "device_command_sent": False, + }, + ) + + worker = threading.Thread( + target=recover_camera, + name=( + "k1-camera-post-recovery-pcl-" + f"{lineage.runtime_producer_generation}" + ), + daemon=True, + ) + try: + worker.start() + except RuntimeError: + with self._lock: + if self._post_recovery_camera_restart_inflight == attempt_claim: + self._post_recovery_camera_restart_inflight = None + raise + return True + + @staticmethod + def _post_recovery_camera_restart_baseline( + camera: Mapping[str, Any], + *, + evidence_session_id: str, + ) -> tuple[int, int | None, int] | None: + """Return exact live/sealed epoch CAS facts admitted for one restart.""" + + recording = camera.get("recording") + error = camera.get("error") + error_code = error.get("code") if isinstance(error, Mapping) else None + if not ( + camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == evidence_session_id + and isinstance(camera.get("generation"), int) + and not isinstance(camera.get("generation"), bool) + and isinstance(recording.get("committed_media_segment_count"), int) + and not isinstance(recording.get("committed_media_segment_count"), bool) + ): + return None + generation = cast(int, camera["generation"]) + media_count = cast(int, recording["committed_media_segment_count"]) + if ( + classify_camera_recording_health( + camera, + stall_milliseconds=ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS, + ) + is None + ): + return None + active_epoch = recording.get("active_epoch") + if isinstance(active_epoch, int) and not isinstance(active_epoch, bool): + return generation, active_epoch, media_count + if active_epoch is not None: + return None + summary = recording.get("last_summary") + completed_epochs = recording.get("completed_epochs") + if not ( + camera.get("phase") == "error" + and error_code == "camera-source-ended" + and recording.get("producer_alive") is False + and recording.get("current_epoch") is None + and isinstance(completed_epochs, int) + and not isinstance(completed_epochs, bool) + and completed_epochs >= 1 + and isinstance(summary, Mapping) + and summary.get("schema_version") == CAMERA_ARCHIVE_SCHEMA + and summary.get("source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and summary.get("codec_epoch") == generation + and summary.get("status") == "interrupted" + and summary.get("failure_code") == error_code + ): + return None + return generation, None, media_count + + @staticmethod + def _post_recovery_camera_successor_admission( + camera: Mapping[str, Any], + *, + key: _PostRecoveryCameraRestartKey, + ) -> _PostRecoveryCameraRestartAdmission | None: + recording = camera.get("recording") + current_epoch = ( + recording.get("current_epoch") if isinstance(recording, Mapping) else None + ) + generation = camera.get("generation") + active_epoch = recording.get("active_epoch") if isinstance(recording, Mapping) else None + if not ( + camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and camera.get("phase") in {"connecting", "streaming"} + and camera.get("error") is None + and isinstance(generation, int) + and not isinstance(generation, bool) + and generation > key.camera_generation + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == key.evidence_session_id + and recording.get("producer_alive") is True + and isinstance(active_epoch, int) + and not isinstance(active_epoch, bool) + and active_epoch == generation + and isinstance(current_epoch, Mapping) + and current_epoch.get("generation") == active_epoch + ): + return None + return _PostRecoveryCameraRestartAdmission( + key=key, + camera_generation=generation, + active_epoch=active_epoch, + ) + + def _finish_post_recovery_camera_restart( + self, + *, + attempt_claim: _PostRecoveryCameraRestartAttempt, + admission: _PostRecoveryCameraRestartAdmission | None = None, + terminal: bool = False, + retry_failure_batch: int | None = None, + ) -> None: + with self._lock: + if self._post_recovery_camera_restart_inflight != attempt_claim: + return + self._post_recovery_camera_restart_inflight = None + if admission is not None: + self._post_recovery_camera_restart_claim = admission + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_terminal = None + elif retry_failure_batch is not None: + delay = min( + CAMERA_POST_RECOVERY_RESTART_BACKOFF_BASE_SECONDS + * (2 ** min(max(0, retry_failure_batch - 1), 8)), + CAMERA_POST_RECOVERY_RESTART_BACKOFF_MAX_SECONDS, + ) + self._post_recovery_camera_restart_retry = ( + _PostRecoveryCameraRestartRetry( + key=attempt_claim.key, + failure_batch=retry_failure_batch, + not_before_monotonic=time.monotonic() + delay, + ) + ) + self._post_recovery_camera_restart_terminal = None + elif terminal: + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_terminal = attempt_claim.key + + def _run_post_recovery_camera_restart_worker( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + attempt_claim: _PostRecoveryCameraRestartAttempt, + ) -> RecoveryDecision: + """Own bounded retries independently of the 10 Hz PCL observer.""" + + key = attempt_claim.key + for attempt in range(1, CAMERA_POST_RECOVERY_RESTART_MAX_ATTEMPTS + 1): + with self._lock: + if self._post_recovery_camera_restart_inflight != attempt_claim: + return "blocked" + if not self._active_stream_recovery_lineage_is_current( + lineage, + require_runtime_reconnecting=False, + ): + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + ) + return "blocked" + with self._camera_restart_commit_gate: + stop_has_priority = bool( + self._camera_stop_priority_counts.get(lineage.acquisition_id, 0) + ) + if stop_has_priority: + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + ) + return "blocked" + + camera = self.camera_preview.snapshot() + admission = self._post_recovery_camera_successor_admission( + camera, + key=key, + ) + if admission is not None: + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + admission=admission, + ) + return "resume" + + with suppress(OSError, RuntimeError, ValueError): + self._recover_active_stream_camera_owned( + lineage, + require_runtime_reconnecting=False, + ) + + camera = self.camera_preview.snapshot() + admission = self._post_recovery_camera_successor_admission( + camera, + key=key, + ) + if admission is not None: + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + admission=admission, + ) + return "resume" + retryable = bool( + self._post_recovery_camera_restart_baseline( + camera, + evidence_session_id=lineage.evidence_session_id, + ) + is not None + or _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=lineage.evidence_session_id, + ) + ) + if not retryable: + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + terminal=True, + ) + return "blocked" + if attempt == CAMERA_POST_RECOVERY_RESTART_MAX_ATTEMPTS: + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_reason_code = ( + "camera-recovery-retry-pending" + ) + self._finish_post_recovery_camera_restart( + attempt_claim=attempt_claim, + retry_failure_batch=attempt_claim.failure_batch + 1, + ) + return "retry" + delay = min( + CAMERA_POST_RECOVERY_RESTART_BACKOFF_BASE_SECONDS + * (2 ** (attempt - 1)), + CAMERA_POST_RECOVERY_RESTART_BACKOFF_MAX_SECONDS, + ) + threading.Event().wait(delay) + + raise AssertionError("bounded camera recovery loop exhausted unexpectedly") + + def _reserve_camera_restart_if_still_active( + self, + *, + acquisition_id: str, + evidence_session_id: str, + start_operation_id: str | None, + runtime_producer_generation: int, + reserve: Callable[[], bool], + ) -> bool: + """Linearize camera detach before any gateway or archive mutation.""" + + return self._camera_restart_action_if_still_active( + acquisition_id=acquisition_id, + evidence_session_id=evidence_session_id, + start_operation_id=start_operation_id, + runtime_producer_generation=runtime_producer_generation, + action=reserve, + ) + + def _commit_camera_restart_if_still_active( + self, + *, + acquisition_id: str, + evidence_session_id: str, + start_operation_id: str | None, + runtime_producer_generation: int, + commit: Callable[[], bool], + ) -> bool: + """Linearize one quick local candidate commit against physical STOP.""" + + return self._camera_restart_action_if_still_active( + acquisition_id=acquisition_id, + evidence_session_id=evidence_session_id, + start_operation_id=start_operation_id, + runtime_producer_generation=runtime_producer_generation, + action=commit, + ) + + def _camera_restart_action_if_still_active( + self, + *, + acquisition_id: str, + evidence_session_id: str, + start_operation_id: str | None, + runtime_producer_generation: int, + action: Callable[[], bool], + ) -> bool: + """Fence one quick camera reservation/commit against physical STOP. + + Plugin-owned camera authority is the complete active acquisition + lineage, not a direct START ledger head. A classified definitely-unsent + STOP may legitimately be the current physical head while the original + START remains active; the canonical matcher proves that bounded chain. + """ + + # The durable coordinator snapshot reloads its ledger under a process + # lock. Take that potentially blocking read before either shared + # facade gate: a STOP can publish its priority intent while this read + # is delayed, then wins the gate/CAS below. Once the gate is ours, + # only bounded in-process lineage checks and the camera-local commit + # remain. + physical = self._physical_command_coordinator.snapshot() + control = self._application_control_session.snapshot() + verified_control_value = control.get("verified_control") + verified_control = ( + cast(Mapping[str, Any], verified_control_value) + if isinstance(verified_control_value, Mapping) + else None + ) + checkpoint_start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=acquisition_id, + evidence_session_id=evidence_session_id, + local_start_operation_id=start_operation_id, + ) + ) + checkpoint_current = ( + isinstance(start_operation_id, str) + and self._active_acquisition_checkpoint_matches_current( + acquisition_id=acquisition_id, + start_operation_id=start_operation_id, + evidence_session_id=evidence_session_id, + verified_control=verified_control, + ) + ) + # Global order: camera-restart priority -> acquisition lifecycle -> + # camera lifecycle (inside ``action``). No caller may enter this + # helper while owning either later lock. + with self._camera_restart_commit_gate: + if self._camera_stop_priority_counts.get(acquisition_id, 0) > 0: + return False + with self._acquisition_lifecycle_access(): + runtime = self.runtime.snapshot() + runtime_recovery = runtime.get("connection_recovery") + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + current_start_operation_id = self._acquisition_start_operation_id + stop_operation_id = self._acquisition_stop_operation_id + recovery_state = self._active_stream_recovery_state + record = physical.get("record") + classified_stop_active = bool( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and isinstance(record, Mapping) + and record.get("action") == "stop" + and isinstance(record.get("operation_id"), str) + and isinstance(start_operation_id, str) + and self._matching_classified_prepared_stop_active( + physical, + acquisition_id=acquisition_id, + stop_operation_id=cast(str, record["operation_id"]), + start_operation_id=start_operation_id, + verified_control=verified_control, + ) + ) + local_lineage_current = bool( + acquisition is not None + and ( + ( + acquisition.control_mode != "plugin-commanded" + and current_start_operation_id + in {None, start_operation_id} + ) + or ( + acquisition.control_mode == "plugin-commanded" + and isinstance(start_operation_id, str) + and ( + current_start_operation_id == start_operation_id + or ( + current_start_operation_id is None + and checkpoint_start_operation_id + == start_operation_id + ) + ) + ) + ) + ) + base_current = bool( + acquisition is not None + and acquisition.acquisition_id == acquisition_id + and acquisition.state + in {"starting", "awaiting_external_start", "acquiring"} + and out_dir is not None + and out_dir.name == evidence_session_id + and local_lineage_current + and ( + stop_operation_id is None + or ( + classified_stop_active + and isinstance(record, Mapping) + and stop_operation_id == record.get("operation_id") + ) + ) + and runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + and runtime.get("producer_generation") + == runtime_producer_generation + ) + plugin_physical_current = bool( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and isinstance(start_operation_id, str) + and checkpoint_current + and recovery_state not in { + "blocked", + "fault", + "force-finishing", + "force-finished", + } + and ( + recovery_state != "reconnecting" + or ( + isinstance(runtime_recovery, Mapping) + and runtime_recovery.get("state") == "recovered" + ) + ) + and self._matching_start_active_confirmed( + physical, + acquisition_id=acquisition_id, + start_operation_id=start_operation_id, + verified_control=verified_control, + ) + ) + if not base_current or ( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and not plugin_physical_current + ): + return False + return action() + + def _observe_camera_producer_stall(self, reason_code: str, generation: int) -> None: + """CAS-restart one exact live camera owner without touching K1 control. + + This callback is owned by the acquisition FFmpeg watchdog, not by a + browser snapshot poll. Camera-only loss stays camera-only: it does not + move the MQTT/PCL runtime to ``reconnecting`` and cannot publish START, + STOP, BLE, Wi-Fi or any other device command. A concurrent STOP or a + newer camera generation wins through the gateway CAS. + """ + + with self._acquisition_lifecycle_access(): + runtime = self.runtime.snapshot() + if runtime.get("phase") != "live" or runtime.get("source_mode") != "live": + # The capture-owned recovery loop handles the same camera fact + # while MQTT/PCL is already reconnecting. + return + camera = self.camera_preview.snapshot() + recording = camera.get("recording") + if not ( + camera.get("generation") == generation + and self._active_stream_camera_recovery_reason(camera) == reason_code + and camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(recording, Mapping) + and recording.get("active") is True + and isinstance(recording.get("session"), str) + ): + return + with self._lock: + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + device_session_id = self._device_session_id + start_operation_id = self._acquisition_start_operation_id + if ( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and out_dir is not None + ): + start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + local_start_operation_id=start_operation_id, + ) + ) + physical = self._physical_command_coordinator.snapshot() + control = self._application_control_session.snapshot() + verified_control = control.get("verified_control") + if not ( + acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and acquisition.state in {"starting", "awaiting_external_start", "acquiring"} + and out_dir is not None + and recording.get("session") == out_dir.name + and isinstance(device_session_id, str) + and isinstance(start_operation_id, str) + and self._matching_start_active_confirmed( + physical, + acquisition_id=acquisition.acquisition_id, + start_operation_id=start_operation_id, + verified_control=( + verified_control + if isinstance(verified_control, Mapping) + else None + ), + ) + and self._active_acquisition_checkpoint_matches_current( + acquisition_id=acquisition.acquisition_id, + start_operation_id=start_operation_id, + evidence_session_id=out_dir.name, + verified_control=( + verified_control + if isinstance(verified_control, Mapping) + else None + ), + ) + ): + return + expected_active_epoch = recording.get("active_epoch") + if expected_active_epoch is not None and ( + not isinstance(expected_active_epoch, int) + or isinstance(expected_active_epoch, bool) + ): + return + expected_media_count = recording.get("committed_media_segment_count") + if ( + not isinstance(expected_media_count, int) + or isinstance(expected_media_count, bool) + or expected_media_count < 0 + ): + return + target = self._camera_target_for_session(device_session_id) + acquisition_id = acquisition.acquisition_id + evidence_session_id = out_dir.name + + try: + reopened = self.camera_preview.restart_recording_producer( + DEFAULT_ACQUISITION_CAMERA_SOURCE, + target, + expected_generation=generation, + expected_recording_session=evidence_session_id, + expected_active_epoch=expected_active_epoch, + expected_recording_media_segment_count=expected_media_count, + pre_detach_fence=lambda reserve: ( + self._reserve_camera_restart_if_still_active( + acquisition_id=acquisition_id, + evidence_session_id=evidence_session_id, + start_operation_id=start_operation_id, + runtime_producer_generation=runtime["producer_generation"], + reserve=reserve, + ) + ), + commit_fence=lambda commit: self._commit_camera_restart_if_still_active( + acquisition_id=acquisition_id, + evidence_session_id=evidence_session_id, + start_operation_id=start_operation_id, + runtime_producer_generation=runtime["producer_generation"], + commit=commit, + ), + committed_before_start=lambda committed: self._bind_live_perception_camera( + evidence_session_id, + committed, + ), + ) + except (OSError, RuntimeError, ValueError): + # A newer epoch/STOP is ordinary CAS supersession. The gateway + # snapshot retains a real local spawn failure for reconciliation. + return + reopened_recording = reopened.get("recording") + current_epoch = ( + reopened_recording.get("current_epoch") + if isinstance(reopened_recording, Mapping) + else None + ) + if not ( + reopened.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(reopened_recording, Mapping) + and reopened_recording.get("active") is True + and reopened_recording.get("session") == evidence_session_id + and isinstance(current_epoch, Mapping) + and current_epoch.get("generation") == reopened_recording.get("active_epoch") + and reopened_recording.get("active_epoch") != expected_active_epoch + ): + return + + logger.info( + "K1 camera watchdog opened a pending evidence epoch", + extra={ + "event_code": "k1_camera_watchdog_epoch_restarted", + "reason_code": reason_code, + "camera_generation": generation, + "replacement_generation": reopened_recording.get("active_epoch"), + "camera_media_ready": reopened_recording.get("media_ready") is True, + "device_command_sent": False, + }, + ) + + def _recover_active_stream_connection(self, attempt: int) -> RecoveryDecision: + """Bridge the capture thread to one serialized read-only rebind attempt.""" + + with self._acquisition_lifecycle_access(): + with self._lock: + acquisition = self._acquisition + stop_operation_id = self._acquisition_stop_operation_id + prepared_outcome = self._settle_prepared_stop_worker_failure( + acquisition_id=( + acquisition.acquisition_id if acquisition is not None else None + ), + stop_operation_id=stop_operation_id, + ) + if prepared_outcome in {"defer", "prepared-pending"}: + # The accepted STOP still owns the publish handoff. The MQTT + # capture loop must remain retryable, but no read-only probe or + # new control worker may overtake that owner. + return "retry" + lineage = self._admit_active_stream_recovery_lineage() + if lineage is None: + return "blocked" + with self._lock: + runtime_loop = self._runtime_event_loop + self._active_stream_recovery_attempt = attempt + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_reason_code = "read-only-rebind-in-progress" + if runtime_loop is None or not runtime_loop.is_running() or runtime_loop.is_closed(): + with self._lock: + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = "runtime-loop-unavailable" + return "blocked" + future = asyncio.run_coroutine_threadsafe( + self._recover_active_stream_connection_owned(lineage, attempt), + runtime_loop, + ) + if not self._active_stream_recovery_lineage_is_current(lineage): + future.cancel() + return "blocked" + with self._lock: + self._active_stream_recovery_future = future + try: + return future.result(timeout=ACTIVE_STREAM_RECOVERY_ATTEMPT_TIMEOUT_SECONDS) + except ConcurrentFutureTimeout: + future.cancel() + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_reason_code = "read-only-rebind-timeout" + return "retry" + except BaseException as exc: + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_reason_code = str( + getattr(exc, "reason_code", "read-only-rebind-cancelled") + ) + return "blocked" if future.cancelled() else "fault" + finally: + with self._lock: + if self._active_stream_recovery_future is future: + self._active_stream_recovery_future = None + + async def _recover_active_stream_connection_owned( + self, + lineage: _ActiveStreamRecoveryLineage, + attempt: int, + ) -> RecoveryDecision: + """Perform route/TCP + DeviceInfo/Status inspection; never publish.""" + + transition_gate_acquired = False + network_process_lease_acquired = False + try: + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + return "retry" + transition_gate_acquired = True + if not await self._await_connection_monitor_quiescence(): + return "retry" + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + self._acquire_k1_lifecycle_process_lease("network") + network_process_lease_acquired = True + + observation = await _run_blocking_operation_without_abandonment( + self._probe_control_endpoint, + lineage.target_ipv4, + ) + if not observation.path.available or not observation.reachable: + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_reason_code = ( + observation.reason_code or "endpoint-not-ready" + ) + return "retry" + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + + control = dict(self._application_control_session.snapshot()) + if control.get("state") != "idle": + if control.get("state") not in {"completed", "closed", "failed"}: + await asyncio.to_thread(self._application_control_session.close) + control = await self._await_local_control_terminal() + if control.get("state") != "idle" and not await ( + self._retire_terminal_control_for_active_stream_recovery_owned(lineage) + ): + return "blocked" + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + + await self._bootstrap_prestart_control_ready_owned( + parent_operation_id=( + f"active-stream-recovery:{lineage.acquisition_id}:{lineage.recovery_generation}" + ), + connection_mode=lineage.connection_mode, + inspection_only=True, + force_new_binding=True, + ) + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + + decision = await self._reconcile_active_stream_physical_state_owned( + lineage, + attempt=attempt, + ) + if decision != "resume": + return decision + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + + with self._lock: + if self._active_stream_recovery_lineage != lineage: + return "blocked" + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_reason_code = "mqtt-resubscribe-pending" + return "resume" + except ConnectionVerificationError as exc: + reason_code = ( + "active-stream-recovery-system-error" + if exc.reason_code == "application-device-system-error" + else exc.reason_code + ) + if reason_code in { + "control-bootstrap-device-identity-unverified", + "physical-command-reconciliation-binding-changed", + }: + failure_decision: RecoveryDecision = "blocked" + elif reason_code in { + "physical-command-ledger-corrupt", + "physical-command-reconciliation-state-unsafe", + "active-stream-recovery-system-error", + }: + failure_decision = "fault" + else: + failure_decision = "retry" + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = ( + "blocked" + if failure_decision == "blocked" + else "fault" + if failure_decision == "fault" + else "reconnecting" + ) + self._active_stream_recovery_reason_code = reason_code + if failure_decision == "fault": + self._active_stream_recovery_terminal_outcome = "fault" + return failure_decision + except (OSError, TimeoutError): + return "retry" + except ApplicationConnectionBindingLost as exc: + # A read-only inspection binding is intentionally created only + # after the old host-path epoch has been retired. macOS may still + # rotate route/association evidence between the correlated TCP + # probe and that worker's first publish. That invalidates only + # this local attempt; the immutable START/acquisition/transport + # lineage remains eligible for the next read-only inspection. + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "reconnecting" + self._active_stream_recovery_reason_code = str( + getattr(exc, "reason_code", "application-connection-binding-lost") + ) + return "retry" + except PhysicalCommandTransitionError: + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = "exact-binding-changed" + return "blocked" + except (RuntimeError, ValueError) as exc: + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "fault" + self._active_stream_recovery_reason_code = str( + getattr(exc, "reason_code", "active-stream-recovery-failed") + ) + self._active_stream_recovery_terminal_outcome = "fault" + return "fault" + finally: + if network_process_lease_acquired: + self._release_k1_lifecycle_process_lease("network") + if transition_gate_acquired: + self._k1_lifecycle_transition_gate.release() + + async def _retire_terminal_control_for_active_stream_recovery_owned( + self, + lineage: _ActiveStreamRecoveryLineage, + ) -> bool: + """Wait for the old socket worker before resetting its local owner.""" + + deadline = time.monotonic() + CONTROL_LOCAL_RETIREMENT_TIMEOUT_SECONDS + while self._active_stream_recovery_lineage_is_current(lineage): + try: + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + return True + except ApplicationAcceptanceError as exc: + if time.monotonic() >= deadline: + raise ConnectionVerificationError( + "локальный control worker не завершился вовремя", + reason_code="control-local-retirement-timeout", + ) from exc + await asyncio.sleep(0.05) + return False + + def _recover_active_stream_camera_owned( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + require_runtime_reconnecting: bool = True, + ) -> RecoveryDecision: + """Reopen only the dead acquisition-owned right-camera producer. + + This is not a public camera-selection intent. The immutable recovery + lineage already owns the source, evidence session and device target; + recovery may only recreate that exact local FFmpeg producer. + """ + + if not self._active_stream_recovery_lineage_is_current( + lineage, + require_runtime_reconnecting=require_runtime_reconnecting, + ): + return "blocked" + camera = self.camera_preview.snapshot() + recording = camera.get("recording") + if not ( + camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == lineage.evidence_session_id + ): + # Link loss may happen during calibration, before the first + # authoritative PCL has admitted any camera producer. MQTT/control + # recovery must remain camera-independent; a later published PCL + # will run the normal one-shot activation edge. + return "resume" + camera_recovery_reason = self._active_stream_camera_recovery_reason(camera) + producer_age_ms = ( + recording.get("producer_age_ms") if isinstance(recording, Mapping) else None + ) + last_segment_age_ms = ( + recording.get("last_segment_age_ms") if isinstance(recording, Mapping) else None + ) + producer_stalled = bool( + isinstance(producer_age_ms, int) + and not isinstance(producer_age_ms, bool) + and producer_age_ms >= ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + and ( + last_segment_age_ms is None + or ( + isinstance(last_segment_age_ms, int) + and not isinstance(last_segment_age_ms, bool) + and last_segment_age_ms >= ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + ) + ) + ) + activation_lineage = ( + lineage.acquisition_id, + lineage.evidence_session_id, + lineage.runtime_producer_generation, + ) + with self._lock: + camera_activation_claimed = activation_lineage in { + self._camera_activation_lineage, + self._camera_activation_retry_lineage, + } + camera_waiting_for_first_media = bool( + camera_activation_claimed + and camera.get("phase") in {"selected", "connecting"} + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == lineage.evidence_session_id + and recording.get("last_segment_age_ms") is None + ) + if camera_waiting_for_first_media: + # Camera activation is downstream of the authoritative PCL stream. + # It must not block the read-only MQTT rebind which supplies its + # later PCL/media recovery edges, even after the generic stall age. + # A process can exit just before its reader publishes ``error``; + # demote that exact active claim now so the next authoritative PCL + # gets the same bounded fresh-epoch retry instead of a permanent + # recovery ``blocked`` decision. + if recording.get("producer_alive") is False: + with self._lock: + if self._active_stream_recovery_lineage != lineage: + return "blocked" + if self._camera_activation_lineage == activation_lineage: + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = activation_lineage + self._camera_activation_retry_not_before_monotonic = max( + self._camera_activation_retry_not_before_monotonic, + time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS, + ) + return "resume" + if camera_recovery_reason is None: + return "resume" + camera_needs_reopen = bool( + camera.get("phase") == "error" + or not isinstance(recording, Mapping) + or recording.get("active_epoch") is None + or recording.get("producer_alive") is False + or producer_stalled + ) + if not camera_needs_reopen: + return "resume" + startup_failure_retryable = _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=lineage.evidence_session_id, + ) + if startup_failure_retryable and require_runtime_reconnecting: + # Camera startup is downstream of the already-authoritative PCL + # stream. A local pre-first-media FFmpeg failure must never block + # the control/MQTT rebind that supplies the later PCL which owns the + # bounded retry. Preserve the exact acquisition/runtime claim and + # let post-publish activation allocate the fresh camera epoch; do + # not restart camera, START/STOP, BLE, or network state here. + with self._lock: + if self._active_stream_recovery_lineage != lineage: + return "blocked" + if self._camera_activation_lineage == activation_lineage: + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = activation_lineage + self._camera_activation_retry_not_before_monotonic = max( + self._camera_activation_retry_not_before_monotonic, + time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS, + ) + return "resume" + if camera_recovery_reason is None and not startup_failure_retryable: + # Storage, artifact and decoder-integrity failures are local + # terminal evidence. They must be reduced by acquisition + # reconciliation, never disguised as a network camera restart. + return "blocked" + if not ( + camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(camera.get("generation"), int) + and not isinstance(camera.get("generation"), bool) + and isinstance(recording, Mapping) + and recording.get("active") is True + and recording.get("session") == lineage.evidence_session_id + ): + return "blocked" + try: + expected_active_epoch = recording.get("active_epoch") + if expected_active_epoch is not None and ( + not isinstance(expected_active_epoch, int) + or isinstance(expected_active_epoch, bool) + ): + raise RuntimeError("camera recovery epoch is invalid") + expected_media_count = recording.get("committed_media_segment_count") + if ( + not isinstance(expected_media_count, int) + or isinstance(expected_media_count, bool) + or expected_media_count < 0 + ): + raise RuntimeError("camera recovery media progress is invalid") + def pre_spawn_fence(reserve: Callable[[], bool]) -> bool: + return self._reserve_camera_restart_if_still_active( + acquisition_id=lineage.acquisition_id, + evidence_session_id=lineage.evidence_session_id, + start_operation_id=lineage.start_operation_id, + runtime_producer_generation=lineage.runtime_producer_generation, + reserve=reserve, + ) + + def commit_fence(commit: Callable[[], bool]) -> bool: + return self._commit_camera_restart_if_still_active( + acquisition_id=lineage.acquisition_id, + evidence_session_id=lineage.evidence_session_id, + start_operation_id=lineage.start_operation_id, + runtime_producer_generation=lineage.runtime_producer_generation, + commit=commit, + ) + + def committed_before_start(committed: Mapping[str, Any]) -> None: + self._bind_live_perception_camera( + lineage.evidence_session_id, + committed, + ) + if startup_failure_retryable: + reopened = self.camera_preview.retry_recording_producer( + DEFAULT_ACQUISITION_CAMERA_SOURCE, + lineage.target_ipv4, + expected_generation=int(camera["generation"]), + expected_recording_session=lineage.evidence_session_id, + pre_retry_fence=pre_spawn_fence, + commit_fence=commit_fence, + committed_before_start=committed_before_start, + ) + else: + reopened = self.camera_preview.restart_recording_producer( + DEFAULT_ACQUISITION_CAMERA_SOURCE, + lineage.target_ipv4, + expected_generation=int(camera["generation"]), + expected_recording_session=lineage.evidence_session_id, + expected_active_epoch=expected_active_epoch, + expected_recording_media_segment_count=expected_media_count, + pre_detach_fence=pre_spawn_fence, + commit_fence=commit_fence, + committed_before_start=committed_before_start, + ) + reopened_recording = reopened.get("recording") + current_epoch = ( + reopened_recording.get("current_epoch") + if isinstance(reopened_recording, Mapping) + else None + ) + if not ( + reopened.get("phase") != "error" + and reopened.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(reopened_recording, Mapping) + and reopened_recording.get("active") is True + and reopened_recording.get("session") == lineage.evidence_session_id + and reopened_recording.get("active_epoch") is not None + and reopened_recording.get("active_epoch") != expected_active_epoch + and reopened_recording.get("producer_alive") is not False + and isinstance(current_epoch, Mapping) + and current_epoch.get("generation") + == reopened_recording.get("active_epoch") + ): + raise RuntimeError("camera recovery did not open a new evidence epoch") + # Callback lineage was installed before reader/watchdog threads. + # ``media_ready`` remains false until this exact epoch durably + # commits init and first media. + except (OSError, RuntimeError, ValueError): + with self._camera_restart_commit_gate: + stop_has_priority = bool( + self._camera_stop_priority_counts.get(lineage.acquisition_id, 0) + ) + if stop_has_priority: + return "blocked" + current_camera = self.camera_preview.snapshot() + if self._active_stream_recovery_lineage_is_current( + lineage, + require_runtime_reconnecting=require_runtime_reconnecting, + ) and ( + self._post_recovery_camera_restart_baseline( + current_camera, + evidence_session_id=lineage.evidence_session_id, + ) + is not None + or _camera_post_pcl_startup_failure_is_retryable( + current_camera, + evidence_session_id=lineage.evidence_session_id, + ) + ): + return "retry" + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "blocked" + self._active_stream_recovery_reason_code = "camera-recovery-failed" + return "blocked" + return ( + "resume" + if self._active_stream_recovery_lineage_is_current( + lineage, + require_runtime_reconnecting=require_runtime_reconnecting, + ) + else "blocked" + ) + + async def _reconcile_active_stream_physical_state_owned( + self, + lineage: _ActiveStreamRecoveryLineage, + *, + attempt: int, + ) -> RecoveryDecision: + """Append one same-lineage read-only state proof without rehydration.""" + + deadline = time.monotonic() + PHYSICAL_RECONCILIATION_PROOF_TIMEOUT_SECONDS + while True: + if not self._active_stream_recovery_lineage_is_current(lineage): + return "blocked" + physical = dict(self._physical_command_coordinator.snapshot()) + control = dict(self._application_control_session.snapshot()) + control_state = str(control.get("state") or "unknown") + if control_state in {"failed", "closed", "completed", "idle"}: + failure = control.get("failure") + reason_code = failure.get("reason_code") if isinstance(failure, Mapping) else None + raise ConnectionVerificationError( + "K1 завершил read-only recovery inspection", + reason_code=( + "active-stream-recovery-system-error" + if reason_code == "application-device-system-error" + else str(reason_code or "control-bootstrap-terminal-failure") + ), + ) + if control_state != "connection-ready": + raise ConnectionVerificationError( + "read-only recovery control generation changed", + reason_code="physical-command-reconciliation-control-changed", + ) + self._application_control_session.validate_physical_reconciliation_binding() + if physical.get("reconciliation_ready") is True: + observed_state = physical.get("observed_session_state") + if observed_state not in {"ready", "scanning", "scan_over"}: + raise ConnectionVerificationError( + "K1 returned an unsafe recovery state", + reason_code="physical-command-reconciliation-state-unsafe", + ) + reconciliation_id = _active_stream_reconciliation_id( + lineage, + attempt=attempt, + ) + reconciliation_record = physical.get("record") + prepared_stop_reconciliation = bool( + physical.get("status") == "unresolved" + and physical.get("requires_reconciliation") is True + and isinstance(reconciliation_record, Mapping) + and reconciliation_record.get("operation_id") + == lineage.physical_operation_id + and reconciliation_record.get("action") == "stop" + and reconciliation_record.get("stage") == "prepared" + ) + with self._lock: + retained_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + retained_prepared_stop = bool( + prepared_stop_reconciliation + and retained_owner is not None + and self._prepared_stop_recovery_owner_is_current(retained_owner) + ) + if retained_prepared_stop: + assert retained_owner is not None + + def reconcile_retained_prepared_stop( + reconciliation_id: str = reconciliation_id, + ) -> Mapping[str, Any]: + return self._physical_command_coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ) + + reconciled = self._classify_retained_prepared_stop_owned( + owner=retained_owner, + reconciliation_id=reconciliation_id, + reconcile=reconcile_retained_prepared_stop, + ) + elif prepared_stop_reconciliation: + reconciled = self._physical_command_coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ) + else: + reconciled = self._physical_command_coordinator.reconcile_resolved_active( + reconciliation_id=reconciliation_id, + ) + if not ( + isinstance(reconciled, Mapping) + and reconciled.get("operation_id") == lineage.physical_operation_id + and reconciled.get("acquisition_id") == lineage.acquisition_id + ): + raise ConnectionVerificationError( + "physical recovery changed immutable START lineage", + reason_code="physical-command-reconciliation-binding-changed", + ) + if prepared_stop_reconciliation and not retained_prepared_stop: + self._settle_classified_prepared_stop_operation( + reconciled_record=reconciled, + reconciliation_id=reconciliation_id, + ) + if observed_state in {"ready", "scan_over"}: + if not self._cease_active_acquisition_checkpoint_from_physical_head( + expected_acquisition_id=lineage.acquisition_id, + expected_start_operation_id=lineage.start_operation_id, + evidence_session_id=lineage.evidence_session_id, + expected_stop_operation_id=lineage.physical_operation_id, + require_reconciliation=True, + ): + raise ConnectionVerificationError( + "terminal recovery checkpoint could not be ceased", + reason_code=( + "active-acquisition-recovery-checkpoint-cease-failed" + ), + ) + with self._lock: + if self._active_stream_recovery_lineage == lineage: + self._active_stream_recovery_state = "standby" + self._active_stream_recovery_reason_code = ( + "device-reported-scan-over" + if observed_state == "scan_over" + else "device-reported-standby" + ) + self._active_stream_recovery_terminal_outcome = "standby" + return "standby" + generation = control.get("session_generation") + revision = control.get("state_revision") + if not isinstance(generation, int) or not isinstance(revision, int): + raise ConnectionVerificationError( + "active recovery lacks an exact control checkpoint", + reason_code="physical-command-reconciliation-control-changed", + ) + self._application_control_session.adopt_reconciled_scanning( + reconciliation_id=reconciliation_id, + expected_session_generation=generation, + expected_state_revision=revision, + ) + adoption_deadline = ( + time.monotonic() + + PHYSICAL_RECONCILIATION_CONTROL_ADOPTION_TIMEOUT_SECONDS + ) + while True: + adopted = dict(self._application_control_session.snapshot()) + adopted_state = str(adopted.get("state") or "unknown") + if adopted_state == "scanning": + self._reconcile_connection_supervisor( + adopted, + self.runtime.snapshot(), + ) + break + if adopted_state in {"idle", "completed", "closed", "failed"}: + raise ConnectionVerificationError( + "active recovery control adoption terminated", + reason_code="physical-command-reconciliation-control-changed", + ) + if time.monotonic() >= adoption_deadline: + logger.error( + "K1 active-stream control did not adopt reconciled SCANNING", + extra={ + "event_code": ( + "k1_physical_reconciliation_adoption_timeout" + ), + "reason_code": ( + "physical-command-reconciliation-control-adoption-timeout" + ), + "device_write_performed": False, + "automatic_retry": False, + }, + ) + raise ConnectionVerificationError( + "active recovery control adoption timed out", + reason_code=( + "physical-command-reconciliation-control-adoption-timeout" + ), + ) + await asyncio.sleep(0.01) + if prepared_stop_reconciliation: + with self._lock: + retained_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + if retained_owner is None: + raise ConnectionVerificationError( + "retained acquisition owner disappeared after classification", + reason_code="physical-command-reconciliation-binding-changed", + ) + retained_adopted = self._adopt_classified_prepared_stop_scanning_in_place( + owner=retained_owner, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled, + checkpoint_trust_token=retained_owner.checkpoint_trust_token, + ) + if not retained_adopted: + return "blocked" + return "resume" + if time.monotonic() >= deadline: + raise ConnectionVerificationError( + "fresh non-retained DeviceStatus was not observed", + reason_code="physical-command-reconciliation-proof-timeout", + ) + await asyncio.sleep(0.05) + + def _active_stream_recovery_projection( + self, + runtime: Mapping[str, Any], + *, + camera: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + runtime_recovery = runtime.get("connection_recovery") + runtime_state = ( + str(runtime_recovery.get("state")) + if isinstance(runtime_recovery, Mapping) + else "inactive" + ) + runtime_attempt = ( + runtime_recovery.get("attempt") if isinstance(runtime_recovery, Mapping) else 0 + ) + runtime_phase = runtime.get("phase") + with self._lock: + lineage = self._active_stream_recovery_lineage + pending = self._restart_rehydration_pending + acquisition = self._acquisition + acquisition_out_dir = self._acquisition_out_dir + acquisition_lease = self._acquisition_session_lease + exact_restart_owner = bool( + lineage is not None + and pending is not None + and acquisition is not None + and acquisition.acquisition_id == pending.token.acquisition_id + and lineage.acquisition_id == pending.token.acquisition_id + and lineage.start_operation_id == pending.token.root_start_operation_id + and lineage.evidence_session_id == pending.new_evidence_session_id + and lineage.runtime_producer_generation + == pending.runtime_producer_generation + and acquisition_out_dir == pending.out_dir + and acquisition_lease is not None + and acquisition_lease.session_root == pending.out_dir + and runtime.get("producer_generation") + == pending.runtime_producer_generation + ) + restart_pre_pcl = bool( + exact_restart_owner + and pending is not None + and pending.phase in {"awaiting-suback", "awaiting-first-pcl"} + ) + restart_blocked = bool( + exact_restart_owner + and pending is not None + and pending.phase == "blocked" + ) + runtime_owns_live_source = bool( + runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and runtime.get("source_ready") is True + ) + if ( + lineage is not None + and runtime_state == "recovered" + and runtime_owns_live_source + and runtime.get("producer_generation") == lineage.runtime_producer_generation + and self._active_stream_recovery_generation == lineage.recovery_generation + and self._active_stream_recovery_state == "reconnecting" + and not exact_restart_owner + ): + self._active_stream_recovery_state = "recovered" + self._active_stream_recovery_reason_code = None + state = self._active_stream_recovery_state + reason_code = self._active_stream_recovery_reason_code + if restart_pre_pcl: + state = "reconnecting" + reason_code = ( + pending.reason_code + if pending is not None and pending.reason_code is not None + else "restart-receiver-awaiting-suback" + if pending is not None and pending.phase == "awaiting-suback" + else "restart-receiver-awaiting-first-pcl" + ) + elif restart_blocked: + state = "blocked" + reason_code = ( + pending.reason_code + if pending is not None and pending.reason_code is not None + else "restart-receiver-setup-failed" + ) + elif ( + state in {"reconnecting", "recovered", "blocked"} + and runtime_phase not in {"live", "reconnecting"} + ) or ( + state == "recovered" and not runtime_owns_live_source + ): + state = "inactive" + if runtime_phase == "reconnecting" and state == "inactive": + state = "reconnecting" + projected_lineage = lineage if state != "inactive" else None + attempt = max( + self._active_stream_recovery_attempt, + runtime_attempt + if isinstance(runtime_attempt, int) and not isinstance(runtime_attempt, bool) + else 0, + ) + started_monotonic = self._active_stream_recovery_started_monotonic + elapsed_ms = ( + int(max(time.monotonic() - started_monotonic, 0.0) * 1000) + if started_monotonic is not None and state in {"reconnecting", "blocked"} + else None + ) + camera_snapshot = camera or self.camera_preview.snapshot() + camera_recording = camera_snapshot.get("recording") + camera_current_epoch = ( + camera_recording.get("current_epoch") + if isinstance(camera_recording, Mapping) + else None + ) + active_acquisition_camera_session = ( + acquisition_out_dir.name + if acquisition is not None + and acquisition.state not in TERMINAL_ACQUISITION_STATES + and acquisition_out_dir is not None + else None + ) + expected_camera_session = ( + projected_lineage.evidence_session_id + if projected_lineage is not None + else active_acquisition_camera_session + ) + exact_camera_owner = bool( + expected_camera_session is not None + and camera_snapshot.get("active_source_id") + == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(camera_recording, Mapping) + and camera_recording.get("active") is True + and camera_recording.get("session") == expected_camera_session + ) + camera_media_ready = bool( + not (restart_pre_pcl or restart_blocked) + and exact_camera_owner + and isinstance(camera_recording, Mapping) + and camera_recording.get("media_ready") is True + and isinstance(camera_current_epoch, Mapping) + and camera_current_epoch.get("generation") + == camera_recording.get("active_epoch") + and camera_current_epoch.get("init_committed") is True + and camera_current_epoch.get("first_media_committed") is True + ) + camera_media_state = ( + "inactive" + if restart_pre_pcl or restart_blocked or not exact_camera_owner + else "ready" + if camera_media_ready + else "pending-epoch" + if not isinstance(camera_current_epoch, Mapping) + else "pending-init" + if camera_current_epoch.get("init_committed") is not True + else "pending-first-media" + ) + camera_pending_owned = bool( + not (restart_pre_pcl or restart_blocked) + and exact_camera_owner + and not camera_media_ready + and runtime_phase in {"live", "reconnecting"} + and ( + camera_snapshot.get("phase") in {"selected", "connecting"} + or self._active_stream_camera_recovery_reason(camera_snapshot) + is not None + ) + ) + return { + "schema_version": ("missioncore.xgrids-k1-active-stream-recovery/v1"), + "state": state, + "generation": self._active_stream_recovery_generation, + "acquisition_id": ( + projected_lineage.acquisition_id + if projected_lineage is not None + else None + ), + "attempt": attempt, + "started_at_utc": self._active_stream_recovery_started_at_utc, + "elapsed_ms": elapsed_ms, + "reason_code": reason_code, + "force_finish_allowed": state in {"reconnecting", "blocked"}, + "automatic_read_only_rebind": projected_lineage is not None, + "automatic_command_retry": False, + "start_performed": False, + "stop_performed": False, + "ble_operation_performed": False, + "network_mutation_performed": False, + "runtime_producer_generation": ( + projected_lineage.runtime_producer_generation + if projected_lineage is not None + else runtime.get("producer_generation") + ), + "camera_recovery": ( + "blocked" + if reason_code == "camera-recovery-failed" + else "owned" + if not (restart_pre_pcl or restart_blocked) + and (projected_lineage is not None or camera_pending_owned) + else "inactive" + ), + # Plane-wise truth: runtime/control may already be recovered + # while the same-lineage camera epoch is still pending media. + "camera_media_state": camera_media_state, + "camera_media_ready": camera_media_ready, + "camera_epoch": ( + dict(camera_current_epoch) + if not (restart_pre_pcl or restart_blocked) + and exact_camera_owner + and isinstance(camera_current_epoch, Mapping) + else None + ), + } def _stop_acquisition_sources( self, @@ -3016,6 +24974,7 @@ class XgridsK1CompatibilityService: camera_error = exc try: self.runtime.stop() + self._reset_live_data_plane_observation() except Exception as exc: runtime_error = exc if camera_error is not None: @@ -3034,6 +24993,15 @@ class XgridsK1CompatibilityService: finally: if out_dir is not None: self.live_perception_ingress.end_session(out_dir.name) + with self._lock: + self._live_perception_camera_binding = None + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = None + self._camera_activation_retry_not_before_monotonic = 0.0 + self._post_recovery_camera_restart_claim = None + self._post_recovery_camera_restart_inflight = None + self._post_recovery_camera_restart_retry = None + self._post_recovery_camera_restart_terminal = None if cleanup_complete: self._release_acquisition_session_lease() @@ -3087,20 +25055,106 @@ class XgridsK1CompatibilityService: def _release_acquisition_session_lease(self) -> None: with self._lock: lease = self._acquisition_session_lease - self._acquisition_session_lease = None + prepared_stop_owner = getattr(self, "_prepared_stop_recovery_owner", None) + operator_manual_acquisition_id = self._operator_manual_acquisition_process_lease_id + # Do not clear the visible ownership fence until the underlying lease + # release itself succeeds. A filesystem/process-lease failure must + # remain retryable and continue to project cleanup_pending=true. if lease is not None: lease.release() + if operator_manual_acquisition_id is not None: + self._release_k1_lifecycle_process_lease("network") + with self._lock: + if self._acquisition_session_lease is lease: + self._acquisition_session_lease = None + if ( + prepared_stop_owner is not None + and prepared_stop_owner.session_lease is lease + and getattr(self, "_prepared_stop_recovery_owner", None) + is prepared_stop_owner + ): + self._prepared_stop_recovery_owner = None + self._prepared_stop_dispatch_lineage = None + if lease is not None: + self._classified_stop_rebind_pending = None + self._classified_stop_rebind_inflight = None + if ( + self._operator_manual_acquisition_process_lease_id + == operator_manual_acquisition_id + ): + self._operator_manual_acquisition_process_lease_id = None + if lease is not None or operator_manual_acquisition_id is not None: + future = self._active_stream_recovery_future + self._active_stream_recovery_generation += 1 + self._active_stream_recovery_lineage = None + self._active_stream_recovery_future = None + self._active_stream_recovery_state = "inactive" + self._active_stream_recovery_attempt = 0 + self._active_stream_recovery_reason_code = None + self._active_stream_recovery_started_at_utc = None + self._active_stream_recovery_started_monotonic = None + self._active_stream_recovery_terminal_outcome = None + else: + future = None + if future is not None: + future.cancel() + + def _release_operator_manual_acquisition_process_lease( + self, + acquisition_id: str, + ) -> None: + with self._lock: + if self._operator_manual_acquisition_process_lease_id != acquisition_id: + return + self._operator_manual_acquisition_process_lease_id = None + self._release_k1_lifecycle_process_lease("network") def _terminalize_acquisition_operations_on_shutdown( self, terminal_error: Exception | None, + *, + fenced_stop_operation_id: str | None, ) -> None: with self._lock: + acquisition = self._acquisition + acquisition_id = acquisition.acquisition_id if acquisition is not None else None + stop_operation_id = ( + self._acquisition_stop_operation_id or fenced_stop_operation_id + ) pending = ( ("start", self._acquisition_start_operation_id), - ("stop", self._acquisition_stop_operation_id), + ("stop", stop_operation_id), + ) + physical = self._physical_command_coordinator.snapshot() + unresolved_stop = self._matching_unresolved_stop_stage( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + resolved_standby_stop_operation_id = ( + self._matching_resolved_stop_standby_owner( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + ) + preserved_stop_operation_id = ( + unresolved_stop[1] + if unresolved_stop is not None + else resolved_standby_stop_operation_id + ) + if isinstance(acquisition_id, str): + self._terminalize_pending_start_for_local_stop_retirement( + acquisition_id=acquisition_id, + stop_operation_id=preserved_stop_operation_id, ) for action_kind, operation_id in pending: + if action_kind == "stop" and operation_id == preserved_stop_operation_id: + # The dispatch-fenced close has already sealed this row as + # definitely not-dispatched or conservatively unknown. Keep its + # exact local pointer until the durable physical row is later + # reconciled/retired; generic shutdown cancellation would lie. + continue if terminal_error is None: self._operations.transition_if_pending( operation_id, @@ -3122,7 +25176,10 @@ class XgridsK1CompatibilityService: ) with self._lock: self._acquisition_start_operation_id = None - self._acquisition_stop_operation_id = None + if self._acquisition_stop_operation_id != preserved_stop_operation_id: + self._acquisition_stop_operation_id = None + if preserved_stop_operation_id is None: + self._prepared_stop_dispatch_lineage = None def _set_operation(self, phase: str, message: str) -> None: with self._lock: @@ -3143,14 +25200,20 @@ class XgridsK1CompatibilityService: *, exclude_operation_id: str, reason_code: str, + preserve_operation_ids: Collection[str] = (), ) -> None: + preserved = frozenset(preserve_operation_ids) with self._lock: pending = ( ("start", self._acquisition_start_operation_id), ("stop", self._acquisition_stop_operation_id), ) for action_kind, operation_id in pending: - if operation_id is None or operation_id == exclude_operation_id: + if ( + operation_id is None + or operation_id == exclude_operation_id + or operation_id in preserved + ): continue self._operations.transition_if_pending( operation_id, @@ -3159,9 +25222,15 @@ class XgridsK1CompatibilityService: message_code=f"acquisition.{action_kind}.{reason_code}", ) with self._lock: - if self._acquisition_start_operation_id != exclude_operation_id: + if self._acquisition_start_operation_id not in { + exclude_operation_id, + *preserved, + }: self._acquisition_start_operation_id = None - if self._acquisition_stop_operation_id != exclude_operation_id: + if self._acquisition_stop_operation_id not in { + exclude_operation_id, + *preserved, + }: self._acquisition_stop_operation_id = None def _ensure_device_context(self) -> tuple[str, str]: @@ -3175,12 +25244,17 @@ class XgridsK1CompatibilityService: def _camera_target_for_session(self, device_session_id: str) -> str: with self._lock: - current_session_id = self._device_session_id + selected_device_id = self._selected_device_id target = self._k1_ip attestation = self._compatibility_attestation connection_mode = self._connection_mode - if current_session_id is None or device_session_id != current_session_id: - raise ValueError("указана неактивная device-сессия") + self._require_camera_device_session(device_session_id) + if selected_device_id is None: + raise ValueError("для camera preview не выбрано устройство K1") + self._require_transport_not_physically_retired( + selected_device_id, + verification=True, + ) if attestation is None: raise ValueError("точный compatibility-профиль K1 не подтверждён") if target is None: @@ -3190,8 +25264,26 @@ class XgridsK1CompatibilityService: raise ValueError("адрес точки доступа K1 не принят для этой device-сессии") if connection_mode == "quick-connect" and target != AP_FALLBACK_IPV4: raise ValueError("Quick Connect camera target не совпадает с AP-адресом K1") + supervisor = self._connection_supervisor.snapshot() + if ( + not supervisor.authority.control_allowed + or supervisor.lease.target is None + or supervisor.lease.target.ipv4 != target + or supervisor.lease.connection_mode != connection_mode + ): + raise ValueError( + "камера недоступна: текущая MQTT/DeviceInfo authority для этой сети K1 отсутствует" + ) return target + def _require_camera_device_session(self, device_session_id: str) -> None: + """Validate local camera ownership without requiring a live target.""" + + with self._lock: + current_session_id = self._device_session_id + if current_session_id is None or device_session_id != current_session_id: + raise ValueError("указана неактивная device-сессия") + def _require_acquisition(self, acquisition_id: str | None) -> AcquisitionRecord: with self._lock: acquisition = self._acquisition @@ -3201,68 +25293,3268 @@ class XgridsK1CompatibilityService: raise ValueError("указана неизвестная acquisition-сессия") return acquisition + def _reconcile_connection_supervisor( + self, + application_control_session: Mapping[str, Any], + runtime: Mapping[str, Any], + ) -> None: + """Reduce live control/data evidence without inventing reconnects. + + A TCP listener is transport evidence only. Control authority is + promoted exclusively by the DeviceInfo proof emitted by the canonical + application dialogue and only for the same current intent, target and + host-route epoch. + """ + + control_state = str(application_control_session.get("state") or "unknown") + verified_control = application_control_session.get("verified_control") + with self._lock: + transport_ref = self._selected_device_id + last_data_at = self._last_live_data_monotonic + last_data_suspend_aware = self._last_live_data_suspend_aware + last_data_session_id = self._last_live_data_session_id + + active_control_states = { + "connection-ready", + "workspace-requested", + "workspace-ready", + "project-requested", + "project-ready", + "start-requested", + "initializing", + "scanning", + "stop-requested", + "stopping", + "awaiting-standby-confirmation", + } + supervisor = self._connection_supervisor.snapshot() + if ( + control_state in active_control_states + and isinstance(verified_control, Mapping) + and supervisor.intent is not None + and supervisor.endpoint.target is not None + and supervisor.endpoint.tcp_state == "reachable" + and supervisor.endpoint.host_path_epoch == supervisor.host_path.epoch + ): + logical_device_id = verified_control.get("logical_device_id") + profile_id = verified_control.get("compatibility_profile_id") + control_session_id = verified_control.get("control_session_id") + bound_intent_id = verified_control.get("intent_id") + bound_transport_ref = verified_control.get("transport_ref") + bound_host_epoch = verified_control.get("host_path_epoch") + bound_target_ipv4 = verified_control.get("target_ipv4") + bound_target_port = verified_control.get("target_port") + bound_connection_mode = verified_control.get("connection_mode") + control_proof_revision = verified_control.get("control_proof_revision") + control_proof_source = verified_control.get("control_proof_source") + control_proof_fresh = verified_control.get("control_proof_fresh") + if ( + all( + isinstance(value, str) and bool(value.strip()) + for value in ( + logical_device_id, + profile_id, + control_session_id, + bound_intent_id, + bound_transport_ref, + bound_target_ipv4, + bound_connection_mode, + ) + ) + and isinstance(bound_host_epoch, int) + and isinstance(bound_target_port, int) + ): + assert isinstance(logical_device_id, str) + assert isinstance(profile_id, str) + assert isinstance(control_session_id, str) + assert isinstance(bound_intent_id, str) + assert isinstance(bound_transport_ref, str) + assert isinstance(bound_target_ipv4, str) + assert isinstance(bound_connection_mode, str) + evidence_is_current = bool( + bound_intent_id == supervisor.intent.intent_id + and transport_ref is not None + and bound_transport_ref == transport_ref + and bound_transport_ref == supervisor.device_network.transport_ref + and bound_host_epoch == supervisor.host_path.epoch + and bound_target_ipv4 == supervisor.endpoint.target.ipv4 + and bound_target_port == supervisor.endpoint.target.port + and bound_connection_mode == supervisor.intent.requested_mode + and profile_id == XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ) + already_current = bool( + supervisor.control_plane.state == "healthy" + and supervisor.control_plane.session_id == control_session_id + and supervisor.device_identity.logical_device_id == logical_device_id + and supervisor.lease.intent_id == supervisor.intent.intent_id + and supervisor.lease.host_path_epoch == supervisor.host_path.epoch + ) + proof_shape_valid = bool( + isinstance(control_proof_revision, int) + and not isinstance(control_proof_revision, bool) + and control_proof_revision >= 1 + and isinstance(control_proof_source, str) + and bool(control_proof_source.strip()) + and isinstance(control_proof_fresh, bool) + ) + exact_current_control = evidence_is_current and already_current + if exact_current_control and ( + not proof_shape_valid or control_proof_fresh is not True + ): + self._connection_supervisor.observe_control_loss( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + control_session_id=control_session_id, + reason_code=( + "control-proof-stale" if proof_shape_valid else "control-proof-invalid" + ), + ) + identity_pin_verified = False + if evidence_is_current and proof_shape_valid and control_proof_fresh is True: + try: + self._pin_or_match_device_identity( + transport_ref=bound_transport_ref, + logical_device_id=logical_device_id, + compatibility_profile_id=profile_id, + ) + identity_pin_verified = True + except DeviceIdentityPinStoreError as exc: + reason_code = getattr( + exc, + "reason_code", + DeviceIdentityPinStoreError.reason_code, + ) + with self._lock: + self._device_identity_pin_store_reason = reason_code + if ( + supervisor.control_plane.state == "healthy" + and supervisor.control_plane.session_id == control_session_id + ): + self._connection_supervisor.observe_control_loss( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + control_session_id=control_session_id, + reason_code=reason_code, + ) + logger.error( + "K1 DeviceInfo identity rejected by durable transport pin", + extra={ + "event_code": "k1_device_identity_pin_rejected", + "reason_code": reason_code, + "automatic_retry": False, + }, + ) + if ( + evidence_is_current + and identity_pin_verified + and proof_shape_valid + and control_proof_fresh is True + and not already_current + ): + assert isinstance(control_proof_revision, int) + assert isinstance(control_proof_source, str) + accepted = self._connection_supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id=bound_intent_id, + transport_ref=bound_transport_ref, + host_path_epoch=bound_host_epoch, + target=EndpointTarget( + validate_private_ipv4(bound_target_ipv4), + bound_target_port, + ), + connection_mode=cast(ConnectionMode, bound_connection_mode), + logical_device_id=logical_device_id, + compatibility_profile_id=profile_id, + control_session_id=control_session_id, + control_proof_revision=control_proof_revision, + control_proof_source=control_proof_source, + ) + ) + if not accepted: + logger.warning( + "K1 DeviceInfo identity became stale before supervisor admission", + extra={ + "event_code": "k1_device_identity_admission_stale", + "reason_code": "device-identity-evidence-stale", + "automatic_retry": False, + }, + ) + elif ( + evidence_is_current + and identity_pin_verified + and proof_shape_valid + and control_proof_fresh is True + and already_current + ): + assert isinstance(control_proof_revision, int) + assert isinstance(control_proof_source, str) + self._connection_supervisor.refresh_control_evidence( + intent_id=bound_intent_id, + host_path_epoch=bound_host_epoch, + control_session_id=control_session_id, + control_proof_revision=control_proof_revision, + control_proof_source=control_proof_source, + ) + + supervisor = self._connection_supervisor.snapshot() + if ( + control_state in {"idle", "completed", "closed", "failed"} + and supervisor.control_plane.state == "healthy" + and supervisor.control_plane.session_id is not None + and supervisor.intent is not None + and supervisor.host_path.epoch > 0 + ): + terminal_reason = { + "completed": "control-session-completed", + "closed": "control-session-closed", + "failed": "control-session-failed", + "idle": "control-session-idle", + }[control_state] + failure = application_control_session.get("failure") + if control_state == "failed" and isinstance(failure, Mapping): + reported = failure.get("reason_code") + if isinstance(reported, str) and reported: + terminal_reason = reported + self._connection_supervisor.observe_control_loss( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + control_session_id=supervisor.control_plane.session_id, + reason_code=terminal_reason, + ) + + supervisor = self._connection_supervisor.snapshot() + exact_data_session = bool( + not supervisor.closed + and supervisor.intent is not None + and last_data_session_id is not None + and supervisor.data_plane.state in {"healthy", "stalled", "lost"} + and supervisor.data_plane.session_id == last_data_session_id + and supervisor.data_plane.host_path_epoch is not None + ) + if runtime.get("source_mode") == "live" and exact_data_session: + assert supervisor.intent is not None + assert last_data_session_id is not None + assert supervisor.data_plane.host_path_epoch is not None + runtime_phase = str(runtime.get("phase") or "unknown") + monotonic_age = time.monotonic() - last_data_at if last_data_at is not None else None + suspend_aware_age = ( + time.time() - last_data_suspend_aware + if last_data_suspend_aware is not None + else None + ) + observed_ages = tuple( + age for age in (monotonic_age, suspend_aware_age) if age is not None + ) + data_age = ( + float("inf") + if any(age < 0 for age in observed_ages) + else max(observed_ages, default=None) + ) + if runtime_phase == "error": + if ( + supervisor.data_plane.state in {"healthy", "stalled"} + and last_data_session_id is not None + ): + self._connection_supervisor.observe_data_plane( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.data_plane.host_path_epoch, + state="lost", + session_id=last_data_session_id, + reason_code="live-runtime-error", + ) + elif ( + data_age is not None + and data_age > LIVE_DATA_PLANE_LOST_SECONDS + and supervisor.data_plane.state in {"healthy", "stalled"} + ): + self._connection_supervisor.observe_data_plane( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.data_plane.host_path_epoch, + state="lost", + session_id=last_data_session_id, + reason_code="live-data-lost", + ) + elif ( + data_age is not None + and data_age > LIVE_DATA_PLANE_STALL_SECONDS + and supervisor.data_plane.state == "healthy" + ): + self._connection_supervisor.observe_data_plane( + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.data_plane.host_path_epoch, + state="stalled", + session_id=last_data_session_id, + reason_code="live-data-stalled", + ) + + def _retire_ephemeral_connection_binding_on_proven_loss( + self, + application_control_session: Mapping[str, Any], + ) -> None: + """Forget one dead runtime binding without touching durable topology. + + Host route and TCP probes are read-only and may fail transiently, so + three consecutive supervisor observations are required. A terminal + terminal MQTT-loop or exact application-binding loss comes from the + canonical socket owner and is conclusive immediately. Retirement never reconnects, + scans, reads GATT, or repeats a network write. A still-fresh + advertisement is independent evidence and remains available for + read-only observation/Verify. If physical state is active or unknown, + provisioning remains forbidden until fresh READY evidence resolves it. + """ + + # A composite-confirmed active START owns the sole automatic exception: + # preserve its exact topology while the data owner performs a bounded, + # inspection-only rebind. This path never authorizes a command retry. + if self._active_stream_recovery_admitted_for_runtime_loss(): + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + return + + # STOP PREPARE replaces the current successful START ledger row before + # the transport can acquire its dispatch fence. Preserve only this + # exact, bounded, still-undispatched owner; a terminal control state, + # deadline, generation drift, or any ledger stage after PREPARED falls + # through to the normal fail-closed loss reducer. + if self._prepared_stop_dispatch_lineage_is_current( + control=application_control_session, + ): + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + return + + supervisor = self._connection_supervisor.snapshot() + with self._lock: + selected_device_id = self._selected_device_id + device_session_id = self._device_session_id + discovery_generation = self._ble_discovery_generation + session_discovery_generation = ( + self._device_session_discovery_generation + if self._device_session_discovery_generation is not None + else discovery_generation + ) + if selected_device_id is None or device_session_id is None: + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + return + + failure = application_control_session.get("failure") + terminal_failure_reason = ( + failure.get("reason_code") if isinstance(failure, Mapping) else None + ) + terminal_control_loss = bool( + application_control_session.get("state") == "failed" + and terminal_failure_reason + in { + "mqtt_network_loop_failed", + "application-connection-binding-lost", + } + ) + terminal_control_generation = application_control_session.get("session_generation") + terminal_control_revision = application_control_session.get("state_revision") + loss_class: str | None = None + reason_code: str | None = None + current_applied_context = bool( + supervisor.intent is not None + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.transport_ref == selected_device_id + and supervisor.device_network.connection_mode == supervisor.intent.requested_mode + and supervisor.device_network.target is not None + ) + if terminal_control_loss: + loss_class = ( + "mqtt-control" + if terminal_failure_reason == "mqtt_network_loop_failed" + else "application-binding" + ) + reason_code = str(terminal_failure_reason) + elif ( + current_applied_context + and not supervisor.host_path.available + and supervisor.host_path.epoch > 0 + ): + loss_class = "host-path" + reason_code = supervisor.host_path.reason_code or "host-path-unavailable" + elif current_applied_context and supervisor.endpoint.tcp_state == "unreachable": + loss_class = "control-endpoint" + reason_code = supervisor.endpoint.reason_code or "endpoint-unreachable" + + if loss_class is None or reason_code is None: + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + return + + intent_id = supervisor.intent.intent_id if supervisor.intent is not None else "" + target_ipv4 = ( + supervisor.device_network.target.ipv4 + if supervisor.device_network.target is not None + else "" + ) + signature_prefix = ( + selected_device_id, + device_session_id, + loss_class, + intent_id, + target_ipv4, + ) + evidence_token: tuple[object, ...] + if loss_class == "host-path": + evidence_token = ( + "host-path", + supervisor.host_path.observed_at, + supervisor.host_path.epoch, + supervisor.host_path.available, + supervisor.host_path.reason_code, + ) + elif loss_class == "control-endpoint": + evidence_token = ( + "control-endpoint", + supervisor.endpoint.observed_at, + supervisor.endpoint.host_path_epoch, + supervisor.endpoint.tcp_state, + supervisor.endpoint.reason_code, + ) + else: + evidence_token = (loss_class, reason_code) + plane_negative_streak = ( + supervisor.host_path_negative_streak + if loss_class == "host-path" + else supervisor.endpoint_negative_streak + if loss_class == "control-endpoint" + else 0 + ) + confirmed = terminal_control_loss + with self._lock: + if not confirmed: + retained_prefix = ( + self._connection_loss_signature[:5] + if self._connection_loss_signature is not None + else None + ) + if retained_prefix != signature_prefix: + self._connection_loss_signature = ( + *signature_prefix, + session_discovery_generation, + ) + self._connection_loss_evidence_token = evidence_token + # The supervisor observes every monitor sample and resets the + # streak on a positive edge, even when no UI/state reduction + # happens between NEG -> POS -> NEG. Repeated state() calls + # therefore cannot manufacture confirmations. + self._connection_loss_confirmation_count = plane_negative_streak + confirmed = self._connection_loss_confirmation_count >= 3 + if not confirmed: + return + retained_signature = self._connection_loss_signature + loss_discovery_generation = ( + session_discovery_generation + if terminal_control_loss or retained_signature is None + else retained_signature[5] + ) + + # Serialize final teardown against a network mutation. If connect is + # already in flight, defer retirement to a later state reduction. + if not self._k1_lifecycle_transition_gate.acquire(blocking=False): + return + dispatch_gate_acquired = False + try: + # The transport holds this gate across fresh path validation, + # durable mark_dispatching and client.publish. Teardown must not + # revoke that topology in the small mark_dispatching -> publish + # window. A genuine route loss still fails the validator before + # publish and releases the gate; the next reduction then retires. + dispatch_gate_acquired = self._k1_command_dispatch_gate.acquire(blocking=False) + if not dispatch_gate_acquired: + return + # Proven-loss teardown is retryable state reduction. Defer it + # instead of blocking a state poll behind an in-flight monitor. + if not self._connection_monitor_contact_gate.acquire(blocking=False): + return + self._connection_monitor_contact_gate.release() + current_supervisor = self._connection_supervisor.snapshot() + current_control = dict(self._application_control_session.snapshot()) + current_failure = current_control.get("failure") + current_failure_reason = ( + current_failure.get("reason_code") + if isinstance(current_failure, Mapping) + else None + ) + current_terminal_control_loss = bool( + terminal_control_loss + and current_control.get("state") == "failed" + and current_failure_reason == terminal_failure_reason + and current_control.get("session_generation") == terminal_control_generation + and current_control.get("state_revision") == terminal_control_revision + ) + current_intent_id = ( + current_supervisor.intent.intent_id if current_supervisor.intent is not None else "" + ) + current_target_ipv4 = ( + current_supervisor.device_network.target.ipv4 + if current_supervisor.device_network.target is not None + else "" + ) + current_applied_context = bool( + current_supervisor.intent is not None + and current_supervisor.device_network.state == "applied" + and current_supervisor.device_network.intent_id + == current_supervisor.intent.intent_id + and current_supervisor.device_network.transport_ref == selected_device_id + and current_supervisor.device_network.connection_mode + == current_supervisor.intent.requested_mode + and current_supervisor.device_network.target is not None + ) + current_loss_still_confirmed = bool( + current_terminal_control_loss + or ( + loss_class == "host-path" + and current_applied_context + and not current_supervisor.host_path.available + and current_supervisor.host_path_negative_streak >= 3 + ) + or ( + loss_class == "control-endpoint" + and current_applied_context + and current_supervisor.endpoint.tcp_state == "unreachable" + and current_supervisor.endpoint_negative_streak >= 3 + ) + ) + if ( + current_intent_id != intent_id + or current_target_ipv4 != target_ipv4 + or current_supervisor.device_network.transport_ref != selected_device_id + or not current_loss_still_confirmed + ): + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + return + if self._request_active_stream_recovery_for_local_loss(reason_code): + # The exact composite START now owns this topology. Leave the + # selected transport/control evidence intact so the existing + # capture thread can perform the reviewed read-only rebind. + with self._lock: + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + logger.info( + "K1 active stream recovery woke before binding retirement", + extra={ + "event_code": "k1_active_stream_recovery_proactive_wake", + "reason_code": reason_code, + "automatic_retry": True, + "device_command_performed": False, + "network_mutation_performed": False, + }, + ) + return + with self._lock: + # Do not let a late monitor result retire a connection that + # was replaced while the evidence was being reduced. + if ( + self._selected_device_id != selected_device_id + or self._device_session_id != device_session_id + ): + return + fresh_devices = self._fresh_ble_devices_locked() + fresh_selected_advertised = any( + str(item.get("device_id")) == selected_device_id for item in fresh_devices + ) + if self._ble_discovery_generation == loss_discovery_generation: + if fresh_selected_advertised: + # The dead LAN/MQTT lease does not invalidate fresh BLE + # evidence from the operator's scan. Keep it as the + # exact target for read-only observation/Verify; this + # teardown itself performs no device I/O and does not + # authorize a new network mutation. + fresh_device_ids = {str(item.get("device_id")) for item in fresh_devices} + self._devices = fresh_devices + self._ble_device_last_seen_monotonic = { + device_id: observed_at + for device_id, observed_at in ( + self._ble_device_last_seen_monotonic.items() + ) + if device_id in fresh_device_ids + } + self._ble_device_last_seen_suspend_aware = { + device_id: observed_at + for device_id, observed_at in ( + self._ble_device_last_seen_suspend_aware.items() + ) + if device_id in fresh_device_ids + } + else: + self._ble_discovery_generation += 1 + self._devices = [] + self._ble_device_last_seen_monotonic = {} + self._ble_device_last_seen_suspend_aware = {} + self._selected_device_id = None + self._k1_ip = None + self._connection_mode = None + self._device_session_id = None + self._device_session_opened_at = None + self._device_session_discovery_generation = None + self._compatibility_attestation = None + self._device_calibration = unavailable_device_calibration_snapshot( + XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ) + self._connection_verification = { + "status": "connection-lost", + "lease_state": "disconnected", + "lease_generation": self._connection_lease_generation, + "endpoint_validation": loss_class, + "network_reachability": "unreachable", + "reason_code": reason_code, + "write_performed": False, + "observed_at": _utc_now_iso(), + } + self._operation_phase = None + self._operation_message = ( + "Старый сетевой адрес K1 недоступен. Свежий Bluetooth-кандидат " + "сохранён только для проверки исходного устройства. " + "Настройки сети недоступны до подтверждённого READY." + if fresh_selected_advertised + else "Текущая связь с K1 потеряна. Выполните новый Bluetooth-поиск " + "и read-only проверку исходного K1." + ) + self._pending_local_control_retirement = True + self._connection_loss_signature = None + self._connection_loss_evidence_token = None + self._connection_loss_confirmation_count = 0 + finally: + if dispatch_gate_acquired: + self._k1_command_dispatch_gate.release() + self._k1_lifecycle_transition_gate.release() + + # This is local teardown only. In particular, do not call any BLE or + # network provisioning helper here: the operator owns the next intent. + invalidate_connected_device_session( + selected_device_id, + device_session_id=device_session_id, + ) + try: + self._retire_application_control_for_network_change() + except (AttributeError, RuntimeError): + logger.warning( + "K1 runtime binding cleared after proven loss while control owner was terminal", + extra={ + "event_code": "k1_connection_binding_retired_after_loss", + "reason_code": reason_code, + "automatic_retry": False, + }, + ) + + def _reconcile_terminal_control_transport( + self, + application_control_session: Mapping[str, Any], + ) -> None: + """Revoke a reachable claim after the canonical MQTT owner has failed. + + The K1 address remains last-known evidence. A terminal network-loop or + exact application-binding failure proves that the process no longer + owns a live control route, so the snapshot must not continue presenting + that address as reachable. + """ + + failure = application_control_session.get("failure") + if ( + application_control_session.get("state") != "failed" + or not isinstance(failure, Mapping) + or failure.get("reason_code") + not in { + "mqtt_network_loop_failed", + "application-connection-binding-lost", + } + ): + return + with self._lock: + current = self._connection_verification + if ( + self._k1_ip is None + or current.get("reason_code") == "connection_lease_control_transport_lost" + or ( + current.get("lease_state") != "reachable" + and current.get("network_reachability") != "reachable" + ) + ): + return + previous_observed_at = current.get("observed_at") + self._connection_verification = { + **current, + "status": "control-transport-lost", + "lease_state": "disconnected", + "endpoint_validation": ( + "terminal-mqtt-network-loop" + if failure.get("reason_code") == "mqtt_network_loop_failed" + else "terminal-application-binding" + ), + "network_reachability": "unreachable", + "reason_code": "connection_lease_control_transport_lost", + "last_known_reachable_at": previous_observed_at, + "observed_at": _utc_now_iso(), + } + + @staticmethod + def _stop_acknowledged_standby_unconfirmed( + terminal_control_proof: Mapping[str, Any] | None, + physical_command_proof: Mapping[str, Any] | None, + ) -> bool: + """Recognize an accepted STOP whose final READY edge is still unknown. + + The control-session failure projection is useful during the first + reduction, but it is intentionally ephemeral. The physical ledger is + the durable fallback after a previous poll has already retired that + socket. Both forms require the same exact semantics: correlated STOP + success plus a fresh SCAN_STOPPING observation; neither authorizes a + second STOP. + """ + + failure = ( + terminal_control_proof.get("failure") + if isinstance(terminal_control_proof, Mapping) + else None + ) + if isinstance(failure, Mapping) and failure.get("network_change_admissible") is True: + reconciliation = failure.get("network_change_reconciliation") + if ( + isinstance(reconciliation, Mapping) + and reconciliation.get("stop_complete") is True + and reconciliation.get("standby_confirmed") is False + and reconciliation.get("device_session_state") == "scan_stopping" + ): + return True + + if not isinstance(physical_command_proof, Mapping): + return False + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + response = record.get("application_response") + last_status = record.get("last_status") + return bool( + physical_command_proof.get("status") == "unresolved" + and record.get("action") == "stop" + and record.get("stage") != "resolved" + and record.get("resolution") is None + and isinstance(response, Mapping) + and response.get("action") == "stop" + and response.get("success") is True + and isinstance(last_status, Mapping) + and last_status.get("session_state") == "scan_stopping" + and last_status.get("mqtt_retained") is False + and last_status.get("project_bound") is True + and last_status.get("system_error_code") is None + ) + + @staticmethod + def _matching_stop_standby_confirmed( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + ) -> bool: + """Recognize the durable READY proof for this exact local STOP. + + The application-control worker is intentionally ephemeral: a host-path + loss may retire it to ``idle`` before the next facade poll. The + physical-command ledger is the durable authority for the already + accepted STOP edge. Only the exact acquisition/operation lineage and + its correlated successful response plus fresh non-retained canonical + READY may complete local teardown. This reducer performs no device or + network I/O and can therefore be repeated safely after any poll. + """ + + if ( + not isinstance(physical_command_proof, Mapping) + or not isinstance(acquisition_id, str) + or not acquisition_id + or not isinstance(stop_operation_id, str) + or not stop_operation_id + or physical_command_proof.get("status") != "resolved" + or physical_command_proof.get("requires_reconciliation") is True + ): + return False + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + response = record.get("application_response") + last_status = record.get("last_status") + return bool( + record.get("operation_id") == stop_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "stop-standby-observed" + and record.get("publish_call_returned") is True + and record.get("qos2_completed") is True + and isinstance(record.get("packet_id"), int) + and not isinstance(record.get("packet_id"), bool) + and isinstance(response, Mapping) + and response.get("operation_id") == stop_operation_id + and response.get("action") == "stop" + and response.get("success") is True + and isinstance(last_status, Mapping) + and last_status.get("session_state") == "ready" + and last_status.get("project_bound") is False + and last_status.get("init_ready") is False + and last_status.get("mqtt_retained") is False + ) + + @staticmethod + def _matching_start_active_confirmed( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + start_operation_id: str | None, + verified_control: Mapping[str, Any] | None = None, + ) -> bool: + """Recognize canonical active proof for this exact local START. + + Point frames may arrive while the K1 still reports ``SCAN_STARTING``; + they are data-plane progress, not proof that the application START + reached its terminal active state. Plugin-commanded acquisition may + therefore be promoted only after the durable physical ledger has + resolved this exact operation/acquisition lineage from its correlated + response, QoS2 completion and fresh non-retained initialized SCANNING + status. This reducer is local-only and safe to repeat after a poll or + process-local control retirement. + """ + + if ( + not isinstance(physical_command_proof, Mapping) + or not isinstance(acquisition_id, str) + or not acquisition_id + or physical_command_proof.get("status") != "resolved" + or physical_command_proof.get("requires_reconciliation") is True + ): + return False + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + response = record.get("application_response") + last_status = record.get("last_status") + normal_start = bool( + isinstance(start_operation_id, str) + and bool(start_operation_id) + and record.get("operation_id") == start_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "start" + and record.get("stage") == "resolved" + and record.get("resolution") == "start-active-observed" + and record.get("publish_call_returned") is True + and record.get("qos2_completed") is True + and isinstance(record.get("packet_id"), int) + and not isinstance(record.get("packet_id"), bool) + and isinstance(response, Mapping) + and response.get("operation_id") == start_operation_id + and response.get("action") == "start" + and response.get("success") is True + and isinstance(last_status, Mapping) + and last_status.get("session_state") == "scanning" + and last_status.get("project_bound") is True + and last_status.get("init_ready") is True + and last_status.get("mqtt_retained") is False + and last_status.get("system_error_code") is None + ) + if normal_start: + return True + reconciliations = record.get("reconciliations") + latest = ( + reconciliations[-1] + if isinstance(reconciliations, list) and reconciliations + else None + ) + original_attempt = ( + latest.get("original_attempt") if isinstance(latest, Mapping) else None + ) + verified_binding = ( + latest.get("verified_binding") if isinstance(latest, Mapping) else None + ) + reconciled_connection = ( + verified_binding.get("connection") + if isinstance(verified_binding, Mapping) + else None + ) + observation = latest.get("observation") if isinstance(latest, Mapping) else None + latest_attempt_sha256 = ( + latest.get("original_attempt_sha256") + if isinstance(latest, Mapping) + else None + ) + latest_project_id_sha256 = ( + observation.get("project_id_sha256") + if isinstance(observation, Mapping) + else None + ) + resolved_ambiguous_rebind_descendant = bool( + isinstance(latest, Mapping) + and latest.get("kind") == "resolved-active-rebind" + and isinstance(reconciliations, list) + and isinstance(latest_attempt_sha256, str) + and isinstance(latest_project_id_sha256, str) + and any( + isinstance(prior, Mapping) + and prior.get("kind") + in {"ambiguous-outcome", "resolved-active-rebind"} + and prior.get("resolution") == "physical-active-observed" + and prior.get("original_attempt") == original_attempt + and prior.get("original_attempt_sha256") == latest_attempt_sha256 + and isinstance(prior.get("observation"), Mapping) + and prior["observation"].get("source") + == "explicit-read-only-reconciliation" + and prior["observation"].get("session_state") == "scanning" + and prior["observation"].get("project_bound") is True + and prior["observation"].get("init_ready") is True + and prior["observation"].get("mqtt_retained") is False + and prior["observation"].get("project_id_sha256") + == latest_project_id_sha256 + for prior in reconciliations[:-1] + ) + ) + expected_binding = ( + XgridsK1CompatibilityService._exact_control_binding_document( + verified_control + ) + if verified_control is not None + else None + ) + if ( + isinstance(start_operation_id, str) + and record.get("operation_id") == start_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "start" + and record.get("stage") == "resolved" + and record.get("resolution") == "physical-active-observed" + and isinstance(latest, Mapping) + and ( + latest.get("kind") == "ambiguous-outcome" + or resolved_ambiguous_rebind_descendant + ) + and latest.get("resolution") == "physical-active-observed" + and isinstance(original_attempt, Mapping) + and original_attempt.get("operation_id") == start_operation_id + and original_attempt.get("acquisition_id") == acquisition_id + and original_attempt.get("action") == "start" + and original_attempt.get("stage") in {"dispatching", "observing"} + and original_attempt.get("resolution") is None + and isinstance(latest_attempt_sha256, str) + and hmac.compare_digest( + latest_attempt_sha256, + _sha256_audit_document(original_attempt), + ) + and isinstance(reconciled_connection, Mapping) + and expected_binding is not None + and dict(reconciled_connection) == expected_binding + and isinstance(observation, Mapping) + and observation.get("source") + == "explicit-read-only-reconciliation" + and observation.get("session_state") == "scanning" + and observation.get("project_bound") is True + and isinstance(latest_project_id_sha256, str) + and observation.get("init_ready") is True + and observation.get("mqtt_retained") is False + ): + return True + current_operation_id = record.get("operation_id") + effective_start_operation_id = ( + start_operation_id + if isinstance(start_operation_id, str) and start_operation_id + else record.get("parent_operation_id") + ) + return bool( + isinstance(current_operation_id, str) + and isinstance(effective_start_operation_id, str) + and bool(effective_start_operation_id) + and XgridsK1CompatibilityService._matching_classified_prepared_stop_active( + physical_command_proof, + acquisition_id=acquisition_id, + stop_operation_id=current_operation_id, + start_operation_id=effective_start_operation_id, + verified_control=verified_control, + ) + ) + + @staticmethod + def _matching_classified_prepared_stop_active( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + start_operation_id: str | None = None, + verified_control: Mapping[str, Any] | None = None, + ) -> bool: + """Recognize exact SCANNING after a PREPARED STOP was not dispatched.""" + + return XgridsK1CompatibilityService._matching_classified_prepared_stop_state( + physical_command_proof, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + start_operation_id=start_operation_id, + verified_control=verified_control, + expected_state="active", + ) + + @staticmethod + def _matching_classified_prepared_stop_standby( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + start_operation_id: str | None = None, + verified_control: Mapping[str, Any] | None = None, + ) -> bool: + """Recognize exact READY/SCAN_OVER after an unsent PREPARED STOP.""" + + return XgridsK1CompatibilityService._matching_classified_prepared_stop_state( + physical_command_proof, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + start_operation_id=start_operation_id, + verified_control=verified_control, + expected_state="standby", + ) + + @staticmethod + def _matching_classified_prepared_stop_state( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + start_operation_id: str | None, + verified_control: Mapping[str, Any] | None, + expected_state: Literal["active", "standby"], + ) -> bool: + """Validate the one leading classification and its monotonic latest audit.""" + + if ( + not isinstance(physical_command_proof, Mapping) + or not isinstance(acquisition_id, str) + or not acquisition_id + or not isinstance(stop_operation_id, str) + or not stop_operation_id + or physical_command_proof.get("status") != "resolved" + or physical_command_proof.get("requires_reconciliation") is True + or ( + expected_state == "active" + and physical_command_proof.get("resolved_active_recovery_required") + is not True + ) + or ( + expected_state == "standby" + and physical_command_proof.get("reconciled_physical_state") != "standby" + ) + ): + return False + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + if not ( + record.get("operation_id") == stop_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "not-dispatched" + ): + return False + reconciliations = record.get("reconciliations") + if not isinstance(reconciliations, list): + return False + same_operation = [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") == stop_operation_id + ] + classification = next( + ( + item + for item in same_operation + if item.get("kind") == "prepared-stop-classification" + ), + None, + ) + latest = same_operation[-1] if same_operation else None + if not isinstance(classification, Mapping) or not isinstance(latest, Mapping): + return False + if same_operation[0] is not classification or sum( + item.get("kind") == "prepared-stop-classification" + for item in same_operation + ) != 1: + return False + classification_attempt = classification.get("original_attempt") + classification_attempt_sha256 = classification.get("original_attempt_sha256") + classification_binding = classification.get("verified_binding") + classification_connection = ( + classification_binding.get("connection") + if isinstance(classification_binding, Mapping) + else None + ) + classification_observation = classification.get("observation") + original_connection = ( + classification_attempt.get("connection") + if isinstance(classification_attempt, Mapping) + else None + ) + record_connection = record.get("connection") + expected_resolution = ( + "physical-active-observed" + if expected_state == "active" + else "physical-standby-observed" + ) + expected_session_states = ( + {"scanning"} if expected_state == "active" else {"ready", "scan_over"} + ) + immediate_parent_operation_id = record.get("parent_operation_id") + active_start_ancestor_matches = bool( + start_operation_id is None + or record.get("parent_operation_id") == start_operation_id + or ( + isinstance(start_operation_id, str) + and bool(start_operation_id) + and isinstance(immediate_parent_operation_id, str) + and XgridsK1CompatibilityService._classified_stop_chain_reaches_start( + reconciliations, + from_operation_id=immediate_parent_operation_id, + start_operation_id=start_operation_id, + acquisition_id=acquisition_id, + ) + ) + ) + if not ( + active_start_ancestor_matches + and classification.get("resolution") == expected_resolution + and isinstance(classification_attempt, Mapping) + and classification_attempt.get("operation_id") == stop_operation_id + and classification_attempt.get("parent_operation_id") + == record.get("parent_operation_id") + and classification_attempt.get("acquisition_id") == acquisition_id + and classification_attempt.get("action") == "stop" + and ( + ( + classification_attempt.get("stage") == "prepared" + and classification_attempt.get("resolution") is None + ) + or ( + classification_attempt.get("stage") == "resolved" + and classification_attempt.get("resolution") == "not-dispatched" + ) + ) + and classification_attempt.get("publish_call_returned") is None + and classification_attempt.get("packet_id") is None + and classification_attempt.get("qos2_completed") is False + and classification_attempt.get("application_response") is None + and classification_attempt.get("last_status") is None + and classification_attempt.get("identity") == record.get("identity") + and classification_attempt.get("compatibility_profile_id") + == record.get("compatibility_profile_id") + and classification_attempt.get("payload_sha256") + == record.get("payload_sha256") + and classification_attempt.get("baseline_status") + == record.get("baseline_status") + and classification_attempt.get("created_at_utc") + == record.get("created_at_utc") + and isinstance(classification_attempt_sha256, str) + and hmac.compare_digest( + classification_attempt_sha256, + _sha256_audit_document(classification_attempt), + ) + and isinstance(classification_connection, Mapping) + and isinstance(original_connection, Mapping) + and isinstance(record_connection, Mapping) + and dict(record_connection) == dict(original_connection) + and isinstance(classification_observation, Mapping) + and classification_observation.get("source") + == "explicit-read-only-reconciliation" + and classification_observation.get("session_state") + in expected_session_states + and classification_observation.get("project_bound") + is (expected_state == "active") + and classification_observation.get("init_ready") + is (expected_state == "active") + and classification_observation.get("mqtt_retained") is False + and ( + isinstance(classification_observation.get("project_id_sha256"), str) + if expected_state == "active" + else classification_observation.get("project_id_sha256") is None + ) + and classification_observation.get("control_session_id") + == classification_connection.get("control_session_id") + and classification_observation.get("host_path_epoch") + == classification_connection.get("host_path_epoch") + and classification_observation.get("producer_generation") + == classification_connection.get("producer_generation") + ): + return False + if any( + item.get("original_attempt") != classification_attempt + or item.get("original_attempt_sha256") != classification_attempt_sha256 + for item in same_operation + ): + return False + latest_binding = latest.get("verified_binding") + reconciliation_connection = ( + latest_binding.get("connection") + if isinstance(latest_binding, Mapping) + else None + ) + observation = latest.get("observation") + allowed_latest_kinds = ( + {"prepared-stop-classification", "resolved-active-rebind"} + if expected_state == "active" + else {"prepared-stop-classification", "resolved-active-cessation"} + ) + if not ( + latest.get("kind") in allowed_latest_kinds + and latest.get("resolution") == expected_resolution + and isinstance(reconciliation_connection, Mapping) + and isinstance(observation, Mapping) + and observation.get("source") == "explicit-read-only-reconciliation" + and observation.get("session_state") in expected_session_states + and observation.get("project_bound") is (expected_state == "active") + and observation.get("init_ready") is (expected_state == "active") + and observation.get("mqtt_retained") is False + and ( + isinstance(observation.get("project_id_sha256"), str) + if expected_state == "active" + else observation.get("project_id_sha256") is None + ) + and observation.get("control_session_id") + == reconciliation_connection.get("control_session_id") + and observation.get("host_path_epoch") + == reconciliation_connection.get("host_path_epoch") + and observation.get("producer_generation") + == reconciliation_connection.get("producer_generation") + ): + return False + if verified_control is None: + return True + exact_fields = ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + return all( + reconciliation_connection.get(field) == verified_control.get(field) + for field in exact_fields + ) + + @staticmethod + def _classified_stop_chain_reaches_start( + reconciliations: list[Any], + *, + from_operation_id: str, + start_operation_id: str, + acquisition_id: str, + ) -> bool: + """Follow bounded immutable active STOP classifications back to START.""" + + cursor = from_operation_id + seen: set[str] = set() + for _ in range(len(reconciliations) + 1): + if cursor == start_operation_id: + return True + if cursor in seen: + return False + seen.add(cursor) + audits = [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") == cursor + ] + if not audits: + return False + classifications = [ + item + for item in audits + if item.get("kind") == "prepared-stop-classification" + ] + latest = audits[-1] + attempt = latest.get("original_attempt") + attempt_sha256 = latest.get("original_attempt_sha256") + if not ( + len(classifications) == 1 + and audits[0] is classifications[0] + and classifications[0].get("resolution") + == "physical-active-observed" + and latest.get("kind") + in {"prepared-stop-classification", "resolved-active-rebind"} + and latest.get("resolution") == "physical-active-observed" + and isinstance(attempt, Mapping) + and attempt.get("operation_id") == cursor + and attempt.get("acquisition_id") == acquisition_id + and attempt.get("action") == "stop" + and isinstance(attempt.get("parent_operation_id"), str) + and isinstance(attempt_sha256, str) + and hmac.compare_digest( + attempt_sha256, + _sha256_audit_document(attempt), + ) + and all( + item.get("original_attempt") == attempt + and item.get("original_attempt_sha256") == attempt_sha256 + for item in audits + ) + ): + return False + cursor = cast(str, attempt["parent_operation_id"]) + return False + + @staticmethod + def _plugin_start_outcome_unknown( + terminal_control_proof: Mapping[str, Any] | None, + ) -> bool: + """Classify a terminal post-checkpoint START without inventing retry safety.""" + + if ( + not isinstance(terminal_control_proof, Mapping) + or terminal_control_proof.get("state") != "failed" + ): + return False + failure = terminal_control_proof.get("failure") + return bool( + isinstance(failure, Mapping) + and failure.get("modeling_command_attempted") is True + and ( + terminal_control_proof.get("outcome_unknown") is True + or failure.get("safe_to_retry") is False + ) + ) + + @staticmethod + def _matching_stop_response_without_terminal_status( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + ) -> bool: + """Match an accepted STOP whose final physical state is still unknown.""" + + if ( + not isinstance(physical_command_proof, Mapping) + or not isinstance(acquisition_id, str) + or not acquisition_id + or not isinstance(stop_operation_id, str) + or not stop_operation_id + or physical_command_proof.get("status") != "unresolved" + or physical_command_proof.get("requires_reconciliation") is not True + ): + return False + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + response = record.get("application_response") + last_status = record.get("last_status") + terminal_status_observed = bool( + isinstance(last_status, Mapping) + and last_status.get("session_state") in {"ready", "scan_stopping"} + ) + return bool( + record.get("operation_id") == stop_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "stop" + and record.get("stage") in {"dispatching", "observing"} + and record.get("resolution") is None + and isinstance(response, Mapping) + and response.get("operation_id") == stop_operation_id + and response.get("action") == "stop" + and response.get("success") is True + and isinstance(response.get("observed_at_utc"), str) + and bool(response.get("observed_at_utc")) + and not terminal_status_observed + ) + + @staticmethod + def _matching_prepared_stop_not_dispatched( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + ) -> bool: + """Match the exact durable STOP row before any publish boundary.""" + + if ( + not isinstance(physical_command_proof, Mapping) + or not isinstance(acquisition_id, str) + or not isinstance(stop_operation_id, str) + ): + return False + record = physical_command_proof.get("record") + return bool( + physical_command_proof.get("status") == "unresolved" + and physical_command_proof.get("requires_reconciliation") is True + and isinstance(record, Mapping) + and record.get("operation_id") == stop_operation_id + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "prepared" + and record.get("resolution") is None + and record.get("publish_call_returned") is None + and record.get("packet_id") is None + and record.get("qos2_completed") is False + and record.get("application_response") is None + and record.get("last_status") is None + ) + + @staticmethod + def _matching_unresolved_stop_stage( + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None = None, + ) -> tuple[Literal["prepared", "dispatching", "observing"], str] | None: + """Return one exact unresolved STOP owner without weakening its stage. + + Local ABORT/force-finish/shutdown paths must not cancel a journal row + merely because the control worker is being retired. The durable row is + the authority for whether publish was still impossible (PREPARED) or + already ambiguous (DISPATCHING/OBSERVING). + """ + + if not isinstance(physical_command_proof, Mapping) or not isinstance( + acquisition_id, str + ): + return None + if physical_command_proof.get("status") != "unresolved" or ( + physical_command_proof.get("requires_reconciliation") is not True + ): + return None + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return None + operation_id = record.get("operation_id") + stage = record.get("stage") + if not ( + isinstance(operation_id, str) + and operation_id + and (stop_operation_id is None or operation_id == stop_operation_id) + and record.get("acquisition_id") == acquisition_id + and record.get("action") == "stop" + and stage in {"prepared", "dispatching", "observing"} + and record.get("resolution") is None + ): + return None + return cast( + "tuple[Literal['prepared', 'dispatching', 'observing'], str]", + (stage, operation_id), + ) + + @classmethod + def _matching_resolved_stop_standby_owner( + cls, + physical_command_proof: Mapping[str, Any] | None, + *, + acquisition_id: str | None, + stop_operation_id: str | None = None, + ) -> str | None: + """Return the exact durable STOP already confirmed in READY.""" + + record = ( + physical_command_proof.get("record") + if isinstance(physical_command_proof, Mapping) + else None + ) + candidate = record.get("operation_id") if isinstance(record, Mapping) else None + if not isinstance(candidate, str) or ( + stop_operation_id is not None and candidate != stop_operation_id + ): + return None + return ( + candidate + if cls._matching_stop_standby_confirmed( + physical_command_proof, + acquisition_id=acquisition_id, + stop_operation_id=candidate, + ) + else None + ) + + @staticmethod + def _prepared_stop_durable_successor_matches( + lineage: _PreparedStopDispatchLineage, + physical_command_proof: Mapping[str, Any], + *, + allow_detached_read_only_owner: bool = False, + ) -> bool: + """Match the exact PREPARED child of the frozen active parent audit.""" + + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + identity = record.get("identity") + reconciliations = record.get("reconciliations") + baseline = record.get("baseline_status") + connection = record.get("connection") + if not ( + physical_command_proof.get("status") == "unresolved" + and physical_command_proof.get("requires_reconciliation") is True + and ( + physical_command_proof.get("active_operation_id") + == lineage.stop_operation_id + or ( + allow_detached_read_only_owner + and physical_command_proof.get("active_operation_id") is None + ) + ) + and record.get("operation_id") == lineage.stop_operation_id + and record.get("parent_operation_id") == lineage.parent_physical_operation_id + and record.get("revision") == lineage.parent_record_revision + 1 + and record.get("acquisition_id") == lineage.acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "prepared" + and record.get("resolution") is None + and record.get("publish_call_returned") is None + and record.get("packet_id") is None + and record.get("qos2_completed") is False + and record.get("application_response") is None + and record.get("last_status") is None + and record.get("compatibility_profile_id") == lineage.compatibility_profile_id + and isinstance(identity, Mapping) + and _sha256_audit_document(identity) == lineage.parent_identity_sha256 + and isinstance(reconciliations, list) + and isinstance(connection, Mapping) + and isinstance(baseline, Mapping) + ): + return False + scoped_reconciliations = [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") + == lineage.parent_physical_operation_id + ] + return bool( + _sha256_audit_document({"reconciliations": scoped_reconciliations}) + == lineage.parent_reconciliations_sha256 + and _sha256_audit_document(cast(Mapping[str, object], connection)) + == lineage.control_binding_sha256 + and baseline.get("control_session_id") == lineage.control_session_id + and baseline.get("host_path_epoch") == lineage.host_path_epoch + and baseline.get("producer_generation") == lineage.control_producer_generation + and baseline.get("session_state") == "scanning" + and baseline.get("project_bound") is True + and baseline.get("init_ready") is True + and baseline.get("mqtt_retained") is False + ) + + @staticmethod + def _resolved_unclassified_stop_successor_matches( + lineage: _PreparedStopDispatchLineage, + physical_command_proof: Mapping[str, Any], + ) -> bool: + """Match the same STOP after a fenced deadline proves no dispatch.""" + + record = physical_command_proof.get("record") + if not isinstance(record, Mapping): + return False + identity = record.get("identity") + reconciliations = record.get("reconciliations") + baseline = record.get("baseline_status") + connection = record.get("connection") + if not ( + physical_command_proof.get("status") == "resolved" + and physical_command_proof.get("active_operation_id") is None + and physical_command_proof.get("resolved_unclassified_stop_recovery_required") + is True + and record.get("operation_id") == lineage.stop_operation_id + and record.get("parent_operation_id") == lineage.parent_physical_operation_id + and record.get("revision") == lineage.parent_record_revision + 2 + and record.get("acquisition_id") == lineage.acquisition_id + and record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "not-dispatched" + and record.get("publish_call_returned") is None + and record.get("packet_id") is None + and record.get("qos2_completed") is False + and record.get("application_response") is None + and record.get("last_status") is None + and record.get("compatibility_profile_id") == lineage.compatibility_profile_id + and isinstance(identity, Mapping) + and _sha256_audit_document(identity) == lineage.parent_identity_sha256 + and isinstance(reconciliations, list) + and isinstance(connection, Mapping) + and isinstance(baseline, Mapping) + ): + return False + scoped_reconciliations = [ + item + for item in reconciliations + if isinstance(item, Mapping) + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") + == lineage.parent_physical_operation_id + ] + return bool( + _sha256_audit_document({"reconciliations": scoped_reconciliations}) + == lineage.parent_reconciliations_sha256 + and _sha256_audit_document(cast(Mapping[str, object], connection)) + == lineage.control_binding_sha256 + and baseline.get("control_session_id") == lineage.control_session_id + and baseline.get("host_path_epoch") == lineage.host_path_epoch + and baseline.get("producer_generation") == lineage.control_producer_generation + and baseline.get("session_state") == "scanning" + and baseline.get("project_bound") is True + and baseline.get("init_ready") is True + and baseline.get("mqtt_retained") is False + ) + + @staticmethod + def _prepared_stop_operation_is_terminal_none(operation: Any) -> bool: + error = getattr(operation, "error", None) + return bool( + getattr(operation, "action", None) == ACTION_ACQUISITION_STOP + and getattr(operation, "status", None) == "failed" + and getattr(operation, "stage_code", None) + == "physical-stop-classified-not-dispatched" + and isinstance(error, Mapping) + and error.get("side_effect_status") == "none" + and error.get("physical_command_sent") is False + and error.get("automatic_replay_allowed") is False + ) + + def _capture_prepared_stop_recovery_owner( + self, + *, + physical: Mapping[str, Any] | None = None, + ) -> _PreparedStopRecoveryOwner | None: + """Capture the same live receiver after exact pre-publish STOP failure.""" + + with self._lock: + lineage = getattr(self, "_prepared_stop_dispatch_lineage", None) + acquisition = self._acquisition + session_lease = self._acquisition_session_lease + out_dir = self._acquisition_out_dir + start_operation_id = self._acquisition_start_operation_id + stop_operation_id = self._acquisition_stop_operation_id + existing = getattr(self, "_prepared_stop_recovery_owner", None) + if not ( + isinstance(lineage, _PreparedStopDispatchLineage) + and acquisition is not None + and acquisition.control_mode == "plugin-commanded" + and acquisition.state == "awaiting_external_stop" + and acquisition.acquisition_id == lineage.acquisition_id + and acquisition.device_id == lineage.device_id + and acquisition.device_session_id == lineage.device_session_id + and session_lease is not None + and out_dir is not None + and out_dir.name == lineage.evidence_session_id + and stop_operation_id == lineage.stop_operation_id + and start_operation_id in { + None, + lineage.local_start_operation_id, + lineage.parent_physical_operation_id, + } + ): + return None + try: + stop_operation = self._operations.get(lineage.stop_operation_id) + except KeyError: + return None + if not self._prepared_stop_operation_is_terminal_none(stop_operation): + return None + runtime = self.runtime.snapshot() + runtime_generation = runtime.get("producer_generation") + if not ( + runtime.get("source_mode") == "live" + and runtime.get("phase") in {"live", "reconnecting"} + and isinstance(runtime_generation, int) + and not isinstance(runtime_generation, bool) + and runtime_generation == lineage.runtime_producer_generation + ): + return None + physical_snapshot = dict(physical or self._physical_command_coordinator.snapshot()) + if not ( + self._prepared_stop_durable_successor_matches(lineage, physical_snapshot) + or self._resolved_unclassified_stop_successor_matches( + lineage, + physical_snapshot, + ) + ): + return None + checkpoint_trust_token = ( + existing.checkpoint_trust_token + if existing is not None + else self._validate_active_acquisition_checkpoint_lineage() + ) + proposed = _PreparedStopRecoveryOwner( + lineage=lineage, + acquisition=acquisition, + session_lease=session_lease, + out_dir=out_dir, + start_operation_id=( + lineage.local_start_operation_id + if lineage.local_start_operation_id is not None + else start_operation_id + ), + runtime_producer_generation=runtime_generation, + checkpoint_trust_token=checkpoint_trust_token, + ) + if existing is not None and existing != proposed: + return None + with self._lock: + current = getattr(self, "_prepared_stop_recovery_owner", None) + if current is not None and current != proposed: + return None + self._prepared_stop_recovery_owner = proposed + return proposed + + def _prepared_stop_recovery_owner_is_current( + self, + owner: _PreparedStopRecoveryOwner, + *, + allow_resolved_classification: bool = False, + allow_resolved_standby: bool = False, + allow_terminal_cleanup: bool = False, + ) -> bool: + """CAS-check a retained owner without requiring the old facade topology.""" + + runtime = self.runtime.snapshot() + physical = dict(self._physical_command_coordinator.snapshot()) + with self._lock: + current_owner = getattr(self, "_prepared_stop_recovery_owner", None) + acquisition = self._acquisition + out_dir = self._acquisition_out_dir + session_lease = self._acquisition_session_lease + start_operation_id = self._acquisition_start_operation_id + stop_operation_id = self._acquisition_stop_operation_id + prepared_successor_current = self._prepared_stop_durable_successor_matches( + owner.lineage, + physical, + # Read-only bootstrap intentionally detaches the process-local + # command observer before installing its fresh DeviceInfo/status + # binding. The immutable ledger row and terminal-NONE operation + # remain exact; requiring coordinator._active here would make the + # safe classification path impossible immediately after detach. + allow_detached_read_only_owner=True, + ) + resolved_unclassified_current = ( + self._resolved_unclassified_stop_successor_matches(owner.lineage, physical) + ) + classified_active_current = bool( + allow_resolved_classification + and self._matching_classified_prepared_stop_active( + physical, + acquisition_id=owner.lineage.acquisition_id, + stop_operation_id=owner.lineage.stop_operation_id, + start_operation_id=owner.start_operation_id, + ) + ) + classified_standby_current = bool( + allow_resolved_standby + and self._matching_classified_prepared_stop_standby( + physical, + acquisition_id=owner.lineage.acquisition_id, + stop_operation_id=owner.lineage.stop_operation_id, + start_operation_id=owner.start_operation_id, + ) + ) + start_owner_terminal = False + if owner.start_operation_id is not None: + try: + start_owner_terminal = ( + self._operations.get(owner.start_operation_id).status + in TERMINAL_OPERATION_STATUSES + ) + except KeyError: + start_owner_terminal = False + start_pointer_current = bool( + start_operation_id == owner.start_operation_id + or ( + start_operation_id is None + and owner.start_operation_id is not None + and start_owner_terminal + ) + ) + terminal_cleanup_current = bool( + allow_terminal_cleanup + and acquisition is not None + and acquisition.state == "failed" + and isinstance(acquisition.result, Mapping) + and acquisition.result.get("read_only_recovery") is True + and acquisition.result.get("device_stop") == "not-sent" + and acquisition.result.get("local_cleanup_retry_pending") is True + ) + base_current = bool( + current_owner == owner + and acquisition is owner.acquisition + and acquisition is not None + and acquisition.acquisition_id == owner.lineage.acquisition_id + and acquisition.control_mode == "plugin-commanded" + and ( + acquisition.state + in {"awaiting_external_stop", "awaiting_external_start", "acquiring"} + or ( + classified_standby_current + and acquisition.state == "finalizing" + ) + or terminal_cleanup_current + ) + and out_dir == owner.out_dir + and session_lease is owner.session_lease + and start_pointer_current + and ( + stop_operation_id == owner.lineage.stop_operation_id + if prepared_successor_current or resolved_unclassified_current + else stop_operation_id in {None, owner.lineage.stop_operation_id} + if classified_active_current or classified_standby_current + else False + ) + and runtime.get("producer_generation") == owner.runtime_producer_generation + and ( + runtime.get("source_mode") == "live" + or ( + classified_standby_current + and runtime.get("source_mode") == "idle" + ) + or ( + terminal_cleanup_current + and runtime.get("source_mode") == "idle" + ) + ) + ) + if not base_current: + return False + if prepared_successor_current or resolved_unclassified_current: + return True + return classified_active_current or classified_standby_current + + def _prepared_stop_recovery_owner_allows_read_only_rebind( + self, + owner: _PreparedStopRecoveryOwner, + ) -> bool: + """Admit only the same retained receiver through a fresh Verify. + + A first inspection may durably classify S0 and then lose its local + control object before the in-place projection completes. That does + not revoke the evidence lease or authorize generic topology rotation; + a later read-only inspection may retry against the exact resolved + active/standby audit. No mode, target or command authority is widened. + """ + + return self._prepared_stop_recovery_owner_is_current( + owner, + allow_resolved_classification=True, + allow_resolved_standby=True, + allow_terminal_cleanup=True, + ) + + def _prepared_stop_local_projection_is_pending( + self, + *, + acquisition_id: str | None, + ) -> bool: + """Fence destructive local actions across durable classification adoption. + + PREPARED STOP classification is committed synchronously, while the + control worker reaches its adopted SCANNING state asynchronously. The + retained receiver is deliberately marked with an exact pending local + projection before the acquisition lifecycle gate is released. ABORT, + force-finish, scenario reset, and shutdown must not consume that owner + in the short commit-to-adoption window. + """ + + with self._lock: + owner = getattr(self, "_prepared_stop_recovery_owner", None) + acquisition = self._acquisition + if ( + owner is None + or acquisition is not owner.acquisition + or acquisition.acquisition_id != acquisition_id + ): + return False + if acquisition.message_code == "acquisition.recovery.scanning_adoption_pending": + return self._prepared_stop_recovery_owner_is_current( + owner, + allow_resolved_classification=True, + ) + if ( + acquisition.state == "finalizing" + and acquisition.message_code + == "acquisition.recovery.device_standby_observed" + ): + return self._prepared_stop_recovery_owner_is_current( + owner, + allow_resolved_standby=True, + ) + return False + + def _fence_existing_stop_before_local_action( + self, + *, + acquisition_id: str, + reject_nonterminal_prepared: bool, + resolved_standby_only: bool = False, + ) -> tuple[_LocalStopRetirementDisposition, str | None] | None: + """Own one exact prior S0 before a local STOP-like action mutates state. + + ``capture-only`` and ABORT are public acquisition actions, so they can + otherwise begin a second journal operation and tear down producers + while an older physical STOP still owns its publish edge. Match only + the acquisition's current local S0 pointer. A live PREPARED worker is + rejected without even closing its socket; terminal PREPARED, + DISPATCHING/OBSERVING, and canonical resolved READY are delegated to + the shared dispatch-fenced retirement proof. + """ + + with self._lock: + stop_operation_id = self._acquisition_stop_operation_id + if not isinstance(stop_operation_id, str): + return None + physical = dict(self._physical_command_coordinator.snapshot()) + unresolved = self._matching_unresolved_stop_stage( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + resolved = self._matching_resolved_stop_standby_owner( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + if resolved_standby_only: + unresolved = None + if unresolved is None and resolved is None: + return None + if ( + reject_nonterminal_prepared + and unresolved is not None + and unresolved[0] == "prepared" + ): + control = dict(self._application_control_session.snapshot()) + if control.get("state") not in {"idle", "completed", "closed", "failed"}: + raise LocalAcquisitionLifecycleError( + "локальный control worker ещё владеет PREPARED STOP", + reason_code="acquisition-stop-worker-retirement-pending", + ) + retirement = self._fence_local_control_for_acquisition_retirement( + acquisition_id=acquisition_id, + ) + if resolved is not None and retirement != ("standby-confirmed", resolved): + raise LocalAcquisitionLifecycleError( + "доказательство READY изменилось во время локального STOP-fence", + reason_code="acquisition-stop-retirement-proof-changed", + ) + return retirement + + def _fence_local_control_for_acquisition_retirement( + self, + *, + acquisition_id: str | None, + ) -> tuple[_LocalStopRetirementDisposition, str | None]: + """Cancel the local worker under the exact publish fence. + + A worker may have passed its early transport-ready check and still be + queued for the facade dispatch lease. Holding that lease while closing + the socket makes a successful close a no-future-publish proof. Only a + fresh PREPARED row plus a terminal worker becomes side-effect ``none``; + DISPATCHING/OBSERVING is sealed conservatively as unknown. + """ + + if self._prepared_stop_local_projection_is_pending( + acquisition_id=acquisition_id, + ): + raise LocalAcquisitionLifecycleError( + "read-only классификация STOP ещё применяется к локальному приёму", + reason_code="acquisition-prepared-stop-adoption-pending", + ) + + if not self._k1_command_dispatch_gate.acquire(blocking=False): + raise LocalAcquisitionLifecycleError( + "локальный control worker ещё владеет publish-переходом STOP", + reason_code="acquisition-stop-dispatch-retirement-pending", + ) + try: + before_physical = dict(self._physical_command_coordinator.snapshot()) + before_close = self._matching_unresolved_stop_stage( + before_physical, + acquisition_id=acquisition_id, + ) + before_resolved = self._matching_resolved_stop_standby_owner( + before_physical, + acquisition_id=acquisition_id, + ) + self._application_control_session.close() + retired_control = dict(self._application_control_session.snapshot()) + if ( + retired_control.get("state") + not in {"idle", "completed", "closed", "failed"} + or retired_control.get("can_open") is False + ): + raise LocalAcquisitionLifecycleError( + ( + "локальный control worker не завершил PREPARED STOP" + if before_close is not None and before_close[0] == "prepared" + else "локальный control worker не завершил STOP после close/join" + ), + reason_code="acquisition-stop-worker-retirement-pending", + ) + current_physical = dict(self._physical_command_coordinator.snapshot()) + current = self._matching_unresolved_stop_stage( + current_physical, + acquisition_id=acquisition_id, + stop_operation_id=(before_close[1] if before_close is not None else None), + ) + resolved_stop_operation_id = self._matching_resolved_stop_standby_owner( + current_physical, + acquisition_id=acquisition_id, + stop_operation_id=( + before_close[1] + if before_close is not None + else before_resolved + ), + ) + if resolved_stop_operation_id is not None: + self._operations.transition_if_pending( + resolved_stop_operation_id, + "succeeded", + stage_code="device-standby-confirmed", + message_code="acquisition.stop.device_standby_confirmed", + result={ + "acquisition_id": acquisition_id, + "device_stop": "protocol-confirmed", + "physical_command_sent": True, + }, + ) + return "standby-confirmed", resolved_stop_operation_id + if current is None: + return "not-applicable", None + stage, stop_operation_id = current + if stage == "prepared": + outcome = self._settle_prepared_stop_worker_failure( + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + dispatch_gate_owned=True, + ) + if outcome == "settled": + return "not-dispatched", stop_operation_id + if outcome in {"defer", "prepared-pending"}: + raise LocalAcquisitionLifecycleError( + "локальный control worker не завершил PREPARED STOP", + reason_code="acquisition-stop-worker-retirement-pending", + ) + raise LocalAcquisitionLifecycleError( + "PREPARED STOP изменился во время локального завершения", + reason_code="acquisition-stop-retirement-proof-changed", + ) + + operation = self._operations.get(stop_operation_id) + if operation.status in TERMINAL_OPERATION_STATUSES: + existing_side_effect = ( + operation.error.get("side_effect_status") + if isinstance(operation.error, Mapping) + else None + ) + if existing_side_effect != "unknown": + raise LocalAcquisitionLifecycleError( + "журнал STOP уже завершён с противоречивым physical outcome", + reason_code="acquisition-stop-terminal-audit-conflict", + ) + else: + self._operations.transition_if_pending( + stop_operation_id, + "failed", + stage_code="physical-stop-outcome-unknown-local-retirement", + message_code="acquisition.stop.physical_outcome_unknown_local_retirement", + error={ + "category": "device", + "code": "physical-stop-outcome-unknown-local-retirement", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + }, + ) + return "outcome-unknown", stop_operation_id + finally: + self._k1_command_dispatch_gate.release() + + def _terminalize_pending_start_for_local_stop_retirement( + self, + *, + acquisition_id: str, + stop_operation_id: str | None, + ) -> None: + """Preserve confirmed START truth when local cleanup ends before PCL. + + STOP is legal as soon as control reports SCANNING, so the original + acquisition START operation may still be pending its first point frame. + An exact unresolved STOP baseline proves that START reached the device; + a local force/reset/shutdown must not relabel that row as cancelled. + """ + + with self._lock: + start_operation_id = self._acquisition_start_operation_id + if not isinstance(start_operation_id, str) or not isinstance( + stop_operation_id, str + ): + return + physical = self._physical_command_coordinator.snapshot() + current = self._matching_unresolved_stop_stage( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + resolved_standby_stop_operation_id = ( + self._matching_resolved_stop_standby_owner( + physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + ) + record = physical.get("record") if isinstance(physical, Mapping) else None + if ( + current is None + and resolved_standby_stop_operation_id is None + ) or not isinstance(record, Mapping): + return + baseline = record.get("baseline_status") + reconciliations = record.get("reconciliations") + immediate_parent = record.get("parent_operation_id") + ancestry_matches = bool( + immediate_parent == start_operation_id + or ( + isinstance(immediate_parent, str) + and isinstance(reconciliations, list) + and self._classified_stop_chain_reaches_start( + reconciliations, + from_operation_id=immediate_parent, + start_operation_id=start_operation_id, + acquisition_id=acquisition_id, + ) + ) + ) + if not ( + ancestry_matches + and isinstance(baseline, Mapping) + and baseline.get("session_state") == "scanning" + and baseline.get("project_bound") is True + and baseline.get("init_ready") is True + and baseline.get("mqtt_retained") is False + ): + return + resolved_ready = resolved_standby_stop_operation_id is not None + self._operations.transition_if_pending( + start_operation_id, + "interrupted", + stage_code=( + "physical-start-active-but-no-point-before-standby" + if resolved_ready + else "physical-start-active-local-retirement-before-point" + ), + message_code=( + "acquisition.start.device_standby_before_point_data" + if resolved_ready + else "acquisition.start.local_retirement_before_point_data" + ), + error={ + "category": "device" if resolved_ready else "stream", + "code": ( + "device-standby-before-first-point-frame" + if resolved_ready + else "local-retirement-before-first-point-frame" + ), + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "succeeded", + "automatic_replay_allowed": False, + }, + ) + + def _settle_prepared_stop_worker_failure( + self, + *, + acquisition_id: str | None, + stop_operation_id: str | None, + dispatch_gate_owned: bool = False, + ) -> Literal["settled", "defer", "prepared-pending", "not-applicable"]: + """Seal a pre-publish worker failure before generic UNKNOWN reducers. + + Both facts are re-read while owning the same dispatch gate used across + mark_dispatching -> MQTT publish. A state snapshot taken just before + the worker crosses that boundary can therefore never be combined with + a newer failed control snapshot and mislabel a dispatched STOP as none. + """ + + if not isinstance(acquisition_id, str) or not isinstance(stop_operation_id, str): + return "not-applicable" + # Only contention while the durable row still appears exactly + # PREPARED is meaningful. An unrelated in-flight command must not + # suppress ordinary acquisition reduction merely because it owns the + # process-wide dispatch gate. + if not self._matching_prepared_stop_not_dispatched( + self._physical_command_coordinator.snapshot(), + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ): + return "not-applicable" + acquired_here = False + if not dispatch_gate_owned: + if not self._k1_command_dispatch_gate.acquire(blocking=False): + return "defer" + acquired_here = True + try: + current_control = dict(self._application_control_session.snapshot()) + current_physical = dict(self._physical_command_coordinator.snapshot()) + if not self._matching_prepared_stop_not_dispatched( + current_physical, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ): + return "not-applicable" + if current_control.get("state") in {"stop-requested", "stopping"}: + # The asynchronous owner has not failed and no publish edge + # exists. Runtime/camera symptoms observed in this tiny + # handoff window are provisional; generic reducers must not + # seal the STOP as UNKNOWN before the worker declares its + # actual pre- or post-dispatch outcome. + return "prepared-pending" + if current_control.get("state") not in {"failed", "closed"}: + return "not-applicable" + failure = current_control.get("failure") + transport = current_control.get("transport") + if not ( + isinstance(failure, Mapping) + and failure.get("reason_code") + == "physical-command-dispatch-deadline-expired" + and failure.get("stop_command_attempted") is False + and failure.get("stop_publish_attempts") == 0 + and current_control.get("outcome_unknown") is False + and isinstance(transport, Mapping) + and isinstance(transport.get("publish_attempts"), int) + ): + # Other terminal PREPARED failures retain the legacy + # read-only classification path. A deadline failure is the + # only new fact whose exact transport diagnostics establish a + # durable immediate no-dispatch transition here. + definite_deadline_no_dispatch = False + else: + definite_deadline_no_dispatch = True + self._operations.transition_if_pending( + stop_operation_id, + "failed", + stage_code="physical-stop-classified-not-dispatched", + message_code="acquisition.stop.physical_command_not_dispatched", + error={ + "category": "device", + "code": "physical-stop-not-dispatched", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "none", + "physical_command_sent": False, + "automatic_replay_allowed": False, + }, + ) + # STOP never crossed the publish boundary. Revoke the camera EOF + # expectation armed for a real device STOP and retain the exact + # live receiver owner for read-only classification. This does not + # reopen or stop the camera and emits no device/network command. + self.camera_preview.cancel_expected_source_end() + if definite_deadline_no_dispatch: + self._physical_command_coordinator.resolve_prepared_not_dispatched("stop") + current_physical = dict(self._physical_command_coordinator.snapshot()) + recovery_owner = self._capture_prepared_stop_recovery_owner( + physical=current_physical + ) + if definite_deadline_no_dispatch and recovery_owner is None: + raise LocalAcquisitionLifecycleError( + "STOP recovery owner disappeared after the no-dispatch commit", + reason_code="prepared-stop-recovery-owner-unavailable", + ) + return "settled" + finally: + if acquired_here: + self._k1_command_dispatch_gate.release() + + def _retire_local_control_after_stop_timeout(self) -> None: + """Close only the host-owned socket after bounded STOP uncertainty.""" + + try: + self._application_control_session.close() + except Exception: + with self._lock: + self._pending_local_control_retirement = True + logger.exception( + "K1 local control close failed after STOP outcome timeout", + extra={ + "event_code": "k1_stop_outcome_timeout_control_close_failed", + "automatic_replay": False, + }, + ) + return + snapshot = self._application_control_session.snapshot() + if snapshot.get("state") not in {"idle", "completed", "closed", "failed"}: + with self._lock: + self._pending_local_control_retirement = True + return + try: + self._retire_application_control_for_network_change( + allow_terminal_failure=True, + ) + except (AttributeError, RuntimeError): + with self._lock: + self._pending_local_control_retirement = True + + def _recovery_only_terminal_without_trusted_checkpoint( + self, + *, + physical_command_proof: Mapping[str, Any] | None, + acquisition_id: str | None, + stop_operation_id: str | None, + ) -> bool: + """Admit only local teardown after an exact recovery-only STOP terminal. + + A corrupt or unavailable checkpoint can never grant START or resume + authority. It also must not strand a STOP-only shell after the durable + physical ledger independently proves that exact successor STOP and a + fresh terminal state. This check performs no mutation and leaves the + checkpoint bytes untouched. + """ + + if getattr( + self, + "_active_acquisition_checkpoint_trust", + "trusted", + ) not in {"corrupt", "unavailable"}: + return False + with self._lock: + acquisition = self._acquisition + result = acquisition.result if acquisition is not None else None + no_local_evidence_owner = bool( + self._acquisition_out_dir is None + and self._acquisition_session_lease is None + ) + if not ( + acquisition is not None + and acquisition.acquisition_id == acquisition_id + and acquisition.control_mode == "plugin-commanded" + and acquisition.state in TERMINAL_ACQUISITION_STATES + and acquisition.message_code + == "acquisition.physical_active_recovered_stop_only" + and isinstance(result, Mapping) + and result.get("recovery_only") is True + and result.get("receiver_stopped") is True + and result.get("device_state") == "scanning" + and result.get("automatic_replay_allowed") is False + and isinstance(result.get("physical_command_operation_id"), str) + and isinstance(result.get("physical_reconciliation_id"), str) + and no_local_evidence_owner + and isinstance(stop_operation_id, str) + ): + return False + ledger_snapshot = self._physical_command_ledger.snapshot() + record = ledger_snapshot.record + projected_record = ( + physical_command_proof.get("record") + if isinstance(physical_command_proof, Mapping) + else None + ) + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and isinstance(projected_record, Mapping) + and projected_record.get("operation_id") == record.operation_id + and projected_record.get("revision") == record.revision + and record.operation_id == stop_operation_id + and record.parent_operation_id + == result.get("physical_command_operation_id") + and record.acquisition_id == acquisition_id + and record.action == "stop" + and record.stage == "resolved" + and record.compatibility_profile_id + == acquisition.compatibility_profile_id + and record.baseline_status.session_state == "scanning" + and record.baseline_status.project_bound + and record.baseline_status.init_ready + and not record.baseline_status.mqtt_retained + and record.baseline_status.project_id_sha256 + == result.get("project_id_sha256") + ): + return False + parent_reconciliation_id = cast(str, result["physical_reconciliation_id"]) + parent_reconciliation = next( + ( + item + for item in record.reconciliations + if item.reconciliation_id == parent_reconciliation_id + and item.original_attempt.operation_id == record.parent_operation_id + and item.resolution == "physical-active-observed" + ), + None, + ) + if not ( + parent_reconciliation is not None + and parent_reconciliation.verified_binding.connection == record.connection + and parent_reconciliation.observation.session_state == "scanning" + and parent_reconciliation.observation.project_id_sha256 + == result.get("project_id_sha256") + and any( + confirmation.stop_operation_id == record.operation_id + and confirmation.reconciliation_id == parent_reconciliation_id + for confirmation in record.operator_confirmations + ) + ): + return False + if record.resolution == "stop-standby-observed": + status = record.last_status + return bool( + record.publish_call_returned is True + and record.packet_id is not None + and record.qos2_completed + and record.application_response is not None + and record.application_response.success + and status is not None + and status.source == "live-control-session" + and status.session_state == "ready" + and not status.project_bound + and not status.init_ready + and not status.mqtt_retained + ) + reconciliation = record.reconciliations[-1] if record.reconciliations else None + if reconciliation is None or not ( + reconciliation.original_attempt.operation_id == stop_operation_id + and reconciliation.observation.source + == "explicit-read-only-reconciliation" + and reconciliation.observation.session_state in {"ready", "scan_over"} + and not reconciliation.observation.project_bound + and not reconciliation.observation.init_ready + and not reconciliation.observation.mqtt_retained + and reconciliation.resolution == "physical-standby-observed" + ): + return False + return bool( + ( + record.resolution == "physical-standby-observed" + and reconciliation.kind == "ambiguous-outcome" + ) + or ( + record.resolution == "not-dispatched" + and reconciliation.kind + in { + "prepared-stop-classification", + "resolved-active-cessation", + } + ) + ) + def _reconcile_acquisition( self, runtime: Mapping[str, Any], camera: Mapping[str, Any], application_control_session: Mapping[str, Any], + *, + terminal_control_proof: Mapping[str, Any] | None = None, + physical_command_proof: Mapping[str, Any] | None = None, + prepared_stop_worker_outcome: Literal[ + "settled", "defer", "prepared-pending", "not-applicable" + ] = "not-applicable", ) -> None: + # The application-session worker may already have reset/retired its + # public projection after publishing terminal READY. Preserve the + # exact pre-retirement terminal proof captured by ``state()`` for this + # one reduction; the durable ledger remains the primary evidence. + terminal_control_completed = bool( + ( + isinstance(terminal_control_proof, Mapping) + and terminal_control_proof.get("state") == "completed" + ) + or application_control_session.get("state") == "completed" + ) + stop_acknowledged_standby_unconfirmed = self._stop_acknowledged_standby_unconfirmed( + terminal_control_proof, + physical_command_proof, + ) with self._lock: current = self._acquisition + current_acquisition_id = current.acquisition_id if current is not None else None + current_start_operation_id = self._acquisition_start_operation_id + current_stop_operation_id = self._acquisition_stop_operation_id + current_out_dir = self._acquisition_out_dir + camera_activation_lineage = self._camera_activation_lineage + camera_activation_retry_lineage = self._camera_activation_retry_lineage + restart_rehydration_pending = self._restart_rehydration_pending + camera_recording = camera.get("recording") + camera_error = camera.get("error") + current_evidence_session_id = ( + current_out_dir.name if current_out_dir is not None else None + ) + restart_rehydration_checkpoint_owner = bool( + restart_rehydration_pending is not None + and current is not None + and current.control_mode == "plugin-commanded" + and current.state + in {"starting", "awaiting_external_start", "awaiting_external_stop"} + and current.acquisition_id + == restart_rehydration_pending.token.acquisition_id + and current_out_dir == restart_rehydration_pending.out_dir + and current_evidence_session_id + == restart_rehydration_pending.new_evidence_session_id + and runtime.get("producer_generation") + == restart_rehydration_pending.runtime_producer_generation + ) + restart_rehydration_reduction_pending = bool( + restart_rehydration_checkpoint_owner + and current is not None + and current.state in {"starting", "awaiting_external_start"} + ) + trusted_recovery_only_lineage = ( + self._trusted_recovery_only_checkpoint_lineage(current) + if current_evidence_session_id is None + else None + ) + canonical_start_operation_id = ( + self._active_acquisition_checkpoint_start_operation_id( + acquisition_id=current.acquisition_id, + evidence_session_id=current_evidence_session_id, + local_start_operation_id=current_start_operation_id, + ) + if current is not None + and current.control_mode == "plugin-commanded" + and current_evidence_session_id is not None + else current_start_operation_id + ) + if restart_rehydration_checkpoint_owner and restart_rehydration_pending is not None: + # Before the first recovered PCL the durable checkpoint still + # names the predecessor evidence session by design. The frozen + # restart owner is nevertheless the exact original START lineage + # required to cease that checkpoint if an explicit STOP wins the + # race. Do not route this through the ordinary current-evidence + # helper: it must remain fail-closed until first-PCL promotion. + canonical_start_operation_id = ( + restart_rehydration_pending.token.root_start_operation_id + ) + checkpoint_cessation_evidence_session_id = current_evidence_session_id + if trusted_recovery_only_lineage is not None: + canonical_start_operation_id = ( + trusted_recovery_only_lineage.original_start_operation_id + ) + checkpoint_cessation_evidence_session_id = ( + trusted_recovery_only_lineage.cessation_evidence_session_id + ) + exact_camera_activation_lineage = ( + ( + current.acquisition_id, + current_evidence_session_id, + runtime.get("producer_generation"), + ) + if current is not None and current_evidence_session_id is not None + else None + ) + if ( + exact_camera_activation_lineage is not None + and current_evidence_session_id is not None + and camera_activation_lineage == exact_camera_activation_lineage + and _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=current_evidence_session_id, + ) + ): + with self._lock: + if self._camera_activation_lineage == exact_camera_activation_lineage: + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = exact_camera_activation_lineage + self._camera_activation_retry_not_before_monotonic = max( + self._camera_activation_retry_not_before_monotonic, + time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS, + ) + camera_activation_lineage = None + camera_activation_retry_lineage = exact_camera_activation_lineage + camera_activation_retry_provisional = bool( + current is not None + and current.state in {"starting", "awaiting_external_start", "acquiring"} + and current_evidence_session_id is not None + and ( + current.acquisition_id, + current_evidence_session_id, + runtime.get("producer_generation"), + ) + in {camera_activation_lineage, camera_activation_retry_lineage} + and camera.get("active_source_id") == DEFAULT_ACQUISITION_CAMERA_SOURCE + and isinstance(camera_recording, Mapping) + and camera_recording.get("active") is True + and camera_recording.get("session") == current_evidence_session_id + and _camera_post_pcl_startup_failure_is_retryable( + camera, + evidence_session_id=current_evidence_session_id, + ) + ) + durable_start_active_confirmed = self._matching_start_active_confirmed( + physical_command_proof, + acquisition_id=current_acquisition_id, + start_operation_id=canonical_start_operation_id, + verified_control=( + application_control_session.get("verified_control") + if isinstance( + application_control_session.get("verified_control"), + Mapping, + ) + else None + ), + ) + durable_checkpoint_active_confirmed = bool( + current is not None + and current.control_mode != "plugin-commanded" + or ( + current is not None + and current_evidence_session_id is not None + and self._active_acquisition_checkpoint_matches_current( + acquisition_id=current.acquisition_id, + start_operation_id=canonical_start_operation_id, + evidence_session_id=current_evidence_session_id, + verified_control=( + application_control_session.get("verified_control") + if isinstance( + application_control_session.get("verified_control"), + Mapping, + ) + else None + ), + ) + ) + ) + camera_recovery_reason = self._active_stream_camera_recovery_reason(camera) + camera_nontransport_error = bool( + camera.get("phase") == "error" + and camera_recovery_reason is None + and not camera_activation_retry_provisional + ) + camera_transport_recovery_pending = bool( + camera_recovery_reason is not None + and current is not None + and current.state in {"starting", "awaiting_external_start", "acquiring"} + and runtime.get("phase") == "live" + and runtime.get("source_mode") == "live" + and current_evidence_session_id is not None + and isinstance(camera_recording, Mapping) + and camera_recording.get("active") is True + and camera_recording.get("session") == current_evidence_session_id + and durable_start_active_confirmed + and durable_checkpoint_active_confirmed + ) + # Camera-only transport loss is supervised by the backend producer + # watchdog and exact camera CAS. Snapshot polling is observational: it + # must not move an otherwise live MQTT/PCL runtime to ``reconnecting``. + # If the host or MQTT plane is also lost, their own evidence wakes the + # capture-owned read-only rebind path independently. + plugin_start_outcome_unknown = self._plugin_start_outcome_unknown( + terminal_control_proof, + ) + canonical_start_control_active = bool( + application_control_session.get("state") == "scanning" + and application_control_session.get("can_stop") is True + ) + durable_stop_standby_confirmed = self._matching_stop_standby_confirmed( + physical_command_proof, + acquisition_id=current_acquisition_id, + stop_operation_id=current_stop_operation_id, + ) + checkpoint_snapshot_status: str | None = None + if self._active_acquisition_checkpoint is not None: + try: + checkpoint_snapshot_status = ( + self._active_acquisition_checkpoint.snapshot().status + ) + except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError): + checkpoint_snapshot_status = None + checkpoint_stop_ceased = bool( + current is None + or current.control_mode != "plugin-commanded" + # Legacy same-process acquisitions predating checkpoint creation + # have no checkpoint to cease. Missing/corrupt storage does not + # enter this compatibility branch and remains fail closed. + or checkpoint_snapshot_status == "empty" + ) + if ( + current is not None + and current.control_mode == "plugin-commanded" + and checkpoint_cessation_evidence_session_id is not None + and isinstance(canonical_start_operation_id, str) + and isinstance(current_stop_operation_id, str) + and (durable_stop_standby_confirmed or terminal_control_completed) + ): + checkpoint_stop_ceased = ( + self._cease_active_acquisition_checkpoint_from_physical_head( + expected_acquisition_id=current.acquisition_id, + expected_start_operation_id=canonical_start_operation_id, + evidence_session_id=checkpoint_cessation_evidence_session_id, + expected_stop_operation_id=current_stop_operation_id, + ) + ) + checkpointless_recovery_terminal = ( + self._recovery_only_terminal_without_trusted_checkpoint( + physical_command_proof=physical_command_proof, + acquisition_id=current_acquisition_id, + stop_operation_id=current_stop_operation_id, + ) + ) + if checkpointless_recovery_terminal: + # This is a local teardown exemption, not a checkpoint repair or + # recovery grant. Keep the checkpoint trust fence in place. + durable_stop_standby_confirmed = True + checkpoint_stop_ceased = True + checkpoint_stop_completion_seen = bool( + durable_stop_standby_confirmed + or terminal_control_completed + ) + if checkpoint_stop_completion_seen and not checkpoint_stop_ceased: + # READY is durable device truth, but local terminal teardown must + # not overtake the checkpoint fsync that makes this acquisition + # non-recoverable. Keep STOP authority/result intact and retry the + # storage-only transition on a later state reduction. + durable_stop_standby_confirmed = False + self._set_active_acquisition_checkpoint_reason( + "active-acquisition-recovery-checkpoint-cease-failed" + ) + stop_response_accepted_without_terminal_status = ( + self._matching_stop_response_without_terminal_status( + physical_command_proof, + acquisition_id=current_acquisition_id, + stop_operation_id=current_stop_operation_id, + ) + ) + stop_response_deadline_reached = bool( + not durable_stop_standby_confirmed + and not stop_acknowledged_standby_unconfirmed + and application_control_session.get("state") != "completed" + and stop_response_accepted_without_terminal_status + and self._operations.deadline_reached(current_stop_operation_id) + ) + with self._lock: + canonical_stop_standby_confirmed = bool( + checkpoint_stop_completion_seen and checkpoint_stop_ceased + ) + terminal_canonical_stop_operation_id = ( + self._acquisition_stop_operation_id + if current is not None + and current.state in TERMINAL_ACQUISITION_STATES + and current.control_mode == "plugin-commanded" + and canonical_stop_standby_confirmed + else None + ) terminal_canonical_cleanup = ( current is not None and current.state in TERMINAL_ACQUISITION_STATES and current.control_mode == "plugin-commanded" - and application_control_session.get("state") == "completed" + and canonical_stop_standby_confirmed and self._acquisition_session_lease is not None ) terminal_camera_status: Literal["complete", "failed"] = ( "complete" if current is not None and current.state == "completed" else "failed" ) - if terminal_canonical_cleanup: - self._stop_acquisition_sources( - camera_status=terminal_camera_status, - camera_failure_code=( - None - if terminal_camera_status == "complete" - else "terminal-local-failure-after-device-standby" - ), + terminal_unconfirmed_stop_operation_id = ( + self._acquisition_stop_operation_id + if current is not None + and current.state in TERMINAL_ACQUISITION_STATES + and current.control_mode == "plugin-commanded" + and stop_acknowledged_standby_unconfirmed + else None ) + terminal_unknown_stop_cleanup = bool( + current is not None + and current.state in TERMINAL_ACQUISITION_STATES + and current.control_mode == "plugin-commanded" + and stop_response_deadline_reached + and self._acquisition_session_lease is not None + ) + terminal_unknown_stop_operation_id = ( + self._acquisition_stop_operation_id + if current is not None + and current.state in TERMINAL_ACQUISITION_STATES + and current.control_mode == "plugin-commanded" + and stop_response_deadline_reached + else None + ) + if stop_response_deadline_reached: + self._retire_local_control_after_stop_timeout() + if terminal_unknown_stop_cleanup: + try: + self._stop_acquisition_sources( + camera_status="failed", + camera_failure_code="stop-accepted-physical-outcome-unknown", + ) + except Exception as exc: + self._operations.transition_if_pending( + terminal_unknown_stop_operation_id, + "failed", + stage_code="local-cleanup-failed-after-stop-outcome-unknown", + message_code="acquisition.stop.outcome_unknown_local_cleanup_failed", + error={ + "category": "stream", + "code": "local-cleanup-failed-after-stop-outcome-unknown", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + "cleanup_error_type": type(exc).__name__, + }, + ) + logger.error( + "K1 local cleanup retry failed after STOP outcome timeout", + extra={ + "event_code": "k1_stop_outcome_timeout_local_cleanup_failed", + "physical_outcome": "unknown", + "automatic_replay": False, + }, + exc_info=exc, + ) + return + with self._lock: + terminal_acquisition = self._acquisition + if ( + terminal_acquisition is not None + and terminal_acquisition.state in TERMINAL_ACQUISITION_STATES + and isinstance(terminal_acquisition.result, dict) + and terminal_acquisition.result.get("device_stop") + == "stop-accepted-physical-outcome-unknown" + ): + terminal_acquisition.result = { + key: value + for key, value in terminal_acquisition.result.items() + if key != "local_cleanup_retry_pending" + } + terminal_acquisition.result["receiver_stopped"] = True + terminal_acquisition.message_code = ( + "acquisition.stop.accepted_physical_outcome_unknown" + ) + terminal_acquisition.state_revision += 1 + terminal_acquisition.updated_at = datetime.now(UTC) + if self._acquisition_stop_operation_id == terminal_unknown_stop_operation_id: + self._acquisition_stop_operation_id = None + return + if terminal_canonical_cleanup: + try: + self._stop_acquisition_sources( + camera_status=terminal_camera_status, + camera_failure_code=( + None + if terminal_camera_status == "complete" + else "terminal-local-failure-after-device-standby" + ), + ) + except Exception as exc: + # READY already proves the K1-side STOP. Keep the local lease + # fenced and report only the retryable host cleanup failure; + # never turn this into another physical STOP opportunity. + self._operations.transition_if_pending( + terminal_canonical_stop_operation_id, + "failed", + stage_code="local-finalization-failed", + message_code="acquisition.stop.local_finalization_failed", + error={ + "category": "stream", + "code": "local-finalization-failed-after-device-standby", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "succeeded", + "cleanup_error_type": type(exc).__name__, + }, + ) + logger.error( + "K1 local cleanup failed after durable STOP READY proof", + extra={ + "event_code": "k1_stop_ready_local_cleanup_failed", + "reason_code": "local-finalization-failed-after-device-standby", + "physical_stop_confirmed": True, + "automatic_replay": False, + }, + exc_info=exc, + ) + return + if terminal_canonical_stop_operation_id is not None: + self._operations.transition_if_pending( + terminal_canonical_stop_operation_id, + "succeeded", + stage_code="device-standby-confirmed-after-local-failure", + message_code="acquisition.stop.completed_after_local_failure", + result={ + "acquisition_id": current.acquisition_id if current is not None else None, + "receiver_stopped": True, + "device_state": "ready", + "device_stop": "protocol-confirmed", + }, + ) + with self._lock: + if self._acquisition_stop_operation_id == terminal_canonical_stop_operation_id: + self._acquisition_stop_operation_id = None + if terminal_unconfirmed_stop_operation_id is not None: + self._operations.transition_if_pending( + terminal_unconfirmed_stop_operation_id, + "failed", + stage_code="stop-acknowledged-standby-unconfirmed", + message_code="acquisition.stop.standby_unconfirmed_after_control_loss", + error={ + "category": "device", + "code": "stop-acknowledged-standby-unconfirmed", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + }, + ) + with self._lock: + if self._acquisition_stop_operation_id == terminal_unconfirmed_stop_operation_id: + self._acquisition_stop_operation_id = None + if terminal_canonical_cleanup or terminal_canonical_stop_operation_id is not None: + return + if terminal_unconfirmed_stop_operation_id is not None: return completed_operation_id: str | None = None failed_operation_id: str | None = None + failed_start_physical_side_effect: Literal["unknown", "succeeded"] | None = None + unknown_start_operation_id: str | None = None + unknown_start_side_effect_status: Literal["unknown", "succeeded"] | None = None failed_stop_operation_id: str | None = None unconfirmed_stop_operation_id: str | None = None + acknowledged_unconfirmed_stop_operation_id: str | None = None completed_stop_operation_id: str | None = None + unknown_stop_operation_id: str | None = None receiver_ready_operation_id: str | None = None receiver_plugin_commanded = False acquisition_id: str | None = None - camera_terminal_status: Literal["complete", "failed"] | None = None + camera_terminal_status: Literal["complete", "interrupted", "failed"] | None = None camera_failure_code: str | None = None stop_runtime_for_camera_failure = False stop_runtime_for_canonical_completion = False + confirmed_stop_local_camera_failure = False + stop_accepted_physical_outcome_unknown = False complete_after_seal = False + recovery_standby_after_seal = False + recovery_standby_start_operation_id: str | None = None + recovery_standby_stop_operation_id: str | None = None completion_message_code = "acquisition.receiver_completed" completion_result: dict[str, Any] = { "receiver_stopped": True, "device_state": "unknown", } + with self._lock: + recovery_lineage = self._active_stream_recovery_lineage + recovery_state = self._active_stream_recovery_state + recovery_terminal_outcome = self._active_stream_recovery_terminal_outcome + recovery_generation = self._active_stream_recovery_generation + prepared_stop_recovery_owner = getattr( + self, + "_prepared_stop_recovery_owner", + None, + ) + process_lease_quarantined = ( + self._application_control_process_lease_quarantine is not None + ) + prepared_stop_recovery_pending = bool( + prepared_stop_recovery_owner is not None + and ( + self._prepared_stop_recovery_owner_is_current( + prepared_stop_recovery_owner, + ) + or ( + current is not None + and current.message_code + == "acquisition.recovery.scanning_adoption_pending" + and self._prepared_stop_recovery_owner_is_current( + prepared_stop_recovery_owner, + allow_resolved_classification=True, + ) + ) + ) + ) + prepared_stop_recovery_standby = bool( + prepared_stop_recovery_owner is not None + and self._prepared_stop_recovery_owner_is_current( + prepared_stop_recovery_owner, + allow_resolved_standby=True, + ) + ) + prepared_stop_cleanup_retry = bool( + prepared_stop_recovery_owner is not None + and self._prepared_stop_recovery_owner_is_current( + prepared_stop_recovery_owner, + allow_resolved_standby=True, + allow_terminal_cleanup=True, + ) + and current is not None + and current.state == "failed" + and isinstance(current.result, Mapping) + and current.result.get("local_cleanup_retry_pending") is True + ) + recovery_nonterminal = bool( + runtime.get("phase") == "reconnecting" + and recovery_lineage is not None + and recovery_lineage.recovery_generation == recovery_generation + and recovery_state in {"reconnecting", "blocked"} + and not camera_nontransport_error + ) with self._lock: acquisition = self._acquisition - if acquisition is None or acquisition.state in TERMINAL_ACQUISITION_STATES: + if acquisition is None or ( + acquisition.state in TERMINAL_ACQUISITION_STATES + and not prepared_stop_cleanup_retry + ): + return + if prepared_stop_cleanup_retry and process_lease_quarantined: + # An ambiguous OS fence release is not a local retry state. + # Preserve the exact evidence/receiver owner and let the + # public snapshot advertise restart-required quarantine. return metrics = runtime.get("metrics") point_frames = metrics.get("pcl_frames", 0) if isinstance(metrics, dict) else 0 phase = runtime.get("phase") source_mode = runtime.get("source_mode") source_ready = runtime.get("source_ready") is True - camera_recording = camera.get("recording") camera_recording_active = ( isinstance(camera_recording, dict) and camera_recording.get("active") is True ) - camera_error = camera.get("error") - if camera.get("phase") == "error" and camera_recording_active: + if prepared_stop_cleanup_retry: + camera_terminal_status = "interrupted" + camera_failure_code = "device-ended-during-link-recovery" + recovery_standby_after_seal = True + recovery_standby_start_operation_id = self._acquisition_start_operation_id + recovery_standby_stop_operation_id = ( + prepared_stop_recovery_owner.lineage.stop_operation_id + if prepared_stop_recovery_owner is not None + else self._acquisition_stop_operation_id + ) + completion_message_code = "acquisition.recovery.device_standby_observed" + completion_result = { + "receiver_stopped": True, + "device_state": "ready-or-scan-over", + "device_stop": "not-sent", + "automatic_command_retry": False, + "read_only_recovery": True, + } + elif ( + acquisition.state == "awaiting_external_stop" + and acquisition.control_mode == "plugin-commanded" + and canonical_stop_standby_confirmed + ): + acquisition.transition( + "finalizing", + message_code="acquisition.stop.device_standby_confirmed", + ) + camera_failed_before_cleanup = bool( + camera.get("phase") == "error" and camera_recording_active + ) + if camera_failed_before_cleanup: + camera_failure_code = ( + str(camera_error.get("code")) + if isinstance(camera_error, dict) and camera_error.get("code") + else "camera-producer-failed" + ) + camera_terminal_status = "failed" + confirmed_stop_local_camera_failure = True + else: + camera_terminal_status = "complete" + complete_after_seal = True + stop_runtime_for_canonical_completion = True + completed_stop_operation_id = self._acquisition_stop_operation_id + # STOP may legitimately be requested after canonical + # SCANNING but before the first PCL reaches the local writer. + # READY proves that original START did have a physical side + # effect, yet it can no longer be completed by point data. + # Seal P truthfully instead of leaving its journal row running + # behind the terminal S1 acquisition. + if point_frames <= 0: + recovery_standby_start_operation_id = ( + self._acquisition_start_operation_id + ) + completion_message_code = "acquisition.stop.completed" + completion_result = { + "receiver_stopped": True, + "device_state": "ready", + "device_stop": "protocol-confirmed", + } + elif ( + acquisition.state == "awaiting_external_stop" + and acquisition.control_mode == "plugin-commanded" + and stop_response_deadline_reached + ): + acquisition.transition( + "finalizing", + message_code="acquisition.stop.accepted_physical_outcome_unknown", + ) + camera_failed_before_cleanup = bool( + camera.get("phase") == "error" and camera_recording_active + ) + camera_failure_code = ( + str(camera_error.get("code")) + if camera_failed_before_cleanup + and isinstance(camera_error, dict) + and camera_error.get("code") + else "stop-accepted-physical-outcome-unknown" + ) + camera_terminal_status = "failed" if camera_failed_before_cleanup else "interrupted" + stop_runtime_for_canonical_completion = True + stop_accepted_physical_outcome_unknown = True + unknown_stop_operation_id = self._acquisition_stop_operation_id + completion_result = { + "receiver_stopped": True, + "device_state": "unknown", + "device_stop": "stop-accepted-physical-outcome-unknown", + "automatic_replay_allowed": False, + } + if camera_failed_before_cleanup: + completion_result["camera_failure_code"] = camera_failure_code + elif ( + acquisition.state == "awaiting_external_stop" + and acquisition.control_mode == "plugin-commanded" + and stop_response_accepted_without_terminal_status + ): + # The exact device response proves STOP was accepted, so a + # concurrent local runtime/control failure must not relabel it + # as a generic retryable failure. Keep the bounded wait until + # the server-owned deadline; the timeout branch above then + # performs local-only cleanup with physical outcome unknown. + pass + elif ( + acquisition.state == "awaiting_external_stop" + and acquisition.control_mode == "plugin-commanded" + and prepared_stop_worker_outcome + in {"settled", "defer", "prepared-pending"} + ): + # Exact PREPARED remains ahead of every MQTT publish edge. + # While its worker is pending (or owns the gate), local + # runtime/camera symptoms cannot overtake it. Once the worker + # terminally fails before dispatch, preserve the receiver for + # read-only classification and seal the old STOP as + # non-replayable/none. + pass + elif prepared_stop_recovery_standby: + acquisition.transition( + "finalizing", + message_code="acquisition.recovery.device_standby_observed", + ) + camera_terminal_status = "interrupted" + camera_failure_code = "device-ended-during-link-recovery" + recovery_standby_after_seal = True + recovery_standby_start_operation_id = self._acquisition_start_operation_id + recovery_standby_stop_operation_id = ( + prepared_stop_recovery_owner.lineage.stop_operation_id + if prepared_stop_recovery_owner is not None + else self._acquisition_stop_operation_id + ) + completion_message_code = "acquisition.recovery.device_standby_observed" + completion_result = { + "receiver_stopped": True, + "device_state": "ready-or-scan-over", + "device_stop": "not-sent", + "automatic_command_retry": False, + "read_only_recovery": True, + } + elif ( + recovery_terminal_outcome == "standby" + and source_mode == "idle" + and recovery_lineage is not None + and recovery_lineage.acquisition_id == acquisition.acquisition_id + ): + acquisition.transition( + "finalizing", + message_code="acquisition.recovery.device_standby_observed", + ) + camera_terminal_status = "interrupted" + camera_failure_code = "device-ended-during-link-recovery" + recovery_standby_after_seal = True + recovery_standby_start_operation_id = self._acquisition_start_operation_id + recovery_standby_stop_operation_id = self._acquisition_stop_operation_id + completion_message_code = "acquisition.recovery.device_standby_observed" + completion_result = { + "receiver_stopped": True, + "device_state": "ready-or-scan-over", + "device_stop": "not-sent", + "automatic_command_retry": False, + "read_only_recovery": True, + } + elif ( + recovery_terminal_outcome == "fault" + and recovery_lineage is not None + and recovery_lineage.acquisition_id == acquisition.acquisition_id + ): + acquisition.transition( + "failed", + message_code="acquisition.recovery.device_fault", + result={ + "receiver_stopped": False, + "device_state": "fault", + "device_stop": "not-sent", + "automatic_command_retry": False, + "read_only_recovery": True, + }, + ) + camera_terminal_status = "failed" + camera_failure_code = "active-stream-recovery-fault" + stop_runtime_for_camera_failure = True + elif recovery_nonterminal: + # Link and FFmpeg failures stay provisional while this exact + # composite START owns the read-only recovery generation. + pass + elif prepared_stop_recovery_pending: + # The exact retained receiver is either awaiting its read-only + # PREPARED classification or has durably committed SCANNING + # while the control worker adopts that same proof. Runtime + # and camera symptoms remain provisional across this local + # commit/adoption seam and cannot release its evidence lease. + pass + elif camera_activation_retry_provisional: + # Exact post-PCL activation startup failures are local and + # generation-fenced. Preserve the active PCL/Rerun session so + # a later authoritative frame can perform the bounded retry. + # Storage/archive failures are intentionally excluded above. + pass + elif camera_transport_recovery_pending: + # The backend camera watchdog owns this exact local CAS. Keep + # PCL/control and the physical STOP authority live while the + # replacement epoch is pending; browser polling is read-only. + pass + elif restart_rehydration_reduction_pending: + # Verify owns an exact restart writer until the PCL callback + # both recognizes the durable checkpoint receipt and retires + # its local pending claim. In particular, a store call may + # fsync successfully and then lose its response. The generic + # point-frame reducer must not infer ``acquiring`` across that + # seam: a later frame retries the same immutable transition. + # Explicit STOP changes the acquisition state first and is + # intentionally outside this fence. + pass + elif ( + acquisition.state in {"starting", "awaiting_external_start"} + and acquisition.control_mode == "plugin-commanded" + and plugin_start_outcome_unknown + ): + # The physical START checkpoint dominates coincident local + # camera/runtime failure. Once the canonical dialogue reports + # a terminal post-dispatch fault, cleanup may be performed only + # by the explicit capture-only action; never relabel the START + # as a retry-safe stream failure or replay a physical command. + acquisition.transition( + "failed", + message_code="acquisition.start.device_outcome_unknown", + result={ + "receiver_stopped": False, + "device_state": "unknown", + "device_start": "outcome-unknown", + "automatic_replay_allowed": False, + }, + ) + unknown_start_operation_id = self._acquisition_start_operation_id + unknown_start_side_effect_status = ( + "succeeded" if durable_start_active_confirmed else "unknown" + ) + elif camera.get("phase") == "error" and camera_recording_active: waiting_for_start = acquisition.state in { "starting", "awaiting_external_start", @@ -3288,30 +28580,26 @@ class XgridsK1CompatibilityService: ) if waiting_for_start: failed_operation_id = self._acquisition_start_operation_id + if acquisition.control_mode == "plugin-commanded": + failed_start_physical_side_effect = ( + "succeeded" if durable_start_active_confirmed else "unknown" + ) if waiting_for_stop: failed_stop_operation_id = self._acquisition_stop_operation_id camera_terminal_status = "failed" stop_runtime_for_camera_failure = True elif ( - acquisition.state == "awaiting_external_stop" - and acquisition.control_mode == "plugin-commanded" - and application_control_session.get("state") == "completed" - ): - acquisition.transition( - "finalizing", - message_code="acquisition.stop.device_standby_confirmed", + acquisition.state in {"starting", "awaiting_external_start"} + and point_frames > 0 + and ( + acquisition.control_mode != "plugin-commanded" + or ( + durable_start_active_confirmed + and durable_checkpoint_active_confirmed + and canonical_start_control_active + ) ) - camera_terminal_status = "complete" - stop_runtime_for_canonical_completion = True - complete_after_seal = True - completed_stop_operation_id = self._acquisition_stop_operation_id - completion_message_code = "acquisition.stop.completed" - completion_result = { - "receiver_stopped": True, - "device_state": "ready", - "device_stop": "protocol-confirmed", - } - elif acquisition.state in {"starting", "awaiting_external_start"} and point_frames > 0: + ): acquisition.transition("acquiring", message_code="acquisition.acquiring") completed_operation_id = self._acquisition_start_operation_id acquisition_id = acquisition.acquisition_id @@ -3336,6 +28624,25 @@ class XgridsK1CompatibilityService: ) receiver_ready_operation_id = self._acquisition_start_operation_id acquisition_id = acquisition.acquisition_id + elif ( + acquisition.state == "awaiting_external_stop" + and acquisition.control_mode == "plugin-commanded" + and stop_acknowledged_standby_unconfirmed + ): + acquisition.transition( + "failed", + message_code="acquisition.stop.standby_unconfirmed_after_control_loss", + result={ + "receiver_stopped": True, + "device_state": "standby-unconfirmed", + "device_stop": "stop-acknowledged-standby-unconfirmed", + "automatic_replay_allowed": False, + }, + ) + camera_terminal_status = "failed" + camera_failure_code = "stop-acknowledged-standby-unconfirmed" + stop_runtime_for_camera_failure = True + acknowledged_unconfirmed_stop_operation_id = self._acquisition_stop_operation_id elif phase == "error": waiting_for_start = acquisition.state in { "starting", @@ -3355,6 +28662,10 @@ class XgridsK1CompatibilityService: stop_runtime_for_camera_failure = True if waiting_for_start: failed_operation_id = self._acquisition_start_operation_id + if acquisition.control_mode == "plugin-commanded": + failed_start_physical_side_effect = ( + "succeeded" if durable_start_active_confirmed else "unknown" + ) if waiting_for_stop: failed_stop_operation_id = self._acquisition_stop_operation_id elif ( @@ -3369,6 +28680,10 @@ class XgridsK1CompatibilityService: camera_terminal_status = "failed" camera_failure_code = "receiver-completed-without-point-data" failed_operation_id = self._acquisition_start_operation_id + if acquisition.control_mode == "plugin-commanded": + failed_start_physical_side_effect = ( + "succeeded" if durable_start_active_confirmed else "unknown" + ) elif acquisition.state == "acquiring" and source_mode == "idle": # Reserve terminal reconciliation before invoking camera or # filesystem callbacks. A nested/concurrent state read now @@ -3390,18 +28705,44 @@ class XgridsK1CompatibilityService: unconfirmed_stop_operation_id = self._acquisition_stop_operation_id reconciliation_error: Exception | None = None + recovery_cleanup_stage: Literal[ + "camera-source", + "camera-process-lease", + "runtime", + "capture-clock", + "perception-ingress", + "evidence-lease", + ] | None = None if camera_terminal_status is not None: terminal_error: Exception | None = None try: - self.camera_preview.stop_recording( - status=camera_terminal_status, - failure_code=camera_failure_code, - ) + if recovery_standby_after_seal: + recovery_cleanup_stage = "camera-source" + self.camera_preview.stop_current() + recovery_cleanup_stage = "camera-process-lease" + self._release_camera_preview_process_lease() + with self._lock: + self._live_perception_camera_binding = None + self._camera_activation_lineage = None + self._camera_activation_retry_lineage = None + self._camera_activation_retry_not_before_monotonic = 0.0 + else: + self.camera_preview.stop_recording( + status=camera_terminal_status, + failure_code=camera_failure_code, + ) except Exception as exc: terminal_error = exc - if stop_runtime_for_camera_failure or stop_runtime_for_canonical_completion: + if ( + stop_runtime_for_camera_failure + or stop_runtime_for_canonical_completion + or (recovery_standby_after_seal and terminal_error is None) + ): try: + if recovery_standby_after_seal: + recovery_cleanup_stage = "runtime" self.runtime.stop() + self._reset_live_data_plane_observation() except Exception as exc: if terminal_error is not None: exc.add_note( @@ -3412,7 +28753,24 @@ class XgridsK1CompatibilityService: try: if terminal_error is not None: raise terminal_error + if recovery_standby_after_seal: + recovery_cleanup_stage = "capture-clock" self._seal_acquisition_capture_clock() + if recovery_standby_after_seal: + # READY/SCAN_OVER is a local-only cleanup transaction. + # Keep the evidence lease (our cleanup_pending fence) + # through camera, runtime, clock and ingress cleanup; the + # lease is released last. Any failure before that point + # remains retryable without another device command. + with self._lock: + finalized_out_dir = self._acquisition_out_dir + if finalized_out_dir is not None: + recovery_cleanup_stage = "perception-ingress" + self.live_perception_ingress.end_session( + finalized_out_dir.name + ) + recovery_cleanup_stage = "evidence-lease" + self._release_acquisition_session_lease() if complete_after_seal: with self._lock: if acquisition.state not in TERMINAL_ACQUISITION_STATES: @@ -3421,21 +28779,137 @@ class XgridsK1CompatibilityService: message_code=completion_message_code, result=completion_result, ) - except Exception as exc: - reconciliation_error = exc - if complete_after_seal: + elif recovery_standby_after_seal: + with self._lock: + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "interrupted", + message_code=completion_message_code, + result=completion_result, + ) + elif prepared_stop_cleanup_retry and ( + self._acquisition is acquisition + and acquisition.state == "failed" + and isinstance(acquisition.result, dict) + and acquisition.result.get("read_only_recovery") is True + and acquisition.result.get("device_stop") == "not-sent" + and acquisition.result.get("local_cleanup_retry_pending") is True + ): + # A retry seals only the exact terminal recovery + # owner that failed its first local cleanup. Keep + # the historical failed state, but project the + # newly proven resource truth before releasing its + # evidence lease/fence below. + acquisition.result = dict(completion_result) + acquisition.message_code = ( + "acquisition.recovery.local_cleanup_completed" + ) + acquisition.state_revision += 1 + acquisition.updated_at = datetime.now(UTC) + elif confirmed_stop_local_camera_failure: with self._lock: if acquisition.state not in TERMINAL_ACQUISITION_STATES: acquisition.transition( "failed", - message_code="acquisition.capture_clock_failed", + message_code="acquisition.stop.local_finalization_failed", + result={ + **completion_result, + "camera_failure_code": camera_failure_code, + }, + ) + elif stop_accepted_physical_outcome_unknown: + with self._lock: + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "failed", + message_code=("acquisition.stop.accepted_physical_outcome_unknown"), + result=completion_result, + ) + except Exception as exc: + reconciliation_error = exc + if ( + complete_after_seal + or recovery_standby_after_seal + or confirmed_stop_local_camera_failure + or stop_accepted_physical_outcome_unknown + ): + with self._lock: + if recovery_standby_after_seal: + cleanup_stage = recovery_cleanup_stage or "unknown" + cleanup_result = { + **completion_result, + "receiver_stopped": cleanup_stage + in { + "capture-clock", + "perception-ingress", + "evidence-lease", + }, + "camera_stopped": cleanup_stage != "camera-source", + "local_cleanup_retry_pending": True, + "local_cleanup_stage": cleanup_stage, + "local_cleanup_error_code": str( + getattr(exc, "reason_code", None) + or type(exc).__name__ + ), + } + cleanup_message_code = { + "camera-source": ( + "acquisition.recovery.camera_cleanup_failed" + ), + "camera-process-lease": ( + "acquisition.recovery.camera_fence_release_failed" + ), + "runtime": ( + "acquisition.recovery.runtime_cleanup_failed" + ), + "capture-clock": ( + "acquisition.recovery.capture_clock_failed" + ), + "perception-ingress": ( + "acquisition.recovery.perception_ingress_cleanup_failed" + ), + "evidence-lease": ( + "acquisition.recovery.evidence_lease_release_failed" + ), + }.get( + cleanup_stage, + "acquisition.recovery.local_cleanup_failed", + ) + if acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "failed", + message_code=cleanup_message_code, + result=cleanup_result, + ) + elif prepared_stop_cleanup_retry: + acquisition.result = cleanup_result + acquisition.message_code = cleanup_message_code + acquisition.state_revision += 1 + acquisition.updated_at = datetime.now(UTC) + elif acquisition.state not in TERMINAL_ACQUISITION_STATES: + acquisition.transition( + "failed", + message_code=( + "acquisition.stop.outcome_unknown_local_cleanup_failed" + if stop_accepted_physical_outcome_unknown + else "acquisition.capture_clock_failed" + ), + result=( + { + **completion_result, + "receiver_stopped": False, + "local_cleanup_retry_pending": True, + } + if stop_accepted_physical_outcome_unknown + else None + ), ) finally: with self._lock: finalized_out_dir = self._acquisition_out_dir - if finalized_out_dir is not None: + if finalized_out_dir is not None and not recovery_standby_after_seal: self.live_perception_ingress.end_session(finalized_out_dir.name) - if reconciliation_error is None: + if reconciliation_error is None and not recovery_standby_after_seal: self._release_acquisition_session_lease() if receiver_ready_operation_id is not None: @@ -3455,12 +28929,54 @@ class XgridsK1CompatibilityService: result={"acquisition_id": acquisition_id}, ) if completed_operation_id is not None: + plugin_commanded_confirmation = acquisition.control_mode == "plugin-commanded" self._operations.transition_if_pending( completed_operation_id, "succeeded", - stage_code="first-point-frame", + stage_code=( + "canonical-start-and-first-point-frame" + if plugin_commanded_confirmation + else "first-point-frame" + ), message_code="acquisition.start.confirmed_by_data", - result={"acquisition_id": acquisition_id, "confirmation": "point-frame"}, + result={ + "acquisition_id": acquisition_id, + "confirmation": ( + "canonical-start-and-point-frame" + if plugin_commanded_confirmation + else "point-frame" + ), + }, + ) + if recovery_standby_start_operation_id is not None: + self._operations.transition_if_pending( + recovery_standby_start_operation_id, + "interrupted", + stage_code="physical-start-active-but-no-point-before-standby", + message_code="acquisition.start.device_standby_before_point_data", + error={ + "category": "device", + "code": "device-standby-before-first-point-frame", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "succeeded", + "automatic_replay_allowed": False, + }, + ) + if unknown_start_operation_id is not None: + self._operations.transition_if_pending( + unknown_start_operation_id, + "failed", + stage_code="device-start-outcome-unknown", + message_code="acquisition.start.device_outcome_unknown", + error={ + "category": "device", + "code": "start-outcome-unknown", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": unknown_start_side_effect_status, + "automatic_replay_allowed": False, + }, ) if failed_operation_id is not None: self._operations.transition_if_pending( @@ -3468,13 +28984,24 @@ class XgridsK1CompatibilityService: "failed", stage_code="runtime-failed", message_code="acquisition.start.runtime_failed", - error={ - "category": "stream", - "code": "runtime-failed", - "retryable": False, - "safe_to_retry": True, - "side_effect_status": "none", - }, + error=( + { + "category": "stream", + "code": "runtime-failed", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": failed_start_physical_side_effect, + "automatic_replay_allowed": False, + } + if failed_start_physical_side_effect is not None + else { + "category": "stream", + "code": "runtime-failed", + "retryable": False, + "safe_to_retry": True, + "side_effect_status": "none", + } + ), ) if failed_stop_operation_id is not None: self._operations.transition_if_pending( @@ -3504,8 +29031,23 @@ class XgridsK1CompatibilityService: "side_effect_status": "unknown", }, ) + if acknowledged_unconfirmed_stop_operation_id is not None: + self._operations.transition_if_pending( + acknowledged_unconfirmed_stop_operation_id, + "failed", + stage_code="stop-acknowledged-standby-unconfirmed", + message_code="acquisition.stop.standby_unconfirmed_after_control_loss", + error={ + "category": "device", + "code": "stop-acknowledged-standby-unconfirmed", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + }, + ) if completed_stop_operation_id is not None: - if reconciliation_error is None: + if reconciliation_error is None and not confirmed_stop_local_camera_failure: self._operations.transition_if_pending( completed_stop_operation_id, "succeeded", @@ -3530,34 +29072,116 @@ class XgridsK1CompatibilityService: "side_effect_status": "succeeded", }, ) + if unknown_stop_operation_id is not None: + if reconciliation_error is None: + self._operations.transition_if_pending( + unknown_stop_operation_id, + "timed_out", + stage_code="stop-accepted-physical-outcome-unknown", + message_code="acquisition.stop.accepted_physical_outcome_unknown", + result={ + "acquisition_id": acquisition.acquisition_id, + "receiver_stopped": True, + "device_state": "unknown", + "device_stop": "stop-accepted-physical-outcome-unknown", + }, + error={ + "category": "device", + "code": "stop-accepted-physical-outcome-unknown", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + }, + ) + else: + self._operations.transition_if_pending( + unknown_stop_operation_id, + "failed", + stage_code="local-cleanup-failed-after-stop-outcome-unknown", + message_code="acquisition.stop.outcome_unknown_local_cleanup_failed", + error={ + "category": "stream", + "code": "local-cleanup-failed-after-stop-outcome-unknown", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + }, + ) with self._lock: if self._acquisition_start_operation_id in { completed_operation_id, failed_operation_id, + unknown_start_operation_id, + recovery_standby_start_operation_id, }: self._acquisition_start_operation_id = None if self._acquisition_stop_operation_id == failed_stop_operation_id: self._acquisition_stop_operation_id = None if self._acquisition_stop_operation_id == unconfirmed_stop_operation_id: self._acquisition_stop_operation_id = None - if self._acquisition_stop_operation_id == completed_stop_operation_id: + if self._acquisition_stop_operation_id == acknowledged_unconfirmed_stop_operation_id: self._acquisition_stop_operation_id = None - if reconciliation_error is not None: + if ( + reconciliation_error is None + and self._acquisition_stop_operation_id == completed_stop_operation_id + ): + self._acquisition_stop_operation_id = None + if ( + reconciliation_error is None + and self._acquisition_stop_operation_id == unknown_stop_operation_id + ): + self._acquisition_stop_operation_id = None + if reconciliation_error is None and recovery_standby_after_seal: + if self._acquisition_stop_operation_id == recovery_standby_stop_operation_id: + self._acquisition_stop_operation_id = None + self._prepared_stop_dispatch_lineage = None + self._prepared_stop_recovery_owner = None + self._classified_stop_rebind_pending = None + self._classified_stop_rebind_inflight = None + if ( + reconciliation_error is not None + and not durable_stop_standby_confirmed + and not stop_response_deadline_reached + ): raise reconciliation_error class XgridsK1ServicePort(Protocol): + def require_snapshot_runtime_id(self, expected_snapshot_runtime_id: str) -> None: ... + + def bind_runtime_event_loop( + self, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: ... + def state(self) -> dict[str, Any]: ... - async def scan_ble(self, duration_seconds: float) -> dict[str, Any]: ... + async def scan_ble(self, request: BleScanRequest) -> dict[str, Any]: ... async def connect(self, request: ConnectRequest) -> dict[str, Any]: ... + def select_connection_mode( + self, + request: DesiredConnectionModeRequest, + ) -> dict[str, Any]: ... + + async def prepare_connection_reconfiguration( + self, + request: PrepareConnectionReconfigurationRequest, + ) -> dict[str, Any]: ... + def inspect_device(self) -> dict[str, Any]: ... - def verify_connection( + async def verify_connection( self, - request: ConnectionVerifyRequest | None = None, + request: ConnectionVerifyRequest, + ) -> dict[str, Any]: ... + + async def probe_configured_endpoint( + self, + request: ConfiguredEndpointProbeRequest | None = None, ) -> dict[str, Any]: ... def read_device_calibration_snapshot(self) -> dict[str, Any]: ... @@ -3572,7 +29196,25 @@ class XgridsK1ServicePort(Protocol): request: EnterApplicationWorkspaceRequest, ) -> dict[str, Any]: ... - def close_application_control_session(self) -> dict[str, Any]: ... + def close_application_control_session( + self, + request: CloseApplicationControlSessionRequest, + ) -> dict[str, Any]: ... + + def reconcile_physical_command( + self, + request: ReconcilePhysicalCommandRequest, + ) -> dict[str, Any]: ... + + def retire_unavailable_physical_command( + self, + request: RetireUnavailablePhysicalCommandRequest, + ) -> dict[str, Any]: ... + + def reopen_retired_physical_command_reconciliation( + self, + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> dict[str, Any]: ... def arm_application_control_shadow( self, @@ -3589,6 +29231,11 @@ class XgridsK1ServicePort(Protocol): def abort_acquisition(self, request: AbortAcquisitionRequest) -> dict[str, Any]: ... + def force_finish_acquisition_locally( + self, + request: ForceFinishAcquisitionRequest, + ) -> dict[str, Any]: ... + def start_live( self, project_name: str, @@ -3608,6 +29255,30 @@ class XgridsK1ServicePort(Protocol): def update_viewer_settings(self, request: ViewerSettingsRequest) -> dict[str, Any]: ... +_SNAPSHOT_RUNTIME_FENCED_MUTATING_ACTIONS = frozenset( + { + ACTION_DISCOVERY_SCAN, + ACTION_NETWORK_PROVISION, + ACTION_CONNECTION_MODE_SELECT, + ACTION_CONNECTION_RECONFIGURE_PREPARE, + ACTION_CONNECTION_VERIFY, + ACTION_CONFIGURED_ENDPOINT_PROBE, + ACTION_ACQUISITION_PREPARE, + ACTION_ACQUISITION_START, + ACTION_ACQUISITION_STOP, + ACTION_ACQUISITION_ABORT, + ACTION_ACQUISITION_FORCE_FINISH, + ACTION_APPLICATION_CONTROL_SESSION_OPEN, + ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER, + ACTION_APPLICATION_CONTROL_SESSION_CLOSE, + ACTION_PHYSICAL_COMMAND_RECONCILE, + ACTION_STREAM_STOP, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + } +) + + class XgridsK1PluginFacade: """Plugin action facade delegating to the proven compatibility service.""" @@ -3620,11 +29291,15 @@ class XgridsK1PluginFacade: ACTION_SENSOR_CATALOG_READ, ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ, ACTION_NETWORK_PROVISION, + ACTION_CONNECTION_MODE_SELECT, + ACTION_CONNECTION_RECONFIGURE_PREPARE, ACTION_CONNECTION_VERIFY, + ACTION_CONFIGURED_ENDPOINT_PROBE, ACTION_ACQUISITION_PREPARE, ACTION_ACQUISITION_START, ACTION_ACQUISITION_STOP, ACTION_ACQUISITION_ABORT, + ACTION_ACQUISITION_FORCE_FINISH, ACTION_ACQUISITION_STATE_READ, ACTION_STREAM_START_LIVE, ACTION_STREAM_START_REPLAY, @@ -3638,6 +29313,9 @@ class XgridsK1PluginFacade: ACTION_APPLICATION_CONTROL_SESSION_OPEN, ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER, ACTION_APPLICATION_CONTROL_SESSION_CLOSE, + ACTION_PHYSICAL_COMMAND_RECONCILE, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, } ) @@ -3655,19 +29333,70 @@ class XgridsK1PluginFacade: except (ValidationError, ValueError, PluginActionNotFoundError): raise except (BleakError, OSError, TimeoutError, RuntimeError) as exc: - raise PluginExecutionError(str(exc)) from exc + reason_code = getattr(exc, "reason_code", None) + raise PluginExecutionError( + str(exc), + http_status_code=_plugin_execution_http_status(reason_code), + reason_code=reason_code if isinstance(reason_code, str) else None, + ) from exc async def _invoke_validated( self, action_id: str, payload: Mapping[str, Any], ) -> dict[str, Any]: + if action_id in _SNAPSHOT_RUNTIME_FENCED_MUTATING_ACTIONS: + expected_snapshot_runtime_id = payload.get("expected_snapshot_runtime_id") + if ( + not isinstance(expected_snapshot_runtime_id, str) + or not expected_snapshot_runtime_id.strip() + ): + raise SnapshotRuntimeConflict() + self.service.require_snapshot_runtime_id(expected_snapshot_runtime_id) + payload = { + key: value + for key, value in payload.items() + if key != "expected_snapshot_runtime_id" + } + # Operator retirement is a local durable CAS and must not initialize + # CoreBluetooth or start the connection monitor. A freshly created + # facade may still carry the old semantic endpoint; binding its loop + # here would let the monitor probe that endpoint before the retirement + # thread wins the lifecycle fence. + if action_id == ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE: + retire_request = RetireUnavailablePhysicalCommandRequest.model_validate(payload) + return await asyncio.to_thread( + self.service.retire_unavailable_physical_command, + retire_request, + ) + if action_id == ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION: + reopen_request = ReopenRetiredPhysicalCommandReconciliationRequest.model_validate( + payload + ) + return await asyncio.to_thread( + self.service.reopen_retired_physical_command_reconciliation, + reopen_request, + ) + # A scenario reset owns retirement of the old local connection + # boundary. Binding the runtime loop first could revive the old + # connection monitor and let it probe the semantic endpoint while the + # reset is still queued behind a lifecycle owner. Dispatch this exact + # local-only request before loop binding; ordinary mode drafts retain + # the existing bind behavior. + if action_id == ACTION_CONNECTION_MODE_SELECT: + mode_request = DesiredConnectionModeRequest.model_validate(payload) + if mode_request.reset_scenario is True: + return await asyncio.to_thread( + self.service.select_connection_mode, + mode_request, + ) + self.service.bind_runtime_event_loop(asyncio.get_running_loop()) if action_id == ACTION_STATE_READ: EmptyRequest.model_validate(payload) return await asyncio.to_thread(self.service.state) if action_id == ACTION_DISCOVERY_SCAN: scan_request = BleScanRequest.model_validate(payload) - return await self.service.scan_ble(scan_request.duration_seconds) + return await self.service.scan_ble(scan_request) if action_id in { ACTION_DEVICE_INSPECT, ACTION_SENSOR_CATALOG_READ, @@ -3683,9 +29412,21 @@ class XgridsK1PluginFacade: if action_id == ACTION_NETWORK_PROVISION: connect_request = ConnectRequest.model_validate(payload) return await self.service.connect(connect_request) + if action_id == ACTION_CONNECTION_MODE_SELECT: + mode_request = DesiredConnectionModeRequest.model_validate(payload) + return await asyncio.to_thread( + self.service.select_connection_mode, + mode_request, + ) + if action_id == ACTION_CONNECTION_RECONFIGURE_PREPARE: + reconfigure_request = PrepareConnectionReconfigurationRequest.model_validate(payload) + return await self.service.prepare_connection_reconfiguration(reconfigure_request) if action_id == ACTION_CONNECTION_VERIFY: verify_request = ConnectionVerifyRequest.model_validate(payload) - return await asyncio.to_thread(self.service.verify_connection, verify_request) + return await self.service.verify_connection(verify_request) + if action_id == ACTION_CONFIGURED_ENDPOINT_PROBE: + probe_request = ConfiguredEndpointProbeRequest.model_validate(payload) + return await self.service.probe_configured_endpoint(probe_request) if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE: EmptyRequest.model_validate(payload) return await asyncio.to_thread(self.service.state) @@ -3711,8 +29452,17 @@ class XgridsK1PluginFacade: workspace_request, ) if action_id == ACTION_APPLICATION_CONTROL_SESSION_CLOSE: - EmptyRequest.model_validate(payload) - return await asyncio.to_thread(self.service.close_application_control_session) + close_request = CloseApplicationControlSessionRequest.model_validate(payload) + return await asyncio.to_thread( + self.service.close_application_control_session, + close_request, + ) + if action_id == ACTION_PHYSICAL_COMMAND_RECONCILE: + reconcile_request = ReconcilePhysicalCommandRequest.model_validate(payload) + return await asyncio.to_thread( + self.service.reconcile_physical_command, + reconcile_request, + ) if action_id == ACTION_ACQUISITION_PREPARE: prepare_request = PrepareAcquisitionRequest.model_validate(payload) return await asyncio.to_thread(self.service.prepare_acquisition, prepare_request) @@ -3725,6 +29475,12 @@ class XgridsK1PluginFacade: if action_id == ACTION_ACQUISITION_ABORT: abort_request = AbortAcquisitionRequest.model_validate(payload) return await asyncio.to_thread(self.service.abort_acquisition, abort_request) + if action_id == ACTION_ACQUISITION_FORCE_FINISH: + force_finish_request = ForceFinishAcquisitionRequest.model_validate(payload) + return await asyncio.to_thread( + self.service.force_finish_acquisition_locally, + force_finish_request, + ) if action_id == ACTION_STREAM_START_LIVE: live_request = LiveRequest.model_validate(payload) return await asyncio.to_thread( @@ -3770,6 +29526,133 @@ def _utc_now_iso() -> str: return datetime.now(UTC).isoformat().replace("+00:00", "Z") +def _utc_now_millisecond_iso() -> str: + """Return the physical-ledger's canonical millisecond UTC wire format.""" + + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _plugin_execution_http_status(reason_code: object) -> int: + """Map stable plugin failure classes without calling expected state a gateway fault.""" + + if isinstance(reason_code, str) and reason_code.startswith("connection-reconfiguration-"): + return 409 + if isinstance(reason_code, str) and reason_code.startswith("connection-scenario-reset-"): + return 409 + if isinstance(reason_code, str) and reason_code.startswith("physical-command-retirement-"): + return 409 + if isinstance(reason_code, str) and reason_code.startswith( + "physical-command-reconciliation-reopen-" + ): + return 409 + if reason_code in { + "connection-verify-candidate-not-fresh", + "connection-verify-resolved-apply-target-mismatch", + "connection-verify-device-not-connectable", + "connection-verify-device-not-rediscovered", + "connection-verify-address-unavailable", + "connection-verify-route-mismatch", + "connection-verify-local-address-conflict", + "connection-verify-candidate-changed", + "connection-verify-discovery-generation-conflict", + "connection-verify-lease-changed", + "connection-verify-busy", + "connection-verify-cleanup-pending", + "connection-verify-lifecycle-busy", + "connection-verify-connection-missing", + "connection-verify-target-not-distinguishable-from-baseline", + "connection-verify-mqtt-unreachable", + "configured-endpoint-unavailable", + "configured-endpoint-topology-corrupt", + "configured-endpoint-changed-during-probe", + "configured-endpoint-probe-lifecycle-busy", + "connection-mode-draft-revision-conflict", + "connection-mode-draft-mismatch", + "connection-mode-switch-pending", + "connection-mode-switch-physical-state-unsafe", + "connection-mode-switch-control-state-unsafe", + "connection-mode-switch-lifecycle-busy", + "connection-mode-switch-acquisition-changed", + "connection-mode-selection-lifecycle-busy", + "connection-mode-selection-physical-state-unsafe", + "connection-mode-selection-control-state-unsafe", + "acquisition-start-lifecycle-busy", + "control-bootstrap-endpoint-unverified", + "control-bootstrap-device-identity-unverified", + "control-bootstrap-terminal-failure", + "physical-command-reconciliation-control-changed", + "physical-command-reconciliation-binding-changed", + "physical-command-reconciliation-state-unsafe", + "connection-supervisor-closed", + "ble-discovery-blocked-by-provisioning", + "ble-runtime-busy", + "ble-runtime-cleanup-pending", + "provisioning-already-running", + "network-provision-candidate-not-fresh", + "network-provision-candidate-changed", + "network-provision-discovery-generation-conflict", + "network-provision-candidate-name-unavailable", + "network-provision-lifecycle-busy", + "network-provision-target-not-distinguishable-from-baseline", + "snapshot-runtime-conflict", + "physical-command-target-retired", + "physical-command-recovery-target-unavailable", + "physical-command-recovery-target-not-observed", + "fresh-ble-candidate-required", + "reconciliation-target-not-observed", + "reconciliation-target-mode-mismatch", + "reconciliation-target-physical-recovery-mismatch", + "network-provisioning-idempotency-operation-mismatch", + "network-provisioning-idempotency-ledger-mismatch", + "network-provision-operation-active", + "control-local-retirement-pending", + "device-calibration-read-active", + "acquisition-active", + "acquisition-cleanup-pending", + "acquisition-start-operation-active", + "acquisition-stop-operation-active", + "local-runtime-active", + "control-session-not-admissible-for-network-change", + "k1-lifecycle-process-lease-control-owned", + "k1-lifecycle-process-lease-active", + "network-write-reconciliation-required", + "application-connection-binding-lost", + "application-control-process-lease-unavailable", + "credential-source-unavailable", + "profile-unavailable", + "profile-ssid-mismatch", + "profile-credential-source-mismatch", + "keychain-authorization-required", + "keychain-authorization-denied", + "keychain-authorization-cancelled", + "keychain-access-failed", + }: + return 409 + if reason_code in { + "connection-verify-runtime-loop-unavailable", + "ble-runtime-owner-loop-conflict", + "ble-runtime-restart-required", + "wifi-interface-unavailable", + }: + return 503 + if reason_code in { + "connection-verify-exact-uuid-scan-timeout", + "connection-verify-status-read-timeout", + "ble-discovery-timeout", + "ble-status-read-timeout", + "ble-provisioning-timeout", + "ble-ap-enable-timeout", + "network-not-found", + "host-wifi-operation-timeout", + "control-bootstrap-timeout", + "physical-command-reconciliation-proof-timeout", + "physical-command-reconciliation-control-adoption-timeout", + "control-local-retirement-timeout", + }: + return 504 + return 502 + + def normalize_project_name(value: str) -> str: """Return a bounded display name that is never interpreted as a path.""" @@ -3789,18 +29672,178 @@ def normalize_project_name(value: str) -> str: return normalized +def _connection_verification_error(exc: Exception) -> ConnectionVerificationError: + if isinstance(exc, ConnectionVerificationError): + return exc + reason_code = getattr(exc, "reason_code", None) + if reason_code in {"ble-runtime-busy", "ble-runtime-cleanup-pending"}: + return ConnectionVerificationError( + ( + "Предыдущий Bluetooth-сеанс ещё завершает очистку" + if reason_code == "ble-runtime-cleanup-pending" + else "Другая Bluetooth-операция уже выполняется" + ), + reason_code=( + "connection-verify-cleanup-pending" + if reason_code == "ble-runtime-cleanup-pending" + else "connection-verify-busy" + ), + ) + if reason_code in { + "ble-runtime-owner-loop-conflict", + "ble-runtime-restart-required", + }: + return ConnectionVerificationError( + "Runtime Bluetooth требует перезапуска перед read-only проверкой", + reason_code="connection-verify-runtime-loop-unavailable", + ) + if reason_code in { + "quick_connect_endpoint_unreachable", + "connection_lease_endpoint_unreachable_after_provision", + "connection_lease_recovered_endpoint_unreachable", + }: + return ConnectionVerificationError( + "K1 найден, но его локальный управляющий endpoint недоступен", + reason_code="connection-verify-mqtt-unreachable", + ) + if reason_code == "connection_lease_changed_during_probe": + return ConnectionVerificationError( + "Подключение K1 изменилось во время read-only проверки", + reason_code="connection-verify-lease-changed", + ) + if isinstance(exc, BleakDeviceNotFoundError): + return ConnectionVerificationError( + "K1 не найден повторно во время read-only проверки Bluetooth", + reason_code="connection-verify-device-not-rediscovered", + ) + if isinstance(exc, TimeoutError): + if getattr(exc, "operation_stage", None) == "exact-uuid-scan": + return ConnectionVerificationError( + "Mission Core не завершил ожидание точного CoreBluetooth UUID вовремя", + reason_code="connection-verify-exact-uuid-scan-timeout", + ) + return ConnectionVerificationError( + "Read-only проверка Bluetooth не завершилась вовремя", + reason_code="connection-verify-status-read-timeout", + ) + + message = str(exc) + classified_messages = ( + ( + ("сначала найдите и выберите устройство", "результаты Bluetooth-поиска устарели"), + "connection-verify-candidate-not-fresh", + "Для read-only проверки нет свежего кандидата Bluetooth", + ), + ( + ("Bluetooth-устройство сейчас недоступно",), + "connection-verify-device-not-connectable", + "Выбранный K1 сейчас недоступен для Bluetooth-подключения", + ), + ( + ("runtime loop недоступен", "runtime event loop"), + "connection-verify-runtime-loop-unavailable", + "Runtime Bluetooth недоступен для read-only проверки", + ), + ( + ("не сообщил актуальный DHCP-адрес", "не сообщил адрес в локальной сети"), + "connection-verify-address-unavailable", + "K1 не сообщил адрес общей локальной сети", + ), + ( + ("прямой локальный маршрут отсутствует", "не имеет прямого маршрута"), + "connection-verify-route-mismatch", + "Компьютер и K1 находятся в разных локальных сетях", + ), + ( + ("MQTT endpoint 1883 недоступен",), + "connection-verify-mqtt-unreachable", + "K1 найден, но его локальный управляющий endpoint недоступен", + ), + ( + ("адрес, принадлежащий этому компьютеру", "уже принадлежит этому компьютеру"), + "connection-verify-local-address-conflict", + "K1 сообщил адрес этого компьютера вместо собственного адреса", + ), + ( + ("не подтвердил read-only операцию", "состояние другого устройства"), + "connection-verify-status-read-invalid", + "Read-only ответ Bluetooth не прошёл проверку идентичности", + ), + ( + ("список найденных Bluetooth-устройств изменился",), + "connection-verify-candidate-changed", + "Кандидат Bluetooth изменился во время read-only проверки", + ), + ( + ("другая операция настройки или проверки", "во время настройки Wi-Fi"), + "connection-verify-busy", + "Другая операция подключения уже выполняется", + ), + ( + ( + "активной acquisition-сессии", + "активном live/replay источнике", + "control-сессии", + "evidence-сессия", + ), + "connection-verify-lifecycle-busy", + "Read-only проверка недоступна во время активной сессии", + ), + ( + ("сначала выберите и подключите K1",), + "connection-verify-connection-missing", + "Активное подключение K1 отсутствует", + ), + ) + for fragments, reason_code, safe_message in classified_messages: + if any(fragment in message for fragment in fragments): + return ConnectionVerificationError(safe_message, reason_code=reason_code) + + if isinstance(exc, BleakError): + return ConnectionVerificationError( + "Read-only статус K1 не удалось прочитать по Bluetooth", + reason_code="connection-verify-status-read-failed", + ) + return ConnectionVerificationError( + "Read-only проверка подключения K1 завершилась ошибкой", + reason_code="connection-verify-status-read-failed", + ) + + +def _configured_endpoint_probe_error(exc: Exception) -> ConfiguredEndpointProbeError: + if isinstance(exc, ConfiguredEndpointProbeError): + return exc + reason_code = getattr(exc, "reason_code", None) + if reason_code == "physical-command-target-retired": + return ConfiguredEndpointProbeError( + "Этот K1 был явно выведен из текущего контура; endpoint не проверялся", + reason_code="physical-command-target-retired", + ) + if reason_code == SemanticTopologyStoreCorrupt.reason_code: + return ConfiguredEndpointProbeError( + "Сохранённая топология K1 недоступна", + reason_code="configured-endpoint-topology-corrupt", + ) + return ConfiguredEndpointProbeError( + "Проверка маршрута и TCP endpoint завершилась ошибкой", + reason_code="configured-endpoint-probe-failed", + ) + + def _operation_error( - exc: Exception, + exc: BaseException, *, category: str, side_effect_status: Literal["none", "possible", "confirmed", "unknown"], + retryable: bool = False, safe_to_retry: bool = False, + host_boundary: HostDiagnosticBoundary | None = None, ) -> dict[str, Any]: reason_code = getattr(exc, "reason_code", None) error: dict[str, Any] = { "category": category, "code": reason_code if isinstance(reason_code, str) and reason_code else type(exc).__name__, - "retryable": False, + "retryable": retryable, "safe_to_retry": safe_to_retry, "side_effect_status": side_effect_status, } @@ -3810,8 +29853,13 @@ def _operation_error( "device_write_confirmed": getattr(exc, "device_write_confirmed", None), "ble_att_error_code": getattr(exc, "att_error_code", None), "ble_att_error_name": getattr(exc, "att_error_name", None), + "resolved_write_mode": getattr(exc, "resolved_write_mode", None), + "max_write_without_response_size": getattr(exc, "max_write_without_response_size", None), + "frame_length": getattr(exc, "frame_length", None), "helper_stage": getattr(exc, "helper_stage", None), "helper_elapsed_ms": getattr(exc, "helper_elapsed_ms", None), + "scan_attempt_count": getattr(exc, "scan_attempt_count", None), + "scan_elapsed_ms": getattr(exc, "scan_elapsed_ms", None), } error.update( { @@ -3820,9 +29868,34 @@ def _operation_error( if isinstance(value, (str, int, bool)) } ) + properties = getattr(exc, "write_characteristic_properties", None) + if isinstance(properties, (list, tuple)) and all( + isinstance(item, str) and item for item in properties + ): + error["write_characteristic_properties"] = list(properties) + host_diagnostic = host_diagnostic_for_exception(exc, boundary=host_boundary) + if host_diagnostic is not None: + error["host_diagnostic"] = host_diagnostic.as_dict() return error +def _application_control_session_public_snapshot( + snapshot: Mapping[str, Any], +) -> dict[str, Any]: + """Attach one reviewed host diagnostic without reflecting failure text.""" + + document = dict(snapshot) + failure = snapshot.get("failure") + if not isinstance(failure, Mapping): + return document + failure_document = dict(failure) + host_diagnostic = host_diagnostic_for_reason(failure.get("reason_code")) + if host_diagnostic is not None: + failure_document["host_diagnostic"] = host_diagnostic.as_dict() + document["failure"] = failure_document + return document + + def _validated_requested_streams( request: PrepareAcquisitionRequest, ) -> tuple[RequestedStreamId, ...]: @@ -3852,6 +29925,8 @@ def _sensor_catalog( active_profile_id: str | None, device_session_id: str | None, camera_preview: Mapping[str, Any], + *, + activation_admitted: bool = True, ) -> dict[str, Any]: camera_profile_active = active_profile_id is not None camera_phase = str(camera_preview.get("phase") or "idle") @@ -3901,7 +29976,10 @@ def _sensor_catalog( "max_active": 1, "selected": selected, "controllable": bool( - camera_profile_active and device_session_id and dependency_available + camera_profile_active + and device_session_id + and dependency_available + and activation_admitted ), }, "delivery": delivery if selected and isinstance(delivery, Mapping) else None, @@ -3974,6 +30052,2398 @@ def _new_operation_session_dir(sessions_root: Path, suffix: str) -> Path: return candidate +def _legacy_connection_projection( + supervisor: ConnectionSupervisorSnapshot, + previous: Mapping[str, Any], +) -> dict[str, Any]: + """Compatibility view derived from the canonical supervisor. + + Historical metadata such as whether a device write occurred is preserved, + but legacy fields can no longer claim ``reachable`` when DeviceInfo/control + authority has been revoked. + """ + + projected = dict(previous) + endpoint_state = supervisor.endpoint.tcp_state + if supervisor.authority.control_allowed: + projected.update( + { + "status": "reachable", + "lease_state": "reachable", + "network_reachability": "reachable", + "reason_code": None, + } + ) + elif supervisor.lease.state == "configured-unverified": + projected.update( + { + "status": ( + "tcp-reachable-device-info-unverified" + if endpoint_state == "reachable" + else "endpoint-unreachable" + if endpoint_state == "unreachable" + else "device-network-applied" + ), + "lease_state": "configured-unverified", + "network_reachability": ( + "reachable" + if endpoint_state == "reachable" + else "unreachable" + if endpoint_state == "unreachable" + else "unknown" + ), + "reason_code": supervisor.lease.reason_code, + } + ) + else: + has_target_context = ( + supervisor.device_network.state == "applied" or supervisor.last_known is not None + ) + explicit_read_only_network_failure = bool( + previous.get("admission_source") == ACTION_CONNECTION_VERIFY + and previous.get("status") == "unreachable" + and previous.get("network_reachability") == "unreachable" + and previous.get("write_performed") is False + and previous.get("reason_code") + in { + "connection-verify-address-unavailable", + "connection-verify-connection-missing", + "connection-verify-route-mismatch", + "connection-verify-mqtt-unreachable", + } + ) + projected.update( + { + "status": ( + "unreachable" + if explicit_read_only_network_failure or supervisor.last_known is not None + else "not-probed" + ), + "lease_state": "disconnected", + "network_reachability": ( + "unreachable" + if explicit_read_only_network_failure + or ( + has_target_context + and (endpoint_state == "unreachable" or not supervisor.host_path.available) + ) + else "unknown" + ), + "reason_code": ( + previous.get("reason_code") + if explicit_read_only_network_failure + else supervisor.lease.reason_code + ), + } + ) + projected["lease_generation"] = supervisor.lease.generation + projected["supervisor_revision"] = supervisor.revision + return projected + + +def _physical_command_reports_active( + physical_command: Mapping[str, object], +) -> bool: + """Project the latest durable physical state without reviving old STARTs. + + Reconciliation audit is inherited by successor commands. Looking only at + the last reconciliation therefore misclassifies a later successful STOP + as active, while looking only at ``start-active-observed`` ignores a newer + read-only standby cessation. Resolve the effective state from the current + record and only the recovery entries attached to that exact START. + """ + + # The coordinator derives this projection from the current ledger row's + # exact latest reconciliation. In particular a PREPARED STOP classified + # read-only as still SCANNING remains action=stop/resolution=not-dispatched, + # so raw resolution whitelists are no longer authoritative. + physical_active = physical_command.get("physical_active") + if isinstance(physical_active, bool): + return physical_active + reconciled_state = physical_command.get("reconciled_physical_state") + if reconciled_state in {"active", "standby"}: + return reconciled_state == "active" + if ( + physical_command.get("status") == "resolved" + and physical_command.get("resolved_active_recovery_required") is True + ): + return True + + record = physical_command.get("record") + if not isinstance(record, Mapping): + return False + resolution = record.get("resolution") + if resolution not in {"start-active-observed", "physical-active-observed"}: + return False + operation_id = record.get("operation_id") + reconciliations = record.get("reconciliations") + latest_recovery: Mapping[str, object] | None = None + if isinstance(reconciliations, list): + for candidate in reconciliations: + if not isinstance(candidate, Mapping): + continue + if candidate.get("kind") not in { + "resolved-active-rebind", + "resolved-active-cessation", + }: + continue + original_attempt = candidate.get("original_attempt") + if ( + isinstance(original_attempt, Mapping) + and original_attempt.get("operation_id") == operation_id + ): + latest_recovery = candidate + return bool( + latest_recovery is None or latest_recovery.get("resolution") == "physical-active-observed" + ) + + +def _physical_command_pending_reopened_reconciliation_mode( + physical_command: Mapping[str, object], +) -> ConnectionMode | None: + """Return the mode pinned by the current pending reopen, if any. + + Reopen/retirement arrays are durable bounded history and are carried into + successor command records. Only a reopen tied to the *current* operation + may freeze the process-local mode draft while its read-only observation is + pending; an older carried audit must never block an unrelated cold draft. + """ + + unresolved_reopen = bool( + physical_command.get("status") == "unresolved" + and physical_command.get("requires_reconciliation") is True + ) + resolved_pending_reopen = bool( + physical_command.get("status") == "resolved" + and physical_command.get("requires_reconciliation") is not True + and physical_command.get("resolved_active_recovery_required") is True + and physical_command.get("reopened_physical_state_recovery_required") is True + and physical_command.get("physical_active") is False + ) + if not (unresolved_reopen or resolved_pending_reopen): + return None + record = physical_command.get("record") + if not isinstance(record, Mapping) or not ( + (unresolved_reopen and record.get("resolution") is None) + or ( + resolved_pending_reopen + and record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "not-dispatched" + ) + ): + return None + operation_id = record.get("operation_id") + revision = record.get("revision") + connection = record.get("connection") + connection_mode = connection.get("connection_mode") if isinstance(connection, Mapping) else None + retirements = record.get("operator_retirements") + reopens = record.get("operator_reconciliation_reopens") + if not ( + isinstance(operation_id, str) + and isinstance(revision, int) + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(retirements, list) + and isinstance(reopens, list) + ): + return None + current_retirement_ids = { + str(retirement.get("retirement_id")) + for retirement in retirements + if isinstance(retirement, Mapping) + and isinstance(retirement.get("retirement_id"), str) + and isinstance(retirement.get("original_attempt"), Mapping) + and retirement["original_attempt"].get("operation_id") == operation_id + } + pending = any( + isinstance(reopen, Mapping) + and reopen.get("retirement_id") in current_retirement_ids + and isinstance(reopen.get("retired_record_revision"), int) + and revision == int(reopen["retired_record_revision"]) + 1 + for reopen in reopens + ) + return cast(ConnectionMode, connection_mode) if pending else None + + +def _physical_command_prepared_stop_ready_successor_pending( + physical_command: Mapping[str, object], +) -> bool: + """Whether READY classified S0 still pins the binding for its fresh START. + + The ledger deliberately requires the successor START to consume the exact + fresh Verify binding recorded by the prepared-STOP classification. Until + that successor is prepared, changing mode/network would strand the only + legal chain even though the device is physically in standby. + """ + + if not ( + physical_command.get("status") == "resolved" + and physical_command.get("requires_reconciliation") is not True + and physical_command.get("reconciled_physical_state") == "standby" + ): + return False + record = physical_command.get("record") + if not isinstance(record, Mapping) or not ( + record.get("action") == "stop" + and record.get("stage") == "resolved" + and record.get("resolution") == "not-dispatched" + ): + return False + operation_id = record.get("operation_id") + reconciliations = record.get("reconciliations") + if not isinstance(operation_id, str) or not isinstance(reconciliations, list): + return False + current = [ + item + for item in reconciliations + if isinstance(item, Mapping) + and item.get("kind") == "prepared-stop-classification" + and isinstance(item.get("original_attempt"), Mapping) + and item["original_attempt"].get("operation_id") == operation_id + ] + return bool( + len(current) == 1 + and current[0].get("resolution") == "physical-standby-observed" + ) + + +def _connection_mode_selection_reason_codes( + *, + acquisition_state: str | None, + acquisition_lease_retained: bool, + acquisition_start_operation_id: str | None, + runtime: Mapping[str, object], + physical_command: Mapping[str, object], + control_state: str, + desired_connection_mode: ConnectionMode, + requested_connection_mode: ConnectionMode | None = None, +) -> list[str]: + """Return the exact safety reasons used by both state and Mode Select. + + Mode Select commits only a local desired-mode draft; it performs no device + or host-network I/O. A disconnected operator may therefore choose the + next path even when durable history still needs explicit reconciliation. + That history continues to fence Scan/Verify/Provision/START independently. + Once an explicit retired-target reopen is durably admitted, its desired-mode + CAS stays frozen until the immediately owned read-only reconciliation + terminalizes; otherwise another tab could strand the consumed checkpoint. + + Once a live control owner exists, however, changing the draft is a real + topology transition. It remains blocked by an ambiguous/active physical + edge, by acquisition/runtime ownership, or by an unsafe control phase. + """ + + physical_start_active = _physical_command_reports_active(physical_command) + acquisition_safe = bool( + acquisition_state is None + or acquisition_state in TERMINAL_ACQUISITION_STATES + or ( + acquisition_state == "prepared" + and not acquisition_lease_retained + and acquisition_start_operation_id is None + ) + ) + disconnected_control = control_state in { + "idle", + "completed", + "closed", + "failed", + } + durable_physical_transition_unsafe = bool( + physical_command.get("requires_reconciliation") is True + or physical_command.get("resolved_active_recovery_required") is True + or _physical_command_prepared_stop_ready_successor_pending(physical_command) + or physical_command.get("status") == "corrupt" + or physical_start_active + ) + pending_reopen_mode = _physical_command_pending_reopened_reconciliation_mode(physical_command) + directed_reopen_mode_recovery = bool( + pending_reopen_mode is not None + and desired_connection_mode != pending_reopen_mode + and (requested_connection_mode is None or requested_connection_mode == pending_reopen_mode) + ) + reopened_reconciliation_blocks = bool( + pending_reopen_mode is not None and not directed_reopen_mode_recovery + ) + reasons: list[str] = [] + if ( + not acquisition_safe + or acquisition_lease_retained + or runtime.get("source_mode") != "idle" + or reopened_reconciliation_blocks + or (durable_physical_transition_unsafe and not disconnected_control) + ): + reasons.append("connection-mode-selection-physical-state-unsafe") + if control_state not in { + "idle", + "completed", + "closed", + "failed", + "connection-ready", + "workspace-ready", + "project-ready", + }: + reasons.append("connection-mode-selection-control-state-unsafe") + return reasons + + +def _control_session_is_network_admissible( + application_control_session: Mapping[str, object], +) -> bool: + control_state = str(application_control_session.get("state") or "unknown") + control_failure = application_control_session.get("failure") + return bool( + control_state in {"idle", "completed", "closed"} + or ( + control_state == "failed" + and isinstance(control_failure, Mapping) + and ( + control_failure.get("safe_to_retry") is True + or control_failure.get("network_change_admissible") is True + ) + ) + ) + + +def _shared_fresh_observation_reasons( + *, + supervisor_closed: bool, + provisioning_active: bool, + acquisition_active: bool, + acquisition_cleanup_pending: bool, + runtime_active: bool, + ble_runtime: Mapping[str, object], + lifecycle_process_lease_holders: Sequence[str], + application_control_session: Mapping[str, object], +) -> list[str]: + """Return blockers shared by Scan-selected read-only observation paths.""" + + reasons: list[str] = [] + if supervisor_closed: + reasons.append("connection-supervisor-closed") + if provisioning_active: + reasons.append("network-provision-operation-active") + if acquisition_active: + reasons.append("acquisition-active") + if acquisition_cleanup_pending: + reasons.append("acquisition-cleanup-pending") + if runtime_active: + reasons.append("local-runtime-active") + if ble_runtime.get("poisoned") is True: + reasons.append("ble-runtime-restart-required") + elif ble_runtime.get("cleanup_pending") is True: + reasons.append("ble-runtime-cleanup-pending") + elif ble_runtime.get("active_operation_kind") is not None: + reasons.append("ble-runtime-busy") + control_state = str(application_control_session.get("state") or "unknown") + if "control" in lifecycle_process_lease_holders and control_state != "failed": + reasons.append("k1-lifecycle-process-lease-control-owned") + return reasons + + +def _durable_connection_health_reasons( + *, + network_provisioning_idempotency: Mapping[str, object], + network_provisioning_idempotency_available: bool, + semantic_topology_store: Mapping[str, object], + device_identity_pin_store: Mapping[str, object], +) -> list[str]: + """Return durable health blockers shared by every network observation.""" + + status = str(network_provisioning_idempotency.get("status") or "unknown") + reasons: list[str] = [] + if not network_provisioning_idempotency_available: + reasons.append("network-provisioning-idempotency-unavailable") + elif status == "corrupt": + reasons.append("network-provisioning-idempotency-corrupt") + elif status not in {"empty", "ready", "blocked"}: + reasons.append("network-provisioning-idempotency-invalid") + if semantic_topology_store.get("status") == "corrupt": + reasons.append("semantic-topology-store-corrupt") + if device_identity_pin_store.get("status") == "corrupt": + reasons.append("device-identity-pin-store-corrupt") + return reasons + + +def _resolved_apply_durable_observation_target( + ledger: NetworkMutationLedgerSnapshot, + semantic_topology_store: Mapping[str, object], +) -> tuple[str, ConnectionMode, str] | None: + """Return one exact durable target proved by a completed network Apply.""" + + ledger_record = ledger.record + semantic_record = semantic_topology_store.get("record") + if not ( + ledger.status == "resolved" + and ledger_record is not None + and ledger_record.stage == "resolved" + and ledger_record.write_confirmed is True + and ledger_record.resolution == "target-observed" + and ledger_record.last_observation is not None + and semantic_topology_store.get("status") == "available" + and isinstance(semantic_record, Mapping) + and semantic_record.get("schema_version") == "missioncore.xgrids-k1-semantic-topology/v1" + and semantic_record.get("compatibility_profile_id") == XGRIDS_K1_COMPATIBILITY_PROFILE_ID + and semantic_record.get("source") in {"ble-post-write-status", "ble-read-only-status"} + ): + return None + + transport_ref = semantic_record.get("transport_ref") + connection_mode = semantic_record.get("connection_mode") + ipv4 = semantic_record.get("ipv4") + if not ( + isinstance(transport_ref, str) + and transport_ref + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(ipv4, str) + and ipv4 + and physical_transport_ref_comparison_key(transport_ref) + == physical_transport_ref_comparison_key(ledger_record.transport_ref) + and connection_mode == ledger_record.intended_mode + and ledger_record.last_observation.ipv4 == ipv4 + ): + return None + return transport_ref, cast(ConnectionMode, connection_mode), ipv4 + + +def _fresh_observation_reasons( + *, + shared_reasons: Sequence[str], + durable_health_reasons: Sequence[str], + application_control_session: Mapping[str, object], + network_ledger_status: str, + network_ledger_operation_id: str | None, + network_ledger_stage: str | None, + network_ledger_resolution: str | None, + network_ledger_transport_ref: str | None, + network_ledger_connection_mode: ConnectionMode | None, + network_provisioning_active_operation_id: str | None, + network_provisioning_active_action: str | None, + network_provisioning_active_stage: str | None, + eligible_fresh_transport_refs: Sequence[str], + physical_recovery_required: bool, + physical_recovery_transport_ref: str | None, + physical_recovery_mode: ConnectionMode | None, +) -> list[str]: + """Compose the exact post-Scan read-only action blockers. + + Reopen admission calls this same reducer against the hypothetical state in + which its one active retirement deny has been removed. Therefore it cannot + consume a durable checkpoint that the immediately owned Verify could not + use for some independent, unchanged reason. + """ + + reasons = [*shared_reasons, *durable_health_reasons] + if not _control_session_is_network_admissible(application_control_session): + reasons.append("control-session-not-admissible-for-network-change") + if network_ledger_status == "corrupt": + reasons.append("network-mutation-ledger-corrupt") + active_idempotency_reconciliation = bool( + network_provisioning_active_operation_id is not None + and network_provisioning_active_action == ACTION_NETWORK_PROVISION + and network_provisioning_active_stage == "unresolved" + ) + # Verify first closes a PREPARED ledger as not-dispatched. It then + # terminalizes one matching unresolved idempotency record for either that + # result or a previously observed target. Project those two deterministic + # local repairs before evaluating the state that would remain at BLE + # admission; every other cross-journal shape is a hard blocker. + prepared_ledger_repairs_to_not_dispatched = bool( + network_ledger_status == "unresolved" + and network_ledger_stage == "prepared" + and network_ledger_operation_id is not None + ) + effective_ledger_status = ( + "resolved" if prepared_ledger_repairs_to_not_dispatched else network_ledger_status + ) + effective_ledger_resolution = ( + "not-dispatched" if prepared_ledger_repairs_to_not_dispatched else network_ledger_resolution + ) + idempotency_repairs_to_terminal = bool( + effective_ledger_status == "resolved" + and network_ledger_operation_id is not None + and effective_ledger_resolution in {"not-dispatched", "target-observed"} + and active_idempotency_reconciliation + and network_provisioning_active_operation_id == network_ledger_operation_id + ) + unresolved_journals_match = bool( + effective_ledger_status == "unresolved" + and network_ledger_stage in {"dispatching", "observing"} + and network_ledger_operation_id is not None + and active_idempotency_reconciliation + and network_provisioning_active_operation_id == network_ledger_operation_id + ) + if effective_ledger_status == "unresolved" and not unresolved_journals_match: + reasons.append("network-provisioning-idempotency-operation-mismatch") + elif ( + network_provisioning_active_operation_id is not None + and not idempotency_repairs_to_terminal + and not unresolved_journals_match + ): + reasons.append("network-provisioning-idempotency-ledger-mismatch") + eligible_keys = { + physical_transport_ref_comparison_key(value) + for value in eligible_fresh_transport_refs + if value + } + + def eligible(value: str | None) -> bool: + return bool(value and physical_transport_ref_comparison_key(value) in eligible_keys) + + if not eligible_keys: + reasons.append("fresh-ble-candidate-required") + if effective_ledger_status == "unresolved" and not eligible(network_ledger_transport_ref): + reasons.append("reconciliation-target-not-observed") + if physical_recovery_required: + if physical_recovery_transport_ref is None or physical_recovery_mode is None: + reasons.append("physical-command-recovery-target-unavailable") + elif not eligible(physical_recovery_transport_ref): + reasons.append("physical-command-recovery-target-not-observed") + if ( + effective_ledger_status == "unresolved" + and network_ledger_transport_ref is not None + and physical_recovery_transport_ref is not None + and physical_transport_ref_comparison_key(network_ledger_transport_ref) + != physical_transport_ref_comparison_key(physical_recovery_transport_ref) + ): + reasons.append("reconciliation-target-physical-recovery-mismatch") + elif ( + effective_ledger_status == "unresolved" + and network_ledger_transport_ref is not None + and physical_recovery_transport_ref is not None + and physical_transport_ref_comparison_key(network_ledger_transport_ref) + == physical_transport_ref_comparison_key(physical_recovery_transport_ref) + and network_ledger_connection_mode is not None + and physical_recovery_mode is not None + and network_ledger_connection_mode != physical_recovery_mode + ): + reasons.append("reconciliation-target-mode-mismatch") + return list(dict.fromkeys(reasons)) + + +def _connection_policy_projection( + *, + supervisor: ConnectionSupervisorSnapshot, + ledger: NetworkMutationLedgerSnapshot, + network_provisioning_idempotency: Mapping[str, object], + network_provisioning_idempotency_available: bool, + semantic_topology_store: Mapping[str, object], + device_identity_pin_store: Mapping[str, object], + physical_command: Mapping[str, object], + ble_runtime: Mapping[str, object], + lifecycle_process_lease_holders: Sequence[str], + fresh_devices: Sequence[Mapping[str, Any]], + current_device_recovery: Mapping[str, object] | None, + provisioning_active: bool, + acquisition_active: bool, + acquisition_state: str | None, + acquisition_cleanup_pending: bool, + runtime_active: bool, + application_control_session: Mapping[str, Any], + desired_connection_mode: ConnectionMode, + active_connection_mode: ConnectionMode | None, + retired_reprovision_transport_ref: str | None = None, +) -> dict[str, Any]: + """Compose every connection fact into one fail-closed UI action policy. + + ``ConnectionSupervisor.allowed_actions`` intentionally describes only the + live host/control/data evidence planes. It cannot authorize BLE writes on + its own because the durable mutation ledger, retained CoreBluetooth scope + and lifecycle gates live outside that reducer. This projection is the API + boundary that combines those independent facts. Browser code may add + local form prerequisites (power acknowledgement and credentials), but it + must never manufacture device authority that is absent here. + """ + + fresh_refs = tuple( + sorted( + { + str(device.get("device_id") or "").strip() + for device in fresh_devices + if str(device.get("device_id") or "").strip() + and device.get("connectable") is not False + } + ) + ) + ledger_record = ledger.record + ledger_transport_ref = ledger_record.transport_ref if ledger_record is not None else None + ledger_connection_mode = ledger_record.intended_mode if ledger_record is not None else None + ledger_operation_id = ledger_record.operation_id if ledger_record is not None else None + ledger_stage = ledger_record.stage if ledger_record is not None else None + ledger_resolution = ledger_record.resolution if ledger_record is not None else None + ledger_unresolved = ledger.status == "unresolved" + ledger_corrupt = ledger.status == "corrupt" + topology_corrupt = semantic_topology_store.get("status") == "corrupt" + semantic_topology_record = semantic_topology_store.get("record") + identity_pin_corrupt = device_identity_pin_store.get("status") == "corrupt" + idempotency_status = str(network_provisioning_idempotency.get("status") or "unknown") + idempotency_active_operation_raw = network_provisioning_idempotency.get("active_operation_id") + idempotency_active_operation_id = ( + idempotency_active_operation_raw.strip() + if isinstance(idempotency_active_operation_raw, str) + and idempotency_active_operation_raw.strip() + else None + ) + idempotency_active_action_raw = network_provisioning_idempotency.get("active_action") + idempotency_active_action = ( + idempotency_active_action_raw.strip() + if isinstance(idempotency_active_action_raw, str) and idempotency_active_action_raw.strip() + else None + ) + idempotency_active_stage_raw = network_provisioning_idempotency.get("active_stage") + idempotency_active_stage = ( + idempotency_active_stage_raw.strip() + if isinstance(idempotency_active_stage_raw, str) and idempotency_active_stage_raw.strip() + else None + ) + expected_active_operation_id = ( + ledger_operation_id + if ledger_unresolved + or ( + ledger.status == "resolved" + and ledger_resolution in {"not-dispatched", "target-observed"} + ) + else None + ) + idempotency_active_mismatch = ( + idempotency_active_operation_id != expected_active_operation_id + if idempotency_active_operation_id is not None or expected_active_operation_id is not None + else False + ) + physical_status = str(physical_command.get("status") or "unknown") + physical_start_active = _physical_command_reports_active(physical_command) + physical_record = physical_command.get("record") + physical_retirements = ( + physical_record.get("operator_retirements") + if isinstance(physical_record, Mapping) + else None + ) + physical_reopens = ( + physical_record.get("operator_reconciliation_reopens") + if isinstance(physical_record, Mapping) + else None + ) + reopened_retirement_ids = { + str(item.get("retirement_id")) + for item in (physical_reopens if isinstance(physical_reopens, list) else []) + if isinstance(item, Mapping) and isinstance(item.get("retirement_id"), str) + } + retired_transport_refs = { + str(item.get("retired_transport_ref")) + for item in (physical_retirements if isinstance(physical_retirements, list) else []) + if isinstance(item, Mapping) + and item.get("retirement_id") not in reopened_retirement_ids + and isinstance(item.get("retired_transport_ref"), str) + and item.get("retired_transport_ref") + } + retired_transport_keys = { + physical_transport_ref_comparison_key(transport_ref) + for transport_ref in retired_transport_refs + } + retired_reprovision_transport_key = ( + physical_transport_ref_comparison_key(retired_reprovision_transport_ref) + if isinstance(retired_reprovision_transport_ref, str) + and retired_reprovision_transport_ref.strip() + else None + ) + eligible_fresh_refs = tuple( + transport_ref + for transport_ref in fresh_refs + if physical_transport_ref_comparison_key(transport_ref) + not in retired_transport_keys + or physical_transport_ref_comparison_key(transport_ref) + == retired_reprovision_transport_key + ) + eligible_fresh_keys = { + physical_transport_ref_comparison_key(transport_ref) + for transport_ref in eligible_fresh_refs + } + + def transport_is_retired(value: object) -> bool: + return bool( + isinstance(value, str) + and value.strip() + and physical_transport_ref_comparison_key(value) in retired_transport_keys + ) + + def transport_is_eligible_fresh(value: object) -> bool: + return bool( + isinstance(value, str) + and value.strip() + and physical_transport_ref_comparison_key(value) in eligible_fresh_keys + ) + + def same_transport(left: object, right: object) -> bool: + return bool( + isinstance(left, str) + and left.strip() + and isinstance(right, str) + and right.strip() + and physical_transport_ref_comparison_key(left) + == physical_transport_ref_comparison_key(right) + ) + + physical_connection = ( + physical_record.get("connection") if isinstance(physical_record, Mapping) else None + ) + physical_recovery_ref_raw = ( + physical_connection.get("transport_ref") + if isinstance(physical_connection, Mapping) + else None + ) + physical_recovery_ref = ( + physical_recovery_ref_raw.strip() + if isinstance(physical_recovery_ref_raw, str) and physical_recovery_ref_raw.strip() + else None + ) + physical_recovery_mode_raw = ( + physical_connection.get("connection_mode") + if isinstance(physical_connection, Mapping) + else None + ) + physical_recovery_mode = ( + physical_recovery_mode_raw + if physical_recovery_mode_raw + in { + "bridge", + "quick-connect", + "direct-connect", + } + else None + ) + resolved_active_recovery_required = bool( + physical_command.get("resolved_active_recovery_required") is True + ) + prepared_stop_ready_successor_pending = ( + _physical_command_prepared_stop_ready_successor_pending(physical_command) + ) + physical_requires_reconciliation = bool( + physical_command.get("requires_reconciliation") is True + or resolved_active_recovery_required + or prepared_stop_ready_successor_pending + or physical_status in {"unresolved", "corrupt"} + ) + physical_read_only_recovery_required = bool( + physical_command.get("requires_reconciliation") is True or resolved_active_recovery_required + ) + ble_active_operation_raw = ble_runtime.get("active_operation_kind") + ble_active_operation = ( + ble_active_operation_raw + if isinstance(ble_active_operation_raw, str) and ble_active_operation_raw + else None + ) + ble_cleanup_pending = ble_runtime.get("cleanup_pending") is True + ble_poisoned = ble_runtime.get("poisoned") is True + lifecycle_holders = tuple( + sorted( + { + holder + for holder in lifecycle_process_lease_holders + if holder in {"control", "network"} + } + ) + ) + network_process_lease_conflict = "control" in lifecycle_holders + control_process_lease_conflict = "network" in lifecycle_holders + recovery_ref_raw = ( + current_device_recovery.get("transport_ref") + if current_device_recovery is not None + else None + ) + recovery_ref = ( + recovery_ref_raw.strip() + if isinstance(recovery_ref_raw, str) and recovery_ref_raw.strip() + else None + ) + recovery_retained = bool( + current_device_recovery is not None + and current_device_recovery.get("handle_retained") is True + ) + recovery_mode_raw = ( + current_device_recovery.get("connection_mode") + if current_device_recovery is not None + else None + ) + recovery_mode = ( + recovery_mode_raw + if recovery_mode_raw in {"bridge", "quick-connect", "direct-connect"} + else None + ) + recovery_advertised = bool( + current_device_recovery is not None + and current_device_recovery.get("advertised_now") is True + ) + control_state = str(application_control_session.get("state") or "unknown") + control_admissible = _control_session_is_network_admissible(application_control_session) + + shared_ble_reasons = _shared_fresh_observation_reasons( + supervisor_closed=supervisor.closed, + provisioning_active=provisioning_active, + acquisition_active=acquisition_active, + acquisition_cleanup_pending=acquisition_cleanup_pending, + runtime_active=runtime_active, + ble_runtime=ble_runtime, + lifecycle_process_lease_holders=lifecycle_holders, + application_control_session=application_control_session, + ) + + idempotency_health_reasons = _durable_connection_health_reasons( + network_provisioning_idempotency=network_provisioning_idempotency, + network_provisioning_idempotency_available=(network_provisioning_idempotency_available), + semantic_topology_store=semantic_topology_store, + device_identity_pin_store=device_identity_pin_store, + ) + # A mismatch between two non-corrupt audit projections is recovered under + # the exclusive lifecycle lease when the next explicit action starts. It + # is diagnostic evidence, not a browser-side admission barrier. + + durable_network_reasons = list(idempotency_health_reasons) + + mutation_reasons = [*shared_ble_reasons, *durable_network_reasons] + if not control_admissible: + mutation_reasons.append("control-session-not-admissible-for-network-change") + if ledger_corrupt: + mutation_reasons.append("network-mutation-ledger-corrupt") + # Discovery and exact read-only observation are the recovery path after a + # battery/control loss. Network mutation is not: an unresolved or last- + # observed-active physical edge must first be classified on a fresh + # DeviceInfo-bound control generation. + if physical_status == "corrupt": + mutation_reasons.append("physical-command-ledger-corrupt") + elif physical_requires_reconciliation: + mutation_reasons.append("physical-command-reconciliation-required") + elif physical_status not in {"empty", "resolved"}: + mutation_reasons.append("physical-command-ledger-unavailable") + # A durable active classification remains true while its process-local + # control adoption is being retried. Keep both facts visible: the exact + # read-only Verify path is still required, and network mutation must never + # mistake the failed local adoption for physical standby. + if physical_start_active: + mutation_reasons.append("physical-device-already-active") + + fresh_mutation_reasons = list(mutation_reasons) + if not eligible_fresh_refs: + fresh_mutation_reasons.append("fresh-ble-candidate-required") + + recovery_reasons = list(mutation_reasons) + if transport_is_retired(recovery_ref): + recovery_reasons.append("physical-command-target-retired") + if not recovery_ref or not recovery_retained: + recovery_reasons.append("retained-recovery-context-unavailable") + if recovery_advertised: + recovery_reasons.append("fresh-candidate-supersedes-retained-recovery") + observation_reasons = _fresh_observation_reasons( + shared_reasons=shared_ble_reasons, + durable_health_reasons=durable_network_reasons, + application_control_session=application_control_session, + network_ledger_status=ledger.status, + network_ledger_operation_id=ledger_operation_id, + network_ledger_stage=ledger_stage, + network_ledger_resolution=ledger_resolution, + network_ledger_transport_ref=ledger_transport_ref, + network_ledger_connection_mode=ledger_connection_mode, + network_provisioning_active_operation_id=(idempotency_active_operation_id), + network_provisioning_active_action=idempotency_active_action, + network_provisioning_active_stage=idempotency_active_stage, + eligible_fresh_transport_refs=eligible_fresh_refs, + physical_recovery_required=physical_read_only_recovery_required, + physical_recovery_transport_ref=physical_recovery_ref, + physical_recovery_mode=physical_recovery_mode, + ) + + retained_observation_reasons = [*shared_ble_reasons, *durable_network_reasons] + if transport_is_retired(recovery_ref): + retained_observation_reasons.append("physical-command-target-retired") + if not control_admissible: + retained_observation_reasons.append("control-session-not-admissible-for-network-change") + if ledger_corrupt: + retained_observation_reasons.append("network-mutation-ledger-corrupt") + if not recovery_ref or not recovery_retained or recovery_mode is None: + retained_observation_reasons.append("retained-recovery-context-unavailable") + if ledger_unresolved and ( + not same_transport(recovery_ref, ledger_transport_ref) or recovery_mode is None + ): + retained_observation_reasons.append("reconciliation-target-not-retained") + if physical_read_only_recovery_required and ( + not same_transport(recovery_ref, physical_recovery_ref) + or recovery_mode != physical_recovery_mode + ): + retained_observation_reasons.append("physical-command-recovery-target-not-retained") + if transport_is_eligible_fresh(recovery_ref): + retained_observation_reasons.append("fresh-candidate-supersedes-retained-recovery") + + semantic_recovery_ref = ( + semantic_topology_record.get("transport_ref") + if isinstance(semantic_topology_record, Mapping) + and isinstance(semantic_topology_record.get("transport_ref"), str) + else None + ) + semantic_recovery_mode = ( + semantic_topology_record.get("connection_mode") + if isinstance(semantic_topology_record, Mapping) + and semantic_topology_record.get("connection_mode") + in {"bridge", "quick-connect", "direct-connect"} + else None + ) + if transport_is_retired(semantic_recovery_ref): + semantic_recovery_ref = None + semantic_recovery_mode = None + resolved_apply_durable_target = _resolved_apply_durable_observation_target( + ledger, + semantic_topology_store, + ) + resolved_apply_durable_target_required = bool( + ledger.status == "resolved" + and ledger.record is not None + and ledger.record.stage == "resolved" + and ledger.record.resolution == "target-observed" + ) + durable_observation_ref = ( + ledger_transport_ref + if ledger_unresolved + else resolved_apply_durable_target[0] + if resolved_apply_durable_target is not None + else None + if resolved_apply_durable_target_required + else semantic_recovery_ref + ) + durable_observation_mode = ( + ledger_connection_mode + if ledger_unresolved + else resolved_apply_durable_target[1] + if resolved_apply_durable_target is not None + else None + if resolved_apply_durable_target_required + else semantic_recovery_mode + ) + # A resolved semantic topology is verified over the LAN after restart. + # BLE runtime health is irrelevant on that path because K1 may stop + # advertising once it has joined the configured network. An unresolved + # Wi-Fi write still takes the GATT reconciliation branch below. + durable_observation_reasons = [*durable_network_reasons] + if supervisor.closed: + durable_observation_reasons.append("connection-supervisor-closed") + if provisioning_active: + durable_observation_reasons.append("network-provision-operation-active") + if acquisition_active: + durable_observation_reasons.append("acquisition-active") + if acquisition_cleanup_pending: + durable_observation_reasons.append("acquisition-cleanup-pending") + if runtime_active: + durable_observation_reasons.append("local-runtime-active") + if network_process_lease_conflict: + durable_observation_reasons.append("k1-lifecycle-process-lease-control-owned") + if not control_admissible: + durable_observation_reasons.append("control-session-not-admissible-for-network-change") + if ledger_corrupt: + durable_observation_reasons.append("network-mutation-ledger-corrupt") + if resolved_apply_durable_target_required and resolved_apply_durable_target is None: + durable_observation_reasons.append("resolved-apply-durable-target-mismatch") + if not durable_observation_ref or durable_observation_mode is None: + durable_observation_reasons.append("durable-recovery-target-unavailable") + if ( + transport_is_eligible_fresh(durable_observation_ref) + and resolved_apply_durable_target is None + ): + durable_observation_reasons.append("fresh-candidate-supersedes-durable-recovery") + if recovery_retained and same_transport(recovery_ref, durable_observation_ref): + durable_observation_reasons.append("retained-recovery-supersedes-durable-recovery") + if physical_read_only_recovery_required: + if physical_recovery_ref is None: + durable_observation_reasons.append("physical-command-recovery-target-unavailable") + elif ( + not same_transport(durable_observation_ref, physical_recovery_ref) + or durable_observation_mode != physical_recovery_mode + ): + durable_observation_reasons.append("physical-command-recovery-target-mismatch") + + prestart_mode_switch_scan = bool( + active_connection_mode is not None + and active_connection_mode != desired_connection_mode + and control_state in {"connection-ready", "workspace-ready", "project-ready"} + ) + locally_prepared_mode_switch_scan = bool( + prestart_mode_switch_scan + and acquisition_active + and acquisition_state == "prepared" + and not acquisition_cleanup_pending + and not runtime_active + ) + scan_reasons = [ + reason + for reason in shared_ble_reasons + if not (prestart_mode_switch_scan and reason == "k1-lifecycle-process-lease-control-owned") + and not (locally_prepared_mode_switch_scan and reason == "acquisition-active") + ] + # BLE discovery performs no device write. Keep it available after local + # acquisition/control cleanup so the operator can identify the exact K1 + # for read-only recovery. The physical barrier remains on provisioning, + # START, STOP and mode mutation until Verify commits exact fresh evidence. + current_configured_endpoint_available = bool( + not supervisor.closed + and supervisor.intent is not None + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.connection_mode == supervisor.intent.requested_mode + and supervisor.device_network.target is not None + and supervisor.device_network.transport_ref + and not transport_is_retired(supervisor.device_network.transport_ref) + ) + durable_configured_endpoint_available = bool( + semantic_topology_store.get("status") == "available" + and semantic_topology_store.get("configured_offline_evidence") is True + and semantic_topology_store.get("live_connection_authority") is False + and isinstance(semantic_topology_record, Mapping) + and semantic_topology_record.get("schema_version") + == "missioncore.xgrids-k1-semantic-topology/v1" + and isinstance(semantic_topology_record.get("transport_ref"), str) + and semantic_topology_record.get("transport_ref") + and not transport_is_retired(semantic_topology_record.get("transport_ref")) + and semantic_topology_record.get("connection_mode") + in {"bridge", "quick-connect", "direct-connect"} + and isinstance(semantic_topology_record.get("ipv4"), str) + and semantic_topology_record.get("ipv4") + ) + configured_endpoint_available = ( + current_configured_endpoint_available or durable_configured_endpoint_available + ) + endpoint_reasons: list[str] = [] + if supervisor.closed: + endpoint_reasons.append("connection-supervisor-closed") + if provisioning_active: + endpoint_reasons.append("network-provision-operation-active") + if topology_corrupt and not current_configured_endpoint_available: + endpoint_reasons.append("semantic-topology-store-corrupt") + if not configured_endpoint_available: + endpoint_reasons.append("configured-endpoint-unavailable") + + supervisor_actions = set(supervisor.allowed_actions) + + def decision( + reasons: list[str], + *, + target_source: str, + required_transport_ref: str | None = None, + required_connection_mode: str | None = None, + requires_live_gatt_validation: bool = False, + ) -> dict[str, object]: + unique_reasons = list(dict.fromkeys(reasons)) + result: dict[str, object] = { + "allowed": not unique_reasons, + "reason_codes": unique_reasons, + "target_source": target_source, + "required_transport_ref": required_transport_ref, + "requires_live_gatt_validation": requires_live_gatt_validation, + "automatic_retry": False, + } + if required_connection_mode is not None: + result["required_connection_mode"] = required_connection_mode + return result + + actions: dict[str, dict[str, object]] = { + "scan-ble": decision(scan_reasons, target_source="none"), + "provision-fresh-device": decision( + fresh_mutation_reasons, + target_source="fresh-scan", + requires_live_gatt_validation=True, + ), + "recover-current-device-network": decision( + recovery_reasons, + target_source="retained-current-process", + required_transport_ref=recovery_ref, + requires_live_gatt_validation=True, + ), + "observe-fresh-device-network": decision( + observation_reasons, + target_source="fresh-scan", + required_transport_ref=( + physical_recovery_ref + if physical_read_only_recovery_required + else ledger_transport_ref + if ledger_unresolved + else None + ), + required_connection_mode=( + physical_recovery_mode + if physical_read_only_recovery_required + else ledger_connection_mode + if ledger_unresolved + else None + ), + requires_live_gatt_validation=True, + ), + "observe-current-device-network": decision( + retained_observation_reasons, + target_source="retained-current-process", + required_transport_ref=( + physical_recovery_ref + if physical_read_only_recovery_required + else ledger_transport_ref + if ledger_unresolved + else recovery_ref + ), + required_connection_mode=( + physical_recovery_mode + if physical_read_only_recovery_required + else ledger_connection_mode + if ledger_unresolved + else recovery_mode + ), + requires_live_gatt_validation=True, + ), + "observe-configured-device-network": decision( + durable_observation_reasons, + target_source="durable-configured-state", + required_transport_ref=( + physical_recovery_ref + if physical_read_only_recovery_required + else durable_observation_ref + ), + required_connection_mode=( + physical_recovery_mode + if physical_read_only_recovery_required + else durable_observation_mode + ), + requires_live_gatt_validation=( + ledger_unresolved or physical_read_only_recovery_required + ), + ), + "inspect-configured-endpoint": decision( + endpoint_reasons, + target_source="configured-topology", + ), + } + # Pure host-route and TCP probes stay independent from BLE, durable + # identity and physical-command state. They observe the host only and can + # never grant control authority by themselves. + for action in ("inspect-host-network", "probe-endpoint"): + actions[action] = decision( + [] if action in supervisor_actions else ["supervisor-action-not-allowed"], + target_source="connection-supervisor", + ) + + control_action_reasons = ( + [] + if "verify-control-device-info" in supervisor_actions + else ["supervisor-action-not-allowed"] + ) + if identity_pin_corrupt: + control_action_reasons.append("device-identity-pin-store-corrupt") + if ble_poisoned: + control_action_reasons.append("ble-runtime-restart-required") + elif ble_cleanup_pending: + control_action_reasons.append("ble-runtime-cleanup-pending") + elif ble_active_operation is not None: + control_action_reasons.append("ble-runtime-busy") + if control_process_lease_conflict: + control_action_reasons.append("k1-lifecycle-process-lease-network-owned") + actions["verify-control-device-info"] = decision( + control_action_reasons, + target_source="connection-supervisor", + ) + + physical_command_reasons: list[str] = [] + if physical_status == "corrupt": + physical_command_reasons.append("physical-command-ledger-corrupt") + elif physical_read_only_recovery_required or physical_status == "unresolved": + physical_command_reasons.append("physical-command-reconciliation-required") + elif physical_status not in {"empty", "resolved"}: + physical_command_reasons.append("physical-command-ledger-unavailable") + if ble_poisoned: + physical_command_reasons.append("ble-runtime-restart-required") + elif ble_cleanup_pending: + physical_command_reasons.append("ble-runtime-cleanup-pending") + elif ble_active_operation is not None: + physical_command_reasons.append("ble-runtime-busy") + if control_process_lease_conflict: + physical_command_reasons.append("k1-lifecycle-process-lease-network-owned") + + physical_mutation_target_ref = ( + supervisor.device_network.transport_ref + if supervisor.device_network.state == "applied" + and isinstance(supervisor.device_network.transport_ref, str) + else recovery_ref + ) + start_reasons = ( + [] if "start-acquisition" in supervisor_actions else ["supervisor-action-not-allowed"] + ) + if active_connection_mode != desired_connection_mode: + start_reasons.append("connection-mode-switch-pending") + control_reports_active_scanning = bool( + control_state == "scanning" + and application_control_session.get("can_stop") is True + and physical_start_active + ) + if physical_start_active: + start_reasons.append("physical-device-already-active") + if identity_pin_corrupt: + start_reasons.append("device-identity-pin-store-corrupt") + if transport_is_retired(physical_mutation_target_ref): + start_reasons.append("physical-command-target-retired") + # A READY classification after an undispatched STOP has completed every + # read-only proof. Its pending marker pins network/mode and the successor + # binding, but must not block the one fresh START that consumes that exact + # verified binding. The ledger remains the final exact-binding CAS. + start_reasons.extend(physical_command_reasons) + actions["start-acquisition"] = decision( + start_reasons, + target_source="connection-supervisor", + ) + + control_healthy = bool( + supervisor.authority.control_allowed and supervisor.control_plane.state == "healthy" + ) + data_lost_with_control = bool( + acquisition_active and supervisor.data_plane.state == "lost" and control_healthy + ) + physical_stop_exposed = bool( + "stop-acquisition" in supervisor_actions + or data_lost_with_control + or control_reports_active_scanning + ) + stop_reasons = [] if physical_stop_exposed else ["supervisor-action-not-allowed"] + if not control_healthy: + stop_reasons.append("physical-control-authority-unavailable") + if transport_is_retired(physical_mutation_target_ref): + stop_reasons.append("physical-command-target-retired") + # A fresh read-only SCANNING classification intentionally leaves the + # durable active row eligible for another read-only rebind until a later + # READY edge. Once that exact inspection worker has adopted SCANNING and + # exposes STOP, the recovery-only acquisition shell may issue its one + # fresh successor STOP. Suppress only this recovery marker here: corrupt + # ledgers, BLE quarantine, process-lease conflicts and all START/network + # barriers remain unchanged. + stop_reasons.extend( + reason + for reason in physical_command_reasons + if not ( + control_reports_active_scanning + and reason == "physical-command-reconciliation-required" + ) + ) + actions["stop-acquisition"] = decision( + stop_reasons, + target_source="connection-supervisor", + ) + + local_receiver_stop_reasons: list[str] = [] + if not (acquisition_active or acquisition_cleanup_pending or runtime_active): + local_receiver_stop_reasons.append("local-acquisition-receiver-not-active") + local_receiver_stop = decision( + local_receiver_stop_reasons, + target_source="local-runtime", + ) + local_receiver_stop.update( + { + "execution_mode": "capture-only", + "physical_command_allowed": False, + "physical_outcome": "unknown", + "operator_follow_up": "manual-device-stop-required", + } + ) + actions["stop-local-receiver"] = local_receiver_stop + + # The supervisor still exposes this legacy suggestion, but no backend + # command implements it. Never advertise a button whose mutation cannot be + # executed exactly. + actions["acknowledge-data-loss"] = decision( + ["action-not-implemented"], + target_source="connection-supervisor", + ) + + preferred_read_only_observation = next( + ( + action_name + for action_name in ( + "observe-current-device-network", + "observe-configured-device-network", + "observe-fresh-device-network", + ) + if actions[action_name]["allowed"] is True + ), + None, + ) + preferred_interrupted_restoration = next( + ( + action_name + for action_name in ( + "observe-current-device-network", + "observe-configured-device-network", + ) + if actions[action_name]["allowed"] is True + ), + None, + ) + interrupted_post_dispatch_audit = bool( + ledger.status == "resolved" + and ledger_stage == "resolved" + and ledger_resolution == "interrupted" + ) + if provisioning_active: + recommended_action = "wait-for-operation" + elif actions["stop-acquisition"]["allowed"] is True and ( + acquisition_active or control_reports_active_scanning + ): + recommended_action = "stop-acquisition" + elif actions["stop-local-receiver"]["allowed"] is True: + recommended_action = "stop-local-receiver" + elif ledger_corrupt: + recommended_action = "diagnose-network-ledger" + elif (ledger_unresolved and preferred_read_only_observation is not None) or ( + interrupted_post_dispatch_audit and preferred_interrupted_restoration is not None + ): + # An unresolved post-dispatch write must first use the most durable + # exact read-only target available. Startup may terminalize that audit + # as interrupted; an independently durable configured topology still + # remains the safer explicit inspection target after the restart. A + # passive scan remains allowed but must not displace that observation. + recommended_observation = ( + preferred_read_only_observation + if ledger_unresolved + else preferred_interrupted_restoration + ) + assert recommended_observation is not None + recommended_action = recommended_observation + elif actions["recover-current-device-network"]["allowed"] is True: + recommended_action = "recover-current-device-network" + elif not eligible_fresh_refs and actions["scan-ble"]["allowed"] is True: + recommended_action = "scan-ble" + else: + next_supervisor_action = next( + ( + action + for action in supervisor.allowed_actions + if action not in {"select-connection-intent", "acknowledge-data-loss"} + and actions.get(action, {}).get("allowed") is True + ), + None, + ) + recommended_action = next_supervisor_action or "select-connection-intent" + + return { + "schema_version": "missioncore.xgrids-k1-connection-policy/v1", + "supervisor_revision": supervisor.revision, + "network_ledger_revision": (ledger_record.revision if ledger_record is not None else None), + "recommended_action": recommended_action, + "allowed_actions": [action for action, item in actions.items() if item["allowed"] is True], + "actions": actions, + "facts": { + "fresh_transport_refs": list(fresh_refs), + "eligible_fresh_transport_refs": list(eligible_fresh_refs), + "retained_transport_ref": recovery_ref, + "retained_connection_mode": recovery_mode, + "desired_connection_mode": desired_connection_mode, + "active_connection_mode": active_connection_mode, + "connection_mode_switch_pending": (active_connection_mode != desired_connection_mode), + "durable_observation_transport_ref": durable_observation_ref, + "durable_observation_connection_mode": durable_observation_mode, + "retained_context_is_presence": False, + "network_mutation_status": ledger.status, + "network_provisioning_idempotency_status": idempotency_status, + "network_provisioning_idempotency_available": ( + network_provisioning_idempotency_available + ), + "network_provisioning_active_operation_id": (idempotency_active_operation_id), + "network_provisioning_active_operation_matches_ledger": ( + not idempotency_active_mismatch + ), + "semantic_topology_store_status": semantic_topology_store.get("status"), + "device_identity_pin_store_status": device_identity_pin_store.get("status"), + "physical_command_status": physical_status, + "physical_command_requires_reconciliation": (physical_requires_reconciliation), + "physical_command_recovery_transport_ref": physical_recovery_ref, + "physical_command_recovery_connection_mode": physical_recovery_mode, + "retired_transport_refs": sorted(retired_transport_refs), + "ble_runtime": { + "active_operation_kind": ble_active_operation, + "cleanup_pending": ble_cleanup_pending, + "poisoned": ble_poisoned, + }, + "lifecycle_process_lease_holders": list(lifecycle_holders), + "control_plane_state": supervisor.control_plane.state, + "data_plane_state": supervisor.data_plane.state, + "physical_network_state": ( + "unknown" if ledger_unresolved or ledger_corrupt else "not-disputed" + ), + }, + } + + +def _network_status_evidence(status: Mapping[str, Any]) -> NetworkStatusEvidence: + mode = status.get("mode") + ipv4 = status.get("ipv4") + status_code = status.get("status_code") + reserved = status.get("reserved") + if mode is not None and not isinstance(mode, str): + raise ValueError("K1 network status mode is invalid") + if ipv4 is not None and not isinstance(ipv4, str): + raise ValueError("K1 network status address is invalid") + if not isinstance(status_code, int) or isinstance(status_code, bool): + raise ValueError("K1 network status code is invalid") + if reserved is not None and (not isinstance(reserved, int) or isinstance(reserved, bool)): + raise ValueError("K1 network status reserved byte is invalid") + return NetworkStatusEvidence( + mode=mode, + ipv4=ipv4, + status_code=status_code, + reserved=reserved, + ) + + +def _optional_network_status_evidence(status: object) -> NetworkStatusEvidence | None: + if not isinstance(status, Mapping): + return None + try: + return _network_status_evidence(cast(Mapping[str, Any], status)) + except (TypeError, ValueError): + return None + + +def _last_network_status_evidence( + observations: object, +) -> NetworkStatusEvidence | None: + if not isinstance(observations, list): + return None + for observation in reversed(observations): + if not isinstance(observation, Mapping): + continue + evidence = _optional_network_status_evidence(observation.get("status")) + if evidence is not None: + return evidence + return None + + +def _recover_network_provisioning_after_process_restart( + repository_root: Path, + *, + network_ledger: NetworkMutationLedger, + idempotency_journal: NetworkProvisioningIdempotencyJournal | None, +) -> NetworkMutationLedgerSnapshot: + """Resolve only crash-safe provisioning states under the global K1 lease. + + A second backend may construct its read-only facade while the process that + admitted a request is still alive. Therefore the mere presence of a + ``prepared`` record is not restart evidence. The OS lease is the authority: + only a process that can acquire it may classify PREPARED as not dispatched. + """ + + try: + recovery_lease = ApplicationControlProcessLease.acquire(repository_root) + except ApplicationControlProcessLeaseUnavailable: + # Another live process still owns the K1 lifecycle. Preserve every + # durable record exactly as-is; this facade remains read-only/blocked. + return network_ledger.snapshot() + + try: + return _recover_network_provisioning_under_exclusive_process_lease( + network_ledger=network_ledger, + idempotency_journal=idempotency_journal, + ) + finally: + recovery_lease.release() + + +def _recover_physical_command_after_process_restart( + repository_root: Path, + *, + physical_ledger: PhysicalCommandLedger, + checkpoint_store: ActiveAcquisitionRecoveryCheckpointStore | None = None, +) -> None: + """Resolve only a PREPARED START after proving the old owner is gone. + + A second backend can construct its facade while the first process is still + between durable preparation and MQTT dispatch. PREPARED therefore proves + ``not-dispatched`` only when this process has acquired the same OS lifecycle + lease used by network, BLE and application-control ownership. + + A STOP is deliberately different: no-dispatch proves what Mission Core did, + but not whether the scanner is still SCANNING or was stopped out of process. + Preserve that exact PREPARED checkpoint so a later explicit read-only Verify + can atomically record both the no-dispatch outcome and fresh READY/SCANNING + evidence. Startup itself performs no device observation or command. + """ + + try: + recovery_lease = ApplicationControlProcessLease.acquire(repository_root) + except ApplicationControlProcessLeaseUnavailable: + return + + try: + physical_snapshot = physical_ledger.snapshot() + if ( + physical_snapshot.status == "unresolved" + and physical_snapshot.record is not None + and physical_snapshot.record.stage == "prepared" + and physical_snapshot.record.action == "start" + ): + physical_ledger.resolve( + physical_snapshot.record.operation_id, + resolution="not-dispatched", + ) + if checkpoint_store is not None: + _settle_restarted_prepared_start_checkpoint( + physical_ledger=physical_ledger, + checkpoint_store=checkpoint_store, + ) + finally: + recovery_lease.release() + + +def _physical_start_checkpoint_immutables_match( + checkpoint: ActiveAcquisitionRecoveryCheckpoint, + record: PhysicalCommandRecord, +) -> bool: + """Match secret-free immutable acquisition identity without inventing authority.""" + + if not ( + record.acquisition_id == checkpoint.acquisition_id + and record.identity.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and record.identity.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and record.compatibility_profile_id == checkpoint.compatibility_profile_id + and checkpoint.prepared_binding.logical_device_id + == checkpoint.identity.logical_device_id + and checkpoint.prepared_binding.vendor_device_id_sha256 + == checkpoint.identity.vendor_device_id_sha256 + and checkpoint.prepared_binding.device_serial_sha256 + == checkpoint.identity.device_serial_sha256 + and checkpoint.prepared_binding.compatibility_profile_id + == checkpoint.compatibility_profile_id + ): + return False + if checkpoint.state != "prepared": + return True + return bool( + record.action == "start" + and record.operation_id == checkpoint.original_start_operation_id + and record.payload_sha256 == checkpoint.start_payload_sha256 + and record.connection.intent_id == checkpoint.prepared_binding.intent_id + and record.connection.transport_ref == checkpoint.connection.transport_ref + == checkpoint.prepared_binding.transport_ref + and record.connection.connection_mode == checkpoint.connection.connection_mode + == checkpoint.prepared_binding.connection_mode + and record.connection.target_ipv4 == checkpoint.connection.target_ipv4 + == checkpoint.prepared_binding.target_ipv4 + and record.connection.target_port == checkpoint.connection.target_port + == checkpoint.prepared_binding.target_port + and record.connection.host_path_epoch + == checkpoint.prepared_binding.host_path_epoch + and record.connection.control_session_id + == checkpoint.prepared_binding.control_session_id + and record.connection.producer_generation + == checkpoint.prepared_binding.producer_generation + ) + + +def _settle_restarted_prepared_start_checkpoint( + *, + physical_ledger: PhysicalCommandLedger, + checkpoint_store: ActiveAcquisitionRecoveryCheckpointStore, +) -> None: + """Cease one exact abandoned PREPARED START without device I/O.""" + + checkpoint_snapshot = checkpoint_store.snapshot() + checkpoint = checkpoint_snapshot.checkpoint + if checkpoint_snapshot.status == "corrupt" or checkpoint is None: + return + if checkpoint.state != "prepared": + return + ledger_snapshot = physical_ledger.snapshot() + record = ledger_snapshot.record + if not ( + ledger_snapshot.status == "resolved" + and record is not None + and record.stage == "resolved" + and record.action == "start" + and record.resolution == "not-dispatched" + and _physical_start_checkpoint_immutables_match(checkpoint, record) + ): + return + proof_material = ( + f"{record.operation_id}\0{record.revision}\0{checkpoint.revision}".encode() + ) + proof = ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=record.revision, + proof_id=f"checkpoint-startup-proof:sha256-{hashlib.sha256(proof_material).hexdigest()}", + operation_id=record.operation_id, + original_start_operation_id=checkpoint.original_start_operation_id, + parent_operation_id=record.parent_operation_id, + acquisition_id=record.acquisition_id, + action="start", + resolution="not-dispatched", + payload_sha256=record.payload_sha256, + original_start_payload_sha256=record.payload_sha256, + reconciliation_kind=None, + reconciliation_resolution=None, + status_message_sha256=None, + observed_session_state=None, + binding=checkpoint.prepared_binding, + composite_complete=False, + edge_terminal=True, + late_start_excluded=True, + stop_fence="none", + observed_at_utc=record.updated_at_utc, + ) + checkpoint_store.cease( + transition_id=( + "checkpoint-startup-cease:sha256-" + + hashlib.sha256(b"cease\0" + proof_material).hexdigest() + ), + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=checkpoint.original_start_operation_id, + physical_proof=proof, + status_proof=None, + ) + + +def _recover_network_provisioning_under_exclusive_process_lease( + *, + network_ledger: NetworkMutationLedger, + idempotency_journal: NetworkProvisioningIdempotencyJournal | None, +) -> NetworkMutationLedgerSnapshot: + """Terminalize records abandoned by a previous host BLE session. + + Exclusive lifecycle ownership proves that no older Mission Core process is + still entitled to dispatch a command. It does *not* prove the device-side + result of a write whose transport was interrupted. Preserve that result as + ``unknown`` audit history, close the old session, and let a later explicit + operator action establish a new BLE/GATT session. No write is retried here. + """ + + ledger_snapshot = network_ledger.snapshot() + # A corrupt independent ledger cannot prove whether a BLE write crossed + # its dispatch boundary. Preserve every idempotency edge as-is and let the + # existing corrupt-ledger fence block both writes and reconciliation. + if ledger_snapshot.status == "corrupt": + return ledger_snapshot + # A durable PREPARED ledger record proves that mark_dispatching never + # crossed its fsynced boundary. No device observation is required. + if ( + ledger_snapshot.status == "unresolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.stage == "prepared" + ): + network_ledger.resolve( + ledger_snapshot.record.operation_id, + expected_revision=ledger_snapshot.record.revision, + resolution="not-dispatched", + ) + ledger_snapshot = network_ledger.snapshot() + + if idempotency_journal is None: + return ledger_snapshot + idempotency_snapshot = idempotency_journal.snapshot() + if idempotency_snapshot.status == "corrupt": + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency journal is corrupt during restart recovery" + ) + active_idempotency = idempotency_snapshot.active_record + + if ( + ledger_snapshot.status == "unresolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.stage in {"dispatching", "observing"} + ): + legacy_record = ledger_snapshot.record + if active_idempotency is None: + # Older installations persisted the authoritative mutation ledger + # before the independent request-idempotency journal existed. A + # direct atomic UNRESOLVED adoption repairs that schema boundary; + # PREPARED would be unsafe because the write already may have run. + active_idempotency = idempotency_journal.adopt_legacy_unresolved( + action=ACTION_NETWORK_PROVISION, + provenance=NetworkProvisioningLegacyAdoptionProvenance( + source_schema_version=legacy_record.schema_version, + operation_id=legacy_record.operation_id, + transport_ref=legacy_record.transport_ref, + intended_mode=legacy_record.intended_mode, + write_mode=legacy_record.write_mode, + baseline_status_sha256=_secret_free_evidence_sha256( + legacy_record.baseline_status.as_dict() + ), + previous_connection_sha256=( + _secret_free_evidence_sha256(legacy_record.previous_connection.as_dict()) + if legacy_record.previous_connection is not None + else None + ), + created_at_utc=legacy_record.created_at_utc, + ), + ) + elif not ( + active_idempotency.stage == "unresolved" + and active_idempotency.action == ACTION_NETWORK_PROVISION + and active_idempotency.operation_id == legacy_record.operation_id + ): + # The journal can contain an older abandoned request if a process + # stopped between two durable publications. It is audit history, + # not device-session authority. Terminalize it before adopting the + # exact current ledger edge. + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="cancelled", + outcome_code="network.provision.superseded", + error_code="network-session-superseded", + side_effect_status=( + "none" if active_idempotency.stage == "prepared" else "reconciled" + ), + retryable=True, + safe_to_retry=(active_idempotency.stage == "prepared"), + ), + ) + active_idempotency = idempotency_journal.adopt_legacy_unresolved( + action=ACTION_NETWORK_PROVISION, + provenance=NetworkProvisioningLegacyAdoptionProvenance( + source_schema_version=legacy_record.schema_version, + operation_id=legacy_record.operation_id, + transport_ref=legacy_record.transport_ref, + intended_mode=legacy_record.intended_mode, + write_mode=legacy_record.write_mode, + baseline_status_sha256=_secret_free_evidence_sha256( + legacy_record.baseline_status.as_dict() + ), + previous_connection_sha256=( + _secret_free_evidence_sha256(legacy_record.previous_connection.as_dict()) + if legacy_record.previous_connection is not None + else None + ), + created_at_utc=legacy_record.created_at_utc, + ), + ) + + # The previous BLE owner is gone. Preserve the ambiguity as a terminal + # audit outcome instead of requiring a special read-only recovery UI. + network_ledger.resolve( + legacy_record.operation_id, + expected_revision=legacy_record.revision, + resolution="interrupted", + ) + ledger_snapshot = network_ledger.snapshot() + + if active_idempotency is None: + return ledger_snapshot + if active_idempotency.stage == "prepared": + # PREPARED plus exclusive process ownership proves that the prior + # backend died before mark_unresolved and before BLE dispatch. + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="service-restart-before-dispatch", + error_code="request-not-dispatched", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + return ledger_snapshot + if ( + active_idempotency.stage == "unresolved" + and active_idempotency.action == ACTION_NETWORK_PROVISION + and ledger_snapshot.status == "resolved" + and ledger_snapshot.record is not None + and ledger_snapshot.record.operation_id == active_idempotency.operation_id + ): + # The independent network ledger closes crash gaps between its + # terminal resolution and terminalizing the idempotency record. + if ledger_snapshot.record.resolution == "not-dispatched": + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="network.provision.not-dispatched", + error_code="request-not-dispatched", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ), + ) + elif ledger_snapshot.record.resolution == "target-observed": + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="succeeded", + outcome_code="network.provision.reconciled", + error_code=None, + side_effect_status="reconciled", + retryable=False, + safe_to_retry=False, + ), + ) + elif ledger_snapshot.record.resolution in {"interrupted", "superseded"}: + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="cancelled", + outcome_code=( + "network.provision.interrupted" + if ledger_snapshot.record.resolution == "interrupted" + else "network.provision.superseded" + ), + error_code=( + "network-session-interrupted" + if ledger_snapshot.record.resolution == "interrupted" + else "network-session-superseded" + ), + side_effect_status="reconciled", + retryable=True, + safe_to_retry=False, + ), + ) + elif active_idempotency.stage == "unresolved": + # A journal-only edge means the previous process stopped after its + # ambiguity marker but before the independent ledger publication (or + # after that ledger was terminalized). Under exclusive ownership it is + # still an abandoned host session, never a permanent admission gate. + idempotency_journal.complete( + active_idempotency.operation_id, + expected_revision=active_idempotency.revision, + terminal=NetworkProvisioningTerminalMetadata( + outcome="cancelled", + outcome_code="network.provision.interrupted", + error_code="network-session-interrupted", + side_effect_status="reconciled", + retryable=True, + safe_to_retry=False, + ), + ) + return ledger_snapshot + + +def _secret_free_evidence_sha256(value: Mapping[str, object]) -> str: + """Canonical digest for an already validated, secret-free ledger value.""" + + return hashlib.sha256( + json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + ).hexdigest() + + +def _network_mutation_reconciliation_projection( + snapshot: NetworkMutationLedgerSnapshot, +) -> dict[str, Any] | None: + if snapshot.status == "corrupt": + return { + "status": "durable-ledger-corrupt", + "operation_id": "unknown", + "transport_ref": "unknown", + "connection_mode": "bridge", + "operation_stage": "ledger-corrupt", + "reason_code": snapshot.reason_code or "network-mutation-ledger-corrupt", + "device_write_confirmed": False, + "required_action": "operator-ledger-diagnosis", + "scope": "durable-ledger", + "ledger_revision": None, + "observed_at": None, + } + record = snapshot.record + if snapshot.status != "unresolved" or record is None: + return None + return { + "status": ( + "prepared-before-dispatch" + if record.stage == "prepared" + else "previous-session-result-unknown" + ), + "operation_id": record.operation_id, + "transport_ref": record.transport_ref, + "connection_mode": record.intended_mode, + "operation_stage": record.stage, + "reason_code": "network-mutation-audit-open", + "device_write_confirmed": bool(record.write_confirmed), + "required_action": "none", + "blocks_new_explicit_intent": False, + "scope": "durable-ledger", + "ledger_revision": record.revision, + "observed_at": record.updated_at_utc, + } + + +def _network_mutation_ledger_public_snapshot( + snapshot: NetworkMutationLedgerSnapshot, +) -> dict[str, Any]: + record = snapshot.record + host_diagnostic = host_diagnostic_for_reason(snapshot.reason_code) + return { + "status": snapshot.status, + # Unresolved is an audit state. The connect path terminalizes it under + # exclusive ownership before admitting a new explicit intent. + "mutation_allowed": snapshot.status != "corrupt", + "reason_code": snapshot.reason_code, + "operation_id": record.operation_id if record is not None else None, + "transport_ref": record.transport_ref if record is not None else None, + "intended_mode": record.intended_mode if record is not None else None, + "stage": record.stage if record is not None else None, + "revision": record.revision if record is not None else None, + "resolution": record.resolution if record is not None else None, + "updated_at_utc": record.updated_at_utc if record is not None else None, + "diagnostic": host_diagnostic.as_dict() if host_diagnostic is not None else None, + } + + +def _network_attempt_matches_live_control_authority( + *, + operation_documents: Sequence[Mapping[str, Any]], + attempt_id: object, + snapshot_runtime_id: str, + supervisor: ConnectionSupervisorSnapshot, + active_binding: Mapping[str, Any] | None, +) -> bool: + """Require historical bootstrap evidence and the exact live supervisor proof.""" + + if not isinstance(attempt_id, str) or not attempt_id: + return False + network_operation = next( + ( + item + for item in reversed(operation_documents) + if item.get("operation_id") == attempt_id + and item.get("action") == ACTION_NETWORK_PROVISION + ), + None, + ) + if not isinstance(network_operation, Mapping): + return False + context = network_operation.get("context") + result = network_operation.get("result") + if not isinstance(context, Mapping) or not isinstance(result, Mapping): + return False + transport_ref = result.get("transport_ref") + connection_mode = result.get("connection_mode") + target_ipv4 = result.get("target_ipv4") + target_port = result.get("target_port") + expected_live_intent_id = attempt_id + recovery_verify = next( + ( + item + for item in reversed(operation_documents) + if item.get("action") == ACTION_CONNECTION_VERIFY + and item.get("status") == "succeeded" + and _recovery_verify_matches_network_attempt(item, network_operation) + ), + None, + ) + if isinstance(recovery_verify, Mapping): + verify_result = recovery_verify.get("result") + verified_binding = ( + verify_result.get("verified_binding") if isinstance(verify_result, Mapping) else None + ) + recovered_intent_id = ( + verified_binding.get("intent_id") if isinstance(verified_binding, Mapping) else None + ) + if not isinstance(recovered_intent_id, str) or not recovered_intent_id: + return False + expected_live_intent_id = recovered_intent_id + return bool( + network_operation.get("status") == "succeeded" + and result.get("phase") == "network_applied" + and context.get("snapshot_runtime_id") == snapshot_runtime_id + and result.get("snapshot_runtime_id") == snapshot_runtime_id + and result.get("parent_intent_id") == attempt_id + and isinstance(transport_ref, str) + and transport_ref + and context.get("transport_ref") == transport_ref + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and context.get("connection_mode") == connection_mode + and isinstance(target_ipv4, str) + and target_ipv4 + and target_port == CONTROL_MQTT_PORT + and isinstance(active_binding, Mapping) + and active_binding.get("intent_id") == expected_live_intent_id + and active_binding.get("transport_ref") == transport_ref + and active_binding.get("connection_mode") == connection_mode + and active_binding.get("target_ipv4") == target_ipv4 + and active_binding.get("target_port") == target_port + and supervisor.authority.control_allowed + and supervisor.intent is not None + and supervisor.intent.intent_id == expected_live_intent_id + and supervisor.intent.requested_mode == connection_mode + and supervisor.device_network.state == "applied" + and supervisor.device_network.intent_id == expected_live_intent_id + and supervisor.device_network.transport_ref == transport_ref + and supervisor.device_network.connection_mode == connection_mode + and supervisor.device_network.target is not None + and supervisor.device_network.target.ipv4 == target_ipv4 + and supervisor.device_network.target.port == target_port + and supervisor.host_path.available + and supervisor.host_path.epoch >= 1 + and supervisor.endpoint.intent_id == expected_live_intent_id + and supervisor.endpoint.target == supervisor.device_network.target + and supervisor.endpoint.tcp_state == "reachable" + and supervisor.endpoint.host_path_epoch == supervisor.host_path.epoch + and supervisor.lease.state == "reachable" + and supervisor.lease.intent_id == expected_live_intent_id + and supervisor.lease.target == supervisor.device_network.target + and supervisor.lease.connection_mode == connection_mode + and supervisor.lease.host_path_epoch == supervisor.host_path.epoch + and supervisor.control_plane.state == "healthy" + and supervisor.control_plane.host_path_epoch == supervisor.host_path.epoch + and active_binding.get("host_path_epoch") == supervisor.host_path.epoch + and isinstance(active_binding.get("control_session_id"), str) + and bool(active_binding.get("control_session_id")) + ) + + +def _owned_control_bootstrap_matches_network_attempt( + bootstrap_operation: Mapping[str, Any], + network_operation: Mapping[str, Any], +) -> bool: + """Join only the service-owned child admitted for this exact Apply proof.""" + + bootstrap_context = bootstrap_operation.get("context") + network_context = network_operation.get("context") + network_result = network_operation.get("result") + if not ( + isinstance(bootstrap_context, Mapping) + and isinstance(network_context, Mapping) + and isinstance(network_result, Mapping) + ): + return False + network_operation_id = network_operation.get("operation_id") + runtime_id = network_result.get("snapshot_runtime_id") + transport_ref = network_result.get("transport_ref") + connection_mode = network_result.get("connection_mode") + target_ipv4 = network_result.get("target_ipv4") + target_port = network_result.get("target_port") + exact_parent = bool( + isinstance(network_operation_id, str) + and network_operation_id + and network_operation.get("status") == "succeeded" + and network_result.get("phase") == "network_applied" + and isinstance(runtime_id, str) + and runtime_id + and network_context.get("snapshot_runtime_id") == runtime_id + and network_context.get("transport_ref") == transport_ref + and network_context.get("connection_mode") == connection_mode + and network_result.get("parent_intent_id") == network_operation_id + and isinstance(transport_ref, str) + and transport_ref + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(target_ipv4, str) + and target_ipv4 + and target_port == CONTROL_MQTT_PORT + and bootstrap_context.get("ownership") == "service-owned-apply-continuation" + and bootstrap_context.get("snapshot_runtime_id") == runtime_id + and bootstrap_context.get("parent_operation_id") == network_operation_id + and bootstrap_context.get("parent_intent_id") == network_operation_id + and bootstrap_context.get("transport_ref") == transport_ref + and bootstrap_context.get("connection_mode") == connection_mode + and bootstrap_context.get("target_ipv4") == target_ipv4 + and bootstrap_context.get("target_port") == target_port + and isinstance(bootstrap_context.get("device_session_id"), str) + and bool(bootstrap_context.get("device_session_id")) + and bootstrap_context.get("network_mutation_performed") is False + and bootstrap_context.get("ble_operation_performed") is False + and bootstrap_context.get("automatic_retry") is False + ) + if not exact_parent: + return False + if bootstrap_operation.get("status") != "succeeded": + return True + bootstrap_result = bootstrap_operation.get("result") + return bool( + isinstance(bootstrap_result, Mapping) + and bootstrap_result.get("connection_mode") == connection_mode + and bootstrap_result.get("control_verified") is True + and bootstrap_result.get("network_mutation_performed") is False + and bootstrap_result.get("ble_operation_performed") is False + and bootstrap_result.get("automatic_retry") is False + ) + + +def _recovery_verify_matches_network_attempt( + verify_operation: Mapping[str, Any], + network_operation: Mapping[str, Any], +) -> bool: + """Join Verify only to the exact network parent it was admitted against.""" + + verify_context = verify_operation.get("context") + network_context = network_operation.get("context") + network_result = network_operation.get("result") + if not ( + isinstance(verify_context, Mapping) + and isinstance(network_context, Mapping) + and isinstance(network_result, Mapping) + ): + return False + network_operation_id = network_operation.get("operation_id") + runtime_id = network_result.get("snapshot_runtime_id") + transport_ref = network_result.get("transport_ref") + connection_mode = network_result.get("connection_mode") + target_ipv4 = network_result.get("target_ipv4") + target_port = network_result.get("target_port") + exact_parent = bool( + isinstance(network_operation_id, str) + and network_operation_id + and isinstance(runtime_id, str) + and runtime_id + and network_context.get("snapshot_runtime_id") == runtime_id + and network_context.get("transport_ref") == transport_ref + and network_context.get("connection_mode") == connection_mode + and network_result.get("parent_intent_id") == network_operation_id + and isinstance(transport_ref, str) + and transport_ref + and connection_mode in {"bridge", "quick-connect", "direct-connect"} + and isinstance(target_ipv4, str) + and target_ipv4 + and target_port == CONTROL_MQTT_PORT + and verify_context.get("snapshot_runtime_id") == runtime_id + and verify_context.get("recovery_parent_operation_id") == network_operation_id + and verify_context.get("recovery_parent_intent_id") == network_operation_id + and verify_context.get("recovery_transport_ref") == transport_ref + and verify_context.get("recovery_connection_mode") == connection_mode + and verify_context.get("recovery_target_ipv4") == target_ipv4 + and verify_context.get("recovery_target_port") == target_port + ) + if not exact_parent: + return False + if verify_operation.get("status") != "succeeded": + return True + verify_result = verify_operation.get("result") + if not isinstance(verify_result, Mapping): + return False + binding = verify_result.get("verified_binding") + return bool( + isinstance(binding, Mapping) + and binding.get("snapshot_runtime_id") == runtime_id + and isinstance(binding.get("intent_id"), str) + and bool(binding.get("intent_id")) + and binding.get("transport_ref") == transport_ref + and binding.get("connection_mode") == connection_mode + and binding.get("target_ipv4") == target_ipv4 + and binding.get("target_port") == target_port + and isinstance(binding.get("host_path_epoch"), int) + and not isinstance(binding.get("host_path_epoch"), bool) + and binding.get("host_path_epoch", 0) >= 1 + and verify_result.get("write_performed") is False + and verify_result.get("control_verified") is True + ) + + +def _connection_attempt_projection( + operation_documents: Sequence[Mapping[str, Any]], +) -> dict[str, Any] | None: + """Project the existing operation journal as one K1 connection attempt. + + This is deliberately a read model, not another lifecycle authority. The + operation journal, durable network ledger and connection supervisor remain + the writers of their respective facts; the projection only makes the + current stage and a safe next action visible to the operator. + """ + + network_operation_index = next( + ( + index + for index in range(len(operation_documents) - 1, -1, -1) + if operation_documents[index].get("action") == ACTION_NETWORK_PROVISION + ), + None, + ) + if network_operation_index is None: + return None + operation = operation_documents[network_operation_index] + network_context = operation.get("context") + network_context_mapping = network_context if isinstance(network_context, Mapping) else {} + control_bootstrap = next( + ( + item + for item in reversed(operation_documents) + if item.get("action") == ACTION_CONNECTION_CONTROL_BOOTSTRAP + and _owned_control_bootstrap_matches_network_attempt(item, operation) + ), + None, + ) + error = operation.get("error") + result = operation.get("result") + error_mapping = error if isinstance(error, Mapping) else {} + result_mapping = result if isinstance(result, Mapping) else {} + context_mapping = network_context_mapping + network_status = str(operation.get("status") or "unknown") + status = network_status + stage = operation.get("stage_code") + control_state: Literal["ready", "control_not_ready", "unknown"] = "unknown" + public_error_code = ( + error_mapping.get("code") if isinstance(error_mapping.get("code"), str) else None + ) + physical_reconciliation: Mapping[str, Any] | None = None + physical_active_observed = False + if network_status == "succeeded" and control_bootstrap is not None: + bootstrap_status = str(control_bootstrap.get("status") or "unknown") + status = bootstrap_status + stage = control_bootstrap.get("stage_code") + bootstrap_result = control_bootstrap.get("result") + bootstrap_reconciliation = ( + bootstrap_result.get("physical_reconciliation") + if isinstance(bootstrap_result, Mapping) + else None + ) + if isinstance(bootstrap_reconciliation, Mapping): + physical_reconciliation = bootstrap_reconciliation + physical_active_observed = bool( + bootstrap_reconciliation.get("performed") is True + and bootstrap_reconciliation.get("resolution") + == "physical-active-observed" + and bootstrap_reconciliation.get("observed_session_state") + == "scanning" + and bootstrap_reconciliation.get("device_write_performed") is False + ) + if bootstrap_status == "succeeded": + control_state = "unknown" if physical_active_observed else "ready" + elif bootstrap_status == "failed": + control_state = "control_not_ready" + bootstrap_error = control_bootstrap.get("error") + if isinstance(bootstrap_error, Mapping): + bootstrap_code = bootstrap_error.get("code") + public_error_code = ( + bootstrap_code if isinstance(bootstrap_code, str) else public_error_code + ) + recovery_verify = next( + ( + item + for item in reversed(operation_documents[network_operation_index + 1 :]) + if item.get("action") == ACTION_CONNECTION_VERIFY + and _recovery_verify_matches_network_attempt(item, operation) + ), + None, + ) + # A later explicit read-only Verify is the canonical recovery for an + # applied network edge whose first DeviceInfo bootstrap failed. It must + # update the visible attempt without rewriting or replaying that edge. + if network_status == "succeeded" and recovery_verify is not None: + status = str(recovery_verify.get("status") or "unknown") + stage = recovery_verify.get("stage_code") + verify_result = recovery_verify.get("result") + verify_reconciliation = ( + verify_result.get("physical_reconciliation") + if isinstance(verify_result, Mapping) + else None + ) + if isinstance(verify_reconciliation, Mapping): + physical_reconciliation = verify_reconciliation + physical_active_observed = bool( + verify_reconciliation.get("performed") is True + and verify_reconciliation.get("resolution") + == "physical-active-observed" + and verify_reconciliation.get("observed_session_state") + == "scanning" + and verify_reconciliation.get("device_write_performed") is False + ) + if status == "succeeded": + control_state = "unknown" if physical_active_observed else "ready" + elif status == "failed": + control_state = "control_not_ready" + else: + control_state = "unknown" + recovery_error = recovery_verify.get("error") + public_error_code = ( + recovery_error.get("code") + if isinstance(recovery_error, Mapping) and isinstance(recovery_error.get("code"), str) + else None + ) + side_effect_status = error_mapping.get( + "side_effect_status", + result_mapping.get("side_effect_status"), + ) + if not isinstance(side_effect_status, str): + side_effect_status = "none" if status in {"accepted", "running"} else "unknown" + phase: Literal[ + "network_applied", + "network_not_applied", + "network_outcome_unknown", + ] + if network_status == "succeeded": + phase = "network_applied" + elif side_effect_status == "unknown": + # Once a device write may have crossed the GATT boundary, a failed + # host request cannot truthfully claim that the network was not + # applied. This remains an audit/read-model distinction only: it does + # not authorize an automatic retry or replay of the old operation. + phase = "network_outcome_unknown" + else: + phase = "network_not_applied" + owned_bootstrap_pending = bool( + control_bootstrap is not None and status in {"accepted", "running"} + ) + recovery_verify_pending = bool( + recovery_verify is not None and status in {"accepted", "running"} + ) + if owned_bootstrap_pending or recovery_verify_pending: + safe_next_action = "wait-for-current-attempt" + elif status == "succeeded" and physical_active_observed: + safe_next_action = "stop-acquisition" + elif status == "succeeded" and control_state == "ready": + safe_next_action = "start-acquisition" + elif network_status == "succeeded": + safe_next_action = "verify-control-read-only" + else: + # Historical ambiguity blocks automatic replay of this attempt, not a + # later explicit scan-select-connect session. + safe_next_action = "scan-select-connect" + events = operation.get("events") + return { + "schema_version": "missioncore.xgrids-k1-connection-attempt/v1", + "attempt_id": operation.get("operation_id"), + "connection_mode": context_mapping.get("connection_mode"), + "status": status, + "phase": phase, + "control_state": control_state, + "stage": stage, + "public_error_code": public_error_code, + "side_effect_status": side_effect_status, + "safe_next_action": safe_next_action, + "physical_reconciliation": ( + dict(physical_reconciliation) + if physical_reconciliation is not None + else None + ), + "automatic_retry": False, + "accepted_at": operation.get("accepted_at"), + "completed_at": operation.get("completed_at"), + "recovery_operation_id": ( + recovery_verify.get("operation_id") if isinstance(recovery_verify, Mapping) else None + ), + "timeline": [ + *( + [dict(item) for item in events if isinstance(item, Mapping)] + if isinstance(events, list) + else [] + ), + *( + [ + {**dict(item), "source_action": ACTION_CONNECTION_CONTROL_BOOTSTRAP} + for item in control_bootstrap.get("events", []) + if isinstance(item, Mapping) + ] + if isinstance(control_bootstrap, Mapping) + and isinstance(control_bootstrap.get("events"), list) + else [] + ), + *( + [ + {**dict(item), "source_action": ACTION_CONNECTION_VERIFY} + for item in recovery_verify.get("events", []) + if isinstance(item, Mapping) + ] + if isinstance(recovery_verify, Mapping) + and isinstance(recovery_verify.get("events"), list) + else [] + ), + ], + } + + def _provisioned_ipv4(result: Mapping[str, Any]) -> str | None: observations = result.get("observations") if not isinstance(observations, list): @@ -3993,6 +32463,142 @@ def _provisioned_ipv4(result: Mapping[str, Any]) -> str | None: return None +def _applied_network_target( + connection_mode: ConnectionMode, + status: object, + *, + expected_network_name: str | None = None, +) -> str | None: + """Return the semantically proven target for one reviewed K1 network mode.""" + + if connection_mode not in CONNECTION_TOPOLOGY_BY_MODE or not isinstance(status, Mapping): + return None + if connection_mode == "quick-connect": + try: + return AP_FALLBACK_IPV4 if is_ap_ready_status(cast(Any, status)) else None + except (KeyError, TypeError, ValueError): + return None + if status.get("mode") != "WIFI_CLIENT": + return None + observed_network_name = status.get("network_name") + if expected_network_name is not None: + # FW 3.0.2 returns the joined SSID in the 7f02 text slot. Exact + # equality proves the requested Bridge/Direct target even when the K1 + # was already attached to that same network before this idempotent + # explicit command. A legacy literal-only WIFI_CLIENT response cannot + # prove an exact requested network and remains fail-closed. + if not isinstance(observed_network_name, str): + return None + if observed_network_name != expected_network_name: + return None + address = status.get("ipv4") + if not isinstance(address, str) or address == AP_FALLBACK_IPV4: + return None + try: + return validate_private_ipv4(address) + except ValueError: + return None + + +def _post_dispatch_network_target( + connection_mode: ConnectionMode, + status: object, + *, + transport_ref: str, + baseline_status: NetworkStatusEvidence, + previous_connection: PreviousConnectionEvidence | None = None, + expected_network_name: str | None = None, +) -> str | None: + """Return a target only when post-write status distinguishes the intent. + + FW 3.0.2 station status exposes the joined network name in the first text + slot. When that name exactly matches the requested Bridge/Direct target, + the desired topology is proven even if the pre-write status was identical: + the explicit command is an idempotent ensure-target operation. Legacy + literal-only ``WIFI_CLIENT`` observations retain the conservative + same-family ambiguity rules below. + """ + + observed_station_name = status.get("network_name") if isinstance(status, Mapping) else None + target = _applied_network_target( + connection_mode, + status, + expected_network_name=( + expected_network_name if isinstance(observed_station_name, str) else None + ), + ) + observed = _optional_network_status_evidence(status) + if target is None or observed is None: + return None + exact_station_target = bool( + connection_mode in {"bridge", "direct-connect"} + and expected_network_name is not None + and observed_station_name == expected_network_name + ) + if exact_station_target: + return target + if observed == baseline_status: + return None + same_transport_previous = bool( + previous_connection is not None and previous_connection.transport_ref == transport_ref + ) + if connection_mode == "quick-connect": + if ( + _applied_network_target( + connection_mode, + baseline_status.as_dict(), + ) + is not None + ): + return None + if ( + same_transport_previous + and previous_connection is not None + and previous_connection.mode == "quick-connect" + ): + return None + return target + if baseline_status.mode == "WIFI_CLIENT": + return None + if ( + same_transport_previous + and previous_connection is not None + and previous_connection.mode in {"bridge", "direct-connect"} + ): + return None + return target + + +def _status_observed_at_utc( + operation_result: Mapping[str, Any], + observation: Mapping[str, Any] | None, +) -> str: + """Return the exact status timestamp, with legacy-test fallback only.""" + + if observation is not None: + observed_at = observation.get("observed_at_utc") + if isinstance(observed_at, str) and observed_at: + return observed_at + completed_at = operation_result.get("completed_at_utc") + if isinstance(completed_at, str) and completed_at: + return completed_at + # Reviewed production BLE helpers always timestamp observations. Keeping a + # current fallback preserves compatibility with synthetic/legacy adapters + # without allowing the timestamp to establish live authority by itself. + return _utc_now_iso() + + +def _live_lan_target_from_status(status_read: Mapping[str, Any]) -> str: + status = status_read.get("status") + observed_target = status.get("ipv4") if isinstance(status, Mapping) else None + if not isinstance(observed_target, str) or observed_target == AP_FALLBACK_IPV4: + raise RuntimeError("K1 не сообщил актуальный DHCP-адрес через BLE status") + target = validate_private_ipv4(observed_target) + if _target_is_local_ipv4(target): + raise RuntimeError("BLE status сообщил адрес, принадлежащий этому компьютеру") + return target + + def _target_is_local_ipv4(target: str) -> bool: """Reject a BLE-reported target when the host route resolves back to itself. @@ -4011,6 +32617,66 @@ def _target_is_local_ipv4(target: str) -> bool: return source_address == target +async def _await_control_endpoint_reachable( + target: str, + *, + endpoint_probe: Callable[[str], _CorrelatedEndpointObservation], + timeout_seconds: float = CONTROL_ENDPOINT_ADMISSION_TIMEOUT_SECONDS, + interval_seconds: float = CONTROL_ENDPOINT_ADMISSION_INTERVAL_SECONDS, +) -> _CorrelatedEndpointObservation: + """Bounded read-only admission probe after a confirmed network change.""" + + if timeout_seconds <= 0: + raise ValueError("control endpoint admission timeout must be positive") + if interval_seconds <= 0: + raise ValueError("control endpoint admission interval must be positive") + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_seconds + latest_observation: _CorrelatedEndpointObservation | None = None + while True: + latest_observation = await _run_blocking_operation_without_abandonment( + endpoint_probe, + target, + ) + if latest_observation.reachable: + return latest_observation + remaining = deadline - loop.time() + if remaining <= 0: + return latest_observation + await asyncio.sleep(min(interval_seconds, remaining)) + + +async def _run_blocking_operation_without_abandonment[T]( + function: Callable[..., T], + /, + *args: Any, + **kwargs: Any, +) -> T: + """Keep a bounded host mutation owned after its HTTP caller is cancelled.""" + + task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs)) + try: + return await asyncio.shield(task) + except asyncio.CancelledError as cancellation: + # ``to_thread`` cannot stop the underlying CoreWLAN/Keychain call. Keep + # the provisioning gate held until that bounded call really finishes, + # consume its outcome, then preserve the caller cancellation. This + # prevents a second host-network mutation from overlapping the first. + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + # Repeated HTTP/task cancellation must not abandon the native + # thread. Delay every cancellation until the bounded worker is + # terminal and the caller can safely release its ownership gate. + continue + except Exception: + break + if task.done() and not task.cancelled(): + task.exception() + raise cancellation + + def _control_endpoint_reachable( target: str, *, @@ -4018,6 +32684,19 @@ def _control_endpoint_reachable( ) -> bool: """Check only the confirmed K1 MQTT endpoint; send no protocol bytes.""" + return _probe_control_endpoint_socket( + target, + timeout_seconds=timeout_seconds, + ).reachable + + +def _probe_control_endpoint_socket( + target: str, + *, + timeout_seconds: float = CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS, +) -> TcpReachabilityProbeResult: + """Classify one exact broker TCP probe without exporting socket details.""" + if timeout_seconds <= 0: raise ValueError("control endpoint probe timeout must be positive") target = validate_private_ipv4(target) @@ -4026,9 +32705,130 @@ def _control_endpoint_reachable( (target, CONTROL_MQTT_PORT), timeout=timeout_seconds, ): - return True + return TcpReachabilityProbeResult(reachable=True) + except OSError as exc: + reason_code = ( + "tcp-connection-refused" + if isinstance(exc, ConnectionRefusedError) or exc.errno == errno.ECONNREFUSED + else "tcp-connection-timeout" + if isinstance(exc, TimeoutError) or exc.errno == errno.ETIMEDOUT + else "tcp-endpoint-unreachable" + ) + return TcpReachabilityProbeResult( + reachable=False, + reason_code=reason_code, + ) + + +def _probe_configured_endpoint_host_only( + target: str, + *, + path_probe: Callable[[str], HostPathProbeResult] | None = None, +) -> _ConfiguredEndpointHostObservation: + """Probe only a bounded kernel route and TCP socket, with no supervisor effects.""" + + sample_path = path_probe or _inspect_host_path + first_path = sample_path(target) + if not first_path.available: + return _ConfiguredEndpointHostObservation( + path=first_path, + reachable=False, + reason_code=first_path.reason_code or "host-route-unavailable", + ) + tcp = _probe_control_endpoint_socket(target) + final_path = sample_path(target) + fingerprint_unchanged = bool( + final_path.available + and first_path.fingerprint is not None + and final_path.fingerprint == first_path.fingerprint + ) + reachable = tcp.reachable and fingerprint_unchanged + reason_code = ( + None + if reachable + else final_path.reason_code or "host-route-unavailable" + if not final_path.available + else "host-route-changed-during-tcp-probe" + if not fingerprint_unchanged + else tcp.reason_code or "tcp-endpoint-unreachable" + ) + return _ConfiguredEndpointHostObservation( + path=final_path, + reachable=reachable, + reason_code=reason_code, + ) + + +def _inspect_host_path(target: str) -> HostPathProbeResult: + """Describe the kernel-selected route to one exact K1 without scanning. + + The fingerprint contains only interface/route facts and the selected local + address. It deliberately excludes SSIDs, credentials and neighbour-table + enumeration. A UDP ``connect`` selects a source address in the kernel but + sends no packet. + """ + + target = validate_private_ipv4(target) + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as route_socket: + route_socket.connect((target, 9)) + source_address = str(route_socket.getsockname()[0]) except OSError: - return False + return HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + if source_address == target: + return HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=source_address, + route_class="unavailable", + reason_code="endpoint-target-is-local-host", + ) + + fields: dict[str, str] = {} + inspection_reason: str | None = None + if sys.platform == "darwin": + try: + result = subprocess.run( + ["route", "-n", "get", target], + check=False, + capture_output=True, + text=True, + timeout=HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.SubprocessError): + result = None + if result is None or result.returncode != 0: + inspection_reason = "host-route-inspection-unavailable" + else: + for line in result.stdout.splitlines(): + key, separator, value = line.strip().partition(":") + if separator: + fields[key] = value.strip() + + interface = fields.get("interface") or None + destination = fields.get("destination", "") + gateway = fields.get("gateway", "") + route_class, classification_reason = _classify_host_route(interface, destination) + inspection_reason = classification_reason or inspection_reason + fingerprint_material = "\x1f".join( + (target, source_address, interface or "", destination, gateway, route_class) + ) + return HostPathProbeResult( + available=True, + fingerprint=hashlib.sha256(fingerprint_material.encode("utf-8")).hexdigest(), + interface=interface, + source_ipv4=source_address, + route_class=route_class, + reason_code=inspection_reason, + ) def _host_route_class(target: str) -> str: @@ -4043,7 +32843,7 @@ def _host_route_class(target: str) -> str: check=False, capture_output=True, text=True, - timeout=2.0, + timeout=HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS, ) except (OSError, subprocess.SubprocessError): return "unknown" @@ -4056,15 +32856,54 @@ def _host_route_class(target: str) -> str: fields[key] = value.strip() interface = fields.get("interface", "") destination = fields.get("destination", "") - if interface.startswith(("utun", "ipsec", "ppp")): + route_class, _reason_code = _classify_host_route(interface or None, destination) + if route_class == "tunnel": return "tunnel" - if destination == "default": + if route_class == "default": return "default-route" - if interface: + if route_class == "direct": return "direct-or-routed" return "unknown" +def _classify_host_route( + interface: str | None, + destination: str, +) -> tuple[Literal["direct", "default", "tunnel", "unknown"], str | None]: + """Classify the exact kernel-selected route with an explicit LAN allowlist.""" + + normalized_interface = (interface or "").strip().casefold() + normalized_destination = destination.strip().casefold() + tunnel_prefixes = ( + "utun", + "ipsec", + "ppp", + "tun", + "tap", + "wg", + "wireguard", + "gif", + "stf", + ) + direct_lan_prefixes = ( + "en", + "eth", + "bridge", + "vlan", + "usb", + "p2p", + ) + if normalized_interface.startswith(tunnel_prefixes): + return "tunnel", "host-route-tunnel" + if normalized_destination == "default": + return "default", "host-route-default" + if normalized_interface.startswith(direct_lan_prefixes): + return "direct", None + if normalized_interface: + return "unknown", "host-route-interface-unreviewed" + return "unknown", "host-route-unclassified" + + def _validate_installed_compatibility_profile(repository_root: Path) -> None: """Run the plugin-owned, fail-closed profile validator before activation.""" @@ -4121,7 +32960,10 @@ def build_xgrids_k1_plugin(repository_root: Path) -> DevicePluginRuntimeContribu service.live_perception_ingress, XGRIDS_K1_PLUGIN_ID, bearer_token=service._live_perception_token, - result_receiver=service.runtime.publish_perception_result, + result_receiver=build_live_perception_result_receiver( + service.live_perception_ingress, + service.runtime.publish_perception_frame, + ), ), ), observation=build_xgrids_k1_observation(repository_root), diff --git a/src/k1link/device_plugins/xgrids_k1/host_diagnostics.py b/src/k1link/device_plugins/xgrids_k1/host_diagnostics.py new file mode 100644 index 0000000..89a83e3 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/host_diagnostics.py @@ -0,0 +1,429 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final, Literal + +from bleak.exc import ( + BleakBluetoothNotAvailableError, + BleakBluetoothNotAvailableReason, +) + +HostDiagnosticBoundary = Literal[ + "corebluetooth", + "corewlan", + "keychain", + "route", + "tcp", + "mqtt", + "filesystem", +] +HostDiagnosticDomain = HostDiagnosticBoundary +HostDiagnosticCode = Literal[ + "host.bluetooth.permission-denied", + "host.bluetooth.adapter-powered-off", + "host.bluetooth.adapter-unavailable", + "host.bluetooth.runtime-unavailable", + "host.bluetooth.operation-timeout", + "host.wifi.permission-denied", + "host.wifi.adapter-powered-off", + "host.wifi.interface-unavailable", + "host.wifi.ssid-unavailable", + "host.wifi.operation-timeout", + "host.wifi.association-failed", + "host.keychain.interaction-required", + "host.keychain.permission-denied", + "host.keychain.unavailable", + "host.route.unavailable", + "host.tcp.connection-refused", + "host.tcp.connection-timeout", + "host.tcp.endpoint-unavailable", + "host.mqtt.connection-timeout", + "host.mqtt.connection-refused", + "host.mqtt.transport-unavailable", + "host.filesystem.permission-denied", + "host.filesystem.ledger-unavailable", +] +HostDiagnosticImpact = Literal[ + "discovery", + "host-network", + "control", + "durable-safety", +] +HostDiagnosticAction = Literal[ + "grant-bluetooth-permission", + "power-on-bluetooth", + "restore-bluetooth-adapter", + "grant-wifi-permission", + "power-on-wifi", + "restore-wifi-interface", + "unlock-or-authorize-keychain", + "review-keychain-access", + "join-expected-network", + "inspect-host-route", + "verify-broker-endpoint", + "inspect-local-storage", + "restart-local-service", + "explicit-retry", +] + + +@dataclass(frozen=True, slots=True) +class HostFailureDiagnostic: + """Secret-free operator diagnostic for one host-side failure boundary.""" + + code: HostDiagnosticCode + domain: HostDiagnosticDomain + impact: HostDiagnosticImpact + operator_action: HostDiagnosticAction + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": "missioncore.host-failure-diagnostic/v1", + "code": self.code, + "domain": self.domain, + "impact": self.impact, + "operator_action": self.operator_action, + "automatic_retry": False, + "redacted": True, + } + + +@dataclass(frozen=True, slots=True) +class _DiagnosticSpec: + code: HostDiagnosticCode + domain: HostDiagnosticDomain + impact: HostDiagnosticImpact + operator_action: HostDiagnosticAction + + def diagnostic(self) -> HostFailureDiagnostic: + return HostFailureDiagnostic( + code=self.code, + domain=self.domain, + impact=self.impact, + operator_action=self.operator_action, + ) + + +_BLUETOOTH_PERMISSION = _DiagnosticSpec( + "host.bluetooth.permission-denied", + "corebluetooth", + "discovery", + "grant-bluetooth-permission", +) +_BLUETOOTH_POWERED_OFF = _DiagnosticSpec( + "host.bluetooth.adapter-powered-off", + "corebluetooth", + "discovery", + "power-on-bluetooth", +) +_BLUETOOTH_UNAVAILABLE = _DiagnosticSpec( + "host.bluetooth.adapter-unavailable", + "corebluetooth", + "discovery", + "restore-bluetooth-adapter", +) +_BLUETOOTH_RUNTIME_UNAVAILABLE = _DiagnosticSpec( + "host.bluetooth.runtime-unavailable", + "corebluetooth", + "discovery", + "restart-local-service", +) +_BLUETOOTH_TIMEOUT = _DiagnosticSpec( + "host.bluetooth.operation-timeout", + "corebluetooth", + "discovery", + "explicit-retry", +) +_WIFI_PERMISSION = _DiagnosticSpec( + "host.wifi.permission-denied", + "corewlan", + "host-network", + "grant-wifi-permission", +) +_WIFI_POWERED_OFF = _DiagnosticSpec( + "host.wifi.adapter-powered-off", + "corewlan", + "host-network", + "power-on-wifi", +) +_WIFI_UNAVAILABLE = _DiagnosticSpec( + "host.wifi.interface-unavailable", + "corewlan", + "host-network", + "restore-wifi-interface", +) +_WIFI_SSID_UNAVAILABLE = _DiagnosticSpec( + "host.wifi.ssid-unavailable", + "corewlan", + "host-network", + "join-expected-network", +) +_WIFI_TIMEOUT = _DiagnosticSpec( + "host.wifi.operation-timeout", + "corewlan", + "host-network", + "explicit-retry", +) +_WIFI_ASSOCIATION_FAILED = _DiagnosticSpec( + "host.wifi.association-failed", + "corewlan", + "host-network", + "join-expected-network", +) +_KEYCHAIN_INTERACTION_REQUIRED = _DiagnosticSpec( + "host.keychain.interaction-required", + "keychain", + "control", + "unlock-or-authorize-keychain", +) +_KEYCHAIN_PERMISSION = _DiagnosticSpec( + "host.keychain.permission-denied", + "keychain", + "control", + "review-keychain-access", +) +_KEYCHAIN_UNAVAILABLE = _DiagnosticSpec( + "host.keychain.unavailable", + "keychain", + "control", + "unlock-or-authorize-keychain", +) +_ROUTE_UNAVAILABLE = _DiagnosticSpec( + "host.route.unavailable", + "route", + "host-network", + "inspect-host-route", +) +_TCP_REFUSED = _DiagnosticSpec( + "host.tcp.connection-refused", + "tcp", + "control", + "verify-broker-endpoint", +) +_TCP_TIMEOUT = _DiagnosticSpec( + "host.tcp.connection-timeout", + "tcp", + "control", + "verify-broker-endpoint", +) +_TCP_UNAVAILABLE = _DiagnosticSpec( + "host.tcp.endpoint-unavailable", + "tcp", + "control", + "verify-broker-endpoint", +) +_MQTT_TIMEOUT = _DiagnosticSpec( + "host.mqtt.connection-timeout", + "mqtt", + "control", + "verify-broker-endpoint", +) +_MQTT_REFUSED = _DiagnosticSpec( + "host.mqtt.connection-refused", + "mqtt", + "control", + "verify-broker-endpoint", +) +_MQTT_UNAVAILABLE = _DiagnosticSpec( + "host.mqtt.transport-unavailable", + "mqtt", + "control", + "verify-broker-endpoint", +) +_FILESYSTEM_PERMISSION = _DiagnosticSpec( + "host.filesystem.permission-denied", + "filesystem", + "durable-safety", + "inspect-local-storage", +) +_LEDGER_UNAVAILABLE = _DiagnosticSpec( + "host.filesystem.ledger-unavailable", + "filesystem", + "durable-safety", + "inspect-local-storage", +) + + +_REASON_SPECS: Final[dict[str, _DiagnosticSpec]] = { + "ble-permission-denied": _BLUETOOTH_PERMISSION, + "ble-adapter-powered-off": _BLUETOOTH_POWERED_OFF, + "ble-adapter-unavailable": _BLUETOOTH_UNAVAILABLE, + "ble-runtime-owner-loop-conflict": _BLUETOOTH_RUNTIME_UNAVAILABLE, + "ble-runtime-restart-required": _BLUETOOTH_RUNTIME_UNAVAILABLE, + "connection-verify-runtime-loop-unavailable": _BLUETOOTH_RUNTIME_UNAVAILABLE, + "ble-discovery-timeout": _BLUETOOTH_TIMEOUT, + "ble-status-read-timeout": _BLUETOOTH_TIMEOUT, + "ble-provisioning-timeout": _BLUETOOTH_TIMEOUT, + "ble-ap-enable-timeout": _BLUETOOTH_TIMEOUT, + "corewlan-permission-denied": _WIFI_PERMISSION, + "corewlan-authorization-denied": _WIFI_PERMISSION, + "wifi-interface-inactive": _WIFI_POWERED_OFF, + "wifi-interface-unavailable": _WIFI_UNAVAILABLE, + "network-not-found": _WIFI_SSID_UNAVAILABLE, + "host-wifi-operation-timeout": _WIFI_TIMEOUT, + "corewlan-error": _WIFI_ASSOCIATION_FAILED, + "keychain-authorization-required": _KEYCHAIN_INTERACTION_REQUIRED, + "keychain-authorization-denied": _KEYCHAIN_PERMISSION, + "keychain-authorization-cancelled": _KEYCHAIN_PERMISSION, + "keychain-access-failed": _KEYCHAIN_UNAVAILABLE, + "application_authority_unavailable": _KEYCHAIN_UNAVAILABLE, + "host-route-unavailable": _ROUTE_UNAVAILABLE, + "host-path-unavailable": _ROUTE_UNAVAILABLE, + "host-route-interface-unavailable": _ROUTE_UNAVAILABLE, + "association-identity-unavailable": _ROUTE_UNAVAILABLE, + "host-path-observation-stale": _ROUTE_UNAVAILABLE, + "host-path-epoch-changed": _ROUTE_UNAVAILABLE, + "host-path-probe-error": _ROUTE_UNAVAILABLE, + "host-path-recheck-error": _ROUTE_UNAVAILABLE, + "connection-monitor-start-failed": _ROUTE_UNAVAILABLE, + "tcp-connection-refused": _TCP_REFUSED, + "tcp-connection-timeout": _TCP_TIMEOUT, + "tcp-endpoint-unreachable": _TCP_UNAVAILABLE, + "tcp-route-lost": _TCP_UNAVAILABLE, + "tcp-probe-error": _TCP_UNAVAILABLE, + "endpoint-unreachable": _TCP_UNAVAILABLE, + "endpoint-observation-stale": _TCP_UNAVAILABLE, + "quick_connect_endpoint_unreachable": _TCP_UNAVAILABLE, + "connection_lease_endpoint_unreachable_after_provision": _TCP_UNAVAILABLE, + "connection_lease_recovered_endpoint_unreachable": _TCP_UNAVAILABLE, + "mqtt_connection_timeout": _MQTT_TIMEOUT, + "mqtt_connect_call_failed": _MQTT_UNAVAILABLE, + "mqtt_connect_rejected": _MQTT_REFUSED, + "mqtt_broker_rejected_connection": _MQTT_REFUSED, + "mqtt_network_loop_failed": _MQTT_UNAVAILABLE, + "mqtt_connection_ended": _MQTT_UNAVAILABLE, + "mqtt_client_unavailable": _MQTT_UNAVAILABLE, + "mqtt_transport_failure": _MQTT_UNAVAILABLE, + "mqtt-control-loop-lost": _MQTT_UNAVAILABLE, + "control-proof-observation-stale": _MQTT_UNAVAILABLE, + "network-mutation-ledger-error": _LEDGER_UNAVAILABLE, + "network-mutation-ledger-corrupt": _LEDGER_UNAVAILABLE, + "network-provisioning-idempotency-error": _LEDGER_UNAVAILABLE, + "network-provisioning-idempotency-corrupt": _LEDGER_UNAVAILABLE, + "physical-command-ledger-error": _LEDGER_UNAVAILABLE, + "physical-command-ledger-corrupt": _LEDGER_UNAVAILABLE, + "semantic-topology-store-error": _LEDGER_UNAVAILABLE, + "semantic-topology-store-corrupt": _LEDGER_UNAVAILABLE, + "device-identity-pin-store-error": _LEDGER_UNAVAILABLE, + "device-identity-pin-store-corrupt": _LEDGER_UNAVAILABLE, + "application-control-process-lease-error": _LEDGER_UNAVAILABLE, + "application-control-process-lease-unavailable": _LEDGER_UNAVAILABLE, +} + + +def host_diagnostic_for_reason(reason_code: object) -> HostFailureDiagnostic | None: + """Map only reviewed reason codes; never reflect arbitrary input.""" + + if not isinstance(reason_code, str): + return None + spec = _REASON_SPECS.get(reason_code) + return spec.diagnostic() if spec is not None else None + + +def host_diagnostic_for_exception( + exc: BaseException, + *, + boundary: HostDiagnosticBoundary | None = None, +) -> HostFailureDiagnostic | None: + """Classify a host exception into a redacted diagnostic whitelist.""" + + explicit = host_diagnostic_for_reason(getattr(exc, "reason_code", None)) + if explicit is not None: + return explicit + if isinstance(exc, ConnectionRefusedError): + if boundary == "tcp": + return _TCP_REFUSED.diagnostic() + if boundary == "mqtt": + return _MQTT_REFUSED.diagnostic() + return None + if isinstance(exc, PermissionError): + if boundary == "corebluetooth": + return _BLUETOOTH_PERMISSION.diagnostic() + if boundary == "corewlan": + return _WIFI_PERMISSION.diagnostic() + if boundary == "keychain": + return _KEYCHAIN_PERMISSION.diagnostic() + if boundary == "filesystem": + return _FILESYSTEM_PERMISSION.diagnostic() + return None + if isinstance(exc, TimeoutError): + if boundary == "corebluetooth": + return _BLUETOOTH_TIMEOUT.diagnostic() + if boundary == "corewlan": + return _WIFI_TIMEOUT.diagnostic() + if boundary == "tcp": + return _TCP_TIMEOUT.diagnostic() + if boundary == "mqtt": + return _MQTT_TIMEOUT.diagnostic() + return None + if boundary == "filesystem" and isinstance(exc, OSError): + return _LEDGER_UNAVAILABLE.diagnostic() + + if boundary == "corebluetooth" and isinstance( + exc, + BleakBluetoothNotAvailableError, + ): + spec = { + BleakBluetoothNotAvailableReason.POWERED_OFF: _BLUETOOTH_POWERED_OFF, + BleakBluetoothNotAvailableReason.DENIED_BY_USER: _BLUETOOTH_PERMISSION, + BleakBluetoothNotAvailableReason.DENIED_BY_SYSTEM: _BLUETOOTH_PERMISSION, + BleakBluetoothNotAvailableReason.DENIED_BY_UNKNOWN: _BLUETOOTH_PERMISSION, + BleakBluetoothNotAvailableReason.NO_BLUETOOTH: _BLUETOOTH_UNAVAILABLE, + BleakBluetoothNotAvailableReason.NO_BLE_CENTRAL_ROLE: _BLUETOOTH_UNAVAILABLE, + BleakBluetoothNotAvailableReason.UNKNOWN: _BLUETOOTH_UNAVAILABLE, + }[exc.reason] + return spec.diagnostic() + + # Older CoreBluetooth surfaces still expose several failures only as a + # human string. Match a narrow, reviewed vocabulary and export none of it. + message = str(exc).casefold() + if boundary == "corebluetooth": + if any( + fragment in message + for fragment in ( + "not authorized", + "permission denied", + "access denied", + "bluetooth permission", + ) + ): + return _BLUETOOTH_PERMISSION.diagnostic() + if any( + fragment in message + for fragment in ("powered off", "power off", "bluetooth is off", "poweredoff") + ): + return _BLUETOOTH_POWERED_OFF.diagnostic() + if any( + fragment in message + for fragment in ( + "adapter unavailable", + "bluetooth unavailable", + "no bluetooth adapter", + ) + ): + return _BLUETOOTH_UNAVAILABLE.diagnostic() + if boundary == "keychain" or "authorityloaderror" in type(exc).__name__.casefold(): + return _KEYCHAIN_UNAVAILABLE.diagnostic() + return None + + +def host_diagnostics_for_reasons(*reason_codes: object) -> tuple[HostFailureDiagnostic, ...]: + diagnostics: list[HostFailureDiagnostic] = [] + seen: set[str] = set() + for reason_code in reason_codes: + diagnostic = host_diagnostic_for_reason(reason_code) + if diagnostic is None or diagnostic.code in seen: + continue + seen.add(diagnostic.code) + diagnostics.append(diagnostic) + return tuple(diagnostics) + + +__all__ = [ + "HostDiagnosticBoundary", + "HostDiagnosticCode", + "HostFailureDiagnostic", + "host_diagnostic_for_exception", + "host_diagnostic_for_reason", + "host_diagnostics_for_reasons", +] diff --git a/src/k1link/device_plugins/xgrids_k1/legacy_api.py b/src/k1link/device_plugins/xgrids_k1/legacy_api.py index 7366cd6..39b7c5e 100644 --- a/src/k1link/device_plugins/xgrids_k1/legacy_api.py +++ b/src/k1link/device_plugins/xgrids_k1/legacy_api.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio from typing import Any -from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect +from fastapi import APIRouter, Header, HTTPException, WebSocket, WebSocketDisconnect from k1link.device_plugins.xgrids_k1.facade import ( ACTION_DISCOVERY_SCAN, @@ -36,23 +36,41 @@ def build_xgrids_k1_legacy_router(runtime: DevicePluginRuntimeTransport) -> APIR return await invoke_device_plugin_runtime(runtime, ACTION_STATE_READ, {}) @router.post("/api/ble/scan", deprecated=True) - async def scan_ble(request: BleScanRequest) -> dict[str, Any]: + async def scan_ble( + request: BleScanRequest, + expected_snapshot_runtime_id: str = Header( + ..., + alias="X-Mission-Core-Snapshot-Runtime-Id", + ), + ) -> dict[str, Any]: try: return await invoke_device_plugin_runtime( runtime, ACTION_DISCOVERY_SCAN, - request.model_dump(), + { + **request.model_dump(), + "expected_snapshot_runtime_id": expected_snapshot_runtime_id, + }, ) except (PluginExecutionError, ValueError) as exc: raise HTTPException(status_code=502, detail=f"Ошибка поиска BLE: {exc}") from exc @router.post("/api/connect", deprecated=True) - async def connect(request: ConnectRequest) -> dict[str, Any]: + async def connect( + request: ConnectRequest, + expected_snapshot_runtime_id: str = Header( + ..., + alias="X-Mission-Core-Snapshot-Runtime-Id", + ), + ) -> dict[str, Any]: try: return await invoke_device_plugin_runtime( runtime, ACTION_NETWORK_PROVISION, - request.model_dump(), + { + **request.model_dump(), + "expected_snapshot_runtime_id": expected_snapshot_runtime_id, + }, ) except (PluginExecutionError, ValueError) as exc: raise HTTPException( diff --git a/src/k1link/device_plugins/xgrids_k1/live_perception_shadow.py b/src/k1link/device_plugins/xgrids_k1/live_perception_shadow.py index 74a8cf8..5b8de79 100644 --- a/src/k1link/device_plugins/xgrids_k1/live_perception_shadow.py +++ b/src/k1link/device_plugins/xgrids_k1/live_perception_shadow.py @@ -13,16 +13,42 @@ from uuid import uuid4 from fastapi import APIRouter, WebSocket, WebSocketDisconnect -from k1link.compute.live_perception import LivePerceptionIngress +from k1link.compute.live_perception import ( + LivePerceptionIngress, + LivePerceptionResultFrame, + decode_live_perception_result, +) TOKEN_BYTES = 32 TOKEN_FILE_NAME = "shadow-worker.token" +def build_live_perception_result_receiver( + ingress: LivePerceptionIngress, + publish_frame: Callable[[LivePerceptionResultFrame], bool], +) -> Callable[[bytes], bool]: + """Bind worker results to the exact active ingress session atomically.""" + + def receive(encoded: bytes) -> bool: + frame = decode_live_perception_result(encoded) + return ingress.admit_result( + session_id=frame.session_id, + session_generation=frame.session_generation, + receiver=lambda: publish_frame(frame), + ) + + return receive + + def ensure_live_shadow_token(repository_root: Path) -> tuple[Path, str]: """Load or create the private bearer used only through the SSH tunnel.""" - token_root = repository_root.resolve() / ".runtime" / "live-perception" + configured_data_root = os.environ.get("MISSIONCORE_DATA_DIR", "").strip() + token_root = ( + Path(configured_data_root).expanduser().resolve() / "live-perception" + if configured_data_root + else repository_root.resolve() / ".runtime" / "live-perception" + ) token_root.mkdir(mode=0o700, parents=True, exist_ok=True) with suppress(OSError): token_root.chmod(0o700) @@ -59,9 +85,7 @@ def build_live_perception_shadow_router( router = APIRouter(include_in_schema=False) - @router.websocket( - f"/api/v1/device-plugins/{plugin_id}/live-perception-shadow" - ) + @router.websocket(f"/api/v1/device-plugins/{plugin_id}/live-perception-shadow") async def live_perception_shadow(websocket: WebSocket) -> None: authorization = websocket.headers.get("authorization", "") supplied = authorization.removeprefix("Bearer ") @@ -103,13 +127,19 @@ def build_live_perception_shadow_router( ) return try: - await asyncio.to_thread(result_receiver, result) + accepted = await asyncio.to_thread(result_receiver, result) except (RuntimeError, ValueError): await websocket.close( code=1008, reason="Shadow result contract is invalid", ) return + if not accepted: + await websocket.close( + code=1008, + reason="Shadow result session is stale", + ) + return client_event = asyncio.create_task(websocket.receive()) if ingress_event in completed: event = ingress_event.result() diff --git a/src/k1link/device_plugins/xgrids_k1/mqtt/capture.py b/src/k1link/device_plugins/xgrids_k1/mqtt/capture.py index ffdcc10..15b71ae 100644 --- a/src/k1link/device_plugins/xgrids_k1/mqtt/capture.py +++ b/src/k1link/device_plugins/xgrids_k1/mqtt/capture.py @@ -8,10 +8,11 @@ import os import re import stat import struct +import threading import time from collections.abc import Callable, Iterator from contextlib import suppress -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from typing import IO, Literal, TypedDict @@ -29,6 +30,19 @@ REPORT_TOPICS: tuple[str, ...] = ( "DeviceStatus", ) +# A fresh subscription proves only broker transport. Recovery of the live +# scene requires one non-retained point-cloud report from that exact MQTT +# client to establish a candidate sequence fence. Pose, DeviceStatus and +# heartbeat remain useful transport/control evidence, but none proves that the +# visible acquisition cloud has resumed. Only the downstream post-publish +# observer can confirm that candidate. +RECOVERY_POINT_CLOUD_TOPICS = frozenset( + { + "RealtimePointcloud", + "lixel/application/report/lio_pcl", + } +) + DEFAULT_MAX_MESSAGE_BYTES = 64 * 1024 * 1024 MAX_CONFIGURABLE_MESSAGE_BYTES = 256 * 1024 * 1024 MAX_TOPIC_BYTES = 65_535 @@ -40,6 +54,8 @@ GROUP_COMMIT_MAX_BYTES = 4 * 1024 * 1024 GROUP_COMMIT_MAX_MESSAGES = 32 CAPTURE_CLOCK_FILENAME = "mqtt.timeline.json" CAPTURE_CLOCK_ORIGIN_FILENAME = "mqtt.timeline.origin.json" +RECOVERY_GAPS_FILENAME = "mqtt.recovery.jsonl" +RECOVERY_GAP_SCHEMA_VERSION = 1 CAPTURE_CLOCK_ORIGIN_SCHEMA_VERSION = 1 CAPTURE_CLOCK_SEALED_PATTERN = re.compile(r"^mqtt\.timeline\.session-[a-f0-9]{64}\.json$") CAPTURE_CLOCK_SCHEMA_VERSION = 1 @@ -61,14 +77,18 @@ StopReason = Literal[ "message_too_large", "connection_failed", "connection_lost", + "recovery_standby", "subscription_failed", "capture_error", ] +RecoveryDecision = Literal["retry", "resume", "standby", "fault", "blocked"] +RecoveryGapOutcome = Literal["recovered", "standby", "fault", "blocked", "interrupted"] class ArtifactPaths(TypedDict): raw: str metadata_jsonl: str + recovery_gaps_jsonl: str capture_clock_origin: str capture_clock: str summary: str @@ -77,6 +97,7 @@ class ArtifactPaths(TypedDict): class ArtifactHashes(TypedDict): raw_sha256: str metadata_jsonl_sha256: str + recovery_gaps_jsonl_sha256: str capture_clock_origin_sha256: str capture_clock_sha256: str @@ -87,6 +108,17 @@ class RawFormat(TypedDict): frame_layout: str +class RecoveryGapSummary(TypedDict): + gap_index: int + started_at_utc: str + started_monotonic_ns: int + ended_at_utc: str + ended_monotonic_ns: int + duration_seconds: float + recovery_attempt: int + outcome: RecoveryGapOutcome + + class CaptureSummary(TypedDict): schema_version: int created_at_utc: str @@ -99,6 +131,11 @@ class CaptureSummary(TypedDict): subscription_qos: int clean_session: bool reconnect_enabled: bool + recovery_attempts: int + successful_recoveries: int + recovery_point_cloud_candidates: int + recovery_blocked: bool + recovery_gaps: list[RecoveryGapSummary] publishing_enabled: bool subscriptions: list[str] requested_duration_seconds: float | None @@ -195,6 +232,13 @@ class _CaptureState: subscription_mid: int | None = None stop_reason: StopReason = "capture_error" error: str | None = None + connection_lost: bool = False + connection_lost_message: str | None = None + recovery_attempts: int = 0 + successful_recoveries: int = 0 + recovery_point_cloud_candidates: int = 0 + recovery_blocked: bool = False + recovery_gaps: list[RecoveryGapSummary] = field(default_factory=list) class _CaptureWriter: @@ -202,6 +246,7 @@ class _CaptureWriter: self.out_dir = out_dir.expanduser().resolve() self.raw_path = self.out_dir / "mqtt.raw.k1mqtt" self.metadata_path = self.out_dir / "mqtt.metadata.jsonl" + self.recovery_gaps_path = self.out_dir / RECOVERY_GAPS_FILENAME self.capture_clock_origin_path = self.out_dir / CAPTURE_CLOCK_ORIGIN_FILENAME self.capture_clock_path = self.out_dir / CAPTURE_CLOCK_FILENAME self.summary_path = self.out_dir / "mqtt.summary.json" @@ -212,6 +257,8 @@ class _CaptureWriter: self.topic_counts: dict[str, int] = {} self._raw: IO[bytes] | None = None self._metadata: IO[str] | None = None + self._recovery_gaps: IO[str] | None = None + self._recovery_gaps_lock = threading.Lock() self._pending_metadata: list[str] = [] self._pending_raw_bytes = 0 self._last_commit_monotonic = time.monotonic() @@ -223,6 +270,7 @@ class _CaptureWriter: artifact_paths = ( self.raw_path, self.metadata_path, + self.recovery_gaps_path, self.capture_clock_origin_path, self.capture_clock_path, self.summary_path, @@ -236,6 +284,7 @@ class _CaptureWriter: self._raw = _open_binary_exclusive(self.raw_path) self._raw.write(RAW_MAGIC) self._metadata = _open_text_exclusive(self.metadata_path) + self._recovery_gaps = _open_text_exclusive(self.recovery_gaps_path) _fsync_directory(self.out_dir) # This is the earliest durable point from which the capture can # accept evidence. The camera is armed only after this writer is @@ -364,9 +413,36 @@ class _CaptureWriter: except OSError as exc: if first_error is None: first_error = exc + with self._recovery_gaps_lock: + recovery_stream = self._recovery_gaps + if recovery_stream is not None and not recovery_stream.closed: + try: + recovery_stream.flush() + os.fsync(recovery_stream.fileno()) + except OSError as exc: + if first_error is None: + first_error = exc + finally: + try: + recovery_stream.close() + except OSError as exc: + if first_error is None: + first_error = exc if first_error is not None: raise first_error + def record_recovery_gap_event(self, record: dict[str, object]) -> None: + """Append and fsync one recovery boundary without touching frame metadata.""" + + payload = json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n" + with self._recovery_gaps_lock: + stream = self._recovery_gaps + if stream is None or stream.closed: + raise RuntimeError("recovery gap journal is not open") + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + def finalize_capture_clock(self) -> CaptureClockEnvelope: """Publish the exact capture envelope after every producer is sealed.""" @@ -748,6 +824,20 @@ def seal_capture_clock(capture_root: Path) -> CaptureClockEnvelope: raise CaptureError(f"could not seal session capture clock: {exc}") from exc +def _recovery_backoff_seconds(attempt: int) -> float: + """Return the canonical capped delay without evaluating an unbounded power.""" + + if attempt <= 1: + return 0.5 + if attempt == 2: + return 1.0 + if attempt == 3: + return 2.0 + if attempt == 4: + return 4.0 + return 5.0 + + def capture_mqtt( host: str, out_dir: Path, @@ -759,9 +849,23 @@ def capture_mqtt( on_ready: Callable[[], None] | None = None, on_message_recorded: Callable[[CapturedMqttMessage], None] | None = None, should_stop: Callable[[], bool] | None = None, + on_connection_lost: Callable[[str], None] | None = None, + consume_connection_recovery_request: Callable[[], str | None] | None = None, + recover_connection: Callable[[int], RecoveryDecision] | None = None, + on_recovery_point_cloud_candidate: Callable[[int, int], None] | None = None, + on_recovery_confirmer_ready: Callable[[Callable[[int], bool]], None] | None = None, _client_factory: Callable[[], mqtt.Client] | None = None, ) -> CaptureSummary: - """Capture the fixed K1 report subscriptions once, without publishing or reconnecting.""" + """Capture fixed K1 reports with an optional externally fenced reconnect. + + The capture layer never decides that an endpoint is still the same K1. A + caller may supply ``recover_connection`` only when a higher-level owner can + revalidate the exact route, DeviceInfo identity and physical SCANNING + lineage. ``consume_connection_recovery_request`` lets that same owner wake + this existing capture before the MQTT keepalive notices a short host-path + outage. The callback returns the only admitted next step; this function + itself never publishes, scans BLE, changes Wi-Fi, or retries START/STOP. + """ target_ipv4 = validate_private_ipv4(host) if not 1 <= port <= 65535: raise ValueError("port must be between 1 and 65535") @@ -774,16 +878,18 @@ def capture_mqtt( f"max_message_bytes must be between 1 and {MAX_CONFIGURABLE_MESSAGE_BYTES}" ) - client = ( - _client_factory() - if _client_factory is not None - else mqtt.Client( - callback_api_version=CallbackAPIVersion.VERSION2, - clean_session=True, - protocol=mqtt.MQTTv311, - reconnect_on_failure=False, + def make_client() -> mqtt.Client: + return ( + _client_factory() + if _client_factory is not None + else mqtt.Client( + callback_api_version=CallbackAPIVersion.VERSION2, + clean_session=True, + protocol=mqtt.MQTTv311, + reconnect_on_failure=False, + ) ) - ) + writer = _CaptureWriter(out_dir, max_message_bytes) try: writer.open() @@ -798,12 +904,119 @@ def capture_mqtt( operation_started = time.monotonic() capture_started: float | None = None failure: CaptureError | None = None + disconnect_expected = False + recovering = False + pending_recovery_attempt: int | None = None + pending_recovery_confirmation_attempt: int | None = None + active_recovery_gap: tuple[int, str, int] | None = None + recovery_lock = threading.Lock() + active_client: mqtt.Client | None = None def fail(reason: StopReason, message: str) -> None: if state.error is None: state.stop_reason = reason state.error = message + def finish_recovery_gap_locked( + outcome: RecoveryGapOutcome, + attempt: int, + ) -> bool: + nonlocal active_recovery_gap, pending_recovery_confirmation_attempt + + gap = active_recovery_gap + if gap is None: + return False + gap_index, started_at_utc, started_monotonic_ns = gap + ended_at_utc = utc_now_iso() + ended_monotonic_ns = time.monotonic_ns() + summary: RecoveryGapSummary = { + "gap_index": gap_index, + "started_at_utc": started_at_utc, + "started_monotonic_ns": started_monotonic_ns, + "ended_at_utc": ended_at_utc, + "ended_monotonic_ns": ended_monotonic_ns, + "duration_seconds": round( + max(ended_monotonic_ns - started_monotonic_ns, 0) / 1_000_000_000, + 9, + ), + "recovery_attempt": attempt, + "outcome": outcome, + } + try: + writer.record_recovery_gap_event( + { + "schema_version": RECOVERY_GAP_SCHEMA_VERSION, + "record_type": "recovery_gap_ended", + **summary, + } + ) + except (OSError, RuntimeError, ValueError) as exc: + fail( + "capture_error", + f"recovery gap journal failed: {type(exc).__name__}: {exc}", + ) + return False + state.recovery_gaps.append(summary) + if outcome == "recovered": + state.successful_recoveries += 1 + active_recovery_gap = None + pending_recovery_confirmation_attempt = None + return True + + def begin_recovery_gap() -> bool: + nonlocal active_recovery_gap, pending_recovery_confirmation_attempt + + with recovery_lock: + # A raw candidate from a client that has already failed can no + # longer prove the still-live scene. Keep the original outage open + # and fence that late downstream callback. + pending_recovery_confirmation_attempt = None + if active_recovery_gap is not None: + return True + gap_index = len(state.recovery_gaps) + 1 + started_at_utc = utc_now_iso() + started_monotonic_ns = time.monotonic_ns() + try: + writer.record_recovery_gap_event( + { + "schema_version": RECOVERY_GAP_SCHEMA_VERSION, + "record_type": "recovery_gap_started", + "gap_index": gap_index, + "started_at_utc": started_at_utc, + "started_monotonic_ns": started_monotonic_ns, + } + ) + except (OSError, RuntimeError, ValueError) as exc: + fail( + "capture_error", + f"recovery gap journal failed: {type(exc).__name__}: {exc}", + ) + return False + active_recovery_gap = (gap_index, started_at_utc, started_monotonic_ns) + return True + + def confirm_recovery(attempt: int) -> bool: + """Consume one exact post-publication proof for the current open gap.""" + + with recovery_lock: + if ( + isinstance(attempt, bool) + or attempt < 1 + or attempt != pending_recovery_confirmation_attempt + or active_recovery_gap is None + ): + return False + return finish_recovery_gap_locked("recovered", attempt) + + def fail_transport_handshake(reason: StopReason, message: str) -> None: + if recovering: + state.connected = False + state.subscribed = False + state.connection_lost = True + state.connection_lost_message = message + return + fail(reason, message) + def on_connect( callback_client: mqtt.Client, _userdata: object, @@ -811,129 +1024,347 @@ def capture_mqtt( reason_code: ReasonCode, _properties: Properties | None, ) -> None: + if callback_client is not active_client: + return if reason_code.is_failure: - fail("connection_failed", f"broker rejected connection: {reason_code}") + fail_transport_handshake( + "connection_failed", + f"broker rejected connection: {reason_code}", + ) return state.connected = True try: result, mid = callback_client.subscribe([(topic, 0) for topic in REPORT_TOPICS]) except (OSError, RuntimeError, ValueError) as exc: - fail( + fail_transport_handshake( "subscription_failed", f"subscribe failed: {type(exc).__name__}: {exc}", ) return if result != mqtt.MQTT_ERR_SUCCESS or mid is None: - fail("subscription_failed", f"subscribe failed: {mqtt.error_string(result)}") + fail_transport_handshake( + "subscription_failed", + f"subscribe failed: {mqtt.error_string(result)}", + ) return state.subscription_mid = mid def on_subscribe( - _callback_client: mqtt.Client, + callback_client: mqtt.Client, _userdata: object, mid: int, reason_codes: list[ReasonCode], _properties: Properties | None, ) -> None: + if callback_client is not active_client: + return if mid != state.subscription_mid: - fail("subscription_failed", f"unexpected SUBACK message id: {mid}") + fail_transport_handshake( + "subscription_failed", + f"unexpected SUBACK message id: {mid}", + ) return if len(reason_codes) != len(REPORT_TOPICS) or any( reason_code.is_failure for reason_code in reason_codes ): - fail("subscription_failed", "broker rejected one or more fixed subscriptions") + fail_transport_handshake( + "subscription_failed", + "broker rejected one or more fixed subscriptions", + ) return state.subscribed = True def on_message( - _callback_client: mqtt.Client, + callback_client: mqtt.Client, _userdata: object, message: mqtt.MQTTMessage, ) -> None: - if state.error is not None: + nonlocal pending_recovery_attempt, pending_recovery_confirmation_attempt, recovering + + if callback_client is not active_client or state.error is not None: return try: recorded = writer.record(message) except MessageTooLargeError as exc: fail("message_too_large", str(exc)) + return except (OSError, RuntimeError, ValueError) as exc: fail("capture_error", f"artifact write failed: {type(exc).__name__}: {exc}") return + recovery_candidate: tuple[int, int] | None = None + if ( + pending_recovery_attempt is not None + and not recorded.retain + and recorded.topic in RECOVERY_POINT_CLOUD_TOPICS + ): + recovered_attempt = pending_recovery_attempt + pending_recovery_attempt = None + recovering = False + with recovery_lock: + if active_recovery_gap is not None: + pending_recovery_confirmation_attempt = recovered_attempt + state.recovery_point_cloud_candidates += 1 + recovery_candidate = (recovered_attempt, recorded.sequence) + # Arm the provisional sequence fence after the raw frame is durable but + # before the preview queue can publish it on another thread. This is + # deliberately not a recovery-success edge: only a later normalized, + # non-empty Rerun publication may consume the candidate. + if recovery_candidate is not None and on_recovery_point_cloud_candidate is not None: + try: + on_recovery_point_cloud_candidate(*recovery_candidate) + except (OSError, RuntimeError, ValueError) as exc: + fail( + "capture_error", + f"recovery candidate callback failed: {type(exc).__name__}: {exc}", + ) + return if on_message_recorded is not None: try: on_message_recorded(recorded) except (OSError, RuntimeError, ValueError) as exc: fail("capture_error", f"preview callback failed: {type(exc).__name__}: {exc}") + return def on_disconnect( - _callback_client: mqtt.Client, + callback_client: mqtt.Client, _userdata: object, _flags: mqtt.DisconnectFlags, reason_code: ReasonCode, _properties: Properties | None, ) -> None: - if not state.stopping: - fail("connection_lost", f"broker connection ended: {reason_code}") + if callback_client is not active_client or state.stopping or disconnect_expected: + return + state.connected = False + state.subscribed = False + state.connection_lost = True + state.connection_lost_message = f"broker connection ended: {reason_code}" - client.on_connect = on_connect - client.on_subscribe = on_subscribe - client.on_message = on_message - client.on_disconnect = on_disconnect + def externally_stopped_or_elapsed() -> bool: + now = time.monotonic() + if should_stop is not None and should_stop(): + state.stop_reason = "external_stop" + return True + if ( + capture_started is not None + and duration_seconds is not None + and now - capture_started >= duration_seconds + ): + state.stop_reason = "duration_elapsed" + return True + return False + + def consume_owner_recovery_request() -> str | None: + if consume_connection_recovery_request is None: + return None + try: + reason = consume_connection_recovery_request() + except (OSError, RuntimeError, ValueError) as exc: + fail( + "capture_error", + "connection recovery request failed: " + f"{type(exc).__name__}: {exc}", + ) + return None + if reason is None: + return None + if not isinstance(reason, str) or not reason.strip(): + fail("capture_error", "connection recovery request returned an invalid reason") + return None + return reason.strip() + + def wait_recovery_backoff(attempt: int) -> bool: + delay = _recovery_backoff_seconds(attempt) + deadline = time.monotonic() + delay + while time.monotonic() < deadline: + if externally_stopped_or_elapsed(): + return False + time.sleep(min(0.1, max(deadline - time.monotonic(), 0.0))) + return True + + if on_recovery_confirmer_ready is not None: + try: + on_recovery_confirmer_ready(confirm_recovery) + except (OSError, RuntimeError, ValueError) as exc: + fail( + "capture_error", + f"recovery confirmer registration failed: {type(exc).__name__}: {exc}", + ) - connect_attempted = False try: - connect_attempted = True - connect_result = client.connect( - target_ipv4, - port=port, - keepalive=KEEPALIVE_SECONDS, - ) - if connect_result != mqtt.MQTT_ERR_SUCCESS: - fail("connection_failed", f"connect failed: {mqtt.error_string(connect_result)}") - while state.error is None: - now = time.monotonic() - if should_stop is not None and should_stop(): - state.stop_reason = "external_stop" + if recovering and externally_stopped_or_elapsed(): break - if state.subscribed and capture_started is None: - capture_started = now - if on_ready is not None: - on_ready() - if ( - capture_started is not None - and duration_seconds is not None - and now - capture_started >= duration_seconds - ): - state.stop_reason = "duration_elapsed" - break - if capture_started is None and now - operation_started >= CONNECT_TIMEOUT_SECONDS: - fail("connection_failed", "timed out waiting for CONNACK/SUBACK") - break - - loop_result = client.loop(timeout=LOOP_INTERVAL_SECONDS) + state.connected = False + state.subscribed = False + state.subscription_mid = None + state.connection_lost = False + state.connection_lost_message = None + client = make_client() + active_client = client + client.on_connect = on_connect + client.on_subscribe = on_subscribe + client.on_message = on_message + client.on_disconnect = on_disconnect + connect_attempted = False try: - writer.maybe_commit() - except OSError as exc: - fail("capture_error", f"group commit failed: {type(exc).__name__}: {exc}") + connect_attempted = True + connect_result = client.connect( + target_ipv4, + port=port, + keepalive=KEEPALIVE_SECONDS, + ) + if connect_result != mqtt.MQTT_ERR_SUCCESS: + message = f"connect failed: {mqtt.error_string(connect_result)}" + if recovering: + state.connection_lost = True + state.connection_lost_message = message + else: + fail("connection_failed", message) + + connect_started = time.monotonic() + while state.error is None and not state.connection_lost: + now = time.monotonic() + if externally_stopped_or_elapsed(): + break + if state.subscribed and capture_started is None: + capture_started = now + if on_ready is not None: + on_ready() + if not state.subscribed and now - connect_started >= CONNECT_TIMEOUT_SECONDS: + message = "timed out waiting for CONNACK/SUBACK" + if recovering: + state.connection_lost = True + state.connection_lost_message = message + else: + fail("connection_failed", message) + break + + loop_result = client.loop(timeout=LOOP_INTERVAL_SECONDS) + owner_recovery_reason = consume_owner_recovery_request() + try: + writer.maybe_commit() + except OSError as exc: + fail( + "capture_error", + f"group commit failed: {type(exc).__name__}: {exc}", + ) + break + if loop_result != mqtt.MQTT_ERR_SUCCESS and state.error is None: + state.connected = False + state.subscribed = False + state.connection_lost = True + state.connection_lost_message = ( + f"MQTT network loop failed: {mqtt.error_string(loop_result)}" + ) + if owner_recovery_reason is not None and not state.connection_lost: + state.connected = False + state.subscribed = False + state.connection_lost = True + state.connection_lost_message = ( + "guarded recovery requested by the connection owner: " + f"{owner_recovery_reason}" + ) + except KeyboardInterrupt: + state.stop_reason = "keyboard_interrupt" + except (OSError, RuntimeError, ValueError) as exc: + message = f"MQTT capture failed: {type(exc).__name__}: {exc}" + if recovering: + state.connection_lost = True + state.connection_lost_message = message + else: + fail("connection_failed", message) + finally: + disconnect_expected = True + active_client = None + if connect_attempted: + try: + client.disconnect() + except (OSError, RuntimeError, ValueError) as exc: + if state.error is None and not state.connection_lost: + fail( + "capture_error", + f"disconnect failed: {type(exc).__name__}: {exc}", + ) + disconnect_expected = False + + if ( + state.stop_reason + in { + "external_stop", + "duration_elapsed", + "keyboard_interrupt", + } + or state.error is not None + ): break - if loop_result != mqtt.MQTT_ERR_SUCCESS and state.error is None: + if not state.connection_lost: + break + # A guarded ``resume`` authorizes exactly one fresh MQTT client. + # If that client never delivers a fresh point-cloud report, consume the + # authorization before inspecting the device again; a later + # READY/SCAN_OVER must be able to end recovery without another + # data-plane connect. + pending_recovery_attempt = None + if not begin_recovery_gap(): + break + if recover_connection is None: fail( "connection_lost", - f"MQTT network loop failed: {mqtt.error_string(loop_result)}", + state.connection_lost_message or "MQTT connection was lost", ) - except KeyboardInterrupt: - state.stop_reason = "keyboard_interrupt" - except (OSError, RuntimeError, ValueError) as exc: - fail("connection_failed", f"MQTT capture failed: {type(exc).__name__}: {exc}") + break + if not recovering and on_connection_lost is not None: + on_connection_lost(state.connection_lost_message or "MQTT connection was lost") + recovering = True + + while state.error is None: + if externally_stopped_or_elapsed(): + break + state.recovery_attempts += 1 + attempt = state.recovery_attempts + try: + decision = recover_connection(attempt) + except (OSError, RuntimeError, ValueError): + decision = "retry" + if decision == "resume": + pending_recovery_attempt = attempt + break + if decision == "standby": + state.stop_reason = "recovery_standby" + break + if decision == "fault": + fail( + "connection_lost", + "exact active-stream recovery rejected the remote state", + ) + break + if decision == "blocked": + state.recovery_blocked = True + while not externally_stopped_or_elapsed(): + time.sleep(LOOP_INTERVAL_SECONDS) + break + if decision != "retry": + fail("capture_error", "invalid active-stream recovery decision") + break + if not wait_recovery_backoff(attempt): + break + + if pending_recovery_attempt is not None: + continue + break finally: state.stopping = True - if connect_attempted: - try: - client.disconnect() - except (OSError, RuntimeError, ValueError) as exc: - if state.error is None: - fail("capture_error", f"disconnect failed: {type(exc).__name__}: {exc}") + with recovery_lock: + if active_recovery_gap is not None: + if state.stop_reason == "recovery_standby": + gap_outcome: RecoveryGapOutcome = "standby" + elif state.recovery_blocked: + gap_outcome = "blocked" + elif state.error is not None: + gap_outcome = "fault" + else: + gap_outcome = "interrupted" + finish_recovery_gap_locked(gap_outcome, state.recovery_attempts) try: writer.close() except OSError as exc: @@ -958,6 +1389,7 @@ def capture_mqtt( created_at_utc=created_at_utc, capture_clock=capture_clock, state=state, + reconnect_enabled=recover_connection is not None, ) try: _write_summary_exclusive(writer.summary_path, summary) @@ -983,6 +1415,7 @@ def _build_summary( created_at_utc: str, capture_clock: CaptureClockEnvelope, state: _CaptureState, + reconnect_enabled: bool, ) -> CaptureSummary: capture_clock_origin = read_capture_clock_origin(writer.capture_clock_origin_path) return { @@ -996,7 +1429,12 @@ def _build_summary( "mqtt_protocol": "3.1.1", "subscription_qos": 0, "clean_session": True, - "reconnect_enabled": False, + "reconnect_enabled": reconnect_enabled, + "recovery_attempts": state.recovery_attempts, + "successful_recoveries": state.successful_recoveries, + "recovery_point_cloud_candidates": state.recovery_point_cloud_candidates, + "recovery_blocked": state.recovery_blocked, + "recovery_gaps": list(state.recovery_gaps), "publishing_enabled": False, "subscriptions": list(REPORT_TOPICS), "requested_duration_seconds": duration_seconds, @@ -1021,6 +1459,7 @@ def _build_summary( "artifacts": { "raw": writer.raw_path.name, "metadata_jsonl": writer.metadata_path.name, + "recovery_gaps_jsonl": writer.recovery_gaps_path.name, "capture_clock_origin": writer.capture_clock_origin_path.name, "capture_clock": writer.capture_clock_path.name, "summary": writer.summary_path.name, @@ -1028,6 +1467,7 @@ def _build_summary( "artifact_hashes": { "raw_sha256": _sha256_file(writer.raw_path), "metadata_jsonl_sha256": _sha256_file(writer.metadata_path), + "recovery_gaps_jsonl_sha256": _sha256_file(writer.recovery_gaps_path), "capture_clock_origin_sha256": capture_clock_origin.artifact_sha256, "capture_clock_sha256": capture_clock.artifact_sha256, }, diff --git a/src/k1link/device_plugins/xgrids_k1/network_mutation_ledger.py b/src/k1link/device_plugins/xgrids_k1/network_mutation_ledger.py new file mode 100644 index 0000000..247245a --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/network_mutation_ledger.py @@ -0,0 +1,958 @@ +from __future__ import annotations + +import fcntl +import ipaddress +import json +import os +import re +import stat +import tempfile +import threading +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, Literal, cast + +from k1link.sessions.store import resolve_missioncore_data_dir + +NETWORK_MUTATION_LEDGER_SCHEMA = "missioncore.xgrids-k1-network-mutation/v2" +NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA = "missioncore.xgrids-k1-network-mutation/v1" +NETWORK_MUTATION_LEDGER_FILENAME = "network-mutation.json" +NETWORK_MUTATION_LEDGER_LOCK_FILENAME = ".network-mutation.lock" +NETWORK_MUTATION_LEDGER_MAX_BYTES = 64 * 1024 + +NetworkConnectionMode = Literal["bridge", "quick-connect", "direct-connect"] +NetworkMutationStage = Literal["prepared", "dispatching", "observing", "resolved"] +NetworkMutationWriteMode = Literal["with_response", "without_response"] +NetworkMutationResolution = Literal[ + "not-dispatched", + "target-observed", + "interrupted", + "superseded", +] +NetworkMutationLedgerStatus = Literal["empty", "unresolved", "resolved", "corrupt"] + +_CONNECTION_MODES = frozenset({"bridge", "quick-connect", "direct-connect"}) +_STAGES = frozenset({"prepared", "dispatching", "observing", "resolved"}) +_WRITE_MODES = frozenset({"with_response", "without_response"}) +_RESOLUTIONS = frozenset( + {"not-dispatched", "target-observed", "interrupted", "superseded"} +) +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_SAFE_STATUS_MODE = re.compile(r"^[A-Z][A-Z0-9_-]{0,31}$") + + +class NetworkMutationLedgerError(RuntimeError): + """Base error for the durable K1 network-mutation fence.""" + + reason_code = "network-mutation-ledger-error" + + +class NetworkMutationBlocked(NetworkMutationLedgerError): + """A prior durable record prevents admission of another device write.""" + + reason_code = "network-mutation-reconciliation-required" + + +class NetworkMutationLedgerCorrupt(NetworkMutationBlocked): + """The durable fence cannot be trusted and therefore fails closed.""" + + reason_code = "network-mutation-ledger-corrupt" + + +class NetworkMutationTransitionError(NetworkMutationLedgerError): + """A caller attempted an invalid ledger state transition.""" + + reason_code = "network-mutation-ledger-transition-invalid" + + +@dataclass(frozen=True, slots=True) +class NetworkStatusEvidence: + """The bounded, non-secret subset of one decoded K1 7f02 status.""" + + mode: str | None + ipv4: str | None + status_code: int + reserved: int | None + + def __post_init__(self) -> None: + if self.mode is not None and _SAFE_STATUS_MODE.fullmatch(self.mode) is None: + raise ValueError("network status mode is outside the secret-free schema") + if self.ipv4 is not None: + try: + parsed = ipaddress.ip_address(self.ipv4) + except ValueError as exc: + raise ValueError("network status address must be an IPv4 address") from exc + if not isinstance(parsed, ipaddress.IPv4Address) or str(parsed) != self.ipv4: + raise ValueError("network status address must be canonical IPv4") + _validate_byte(self.status_code, field_name="status_code") + if self.reserved is not None: + _validate_byte(self.reserved, field_name="reserved") + + def as_dict(self) -> dict[str, object]: + return { + "mode": self.mode, + "ipv4": self.ipv4, + "status_code": self.status_code, + "reserved": self.reserved, + } + + +@dataclass(frozen=True, slots=True) +class PreviousConnectionEvidence: + """Last admitted topology retained as evidence, never as live reachability.""" + + transport_ref: str + mode: NetworkConnectionMode + ipv4: str | None + device_session_id: str | None + + def __post_init__(self) -> None: + _validate_identifier(self.transport_ref, field_name="transport_ref") + _validate_connection_mode(self.mode) + if self.ipv4 is not None: + try: + parsed = ipaddress.ip_address(self.ipv4) + except ValueError as exc: + raise ValueError("previous connection address must be an IPv4 address") from exc + if not isinstance(parsed, ipaddress.IPv4Address) or str(parsed) != self.ipv4: + raise ValueError("previous connection address must be canonical IPv4") + if self.device_session_id is not None: + _validate_identifier(self.device_session_id, field_name="device_session_id") + + def as_dict(self) -> dict[str, object]: + return { + "transport_ref": self.transport_ref, + "mode": self.mode, + "ipv4": self.ipv4, + "device_session_id": self.device_session_id, + } + + +@dataclass(frozen=True, slots=True) +class NetworkMutationRecord: + schema_version: str + revision: int + operation_id: str + transport_ref: str + intended_mode: NetworkConnectionMode + stage: NetworkMutationStage + write_mode: NetworkMutationWriteMode + baseline_status: NetworkStatusEvidence + previous_connection: PreviousConnectionEvidence | None + write_confirmed: bool | None + last_observation: NetworkStatusEvidence | None + resolution: NetworkMutationResolution | None + created_at_utc: str + updated_at_utc: str + + @property + def unresolved(self) -> bool: + return self.stage != "resolved" + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "revision": self.revision, + "operation_id": self.operation_id, + "transport_ref": self.transport_ref, + "intended_mode": self.intended_mode, + "stage": self.stage, + "write_mode": self.write_mode, + "baseline_status": self.baseline_status.as_dict(), + "previous_connection": ( + self.previous_connection.as_dict() if self.previous_connection is not None else None + ), + "write_confirmed": self.write_confirmed, + "last_observation": ( + self.last_observation.as_dict() if self.last_observation is not None else None + ), + "resolution": self.resolution, + "created_at_utc": self.created_at_utc, + "updated_at_utc": self.updated_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class NetworkMutationLedgerSnapshot: + status: NetworkMutationLedgerStatus + record: NetworkMutationRecord | None + reason_code: str | None + + @property + def mutation_allowed(self) -> bool: + return self.status in {"empty", "resolved"} + + +class NetworkMutationLedger: + """One durable, secret-free fence around K1 network side effects. + + The ledger is intentionally independent from the BLE helpers. Integration + writes ``dispatching`` durably immediately before entering + ``write_gatt_char``. A process crash can therefore create a conservative + false-positive fence, but can never silently authorize a second write. + """ + + def __init__( + self, + repository_root: Path, + *, + clock: Callable[[], datetime] | None = None, + ) -> None: + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = data_dir / "xgrids-k1" / NETWORK_MUTATION_LEDGER_FILENAME + self._process_lock_path = data_dir / "xgrids-k1" / NETWORK_MUTATION_LEDGER_LOCK_FILENAME + self._data_dir = data_dir + self._clock = clock or (lambda: datetime.now(UTC)) + self._lock = threading.RLock() + self._record: NetworkMutationRecord | None = None + self._corrupt = False + with self._lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> NetworkMutationLedgerSnapshot: + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return NetworkMutationLedgerSnapshot( + status="corrupt", + record=None, + reason_code=NetworkMutationLedgerCorrupt.reason_code, + ) + if self._record is None: + return NetworkMutationLedgerSnapshot(status="empty", record=None, reason_code=None) + return NetworkMutationLedgerSnapshot( + status="unresolved" if self._record.unresolved else "resolved", + record=self._record, + reason_code=( + NetworkMutationBlocked.reason_code if self._record.unresolved else None + ), + ) + + def require_mutation_allowed(self) -> None: + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_mutation_allowed_locked() + + def prepare( + self, + *, + operation_id: str, + transport_ref: str, + intended_mode: NetworkConnectionMode, + write_mode: NetworkMutationWriteMode, + baseline_status: NetworkStatusEvidence, + previous_connection: PreviousConnectionEvidence | None = None, + ) -> NetworkMutationRecord: + """Persist a pre-write record after the live baseline has been read.""" + + _validate_identifier(operation_id, field_name="operation_id") + _validate_identifier(transport_ref, field_name="transport_ref") + _validate_connection_mode(intended_mode) + _validate_write_mode(write_mode) + if not isinstance(baseline_status, NetworkStatusEvidence): + raise TypeError("baseline_status must be NetworkStatusEvidence") + if previous_connection is not None and not isinstance( + previous_connection, PreviousConnectionEvidence + ): + raise TypeError("previous_connection must be PreviousConnectionEvidence") + + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_mutation_allowed_locked() + previous_revision = self._record.revision if self._record is not None else 0 + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=(self._record.updated_at_utc if self._record is not None else None), + ) + record = NetworkMutationRecord( + schema_version=NETWORK_MUTATION_LEDGER_SCHEMA, + revision=previous_revision + 1, + operation_id=operation_id, + transport_ref=transport_ref, + intended_mode=intended_mode, + stage="prepared", + write_mode=write_mode, + baseline_status=baseline_status, + previous_connection=previous_connection, + write_confirmed=None, + last_observation=None, + resolution=None, + created_at_utc=now, + updated_at_utc=now, + ) + self._persist_locked(record) + return record + + def mark_dispatching( + self, + operation_id: str, + *, + expected_revision: int, + ) -> NetworkMutationRecord: + """Durably cross the side-effect boundary before the BLE write call.""" + + _positive_int(expected_revision, field_name="expected_revision") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + self._require_expected_revision(current, expected_revision) + if current.stage != "prepared": + raise NetworkMutationTransitionError( + "network mutation may dispatch only from prepared" + ) + return self._transition_locked(current, stage="dispatching") + + def confirm_dispatching_after_uncertain_return( + self, + operation_id: str, + *, + expected_prepared: NetworkMutationRecord, + ) -> NetworkMutationRecord: + """Re-fsync one exact DISPATCHING commit after its return path failed. + + Reading a replaced file is not sufficient proof that its directory + entry reached stable storage. This recovery primitive verifies the + complete immutable predecessor/operation tuple under the store lock, + then republishes the identical record through the normal file+parent + fsync path. It never advances revision or creates dispatch authority + for a merely PREPARED row. + """ + + _validate_identifier(operation_id, field_name="operation_id") + if not isinstance(expected_prepared, NetworkMutationRecord): + raise TypeError("expected_prepared must be NetworkMutationRecord") + if ( + expected_prepared.operation_id != operation_id + or expected_prepared.stage != "prepared" + or expected_prepared.write_confirmed is not None + or expected_prepared.last_observation is not None + or expected_prepared.resolution is not None + ): + raise NetworkMutationTransitionError( + "dispatch confirmation requires the exact PREPARED predecessor" + ) + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt or self._record is None: + raise NetworkMutationLedgerCorrupt( + "uncertain network dispatch record cannot be trusted" + ) + current = self._record + if not ( + current.operation_id == expected_prepared.operation_id + and current.transport_ref == expected_prepared.transport_ref + and current.intended_mode == expected_prepared.intended_mode + and current.write_mode == expected_prepared.write_mode + and current.baseline_status == expected_prepared.baseline_status + and current.previous_connection == expected_prepared.previous_connection + and current.created_at_utc == expected_prepared.created_at_utc + and current.stage == "dispatching" + and current.revision == expected_prepared.revision + 1 + and current.write_confirmed is None + and current.last_observation is None + and current.resolution is None + ): + raise NetworkMutationTransitionError( + "uncertain network dispatch does not match its predecessor" + ) + self._persist_locked(current) + return current + + def mark_observing( + self, + operation_id: str, + *, + expected_revision: int, + write_confirmed: bool, + observation: NetworkStatusEvidence | None = None, + ) -> NetworkMutationRecord: + """Record transport acknowledgement and bounded post-write evidence.""" + + _positive_int(expected_revision, field_name="expected_revision") + if not isinstance(write_confirmed, bool): + raise TypeError("write_confirmed must be bool") + if observation is not None and not isinstance(observation, NetworkStatusEvidence): + raise TypeError("observation must be NetworkStatusEvidence") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + self._require_expected_revision(current, expected_revision) + if current.stage not in {"dispatching", "observing"}: + raise NetworkMutationTransitionError( + "network mutation may observe only after dispatch" + ) + if current.write_confirmed is True and not write_confirmed: + raise NetworkMutationTransitionError( + "network mutation write confirmation cannot regress" + ) + return self._transition_locked( + current, + stage="observing", + write_confirmed=write_confirmed, + last_observation=observation, + ) + + def resolve( + self, + operation_id: str, + *, + expected_revision: int, + resolution: NetworkMutationResolution, + observation: NetworkStatusEvidence | None = None, + ) -> NetworkMutationRecord: + """Terminalize one mutation without authorizing an automatic retry. + + ``interrupted`` and ``superseded`` are audit outcomes. They make no + claim about the device-side result of a previously dispatched write; + they only state that the old host session no longer owns the next + explicit operator action. + """ + + _positive_int(expected_revision, field_name="expected_revision") + _validate_resolution(resolution) + if observation is not None and not isinstance(observation, NetworkStatusEvidence): + raise TypeError("observation must be NetworkStatusEvidence") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + self._require_expected_revision(current, expected_revision) + if current.stage == "resolved": + if current.resolution == resolution: + return current + raise NetworkMutationTransitionError( + "resolved network mutation cannot change its resolution" + ) + if resolution == "not-dispatched": + if current.stage != "prepared": + raise NetworkMutationTransitionError( + "not-dispatched resolution requires prepared stage" + ) + if observation is not None: + raise NetworkMutationTransitionError( + "not-dispatched resolution cannot attach post-write evidence" + ) + elif resolution == "target-observed": + if current.stage not in {"dispatching", "observing"}: + raise NetworkMutationTransitionError( + "target-observed resolution requires a dispatched mutation" + ) + observation = observation or current.last_observation + if observation is None: + raise NetworkMutationTransitionError( + "target-observed resolution requires bounded status evidence" + ) + else: + if current.stage not in {"dispatching", "observing"}: + raise NetworkMutationTransitionError( + f"{resolution} resolution requires a dispatched mutation" + ) + # Session termination is not device-state evidence. Retain a + # previously captured bounded observation when one exists, but + # never require a read-only reconciliation before allowing a + # later explicit action. + observation = observation or current.last_observation + return self._transition_locked( + current, + stage="resolved", + last_observation=observation, + resolution=resolution, + ) + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + """Serialize one complete ledger transaction across local processes. + + Atomic replacement protects the JSON from partial publication, but it + does not make the preceding read/check/write sequence atomic. A stable, + separately opened lock inode fences that whole sequence so two backend + processes cannot both admit a device mutation from the same revision. + """ + + # The shared Mission Core data root may predate this plugin and is + # normalized by the same helper used by atomic publication. The + # ledger-specific directory must already be private or fail closed. + data_dir_created = _ensure_private_directory(self._data_dir, parents=True) + parent_created = _ensure_private_lock_directory(self.path.parent, parents=True) + if data_dir_created or parent_created: + _fsync_directory(self._data_dir) + + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self._process_lock_path, flags, 0o600) + except OSError as exc: + raise NetworkMutationLedgerCorrupt( + "network mutation ledger lock cannot be opened safely" + ) from exc + lock_stream: IO[bytes] | None = None + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600: + raise NetworkMutationLedgerCorrupt( + "network mutation ledger lock is not a private regular file" + ) + lock_stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(lock_stream.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(lock_stream.fileno(), fcntl.LOCK_UN) + finally: + if lock_stream is not None: + lock_stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _current_operation_locked(self, operation_id: str) -> NetworkMutationRecord: + _validate_identifier(operation_id, field_name="operation_id") + self._reload_locked() + if self._corrupt: + raise NetworkMutationLedgerCorrupt( + "network mutation ledger is corrupt; device writes remain blocked" + ) + current = self._record + if current is None or current.operation_id != operation_id: + raise NetworkMutationTransitionError("network mutation operation does not match ledger") + return current + + def _require_mutation_allowed_locked(self) -> None: + if self._corrupt: + raise NetworkMutationLedgerCorrupt( + "network mutation ledger is corrupt; device writes remain blocked" + ) + if self._record is not None and self._record.unresolved: + raise NetworkMutationBlocked( + "previous network mutation is unresolved; another device write is blocked" + ) + + @staticmethod + def _require_expected_revision( + record: NetworkMutationRecord, + expected_revision: int, + ) -> None: + if record.revision != expected_revision: + raise NetworkMutationTransitionError( + "network mutation transition used a stale record revision" + ) + + def _transition_locked( + self, + current: NetworkMutationRecord, + *, + stage: NetworkMutationStage, + write_confirmed: bool | None = None, + last_observation: NetworkStatusEvidence | None = None, + resolution: NetworkMutationResolution | None = None, + ) -> NetworkMutationRecord: + record = replace( + current, + revision=current.revision + 1, + stage=stage, + write_confirmed=( + write_confirmed if write_confirmed is not None else current.write_confirmed + ), + last_observation=( + last_observation if last_observation is not None else current.last_observation + ), + resolution=resolution, + updated_at_utc=_nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ), + ) + self._persist_locked(record) + return record + + def _persist_locked(self, record: NetworkMutationRecord) -> None: + _write_private_json_atomic( + self.path, + record.as_dict(), + data_dir=self._data_dir, + ) + self._record = record + self._corrupt = False + + def _reload_locked(self) -> None: + try: + metadata = self.path.lstat() + except FileNotFoundError: + self._record = None + self._corrupt = False + return + except OSError: + self._record = None + self._corrupt = True + return + try: + if not stat.S_ISREG(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError("ledger file is not a private regular file") + parent_metadata = self.path.parent.lstat() + if ( + not stat.S_ISDIR(parent_metadata.st_mode) + or stat.S_IMODE(parent_metadata.st_mode) != 0o700 + ): + raise ValueError("ledger directory is not private") + if metadata.st_size > NETWORK_MUTATION_LEDGER_MAX_BYTES: + raise ValueError("ledger file exceeds the bounded size") + payload = json.loads( + self.path.read_text(encoding="utf-8"), + object_pairs_hook=_unique_json_object, + ) + legacy_schema = ( + isinstance(payload, dict) + and payload.get("schema_version") == NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA + ) + record = _record_from_mapping(payload) + if legacy_schema: + # v1 was briefly published with two shapes: the original + # previous_connection lacked transport_ref, while the final + # in-tree shape already carried it. Only the latter can be + # migrated without inventing which physical K1 owned the + # previous topology. _record_from_mapping deliberately + # rejects the ambiguous shape and preserves it fail-closed. + _write_private_json_atomic( + self.path, + record.as_dict(), + data_dir=self._data_dir, + ) + except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): + self._record = None + self._corrupt = True + return + self._record = record + self._corrupt = False + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = ( + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + if len(serialized) > NETWORK_MUTATION_LEDGER_MAX_BYTES: + raise ValueError("network mutation ledger exceeds the bounded size") + + data_dir_created = _ensure_private_directory(data_dir, parents=True) + parent_created = _ensure_private_directory(path.parent, parents=True) + if data_dir_created or parent_created: + _fsync_directory(data_dir) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temp_path, path) + path.chmod(0o600) + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + # A peer process may have created the shared private directory + # between lstat and mkdir. Validate that winner below instead of + # failing a safe concurrent ledger acquisition. + metadata = path.lstat() + else: + path.chmod(0o700) + return True + if not stat.S_ISDIR(metadata.st_mode): + raise NetworkMutationLedgerCorrupt( + "network mutation ledger directory is not a private directory" + ) + path.chmod(0o700) + return False + + +def _ensure_private_lock_directory(path: Path, *, parents: bool) -> bool: + """Create a lock parent privately or reject an unsafe existing parent.""" + + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + return True + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + raise NetworkMutationLedgerCorrupt("network mutation ledger lock directory is not private") + return False + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("network mutation ledger contains duplicate fields") + document[key] = value + return document + + +def _record_from_mapping(value: object) -> NetworkMutationRecord: + document = _exact_mapping( + value, + { + "schema_version", + "revision", + "operation_id", + "transport_ref", + "intended_mode", + "stage", + "write_mode", + "baseline_status", + "previous_connection", + "write_confirmed", + "last_observation", + "resolution", + "created_at_utc", + "updated_at_utc", + }, + label="ledger", + ) + if document["schema_version"] not in { + NETWORK_MUTATION_LEDGER_SCHEMA, + NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA, + }: + raise ValueError("unsupported network mutation ledger schema") + revision = _positive_int(document["revision"], field_name="revision") + operation_id = _required_string(document["operation_id"], field_name="operation_id") + transport_ref = _required_string(document["transport_ref"], field_name="transport_ref") + _validate_identifier(operation_id, field_name="operation_id") + _validate_identifier(transport_ref, field_name="transport_ref") + + intended_mode_raw = _required_string(document["intended_mode"], field_name="intended_mode") + _validate_connection_mode(intended_mode_raw) + intended_mode = cast(NetworkConnectionMode, intended_mode_raw) + stage_raw = _required_string(document["stage"], field_name="stage") + if stage_raw not in _STAGES: + raise ValueError("unsupported network mutation stage") + stage = cast(NetworkMutationStage, stage_raw) + write_mode_raw = _required_string(document["write_mode"], field_name="write_mode") + _validate_write_mode(write_mode_raw) + write_mode = cast(NetworkMutationWriteMode, write_mode_raw) + baseline_status = _status_from_mapping(document["baseline_status"]) + + previous_raw = document["previous_connection"] + previous_connection = ( + None if previous_raw is None else _previous_connection_from_mapping(previous_raw) + ) + write_confirmed_raw = document["write_confirmed"] + if write_confirmed_raw is not None and not isinstance(write_confirmed_raw, bool): + raise ValueError("write_confirmed must be bool or null") + observation_raw = document["last_observation"] + last_observation = None if observation_raw is None else _status_from_mapping(observation_raw) + resolution_raw = document["resolution"] + if resolution_raw is None: + resolution = None + else: + resolution_string = _required_string(resolution_raw, field_name="resolution") + _validate_resolution(resolution_string) + resolution = cast(NetworkMutationResolution, resolution_string) + created_at = _validated_timestamp(document["created_at_utc"], field_name="created_at_utc") + updated_at = _validated_timestamp(document["updated_at_utc"], field_name="updated_at_utc") + if updated_at < created_at: + raise ValueError("ledger update precedes creation") + + if stage == "prepared": + if ( + write_confirmed_raw is not None + or last_observation is not None + or resolution is not None + ): + raise ValueError("prepared ledger contains post-write fields") + elif stage in {"dispatching", "observing"}: + if resolution is not None: + raise ValueError("unresolved ledger contains a resolution") + if stage == "dispatching" and ( + write_confirmed_raw is not None or last_observation is not None + ): + raise ValueError("dispatching ledger contains observation fields") + if stage == "observing" and write_confirmed_raw is None: + raise ValueError("observing ledger lacks write acknowledgement status") + else: + if resolution is None: + raise ValueError("resolved ledger lacks a resolution") + if resolution == "not-dispatched" and ( + write_confirmed_raw is not None or last_observation is not None + ): + raise ValueError("not-dispatched resolution contains post-write fields") + if resolution == "target-observed" and last_observation is None: + raise ValueError("target-observed resolution lacks status evidence") + + return NetworkMutationRecord( + schema_version=NETWORK_MUTATION_LEDGER_SCHEMA, + revision=revision, + operation_id=operation_id, + transport_ref=transport_ref, + intended_mode=intended_mode, + stage=stage, + write_mode=write_mode, + baseline_status=baseline_status, + previous_connection=previous_connection, + write_confirmed=write_confirmed_raw, + last_observation=last_observation, + resolution=resolution, + created_at_utc=_timestamp(created_at), + updated_at_utc=_timestamp(updated_at), + ) + + +def _status_from_mapping(value: object) -> NetworkStatusEvidence: + document = _exact_mapping( + value, + {"mode", "ipv4", "status_code", "reserved"}, + label="network status", + ) + mode = _optional_string(document["mode"], field_name="mode") + ipv4 = _optional_string(document["ipv4"], field_name="ipv4") + status_code = _byte(document["status_code"], field_name="status_code") + reserved_raw = document["reserved"] + reserved = None if reserved_raw is None else _byte(reserved_raw, field_name="reserved") + return NetworkStatusEvidence( + mode=mode, + ipv4=ipv4, + status_code=status_code, + reserved=reserved, + ) + + +def _previous_connection_from_mapping(value: object) -> PreviousConnectionEvidence: + document = _exact_mapping( + value, + {"transport_ref", "mode", "ipv4", "device_session_id"}, + label="previous connection", + ) + mode_raw = _required_string(document["mode"], field_name="mode") + _validate_connection_mode(mode_raw) + return PreviousConnectionEvidence( + transport_ref=_required_string(document["transport_ref"], field_name="transport_ref"), + mode=cast(NetworkConnectionMode, mode_raw), + ipv4=_optional_string(document["ipv4"], field_name="ipv4"), + device_session_id=_optional_string( + document["device_session_id"], field_name="device_session_id" + ), + ) + + +def _exact_mapping(value: object, keys: set[str], *, label: str) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"{label} does not match the secret-free schema") + return cast(Mapping[str, object], value) + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _validate_connection_mode(value: str) -> None: + if value not in _CONNECTION_MODES: + raise ValueError("unsupported network connection mode") + + +def _validate_write_mode(value: str) -> None: + if value not in _WRITE_MODES: + raise ValueError("unsupported BLE write mode") + + +def _validate_resolution(value: str) -> None: + if value not in _RESOLUTIONS: + raise ValueError("unsupported network mutation resolution") + + +def _validate_byte(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 255: + raise ValueError(f"{field_name} must be an unsigned byte") + + +def _byte(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 255: + raise ValueError(f"{field_name} must be an unsigned byte") + return value + + +def _positive_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{field_name} must be a positive integer") + return value + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _optional_string(value: object, *, field_name: str) -> str | None: + if value is None: + return None + return _required_string(value, field_name=field_name) + + +def _validated_timestamp(value: object, *, field_name: str) -> datetime: + raw = _required_string(value, field_name=field_name) + if not raw.endswith("Z"): + raise ValueError(f"{field_name} must be UTC") + try: + parsed = datetime.fromisoformat(raw.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise ValueError(f"{field_name} is invalid") from exc + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): + raise ValueError(f"{field_name} must be UTC") + return parsed.astimezone(UTC) + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("network mutation ledger clock must be timezone-aware") + return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _nondecreasing_audit_timestamp(value: datetime, *, floor: str | None) -> str: + """Canonicalize wall time without using it as transition authority. + + Serialized revision and stage checks order ledger transitions. A host clock + may move backwards (NTP correction, RTC repair, suspend/resume), so the + human-readable audit timestamp is clamped to the prior durable value rather + than allowing a valid transition to publish a record that fails its own + ``updated >= created`` structural check after restart. + """ + + candidate = _validated_timestamp(_timestamp(value), field_name="ledger clock") + if floor is None: + return _timestamp(candidate) + floor_value = _validated_timestamp(floor, field_name="audit timestamp floor") + return _timestamp(max(candidate, floor_value)) diff --git a/src/k1link/device_plugins/xgrids_k1/network_provisioning_idempotency_journal.py b/src/k1link/device_plugins/xgrids_k1/network_provisioning_idempotency_journal.py new file mode 100644 index 0000000..38789ae --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/network_provisioning_idempotency_journal.py @@ -0,0 +1,1153 @@ +from __future__ import annotations + +import fcntl +import hashlib +import hmac +import json +import os +import re +import stat +import tempfile +import threading +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, Literal, cast + +from k1link.sessions.store import resolve_missioncore_data_dir + +NETWORK_PROVISIONING_IDEMPOTENCY_SCHEMA = ( + "missioncore.xgrids-k1-network-provisioning-idempotency/v1" +) +NETWORK_PROVISIONING_IDEMPOTENCY_FILENAME = "network-provisioning-idempotency.json" +NETWORK_PROVISIONING_IDEMPOTENCY_LOCK_FILENAME = ".network-provisioning-idempotency.lock" +NETWORK_PROVISIONING_IDEMPOTENCY_MAX_BYTES = 256 * 1024 +NETWORK_PROVISIONING_IDEMPOTENCY_MAX_TERMINAL_RECORDS = 64 +NETWORK_PROVISIONING_IDEMPOTENCY_HARD_MAX_RECORDS = 256 +NETWORK_PROVISIONING_LEGACY_ADOPTION_PROVENANCE_SCHEMA = ( + "missioncore.xgrids-k1-network-provisioning-legacy-adoption/v1" +) +NETWORK_PROVISIONING_LEGACY_SOURCE_SCHEMA = ( + "missioncore.xgrids-k1-network-mutation/v2" +) + +NetworkProvisioningIdempotencyStage = Literal["prepared", "unresolved", "terminal"] +NetworkProvisioningIdempotencyDisposition = Literal["admitted", "terminal-replay"] +NetworkProvisioningIdempotencyStatus = Literal["empty", "ready", "blocked", "corrupt"] +NetworkProvisioningTerminalOutcome = Literal["succeeded", "failed", "cancelled"] +NetworkProvisioningSideEffectStatus = Literal["none", "applied", "reconciled"] + +_STAGES = frozenset({"prepared", "unresolved", "terminal"}) +_OUTCOMES = frozenset({"succeeded", "failed", "cancelled"}) +_SIDE_EFFECT_STATUSES = frozenset({"none", "applied", "reconciled"}) +_SAFE_ACTION = re.compile(r"^[a-z][a-z0-9._-]{0,95}$") +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_SAFE_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_MAX_REVISION = (1 << 63) - 1 + + +class NetworkProvisioningIdempotencyError(RuntimeError): + """Base error for the durable network-provisioning request fence.""" + + reason_code = "network-provisioning-idempotency-error" + + +class NetworkProvisioningIdempotencyConflict(NetworkProvisioningIdempotencyError): + """A durable key or operation id is already bound to another request.""" + + reason_code = "network-provisioning-idempotency-conflict" + + +class NetworkProvisioningIdempotencyBlocked(NetworkProvisioningIdempotencyError): + """A live concurrent request owns the single network side-effect slot.""" + + reason_code = "network-provisioning-idempotency-reconciliation-required" + + def __init__( + self, + message: str, + *, + record: NetworkProvisioningIdempotencyRecord | None = None, + ) -> None: + super().__init__(message) + self.record = record + + +class NetworkProvisioningIdempotencyCorrupt(NetworkProvisioningIdempotencyBlocked): + """The durable journal is untrusted and therefore fails closed.""" + + reason_code = "network-provisioning-idempotency-corrupt" + + +class NetworkProvisioningIdempotencyTransitionError(NetworkProvisioningIdempotencyError): + """A stale or invalid journal transition was requested.""" + + reason_code = "network-provisioning-idempotency-transition-invalid" + + +@dataclass(frozen=True, slots=True) +class NetworkProvisioningLegacyAdoptionProvenance: + """Immutable, secret-free identity of one pre-idempotency ledger mutation. + + The legacy mutation ledger already contains only bounded evidence. The + nested evidence is supplied here as canonical SHA-256 digests so this + migration API cannot accidentally become a persistence path for an SSID, + password, or vendor payload. Mutable ledger state (stage, observation and + revision) is intentionally excluded: adopting the same physical mutation + from ``dispatching`` or ``observing`` must derive the same journal identity. + """ + + source_schema_version: str + operation_id: str + transport_ref: str + intended_mode: str + write_mode: str + baseline_status_sha256: str + previous_connection_sha256: str | None + created_at_utc: str + + def __post_init__(self) -> None: + if self.source_schema_version != NETWORK_PROVISIONING_LEGACY_SOURCE_SCHEMA: + raise ValueError("unsupported legacy network mutation schema") + _validate_identifier(self.operation_id, field_name="operation_id") + _validate_identifier(self.transport_ref, field_name="transport_ref") + _validate_safe_code(self.intended_mode, field_name="intended_mode") + _validate_safe_code(self.write_mode, field_name="write_mode") + _validate_sha256( + self.baseline_status_sha256, + field_name="baseline_status_sha256", + ) + if self.previous_connection_sha256 is not None: + _validate_sha256( + self.previous_connection_sha256, + field_name="previous_connection_sha256", + ) + _validated_timestamp(self.created_at_utc, field_name="created_at_utc") + + def canonical_bytes(self) -> bytes: + return json.dumps( + { + "schema_version": NETWORK_PROVISIONING_LEGACY_ADOPTION_PROVENANCE_SCHEMA, + "source_schema_version": self.source_schema_version, + "operation_id": self.operation_id, + "transport_ref": self.transport_ref, + "intended_mode": self.intended_mode, + "write_mode": self.write_mode, + "baseline_status_sha256": self.baseline_status_sha256, + "previous_connection_sha256": self.previous_connection_sha256, + "created_at_utc": self.created_at_utc, + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + + +@dataclass(frozen=True, slots=True) +class NetworkProvisioningTerminalMetadata: + """Bounded replay metadata that cannot carry credentials or vendor payloads.""" + + outcome: NetworkProvisioningTerminalOutcome + outcome_code: str + error_code: str | None + side_effect_status: NetworkProvisioningSideEffectStatus + retryable: bool + safe_to_retry: bool + + def __post_init__(self) -> None: + if self.outcome not in _OUTCOMES: + raise ValueError("unsupported network provisioning terminal outcome") + _validate_safe_code(self.outcome_code, field_name="outcome_code") + if self.error_code is not None: + _validate_safe_code(self.error_code, field_name="error_code") + if self.side_effect_status not in _SIDE_EFFECT_STATUSES: + raise ValueError("unsupported network provisioning side-effect status") + if not isinstance(self.retryable, bool) or not isinstance(self.safe_to_retry, bool): + raise TypeError("retryability fields must be bool") + if self.outcome == "succeeded" and self.error_code is not None: + raise ValueError("successful terminal metadata cannot carry an error code") + if self.outcome != "succeeded" and self.error_code is None: + raise ValueError("failed or cancelled terminal metadata requires an error code") + if self.safe_to_retry and (not self.retryable or self.side_effect_status != "none"): + raise ValueError("safe retry requires a retryable no-side-effect outcome") + + def as_dict(self) -> dict[str, object]: + return { + "outcome": self.outcome, + "outcome_code": self.outcome_code, + "error_code": self.error_code, + "side_effect_status": self.side_effect_status, + "retryable": self.retryable, + "safe_to_retry": self.safe_to_retry, + } + + +@dataclass(frozen=True, slots=True) +class NetworkProvisioningIdempotencyRecord: + idempotency_key_sha256: str + action: str + operation_id: str + request_binding_sha256: str + stage: NetworkProvisioningIdempotencyStage + terminal: NetworkProvisioningTerminalMetadata | None + created_revision: int + revision: int + previous_revision: int | None + created_at_utc: str + updated_at_utc: str + + @property + def reconciliation_required(self) -> bool: + return self.stage != "terminal" + + def as_dict(self) -> dict[str, object]: + return { + "idempotency_key_sha256": self.idempotency_key_sha256, + "action": self.action, + "operation_id": self.operation_id, + "request_binding_sha256": self.request_binding_sha256, + "stage": self.stage, + "terminal": self.terminal.as_dict() if self.terminal is not None else None, + "created_revision": self.created_revision, + "revision": self.revision, + "previous_revision": self.previous_revision, + "created_at_utc": self.created_at_utc, + "updated_at_utc": self.updated_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class NetworkProvisioningIdempotencyAdmission: + disposition: NetworkProvisioningIdempotencyDisposition + record: NetworkProvisioningIdempotencyRecord + + @property + def created(self) -> bool: + return self.disposition == "admitted" + + +@dataclass(frozen=True, slots=True) +class NetworkProvisioningIdempotencySnapshot: + status: NetworkProvisioningIdempotencyStatus + revision: int + records: tuple[NetworkProvisioningIdempotencyRecord, ...] + reason_code: str | None + + @property + def mutation_allowed(self) -> bool: + return self.status in {"empty", "ready"} + + @property + def active_record(self) -> NetworkProvisioningIdempotencyRecord | None: + return next((record for record in self.records if record.reconciliation_required), None) + + +@dataclass(frozen=True, slots=True) +class _JournalDocument: + revision: int + previous_revision: int | None + records: tuple[NetworkProvisioningIdempotencyRecord, ...] + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": NETWORK_PROVISIONING_IDEMPOTENCY_SCHEMA, + "revision": self.revision, + "previous_revision": self.previous_revision, + "records": [record.as_dict() for record in self.records], + } + + +def derive_request_binding_sha256( + idempotency_key: str, + *, + action: str, + canonical_request: bytes, +) -> str: + """Derive a restart-stable, privacy-safe binding for the complete request. + + ``canonical_request`` may contain request secrets transiently. It is never + persisted by this module. HMAC binds the digest to the request's own + idempotency key, so the durable value is not a plain password/SSID hash. + Callers must use a stable canonical encoding and a high-entropy key. + """ + + key = _idempotency_key_bytes(idempotency_key) + _validate_action(action) + if not isinstance(canonical_request, bytes): + raise TypeError("canonical_request must be bytes") + derived_key = hashlib.sha256( + b"missioncore.network-provisioning.idempotency-key/v1\0" + key + ).digest() + payload = ( + b"missioncore.network-provisioning.request-binding/v1\0" + + action.encode("ascii") + + b"\0" + + canonical_request + ) + return hmac.new(derived_key, payload, hashlib.sha256).hexdigest() + + +class NetworkProvisioningIdempotencyJournal: + """Durable, bounded idempotency and audit journal for K1 network writes. + + Admission publishes ``prepared`` before any side effect. Integration must + publish ``unresolved`` immediately before entering a BLE write. Any + non-terminal record fences concurrent admission while its process owns the + lifecycle lease. Restart/session recovery terminalizes abandoned records; + no read-only reconciliation is required before a new explicit operator + action. Terminal records are replayable by the same key/action/request + binding across restarts and backend processes. + """ + + def __init__( + self, + repository_root: Path, + *, + max_terminal_records: int = NETWORK_PROVISIONING_IDEMPOTENCY_MAX_TERMINAL_RECORDS, + clock: Callable[[], datetime] | None = None, + ) -> None: + if not 1 <= max_terminal_records <= NETWORK_PROVISIONING_IDEMPOTENCY_HARD_MAX_RECORDS: + raise ValueError("max_terminal_records is outside the bounded range") + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = data_dir / "xgrids-k1" / NETWORK_PROVISIONING_IDEMPOTENCY_FILENAME + self._process_lock_path = ( + data_dir / "xgrids-k1" / NETWORK_PROVISIONING_IDEMPOTENCY_LOCK_FILENAME + ) + self._data_dir = data_dir + self._max_terminal_records = max_terminal_records + self._clock = clock or (lambda: datetime.now(UTC)) + self._lock = threading.RLock() + self._document = _JournalDocument(revision=0, previous_revision=None, records=()) + self._corrupt = False + with self._lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> NetworkProvisioningIdempotencySnapshot: + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return NetworkProvisioningIdempotencySnapshot( + status="corrupt", + revision=0, + records=(), + reason_code=NetworkProvisioningIdempotencyCorrupt.reason_code, + ) + records = self._document.records + active = next((record for record in records if record.reconciliation_required), None) + if active is not None: + status: NetworkProvisioningIdempotencyStatus = "blocked" + reason_code = NetworkProvisioningIdempotencyBlocked.reason_code + elif records: + status = "ready" + reason_code = None + else: + status = "empty" + reason_code = None + return NetworkProvisioningIdempotencySnapshot( + status=status, + revision=self._document.revision, + records=records, + reason_code=reason_code, + ) + + def begin( + self, + *, + idempotency_key: str, + action: str, + operation_id: str, + request_binding_sha256: str, + ) -> NetworkProvisioningIdempotencyAdmission: + """Admit one new request or replay its exact terminal predecessor.""" + + key_digest = hashlib.sha256(_idempotency_key_bytes(idempotency_key)).hexdigest() + _validate_action(action) + _validate_identifier(operation_id, field_name="operation_id") + _validate_sha256(request_binding_sha256, field_name="request_binding_sha256") + + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_trusted_locked() + matching_key = next( + ( + record + for record in self._document.records + if record.idempotency_key_sha256 == key_digest + ), + None, + ) + if matching_key is not None: + if ( + matching_key.action != action + or matching_key.request_binding_sha256 != request_binding_sha256 + ): + raise NetworkProvisioningIdempotencyConflict( + "idempotency key is already bound to another request" + ) + if matching_key.stage != "terminal": + raise NetworkProvisioningIdempotencyBlocked( + "idempotent network request is non-terminal; reconciliation is required", + record=matching_key, + ) + return NetworkProvisioningIdempotencyAdmission( + disposition="terminal-replay", + record=matching_key, + ) + + matching_operation = next( + ( + record + for record in self._document.records + if record.operation_id == operation_id + ), + None, + ) + if matching_operation is not None: + raise NetworkProvisioningIdempotencyConflict( + "operation id is already bound to another idempotency key" + ) + active = next( + (record for record in self._document.records if record.reconciliation_required), + None, + ) + if active is not None: + raise NetworkProvisioningIdempotencyBlocked( + "another network request is non-terminal; reconciliation is required", + record=active, + ) + + revision = self._next_revision_locked() + now = _timestamp(self._clock()) + record = NetworkProvisioningIdempotencyRecord( + idempotency_key_sha256=key_digest, + action=action, + operation_id=operation_id, + request_binding_sha256=request_binding_sha256, + stage="prepared", + terminal=None, + created_revision=revision, + revision=revision, + previous_revision=(self._document.revision or None), + created_at_utc=now, + updated_at_utc=now, + ) + records = (*self._document.records, record) + self._persist_next_locked(records) + return NetworkProvisioningIdempotencyAdmission( + disposition="admitted", + record=record, + ) + + def adopt_legacy_unresolved( + self, + *, + action: str, + provenance: NetworkProvisioningLegacyAdoptionProvenance, + ) -> NetworkProvisioningIdempotencyRecord: + """Atomically fence one dispatched legacy mutation as unresolved. + + This is a recovery-only bridge for network-ledger v2 records created + before the independent idempotency journal existed. It deliberately + publishes ``unresolved`` in the first durable document; using + ``begin`` followed by ``mark_unresolved`` would create a crash window + in which restart recovery could incorrectly classify a dispatched + device write as not dispatched. + + The caller must already own the process-wide K1 lifecycle lease. This + method additionally owns the journal flock, rechecks the complete + document, and refuses every competing or conflicting active record. + """ + + _validate_action(action) + if not isinstance(provenance, NetworkProvisioningLegacyAdoptionProvenance): + raise TypeError( + "provenance must be NetworkProvisioningLegacyAdoptionProvenance" + ) + canonical_provenance = provenance.canonical_bytes() + key_digest = hashlib.sha256( + b"missioncore.network-provisioning.legacy-adoption-key/v1\0" + + action.encode("ascii") + + b"\0" + + canonical_provenance + ).hexdigest() + request_binding_sha256 = hashlib.sha256( + b"missioncore.network-provisioning.legacy-adoption-binding/v1\0" + + action.encode("ascii") + + b"\0" + + canonical_provenance + ).hexdigest() + + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_trusted_locked() + + matching_operation = next( + ( + record + for record in self._document.records + if record.operation_id == provenance.operation_id + ), + None, + ) + if matching_operation is not None: + if ( + matching_operation.action == action + and matching_operation.idempotency_key_sha256 == key_digest + and matching_operation.request_binding_sha256 + == request_binding_sha256 + and matching_operation.stage == "unresolved" + and matching_operation.terminal is None + ): + # Recovery may crash after the atomic replace and before + # returning to the caller. The exact adoption is therefore + # deliberately idempotent across another restart. + return matching_operation + raise NetworkProvisioningIdempotencyConflict( + "legacy network mutation operation is already bound to another record" + ) + + matching_key = next( + ( + record + for record in self._document.records + if record.idempotency_key_sha256 == key_digest + ), + None, + ) + if matching_key is not None: + raise NetworkProvisioningIdempotencyConflict( + "legacy network mutation identity collides with another record" + ) + active = next( + (record for record in self._document.records if record.reconciliation_required), + None, + ) + if active is not None: + raise NetworkProvisioningIdempotencyBlocked( + "another network request is non-terminal; legacy adoption is blocked", + record=active, + ) + + revision = self._next_revision_locked() + now = _timestamp(self._clock()) + adopted = NetworkProvisioningIdempotencyRecord( + idempotency_key_sha256=key_digest, + action=action, + operation_id=provenance.operation_id, + request_binding_sha256=request_binding_sha256, + stage="unresolved", + terminal=None, + created_revision=revision, + revision=revision, + previous_revision=(self._document.revision or None), + created_at_utc=now, + updated_at_utc=now, + ) + self._persist_next_locked((*self._document.records, adopted)) + return adopted + + def mark_unresolved( + self, + operation_id: str, + *, + expected_revision: int, + ) -> NetworkProvisioningIdempotencyRecord: + """Durably cross the ambiguity boundary immediately before a BLE write.""" + + _validate_identifier(operation_id, field_name="operation_id") + _validate_positive_revision(expected_revision, field_name="expected_revision") + with self._lock, self._process_lock_locked(): + current, index = self._current_operation_locked(operation_id) + self._require_expected_revision(current, expected_revision) + if current.stage != "prepared": + raise NetworkProvisioningIdempotencyTransitionError( + "network request may become unresolved only from prepared" + ) + return self._transition_locked(current, index=index, stage="unresolved") + + def complete( + self, + operation_id: str, + *, + expected_revision: int, + terminal: NetworkProvisioningTerminalMetadata, + ) -> NetworkProvisioningIdempotencyRecord: + """Publish the redacted terminal result after proof or reconciliation.""" + + _validate_identifier(operation_id, field_name="operation_id") + _validate_positive_revision(expected_revision, field_name="expected_revision") + if not isinstance(terminal, NetworkProvisioningTerminalMetadata): + raise TypeError("terminal must be NetworkProvisioningTerminalMetadata") + with self._lock, self._process_lock_locked(): + current, index = self._current_operation_locked(operation_id) + if current.stage == "terminal": + if current.terminal == terminal: + return current + raise NetworkProvisioningIdempotencyTransitionError( + "terminal network request cannot change its outcome" + ) + self._require_expected_revision(current, expected_revision) + if current.stage == "prepared" and terminal.side_effect_status != "none": + raise NetworkProvisioningIdempotencyTransitionError( + "prepared request can terminate only with no-side-effect proof" + ) + return self._transition_locked( + current, + index=index, + stage="terminal", + terminal=terminal, + ) + + def _current_operation_locked( + self, + operation_id: str, + ) -> tuple[NetworkProvisioningIdempotencyRecord, int]: + self._reload_locked() + self._require_trusted_locked() + for index, record in enumerate(self._document.records): + if record.operation_id == operation_id: + return record, index + raise NetworkProvisioningIdempotencyTransitionError( + "network provisioning operation does not match the durable journal" + ) + + def _require_trusted_locked(self) -> None: + if self._corrupt: + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency journal is corrupt; writes remain blocked" + ) + + @staticmethod + def _require_expected_revision( + record: NetworkProvisioningIdempotencyRecord, + expected_revision: int, + ) -> None: + if record.revision != expected_revision: + raise NetworkProvisioningIdempotencyTransitionError( + "network provisioning transition used a stale record revision" + ) + + def _next_revision_locked(self) -> int: + revision = self._document.revision + 1 + if revision > _MAX_REVISION: + raise NetworkProvisioningIdempotencyTransitionError( + "network provisioning journal revision is exhausted" + ) + return revision + + def _transition_locked( + self, + current: NetworkProvisioningIdempotencyRecord, + *, + index: int, + stage: NetworkProvisioningIdempotencyStage, + terminal: NetworkProvisioningTerminalMetadata | None = None, + ) -> NetworkProvisioningIdempotencyRecord: + revision = self._next_revision_locked() + updated = replace( + current, + stage=stage, + terminal=terminal, + revision=revision, + previous_revision=current.revision, + updated_at_utc=_nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ), + ) + records = list(self._document.records) + records[index] = updated + self._persist_next_locked(tuple(records)) + return updated + + def _persist_next_locked( + self, + records: tuple[NetworkProvisioningIdempotencyRecord, ...], + ) -> None: + revision = self._next_revision_locked() + terminal_records = [record for record in records if record.stage == "terminal"] + excess = len(terminal_records) - self._max_terminal_records + if excess > 0: + evicted = {record.operation_id for record in terminal_records[:excess]} + records = tuple(record for record in records if record.operation_id not in evicted) + document = _JournalDocument( + revision=revision, + previous_revision=(self._document.revision or None), + records=records, + ) + _write_private_json_atomic( + self.path, + document.as_dict(), + data_dir=self._data_dir, + ) + self._document = document + self._corrupt = False + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + data_dir_created = _ensure_private_directory(self._data_dir, parents=True) + parent_created = _ensure_private_directory(self.path.parent, parents=True) + if data_dir_created or parent_created: + _fsync_directory(self._data_dir) + + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + self._process_lock_path.lstat() + except FileNotFoundError: + lock_may_be_new = True + else: + lock_may_be_new = False + try: + descriptor = os.open(self._process_lock_path, flags, 0o600) + except OSError as exc: + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency lock cannot be opened safely" + ) from exc + lock_stream: IO[bytes] | None = None + try: + try: + metadata = os.fstat(descriptor) + _validate_private_file_metadata(metadata, label="idempotency lock") + path_metadata = self._process_lock_path.lstat() + if (path_metadata.st_dev, path_metadata.st_ino) != ( + metadata.st_dev, + metadata.st_ino, + ): + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency lock path changed during open" + ) + lock_stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(lock_stream.fileno(), fcntl.LOCK_EX) + if lock_may_be_new: + # Persist the stable lock inode before trusting it as the + # cross-process serialization authority. + os.fsync(lock_stream.fileno()) + _fsync_directory(self.path.parent) + locked_metadata = os.fstat(lock_stream.fileno()) + _validate_private_file_metadata(locked_metadata, label="idempotency lock") + locked_path_metadata = self._process_lock_path.lstat() + if (locked_path_metadata.st_dev, locked_path_metadata.st_ino) != ( + locked_metadata.st_dev, + locked_metadata.st_ino, + ): + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency lock was replaced" + ) + except (OSError, ValueError) as exc: + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency lock is unsafe" + ) from exc + try: + yield + finally: + assert lock_stream is not None + fcntl.flock(lock_stream.fileno(), fcntl.LOCK_UN) + finally: + if lock_stream is not None: + lock_stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _reload_locked(self) -> None: + try: + payload = _read_private_json(self.path) + except FileNotFoundError: + self._document = _JournalDocument(revision=0, previous_revision=None, records=()) + self._corrupt = False + return + except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): + self._document = _JournalDocument(revision=0, previous_revision=None, records=()) + self._corrupt = True + return + try: + document = _document_from_mapping( + payload, + max_terminal_records=self._max_terminal_records, + ) + except (TypeError, ValueError): + self._document = _JournalDocument(revision=0, previous_revision=None, records=()) + self._corrupt = True + return + self._document = document + self._corrupt = False + + +def _read_private_json(path: Path) -> object: + parent_metadata = path.parent.lstat() + if not stat.S_ISDIR(parent_metadata.st_mode) or stat.S_IMODE(parent_metadata.st_mode) != 0o700: + raise ValueError("idempotency journal directory is not private") + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + metadata = os.fstat(descriptor) + _validate_private_file_metadata(metadata, label="idempotency journal") + if metadata.st_size > NETWORK_PROVISIONING_IDEMPOTENCY_MAX_BYTES: + raise ValueError("idempotency journal exceeds the bounded size") + path_metadata = path.lstat() + if (path_metadata.st_dev, path_metadata.st_ino) != (metadata.st_dev, metadata.st_ino): + raise ValueError("idempotency journal path changed during open") + with os.fdopen(descriptor, "rb", closefd=True) as stream: + descriptor = -1 + encoded = stream.read(NETWORK_PROVISIONING_IDEMPOTENCY_MAX_BYTES + 1) + finally: + if descriptor >= 0: + os.close(descriptor) + if len(encoded) > NETWORK_PROVISIONING_IDEMPOTENCY_MAX_BYTES: + raise ValueError("idempotency journal exceeds the bounded size") + return json.loads(encoded.decode("utf-8"), object_pairs_hook=_unique_json_object) + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = ( + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + if len(serialized) > NETWORK_PROVISIONING_IDEMPOTENCY_MAX_BYTES: + raise ValueError("network provisioning idempotency journal exceeds the bounded size") + data_dir_created = _ensure_private_directory(data_dir, parents=True) + parent_created = _ensure_private_directory(path.parent, parents=True) + if data_dir_created or parent_created: + _fsync_directory(data_dir) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + metadata = os.fstat(descriptor) + _validate_private_file_metadata(metadata, label="temporary idempotency journal") + with os.fdopen(descriptor, "wb", closefd=True) as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temp_path, path) + published_flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + published_flags |= getattr(os, "O_NOFOLLOW", 0) + published_descriptor = os.open(path, published_flags) + try: + _validate_private_file_metadata( + os.fstat(published_descriptor), + label="published idempotency journal", + ) + finally: + os.close(published_descriptor) + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + return True + if not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700: + raise NetworkProvisioningIdempotencyCorrupt( + "network provisioning idempotency directory is not private" + ) + return False + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _validate_private_file_metadata(metadata: os.stat_result, *, label: str) -> None: + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} is not a regular file") + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError(f"{label} is not private") + if metadata.st_nlink != 1: + raise ValueError(f"{label} must have exactly one hard link") + + +def _document_from_mapping( + value: object, + *, + max_terminal_records: int, +) -> _JournalDocument: + document = _exact_mapping( + value, + {"schema_version", "revision", "previous_revision", "records"}, + label="idempotency journal", + ) + if document["schema_version"] != NETWORK_PROVISIONING_IDEMPOTENCY_SCHEMA: + raise ValueError("unsupported network provisioning idempotency schema") + revision = _positive_revision(document["revision"], field_name="revision") + previous_revision = _optional_revision( + document["previous_revision"], + field_name="previous_revision", + ) + if revision == 1: + if previous_revision is not None: + raise ValueError("first idempotency journal revision cannot have a predecessor") + elif previous_revision != revision - 1: + raise ValueError("idempotency journal revision lineage is discontinuous") + records_raw = document["records"] + if not isinstance(records_raw, list): + raise ValueError("idempotency journal records must be a list") + if len(records_raw) > max_terminal_records + 1: + raise ValueError("idempotency journal exceeds its bounded history") + records = tuple(_record_from_mapping(item) for item in records_raw) + terminal_count = sum(record.stage == "terminal" for record in records) + if terminal_count > max_terminal_records: + raise ValueError("idempotency journal terminal history exceeds its bound") + if sum(record.reconciliation_required for record in records) > 1: + raise ValueError("idempotency journal contains multiple active requests") + key_hashes = [record.idempotency_key_sha256 for record in records] + operation_ids = [record.operation_id for record in records] + if len(set(key_hashes)) != len(key_hashes): + raise ValueError("idempotency journal contains duplicate key hashes") + if len(set(operation_ids)) != len(operation_ids): + raise ValueError("idempotency journal contains duplicate operation ids") + created_revisions = [record.created_revision for record in records] + if created_revisions != sorted(created_revisions) or len(set(created_revisions)) != len( + created_revisions + ): + raise ValueError("idempotency journal record order is not canonical") + if any(record.revision > revision for record in records): + raise ValueError("idempotency record revision exceeds the journal revision") + return _JournalDocument( + revision=revision, + previous_revision=previous_revision, + records=records, + ) + + +def _record_from_mapping(value: object) -> NetworkProvisioningIdempotencyRecord: + document = _exact_mapping( + value, + { + "idempotency_key_sha256", + "action", + "operation_id", + "request_binding_sha256", + "stage", + "terminal", + "created_revision", + "revision", + "previous_revision", + "created_at_utc", + "updated_at_utc", + }, + label="idempotency record", + ) + key_hash = _required_string( + document["idempotency_key_sha256"], + field_name="idempotency_key_sha256", + ) + _validate_sha256(key_hash, field_name="idempotency_key_sha256") + action = _required_string(document["action"], field_name="action") + _validate_action(action) + operation_id = _required_string(document["operation_id"], field_name="operation_id") + _validate_identifier(operation_id, field_name="operation_id") + request_binding = _required_string( + document["request_binding_sha256"], + field_name="request_binding_sha256", + ) + _validate_sha256(request_binding, field_name="request_binding_sha256") + stage_raw = _required_string(document["stage"], field_name="stage") + if stage_raw not in _STAGES: + raise ValueError("unsupported network provisioning idempotency stage") + stage = cast(NetworkProvisioningIdempotencyStage, stage_raw) + terminal_raw = document["terminal"] + terminal = None if terminal_raw is None else _terminal_from_mapping(terminal_raw) + if stage == "terminal" and terminal is None: + raise ValueError("terminal idempotency record lacks replay metadata") + if stage != "terminal" and terminal is not None: + raise ValueError("non-terminal idempotency record carries replay metadata") + created_revision = _positive_revision( + document["created_revision"], + field_name="created_revision", + ) + revision = _positive_revision(document["revision"], field_name="record revision") + previous_revision = _optional_revision( + document["previous_revision"], + field_name="record previous_revision", + ) + if created_revision > revision: + raise ValueError("idempotency record revision precedes its creation") + if created_revision == revision: + expected_predecessor = None if revision == 1 else revision - 1 + if previous_revision != expected_predecessor: + raise ValueError("created idempotency record has discontinuous lineage") + elif previous_revision != revision - 1: + raise ValueError("updated idempotency record has discontinuous lineage") + created_at = _validated_timestamp(document["created_at_utc"], field_name="created_at_utc") + updated_at = _validated_timestamp(document["updated_at_utc"], field_name="updated_at_utc") + if updated_at < created_at: + raise ValueError("idempotency record update precedes creation") + return NetworkProvisioningIdempotencyRecord( + idempotency_key_sha256=key_hash, + action=action, + operation_id=operation_id, + request_binding_sha256=request_binding, + stage=stage, + terminal=terminal, + created_revision=created_revision, + revision=revision, + previous_revision=previous_revision, + created_at_utc=_timestamp(created_at), + updated_at_utc=_timestamp(updated_at), + ) + + +def _terminal_from_mapping(value: object) -> NetworkProvisioningTerminalMetadata: + document = _exact_mapping( + value, + { + "outcome", + "outcome_code", + "error_code", + "side_effect_status", + "retryable", + "safe_to_retry", + }, + label="terminal idempotency metadata", + ) + outcome_raw = _required_string(document["outcome"], field_name="outcome") + if outcome_raw not in _OUTCOMES: + raise ValueError("unsupported network provisioning terminal outcome") + side_effect_raw = _required_string( + document["side_effect_status"], + field_name="side_effect_status", + ) + if side_effect_raw not in _SIDE_EFFECT_STATUSES: + raise ValueError("unsupported network provisioning side-effect status") + retryable = document["retryable"] + safe_to_retry = document["safe_to_retry"] + if not isinstance(retryable, bool) or not isinstance(safe_to_retry, bool): + raise ValueError("retryability fields must be bool") + return NetworkProvisioningTerminalMetadata( + outcome=cast(NetworkProvisioningTerminalOutcome, outcome_raw), + outcome_code=_required_string(document["outcome_code"], field_name="outcome_code"), + error_code=_optional_string(document["error_code"], field_name="error_code"), + side_effect_status=cast(NetworkProvisioningSideEffectStatus, side_effect_raw), + retryable=retryable, + safe_to_retry=safe_to_retry, + ) + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("network provisioning idempotency journal has duplicate fields") + document[key] = value + return document + + +def _exact_mapping(value: object, keys: set[str], *, label: str) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"{label} does not match the secret-free schema") + return cast(Mapping[str, object], value) + + +def _idempotency_key_bytes(value: str) -> bytes: + if not isinstance(value, str): + raise TypeError("idempotency_key must be str") + encoded = value.encode("utf-8") + if not 1 <= len(encoded) <= 512: + raise ValueError("idempotency key must contain 1..512 UTF-8 bytes") + return encoded + + +def _validate_action(value: str) -> None: + if not isinstance(value, str) or _SAFE_ACTION.fullmatch(value) is None: + raise ValueError("action is outside the secret-free schema") + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _validate_safe_code(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SAFE_CODE.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the redacted code schema") + + +def _validate_sha256(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ValueError(f"{field_name} must be a lowercase SHA-256 digest") + + +def _positive_revision(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= _MAX_REVISION: + raise ValueError(f"{field_name} must be a bounded positive integer") + return value + + +def _validate_positive_revision(value: int, *, field_name: str) -> None: + _positive_revision(value, field_name=field_name) + + +def _optional_revision(value: object, *, field_name: str) -> int | None: + if value is None: + return None + return _positive_revision(value, field_name=field_name) + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _optional_string(value: object, *, field_name: str) -> str | None: + if value is None: + return None + return _required_string(value, field_name=field_name) + + +def _validated_timestamp(value: object, *, field_name: str) -> datetime: + raw = _required_string(value, field_name=field_name) + if not raw.endswith("Z"): + raise ValueError(f"{field_name} must be UTC") + try: + parsed = datetime.fromisoformat(raw.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise ValueError(f"{field_name} is invalid") from exc + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): + raise ValueError(f"{field_name} must be UTC") + return parsed.astimezone(UTC) + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("network provisioning idempotency clock must be timezone-aware") + return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _nondecreasing_audit_timestamp(value: datetime, *, floor: str) -> str: + candidate = _validated_timestamp(_timestamp(value), field_name="idempotency clock") + floor_value = _validated_timestamp(floor, field_name="audit timestamp floor") + return _timestamp(max(candidate, floor_value)) diff --git a/src/k1link/device_plugins/xgrids_k1/observation.py b/src/k1link/device_plugins/xgrids_k1/observation.py index 4d80a3b..52b9d44 100644 --- a/src/k1link/device_plugins/xgrids_k1/observation.py +++ b/src/k1link/device_plugins/xgrids_k1/observation.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import stat import threading from collections.abc import Mapping @@ -41,8 +42,16 @@ XGRIDS_K1_PLUGIN_ID = "nodedc.device.xgrids-lixelkity-k1" def build_xgrids_k1_observation(repository_root: Path) -> ObservationRuntimeContribution: """Compose every K1 evidence root behind the generic observation ABI.""" + configured_legacy_root = os.environ.get( + "MISSIONCORE_LEGACY_SESSIONS_DIR", "" + ).strip() + legacy_root = ( + Path(configured_legacy_root).expanduser().resolve() + if configured_legacy_root + else repository_root.resolve() / "sessions" + ) roots = ( - ("xgrids-k1.viewer-live.repository", repository_root.resolve() / "sessions"), + ("xgrids-k1.viewer-live.repository", legacy_root), ( "xgrids-k1.viewer-live.evidence", resolve_missioncore_evidence_dir(repository_root), diff --git a/src/k1link/device_plugins/xgrids_k1/physical_command_coordinator.py b/src/k1link/device_plugins/xgrids_k1/physical_command_coordinator.py new file mode 100644 index 0000000..2b35988 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/physical_command_coordinator.py @@ -0,0 +1,1115 @@ +from __future__ import annotations + +import hashlib +import threading +from collections.abc import Callable +from dataclasses import dataclass, replace +from typing import Literal, Protocol + +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PhysicalCommandApplicationResponse, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandLedger, + PhysicalCommandOperatorRetirementReason, + PhysicalCommandReconciliationResolution, + PhysicalCommandRecord, + PhysicalCommandStatusEvidence, + PhysicalCommandVerifiedBinding, + active_operator_retirements, +) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttEvidenceObserver, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, + ApplicationMqttTransportError, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import MODELING_STATE_BASE + +PhysicalCommandAction = Literal["start", "stop"] + + +class PhysicalCommandPrepareSettlementError(RuntimeError): + """A failed durable prepare could not be proven absent or sealed safely.""" + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandIntentContext: + """Facade-owned operation lineage for one explicit physical edge.""" + + operation_id: str + parent_operation_id: str | None + acquisition_id: str + operator_confirmation_id: str | None = None + operator_confirmed_at_utc: str | None = None + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandRuntimeBinding: + """Exact DeviceInfo-verified session and host-path generation.""" + + vendor_device_id_sha256: str + device_serial_sha256: str + compatibility_profile_id: str + intent_id: str + transport_ref: str + connection_mode: Literal["bridge", "quick-connect", "direct-connect"] + target_ipv4: str + target_port: int + host_path_epoch: int + control_session_id: str + producer_generation: int + + +class PhysicalCommandCoordinator(ApplicationMqttEvidenceObserver, Protocol): + """Session-owned policy boundary between MQTT evidence and durable state.""" + + def bind_control_session(self, binding: PhysicalCommandRuntimeBinding) -> None: ... + + def prepare( + self, + context: PhysicalCommandIntentContext, + *, + action: PhysicalCommandAction, + envelope: OneShotPublishEnvelope, + ) -> None: ... + + def prepare_read_only_bootstrap(self) -> None: ... + + def resolve(self, action: PhysicalCommandAction) -> None: ... + + def resolve_prepared_not_dispatched(self, action: PhysicalCommandAction) -> None: ... + + def reconcile_unresolved(self, *, reconciliation_id: str) -> dict[str, object]: ... + + def retire_unavailable_target( + self, + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, + ) -> dict[str, object]: ... + + def retire_connection_scenario( + self, + *, + retirement_id: str, + expected_revision: int | None = None, + ) -> dict[str, object]: ... + + def preview_connection_scenario_retirement( + self, + *, + retirement_id: str, + ) -> dict[str, object]: ... + + def reopen_retired_reconciliation( + self, + *, + reopening_id: str, + expected_revision: int, + expected_retirement_id: str, + expected_transport_ref: str, + expected_discovery_generation: int, + reason: Literal[ + "device-returned-for-explicit-reconciliation", + "reset-network-intent-read-only-settlement", + ], + ) -> dict[str, object]: ... + + def reconcile_resolved_active(self, *, reconciliation_id: str) -> dict[str, object]: ... + + def snapshot(self) -> dict[str, object]: ... + + +@dataclass(frozen=True, slots=True) +class _PreparedPhysicalCommand: + context: PhysicalCommandIntentContext + action: PhysicalCommandAction + payload_sha256: str + + +class LedgerPhysicalCommandCoordinator: + """Translate typed live evidence into the storage-only physical ledger. + + This object never performs device or network I/O. It deliberately ignores + non-modeling MQTT writes and never replays a command. A normal session may + resolve only its own exact START/STOP edge; restart reconciliation remains + a separate, explicitly read-only workflow. + """ + + def __init__(self, ledger: PhysicalCommandLedger) -> None: + self._ledger = ledger + self._lock = threading.RLock() + self._runtime: PhysicalCommandRuntimeBinding | None = None + self._pending_device_info: ApplicationMqttResponseEvidence | None = None + self._verified_device_info: ApplicationMqttResponseEvidence | None = None + self._latest_status: PhysicalCommandStatusEvidence | None = None + self._active: _PreparedPhysicalCommand | None = None + + def bind_control_session(self, binding: PhysicalCommandRuntimeBinding) -> None: + if not isinstance(binding, PhysicalCommandRuntimeBinding): + raise TypeError("binding must be PhysicalCommandRuntimeBinding") + # Construct the ledger-owned types now so malformed generation/path + # data fails before any physical command can be prepared. + identity = self._identity(binding) + connection = self._connection(binding) + with self._lock: + try: + self._ledger.require_runtime_not_retired( + identity=identity, + connection=connection, + ) + except BaseException: + # DeviceInfo has identified a retired physical target. Revoke + # every process-local proof before surfacing the failure so a + # later session cannot consume or continue this bootstrap. + self._runtime = None + self._pending_device_info = None + self._verified_device_info = None + self._latest_status = None + self._active = None + raise + self._runtime = binding + # The correlated bootstrap DeviceInfo response is observed before + # ApplicationControlSession can construct the exact runtime binding. + # Consume it once here so no later control-session generation can + # accidentally inherit an older identity proof. + self._verified_device_info = self._pending_device_info + self._pending_device_info = None + self._latest_status = None + # A new control-session binding can never continue a physical + # edge from the previous session. In particular, an unresolved + # START/STOP must consume fresh DeviceInfo + DeviceStatus only as + # read-only reconciliation evidence. Keeping the old in-memory + # command active here would try to append the new session's status + # to a ledger record bound to the old host path, poison the MQTT + # callback, and prevent reconciliation from running at all. + self._active = None + + def prepare( + self, + context: PhysicalCommandIntentContext, + *, + action: PhysicalCommandAction, + envelope: OneShotPublishEnvelope, + ) -> None: + if not isinstance(context, PhysicalCommandIntentContext): + raise TypeError("context must be PhysicalCommandIntentContext") + expected_operation_key = f"modeling:{action}" + if envelope.operation_key != expected_operation_key: + raise ValueError("physical command envelope action does not match preparation") + with self._lock: + if self._active is not None: + raise RuntimeError("a physical command is already prepared in this session") + runtime = self._require_runtime_locked() + baseline = self._latest_status + if baseline is None: + raise RuntimeError("fresh non-retained DeviceStatus baseline is unavailable") + expected_identity = self._identity(runtime) + expected_connection = self._connection(runtime) + try: + self._ledger.prepare( + operation_id=context.operation_id, + parent_operation_id=context.parent_operation_id, + acquisition_id=context.acquisition_id, + action=action, + identity=expected_identity, + connection=expected_connection, + compatibility_profile_id=runtime.compatibility_profile_id, + payload_sha256=envelope.payload_sha256, + baseline_status=baseline, + operator_confirmation_id=context.operator_confirmation_id, + operator_confirmed_at_utc=context.operator_confirmed_at_utc, + ) + except BaseException as prepare_error: + # A storage exception can occur after the PREPARED document was + # atomically replaced but before ``prepare`` returned. Since + # no transport publish can begin until this method succeeds, + # seal that exact row as definitely not dispatched. Anything + # other than either absence or the exact PREPARED edge is an + # unresolved safety fault and must be surfaced distinctly. + try: + snapshot = self._ledger.snapshot() + record = snapshot.record + exact_prepared = bool( + snapshot.status == "unresolved" + and record is not None + and record.operation_id == context.operation_id + and record.parent_operation_id == context.parent_operation_id + and record.acquisition_id == context.acquisition_id + and record.action == action + and record.stage == "prepared" + and record.identity == expected_identity + and record.connection == expected_connection + and record.compatibility_profile_id + == runtime.compatibility_profile_id + and record.payload_sha256 == envelope.payload_sha256 + and record.baseline_status == baseline + and self._prepared_confirmation_matches_context( + record, + context, + ) + ) + same_operation = bool( + record is not None + and record.operation_id == context.operation_id + ) + if exact_prepared and record is not None: + self._ledger.resolve( + record.operation_id, + resolution="not-dispatched", + ) + elif same_operation: + raise RuntimeError( + "failed prepare left a non-PREPARED or mismatched physical edge" + ) + except BaseException as settlement_error: + raise PhysicalCommandPrepareSettlementError( + "failed physical command prepare could not be settled as no-dispatch" + ) from settlement_error + raise prepare_error + self._active = _PreparedPhysicalCommand( + context=context, + action=action, + payload_sha256=envelope.payload_sha256, + ) + + @staticmethod + def _prepared_confirmation_matches_context( + record: PhysicalCommandRecord, + context: PhysicalCommandIntentContext, + ) -> bool: + matching = tuple( + item + for item in record.operator_confirmations + if item.stop_operation_id == context.operation_id + ) + if context.operator_confirmation_id is None: + return context.operator_confirmed_at_utc is None and not matching + return bool( + context.operator_confirmed_at_utc is not None + and len(matching) == 1 + and matching[0].confirmation_id == context.operator_confirmation_id + and matching[0].confirmed_at_utc == context.operator_confirmed_at_utc + ) + + def prepare_read_only_bootstrap(self) -> None: + """Detach an undispatched STOP before installing a Verify observer. + + The MQTT observer is installed before a new control binding exists, so + a retained/stale status may arrive during transport bootstrap. Leaving + a process-local PREPARED STOP active would route that callback into + ``record_status_observation`` and poison DeviceInfo bootstrap even + though no publish boundary was crossed. This local detach preserves + the durable PREPARED proof for the later atomic read-only classification. + + The facade calls this only after the old ApplicationControlSession is + terminal/retired while holding lifecycle ownership. Any exact current + unresolved edge may then detach: PREPARED retains definite no-dispatch + proof; DISPATCHING/OBSERVING retains its durable ambiguity for the + unchanged ``reconcile_ambiguous`` path. A mismatched or resolved row + fails closed. + """ + + with self._lock: + active = self._active + if active is None: + return + snapshot = self._ledger.snapshot() + record = snapshot.record + if ( + snapshot.status != "unresolved" + or record is None + or record.operation_id != active.context.operation_id + or record.stage not in {"prepared", "dispatching", "observing"} + or record.action != active.action + ): + raise RuntimeError( + "only the exact durable unresolved edge may detach for read-only bootstrap" + ) + self._active = None + self._runtime = None + self._pending_device_info = None + self._verified_device_info = None + self._latest_status = None + + def resolve(self, action: PhysicalCommandAction) -> None: + with self._lock: + active = self._require_active_locked(action) + observation = self._latest_status + if observation is None: + raise RuntimeError("physical command target status evidence is unavailable") + self._ledger.resolve( + active.context.operation_id, + resolution=( + "start-active-observed" if action == "start" else "stop-standby-observed" + ), + observation=observation, + ) + self._active = None + + def resolve_prepared_not_dispatched(self, action: PhysicalCommandAction) -> None: + """Durably seal one exact process-local PREPARED edge as no-dispatch. + + The caller must independently fence the transport publish boundary. + This storage-only method refuses DISPATCHING/OBSERVING and clears the + process-local owner only after the ledger transition is durable. + """ + + with self._lock: + active = self._require_active_locked(action) + snapshot = self._ledger.snapshot() + record = snapshot.record + if not ( + snapshot.status == "unresolved" + and record is not None + and record.operation_id == active.context.operation_id + and record.action == action + and record.stage == "prepared" + ): + raise RuntimeError( + "not-dispatched resolution requires the exact active PREPARED edge" + ) + self._ledger.resolve( + active.context.operation_id, + resolution="not-dispatched", + ) + self._active = None + + def publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: + action = self._physical_action(evidence.operation_key) + if action is None: + return + with self._lock: + active = self._require_matching_active_locked(action, evidence.payload_sha256) + # This is the physical STOP's last reversible boundary. The + # coordinator lock keeps the exact PREPARED owner stable across + # the predicate and the ledger's durable DISPATCHING transition, + # eliminating the transport-level check/mark TOCTOU. + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + # Commit the still-fresh physical-acceptance reservation under the + # same owner lock, immediately before starting the durable mark. + # A permit-expiry error leaves PREPARED unchanged. If the ledger + # mark itself fails, the consumed permit and PREPARED/ambiguous + # disk evidence remain fail-closed and can never authorize retry. + if dispatch_admission_commit is not None: + dispatch_admission_commit() + self._ledger.mark_dispatching(active.context.operation_id) + + def publish_result( + self, + evidence: ApplicationMqttPublishEvidence, + *, + publish_call_returned: bool, + ) -> None: + action = self._physical_action(evidence.operation_key) + if action is None: + return + with self._lock: + active = self._require_matching_active_locked(action, evidence.payload_sha256) + self._ledger.mark_observing( + active.context.operation_id, + publish_call_returned=publish_call_returned, + packet_id=evidence.packet_id, + ) + + def qos2_completed(self, evidence: ApplicationMqttPublishEvidence) -> None: + action = self._physical_action(evidence.operation_key) + if action is None: + return + if evidence.packet_id is None: + raise RuntimeError("QoS2 evidence omitted the exact packet identifier") + with self._lock: + active = self._require_matching_active_locked(action, evidence.payload_sha256) + self._ledger.mark_qos2_completed( + active.context.operation_id, + packet_id=evidence.packet_id, + ) + + def application_response(self, evidence: ApplicationMqttResponseEvidence) -> None: + action = self._physical_action(evidence.operation_key) + if action is None: + if self._is_bootstrap_device_info(evidence.operation_key): + with self._lock: + self._pending_device_info = evidence + return + if ( + evidence.modeling_action != action + or evidence.result_code is None + or evidence.success is None + ): + raise RuntimeError("modeling response evidence is incomplete or mismatched") + with self._lock: + active = self._require_active_locked(action) + runtime = self._require_runtime_locked() + self._ledger.record_application_response( + active.context.operation_id, + PhysicalCommandApplicationResponse( + operation_id=active.context.operation_id, + action=action, + control_session_id=runtime.control_session_id, + host_path_epoch=runtime.host_path_epoch, + producer_generation=runtime.producer_generation, + result_code=evidence.result_code, + success=evidence.success, + payload_sha256=evidence.payload_sha256, + observed_at_utc=evidence.observed_at_utc, + ), + ) + + def reconcile_unresolved(self, *, reconciliation_id: str) -> dict[str, object]: + """Resolve or classify one fenced edge from a read-only dialogue. + + This method never publishes and never retries START/STOP. It accepts + only the correlated bootstrap DeviceInfo proof consumed by the current + runtime binding plus a later fresh, non-retained DeviceStatus from that + same binding. + """ + + with self._lock: + ledger_snapshot = self._ledger.snapshot() + record = ledger_snapshot.record + resolved_unclassified_stop_recovery_required = bool( + record is not None + and record.resolved_unclassified_stop_recovery_required + ) + if record is None or not ( + ledger_snapshot.requires_reconciliation + or resolved_unclassified_stop_recovery_required + ): + raise RuntimeError("physical command reconciliation is not required") + runtime = self._require_runtime_locked() + device_info = self._verified_device_info + if device_info is None: + raise RuntimeError("fresh correlated DeviceInfo proof is unavailable") + observation = self._latest_status + if observation is None or observation.mqtt_retained: + raise RuntimeError("fresh non-retained DeviceStatus proof is unavailable") + resolution: PhysicalCommandReconciliationResolution + if ( + observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ): + resolution = "physical-standby-observed" + elif ( + observation.session_state == "scanning" + and observation.project_bound + and observation.init_ready + ): + resolution = "physical-active-observed" + else: + raise RuntimeError("K1 is neither in canonical READY nor canonical SCANNING state") + verified_binding = PhysicalCommandVerifiedBinding( + verification_id=f"{reconciliation_id}.device-info", + identity=self._identity(runtime), + connection=self._connection(runtime), + device_info_message_sha256=device_info.payload_sha256, + verified_at_utc=device_info.observed_at_utc, + ) + explicit_observation = replace( + observation, + source="explicit-read-only-reconciliation", + ) + if ( + record.stage == "prepared" + or resolved_unclassified_stop_recovery_required + ): + if record.action != "stop": + raise RuntimeError( + "only definitely undispatched STOP supports combined " + "read-only classification" + ) + reconciled = self._ledger.reconcile_prepared_stop_not_dispatched( + record.operation_id, + expected_revision=record.revision, + reconciliation_id=reconciliation_id, + verified_binding=verified_binding, + observation=explicit_observation, + ) + else: + reconciled = self._ledger.reconcile_ambiguous( + record.operation_id, + reconciliation_id=reconciliation_id, + resolution=resolution, + verified_binding=verified_binding, + observation=explicit_observation, + ) + self._active = None + return reconciled.as_dict() + + def retire_unavailable_target( + self, + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, + ) -> dict[str, object]: + """Retire one unreachable target locally without publishing anything.""" + + with self._lock: + if self._active is not None: + raise RuntimeError("physical command owner is still active in this process") + retired = self._ledger.retire_unavailable_target( + retirement_id=retirement_id, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_transport_ref=expected_transport_ref, + reason=reason, + ) + # The durable transition succeeded, so stale in-memory proof from + # that unreachable target can no longer authorize a continuation. + # Clearing these process-local references performs no transport or + # device operation. + self._runtime = None + self._pending_device_info = None + self._verified_device_info = None + self._latest_status = None + return retired.as_dict() + + def preview_connection_scenario_retirement( + self, + *, + retirement_id: str, + ) -> dict[str, object]: + """Validate the durable physical reset boundary without mutating it.""" + + with self._lock: + snapshot = self._ledger.snapshot() + record = snapshot.record + if snapshot.status == "corrupt": + raise RuntimeError("physical command ledger is corrupt; scenario reset is blocked") + if record is None: + disposition = "empty" + elif any( + retirement.reason == "connection-scenario-reset-by-operator" + for retirement in active_operator_retirements(record) + ): + disposition = "operator-retired-outcome-unknown" + elif record.stage == "prepared": + self._ledger.preflight_resolve_not_dispatched(record.operation_id) + disposition = "not-dispatched" + elif record.stage in {"dispatching", "observing"} or ( + record.stage == "resolved" + and ( + record.reconciled_physical_state == "active" + or record.reopened_physical_state_requires_reconciliation + or record.classified_stop_standby_successor_pending + ) + ): + # Ask the ledger to build and fully validate the exact + # prospective append/rollover before any caller tears down its + # local receiver, control session or network ownership. + self._ledger.preflight_retire_unavailable_target( + retirement_id=retirement_id, + expected_operation_id=record.operation_id, + expected_revision=record.revision, + expected_transport_ref=record.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + disposition = "operator-retired-outcome-unknown" + else: + disposition = "already-terminal" + return { + "disposition": disposition, + "record_revision": record.revision if record is not None else None, + "operation_id": record.operation_id if record is not None else None, + "transport_ref": (record.connection.transport_ref if record is not None else None), + } + + def retire_connection_scenario( + self, + *, + retirement_id: str, + expected_revision: int | None = None, + ) -> dict[str, object]: + """Seal the old local scenario without inventing a device STOP. + + A prepared edge is resolved as definitely not dispatched. An + ambiguous dispatched or last-proven-active edge is retained as an + explicit operator retirement whose physical outcome stays unknown. + A terminal standby/empty chain normally needs no additional durable + mutation. The one exception is a classified undispatched STOP whose + future START binding is explicitly abandoned by this scenario reset; + that binding is retired locally while its READY proof stays in audit. + No branch publishes, opens a transport, or contacts the K1. + """ + + with self._lock: + snapshot = self._ledger.snapshot() + record = snapshot.record + if snapshot.status == "corrupt": + raise RuntimeError("physical command ledger is corrupt; scenario reset is blocked") + if ( + expected_revision is not None + and record is not None + and record.revision != expected_revision + and not any( + retirement.reason == "connection-scenario-reset-by-operator" + for retirement in active_operator_retirements(record) + ) + ): + raise RuntimeError("physical command ledger changed after scenario-reset preflight") + if record is None: + disposition = "empty" + result: dict[str, object] = { + "disposition": disposition, + "record": None, + } + elif any( + retirement.reason == "connection-scenario-reset-by-operator" + for retirement in active_operator_retirements(record) + ): + result = { + "disposition": "operator-retired-outcome-unknown", + "record": record.as_dict(), + } + elif record.stage == "prepared": + resolved = self._ledger.resolve( + record.operation_id, + resolution="not-dispatched", + ) + result = { + "disposition": "not-dispatched", + "record": resolved.as_dict(), + } + else: + retireable = bool( + record.stage in {"dispatching", "observing"} + or ( + record.stage == "resolved" + and ( + record.reconciled_physical_state == "active" + or record.reopened_physical_state_requires_reconciliation + or record.classified_stop_standby_successor_pending + ) + ) + ) + if retireable: + retired = self._ledger.retire_unavailable_target( + retirement_id=retirement_id, + expected_operation_id=record.operation_id, + expected_revision=record.revision, + expected_transport_ref=record.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + result = { + "disposition": "operator-retired-outcome-unknown", + "record": retired.as_dict(), + } + else: + result = { + "disposition": "already-terminal", + "record": record.as_dict(), + } + self._active = None + self._runtime = None + self._pending_device_info = None + self._verified_device_info = None + self._latest_status = None + return result + + def reopen_retired_reconciliation( + self, + *, + reopening_id: str, + expected_revision: int, + expected_retirement_id: str, + expected_transport_ref: str, + expected_discovery_generation: int, + reason: Literal[ + "device-returned-for-explicit-reconciliation", + "reset-network-intent-read-only-settlement", + ], + ) -> dict[str, object]: + """Re-admit one retired UUID locally for exact read-only settlement.""" + + with self._lock: + if self._active is not None: + raise RuntimeError("physical command owner is still active in this process") + reopened = self._ledger.reopen_retired_reconciliation( + reopening_id=reopening_id, + expected_revision=expected_revision, + expected_retirement_id=expected_retirement_id, + expected_transport_ref=expected_transport_ref, + expected_discovery_generation=expected_discovery_generation, + reason=reason, + ) + # The action grants no runtime authority. Explicit recovery or the + # exact reset-owned Apply continuation must establish fresh + # DeviceInfo and DeviceStatus proofs afterwards. + self._runtime = None + self._pending_device_info = None + self._verified_device_info = None + self._latest_status = None + return reopened.as_dict() + + def reconcile_resolved_active(self, *, reconciliation_id: str) -> dict[str, object]: + """Re-observe one proven START after its original control path was lost. + + A successful START is not ambiguous, but a later battery removal or + hard shutdown can make its *current* physical state unknown. A new + explicit Verify may therefore attach fresh DeviceInfo plus a later + non-retained DeviceStatus to that immutable START. READY records a + cessation; SCANNING rebinds the active state so an explicit STOP can + use the new exact control generation. Neither branch publishes. + """ + + with self._lock: + ledger_snapshot = self._ledger.snapshot() + record = ledger_snapshot.record + scan_over_ready_required = _resolved_scan_over_ready_required(record) + reopened_recovery_required = bool( + record is not None + and record.reopened_physical_state_requires_reconciliation + ) + reopened_resolved_start_recovery_required = bool( + record is not None + and getattr( + record, + "reopened_resolved_start_requires_reconciliation", + False, + ) + ) + if ( + record is None + or not ( + ( + ledger_snapshot.status == "resolved" + and record.resolution + in { + "start-active-observed", + "physical-active-observed", + "physical-standby-observed", + "not-dispatched", + } + ) + or reopened_resolved_start_recovery_required + ) + or ( + record.reconciled_physical_state != "active" + and not scan_over_ready_required + and not reopened_recovery_required + and not reopened_resolved_start_recovery_required + ) + ): + raise RuntimeError("resolved active physical recovery is not required") + runtime = self._require_runtime_locked() + device_info = self._verified_device_info + if device_info is None: + raise RuntimeError("fresh correlated DeviceInfo proof is unavailable") + observation = self._latest_status + if observation is None or observation.mqtt_retained: + raise RuntimeError("fresh non-retained DeviceStatus proof is unavailable") + verified_binding = PhysicalCommandVerifiedBinding( + verification_id=f"{reconciliation_id}.device-info", + identity=self._identity(runtime), + connection=self._connection(runtime), + device_info_message_sha256=device_info.payload_sha256, + verified_at_utc=device_info.observed_at_utc, + ) + explicit_observation = replace( + observation, + source="explicit-read-only-reconciliation", + ) + canonical_standby_observation = bool( + ( + observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ) + or (observation.session_state == "scan_over" and not scan_over_ready_required) + ) + if canonical_standby_observation: + reconciled = self._ledger.observe_resolved_active_standby( + record.operation_id, + reconciliation_id=reconciliation_id, + verified_binding=verified_binding, + observation=explicit_observation, + ) + elif ( + observation.session_state == "scanning" + and observation.project_bound + and observation.init_ready + and not scan_over_ready_required + ): + reconciled = self._ledger.rebind_resolved_active( + record.operation_id, + reconciliation_id=reconciliation_id, + verified_binding=verified_binding, + observation=explicit_observation, + ) + else: + raise RuntimeError("K1 is not in canonical READY, SCAN_OVER or SCANNING state") + self._active = None + return reconciled.as_dict() + + def device_status(self, evidence: ApplicationMqttDeviceStatusEvidence) -> None: + with self._lock: + runtime = self._runtime + if runtime is None: + # DeviceStatus delivered before correlated DeviceInfo cannot be + # attached to a verified control-session generation. + return + status = self._status(runtime, evidence) + self._latest_status = status + active = self._active + if active is not None and self._is_target(active.action, status): + self._ledger.record_status_observation( + active.context.operation_id, + status, + ) + + def snapshot(self) -> dict[str, object]: + with self._lock: + ledger_snapshot = self._ledger.snapshot() + record = ledger_snapshot.record + reopened_physical_state_recovery_required = bool( + ledger_snapshot.status == "resolved" + and record is not None + and record.reopened_physical_state_requires_reconciliation + ) + reopened_resolved_start_recovery_required = bool( + record is not None + and getattr( + record, + "reopened_resolved_start_requires_reconciliation", + False, + ) + ) + resolved_active_recovery_required = bool( + record is not None + and ( + ( + ledger_snapshot.status == "resolved" + and ( + record.reconciled_physical_state == "active" + or reopened_physical_state_recovery_required + ) + ) + or reopened_resolved_start_recovery_required + ) + ) + resolved_scan_over_recovery_required = _resolved_scan_over_ready_required(record) + resolved_unclassified_stop_recovery_required = bool( + ledger_snapshot.status == "resolved" + and record is not None + and record.resolved_unclassified_stop_recovery_required + ) + read_only_recovery_required = bool( + ledger_snapshot.requires_reconciliation + or resolved_active_recovery_required + or resolved_scan_over_recovery_required + or resolved_unclassified_stop_recovery_required + ) + return { + "status": ledger_snapshot.status, + "reason_code": ( + "physical-command-scan-over-awaiting-ready" + if resolved_scan_over_recovery_required + else "physical-command-stop-awaiting-state-classification" + if resolved_unclassified_stop_recovery_required + else ledger_snapshot.reason_code + ), + "requires_reconciliation": ( + ledger_snapshot.requires_reconciliation + or resolved_scan_over_recovery_required + or resolved_unclassified_stop_recovery_required + ), + "resolved_active_recovery_required": (resolved_active_recovery_required), + "reopened_physical_state_recovery_required": ( + reopened_physical_state_recovery_required + ), + "reopened_resolved_start_recovery_required": ( + reopened_resolved_start_recovery_required + ), + "resolved_scan_over_recovery_required": (resolved_scan_over_recovery_required), + "resolved_unclassified_stop_recovery_required": ( + resolved_unclassified_stop_recovery_required + ), + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": ( + "explicit-read-only-deviceinfo-and-non-retained-devicestatus" + if read_only_recovery_required + else None + ), + "runtime_bound": self._runtime is not None, + "reconciliation_ready": ( + read_only_recovery_required + and self._runtime is not None + and self._verified_device_info is not None + and self._latest_status is not None + and not self._latest_status.mqtt_retained + ), + "observed_session_state": ( + self._latest_status.session_state if self._latest_status is not None else None + ), + "reconciled_physical_state": ( + record.reconciled_physical_state if record is not None else None + ), + "physical_active": bool( + record is not None and record.reconciled_physical_state == "active" + ), + "active_operation_id": ( + self._active.context.operation_id if self._active is not None else None + ), + "record": ( + ledger_snapshot.record.as_dict() if ledger_snapshot.record is not None else None + ), + } + + def _require_runtime_locked(self) -> PhysicalCommandRuntimeBinding: + if self._runtime is None: + raise RuntimeError("physical command control-session binding is unavailable") + return self._runtime + + def _require_active_locked(self, action: PhysicalCommandAction) -> _PreparedPhysicalCommand: + active = self._active + if active is None or active.action != action: + raise RuntimeError("physical command has no matching durable preparation") + return active + + def _require_matching_active_locked( + self, + action: PhysicalCommandAction, + payload_sha256: str, + ) -> _PreparedPhysicalCommand: + active = self._require_active_locked(action) + if active.payload_sha256 != payload_sha256: + raise RuntimeError("physical command payload changed after durable preparation") + return active + + @staticmethod + def _physical_action(operation_key: str) -> PhysicalCommandAction | None: + if operation_key == "modeling:start": + return "start" + if operation_key == "modeling:stop": + return "stop" + return None + + @staticmethod + def _is_bootstrap_device_info(operation_key: str) -> bool: + return operation_key.startswith("bootstrap:") and operation_key.endswith( + ":DeviceInfoRequest" + ) + + @staticmethod + def _identity(binding: PhysicalCommandRuntimeBinding) -> PhysicalCommandIdentity: + return PhysicalCommandIdentity( + vendor_device_id_sha256=binding.vendor_device_id_sha256, + device_serial_sha256=binding.device_serial_sha256, + ) + + @staticmethod + def _connection( + binding: PhysicalCommandRuntimeBinding, + ) -> PhysicalCommandConnectionBinding: + return PhysicalCommandConnectionBinding( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode=binding.connection_mode, + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id=binding.control_session_id, + producer_generation=binding.producer_generation, + ) + + @staticmethod + def _status( + runtime: PhysicalCommandRuntimeBinding, + evidence: ApplicationMqttDeviceStatusEvidence, + ) -> PhysicalCommandStatusEvidence: + if ( + evidence.vendor_device_id_sha256 != runtime.vendor_device_id_sha256 + or evidence.device_serial_sha256 != runtime.device_serial_sha256 + ): + raise RuntimeError("DeviceStatus identity does not match the bound control session") + if evidence.session_state is None: + raise RuntimeError("DeviceStatus state is not recognized by the physical ledger") + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=runtime.vendor_device_id_sha256, + device_serial_sha256=runtime.device_serial_sha256, + control_session_id=runtime.control_session_id, + host_path_epoch=runtime.host_path_epoch, + producer_generation=runtime.producer_generation, + session_state=evidence.session_state, # type: ignore[arg-type] + # DeviceStatus carries the vendor wire enum as + # MODELING_STATE_BASE + SessionState (for example READY is + # 302252332). The durable ledger deliberately stores the stable + # semantic enum value (READY == 300). Feeding the raw wire value + # into the ledger poisoned an otherwise healthy live control + # session as soon as the first READY report arrived. + session_state_code=evidence.session_state_code - MODELING_STATE_BASE, + project_bound=evidence.project_bound, + project_id_sha256=evidence.project_id_sha256, + init_ready=evidence.init_ready, + status_message_sha256=evidence.status_message_sha256, + mqtt_retained=evidence.mqtt_retained, + observed_at_utc=evidence.observed_at_utc, + ) + + @staticmethod + def _is_target( + action: PhysicalCommandAction, + status: PhysicalCommandStatusEvidence, + ) -> bool: + if status.mqtt_retained: + return False + if action == "start": + return status.session_state == "scanning" and status.project_bound and status.init_ready + return ( + status.session_state == "ready" and not status.project_bound and not status.init_ready + ) + + +def _resolved_scan_over_ready_required( + record: PhysicalCommandRecord | None, +) -> bool: + """Return whether a durable cessation still awaits canonical READY.""" + + if not isinstance(record, PhysicalCommandRecord) or record.stage != "resolved": + return False + for reconciliation in reversed(record.reconciliations): + if reconciliation.kind not in { + "resolved-active-rebind", + "resolved-active-cessation", + }: + continue + if reconciliation.original_attempt.operation_id != record.operation_id: + continue + return bool( + reconciliation.resolution == "physical-standby-observed" + and reconciliation.observation.session_state == "scan_over" + ) + return False + + +def hash_physical_identity(value: str) -> str: + """Hash one DeviceInfo identity before it crosses the coordinator API.""" + + return hashlib.sha256(value.encode("utf-8")).hexdigest() diff --git a/src/k1link/device_plugins/xgrids_k1/physical_command_ledger.py b/src/k1link/device_plugins/xgrids_k1/physical_command_ledger.py new file mode 100644 index 0000000..4615a35 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/physical_command_ledger.py @@ -0,0 +1,6914 @@ +from __future__ import annotations + +import fcntl +import hashlib +import ipaddress +import json +import os +import re +import stat +import tempfile +import threading +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, Literal, cast + +from k1link.sessions.store import resolve_missioncore_data_dir + +PHYSICAL_COMMAND_LEDGER_SCHEMA = "missioncore.xgrids-k1-physical-command/v5" +PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA = "missioncore.xgrids-k1-physical-command/v4" +PHYSICAL_COMMAND_LEDGER_V3_SCHEMA = "missioncore.xgrids-k1-physical-command/v3" +PHYSICAL_COMMAND_LEDGER_V2_SCHEMA = "missioncore.xgrids-k1-physical-command/v2" +PHYSICAL_COMMAND_LEDGER_LEGACY_SCHEMA = "missioncore.xgrids-k1-physical-command/v1" +PHYSICAL_COMMAND_LEDGER_FILENAME = "physical-command.json" +PHYSICAL_COMMAND_LEDGER_LOCK_FILENAME = ".physical-command.lock" +PHYSICAL_COMMAND_ARCHIVE_DIRECTORY = "physical-command-archive" +PHYSICAL_COMMAND_ARCHIVE_SCHEMA = "missioncore.xgrids-k1-physical-command-archive/v1" +PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA: Literal[ + "missioncore.xgrids-k1-classified-stop-ancestry/v1" +] = ( + "missioncore.xgrids-k1-classified-stop-ancestry/v1" +) +PHYSICAL_COMMAND_LEDGER_MAX_BYTES = 64 * 1024 +PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES = 128 * 1024 +PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS = 4096 +PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES = 512 * 1024 * 1024 +# Private, inert test failpoint used by the subprocess crash-recovery gate. A +# real process death is required to prove the hard-link publication window; +# production never assigns this callable. +_PHYSICAL_COMMAND_ARCHIVE_AFTER_LINK_FSYNC_FAILPOINT: Callable[[], None] | None = None +# Reviewed XGRIDS OpenAPI ModelingResponse success value. Kept local so this +# storage-only module does not import the wire codec or create a protocol cycle. +PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE = 302252033 + +PhysicalCommandAction = Literal["start", "stop"] +PhysicalCommandConnectionMode = Literal["bridge", "quick-connect", "direct-connect"] +PhysicalCommandStage = Literal["prepared", "dispatching", "observing", "resolved"] +PhysicalCommandResolution = Literal[ + "not-dispatched", + "start-active-observed", + "stop-standby-observed", + "physical-active-observed", + "physical-standby-observed", + "operator-retired-outcome-unknown", +] +PhysicalCommandOrdinaryResolution = Literal[ + "not-dispatched", + "start-active-observed", + "stop-standby-observed", +] +PhysicalCommandReconciliationResolution = Literal[ + "physical-active-observed", + "physical-standby-observed", +] +PhysicalCommandReconciliationKind = Literal[ + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", +] +PhysicalCommandLedgerStatus = Literal["empty", "unresolved", "resolved", "corrupt"] +PhysicalCommandOriginalOutcome = Literal["unknown", "not-dispatched", "succeeded"] +PhysicalCommandObservedState = Literal["active", "standby"] +PhysicalCommandOperatorRetirementReason = Literal[ + "device-permanently-unavailable-or-replaced", + "connection-scenario-reset-by-operator", +] +_PHYSICAL_COMMAND_OPERATOR_RETIREMENT_REASONS = frozenset( + { + "device-permanently-unavailable-or-replaced", + "connection-scenario-reset-by-operator", + } +) +PhysicalCommandOperatorReconciliationReopenReason = Literal[ + "device-returned-for-explicit-reconciliation", + "reset-network-intent-read-only-settlement", +] +_PHYSICAL_COMMAND_OPERATOR_RECONCILIATION_REOPEN_REASONS = frozenset( + { + "device-returned-for-explicit-reconciliation", + "reset-network-intent-read-only-settlement", + } +) +PhysicalStatusSource = Literal[ + "live-control-session", + "explicit-read-only-reconciliation", +] +PhysicalSessionState = Literal[ + "other_status", + "ready", + "scan_starting", + "scanning", + "scan_stopping", + "scan_over", + "disk_error", + "save_error", + "algorithm_error", + "no_continue_error", + "no_space_error", + "camera_error", + "continue_success", + "continue_fail", + "usb_disk", + "sd_space_not_enough", + "memory_not_enough", + "lidar_data_error", + "mapping_error", +] + +_ACTIONS = frozenset({"start", "stop"}) +_CONNECTION_MODES = frozenset({"bridge", "quick-connect", "direct-connect"}) +_STAGES = frozenset({"prepared", "dispatching", "observing", "resolved"}) +_RESOLUTIONS = frozenset( + { + "not-dispatched", + "start-active-observed", + "stop-standby-observed", + "physical-active-observed", + "physical-standby-observed", + "operator-retired-outcome-unknown", + } +) +_ORDINARY_RESOLUTIONS = frozenset( + {"not-dispatched", "start-active-observed", "stop-standby-observed"} +) +_RECONCILIATION_RESOLUTIONS = frozenset({"physical-active-observed", "physical-standby-observed"}) +_RECONCILIATION_KINDS = frozenset( + { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", + } +) +_STATUS_SOURCES = frozenset({"live-control-session", "explicit-read-only-reconciliation"}) +_SESSION_STATE_CODES: dict[str, int] = { + "other_status": 0, + "ready": 300, + "scan_starting": 301, + "scanning": 302, + "scan_stopping": 303, + "scan_over": 304, + "disk_error": 305, + "save_error": 306, + "algorithm_error": 307, + "no_continue_error": 308, + "no_space_error": 309, + "camera_error": 310, + "continue_success": 311, + "continue_fail": 312, + "usb_disk": 313, + "sd_space_not_enough": 314, + "memory_not_enough": 315, + "lidar_data_error": 316, + "mapping_error": 317, +} +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_MAX_REVISION = (1 << 63) - 1 +_MAX_RECONCILIATIONS = 4096 +_MAX_OPERATOR_CONFIRMATIONS = 4096 +PHYSICAL_COMMAND_MAX_OPERATOR_RETIREMENTS = 4096 +_MAX_OPERATOR_RETIREMENTS = PHYSICAL_COMMAND_MAX_OPERATOR_RETIREMENTS +_MAX_OPERATOR_RECONCILIATION_REOPENS = 4096 + + +def physical_transport_ref_comparison_key(value: str) -> str: + """Return a case-insensitive key without changing persisted/CAS evidence.""" + + if not isinstance(value, str): + raise TypeError("physical transport_ref must be a string") + key = value.strip().casefold() + if not key: + raise ValueError("physical transport_ref cannot be empty") + return key + + +class PhysicalCommandLedgerError(RuntimeError): + """Base error for the durable K1 START/STOP edge ledger.""" + + reason_code = "physical-command-ledger-error" + + +class PhysicalCommandBlocked(PhysicalCommandLedgerError): + """An unresolved or unsafe prior edge blocks another physical command.""" + + reason_code = "physical-command-reconciliation-required" + + +class PhysicalCommandLedgerCorrupt(PhysicalCommandBlocked): + """The durable evidence cannot be trusted and therefore fails closed.""" + + reason_code = "physical-command-ledger-corrupt" + + +class PhysicalCommandTransitionError(PhysicalCommandLedgerError): + """A caller attempted an invalid or semantically unsafe ledger transition.""" + + reason_code = "physical-command-ledger-transition-invalid" + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandIdentity: + """Secret-free hashes of both independently recovered K1 identities.""" + + vendor_device_id_sha256: str + device_serial_sha256: str + + def __post_init__(self) -> None: + _validate_sha256( + self.vendor_device_id_sha256, + field_name="vendor_device_id_sha256", + ) + _validate_sha256(self.device_serial_sha256, field_name="device_serial_sha256") + + def as_dict(self) -> dict[str, object]: + return { + "vendor_device_id_sha256": self.vendor_device_id_sha256, + "device_serial_sha256": self.device_serial_sha256, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandConnectionBinding: + """Exact live-path generation that owned one command edge.""" + + intent_id: str + transport_ref: str + connection_mode: PhysicalCommandConnectionMode + target_ipv4: str + target_port: int + host_path_epoch: int + control_session_id: str + producer_generation: int + + def __post_init__(self) -> None: + _validate_identifier(self.intent_id, field_name="intent_id") + _validate_identifier(self.transport_ref, field_name="transport_ref") + _validate_connection_mode(self.connection_mode) + _validate_ipv4(self.target_ipv4, field_name="target_ipv4") + _validate_port(self.target_port, field_name="target_port") + _validate_positive_int(self.host_path_epoch, field_name="host_path_epoch") + _validate_identifier(self.control_session_id, field_name="control_session_id") + _validate_positive_int( + self.producer_generation, + field_name="producer_generation", + ) + + def as_dict(self) -> dict[str, object]: + return { + "intent_id": self.intent_id, + "transport_ref": self.transport_ref, + "connection_mode": self.connection_mode, + "target_ipv4": self.target_ipv4, + "target_port": self.target_port, + "host_path_epoch": self.host_path_epoch, + "control_session_id": self.control_session_id, + "producer_generation": self.producer_generation, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandStatusEvidence: + """Bounded identity- and generation-bound DeviceStatus evidence.""" + + source: PhysicalStatusSource + vendor_device_id_sha256: str + device_serial_sha256: str + control_session_id: str + host_path_epoch: int + producer_generation: int + session_state: PhysicalSessionState + session_state_code: int + project_bound: bool + project_id_sha256: str | None + init_ready: bool + status_message_sha256: str + mqtt_retained: bool + observed_at_utc: str + + def __post_init__(self) -> None: + if self.source not in _STATUS_SOURCES: + raise ValueError("unsupported physical status evidence source") + _validate_sha256( + self.vendor_device_id_sha256, + field_name="vendor_device_id_sha256", + ) + _validate_sha256(self.device_serial_sha256, field_name="device_serial_sha256") + _validate_identifier(self.control_session_id, field_name="control_session_id") + _validate_positive_int(self.host_path_epoch, field_name="host_path_epoch") + _validate_positive_int( + self.producer_generation, + field_name="producer_generation", + ) + expected_code = _SESSION_STATE_CODES.get(self.session_state) + if expected_code is None or self.session_state_code != expected_code: + raise ValueError("physical session state and state code do not match") + if not isinstance(self.project_bound, bool): + raise TypeError("project_bound must be bool") + if self.project_bound: + if self.project_id_sha256 is None: + raise ValueError("bound physical status requires a project hash") + _validate_sha256(self.project_id_sha256, field_name="project_id_sha256") + elif self.project_id_sha256 is not None: + raise ValueError("unbound physical status cannot carry a project hash") + if not isinstance(self.init_ready, bool): + raise TypeError("init_ready must be bool") + _validate_sha256(self.status_message_sha256, field_name="status_message_sha256") + if not isinstance(self.mqtt_retained, bool): + raise TypeError("mqtt_retained must be bool") + _validated_timestamp(self.observed_at_utc, field_name="observed_at_utc") + + def as_dict(self) -> dict[str, object]: + return { + "source": self.source, + "vendor_device_id_sha256": self.vendor_device_id_sha256, + "device_serial_sha256": self.device_serial_sha256, + "control_session_id": self.control_session_id, + "host_path_epoch": self.host_path_epoch, + "producer_generation": self.producer_generation, + "session_state": self.session_state, + "session_state_code": self.session_state_code, + "project_bound": self.project_bound, + "project_id_sha256": self.project_id_sha256, + "init_ready": self.init_ready, + "status_message_sha256": self.status_message_sha256, + "mqtt_retained": self.mqtt_retained, + "observed_at_utc": self.observed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandApplicationResponse: + """Exact correlated application response, without raw vendor payload.""" + + operation_id: str + action: PhysicalCommandAction + control_session_id: str + host_path_epoch: int + producer_generation: int + result_code: int + success: bool + payload_sha256: str + observed_at_utc: str + + def __post_init__(self) -> None: + _validate_identifier(self.operation_id, field_name="operation_id") + _validate_action(self.action) + _validate_identifier(self.control_session_id, field_name="control_session_id") + _validate_positive_int(self.host_path_epoch, field_name="host_path_epoch") + _validate_positive_int( + self.producer_generation, + field_name="producer_generation", + ) + _validate_nonnegative_int(self.result_code, field_name="result_code") + if not isinstance(self.success, bool): + raise TypeError("success must be bool") + if self.success is not (self.result_code == PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE): + raise ValueError("application success must agree with result_code") + _validate_sha256(self.payload_sha256, field_name="payload_sha256") + _validated_timestamp(self.observed_at_utc, field_name="observed_at_utc") + + def as_dict(self) -> dict[str, object]: + return { + "operation_id": self.operation_id, + "action": self.action, + "control_session_id": self.control_session_id, + "host_path_epoch": self.host_path_epoch, + "producer_generation": self.producer_generation, + "result_code": self.result_code, + "success": self.success, + "payload_sha256": self.payload_sha256, + "observed_at_utc": self.observed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandAttemptAudit: + """Immutable full snapshot of one command before local recovery disposition.""" + + revision: int + operation_id: str + parent_operation_id: str | None + acquisition_id: str + action: PhysicalCommandAction + stage: PhysicalCommandStage + resolution: PhysicalCommandResolution | None + identity: PhysicalCommandIdentity + connection: PhysicalCommandConnectionBinding + compatibility_profile_id: str + payload_sha256: str + baseline_status: PhysicalCommandStatusEvidence + publish_call_returned: bool | None + packet_id: int | None + qos2_completed: bool + application_response: PhysicalCommandApplicationResponse | None + last_status: PhysicalCommandStatusEvidence | None + created_at_utc: str + updated_at_utc: str + + def __post_init__(self) -> None: + _validate_attempt_audit(self) + + def as_dict(self) -> dict[str, object]: + return { + "revision": self.revision, + "operation_id": self.operation_id, + "parent_operation_id": self.parent_operation_id, + "acquisition_id": self.acquisition_id, + "action": self.action, + "stage": self.stage, + "resolution": self.resolution, + "identity": self.identity.as_dict(), + "connection": self.connection.as_dict(), + "compatibility_profile_id": self.compatibility_profile_id, + "payload_sha256": self.payload_sha256, + "baseline_status": self.baseline_status.as_dict(), + "publish_call_returned": self.publish_call_returned, + "packet_id": self.packet_id, + "qos2_completed": self.qos2_completed, + "application_response": ( + self.application_response.as_dict() + if self.application_response is not None + else None + ), + "last_status": self.last_status.as_dict() if self.last_status is not None else None, + "created_at_utc": self.created_at_utc, + "updated_at_utc": self.updated_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandVerifiedBinding: + """Audited DeviceInfo verification for a new read-only control binding.""" + + verification_id: str + identity: PhysicalCommandIdentity + connection: PhysicalCommandConnectionBinding + device_info_message_sha256: str + verified_at_utc: str + + def __post_init__(self) -> None: + _validate_identifier(self.verification_id, field_name="verification_id") + if not isinstance(self.identity, PhysicalCommandIdentity): + raise TypeError("identity must be PhysicalCommandIdentity") + if not isinstance(self.connection, PhysicalCommandConnectionBinding): + raise TypeError("connection must be PhysicalCommandConnectionBinding") + _validate_sha256( + self.device_info_message_sha256, + field_name="device_info_message_sha256", + ) + _validated_timestamp(self.verified_at_utc, field_name="verified_at_utc") + + def as_dict(self) -> dict[str, object]: + return { + "verification_id": self.verification_id, + "identity": self.identity.as_dict(), + "connection": self.connection.as_dict(), + "device_info_message_sha256": self.device_info_message_sha256, + "verified_at_utc": self.verified_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandReconciliation: + """Read-only state classification or resolved-active control rebind. + + ``ambiguous-outcome`` never claims that its original command succeeded. + ``prepared-stop-classification`` records two facts atomically: the original + STOP is definitely ``not-dispatched``, while a fresh read-only dialogue + independently classifies the scanner as READY or SCANNING. In particular, + observing READY is never represented as successful STOP execution. + ``resolved-active-rebind`` preserves an already proven successful START + while replacing only the dead control-session authority needed for an + explicit operator-confirmed STOP. + """ + + reconciliation_id: str + kind: PhysicalCommandReconciliationKind + resolution: PhysicalCommandReconciliationResolution + original_attempt: PhysicalCommandAttemptAudit + original_attempt_sha256: str + verified_binding: PhysicalCommandVerifiedBinding + observation: PhysicalCommandStatusEvidence + reconciled_at_utc: str + + def __post_init__(self) -> None: + _validate_reconciliation(self) + + def as_dict(self) -> dict[str, object]: + return { + "reconciliation_id": self.reconciliation_id, + "kind": self.kind, + "resolution": self.resolution, + "original_attempt": self.original_attempt.as_dict(), + "original_attempt_sha256": self.original_attempt_sha256, + "verified_binding": self.verified_binding.as_dict(), + "observation": self.observation.as_dict(), + "reconciled_at_utc": self.reconciled_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandOperatorConfirmation: + """One explicit operator confirmation consumed by exactly one STOP attempt.""" + + confirmation_id: str + reconciliation_id: str + stop_operation_id: str + confirmed_at_utc: str + + def __post_init__(self) -> None: + _validate_identifier(self.confirmation_id, field_name="confirmation_id") + _validate_identifier(self.reconciliation_id, field_name="reconciliation_id") + _validate_identifier(self.stop_operation_id, field_name="stop_operation_id") + _validated_timestamp(self.confirmed_at_utc, field_name="confirmed_at_utc") + + def as_dict(self) -> dict[str, object]: + return { + "confirmation_id": self.confirmation_id, + "reconciliation_id": self.reconciliation_id, + "stop_operation_id": self.stop_operation_id, + "confirmed_at_utc": self.confirmed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandOperatorRetirement: + """Explicit local retirement of one unreachable physical target. + + Retirement transfers no command authority and makes no claim that the + scanner stopped. The complete ambiguous or last-proven-active attempt is + retained so a later replacement device can start a separate physical chain + without erasing or reinterpreting the old device-side outcome. + """ + + retirement_id: str + reason: PhysicalCommandOperatorRetirementReason + original_attempt: PhysicalCommandAttemptAudit + original_attempt_sha256: str + retired_transport_ref: str + retired_at_utc: str + + def __post_init__(self) -> None: + _validate_operator_retirement(self) + + def as_dict(self) -> dict[str, object]: + return { + "retirement_id": self.retirement_id, + "reason": self.reason, + "original_attempt": self.original_attempt.as_dict(), + "original_attempt_sha256": self.original_attempt_sha256, + "retired_transport_ref": self.retired_transport_ref, + "retired_at_utc": self.retired_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandOperatorReconciliationReopen: + """Local audit that re-admits one retired UUID for read-only settlement. + + This record is not a device identity proof and grants no command authority. + It merely records that the operator saw the exact retired CoreBluetooth UUID + in one fresh discovery generation and chose to reconcile that device again, + either through explicit recovery or the exact reset-owned Apply continuation. + """ + + reopening_id: str + retirement_id: str + retirement_sha256: str + reason: PhysicalCommandOperatorReconciliationReopenReason + retired_record_revision: int + reopened_transport_ref: str + discovery_generation: int + reopened_at_utc: str + + def __post_init__(self) -> None: + _validate_operator_reconciliation_reopen(self) + + def as_dict(self) -> dict[str, object]: + return { + "reopening_id": self.reopening_id, + "retirement_id": self.retirement_id, + "retirement_sha256": self.retirement_sha256, + "reason": self.reason, + "retired_record_revision": self.retired_record_revision, + "reopened_transport_ref": self.reopened_transport_ref, + "discovery_generation": self.discovery_generation, + "reopened_at_utc": self.reopened_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandRecord: + schema_version: str + revision: int + operation_id: str + parent_operation_id: str | None + acquisition_id: str + action: PhysicalCommandAction + stage: PhysicalCommandStage + identity: PhysicalCommandIdentity + connection: PhysicalCommandConnectionBinding + compatibility_profile_id: str + payload_sha256: str + baseline_status: PhysicalCommandStatusEvidence + publish_call_returned: bool | None + packet_id: int | None + qos2_completed: bool + application_response: PhysicalCommandApplicationResponse | None + last_status: PhysicalCommandStatusEvidence | None + reconciliations: tuple[PhysicalCommandReconciliation, ...] + operator_confirmations: tuple[PhysicalCommandOperatorConfirmation, ...] + operator_retirements: tuple[PhysicalCommandOperatorRetirement, ...] + resolution: PhysicalCommandResolution | None + created_at_utc: str + updated_at_utc: str + operator_reconciliation_reopens: tuple[PhysicalCommandOperatorReconciliationReopen, ...] = () + archive_head_sha256: str | None = None + archive_segment_count: int = 0 + archive_total_bytes: int = 0 + + @property + def unresolved(self) -> bool: + return self.stage != "resolved" + + @property + def automatic_replay_allowed(self) -> bool: + """Physical command edges are never safe to replay automatically.""" + + return False + + @property + def original_command_outcome(self) -> PhysicalCommandOriginalOutcome: + """Separate original edge outcome from read-only physical classification.""" + + if self.resolution == "not-dispatched": + return "not-dispatched" + if self.resolution == "operator-retired-outcome-unknown": + retirement = next( + ( + item + for item in reversed(self.operator_retirements) + if item.original_attempt.operation_id == self.operation_id + ), + None, + ) + if retirement is not None and retirement.original_attempt.resolution == ( + "not-dispatched" + ): + return "not-dispatched" + if self.resolution in {"start-active-observed", "stop-standby-observed"}: + return "succeeded" + return "unknown" + + @property + def reconciled_physical_state(self) -> PhysicalCommandObservedState | None: + if self.resolution == "operator-retired-outcome-unknown": + return None + if self.reopened_physical_state_requires_reconciliation: + # Re-admitting the retired CoreBluetooth UUID transfers no state or + # command authority. The last pre-retirement SCANNING proof stays + # in audit, but current truth remains unknown until a new exact + # DeviceInfo + non-retained DeviceStatus generation is appended. + return None + latest_recovery = _latest_current_physical_state_recovery(self) + if latest_recovery is not None: + return ( + "active" + if latest_recovery.resolution == "physical-active-observed" + else "standby" + ) + if self.resolution == "physical-active-observed": + return "active" + if self.resolution == "physical-standby-observed": + return "standby" + if self.resolution == "start-active-observed": + return "active" + return None + + @property + def reopened_physical_state_requires_reconciliation(self) -> bool: + """Whether a reopened classified STOP still lacks post-reopen proof.""" + + if self.stage != "resolved" or self.resolution != "not-dispatched": + return False + reopening = _latest_current_classified_stop_reopen(self) + return bool( + reopening is not None + and self.revision == reopening.retired_record_revision + 1 + ) + + @property + def classified_stop_standby_successor_pending(self) -> bool: + """Whether a classified undispatched STOP still owns a future START. + + A normal continuation must preserve this binding. An explicit + connection-scenario reset may retire it locally, however: READY is a + durable standby proof, and keeping the old future-START binding after + that reset would make a newly selected BLE device impossible to + provision. Retirement preserves the full STOP and READY audit and + grants no physical command authority. + """ + + if not ( + self.stage == "resolved" + and self.action == "stop" + and self.resolution == "not-dispatched" + and not self.reopened_physical_state_requires_reconciliation + ): + return False + recovery = _latest_prepared_stop_recovery(self) + return bool( + recovery is not None + and recovery.resolution == "physical-standby-observed" + ) + + @property + def reopened_resolved_start_requires_reconciliation(self) -> bool: + """Whether a proven START was reopened and still needs fresh state. + + Scenario reset keeps the immutable successful START and all later + read-only active rebinds, but temporarily moves the current record back + to ``observing``. That shape is not an ambiguous original command: a + fresh READY/SCANNING proof must extend the resolved-active lineage so a + repeated reset cannot append a second ``ambiguous-outcome`` for the + same START. + """ + + return _reopened_resolved_start_attempt(self) is not None + + @property + def resolved_unclassified_stop_recovery_required(self) -> bool: + """Whether a proven no-dispatch STOP still lacks current device state. + + Older startup recovery could durably close a PREPARED STOP as + ``not-dispatched`` before explicit Verify had a chance to append the + independent READY/SCANNING classification. That local command outcome + is definite, but it grants no successor authority until a fresh + DeviceInfo binding and non-retained DeviceStatus are recorded. + """ + + return bool( + self.stage == "resolved" + and self.action == "stop" + and self.resolution == "not-dispatched" + and self.publish_call_returned is None + and self.packet_id is None + and not self.qos2_completed + and self.application_response is None + and self.last_status is None + and not any( + item.original_attempt.operation_id == self.operation_id + for item in self.reconciliations + ) + ) + + @property + def successor_stop_requires_operator_confirmation(self) -> bool: + """Whether the next STOP consumes a reconciliation-only confirmation. + + Normal same-session START -> STOP edges already have durable command + lineage and must not be mislabeled as restart reconciliation. An + additional confirmation id is consumed only when read-only recovery + observed an active scanner, or when retrying a STOP that had already + consumed such a recovery confirmation but was proven not dispatched. + """ + + if self.unresolved: + return False + if self.resolution == "physical-active-observed": + return True + if self.resolution == "start-active-observed": + latest = _latest_resolved_start_recovery(self) + return latest is not None and latest.resolution == "physical-active-observed" + if self.resolution == "not-dispatched" and self.action == "stop": + latest = _latest_current_physical_state_recovery(self) + if latest is not None: + return latest.resolution == "physical-active-observed" + return any( + confirmation.stop_operation_id == self.operation_id + for confirmation in self.operator_confirmations + ) + return False + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "revision": self.revision, + "operation_id": self.operation_id, + "parent_operation_id": self.parent_operation_id, + "acquisition_id": self.acquisition_id, + "action": self.action, + "stage": self.stage, + "identity": self.identity.as_dict(), + "connection": self.connection.as_dict(), + "compatibility_profile_id": self.compatibility_profile_id, + "payload_sha256": self.payload_sha256, + "baseline_status": self.baseline_status.as_dict(), + "publish_call_returned": self.publish_call_returned, + "packet_id": self.packet_id, + "qos2_completed": self.qos2_completed, + "application_response": ( + self.application_response.as_dict() + if self.application_response is not None + else None + ), + "last_status": self.last_status.as_dict() if self.last_status is not None else None, + "reconciliations": [item.as_dict() for item in self.reconciliations], + "operator_confirmations": [item.as_dict() for item in self.operator_confirmations], + "operator_retirements": [item.as_dict() for item in self.operator_retirements], + "operator_reconciliation_reopens": [ + item.as_dict() for item in self.operator_reconciliation_reopens + ], + "resolution": self.resolution, + "created_at_utc": self.created_at_utc, + "updated_at_utc": self.updated_at_utc, + "archive_head_sha256": self.archive_head_sha256, + "archive_segment_count": self.archive_segment_count, + "archive_total_bytes": self.archive_total_bytes, + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandArchiveSegment: + """Content-addressed immutable predecessor snapshot for ledger rollover.""" + + sequence: int + predecessor_sha256: str | None + record_sha256: str + record: PhysicalCommandRecord + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": PHYSICAL_COMMAND_ARCHIVE_SCHEMA, + "sequence": self.sequence, + "predecessor_sha256": self.predecessor_sha256, + "record_sha256": self.record_sha256, + "record": self.record.as_dict(), + } + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandClassifiedStopAncestryProof: + """Bounded commitment to an operator-confirmed STOP ancestry chain. + + The full reconciliation/confirmation path remains in the physical ledger + and its immutable archive. This fixed-size certificate is safe to copy + into the active-acquisition checkpoint; callers must recompute it through + :meth:`PhysicalCommandLedger.prove_classified_stop_ancestry` after restart. + """ + + schema_version: Literal["missioncore.xgrids-k1-classified-stop-ancestry/v1"] + ledger_revision: int + head_operation_id: str + head_parent_operation_id: str + root_start_operation_id: str + acquisition_id: str + depth: int + chain_sha256: str + + def __post_init__(self) -> None: + if self.schema_version != PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA: + raise ValueError("unsupported classified STOP ancestry schema") + _validate_positive_int(self.ledger_revision, field_name="ledger_revision") + for field_name, value in ( + ("head_operation_id", self.head_operation_id), + ("head_parent_operation_id", self.head_parent_operation_id), + ("root_start_operation_id", self.root_start_operation_id), + ("acquisition_id", self.acquisition_id), + ): + _validate_identifier(value, field_name=field_name) + if self.head_operation_id in { + self.head_parent_operation_id, + self.root_start_operation_id, + }: + raise ValueError("classified STOP ancestry head is not a successor") + if self.head_parent_operation_id == self.root_start_operation_id: + raise ValueError("direct START parent must not carry an ancestry certificate") + _validate_positive_int(self.depth, field_name="depth") + if self.depth > _MAX_RECONCILIATIONS: + raise ValueError("classified STOP ancestry exceeds the bounded depth") + _validate_sha256(self.chain_sha256, field_name="chain_sha256") + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "ledger_revision": self.ledger_revision, + "head_operation_id": self.head_operation_id, + "head_parent_operation_id": self.head_parent_operation_id, + "root_start_operation_id": self.root_start_operation_id, + "acquisition_id": self.acquisition_id, + "depth": self.depth, + "chain_sha256": self.chain_sha256, + } + + +@dataclass(frozen=True, slots=True) +class _PhysicalCommandArchiveHistory: + """Bounded, fully verified identity index over immutable archive segments.""" + + head_sha256: str | None = None + segment_count: int = 0 + total_bytes: int = 0 + # The newest segment is the exact full record immediately preceding the + # compacted main ledger. It is populated only after the complete + # content-addressed archive chain has been reloaded and verified. + head_record: PhysicalCommandRecord | None = None + operation_ids: frozenset[str] = frozenset() + reconciliation_ids: frozenset[str] = frozenset() + verification_ids: frozenset[str] = frozenset() + verified_control_session_ids: frozenset[str] = frozenset() + confirmation_ids: frozenset[str] = frozenset() + retirement_ids: frozenset[str] = frozenset() + reopening_ids: frozenset[str] = frozenset() + lineage_reconciliations_by_id: tuple[ + tuple[str, PhysicalCommandReconciliation], ... + ] = () + lineage_classification_ids_by_operation: tuple[tuple[str, str], ...] = () + lineage_confirmations_by_stop_operation: tuple[ + tuple[str, PhysicalCommandOperatorConfirmation], ... + ] = () + lineage_attempt_sha256_by_operation: tuple[tuple[str, str], ...] = () + + +@dataclass(frozen=True, slots=True) +class _PhysicalCommandPersistencePlan: + """Fully validated write plan shared by preflight and publication.""" + + record: PhysicalCommandRecord + history: _PhysicalCommandArchiveHistory + archive_head_sha256: str | None = None + archive_payload: bytes | None = None + + +@dataclass(frozen=True, slots=True) +class PhysicalCommandLedgerSnapshot: + status: PhysicalCommandLedgerStatus + record: PhysicalCommandRecord | None + reason_code: str | None + + @property + def requires_reconciliation(self) -> bool: + return self.status in {"unresolved", "corrupt"} + + @property + def automatic_replay_allowed(self) -> bool: + return False + + +def active_operator_retirements( + record: PhysicalCommandRecord, +) -> tuple[PhysicalCommandOperatorRetirement, ...]: + """Return only retirements that still fence their physical target.""" + + reopened_retirement_ids = { + item.retirement_id for item in record.operator_reconciliation_reopens + } + return tuple( + item + for item in record.operator_retirements + if item.retirement_id not in reopened_retirement_ids + ) + + +def _compact_record_after_archive( + record: PhysicalCommandRecord, + *, + archive_head_sha256: str, + archive_segment_count: int, + archive_total_bytes: int, +) -> PhysicalCommandRecord: + """Retain the complete live safety frontier after archiving a predecessor. + + The immutable segment keeps the full former document. The main file keeps + all active retirement deny-fences, every reconciliation belonging to the + current physical operation, confirmation dependencies, and the newest + retire/reopen pair for lost-response idempotency. + """ + + active_retirement_ids = {item.retirement_id for item in active_operator_retirements(record)} + retained_retirement_ids = set(active_retirement_ids) + retained_reopening_ids: set[str] = set() + if record.operator_retirements: + retained_retirement_ids.add(record.operator_retirements[-1].retirement_id) + if record.operator_reconciliation_reopens: + latest_reopening = record.operator_reconciliation_reopens[-1] + retained_reopening_ids.add(latest_reopening.reopening_id) + retained_retirement_ids.add(latest_reopening.retirement_id) + + retirements = tuple( + item + for item in record.operator_retirements + if item.retirement_id in retained_retirement_ids + ) + retained_retirement_ids = {item.retirement_id for item in retirements} + reopens = tuple( + item + for item in record.operator_reconciliation_reopens + if item.reopening_id in retained_reopening_ids + and item.retirement_id in retained_retirement_ids + ) + + confirmations = tuple( + item + for item in record.operator_confirmations + if item.stop_operation_id == record.operation_id + ) + required_reconciliation_ids = {item.reconciliation_id for item in confirmations} + required_reconciliation_operation_ids = { + item.original_attempt.operation_id + for item in retirements + if item.original_attempt.resolution == "not-dispatched" + } + required_reconciliation_operation_ids.update( + item.original_attempt.operation_id + for item in record.reconciliations + if item.reconciliation_id in required_reconciliation_ids + ) + reconciliations = tuple( + item + for item in record.reconciliations + if item.original_attempt.operation_id == record.operation_id + or item.reconciliation_id in required_reconciliation_ids + or item.original_attempt.operation_id + in required_reconciliation_operation_ids + ) + + return replace( + record, + reconciliations=reconciliations, + operator_confirmations=confirmations, + operator_retirements=retirements, + operator_reconciliation_reopens=reopens, + archive_head_sha256=archive_head_sha256, + archive_segment_count=archive_segment_count, + archive_total_bytes=archive_total_bytes, + ) + + +def _register_lineage_proof_indexes( + record: PhysicalCommandRecord, + *, + reconciliations_by_id: dict[str, PhysicalCommandReconciliation], + classification_ids_by_operation: dict[str, str], + confirmations_by_stop_operation: dict[str, PhysicalCommandOperatorConfirmation], + attempt_sha256_by_operation: dict[str, str], +) -> None: + """Retain immutable typed evidence needed for archived STOP ancestry.""" + + for reconciliation in record.reconciliations: + prior_reconciliation = reconciliations_by_id.setdefault( + reconciliation.reconciliation_id, + reconciliation, + ) + if prior_reconciliation != reconciliation: + raise PhysicalCommandLedgerCorrupt( + "physical command reconciliation changed across lineage history" + ) + attempt = reconciliation.original_attempt + if not _is_exact_not_dispatched_stop_attempt(attempt): + continue + prior_attempt_sha256 = attempt_sha256_by_operation.setdefault( + attempt.operation_id, + reconciliation.original_attempt_sha256, + ) + if prior_attempt_sha256 != reconciliation.original_attempt_sha256: + raise PhysicalCommandLedgerCorrupt( + "classified STOP attempt changed across lineage history" + ) + if reconciliation.kind != "prepared-stop-classification": + continue + prior_classification_id = classification_ids_by_operation.setdefault( + attempt.operation_id, + reconciliation.reconciliation_id, + ) + if prior_classification_id != reconciliation.reconciliation_id: + raise PhysicalCommandLedgerCorrupt( + "classified STOP operation has conflicting classification audits" + ) + + for confirmation in record.operator_confirmations: + prior_confirmation = confirmations_by_stop_operation.setdefault( + confirmation.stop_operation_id, + confirmation, + ) + if prior_confirmation != confirmation: + raise PhysicalCommandLedgerCorrupt( + "successor STOP has conflicting operator confirmations" + ) + + +def _extend_archive_history( + history: _PhysicalCommandArchiveHistory, + *, + archived_record: PhysicalCommandRecord, + head_sha256: str, + segment_bytes: int, +) -> _PhysicalCommandArchiveHistory: + operation_ids = set(history.operation_ids) + reconciliation_ids = set(history.reconciliation_ids) + verification_ids = set(history.verification_ids) + verified_control_session_ids = set(history.verified_control_session_ids) + confirmation_ids = set(history.confirmation_ids) + retirement_ids = set(history.retirement_ids) + reopening_ids = set(history.reopening_ids) + lineage_reconciliations_by_id = dict(history.lineage_reconciliations_by_id) + lineage_classification_ids_by_operation = dict( + history.lineage_classification_ids_by_operation + ) + lineage_confirmations_by_stop_operation = dict( + history.lineage_confirmations_by_stop_operation + ) + lineage_attempt_sha256_by_operation = dict( + history.lineage_attempt_sha256_by_operation + ) + _add_record_identities_to_archive_history( + archived_record, + operation_ids=operation_ids, + reconciliation_ids=reconciliation_ids, + verification_ids=verification_ids, + verified_control_session_ids=verified_control_session_ids, + confirmation_ids=confirmation_ids, + retirement_ids=retirement_ids, + reopening_ids=reopening_ids, + ) + _register_lineage_proof_indexes( + archived_record, + reconciliations_by_id=lineage_reconciliations_by_id, + classification_ids_by_operation=lineage_classification_ids_by_operation, + confirmations_by_stop_operation=lineage_confirmations_by_stop_operation, + attempt_sha256_by_operation=lineage_attempt_sha256_by_operation, + ) + return _PhysicalCommandArchiveHistory( + head_sha256=head_sha256, + segment_count=history.segment_count + 1, + total_bytes=history.total_bytes + segment_bytes, + head_record=archived_record, + operation_ids=frozenset(operation_ids), + reconciliation_ids=frozenset(reconciliation_ids), + verification_ids=frozenset(verification_ids), + verified_control_session_ids=frozenset(verified_control_session_ids), + confirmation_ids=frozenset(confirmation_ids), + retirement_ids=frozenset(retirement_ids), + reopening_ids=frozenset(reopening_ids), + lineage_reconciliations_by_id=tuple( + sorted(lineage_reconciliations_by_id.items()) + ), + lineage_classification_ids_by_operation=tuple( + sorted(lineage_classification_ids_by_operation.items()) + ), + lineage_confirmations_by_stop_operation=tuple( + sorted(lineage_confirmations_by_stop_operation.items()) + ), + lineage_attempt_sha256_by_operation=tuple( + sorted(lineage_attempt_sha256_by_operation.items()) + ), + ) + + +def _add_record_identities_to_archive_history( + record: PhysicalCommandRecord, + *, + operation_ids: set[str], + reconciliation_ids: set[str], + verification_ids: set[str], + verified_control_session_ids: set[str], + confirmation_ids: set[str], + retirement_ids: set[str], + reopening_ids: set[str], +) -> None: + operation_ids.add(record.operation_id) + if record.parent_operation_id is not None: + operation_ids.add(record.parent_operation_id) + verified_control_session_ids.add(record.connection.control_session_id) + for reconciliation in record.reconciliations: + operation_ids.add(reconciliation.original_attempt.operation_id) + if reconciliation.original_attempt.parent_operation_id is not None: + operation_ids.add(reconciliation.original_attempt.parent_operation_id) + reconciliation_ids.add(reconciliation.reconciliation_id) + verification_ids.add(reconciliation.verified_binding.verification_id) + verified_control_session_ids.add( + reconciliation.verified_binding.connection.control_session_id + ) + verified_control_session_ids.add( + reconciliation.original_attempt.connection.control_session_id + ) + for confirmation in record.operator_confirmations: + confirmation_ids.add(confirmation.confirmation_id) + operation_ids.add(confirmation.stop_operation_id) + for retirement in record.operator_retirements: + retirement_ids.add(retirement.retirement_id) + operation_ids.add(retirement.original_attempt.operation_id) + verified_control_session_ids.add(retirement.original_attempt.connection.control_session_id) + if retirement.original_attempt.parent_operation_id is not None: + operation_ids.add(retirement.original_attempt.parent_operation_id) + for reopening in record.operator_reconciliation_reopens: + reopening_ids.add(reopening.reopening_id) + + +class PhysicalCommandLedger: + """Durable, secret-free fence for edge-triggered K1 START and STOP. + + ``dispatching`` is persisted immediately before the MQTT publish call. A + crash at or after that point is deliberately ambiguous and never becomes + an automatic retry. Resolution after dispatch requires three independent + facts for the same operation and live-path generation: QoS2 completion, an + exactly correlated successful application response, and the exact target + DeviceStatus. Status alone is evidence, not proof that this edge applied. + + Wall UTC fields are audit metadata, not transition authority. The caller + delivers MQTT evidence serially; exact operation id, stage/revision, + identity, control-session id, host-path epoch, producer generation, packet + id and one-shot reconciliation/confirmation ids carry the ordering and + binding invariants even when the host clock moves backwards. + + After restart, ``reconcile_ambiguous`` may instead classify only the + currently observed physical state from a new DeviceInfo-verified binding. + That resolution deliberately keeps the original command outcome unknown; + its immutable attempt snapshot, new binding, and read-only status are all + retained. An observed active state can feed only an explicitly confirmed + STOP on that binding, and no reconciliation ever enables automatic replay. + + This class performs no network or device I/O. It also does not grant a + generic command admission: callers still own live lease, topology, identity + and current-status gates. Its transition rules only ensure that the + durable edge history itself cannot authorize START/STOP inversions. + """ + + def __init__( + self, + repository_root: Path, + *, + clock: Callable[[], datetime] | None = None, + ) -> None: + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = data_dir / "xgrids-k1" / PHYSICAL_COMMAND_LEDGER_FILENAME + self._process_lock_path = data_dir / "xgrids-k1" / PHYSICAL_COMMAND_LEDGER_LOCK_FILENAME + self._archive_dir = data_dir / "xgrids-k1" / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + self._data_dir = data_dir + self._clock = clock or (lambda: datetime.now(UTC)) + self._lock = threading.RLock() + self._record: PhysicalCommandRecord | None = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = False + with self._lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> PhysicalCommandLedgerSnapshot: + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return PhysicalCommandLedgerSnapshot( + status="corrupt", + record=None, + reason_code=PhysicalCommandLedgerCorrupt.reason_code, + ) + if self._record is None: + return PhysicalCommandLedgerSnapshot( + status="empty", + record=None, + reason_code=None, + ) + return PhysicalCommandLedgerSnapshot( + status="unresolved" if self._record.unresolved else "resolved", + record=self._record, + reason_code=( + PhysicalCommandBlocked.reason_code if self._record.unresolved else None + ), + ) + + def prove_classified_stop_ancestry( + self, + *, + head_operation_id: str, + expected_start_operation_id: str, + acquisition_id: str, + ) -> PhysicalCommandClassifiedStopAncestryProof | None: + """Prove the exact current STOP head back to its original START. + + A direct ``STOP -> START`` head needs no certificate and returns + ``None``. A longer chain is accepted only when every intervening STOP + was durably proven not dispatched, independently observed active, and + consumed by the exact successor's operator confirmation. Archive + objects are reloaded and fully verified under the process lock on every + call, so this method is also the restart revalidation boundary. + """ + + _validate_identifier(head_operation_id, field_name="head_operation_id") + _validate_identifier( + expected_start_operation_id, + field_name="expected_start_operation_id", + ) + _validate_identifier(acquisition_id, field_name="acquisition_id") + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "classified STOP ancestry cannot use a corrupt ledger" + ) + record = self._record + if record is None: + raise PhysicalCommandTransitionError( + "classified STOP ancestry requires a current physical head" + ) + try: + return _prove_classified_stop_ancestry( + record, + archive_history=self._archive_history, + head_operation_id=head_operation_id, + expected_start_operation_id=expected_start_operation_id, + acquisition_id=acquisition_id, + ) + except PhysicalCommandLedgerCorrupt: + self._corrupt = True + raise + + def prove_immediate_archived_reconciliation_successor( + self, + *, + expected_current: PhysicalCommandRecord, + predecessor: PhysicalCommandRecord, + ) -> bool: + """Prove one exact reconciliation that archived its predecessor. + + A reconciliation can cross the bounded main-ledger size and compact + historical audits. Callers must not mistake that canonical + compaction for a stale checkpoint token, but accepting an arbitrary + archived ancestor would make old tokens current again. This proof is + therefore deliberately limited to the exact immediate predecessor of + the current main record and one appended read-only reconciliation. + + The archive loader verifies every content address, embedded record + digest, predecessor link, sequence, revision and byte count while this + method owns the ledger process lock. ``head_record`` is consequently + authority only inside this locked, freshly reloaded comparison. + """ + + if not isinstance(expected_current, PhysicalCommandRecord): + raise TypeError("expected_current must be PhysicalCommandRecord") + if not isinstance(predecessor, PhysicalCommandRecord): + raise TypeError("predecessor must be PhysicalCommandRecord") + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "archived predecessor proof cannot use a corrupt ledger" + ) + current = self._record + if current is None or current != expected_current: + return False + history = self._archive_history + archive_head_sha256 = current.archive_head_sha256 + if not ( + archive_head_sha256 is not None + and + history.head_record == predecessor + and history.head_sha256 == archive_head_sha256 + and history.segment_count == current.archive_segment_count + and history.total_bytes == current.archive_total_bytes + and current.revision == predecessor.revision + 1 + and current.archive_segment_count + == predecessor.archive_segment_count + 1 + and current.schema_version == predecessor.schema_version + and current.operation_id == predecessor.operation_id + and current.parent_operation_id == predecessor.parent_operation_id + and current.acquisition_id == predecessor.acquisition_id + and current.action == predecessor.action + and current.identity == predecessor.identity + and current.connection == predecessor.connection + and current.compatibility_profile_id + == predecessor.compatibility_profile_id + and current.payload_sha256 == predecessor.payload_sha256 + and current.baseline_status == predecessor.baseline_status + and current.created_at_utc == predecessor.created_at_utc + ): + return False + + predecessor_reconciliation_ids = { + item.reconciliation_id for item in predecessor.reconciliations + } + appended = tuple( + item + for item in current.reconciliations + if item.reconciliation_id not in predecessor_reconciliation_ids + ) + allowed_reconciliation = { + ("ambiguous-outcome", "physical-active-observed"), + ("ambiguous-outcome", "physical-standby-observed"), + ("prepared-stop-classification", "physical-active-observed"), + ("prepared-stop-classification", "physical-standby-observed"), + ("resolved-active-rebind", "physical-active-observed"), + ("resolved-active-cessation", "physical-standby-observed"), + } + if len(appended) != 1 or ( + appended[0].kind, + appended[0].resolution, + ) not in allowed_reconciliation: + return False + uncompacted = replace( + predecessor, + revision=current.revision, + stage=current.stage, + reconciliations=(*predecessor.reconciliations, appended[0]), + resolution=current.resolution, + updated_at_utc=current.updated_at_utc, + ) + return ( + _compact_record_after_archive( + uncompacted, + archive_head_sha256=archive_head_sha256, + archive_segment_count=current.archive_segment_count, + archive_total_bytes=current.archive_total_bytes, + ) + == current + ) + + def require_no_unresolved_attempt(self) -> None: + """Require only a clear durable attempt fence, not live command safety.""" + + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_no_unresolved_attempt_locked() + + def require_runtime_not_retired( + self, + *, + identity: PhysicalCommandIdentity, + connection: PhysicalCommandConnectionBinding, + ) -> None: + """Reject a DeviceInfo/runtime binding overlapping any retired K1. + + This check runs at control-session admission, before workspace/project + commands can use the verified DeviceInfo identity. It intentionally + shares the ledger process lock with retirement so a bind observes one + complete durable history rather than racing its publication. + """ + + if not isinstance(identity, PhysicalCommandIdentity): + raise TypeError("identity must be PhysicalCommandIdentity") + if not isinstance(connection, PhysicalCommandConnectionBinding): + raise TypeError("connection must be PhysicalCommandConnectionBinding") + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise PhysicalCommandBlocked( + "physical command audit is corrupt; runtime binding is blocked" + ) + current = self._record + if current is None: + return + if any( + retirement.original_attempt.identity.vendor_device_id_sha256 + == identity.vendor_device_id_sha256 + or retirement.original_attempt.identity.device_serial_sha256 + == identity.device_serial_sha256 + or physical_transport_ref_comparison_key(retirement.retired_transport_ref) + == physical_transport_ref_comparison_key(connection.transport_ref) + for retirement in active_operator_retirements(current) + ): + raise PhysicalCommandBlocked( + "retired physical target cannot bind a control session" + ) + + def prepare( + self, + *, + operation_id: str, + parent_operation_id: str | None, + acquisition_id: str, + action: PhysicalCommandAction, + identity: PhysicalCommandIdentity, + connection: PhysicalCommandConnectionBinding, + compatibility_profile_id: str, + payload_sha256: str, + baseline_status: PhysicalCommandStatusEvidence, + operator_confirmation_id: str | None = None, + operator_confirmed_at_utc: str | None = None, + ) -> PhysicalCommandRecord: + """Persist one edge after the caller has acquired all live safety gates.""" + + _validate_identifier(operation_id, field_name="operation_id") + if parent_operation_id is not None: + _validate_identifier(parent_operation_id, field_name="parent_operation_id") + _validate_identifier(acquisition_id, field_name="acquisition_id") + _validate_action(action) + if not isinstance(identity, PhysicalCommandIdentity): + raise TypeError("identity must be PhysicalCommandIdentity") + if not isinstance(connection, PhysicalCommandConnectionBinding): + raise TypeError("connection must be PhysicalCommandConnectionBinding") + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + _validate_sha256(payload_sha256, field_name="payload_sha256") + if not isinstance(baseline_status, PhysicalCommandStatusEvidence): + raise TypeError("baseline_status must be PhysicalCommandStatusEvidence") + if (operator_confirmation_id is None) is not (operator_confirmed_at_utc is None): + raise ValueError("operator confirmation id and timestamp must be supplied together") + if operator_confirmation_id is not None: + _validate_identifier( + operator_confirmation_id, + field_name="operator_confirmation_id", + ) + if operator_confirmed_at_utc is not None: + _validated_timestamp( + operator_confirmed_at_utc, + field_name="operator_confirmed_at_utc", + ) + _require_bound_evidence(identity, connection, baseline_status) + _require_live_baseline(action, baseline_status) + + with self._lock, self._process_lock_locked(): + self._reload_locked() + self._require_no_unresolved_attempt_locked() + previous = self._record + required_reconciliation_id = _require_safe_edge_successor( + previous, + operation_id=operation_id, + parent_operation_id=parent_operation_id, + acquisition_id=acquisition_id, + action=action, + identity=identity, + connection=connection, + compatibility_profile_id=compatibility_profile_id, + archived_operation_ids=self._archive_history.operation_ids, + ) + _require_successor_baseline_after_reconciliation( + previous, + required_reconciliation_id=required_reconciliation_id, + baseline_status=baseline_status, + ) + previous_revision = previous.revision if previous is not None else 0 + if previous_revision >= _MAX_REVISION: + raise PhysicalCommandTransitionError("physical command revision is exhausted") + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=(previous.updated_at_utc if previous is not None else None), + ) + reconciliations = previous.reconciliations if previous is not None else () + confirmations = previous.operator_confirmations if previous is not None else () + retirements = previous.operator_retirements if previous is not None else () + reconciliation_reopens = ( + previous.operator_reconciliation_reopens if previous is not None else () + ) + if required_reconciliation_id is None: + if operator_confirmation_id is not None: + raise PhysicalCommandTransitionError( + "operator confirmation is valid only for reconciled active STOP" + ) + else: + if operator_confirmation_id is None or operator_confirmed_at_utc is None: + raise PhysicalCommandTransitionError( + "reconciled active state requires explicit operator-confirmed STOP" + ) + if any(item.confirmation_id == operator_confirmation_id for item in confirmations): + raise PhysicalCommandTransitionError( + "operator confirmation id has already been consumed" + ) + if operator_confirmation_id in self._archive_history.confirmation_ids: + raise PhysicalCommandTransitionError( + "operator confirmation id has already been consumed" + ) + _reconciliation_by_id( + reconciliations, + required_reconciliation_id, + ) + if len(confirmations) >= _MAX_OPERATOR_CONFIRMATIONS: + raise PhysicalCommandBlocked( + "physical confirmation audit is full and requires archival" + ) + confirmations = ( + *confirmations, + PhysicalCommandOperatorConfirmation( + confirmation_id=operator_confirmation_id, + reconciliation_id=required_reconciliation_id, + stop_operation_id=operation_id, + confirmed_at_utc=operator_confirmed_at_utc, + ), + ) + record = PhysicalCommandRecord( + schema_version=PHYSICAL_COMMAND_LEDGER_SCHEMA, + revision=previous_revision + 1, + operation_id=operation_id, + parent_operation_id=parent_operation_id, + acquisition_id=acquisition_id, + action=action, + stage="prepared", + identity=identity, + connection=connection, + compatibility_profile_id=compatibility_profile_id, + payload_sha256=payload_sha256, + baseline_status=baseline_status, + publish_call_returned=None, + packet_id=None, + qos2_completed=False, + application_response=None, + last_status=None, + reconciliations=reconciliations, + operator_confirmations=confirmations, + operator_retirements=retirements, + resolution=None, + created_at_utc=now, + updated_at_utc=now, + operator_reconciliation_reopens=reconciliation_reopens, + archive_head_sha256=( + previous.archive_head_sha256 if previous is not None else None + ), + archive_segment_count=( + previous.archive_segment_count if previous is not None else 0 + ), + archive_total_bytes=(previous.archive_total_bytes if previous is not None else 0), + ) + return self._persist_locked(record) + + def mark_dispatching(self, operation_id: str) -> PhysicalCommandRecord: + """Durably cross the no-replay boundary immediately before publish.""" + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage != "prepared": + raise PhysicalCommandTransitionError( + "physical command may dispatch only from prepared" + ) + return self._transition_locked(current, stage="dispatching") + + def mark_observing( + self, + operation_id: str, + *, + publish_call_returned: bool, + packet_id: int | None = None, + ) -> PhysicalCommandRecord: + """Record the local publish-call outcome; it is not application success.""" + + if not isinstance(publish_call_returned, bool): + raise TypeError("publish_call_returned must be bool") + if packet_id is not None: + _validate_positive_int(packet_id, field_name="packet_id") + if not publish_call_returned: + raise ValueError("packet_id requires a returned publish call") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "physical command may observe only after dispatch" + ) + if ( + current.publish_call_returned is not None + and current.publish_call_returned is not publish_call_returned + ): + raise PhysicalCommandTransitionError( + "publish-call outcome cannot change within one physical attempt" + ) + if ( + current.packet_id is not None + and packet_id is not None + and current.packet_id != packet_id + ): + raise PhysicalCommandTransitionError("physical command packet_id cannot change") + effective_packet_id = current.packet_id if packet_id is None else packet_id + if ( + current.stage == "observing" + and current.publish_call_returned is publish_call_returned + and current.packet_id == effective_packet_id + ): + return current + return self._transition_locked( + current, + stage="observing", + publish_call_returned=publish_call_returned, + packet_id=effective_packet_id, + ) + + def mark_qos2_completed( + self, + operation_id: str, + *, + packet_id: int, + ) -> PhysicalCommandRecord: + """Record the exact QoS2 publication completion, never retry it.""" + + _validate_positive_int(packet_id, field_name="packet_id") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "QoS2 completion requires a dispatched physical command" + ) + if current.publish_call_returned is False: + raise PhysicalCommandTransitionError( + "QoS2 completion cannot reverse a failed publish call" + ) + if current.packet_id is not None and current.packet_id != packet_id: + raise PhysicalCommandTransitionError( + "QoS2 packet_id does not match the physical command" + ) + if current.qos2_completed: + return current + return self._transition_locked( + current, + stage="observing", + publish_call_returned=True, + packet_id=packet_id, + qos2_completed=True, + ) + + def record_application_response( + self, + operation_id: str, + response: PhysicalCommandApplicationResponse, + ) -> PhysicalCommandRecord: + """Persist an exact response without treating it as target-state proof.""" + + if not isinstance(response, PhysicalCommandApplicationResponse): + raise TypeError("response must be PhysicalCommandApplicationResponse") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "application response requires a dispatched physical command" + ) + _require_bound_response(current, response) + if current.application_response is not None: + if current.application_response == response: + return current + raise PhysicalCommandTransitionError( + "physical command application response cannot change" + ) + return self._transition_locked( + current, + stage="observing", + application_response=response, + ) + + def record_status_observation( + self, + operation_id: str, + observation: PhysicalCommandStatusEvidence, + ) -> PhysicalCommandRecord: + """Persist status as evidence; status alone can never resolve an edge.""" + + if not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "status observation requires a dispatched physical command" + ) + _require_bound_evidence(current.identity, current.connection, observation) + if current.last_status is not None and current.last_status == observation: + return current + return self._transition_locked( + current, + stage="observing", + last_status=observation, + ) + + def resolve( + self, + operation_id: str, + *, + resolution: PhysicalCommandOrdinaryResolution, + observation: PhysicalCommandStatusEvidence | None = None, + ) -> PhysicalCommandRecord: + """Resolve only definite no-dispatch or the complete exact target gate.""" + + if resolution not in _ORDINARY_RESOLUTIONS: + raise PhysicalCommandTransitionError( + "physical-state reconciliation requires reconcile_ambiguous" + ) + if observation is not None and not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage == "resolved": + if current.resolution == resolution: + return current + raise PhysicalCommandTransitionError( + "resolved physical command cannot change its resolution" + ) + if resolution == "not-dispatched": + if current.stage != "prepared": + raise PhysicalCommandTransitionError( + "not-dispatched resolution requires prepared stage" + ) + if observation is not None: + raise PhysicalCommandTransitionError( + "not-dispatched resolution cannot attach post-dispatch status" + ) + resolved_status = None + else: + expected_resolution = ( + "start-active-observed" + if current.action == "start" + else "stop-standby-observed" + ) + if resolution != expected_resolution: + raise PhysicalCommandTransitionError( + "physical resolution does not match the command action" + ) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "target resolution requires a dispatched physical command" + ) + resolved_status = observation or current.last_status + if resolved_status is None: + raise PhysicalCommandTransitionError( + "target resolution requires exact DeviceStatus evidence" + ) + _require_bound_evidence( + current.identity, + current.connection, + resolved_status, + ) + _require_complete_target_gate(current, resolution, resolved_status) + return self._transition_locked( + current, + stage="resolved", + last_status=resolved_status, + resolution=resolution, + ) + + def preflight_resolve_not_dispatched( + self, + operation_id: str, + ) -> PhysicalCommandRecord: + """Prepare the exact no-dispatch plan without replacing the main ledger. + + Rollover may idempotently prepublish its immutable predecessor so all + deterministic archive-path failures occur before local teardown. The + authoritative revision, resolution and CAS remain unchanged. + """ + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage == "resolved": + if current.resolution == "not-dispatched": + return current + raise PhysicalCommandTransitionError( + "resolved physical command cannot change its resolution" + ) + if current.stage != "prepared": + raise PhysicalCommandTransitionError( + "not-dispatched resolution requires prepared stage" + ) + candidate = self._transition_record_locked( + current, + stage="resolved", + last_status=None, + resolution="not-dispatched", + ) + try: + plan = self._plan_persist_locked(candidate) + self._publish_archive_plan_locked(plan) + return plan.record + except ValueError as exc: + if str(exc) == "physical command ledger exceeds the bounded size": + raise PhysicalCommandBlocked( + "physical command no-dispatch audit is full and requires archival" + ) from exc + raise + + def retire_unavailable_target( + self, + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, + ) -> PhysicalCommandRecord: + """Retire one permanently unavailable target without device I/O. + + The exact unresolved or last-proven-active record is terminalized only + as a local ownership disposition. The full attempt is preserved in + the retirement audit, no standby outcome is invented, and the retired + target can never receive a successor physical command from this ledger. + """ + + _validate_retirement_request( + retirement_id=retirement_id, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_transport_ref=expected_transport_ref, + reason=reason, + ) + + with self._lock, self._process_lock_locked(): + self._reload_locked() + candidate, already_committed = self._retirement_candidate_locked( + retirement_id=retirement_id, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_transport_ref=expected_transport_ref, + reason=reason, + ) + if already_committed: + return candidate + try: + return self._persist_locked(candidate) + except ValueError as exc: + if str(exc) == "physical command ledger exceeds the bounded size": + raise PhysicalCommandBlocked( + "physical operator-retirement audit is full and requires archival" + ) from exc + raise + + def preflight_retire_unavailable_target( + self, + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, + ) -> PhysicalCommandRecord: + """Validate the exact retirement plan without replacing the main ledger. + + This uses the same candidate builder and persistence planner as the + committing transition, including serialization, immutable segment, + archive count/byte and compacted-live-frontier bounds. When rollover + is required it may durably prepare the immutable content-addressed + predecessor, but never replaces the main ledger or commits retirement. + """ + + _validate_retirement_request( + retirement_id=retirement_id, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_transport_ref=expected_transport_ref, + reason=reason, + ) + with self._lock, self._process_lock_locked(): + self._reload_locked() + candidate, already_committed = self._retirement_candidate_locked( + retirement_id=retirement_id, + expected_operation_id=expected_operation_id, + expected_revision=expected_revision, + expected_transport_ref=expected_transport_ref, + reason=reason, + ) + if already_committed: + return candidate + try: + plan = self._plan_persist_locked(candidate) + # A rollover preflight publishes only the immutable, + # content-addressed predecessor. The authoritative main + # record and retirement CAS remain unchanged. This proves + # the exact owner/mode/symlink/collision filesystem path now, + # before a facade tears down any local session, and commit + # later reuses the same bytes idempotently. + self._publish_archive_plan_locked(plan) + return plan.record + except ValueError as exc: + if str(exc) == "physical command ledger exceeds the bounded size": + raise PhysicalCommandBlocked( + "physical operator-retirement audit is full and requires archival" + ) from exc + raise + + def reopen_retired_reconciliation( + self, + *, + reopening_id: str, + expected_revision: int, + expected_retirement_id: str, + expected_transport_ref: str, + expected_discovery_generation: int, + reason: PhysicalCommandOperatorReconciliationReopenReason, + ) -> PhysicalCommandRecord: + """Re-admit one freshly rediscovered retired UUID for read-only settlement. + + The transition performs no transport operation, preserves the original + retirement and unknown command outcome, and returns the exact historical + attempt to unresolved ``observing``. Only a later exact DeviceInfo + + fresh DeviceStatus reconciliation may resolve the physical state. + """ + + _validate_identifier(reopening_id, field_name="reopening_id") + _validate_positive_int(expected_revision, field_name="expected_revision") + _validate_identifier( + expected_retirement_id, + field_name="expected_retirement_id", + ) + _validate_identifier( + expected_transport_ref, + field_name="expected_transport_ref", + ) + _validate_positive_int( + expected_discovery_generation, + field_name="expected_discovery_generation", + ) + if reason not in _PHYSICAL_COMMAND_OPERATOR_RECONCILIATION_REOPEN_REASONS: + raise ValueError("unsupported physical operator-reconciliation reopen reason") + + with self._lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger is corrupt; reconciliation reopen is blocked" + ) + current = self._record + if current is None: + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen target is unavailable" + ) + + matching_reopen = next( + ( + item + for item in current.operator_reconciliation_reopens + if item.reopening_id == reopening_id + ), + None, + ) + if matching_reopen is not None: + if ( + matching_reopen.retirement_id == expected_retirement_id + and matching_reopen.retired_record_revision == expected_revision + and matching_reopen.reopened_transport_ref == expected_transport_ref + and matching_reopen.discovery_generation == expected_discovery_generation + and matching_reopen.reason == reason + ): + return current + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen conflicts with its audit" + ) + if reopening_id in self._archive_history.reopening_ids: + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen id has already been used" + ) + + if current.revision != expected_revision: + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen used a stale record checkpoint" + ) + if ( + current.stage != "resolved" + or current.resolution != "operator-retired-outcome-unknown" + ): + raise PhysicalCommandTransitionError( + "only an operator-retired ambiguous command may be reopened" + ) + retirement = next( + ( + item + for item in reversed(active_operator_retirements(current)) + if item.retirement_id == expected_retirement_id + and item.original_attempt.operation_id == current.operation_id + ), + None, + ) + if retirement is None: + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen retirement is not active" + ) + if physical_transport_ref_comparison_key( + retirement.retired_transport_ref + ) != physical_transport_ref_comparison_key(expected_transport_ref): + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen target does not match retirement" + ) + if len(current.operator_reconciliation_reopens) >= ( + _MAX_OPERATOR_RECONCILIATION_REOPENS + ): + raise PhysicalCommandBlocked( + "physical reconciliation-reopen audit is full and requires archival" + ) + if any( + item.retirement_id == expected_retirement_id + for item in current.operator_reconciliation_reopens + ): + raise PhysicalCommandTransitionError( + "physical operator retirement has already been reopened" + ) + reopened_at_utc = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + reopening = PhysicalCommandOperatorReconciliationReopen( + reopening_id=reopening_id, + retirement_id=expected_retirement_id, + retirement_sha256=_sha256_document(retirement.as_dict()), + reason=reason, + retired_record_revision=expected_revision, + reopened_transport_ref=expected_transport_ref, + discovery_generation=expected_discovery_generation, + reopened_at_utc=reopened_at_utc, + ) + try: + reopen_classified_stop = bool( + retirement.original_attempt.stage == "resolved" + and retirement.original_attempt.action == "stop" + and retirement.original_attempt.resolution == "not-dispatched" + ) + return self._transition_locked( + current, + # An ambiguous attempt retains its exact dispatch stage. + # A proven-active retirement reopens as an unresolved + # observation checkpoint: it preserves all original proof + # while requiring a new explicit read-only classification. + stage=( + "resolved" + if reopen_classified_stop + else "observing" + if retirement.original_attempt.stage == "resolved" + else retirement.original_attempt.stage + ), + resolution=( + "not-dispatched" if reopen_classified_stop else None + ), + operator_reconciliation_reopens=( + *current.operator_reconciliation_reopens, + reopening, + ), + audit_floor=reopened_at_utc, + ) + except ValueError as exc: + if str(exc) == "physical command ledger exceeds the bounded size": + raise PhysicalCommandBlocked( + "physical reconciliation-reopen audit is full and requires archival" + ) from exc + raise + + def reconcile_ambiguous( + self, + operation_id: str, + *, + reconciliation_id: str, + resolution: PhysicalCommandReconciliationResolution, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + ) -> PhysicalCommandRecord: + """Classify current physical state without claiming original edge success. + + This is a read-only recovery transition. The caller must first create + a new DeviceInfo-verified binding, then receive a fresh, non-retained + DeviceStatus on that exact binding. The complete original ambiguous + attempt and both new proofs are retained in the reconciliation audit. + """ + + _validate_identifier(reconciliation_id, field_name="reconciliation_id") + if resolution not in _RECONCILIATION_RESOLUTIONS: + raise ValueError("unsupported physical reconciliation resolution") + if not isinstance(verified_binding, PhysicalCommandVerifiedBinding): + raise TypeError("verified_binding must be PhysicalCommandVerifiedBinding") + if not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + if current.stage == "resolved": + if current.resolution == resolution and current.reconciliations: + latest = current.reconciliations[-1] + if ( + latest.reconciliation_id == reconciliation_id + and latest.verified_binding == verified_binding + and latest.observation == observation + ): + return current + raise PhysicalCommandTransitionError( + "resolved physical command cannot be reconciled again" + ) + if current.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "read-only reconciliation requires an ambiguous dispatched command" + ) + if len(current.reconciliations) >= _MAX_RECONCILIATIONS: + raise PhysicalCommandBlocked( + "physical reconciliation audit is full and requires archival" + ) + if any(item.reconciliation_id == reconciliation_id for item in current.reconciliations): + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if reconciliation_id in self._archive_history.reconciliation_ids: + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if any( + item.verified_binding.verification_id == verified_binding.verification_id + for item in current.reconciliations + ): + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + if verified_binding.verification_id in self._archive_history.verification_ids: + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + historical_control_sessions = { + item.original_attempt.connection.control_session_id + for item in current.reconciliations + } | { + item.verified_binding.connection.control_session_id + for item in current.reconciliations + } + if verified_binding.connection.control_session_id in historical_control_sessions: + raise PhysicalCommandTransitionError( + "reconciliation control-session binding is not globally fresh" + ) + if ( + verified_binding.connection.control_session_id + in self._archive_history.verified_control_session_ids + ): + raise PhysicalCommandTransitionError( + "reconciliation control-session binding is not globally fresh" + ) + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + _require_fresh_reconciliation_evidence( + current, + resolution=resolution, + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + original_attempt = _attempt_from_record(current) + reconciliation = PhysicalCommandReconciliation( + reconciliation_id=reconciliation_id, + kind="ambiguous-outcome", + resolution=resolution, + original_attempt=original_attempt, + original_attempt_sha256=_sha256_document(original_attempt.as_dict()), + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + return self._transition_locked( + current, + stage="resolved", + reconciliations=(*current.reconciliations, reconciliation), + resolution=resolution, + audit_floor=now, + ) + + def reconcile_prepared_stop_not_dispatched( + self, + operation_id: str, + *, + expected_revision: int, + reconciliation_id: str, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + ) -> PhysicalCommandRecord: + """Classify a definitely undispatched STOP and the current scanner. + + PREPARED proves that the STOP publish boundary was never crossed. A + later explicit Verify can nevertheless observe READY (for example, + after a manual stop) or SCANNING (the scan continued). This transition + persists both facts without conflating them: the record resolution is + always ``not-dispatched`` and the read-only reconciliation independently + carries ``physical-standby-observed`` or ``physical-active-observed``. + + Compatibility startup recovery may already have advanced that exact + PREPARED edge to RESOLVED/not-dispatched without a physical-state + classification. That shape is accepted as an equally strict input: + it has no publish, packet, QoS2, response or status fields, and no + classification for the current operation. + + No device operation occurs here. ``expected_revision`` is the exact + current checkpoint. Only an exact retry of the latest committed audit + is idempotent; stale/nonlatest checkpoints and globally reused proof + identities fail closed under the ledger process lock. + """ + + _validate_positive_int(expected_revision, field_name="expected_revision") + _validate_identifier(reconciliation_id, field_name="reconciliation_id") + if not isinstance(verified_binding, PhysicalCommandVerifiedBinding): + raise TypeError("verified_binding must be PhysicalCommandVerifiedBinding") + if not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + latest = current.reconciliations[-1] if current.reconciliations else None + if ( + current.stage == "resolved" + and current.action == "stop" + and current.resolution == "not-dispatched" + and current.revision == expected_revision + 1 + and latest is not None + and latest.kind == "prepared-stop-classification" + and latest.reconciliation_id == reconciliation_id + and latest.original_attempt.revision == expected_revision + and latest.verified_binding == verified_binding + and latest.observation == observation + ): + return current + if ( + current.stage == "resolved" + and current.action == "stop" + and current.resolution == "not-dispatched" + and _prepared_stop_classification(current) is not None + ): + raise PhysicalCommandTransitionError( + "prepared STOP classification conflicts with the current record" + ) + exact_prepared = bool( + current.stage == "prepared" + and current.action == "stop" + and current.revision == expected_revision + ) + exact_resolved_unclassified = bool( + current.revision == expected_revision + and current.resolved_unclassified_stop_recovery_required + ) + if not (exact_prepared or exact_resolved_unclassified): + raise PhysicalCommandTransitionError( + "undispatched STOP classification used a stale or ineligible checkpoint" + ) + if len(current.reconciliations) >= _MAX_RECONCILIATIONS: + raise PhysicalCommandBlocked( + "physical reconciliation audit is full and requires archival" + ) + if any( + item.reconciliation_id == reconciliation_id + for item in current.reconciliations + ) or reconciliation_id in self._archive_history.reconciliation_ids: + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if any( + item.verified_binding.verification_id + == verified_binding.verification_id + for item in current.reconciliations + ) or ( + verified_binding.verification_id + in self._archive_history.verification_ids + ): + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + historical_control_sessions = { + current.connection.control_session_id, + *( + item.verified_binding.connection.control_session_id + for item in current.reconciliations + ), + } + if ( + verified_binding.connection.control_session_id + in historical_control_sessions + or verified_binding.connection.control_session_id + in self._archive_history.verified_control_session_ids + ): + raise PhysicalCommandTransitionError( + "undispatched STOP classification requires a globally fresh " + "control-session binding" + ) + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + original_attempt = _attempt_from_record(current) + resolution = _validate_prepared_stop_classification_components( + original_attempt, + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + reconciliation = PhysicalCommandReconciliation( + reconciliation_id=reconciliation_id, + kind="prepared-stop-classification", + resolution=resolution, + original_attempt=original_attempt, + original_attempt_sha256=_sha256_document( + original_attempt.as_dict() + ), + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + return self._transition_locked( + current, + stage="resolved", + reconciliations=(*current.reconciliations, reconciliation), + resolution="not-dispatched", + audit_floor=now, + ) + + def rebind_resolved_active( + self, + operation_id: str, + *, + reconciliation_id: str, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + ) -> PhysicalCommandRecord: + """Rebind a resolved active state to a fresh read-only control session. + + The original START remains a successful, fully proven edge. This + transition publishes nothing and grants no START authority. It only + records fresh DeviceInfo plus non-retained SCANNING evidence so one + later, explicitly operator-confirmed STOP can use the new binding. + """ + + _validate_identifier(reconciliation_id, field_name="reconciliation_id") + if not isinstance(verified_binding, PhysicalCommandVerifiedBinding): + raise TypeError("verified_binding must be PhysicalCommandVerifiedBinding") + if not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + reopened_resolved_start = _reopened_resolved_start_attempt(current) + resolved_active_shape = bool( + current.stage == "resolved" + and current.resolution + in { + "start-active-observed", + "physical-active-observed", + "physical-standby-observed", + "not-dispatched", + } + ) + if not (resolved_active_shape or reopened_resolved_start is not None): + raise PhysicalCommandTransitionError( + "active rebind requires a resolved active physical state" + ) + prepared_stop_recovery = _latest_prepared_stop_recovery(current) + latest_recovery = ( + prepared_stop_recovery + if prepared_stop_recovery is not None + else _latest_resolved_start_recovery(current) + ) + if ( + latest_recovery is not None + and latest_recovery.resolution == "physical-standby-observed" + ): + raise PhysicalCommandTransitionError( + "standby cessation cannot be rebound as the original active START" + ) + if ( + current.reconciled_physical_state != "active" + and not current.reopened_physical_state_requires_reconciliation + ): + raise PhysicalCommandTransitionError( + "active rebind requires a resolved active physical state" + ) + if current.reconciliations: + latest = current.reconciliations[-1] + if latest.reconciliation_id == reconciliation_id: + if ( + latest.kind == "resolved-active-rebind" + and latest.resolution == "physical-active-observed" + and latest.verified_binding == verified_binding + and latest.observation == observation + ): + return current + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if len(current.reconciliations) >= _MAX_RECONCILIATIONS: + raise PhysicalCommandBlocked( + "physical reconciliation audit is full and requires archival" + ) + if any(item.reconciliation_id == reconciliation_id for item in current.reconciliations): + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if reconciliation_id in self._archive_history.reconciliation_ids: + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if any( + item.verified_binding.verification_id == verified_binding.verification_id + for item in current.reconciliations + ): + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + if verified_binding.verification_id in self._archive_history.verification_ids: + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + historical_control_sessions = { + current.connection.control_session_id, + *( + item.verified_binding.connection.control_session_id + for item in current.reconciliations + ), + } + if verified_binding.connection.control_session_id in historical_control_sessions: + raise PhysicalCommandTransitionError( + "active rebind requires a new verified globally fresh control-session binding" + ) + if ( + verified_binding.connection.control_session_id + in self._archive_history.verified_control_session_ids + ): + raise PhysicalCommandTransitionError( + "active rebind requires a new verified globally fresh control-session binding" + ) + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + prepared_stop_classification = _prepared_stop_classification(current) + original_start = ( + prepared_stop_classification.original_attempt + if prepared_stop_classification is not None + else reopened_resolved_start + if reopened_resolved_start is not None + else _attempt_from_record(current) + if current.resolution == "start-active-observed" + else _latest_reconciliation_for_operation(current).original_attempt + ) + if prepared_stop_classification is not None: + _validate_prepared_stop_followup_components( + prepared_stop_classification.original_attempt, + verified_binding=verified_binding, + observation=observation, + required_state="scanning", + reconciled_at_utc=now, + ) + else: + _validate_active_rebind_components( + original_start, + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + prior_active_observation = ( + latest_recovery.observation + if latest_recovery is not None + else current.last_status + if current.resolution == "start-active-observed" + else _latest_reconciliation_for_operation(current).observation + ) + if ( + prior_active_observation is None + or prior_active_observation.project_id_sha256 is None + or observation.project_id_sha256 != prior_active_observation.project_id_sha256 + ): + raise PhysicalCommandTransitionError( + "active rebind project does not match the current active state" + ) + reconciliation = PhysicalCommandReconciliation( + reconciliation_id=reconciliation_id, + kind="resolved-active-rebind", + resolution="physical-active-observed", + original_attempt=original_start, + original_attempt_sha256=_sha256_document(original_start.as_dict()), + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + return self._transition_locked( + current, + stage="resolved", + reconciliations=(*current.reconciliations, reconciliation), + resolution=( + "start-active-observed" + if reopened_resolved_start is not None + else current.resolution + ), + audit_floor=now, + ) + + def observe_resolved_active_standby( + self, + operation_id: str, + *, + reconciliation_id: str, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + ) -> PhysicalCommandRecord: + """Record that a previously proven START is now physically standby. + + A power loss, hard shutdown or out-of-process device stop can end the + physical scan without a STOP response owned by Mission Core. This + read-only transition preserves the original successful START and does + not invent STOP success. It only releases the edge chain for a later + fresh START after exact unbound, non-initialized READY is observed on a + new DeviceInfo-verified binding. + """ + + _validate_identifier(reconciliation_id, field_name="reconciliation_id") + if not isinstance(verified_binding, PhysicalCommandVerifiedBinding): + raise TypeError("verified_binding must be PhysicalCommandVerifiedBinding") + if not isinstance(observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + + with self._lock, self._process_lock_locked(): + current = self._current_operation_locked(operation_id) + reopened_resolved_start = _reopened_resolved_start_attempt(current) + resolved_active_shape = bool( + current.stage == "resolved" + and current.resolution + in { + "start-active-observed", + "physical-active-observed", + "not-dispatched", + } + ) + if not (resolved_active_shape or reopened_resolved_start is not None): + raise PhysicalCommandTransitionError( + "active cessation requires a resolved active physical state" + ) + prepared_stop_recovery = _latest_prepared_stop_recovery(current) + latest_recovery = ( + prepared_stop_recovery + if prepared_stop_recovery is not None + else _latest_resolved_start_recovery(current) + ) + awaiting_ready_after_scan_over = bool( + latest_recovery is not None + and latest_recovery.resolution == "physical-standby-observed" + and latest_recovery.observation.session_state == "scan_over" + ) + if current.reconciliations: + latest = current.reconciliations[-1] + if latest.reconciliation_id == reconciliation_id: + if ( + current.reconciled_physical_state == "standby" + and latest.kind == "resolved-active-cessation" + and latest.resolution == "physical-standby-observed" + and latest.verified_binding == verified_binding + and latest.observation == observation + ): + return current + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if ( + latest_recovery is not None + and latest_recovery.resolution == ("physical-standby-observed") + and not ( + awaiting_ready_after_scan_over + and observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ) + ): + raise PhysicalCommandTransitionError( + "successful START is already reconciled to standby" + ) + if ( + current.reconciled_physical_state != "active" + and not current.reopened_physical_state_requires_reconciliation + and not awaiting_ready_after_scan_over + ): + raise PhysicalCommandTransitionError( + "active cessation requires a resolved active physical state" + ) + if len(current.reconciliations) >= _MAX_RECONCILIATIONS: + raise PhysicalCommandBlocked( + "physical reconciliation audit is full and requires archival" + ) + if any(item.reconciliation_id == reconciliation_id for item in current.reconciliations): + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if reconciliation_id in self._archive_history.reconciliation_ids: + raise PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + if any( + item.verified_binding.verification_id == verified_binding.verification_id + for item in current.reconciliations + ): + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + if verified_binding.verification_id in self._archive_history.verification_ids: + raise PhysicalCommandTransitionError( + "physical binding verification id has already been used" + ) + historical_control_sessions = { + current.connection.control_session_id, + *( + item.verified_binding.connection.control_session_id + for item in current.reconciliations + ), + } + if verified_binding.connection.control_session_id in historical_control_sessions: + raise PhysicalCommandTransitionError( + "active cessation requires a new verified globally fresh " + "control-session binding" + ) + if ( + verified_binding.connection.control_session_id + in self._archive_history.verified_control_session_ids + ): + raise PhysicalCommandTransitionError( + "active cessation requires a new verified globally fresh " + "control-session binding" + ) + now = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + prepared_stop_classification = _prepared_stop_classification(current) + original_start = ( + prepared_stop_classification.original_attempt + if prepared_stop_classification is not None + else reopened_resolved_start + if reopened_resolved_start is not None + else _attempt_from_record(current) + if current.resolution == "start-active-observed" + else _latest_reconciliation_for_operation(current).original_attempt + ) + if prepared_stop_classification is not None: + _validate_prepared_stop_followup_components( + prepared_stop_classification.original_attempt, + verified_binding=verified_binding, + observation=observation, + required_state="standby", + reconciled_at_utc=now, + ) + else: + _validate_resolved_active_standby_components( + original_start, + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + reconciliation = PhysicalCommandReconciliation( + reconciliation_id=reconciliation_id, + kind="resolved-active-cessation", + resolution="physical-standby-observed", + original_attempt=original_start, + original_attempt_sha256=_sha256_document(original_start.as_dict()), + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=now, + ) + return self._transition_locked( + current, + stage="resolved", + reconciliations=(*current.reconciliations, reconciliation), + resolution=( + "start-active-observed" + if current.resolution == "start-active-observed" + or reopened_resolved_start is not None + else "not-dispatched" + if current.resolution == "not-dispatched" + else "physical-standby-observed" + ), + audit_floor=now, + ) + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + try: + data_created = _ensure_private_directory(self._data_dir, parents=True) + parent_created = _ensure_private_directory(self.path.parent, parents=True) + except (OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger directories are not private" + ) from exc + if data_created: + _fsync_directory(self._data_dir.parent) + if parent_created: + _fsync_directory(self._data_dir) + + existed_before = self._process_lock_path.exists() + flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self._process_lock_path, flags, 0o600) + except OSError as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger lock cannot be opened safely" + ) from exc + stream: IO[bytes] | None = None + try: + try: + before = os.fstat(descriptor) + _require_private_regular_file(before, label="ledger lock", empty=True) + stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + after = os.fstat(stream.fileno()) + path_metadata = self._process_lock_path.lstat() + _require_private_regular_file(after, label="ledger lock", empty=True) + if (after.st_dev, after.st_ino) != ( + path_metadata.st_dev, + path_metadata.st_ino, + ): + raise PhysicalCommandLedgerCorrupt( + "physical command ledger lock identity changed" + ) + if not existed_before: + _fsync_directory(self.path.parent) + except (OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger lock failed closed" + ) from exc + try: + yield + finally: + if stream is not None: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + if stream is not None: + stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _current_operation_locked(self, operation_id: str) -> PhysicalCommandRecord: + _validate_identifier(operation_id, field_name="operation_id") + self._reload_locked() + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger is corrupt; commands remain blocked" + ) + current = self._record + if current is None or current.operation_id != operation_id: + raise PhysicalCommandTransitionError("physical command operation does not match ledger") + return current + + def _require_no_unresolved_attempt_locked(self) -> None: + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger is corrupt; commands remain blocked" + ) + if self._record is not None and self._record.unresolved: + raise PhysicalCommandBlocked( + "previous physical command is unresolved; automatic retry is blocked" + ) + + def _retirement_candidate_locked( + self, + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, + ) -> tuple[PhysicalCommandRecord, bool]: + """Return the exact prospective retirement and its replay disposition.""" + + if self._corrupt: + raise PhysicalCommandLedgerCorrupt( + "physical command ledger is corrupt; retirement is blocked" + ) + current = self._record + if current is None: + raise PhysicalCommandTransitionError( + "physical command retirement target is unavailable" + ) + if current.stage == "resolved": + matching = next( + ( + item + for item in current.operator_retirements + if item.retirement_id == retirement_id + ), + None, + ) + if ( + current.operation_id == expected_operation_id + and current.revision == expected_revision + 1 + and current.resolution == "operator-retired-outcome-unknown" + and matching is not None + and matching.original_attempt.operation_id == expected_operation_id + and matching.original_attempt.revision == expected_revision + and matching.retired_transport_ref == expected_transport_ref + and matching.reason == reason + ): + return current, True + if ( + current.reconciled_physical_state != "active" + and not current.reopened_physical_state_requires_reconciliation + and not ( + reason == "connection-scenario-reset-by-operator" + and current.classified_stop_standby_successor_pending + ) + ): + raise PhysicalCommandTransitionError( + "physical command retirement conflicts with the current record" + ) + if ( + current.operation_id != expected_operation_id + or current.revision != expected_revision + or current.connection.transport_ref != expected_transport_ref + ): + raise PhysicalCommandTransitionError( + "physical command retirement used a stale record checkpoint" + ) + retireable_ambiguous = current.stage in {"dispatching", "observing"} + retireable_proven_active = bool( + current.stage == "resolved" + and ( + current.reconciled_physical_state == "active" + # A freshly reopened classified STOP deliberately has no + # current physical-state claim. If that exact UUID disappears + # again before Verify can append fresh evidence, allow another + # local retirement fence so recovery cannot become permanently + # stuck. The retirement audit still preserves the original + # STOP as not-dispatched and grants no command authority. + or current.reopened_physical_state_requires_reconciliation + ) + ) + retireable_reset_standby_successor = bool( + reason == "connection-scenario-reset-by-operator" + and current.classified_stop_standby_successor_pending + ) + if not ( + retireable_ambiguous + or retireable_proven_active + or retireable_reset_standby_successor + ): + raise PhysicalCommandTransitionError( + "only an ambiguous dispatched or proven-active command may be retired" + ) + if len(current.operator_retirements) >= _MAX_OPERATOR_RETIREMENTS: + raise PhysicalCommandBlocked( + "physical operator-retirement audit is full and requires archival" + ) + if any(item.retirement_id == retirement_id for item in current.operator_retirements): + raise PhysicalCommandTransitionError( + "physical operator-retirement id has already been used" + ) + if retirement_id in self._archive_history.retirement_ids: + raise PhysicalCommandTransitionError( + "physical operator-retirement id has already been used" + ) + original_attempt = _retirement_attempt_from_record(current) + retired_at_utc = _nondecreasing_audit_timestamp( + self._clock(), + floor=current.updated_at_utc, + ) + retirement = PhysicalCommandOperatorRetirement( + retirement_id=retirement_id, + reason=reason, + original_attempt=original_attempt, + original_attempt_sha256=_sha256_document(original_attempt.as_dict()), + retired_transport_ref=current.connection.transport_ref, + retired_at_utc=retired_at_utc, + ) + return ( + self._transition_record_locked( + current, + stage="resolved", + resolution="operator-retired-outcome-unknown", + operator_retirements=(*current.operator_retirements, retirement), + audit_floor=retired_at_utc, + ), + False, + ) + + def _transition_locked( + self, + current: PhysicalCommandRecord, + *, + stage: PhysicalCommandStage, + publish_call_returned: bool | None = None, + packet_id: int | None = None, + qos2_completed: bool | None = None, + application_response: PhysicalCommandApplicationResponse | None = None, + last_status: PhysicalCommandStatusEvidence | None = None, + reconciliations: tuple[PhysicalCommandReconciliation, ...] | None = None, + operator_retirements: tuple[PhysicalCommandOperatorRetirement, ...] | None = None, + operator_reconciliation_reopens: tuple[PhysicalCommandOperatorReconciliationReopen, ...] + | None = None, + resolution: PhysicalCommandResolution | None = None, + audit_floor: str | None = None, + ) -> PhysicalCommandRecord: + return self._persist_locked( + self._transition_record_locked( + current, + stage=stage, + publish_call_returned=publish_call_returned, + packet_id=packet_id, + qos2_completed=qos2_completed, + application_response=application_response, + last_status=last_status, + reconciliations=reconciliations, + operator_retirements=operator_retirements, + operator_reconciliation_reopens=operator_reconciliation_reopens, + resolution=resolution, + audit_floor=audit_floor, + ) + ) + + def _transition_record_locked( + self, + current: PhysicalCommandRecord, + *, + stage: PhysicalCommandStage, + publish_call_returned: bool | None = None, + packet_id: int | None = None, + qos2_completed: bool | None = None, + application_response: PhysicalCommandApplicationResponse | None = None, + last_status: PhysicalCommandStatusEvidence | None = None, + reconciliations: tuple[PhysicalCommandReconciliation, ...] | None = None, + operator_retirements: tuple[PhysicalCommandOperatorRetirement, ...] | None = None, + operator_reconciliation_reopens: tuple[PhysicalCommandOperatorReconciliationReopen, ...] + | None = None, + resolution: PhysicalCommandResolution | None = None, + audit_floor: str | None = None, + ) -> PhysicalCommandRecord: + if current.revision >= _MAX_REVISION: + raise PhysicalCommandTransitionError("physical command revision is exhausted") + return replace( + current, + revision=current.revision + 1, + stage=stage, + publish_call_returned=( + publish_call_returned + if publish_call_returned is not None + else current.publish_call_returned + ), + packet_id=packet_id if packet_id is not None else current.packet_id, + qos2_completed=( + qos2_completed if qos2_completed is not None else current.qos2_completed + ), + application_response=( + application_response + if application_response is not None + else current.application_response + ), + last_status=last_status if last_status is not None else current.last_status, + reconciliations=( + reconciliations if reconciliations is not None else current.reconciliations + ), + operator_retirements=( + operator_retirements + if operator_retirements is not None + else current.operator_retirements + ), + operator_reconciliation_reopens=( + operator_reconciliation_reopens + if operator_reconciliation_reopens is not None + else current.operator_reconciliation_reopens + ), + resolution=resolution, + updated_at_utc=_nondecreasing_audit_timestamp( + self._clock(), + floor=_max_audit_floor(current.updated_at_utc, audit_floor), + ), + ) + + def _persist_locked(self, record: PhysicalCommandRecord) -> PhysicalCommandRecord: + """Publish one transition, rolling immutable history before it can block safety. + + The current main document is archived *before* a compacted successor is + published. A crash before the main-file replace can therefore leave at + most an unreferenced content-addressed segment; retrying publishes the + same bytes and never duplicates the chain. Active retirement fences + and every reconciliation needed by the current physical operation stay + in the main record. + """ + + plan = self._plan_persist_locked(record) + self._publish_archive_plan_locked(plan) + + _write_private_json_atomic(self.path, plan.record.as_dict(), data_dir=self._data_dir) + self._record = plan.record + self._archive_history = plan.history + self._corrupt = False + return plan.record + + def _publish_archive_plan_locked( + self, + plan: _PhysicalCommandPersistencePlan, + ) -> None: + if plan.archive_payload is None: + return + assert plan.archive_head_sha256 is not None + try: + _write_private_content_addressed_archive( + self._archive_dir, + expected_sha256=plan.archive_head_sha256, + payload=plan.archive_payload, + data_dir=self._data_dir, + ) + except PhysicalCommandLedgerError: + raise + except (OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive publication failed closed" + ) from exc + + def _plan_persist_locked( + self, + record: PhysicalCommandRecord, + ) -> _PhysicalCommandPersistencePlan: + """Validate and materialize a transition plan without filesystem writes.""" + + _validate_record_semantics(record) + serialized = _canonical_json_bytes(record.as_dict()) + persisted = record + history = self._archive_history + archive_head_sha256: str | None = None + segment_bytes: bytes | None = None + if len(serialized) > PHYSICAL_COMMAND_LEDGER_MAX_BYTES: + previous = self._record + if previous is None or record.revision != previous.revision + 1: + raise ValueError("physical command ledger exceeds the bounded size") + if history.segment_count >= PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS: + raise PhysicalCommandBlocked( + "physical command archive reached its bounded segment limit" + ) + segment = PhysicalCommandArchiveSegment( + sequence=history.segment_count + 1, + predecessor_sha256=history.head_sha256, + record_sha256=_sha256_document(previous.as_dict()), + record=previous, + ) + segment_bytes = _canonical_json_bytes(segment.as_dict()) + if len(segment_bytes) > PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES: + raise PhysicalCommandBlocked( + "physical command predecessor exceeds the bounded archive segment size" + ) + archive_total_bytes = history.total_bytes + len(segment_bytes) + if archive_total_bytes > PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES: + raise PhysicalCommandBlocked( + "physical command archive reached its bounded byte limit" + ) + archive_head_sha256 = hashlib.sha256(segment_bytes).hexdigest() + persisted = _compact_record_after_archive( + record, + archive_head_sha256=archive_head_sha256, + archive_segment_count=history.segment_count + 1, + archive_total_bytes=archive_total_bytes, + ) + _validate_record_semantics(persisted) + if len(_canonical_json_bytes(persisted.as_dict())) > ( + PHYSICAL_COMMAND_LEDGER_MAX_BYTES + ): + raise ValueError("physical command ledger exceeds the bounded size") + history = _extend_archive_history( + history, + archived_record=previous, + head_sha256=archive_head_sha256, + segment_bytes=len(segment_bytes), + ) + return _PhysicalCommandPersistencePlan( + record=persisted, + history=history, + archive_head_sha256=archive_head_sha256, + archive_payload=segment_bytes, + ) + + def _reload_locked(self) -> None: + try: + parent = self.path.parent.lstat() + _require_private_directory_metadata(parent, label="ledger directory") + except FileNotFoundError: + self._record = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = False + return + except (OSError, ValueError): + self._record = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = True + return + + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self.path, flags) + except FileNotFoundError: + self._record = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = False + return + except OSError: + self._record = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = True + return + try: + metadata = os.fstat(descriptor) + _require_private_regular_file(metadata, label="ledger", empty=False) + if metadata.st_size > PHYSICAL_COMMAND_LEDGER_MAX_BYTES: + raise ValueError("physical command ledger exceeds the bounded size") + payload_bytes = _read_bounded(descriptor, PHYSICAL_COMMAND_LEDGER_MAX_BYTES) + after = os.fstat(descriptor) + path_metadata = self.path.lstat() + _require_private_regular_file(after, label="ledger", empty=False) + if (after.st_dev, after.st_ino) != ( + path_metadata.st_dev, + path_metadata.st_ino, + ): + raise ValueError("physical command ledger identity changed while reading") + payload = json.loads( + payload_bytes.decode("utf-8"), + object_pairs_hook=_unique_json_object, + ) + record = _record_from_mapping(payload) + archive_history = self._load_archive_history_locked(record) + except ( + OSError, + UnicodeError, + json.JSONDecodeError, + TypeError, + ValueError, + PhysicalCommandLedgerError, + ): + self._record = None + self._archive_history = _PhysicalCommandArchiveHistory() + self._corrupt = True + return + finally: + os.close(descriptor) + self._record = record + self._archive_history = archive_history + self._corrupt = False + + def _load_archive_history_locked( + self, + record: PhysicalCommandRecord, + ) -> _PhysicalCommandArchiveHistory: + return _load_private_archive_history( + self._archive_dir, + record=record, + ) + + +def _require_safe_edge_successor( + previous: PhysicalCommandRecord | None, + *, + operation_id: str, + parent_operation_id: str | None, + acquisition_id: str, + action: PhysicalCommandAction, + identity: PhysicalCommandIdentity, + connection: PhysicalCommandConnectionBinding, + compatibility_profile_id: str, + archived_operation_ids: frozenset[str] = frozenset(), +) -> str | None: + if previous is None: + if parent_operation_id is not None: + raise PhysicalCommandTransitionError( + "initial physical command cannot name a parent operation" + ) + if action != "start": + raise PhysicalCommandTransitionError( + "initial STOP lacks durable same-session START authority" + ) + return None + if previous.stage != "resolved" or previous.resolution is None: + raise PhysicalCommandBlocked("previous physical command is unresolved") + if operation_id == previous.operation_id: + raise PhysicalCommandTransitionError("physical operation_id cannot be reused") + historical_operation_ids: set[str | None] = ( + { + reconciliation.original_attempt.operation_id + for reconciliation in previous.reconciliations + } + | { + reconciliation.original_attempt.parent_operation_id + for reconciliation in previous.reconciliations + } + | {confirmation.stop_operation_id for confirmation in previous.operator_confirmations} + | {retirement.original_attempt.operation_id for retirement in previous.operator_retirements} + | { + retirement.original_attempt.parent_operation_id + for retirement in previous.operator_retirements + } + ) + historical_operation_ids.discard(None) + if operation_id in historical_operation_ids: + raise PhysicalCommandTransitionError( + "physical operation_id was already consumed in reconciliation audit" + ) + if operation_id in archived_operation_ids: + raise PhysicalCommandTransitionError( + "physical operation_id was already consumed in archived audit" + ) + if parent_operation_id != previous.operation_id: + raise PhysicalCommandTransitionError( + "next physical command must chain the prior durable operation" + ) + active_retirements = active_operator_retirements(previous) + if any( + retirement.original_attempt.identity.vendor_device_id_sha256 + == identity.vendor_device_id_sha256 + or retirement.original_attempt.identity.device_serial_sha256 + == identity.device_serial_sha256 + or physical_transport_ref_comparison_key(retirement.retired_transport_ref) + == physical_transport_ref_comparison_key(connection.transport_ref) + for retirement in active_retirements + ): + raise PhysicalCommandBlocked("retired physical target cannot receive another command") + if any( + retirement.original_attempt.acquisition_id == acquisition_id + for retirement in active_retirements + ): + raise PhysicalCommandBlocked("retired physical target acquisition id cannot be reused") + + if previous.resolution == "operator-retired-outcome-unknown": + retirement = next( + ( + item + for item in reversed(previous.operator_retirements) + if item.original_attempt.operation_id == previous.operation_id + ), + None, + ) + if retirement is None: + raise PhysicalCommandTransitionError("operator-retired physical state lacks its audit") + if action != "start": + raise PhysicalCommandTransitionError("replacement physical chain must begin with START") + if identity == previous.identity or physical_transport_ref_comparison_key( + connection.transport_ref + ) == physical_transport_ref_comparison_key(previous.connection.transport_ref): + raise PhysicalCommandBlocked( + "operator retirement cannot authorize the same physical target" + ) + return None + + if identity != previous.identity: + raise PhysicalCommandTransitionError("physical command device identity changed") + if compatibility_profile_id != previous.compatibility_profile_id: + raise PhysicalCommandTransitionError( + "physical command compatibility profile changed within the ledger chain" + ) + + if previous.resolution == "start-active-observed": + latest_recovery = _latest_resolved_start_recovery(previous) + if ( + latest_recovery is not None + and latest_recovery.resolution == "physical-standby-observed" + ): + if latest_recovery.observation.session_state == "scan_over": + raise PhysicalCommandBlocked( + "SCAN_OVER cessation requires a later fresh READY reconciliation" + ) + if action != "start": + raise PhysicalCommandTransitionError( + "observed standby after successful START permits only a fresh START" + ) + return None + if action != "stop": + raise PhysicalCommandTransitionError( + "active START may be followed only by same-session STOP" + ) + required_connection = ( + latest_recovery.verified_binding.connection + if latest_recovery is not None + else previous.connection + ) + if acquisition_id != previous.acquisition_id or connection != required_connection: + raise PhysicalCommandTransitionError( + "STOP must retain START acquisition and latest verified " + "live control-session binding" + ) + return latest_recovery.reconciliation_id if latest_recovery is not None else None + elif previous.resolution == "stop-standby-observed": + if action != "start": + raise PhysicalCommandTransitionError( + "standby STOP may be followed only by a fresh START" + ) + return None + elif previous.resolution == "physical-active-observed": + reconciliation = _latest_reconciliation_for_operation(previous) + if action != "stop": + raise PhysicalCommandTransitionError( + "reconciled active state may authorize only explicit STOP" + ) + if acquisition_id != previous.acquisition_id: + raise PhysicalCommandTransitionError( + "reconciled active STOP must retain the original acquisition" + ) + if connection != reconciliation.verified_binding.connection: + raise PhysicalCommandTransitionError( + "reconciled active STOP must use the freshly verified binding" + ) + return reconciliation.reconciliation_id + elif previous.resolution == "physical-standby-observed": + latest_reconciliation = _latest_reconciliation_for_operation(previous) + if latest_reconciliation.observation.session_state == "scan_over": + raise PhysicalCommandBlocked( + "SCAN_OVER cessation requires a later fresh READY reconciliation" + ) + if action != "start": + raise PhysicalCommandTransitionError( + "reconciled standby state may be followed only by fresh START" + ) + return None + elif previous.resolution == "not-dispatched": + if previous.reopened_physical_state_requires_reconciliation: + raise PhysicalCommandBlocked( + "reopened physical target requires fresh read-only reconciliation" + ) + if previous.resolved_unclassified_stop_recovery_required: + raise PhysicalCommandBlocked( + "undispatched STOP requires fresh read-only physical classification" + ) + prepared_stop_recovery = _latest_prepared_stop_recovery(previous) + if prepared_stop_recovery is not None: + if prepared_stop_recovery.resolution == "physical-standby-observed": + # READY carries no active-session command authority. The + # successor START is instead bound above to its own fresh, + # non-retained live READY generation, so a process restart may + # replace the read-only verification connection safely. + if action != "start": + raise PhysicalCommandTransitionError( + "read-only READY after undispatched STOP permits only fresh START" + ) + return None + required_connection = prepared_stop_recovery.verified_binding.connection + if connection != required_connection: + raise PhysicalCommandTransitionError( + "prepared STOP recovery successor must use the freshly verified binding" + ) + if action != "stop": + raise PhysicalCommandTransitionError( + "read-only SCANNING after undispatched STOP permits only explicit STOP" + ) + if acquisition_id != previous.acquisition_id: + raise PhysicalCommandTransitionError( + "recovered active STOP must retain the original acquisition" + ) + return prepared_stop_recovery.reconciliation_id + if action != previous.action: + raise PhysicalCommandTransitionError( + "not-dispatched edge may be superseded only by the same action" + ) + if action == "stop" and ( + acquisition_id != previous.acquisition_id or connection != previous.connection + ): + raise PhysicalCommandTransitionError( + "undispatched STOP may be superseded only in the same live session" + ) + if action == "stop": + confirmation = next( + ( + item + for item in reversed(previous.operator_confirmations) + if item.stop_operation_id == previous.operation_id + ), + None, + ) + if confirmation is not None: + return confirmation.reconciliation_id + return None + raise PhysicalCommandTransitionError("unsupported prior physical command resolution") + + +def _prove_classified_stop_ancestry( + record: PhysicalCommandRecord, + *, + archive_history: _PhysicalCommandArchiveHistory, + head_operation_id: str, + expected_start_operation_id: str, + acquisition_id: str, +) -> PhysicalCommandClassifiedStopAncestryProof | None: + if ( + record.operation_id != head_operation_id + or record.action != "stop" + or record.acquisition_id != acquisition_id + ): + raise PhysicalCommandTransitionError( + "classified STOP ancestry does not match the current head/acquisition" + ) + parent_operation_id = record.parent_operation_id + if parent_operation_id is None or head_operation_id == expected_start_operation_id: + raise PhysicalCommandTransitionError( + "classified STOP ancestry lacks a distinct START predecessor" + ) + if parent_operation_id == expected_start_operation_id: + return None + + reconciliations_by_id = dict(archive_history.lineage_reconciliations_by_id) + classification_ids_by_operation = dict( + archive_history.lineage_classification_ids_by_operation + ) + confirmations_by_stop_operation = dict( + archive_history.lineage_confirmations_by_stop_operation + ) + attempt_sha256_by_operation = dict( + archive_history.lineage_attempt_sha256_by_operation + ) + _register_lineage_proof_indexes( + record, + reconciliations_by_id=reconciliations_by_id, + classification_ids_by_operation=classification_ids_by_operation, + confirmations_by_stop_operation=confirmations_by_stop_operation, + attempt_sha256_by_operation=attempt_sha256_by_operation, + ) + + first_parent_operation_id = parent_operation_id + child_operation_id = head_operation_id + seen = {head_operation_id} + links: list[dict[str, object]] = [] + while parent_operation_id != expected_start_operation_id: + if parent_operation_id in seen: + raise PhysicalCommandLedgerCorrupt( + "classified STOP ancestry contains an operation cycle" + ) + if len(links) >= _MAX_RECONCILIATIONS: + raise PhysicalCommandLedgerCorrupt( + "classified STOP ancestry exceeds its bounded depth" + ) + seen.add(parent_operation_id) + confirmation = confirmations_by_stop_operation.get(child_operation_id) + if confirmation is None: + raise PhysicalCommandTransitionError( + "classified STOP ancestry lacks an exact successor confirmation" + ) + authorization = reconciliations_by_id.get(confirmation.reconciliation_id) + classification_id = classification_ids_by_operation.get(parent_operation_id) + classification = ( + reconciliations_by_id.get(classification_id) + if classification_id is not None + else None + ) + if authorization is None or classification is None: + raise PhysicalCommandLedgerCorrupt( + "classified STOP ancestry references missing durable evidence" + ) + attempt = authorization.original_attempt + attempt_sha256 = attempt_sha256_by_operation.get(parent_operation_id) + if not ( + confirmation.stop_operation_id == child_operation_id + and authorization.reconciliation_id == confirmation.reconciliation_id + and authorization.kind + in {"prepared-stop-classification", "resolved-active-rebind"} + and authorization.resolution == "physical-active-observed" + and classification.kind == "prepared-stop-classification" + and classification.resolution == "physical-active-observed" + and classification.original_attempt == attempt + and classification.original_attempt_sha256 + == authorization.original_attempt_sha256 + and attempt_sha256 == authorization.original_attempt_sha256 + and attempt.operation_id == parent_operation_id + and attempt.acquisition_id == acquisition_id + and attempt.identity == record.identity + and attempt.compatibility_profile_id == record.compatibility_profile_id + and _is_exact_not_dispatched_stop_attempt(attempt) + and isinstance(attempt.parent_operation_id, str) + ): + raise PhysicalCommandLedgerCorrupt( + "classified STOP ancestry evidence is internally inconsistent" + ) + links.append( + { + "child_operation_id": child_operation_id, + "parent_operation_id": parent_operation_id, + "parent_parent_operation_id": attempt.parent_operation_id, + "parent_attempt_sha256": authorization.original_attempt_sha256, + "classification_sha256": _sha256_document(classification.as_dict()), + "authorization_reconciliation_sha256": _sha256_document( + authorization.as_dict() + ), + "operator_confirmation_sha256": _sha256_document( + confirmation.as_dict() + ), + } + ) + child_operation_id = parent_operation_id + parent_operation_id = attempt.parent_operation_id + + chain_document: dict[str, object] = { + "schema_version": PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + "ledger_revision": record.revision, + "head_operation_id": head_operation_id, + "head_parent_operation_id": first_parent_operation_id, + "root_start_operation_id": expected_start_operation_id, + "acquisition_id": acquisition_id, + "links": links, + } + chain_sha256 = hashlib.sha256( + b"missioncore.xgrids-k1-classified-stop-ancestry/v1\x00" + + _canonical_json_bytes(chain_document) + ).hexdigest() + return PhysicalCommandClassifiedStopAncestryProof( + schema_version=PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + ledger_revision=record.revision, + head_operation_id=head_operation_id, + head_parent_operation_id=first_parent_operation_id, + root_start_operation_id=expected_start_operation_id, + acquisition_id=acquisition_id, + depth=len(links), + chain_sha256=chain_sha256, + ) + + +def _attempt_from_record(record: PhysicalCommandRecord) -> PhysicalCommandAttemptAudit: + return PhysicalCommandAttemptAudit( + revision=record.revision, + operation_id=record.operation_id, + parent_operation_id=record.parent_operation_id, + acquisition_id=record.acquisition_id, + action=record.action, + stage=record.stage, + resolution=( + record.resolution + if record.resolution in {*_ORDINARY_RESOLUTIONS, "physical-active-observed"} + else None + ), + identity=record.identity, + connection=record.connection, + compatibility_profile_id=record.compatibility_profile_id, + payload_sha256=record.payload_sha256, + baseline_status=record.baseline_status, + publish_call_returned=record.publish_call_returned, + packet_id=record.packet_id, + qos2_completed=record.qos2_completed, + application_response=record.application_response, + last_status=record.last_status, + created_at_utc=record.created_at_utc, + updated_at_utc=record.updated_at_utc, + ) + + +def _retirement_attempt_from_record( + record: PhysicalCommandRecord, +) -> PhysicalCommandAttemptAudit: + # Unlike the reconciliation audit (which preserves the original ambiguous + # edge), retirement binds the exact current record revision. For a + # physical-active resolution the enclosing ledger record retains and + # validates the complete DeviceInfo/DeviceStatus proof chain. + return _attempt_from_record(record) + + +def _same_attempt_evidence_except_revision_and_update( + left: PhysicalCommandAttemptAudit, + right: PhysicalCommandAttemptAudit, +) -> bool: + """Compare a retire/reopen cycle without erasing its local audit edge.""" + + compatible_recovery_shape = bool( + (left.stage == right.stage and left.resolution == right.resolution) + or ( + left.stage == "resolved" + and left.resolution in {"start-active-observed", "physical-active-observed"} + and right.stage in {"dispatching", "observing", "resolved"} + and right.resolution in {None, "physical-active-observed"} + ) + or ( + left.stage == "resolved" + and left.resolution == "not-dispatched" + and right.stage in {"prepared", "resolved"} + and right.resolution in {None, "not-dispatched"} + ) + ) + return ( + left.operation_id == right.operation_id + and left.parent_operation_id == right.parent_operation_id + and left.acquisition_id == right.acquisition_id + and left.action == right.action + and compatible_recovery_shape + and left.identity == right.identity + and left.connection == right.connection + and left.compatibility_profile_id == right.compatibility_profile_id + and left.payload_sha256 == right.payload_sha256 + and left.baseline_status == right.baseline_status + and left.publish_call_returned == right.publish_call_returned + and left.packet_id == right.packet_id + and left.qos2_completed == right.qos2_completed + and left.application_response == right.application_response + and left.last_status == right.last_status + and left.created_at_utc == right.created_at_utc + ) + + +def _require_successor_baseline_after_reconciliation( + previous: PhysicalCommandRecord | None, + *, + required_reconciliation_id: str | None, + baseline_status: PhysicalCommandStatusEvidence, +) -> None: + if previous is None: + return + if previous.resolution in { + "physical-active-observed", + "physical-standby-observed", + }: + _latest_reconciliation_for_operation(previous) + elif required_reconciliation_id is not None: + _reconciliation_by_id( + previous.reconciliations, + required_reconciliation_id, + ) + expected_project_id_sha256 = _effective_active_project_id(previous) + if expected_project_id_sha256 is not None and ( + not baseline_status.project_bound + or baseline_status.project_id_sha256 != expected_project_id_sha256 + ): + raise PhysicalCommandTransitionError( + "physical STOP baseline project does not match proven active project" + ) + + +def _reconciliation_by_id( + reconciliations: tuple[PhysicalCommandReconciliation, ...], + reconciliation_id: str, +) -> PhysicalCommandReconciliation: + matches = [item for item in reconciliations if item.reconciliation_id == reconciliation_id] + if len(matches) != 1: + raise PhysicalCommandTransitionError( + "operator STOP lacks one exact physical reconciliation" + ) + return matches[0] + + +def _latest_reconciliation_for_operation( + record: PhysicalCommandRecord, +) -> PhysicalCommandReconciliation: + if not record.reconciliations: + raise PhysicalCommandTransitionError("physical-state resolution lacks reconciliation audit") + reconciliation = record.reconciliations[-1] + if ( + reconciliation.original_attempt.operation_id != record.operation_id + or reconciliation.resolution != record.resolution + ): + raise PhysicalCommandTransitionError( + "physical-state resolution does not match its reconciliation audit" + ) + return reconciliation + + +def _latest_resolved_start_recovery( + record: PhysicalCommandRecord, +) -> PhysicalCommandReconciliation | None: + matches = [ + item + for item in record.reconciliations + if item.kind in {"resolved-active-rebind", "resolved-active-cessation"} + and item.original_attempt.operation_id == record.operation_id + ] + return matches[-1] if matches else None + + +def _latest_current_physical_state_recovery( + record: PhysicalCommandRecord, +) -> PhysicalCommandReconciliation | None: + """Return only the newest physical classification for this operation. + + Reconciliation arrays are intentionally inherited by successors. Filtering + by the current operation keeps an older START/STOP classification from + changing the meaning of a later record. + """ + + matches = [ + item + for item in record.reconciliations + if item.kind + in { + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", + } + and item.original_attempt.operation_id == record.operation_id + ] + return matches[-1] if matches else None + + +def _latest_prepared_stop_recovery( + record: PhysicalCommandRecord, +) -> PhysicalCommandReconciliation | None: + classification = _prepared_stop_classification(record) + if classification is None: + return None + latest = _latest_current_physical_state_recovery(record) + if latest is None: + raise PhysicalCommandTransitionError( + "prepared STOP classification lacks current physical state" + ) + return latest + + +def _prepared_stop_classification( + record: PhysicalCommandRecord, +) -> PhysicalCommandReconciliation | None: + return next( + ( + item + for item in record.reconciliations + if item.kind == "prepared-stop-classification" + and item.original_attempt.operation_id == record.operation_id + ), + None, + ) + + +def _latest_current_classified_stop_reopen( + record: PhysicalCommandRecord, +) -> PhysicalCommandOperatorReconciliationReopen | None: + classified_retirement_ids = { + item.retirement_id + for item in record.operator_retirements + if item.original_attempt.operation_id == record.operation_id + and item.original_attempt.action == "stop" + and item.original_attempt.stage == "resolved" + and item.original_attempt.resolution == "not-dispatched" + } + matches = [ + item + for item in record.operator_reconciliation_reopens + if item.retirement_id in classified_retirement_ids + ] + return matches[-1] if matches else None + + +def _reopened_resolved_start_attempt( + record: PhysicalCommandRecord, +) -> PhysicalCommandAttemptAudit | None: + """Return the immutable successful START behind one current reopen. + + The latest reopen must be the exact current revision and every retirement + must have a matching reopen. The successful START may be present directly + in a retirement snapshot or as the original attempt of the newest active + rebind retained by the validated main/archive chain. + """ + + if not ( + record.action == "start" + and record.stage == "observing" + and record.resolution is None + and record.operator_reconciliation_reopens + and not active_operator_retirements(record) + ): + return None + latest_reopen = record.operator_reconciliation_reopens[-1] + if ( + latest_reopen.reason != "reset-network-intent-read-only-settlement" + or record.revision != latest_reopen.retired_record_revision + 1 + ): + return None + if not any( + retirement.retirement_id == latest_reopen.retirement_id + and retirement.original_attempt.operation_id == record.operation_id + for retirement in record.operator_retirements + ): + return None + latest_recovery = _latest_resolved_start_recovery(record) + if ( + latest_recovery is not None + and latest_recovery.resolution == "physical-active-observed" + and latest_recovery.original_attempt.operation_id == record.operation_id + and latest_recovery.original_attempt.stage == "resolved" + and latest_recovery.original_attempt.action == "start" + and latest_recovery.original_attempt.resolution == "start-active-observed" + ): + return latest_recovery.original_attempt + return next( + ( + retirement.original_attempt + for retirement in reversed(record.operator_retirements) + if retirement.original_attempt.operation_id == record.operation_id + and retirement.original_attempt.stage == "resolved" + and retirement.original_attempt.action == "start" + and retirement.original_attempt.resolution == "start-active-observed" + ), + None, + ) + + +def _effective_active_project_id( + record: PhysicalCommandRecord | None, +) -> str | None: + if record is None: + return None + if record.resolution == "start-active-observed": + latest_recovery = _latest_resolved_start_recovery(record) + if latest_recovery is not None: + return ( + latest_recovery.observation.project_id_sha256 + if latest_recovery.resolution == "physical-active-observed" + else None + ) + return record.last_status.project_id_sha256 if record.last_status is not None else None + if record.resolution == "physical-active-observed": + return _latest_reconciliation_for_operation(record).observation.project_id_sha256 + if record.resolution == "not-dispatched" and record.action == "stop": + latest = _latest_prepared_stop_recovery(record) + if latest is not None: + return ( + latest.observation.project_id_sha256 + if latest.resolution == "physical-active-observed" + else None + ) + return record.baseline_status.project_id_sha256 + return None + + +def _require_fresh_reconciliation_evidence( + record: PhysicalCommandRecord, + *, + resolution: PhysicalCommandReconciliationResolution, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + reconciled_at_utc: str, +) -> None: + _validate_reconciliation_components( + _attempt_from_record(record), + resolution=resolution, + verified_binding=verified_binding, + observation=observation, + reconciled_at_utc=reconciled_at_utc, + ) + + +def _validate_reconciliation_components( + original_attempt: PhysicalCommandAttemptAudit, + *, + resolution: PhysicalCommandReconciliationResolution, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + reconciled_at_utc: str, +) -> None: + if original_attempt.stage not in {"dispatching", "observing"}: + raise PhysicalCommandTransitionError( + "reconciliation original attempt was not ambiguously dispatched" + ) + if verified_binding.identity != original_attempt.identity: + raise PhysicalCommandTransitionError( + "reconciliation DeviceInfo identity does not match original command" + ) + if ( + verified_binding.connection.control_session_id + == original_attempt.connection.control_session_id + ): + raise PhysicalCommandTransitionError( + "reconciliation requires a new verified control-session binding" + ) + _require_bound_evidence( + verified_binding.identity, + verified_binding.connection, + observation, + ) + if observation.source != "explicit-read-only-reconciliation": + raise PhysicalCommandTransitionError( + "reconciliation requires explicit read-only status evidence" + ) + if observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "reconciliation cannot use retained DeviceStatus evidence" + ) + _validated_timestamp(reconciled_at_utc, field_name="reconciled_at_utc") + if resolution == "physical-active-observed": + if ( + observation.session_state != "scanning" + or not observation.project_bound + or not observation.init_ready + ): + raise PhysicalCommandTransitionError( + "active reconciliation requires bound initialized SCANNING" + ) + elif ( + observation.session_state != "ready" or observation.project_bound or observation.init_ready + ): + raise PhysicalCommandTransitionError( + "standby reconciliation requires unbound non-initialized READY" + ) + + +def _validate_active_rebind_components( + original_start: PhysicalCommandAttemptAudit, + *, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + reconciled_at_utc: str, +) -> None: + proven_start = bool( + original_start.stage == "resolved" + and original_start.action == "start" + and original_start.resolution == "start-active-observed" + ) + ambiguous_attempt = bool( + original_start.stage in {"dispatching", "observing"} and original_start.resolution is None + ) + if not (proven_start or ambiguous_attempt): + raise PhysicalCommandTransitionError( + "active rebind requires a successful START or ambiguous attempt audit" + ) + if verified_binding.identity != original_start.identity: + raise PhysicalCommandTransitionError( + "active rebind DeviceInfo identity does not match successful START" + ) + if ( + verified_binding.connection.control_session_id + == original_start.connection.control_session_id + ): + raise PhysicalCommandTransitionError( + "active rebind requires a new verified control-session binding" + ) + _require_bound_evidence( + verified_binding.identity, + verified_binding.connection, + observation, + ) + if observation.source != "explicit-read-only-reconciliation": + raise PhysicalCommandTransitionError( + "active rebind requires explicit read-only status evidence" + ) + if observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "active rebind cannot use retained DeviceStatus evidence" + ) + if ( + observation.session_state != "scanning" + or not observation.project_bound + or not observation.init_ready + ): + raise PhysicalCommandTransitionError("active rebind requires bound initialized SCANNING") + if proven_start: + proven_status = original_start.last_status + if ( + proven_status is None + or proven_status.project_id_sha256 is None + or observation.project_id_sha256 != proven_status.project_id_sha256 + ): + raise PhysicalCommandTransitionError( + "active rebind project does not match the proven successful START" + ) + _validated_timestamp(reconciled_at_utc, field_name="reconciled_at_utc") + + +def _is_exact_not_dispatched_stop_attempt( + attempt: PhysicalCommandAttemptAudit, +) -> bool: + """Accept PREPARED or its evidence-identical local no-dispatch closure.""" + + return bool( + attempt.action == "stop" + and ( + (attempt.stage == "prepared" and attempt.resolution is None) + or ( + attempt.stage == "resolved" + and attempt.resolution == "not-dispatched" + ) + ) + and attempt.publish_call_returned is None + and attempt.packet_id is None + and not attempt.qos2_completed + and attempt.application_response is None + and attempt.last_status is None + ) + + +def _validate_prepared_stop_classification_components( + original_stop: PhysicalCommandAttemptAudit, + *, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + reconciled_at_utc: str, +) -> PhysicalCommandReconciliationResolution: + """Validate fresh read-only state without reinterpreting the STOP edge.""" + + if not _is_exact_not_dispatched_stop_attempt(original_stop): + raise PhysicalCommandTransitionError( + "STOP classification requires an exact non-dispatched STOP" + ) + if verified_binding.identity != original_stop.identity: + raise PhysicalCommandTransitionError( + "prepared STOP DeviceInfo identity does not match the original command" + ) + if ( + verified_binding.connection.control_session_id + == original_stop.connection.control_session_id + ): + raise PhysicalCommandTransitionError( + "prepared STOP classification requires a new verified control-session binding" + ) + _require_bound_evidence( + verified_binding.identity, + verified_binding.connection, + observation, + ) + if observation.source != "explicit-read-only-reconciliation": + raise PhysicalCommandTransitionError( + "prepared STOP classification requires explicit read-only status evidence" + ) + if observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "prepared STOP classification cannot use retained DeviceStatus evidence" + ) + _validated_timestamp(reconciled_at_utc, field_name="reconciled_at_utc") + if ( + observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ): + return "physical-standby-observed" + if ( + observation.session_state == "scanning" + and observation.project_bound + and observation.init_ready + and observation.project_id_sha256 + == original_stop.baseline_status.project_id_sha256 + ): + return "physical-active-observed" + raise PhysicalCommandTransitionError( + "prepared STOP classification requires canonical READY or same-project SCANNING" + ) + + +def _validate_prepared_stop_followup_components( + original_stop: PhysicalCommandAttemptAudit, + *, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + required_state: Literal["scanning", "standby"], + reconciled_at_utc: str, +) -> None: + """Validate a later read-only generation without changing STOP outcome.""" + + if not _is_exact_not_dispatched_stop_attempt(original_stop): + raise PhysicalCommandTransitionError( + "prepared STOP follow-up lacks its original attempt" + ) + if verified_binding.identity != original_stop.identity: + raise PhysicalCommandTransitionError( + "prepared STOP follow-up DeviceInfo identity changed" + ) + if ( + verified_binding.connection.control_session_id + == original_stop.connection.control_session_id + ): + raise PhysicalCommandTransitionError( + "prepared STOP follow-up requires a new verified control-session binding" + ) + _require_bound_evidence( + verified_binding.identity, + verified_binding.connection, + observation, + ) + if observation.source != "explicit-read-only-reconciliation": + raise PhysicalCommandTransitionError( + "prepared STOP follow-up requires explicit read-only status evidence" + ) + if observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "prepared STOP follow-up cannot use retained DeviceStatus evidence" + ) + if required_state == "scanning": + if ( + observation.session_state != "scanning" + or not observation.project_bound + or not observation.init_ready + or observation.project_id_sha256 + != original_stop.baseline_status.project_id_sha256 + ): + raise PhysicalCommandTransitionError( + "prepared STOP active follow-up requires same-project SCANNING" + ) + elif not ( + observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ): + raise PhysicalCommandTransitionError( + "prepared STOP standby follow-up requires canonical READY" + ) + _validated_timestamp(reconciled_at_utc, field_name="reconciled_at_utc") + + +def _validate_resolved_active_standby_components( + original_start: PhysicalCommandAttemptAudit, + *, + verified_binding: PhysicalCommandVerifiedBinding, + observation: PhysicalCommandStatusEvidence, + reconciled_at_utc: str, +) -> None: + proven_start = bool( + original_start.stage == "resolved" + and original_start.action == "start" + and original_start.resolution == "start-active-observed" + ) + ambiguous_attempt = bool( + original_start.stage in {"dispatching", "observing"} and original_start.resolution is None + ) + if not (proven_start or ambiguous_attempt): + raise PhysicalCommandTransitionError( + "active cessation requires a successful START or ambiguous attempt audit" + ) + if verified_binding.identity != original_start.identity: + raise PhysicalCommandTransitionError( + "active cessation DeviceInfo identity does not match successful START" + ) + if ( + verified_binding.connection.control_session_id + == original_start.connection.control_session_id + ): + raise PhysicalCommandTransitionError( + "active cessation requires a new verified control-session binding" + ) + _require_bound_evidence( + verified_binding.identity, + verified_binding.connection, + observation, + ) + if observation.source != "explicit-read-only-reconciliation": + raise PhysicalCommandTransitionError( + "active cessation requires explicit read-only status evidence" + ) + if observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "active cessation cannot use retained DeviceStatus evidence" + ) + canonical_ready = bool( + observation.session_state == "ready" + and not observation.project_bound + and not observation.init_ready + ) + # SCAN_OVER proves that the old scan is no longer producing, but it is not + # yet authority for a new START. Persist it as a cessation audit so the + # previous START is no longer modeled as active; the successor gate below + # still requires a later, fresh unbound READY reconciliation. + canonical_scan_over = observation.session_state == "scan_over" + if not (canonical_ready or canonical_scan_over): + raise PhysicalCommandTransitionError( + "active cessation requires SCAN_OVER or unbound non-initialized READY" + ) + _validated_timestamp(reconciled_at_utc, field_name="reconciled_at_utc") + + +def _validate_attempt_audit(attempt: PhysicalCommandAttemptAudit) -> None: + _validate_positive_int(attempt.revision, field_name="revision") + if attempt.revision > _MAX_REVISION: + raise ValueError("physical attempt audit revision exceeds supported range") + _validate_identifier(attempt.operation_id, field_name="operation_id") + if attempt.parent_operation_id is not None: + _validate_identifier( + attempt.parent_operation_id, + field_name="parent_operation_id", + ) + _validate_identifier(attempt.acquisition_id, field_name="acquisition_id") + _validate_action(attempt.action) + if attempt.stage not in {"prepared", "dispatching", "observing", "resolved"}: + raise ValueError("physical attempt audit has unsupported stage") + if attempt.resolution is not None and attempt.resolution not in { + *_ORDINARY_RESOLUTIONS, + "physical-active-observed", + }: + raise ValueError("physical attempt audit has unsupported resolution") + if not isinstance(attempt.identity, PhysicalCommandIdentity): + raise TypeError("attempt identity must be PhysicalCommandIdentity") + if not isinstance(attempt.connection, PhysicalCommandConnectionBinding): + raise TypeError("attempt connection must be PhysicalCommandConnectionBinding") + _validate_identifier( + attempt.compatibility_profile_id, + field_name="compatibility_profile_id", + ) + _validate_sha256(attempt.payload_sha256, field_name="payload_sha256") + _require_bound_evidence( + attempt.identity, + attempt.connection, + attempt.baseline_status, + ) + _require_live_baseline(attempt.action, attempt.baseline_status) + created = _as_datetime(attempt.created_at_utc) + updated = _as_datetime(attempt.updated_at_utc) + if updated < created: + raise ValueError("physical attempt audit update precedes creation") + if attempt.packet_id is not None: + _validate_positive_int(attempt.packet_id, field_name="packet_id") + if attempt.publish_call_returned is not True: + raise ValueError("physical attempt packet lacks returned publish") + if attempt.qos2_completed and ( + attempt.publish_call_returned is not True or attempt.packet_id is None + ): + raise ValueError("physical attempt QoS2 completion lacks packet proof") + if attempt.application_response is not None: + _require_response_binding( + operation_id=attempt.operation_id, + action=attempt.action, + connection=attempt.connection, + response=attempt.application_response, + ) + if attempt.last_status is not None: + _require_bound_evidence( + attempt.identity, + attempt.connection, + attempt.last_status, + ) + post_fields_present = any( + ( + attempt.publish_call_returned is not None, + attempt.packet_id is not None, + attempt.qos2_completed, + attempt.application_response is not None, + attempt.last_status is not None, + ) + ) + if attempt.stage == "prepared" and ( + post_fields_present or attempt.resolution is not None + ): + raise ValueError("prepared attempt audit contains post-dispatch fields") + if attempt.stage == "dispatching" and (post_fields_present or attempt.resolution is not None): + raise ValueError("dispatching attempt audit contains observation fields") + if attempt.stage == "observing" and (not post_fields_present or attempt.resolution is not None): + raise ValueError("observing attempt audit lacks evidence fields") + if attempt.stage == "resolved": + if attempt.resolution == "start-active-observed": + if attempt.action != "start": + raise ValueError("resolved attempt audit must preserve successful START") + if ( + not attempt.qos2_completed + or attempt.application_response is None + or not attempt.application_response.success + or attempt.last_status is None + ): + raise ValueError("resolved START audit lacks complete success evidence") + if ( + attempt.last_status.source != "live-control-session" + or attempt.last_status.mqtt_retained + or attempt.last_status.session_state != "scanning" + or not attempt.last_status.project_bound + or not attempt.last_status.init_ready + ): + raise ValueError("resolved START audit lacks exact active status evidence") + elif attempt.resolution == "physical-active-observed": + # The command fields stay immutable across a read-only physical + # reconciliation, so its fresh DeviceInfo/DeviceStatus proof lives + # in the enclosing record's reconciliation audit. The record + # validator binds this exact current-revision snapshot to that + # complete proof chain before accepting a retirement document. + pass + elif ( + attempt.action == "stop" + and attempt.resolution == "not-dispatched" + and not post_fields_present + ): + # The enclosing record must additionally bind this snapshot to an + # active ``prepared-stop-classification`` reconciliation. Keeping + # that cross-object condition out of the immutable attempt type + # lets the attempt remain a faithful no-dispatch edge snapshot. + pass + else: + raise ValueError("resolved attempt audit lacks active physical evidence") + + +def _validate_reconciliation(reconciliation: PhysicalCommandReconciliation) -> None: + _validate_identifier( + reconciliation.reconciliation_id, + field_name="reconciliation_id", + ) + if reconciliation.kind not in _RECONCILIATION_KINDS: + raise ValueError("unsupported physical reconciliation kind") + if reconciliation.resolution not in _RECONCILIATION_RESOLUTIONS: + raise ValueError("unsupported physical reconciliation resolution") + if not isinstance(reconciliation.original_attempt, PhysicalCommandAttemptAudit): + raise TypeError("original_attempt must be PhysicalCommandAttemptAudit") + _validate_sha256( + reconciliation.original_attempt_sha256, + field_name="original_attempt_sha256", + ) + if reconciliation.original_attempt_sha256 != _sha256_document( + reconciliation.original_attempt.as_dict() + ): + raise ValueError("physical reconciliation original-attempt hash does not match") + if not isinstance(reconciliation.verified_binding, PhysicalCommandVerifiedBinding): + raise TypeError("verified_binding must be PhysicalCommandVerifiedBinding") + if not isinstance(reconciliation.observation, PhysicalCommandStatusEvidence): + raise TypeError("observation must be PhysicalCommandStatusEvidence") + if reconciliation.kind == "ambiguous-outcome": + _validate_reconciliation_components( + reconciliation.original_attempt, + resolution=reconciliation.resolution, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + elif reconciliation.kind == "prepared-stop-classification": + expected_resolution = _validate_prepared_stop_classification_components( + reconciliation.original_attempt, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + if reconciliation.resolution != expected_resolution: + raise ValueError( + "prepared STOP classification resolution does not match observation" + ) + elif reconciliation.kind == "resolved-active-rebind": + if reconciliation.resolution != "physical-active-observed": + raise ValueError("resolved-active rebind must observe active state") + if _is_exact_not_dispatched_stop_attempt(reconciliation.original_attempt): + _validate_prepared_stop_followup_components( + reconciliation.original_attempt, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + required_state="scanning", + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + else: + _validate_active_rebind_components( + reconciliation.original_attempt, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + else: + if reconciliation.resolution != "physical-standby-observed": + raise ValueError("resolved-active cessation must observe standby state") + if _is_exact_not_dispatched_stop_attempt(reconciliation.original_attempt): + _validate_prepared_stop_followup_components( + reconciliation.original_attempt, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + required_state="standby", + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + else: + _validate_resolved_active_standby_components( + reconciliation.original_attempt, + verified_binding=reconciliation.verified_binding, + observation=reconciliation.observation, + reconciled_at_utc=reconciliation.reconciled_at_utc, + ) + + +def _validate_operator_retirement( + retirement: PhysicalCommandOperatorRetirement, +) -> None: + _validate_identifier(retirement.retirement_id, field_name="retirement_id") + if retirement.reason not in _PHYSICAL_COMMAND_OPERATOR_RETIREMENT_REASONS: + raise ValueError("unsupported physical operator-retirement reason") + if not isinstance(retirement.original_attempt, PhysicalCommandAttemptAudit): + raise TypeError("retirement original_attempt must be PhysicalCommandAttemptAudit") + _validate_sha256( + retirement.original_attempt_sha256, + field_name="original_attempt_sha256", + ) + if retirement.original_attempt_sha256 != _sha256_document( + retirement.original_attempt.as_dict() + ): + raise ValueError("physical retirement original-attempt hash does not match") + original_attempt = retirement.original_attempt + ambiguous_attempt = bool( + original_attempt.stage in {"dispatching", "observing"} + and original_attempt.resolution is None + ) + proven_active_attempt = bool( + original_attempt.stage == "resolved" + and original_attempt.resolution in {"start-active-observed", "physical-active-observed"} + ) + classified_active_not_dispatched = bool( + original_attempt.stage == "resolved" + and original_attempt.action == "stop" + and original_attempt.resolution == "not-dispatched" + ) + if not ( + ambiguous_attempt + or proven_active_attempt + or classified_active_not_dispatched + ): + raise ValueError("physical retirement must preserve an ambiguous or proven-active attempt") + _validate_identifier( + retirement.retired_transport_ref, + field_name="retired_transport_ref", + ) + if retirement.retired_transport_ref != retirement.original_attempt.connection.transport_ref: + raise ValueError("physical retirement transport does not match original attempt") + retired = _validated_timestamp( + retirement.retired_at_utc, + field_name="retired_at_utc", + ) + if retired < _as_datetime(retirement.original_attempt.updated_at_utc): + raise ValueError("physical retirement precedes the original attempt") + + +def _validate_operator_reconciliation_reopen( + reopening: PhysicalCommandOperatorReconciliationReopen, +) -> None: + _validate_identifier(reopening.reopening_id, field_name="reopening_id") + _validate_identifier(reopening.retirement_id, field_name="retirement_id") + _validate_sha256(reopening.retirement_sha256, field_name="retirement_sha256") + if reopening.reason not in _PHYSICAL_COMMAND_OPERATOR_RECONCILIATION_REOPEN_REASONS: + raise ValueError("unsupported physical operator-reconciliation reopen reason") + _validate_positive_int( + reopening.retired_record_revision, + field_name="retired_record_revision", + ) + _validate_identifier( + reopening.reopened_transport_ref, + field_name="reopened_transport_ref", + ) + _validate_positive_int( + reopening.discovery_generation, + field_name="discovery_generation", + ) + _validated_timestamp(reopening.reopened_at_utc, field_name="reopened_at_utc") + + +def _require_reconciliation_matches_resolved_record( + record: PhysicalCommandRecord, + reconciliation: PhysicalCommandReconciliation, +) -> None: + attempt = reconciliation.original_attempt + if record.resolution != reconciliation.resolution: + raise ValueError("physical reconciliation resolution does not match ledger") + operation_reconciliation_count = sum( + item.original_attempt.operation_id == attempt.operation_id + for item in record.reconciliations + ) + if record.revision != attempt.revision + operation_reconciliation_count: + raise ValueError("physical reconciliation did not follow its original revision") + if ( + record.operation_id != attempt.operation_id + or record.parent_operation_id != attempt.parent_operation_id + or record.acquisition_id != attempt.acquisition_id + or record.action != attempt.action + or record.identity != attempt.identity + or record.connection != attempt.connection + or record.compatibility_profile_id != attempt.compatibility_profile_id + or record.payload_sha256 != attempt.payload_sha256 + or record.baseline_status != attempt.baseline_status + or record.publish_call_returned != attempt.publish_call_returned + or record.packet_id != attempt.packet_id + or record.qos2_completed != attempt.qos2_completed + or record.application_response != attempt.application_response + or record.last_status != attempt.last_status + or record.created_at_utc != attempt.created_at_utc + ): + raise ValueError("physical reconciliation altered original command evidence") + if _as_datetime(record.updated_at_utc) < _as_datetime(attempt.updated_at_utc): + raise ValueError("physical reconciliation ledger time regressed") + + +def _prepared_stop_retained_transition_revision( + record: PhysicalCommandRecord, + *, + classification: PhysicalCommandReconciliation, +) -> int: + operation_reconciliation_count = sum( + item.original_attempt.operation_id == record.operation_id + and item.kind + in { + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", + } + for item in record.reconciliations + ) + operation_retirement_ids = { + item.retirement_id + for item in record.operator_retirements + if item.original_attempt.operation_id == record.operation_id + } + operation_reopen_count = sum( + item.retirement_id in operation_retirement_ids + for item in record.operator_reconciliation_reopens + ) + return ( + classification.original_attempt.revision + + operation_reconciliation_count + + len(operation_retirement_ids) + + operation_reopen_count + ) + + +def _require_prepared_stop_archive_gap_matches_audits( + record: PhysicalCommandRecord, + *, + archived_retirement_ids: set[str], + archived_reopens: dict[str, str], +) -> None: + """Bind a compact classified-STOP revision gap to immutable pair audits.""" + + if record.stage != "resolved" or record.resolution != "not-dispatched": + return + classification = _prepared_stop_classification(record) + if classification is None: + return + retained_revision = _prepared_stop_retained_transition_revision( + record, + classification=classification, + ) + revision_gap = record.revision - retained_revision + live_retirement_ids = { + item.retirement_id + for item in record.operator_retirements + if item.original_attempt.operation_id == record.operation_id + } + live_reopening_ids = { + item.reopening_id + for item in record.operator_reconciliation_reopens + if item.retirement_id in live_retirement_ids + } + omitted_retirement_ids = archived_retirement_ids - live_retirement_ids + omitted_reopens = { + reopening_id: retirement_id + for reopening_id, retirement_id in archived_reopens.items() + if reopening_id not in live_reopening_ids + } + if ( + set(omitted_reopens.values()) != omitted_retirement_ids + or len(set(omitted_reopens.values())) != len(omitted_reopens) + or revision_gap + != len(omitted_retirement_ids) + len(omitted_reopens) + ): + raise PhysicalCommandLedgerCorrupt( + "prepared STOP compact revision gap lacks immutable retire/reopen pairs" + ) + + +def _require_prepared_stop_classification_matches_record( + record: PhysicalCommandRecord, + classification: PhysicalCommandReconciliation, +) -> None: + """Bind ``not-dispatched`` and its independent physical-state audit.""" + + attempt = classification.original_attempt + operation_reconciliations = [ + item + for item in record.reconciliations + if item.original_attempt.operation_id == record.operation_id + and item.kind + in { + "prepared-stop-classification", + "resolved-active-rebind", + "resolved-active-cessation", + } + ] + if ( + classification.kind != "prepared-stop-classification" + or not operation_reconciliations + or operation_reconciliations[0] != classification + or sum( + item.kind == "prepared-stop-classification" + for item in operation_reconciliations + ) + != 1 + ): + raise ValueError("not-dispatched STOP lacks one leading physical classification") + retained_transition_revision = _prepared_stop_retained_transition_revision( + record, + classification=classification, + ) + archived_pair_revision_gap = record.revision - retained_transition_revision + if archived_pair_revision_gap < 0 or ( + archived_pair_revision_gap != 0 + and ( + record.archive_head_sha256 is None + # Compaction may omit only completed retirement/reopen pairs for + # this operation. Each omitted pair accounts for exactly two + # durable revisions; reconciliations for the current operation and + # the newest retire/reopen tail remain live. + or archived_pair_revision_gap % 2 != 0 + ) + ): + raise ValueError("prepared STOP classification revision chain is not contiguous") + if any( + item.original_attempt != attempt + or item.original_attempt_sha256 != classification.original_attempt_sha256 + for item in operation_reconciliations + ): + raise ValueError("prepared STOP recovery changed its original attempt") + if ( + record.operation_id != attempt.operation_id + or record.parent_operation_id != attempt.parent_operation_id + or record.acquisition_id != attempt.acquisition_id + or record.action != "stop" + or attempt.action != "stop" + or record.identity != attempt.identity + or record.connection != attempt.connection + or record.compatibility_profile_id != attempt.compatibility_profile_id + or record.payload_sha256 != attempt.payload_sha256 + or record.baseline_status != attempt.baseline_status + or record.publish_call_returned is not None + or record.packet_id is not None + or record.qos2_completed + or record.application_response is not None + or record.last_status is not None + or record.created_at_utc != attempt.created_at_utc + or record.resolution != "not-dispatched" + ): + raise ValueError("prepared STOP classification altered original command evidence") + if _as_datetime(record.updated_at_utc) < _as_datetime(attempt.updated_at_utc): + raise ValueError("prepared STOP classification ledger time regressed") + + +def _require_prepared_stop_classification_matches_attempt( + attempt: PhysicalCommandAttemptAudit, + classification: PhysicalCommandReconciliation, +) -> None: + original = classification.original_attempt + if ( + classification.kind != "prepared-stop-classification" + or classification.resolution + not in {"physical-active-observed", "physical-standby-observed"} + or attempt.stage != "resolved" + or attempt.resolution != "not-dispatched" + or attempt.revision <= original.revision + or attempt.operation_id != original.operation_id + or attempt.parent_operation_id != original.parent_operation_id + or attempt.acquisition_id != original.acquisition_id + or attempt.action != original.action + or attempt.identity != original.identity + or attempt.connection != original.connection + or attempt.compatibility_profile_id != original.compatibility_profile_id + or attempt.payload_sha256 != original.payload_sha256 + or attempt.baseline_status != original.baseline_status + or attempt.publish_call_returned is not None + or attempt.packet_id is not None + or attempt.qos2_completed + or attempt.application_response is not None + or attempt.last_status is not None + or attempt.created_at_utc != original.created_at_utc + ): + raise ValueError("retired prepared STOP classification changed its original audit") + + +def _require_bound_evidence( + identity: PhysicalCommandIdentity, + connection: PhysicalCommandConnectionBinding, + evidence: PhysicalCommandStatusEvidence, +) -> None: + if ( + evidence.vendor_device_id_sha256 != identity.vendor_device_id_sha256 + or evidence.device_serial_sha256 != identity.device_serial_sha256 + ): + raise PhysicalCommandTransitionError("physical status identity does not match the command") + if ( + evidence.control_session_id != connection.control_session_id + or evidence.host_path_epoch != connection.host_path_epoch + or evidence.producer_generation != connection.producer_generation + ): + raise PhysicalCommandTransitionError( + "physical status does not match the live-path generation" + ) + + +def _require_bound_response( + record: PhysicalCommandRecord, + response: PhysicalCommandApplicationResponse, +) -> None: + _require_response_binding( + operation_id=record.operation_id, + action=record.action, + connection=record.connection, + response=response, + ) + + +def _require_response_binding( + *, + operation_id: str, + action: PhysicalCommandAction, + connection: PhysicalCommandConnectionBinding, + response: PhysicalCommandApplicationResponse, +) -> None: + if response.operation_id != operation_id or response.action != action: + raise PhysicalCommandTransitionError( + "application response does not correlate to the physical command" + ) + if ( + response.control_session_id != connection.control_session_id + or response.host_path_epoch != connection.host_path_epoch + or response.producer_generation != connection.producer_generation + ): + raise PhysicalCommandTransitionError( + "application response does not match the live-path generation" + ) + + +def _require_live_baseline( + action: PhysicalCommandAction, + baseline: PhysicalCommandStatusEvidence, +) -> None: + if baseline.source != "live-control-session" or baseline.mqtt_retained: + raise PhysicalCommandTransitionError( + "physical command baseline must be a non-retained live-session report" + ) + if action == "start": + if baseline.session_state != "ready" or baseline.project_bound or baseline.init_ready: + raise PhysicalCommandTransitionError( + "START baseline must be unbound READY and not initialized" + ) + elif ( + baseline.session_state != "scanning" + or not baseline.project_bound + or not baseline.init_ready + ): + raise PhysicalCommandTransitionError("STOP baseline must be bound initialized SCANNING") + + +def _require_complete_target_gate( + record: PhysicalCommandRecord, + resolution: PhysicalCommandResolution, + observation: PhysicalCommandStatusEvidence, +) -> None: + response = record.application_response + if not record.qos2_completed or response is None or not response.success: + raise PhysicalCommandTransitionError( + "target resolution requires QoS2 and exact successful application response" + ) + _require_bound_response(record, response) + if observation.source != "live-control-session" or observation.mqtt_retained: + raise PhysicalCommandTransitionError( + "target resolution requires a non-retained live-session status" + ) + if resolution == "start-active-observed": + if ( + observation.session_state != "scanning" + or not observation.project_bound + or not observation.init_ready + ): + raise PhysicalCommandTransitionError( + "START resolution requires bound initialized SCANNING" + ) + elif ( + observation.session_state != "ready" or observation.project_bound or observation.init_ready + ): + raise PhysicalCommandTransitionError( + "STOP resolution requires unbound non-initialized READY" + ) + + +def _validate_record_semantics(record: PhysicalCommandRecord) -> None: + if record.schema_version != PHYSICAL_COMMAND_LEDGER_SCHEMA: + raise ValueError("unsupported physical command ledger schema") + _validate_positive_int(record.revision, field_name="revision") + if record.revision > _MAX_REVISION: + raise ValueError("physical command revision exceeds the supported range") + _validate_identifier(record.operation_id, field_name="operation_id") + if record.parent_operation_id is not None: + _validate_identifier(record.parent_operation_id, field_name="parent_operation_id") + _validate_identifier(record.acquisition_id, field_name="acquisition_id") + _validate_action(record.action) + if record.stage not in _STAGES: + raise ValueError("unsupported physical command stage") + _validate_identifier( + record.compatibility_profile_id, + field_name="compatibility_profile_id", + ) + _validate_sha256(record.payload_sha256, field_name="payload_sha256") + _require_bound_evidence(record.identity, record.connection, record.baseline_status) + _require_live_baseline(record.action, record.baseline_status) + created = _validated_timestamp(record.created_at_utc, field_name="created_at_utc") + updated = _validated_timestamp(record.updated_at_utc, field_name="updated_at_utc") + if updated < created: + raise ValueError("physical command update precedes creation") + if record.archive_head_sha256 is None: + if record.archive_segment_count != 0 or record.archive_total_bytes != 0: + raise ValueError("physical command archive metadata lacks a head") + else: + _validate_sha256(record.archive_head_sha256, field_name="archive_head_sha256") + _validate_positive_int( + record.archive_segment_count, + field_name="archive_segment_count", + ) + _validate_positive_int( + record.archive_total_bytes, + field_name="archive_total_bytes", + ) + if record.archive_segment_count > PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS: + raise ValueError("physical command archive segment count exceeds the bound") + if record.archive_total_bytes > PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES: + raise ValueError("physical command archive byte count exceeds the bound") + if ( + not isinstance(record.reconciliations, tuple) + or len(record.reconciliations) > _MAX_RECONCILIATIONS + ): + raise ValueError("physical reconciliation audit is not a bounded tuple") + if ( + not isinstance(record.operator_confirmations, tuple) + or len(record.operator_confirmations) > _MAX_OPERATOR_CONFIRMATIONS + ): + raise ValueError("physical operator-confirmation audit is not a bounded tuple") + if ( + not isinstance(record.operator_retirements, tuple) + or len(record.operator_retirements) > _MAX_OPERATOR_RETIREMENTS + ): + raise ValueError("physical operator-retirement audit is not a bounded tuple") + if ( + not isinstance(record.operator_reconciliation_reopens, tuple) + or len(record.operator_reconciliation_reopens) > _MAX_OPERATOR_RECONCILIATION_REOPENS + ): + raise ValueError("physical reconciliation-reopen audit is not a bounded tuple") + reconciliation_ids = [item.reconciliation_id for item in record.reconciliations] + ambiguous_reconciliation_operations = [ + item.original_attempt.operation_id + for item in record.reconciliations + if item.kind == "ambiguous-outcome" + ] + prepared_stop_classification_operations = [ + item.original_attempt.operation_id + for item in record.reconciliations + if item.kind == "prepared-stop-classification" + ] + verification_ids = [item.verified_binding.verification_id for item in record.reconciliations] + verified_control_sessions = [ + item.verified_binding.connection.control_session_id for item in record.reconciliations + ] + if len(set(reconciliation_ids)) != len(reconciliation_ids) or len( + set(ambiguous_reconciliation_operations) + ) != len(ambiguous_reconciliation_operations): + raise ValueError("physical reconciliation audit contains duplicate identities") + if len(set(prepared_stop_classification_operations)) != len( + prepared_stop_classification_operations + ): + raise ValueError("prepared STOP classification was committed more than once") + if len(set(verification_ids)) != len(verification_ids) or len( + set(verified_control_sessions) + ) != len(verified_control_sessions): + raise ValueError("physical verified-binding audit contains duplicate identities") + for reconciliation in record.reconciliations: + if not isinstance(reconciliation, PhysicalCommandReconciliation): + raise TypeError("reconciliation audit item has invalid type") + _validate_reconciliation(reconciliation) + historical_retirement = next( + ( + retirement + for retirement in record.operator_retirements + if retirement.original_attempt.identity == reconciliation.original_attempt.identity + and retirement.original_attempt.compatibility_profile_id + == reconciliation.original_attempt.compatibility_profile_id + ), + None, + ) + if not ( + ( + reconciliation.original_attempt.identity == record.identity + and reconciliation.original_attempt.compatibility_profile_id + == record.compatibility_profile_id + ) + or historical_retirement is not None + ): + raise ValueError("physical reconciliation left the identity/profile chain") + active_projects_by_operation: dict[str, str | None] = {} + scan_over_ready_pending_by_operation: dict[str, bool] = {} + for reconciliation in record.reconciliations: + operation_id = reconciliation.original_attempt.operation_id + if ( + operation_id not in active_projects_by_operation + and reconciliation.original_attempt.stage == "resolved" + and reconciliation.original_attempt.resolution == "start-active-observed" + ): + original_status = reconciliation.original_attempt.last_status + active_projects_by_operation[operation_id] = ( + original_status.project_id_sha256 if original_status is not None else None + ) + if reconciliation.kind in { + "ambiguous-outcome", + "prepared-stop-classification", + }: + active_projects_by_operation[operation_id] = ( + reconciliation.observation.project_id_sha256 + if reconciliation.resolution == "physical-active-observed" + else None + ) + scan_over_ready_pending_by_operation[operation_id] = False + elif reconciliation.kind == "resolved-active-rebind": + prior_project = active_projects_by_operation.get(operation_id) + if ( + prior_project is None + or reconciliation.observation.project_id_sha256 != prior_project + ): + raise ValueError("resolved-active rebind changed or lacked the active project") + scan_over_ready_pending_by_operation[operation_id] = False + elif reconciliation.kind == "resolved-active-cessation": + if active_projects_by_operation.get(operation_id) is None and not ( + scan_over_ready_pending_by_operation.get(operation_id) is True + and reconciliation.observation.session_state == "ready" + and not reconciliation.observation.project_bound + and not reconciliation.observation.init_ready + ): + raise ValueError("resolved-active cessation lacks a prior active state") + active_projects_by_operation[operation_id] = None + scan_over_ready_pending_by_operation[operation_id] = ( + reconciliation.observation.session_state == "scan_over" + ) + confirmation_ids = [item.confirmation_id for item in record.operator_confirmations] + confirmed_operations = [item.stop_operation_id for item in record.operator_confirmations] + if len(set(confirmation_ids)) != len(confirmation_ids) or len(set(confirmed_operations)) != len( + confirmed_operations + ): + raise ValueError("physical operator confirmation was consumed more than once") + for confirmation in record.operator_confirmations: + if not isinstance(confirmation, PhysicalCommandOperatorConfirmation): + raise TypeError("operator confirmation audit item has invalid type") + reconciliation = _reconciliation_by_id( + record.reconciliations, + confirmation.reconciliation_id, + ) + if reconciliation.resolution != "physical-active-observed": + raise ValueError("operator STOP confirmation lacks active reconciliation") + retirement_ids = [item.retirement_id for item in record.operator_retirements] + if len(set(retirement_ids)) != len(retirement_ids): + raise ValueError("physical operator-retirement audit contains duplicate identities") + previous_retired_at: datetime | None = None + for retirement in record.operator_retirements: + if not isinstance(retirement, PhysicalCommandOperatorRetirement): + raise TypeError("operator retirement audit item has invalid type") + _validate_operator_retirement(retirement) + if retirement.original_attempt.resolution == "not-dispatched": + allowed_classification_resolutions = { + "physical-active-observed", + } + if retirement.reason == "connection-scenario-reset-by-operator": + # A reset may explicitly abandon the future-START binding of + # a classified READY/standby STOP. The complete classification + # remains in audit; only the old scenario ownership is retired. + allowed_classification_resolutions.add( + "physical-standby-observed" + ) + matching_classifications = [ + item + for item in record.reconciliations + if item.kind == "prepared-stop-classification" + and item.original_attempt.operation_id + == retirement.original_attempt.operation_id + and item.resolution in allowed_classification_resolutions + ] + if len(matching_classifications) != 1: + raise ValueError( + "retired not-dispatched STOP lacks one eligible classification" + ) + _require_prepared_stop_classification_matches_attempt( + retirement.original_attempt, + matching_classifications[0], + ) + retired_at = _as_datetime(retirement.retired_at_utc) + if previous_retired_at is not None and retired_at < previous_retired_at: + raise ValueError("physical operator-retirement audit time regressed") + previous_retired_at = retired_at + + reopening_ids = [item.reopening_id for item in record.operator_reconciliation_reopens] + reopened_retirement_ids = [ + item.retirement_id for item in record.operator_reconciliation_reopens + ] + if len(set(reopening_ids)) != len(reopening_ids) or len(set(reopened_retirement_ids)) != len( + reopened_retirement_ids + ): + raise ValueError("physical reconciliation-reopen audit contains duplicate identities") + previous_reopened_at: datetime | None = None + for reopening in record.operator_reconciliation_reopens: + if not isinstance( + reopening, + PhysicalCommandOperatorReconciliationReopen, + ): + raise TypeError("operator reconciliation-reopen audit item has invalid type") + _validate_operator_reconciliation_reopen(reopening) + matched_retirement = next( + ( + item + for item in record.operator_retirements + if item.retirement_id == reopening.retirement_id + ), + None, + ) + if matched_retirement is None: + raise ValueError("physical reconciliation reopen lacks retirement audit") + if reopening.retirement_sha256 != _sha256_document(matched_retirement.as_dict()): + raise ValueError("physical reconciliation reopen retirement hash does not match") + if reopening.retired_record_revision != matched_retirement.original_attempt.revision + 1: + raise ValueError("physical reconciliation reopen revision does not follow retirement") + if physical_transport_ref_comparison_key( + reopening.reopened_transport_ref + ) != physical_transport_ref_comparison_key(matched_retirement.retired_transport_ref): + raise ValueError("physical reconciliation reopen transport does not match retirement") + reopened_at = _as_datetime(reopening.reopened_at_utc) + if reopened_at < _as_datetime(matched_retirement.retired_at_utc): + raise ValueError("physical reconciliation reopen precedes retirement") + if previous_reopened_at is not None and reopened_at < previous_reopened_at: + raise ValueError("physical reconciliation-reopen audit time regressed") + previous_reopened_at = reopened_at + + reopens_by_retirement_id = { + item.retirement_id: item for item in record.operator_reconciliation_reopens + } + retirements_by_operation: dict[str, list[PhysicalCommandOperatorRetirement]] = {} + for retirement in record.operator_retirements: + retirements_by_operation.setdefault( + retirement.original_attempt.operation_id, + [], + ).append(retirement) + active_retirement_ids = {item.retirement_id for item in active_operator_retirements(record)} + for operation_retirements in retirements_by_operation.values(): + if sum(item.retirement_id in active_retirement_ids for item in operation_retirements) > 1: + raise ValueError("physical operator-retirement cycles contain multiple active fences") + for prior, current_retirement in zip( + operation_retirements, + operation_retirements[1:], + strict=False, + ): + prior_reopen = reopens_by_retirement_id.get(prior.retirement_id) + if prior_reopen is None: + raise ValueError("physical operator-retirement cycle lacks its prior reopen") + prior_attempt = prior.original_attempt + current_attempt = current_retirement.original_attempt + reopened_revision = prior_reopen.retired_record_revision + 1 + if current_attempt.resolution == "not-dispatched": + # A reopened classified STOP can disappear immediately (zero + # intervening transitions), or after exactly one fresh + # SCANNING rebind. Its followup reconciliation intentionally + # keeps the immutable PREPARED attempt as original evidence, + # so that audit's revision cannot be used as a cycle marker. + if current_attempt.revision not in { + reopened_revision, + reopened_revision + 1, + }: + raise ValueError( + "classified STOP retirement cycle revision is not contiguous" + ) + else: + post_reopen_reconciliation_count = sum( + item.original_attempt.operation_id == current_attempt.operation_id + and item.original_attempt.revision >= reopened_revision + and item.original_attempt.revision < current_attempt.revision + for item in record.reconciliations + ) + if current_attempt.revision != ( + reopened_revision + post_reopen_reconciliation_count + ): + raise ValueError( + "physical operator-retirement cycle revision is not contiguous" + ) + if not _same_attempt_evidence_except_revision_and_update( + prior_attempt, + current_attempt, + ): + raise ValueError("physical operator-retirement cycle changed original evidence") + if _as_datetime(current_attempt.updated_at_utc) < _as_datetime( + prior_reopen.reopened_at_utc + ): + raise ValueError("physical operator-retirement cycle precedes its prior reopen") + + if record.packet_id is not None: + _validate_positive_int(record.packet_id, field_name="packet_id") + if record.publish_call_returned is not True: + raise ValueError("physical packet_id lacks returned publish call") + if record.qos2_completed and ( + record.publish_call_returned is not True or record.packet_id is None + ): + raise ValueError("QoS2 completion lacks a returned publish and packet_id") + if record.application_response is not None: + _require_bound_response(record, record.application_response) + if record.last_status is not None: + _require_bound_evidence(record.identity, record.connection, record.last_status) + + post_fields_present = any( + ( + record.publish_call_returned is not None, + record.packet_id is not None, + record.qos2_completed, + record.application_response is not None, + record.last_status is not None, + ) + ) + reopened_current = next( + ( + reopening + for reopening in reversed(record.operator_reconciliation_reopens) + if any( + retirement.retirement_id == reopening.retirement_id + and retirement.original_attempt.operation_id == record.operation_id + for retirement in record.operator_retirements + ) + ), + None, + ) + if record.stage == "prepared": + if post_fields_present or record.resolution is not None: + raise ValueError("prepared physical command contains post-dispatch fields") + elif record.stage == "dispatching": + if post_fields_present or record.resolution is not None: + raise ValueError("dispatching physical command contains observation fields") + if reopened_current is not None and record.revision != ( + reopened_current.retired_record_revision + 1 + ): + raise ValueError("unresolved physical reconciliation reopen has invalid revision") + elif record.stage == "observing": + if not post_fields_present or record.resolution is not None: + raise ValueError("observing physical command has invalid evidence fields") + if reopened_current is not None and record.revision != ( + reopened_current.retired_record_revision + 1 + ): + raise ValueError("unresolved physical reconciliation reopen has invalid revision") + else: + if record.resolution is None: + raise ValueError("resolved physical command lacks a resolution") + _validate_resolution(record.resolution) + if record.resolution == "not-dispatched": + if post_fields_present: + raise ValueError("not-dispatched resolution contains post-dispatch fields") + current_classifications = [ + item + for item in record.reconciliations + if item.kind == "prepared-stop-classification" + and item.original_attempt.operation_id == record.operation_id + ] + if current_classifications: + _require_prepared_stop_classification_matches_record( + record, + current_classifications[0], + ) + elif record.resolution in { + "start-active-observed", + "stop-standby-observed", + }: + if record.last_status is None: + raise ValueError("target physical resolution lacks status evidence") + expected = ( + "start-active-observed" if record.action == "start" else "stop-standby-observed" + ) + if record.resolution != expected: + raise ValueError("target resolution does not match physical action") + _require_complete_target_gate(record, record.resolution, record.last_status) + elif record.resolution == "operator-retired-outcome-unknown": + matching_retirement = next( + ( + item + for item in reversed(record.operator_retirements) + if item.original_attempt.operation_id == record.operation_id + ), + None, + ) + if matching_retirement is None: + raise ValueError("retired physical command lacks operator-retirement audit") + attempt = matching_retirement.original_attempt + if ( + record.revision != attempt.revision + 1 + or record.parent_operation_id != attempt.parent_operation_id + or record.acquisition_id != attempt.acquisition_id + or record.action != attempt.action + or record.identity != attempt.identity + or record.connection != attempt.connection + or record.compatibility_profile_id != attempt.compatibility_profile_id + or record.payload_sha256 != attempt.payload_sha256 + or record.baseline_status != attempt.baseline_status + or record.publish_call_returned != attempt.publish_call_returned + or record.packet_id != attempt.packet_id + or record.qos2_completed != attempt.qos2_completed + or record.application_response != attempt.application_response + or record.last_status != attempt.last_status + or record.created_at_utc != attempt.created_at_utc + ): + raise ValueError("operator retirement altered original command evidence") + if attempt.resolution == "physical-active-observed": + if not record.reconciliations: + raise ValueError("operator retirement lacks its resolved-active reconciliation") + reconciliation = record.reconciliations[-1] + operation_reconciliation_count = sum( + item.original_attempt.operation_id == attempt.operation_id + for item in record.reconciliations + ) + if ( + reconciliation.original_attempt.operation_id != attempt.operation_id + or reconciliation.resolution != "physical-active-observed" + or attempt.revision + != (reconciliation.original_attempt.revision + operation_reconciliation_count) + or not _same_attempt_evidence_except_revision_and_update( + attempt, + reconciliation.original_attempt, + ) + ): + raise ValueError("operator retirement lost its resolved-active reconciliation") + elif attempt.resolution == "not-dispatched": + classification = _prepared_stop_classification(record) + allowed_classification_resolutions = { + "physical-active-observed", + } + if matching_retirement.reason == ( + "connection-scenario-reset-by-operator" + ): + allowed_classification_resolutions.add( + "physical-standby-observed" + ) + if ( + classification is None + or classification.resolution + not in allowed_classification_resolutions + ): + raise ValueError( + "operator retirement lacks eligible prepared STOP classification" + ) + _require_prepared_stop_classification_matches_attempt( + attempt, + classification, + ) + if record.original_command_outcome != "not-dispatched": + raise ValueError( + "operator retirement changed prepared STOP no-dispatch outcome" + ) + elif record.original_command_outcome != "unknown": + raise ValueError("operator retirement must preserve unknown command outcome") + else: + reconciliation = _latest_reconciliation_for_operation(record) + _require_reconciliation_matches_resolved_record(record, reconciliation) + + +def _canonical_json_bytes(payload: Mapping[str, object]) -> bytes: + return ( + json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n" + ).encode("utf-8") + + +def _write_private_content_addressed_archive( + archive_dir: Path, + *, + expected_sha256: str, + payload: bytes, + data_dir: Path, +) -> None: + """Create one immutable archive object without ever replacing an existing one.""" + + _validate_sha256(expected_sha256, field_name="archive segment sha256") + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise PhysicalCommandLedgerCorrupt("physical command archive payload hash changed") + if not payload or len(payload) > PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES: + raise PhysicalCommandLedgerCorrupt("physical command archive payload is outside its bound") + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(archive_dir.parent, parents=True) + archive_created = _ensure_private_directory(archive_dir, parents=False) + if archive_created: + _fsync_directory(archive_dir.parent) + destination = archive_dir / f"{expected_sha256}.json" + + _recover_content_addressed_archive_publication( + archive_dir, + destination=destination, + expected_sha256=expected_sha256, + payload=payload, + ) + + try: + existing = _read_private_file_bounded( + destination, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="archive segment", + ) + except FileNotFoundError: + existing = None + except (OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive content address is unsafe" + ) from exc + if existing is not None: + if existing != payload: + raise PhysicalCommandLedgerCorrupt( + "physical command archive content address contains different bytes" + ) + return + + descriptor, temp_name = tempfile.mkstemp( + dir=archive_dir, + prefix=f".{expected_sha256}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + temp_metadata = os.fstat(descriptor) + _require_private_regular_file(temp_metadata, label="archive temporary file", empty=False) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temp_path, destination, follow_symlinks=False) + except FileExistsError: + pass + else: + _fsync_directory(archive_dir) + failpoint = _PHYSICAL_COMMAND_ARCHIVE_AFTER_LINK_FSYNC_FAILPOINT + if failpoint is not None: + failpoint() + finally: + try: + temp_path.unlink() + except FileNotFoundError: + pass + else: + _fsync_directory(archive_dir) + published = _read_private_file_bounded( + destination, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="archive segment", + ) + if published != payload: + raise PhysicalCommandLedgerCorrupt( + "physical command archive content address contains different bytes" + ) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _recover_content_addressed_archive_publication( + archive_dir: Path, + *, + destination: Path, + expected_sha256: str, + payload: bytes, +) -> None: + """Finish only an exact private temp publication left by process death. + + Publication deliberately uses a hard link so an existing content address + can never be replaced. Death after linking and directory fsync but before + temp unlink leaves precisely two names for the same inode. We accept that + state only when one strictly named, owner-only temp alias has the exact + expected bytes and is the destination's sole second link. Any unrelated + hard link, symlink, metadata mismatch or extra candidate fails closed. + """ + + prefix = f".{expected_sha256}." + candidates: list[Path] = [] + scanned_entries = 0 + try: + with os.scandir(archive_dir) as entries: + for entry in entries: + scanned_entries += 1 + if scanned_entries > PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS + 2: + raise PhysicalCommandLedgerCorrupt( + "physical command archive directory exceeds its bounded entries" + ) + name = entry.name + if not name.startswith(prefix) or not name.endswith(".tmp"): + continue + random_part = name[len(prefix) : -len(".tmp")] + if not re.fullmatch(r"[A-Za-z0-9_-]{6,32}", random_part): + raise PhysicalCommandLedgerCorrupt( + "physical command archive temporary name is unsafe" + ) + candidates.append(archive_dir / name) + except OSError as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive temporary scan failed closed" + ) from exc + if not candidates: + return + if len(candidates) != 1: + raise PhysicalCommandLedgerCorrupt( + "physical command archive has ambiguous publication temporaries" + ) + + temporary = candidates[0] + try: + temporary_payload, temporary_metadata = _read_private_file_bounded_with_links( + temporary, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="archive publication temporary", + allowed_link_counts=frozenset({1, 2}), + ) + except (FileNotFoundError, OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive publication temporary is unsafe" + ) from exc + if temporary_payload != payload: + raise PhysicalCommandLedgerCorrupt( + "physical command archive publication temporary changed bytes" + ) + + try: + destination_payload, destination_metadata = _read_private_file_bounded_with_links( + destination, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="recovering archive segment", + allowed_link_counts=frozenset({1, 2}), + ) + except FileNotFoundError: + destination_payload = None + destination_metadata = None + except (OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive destination is unsafe during recovery" + ) from exc + + if destination_payload is None: + if temporary_metadata.st_nlink != 1: + raise PhysicalCommandLedgerCorrupt( + "physical command archive temporary has an unrelated hard link" + ) + try: + os.link(temporary, destination, follow_symlinks=False) + except OSError as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive temporary recovery failed" + ) from exc + _fsync_directory(archive_dir) + destination_metadata = destination.lstat() + if ( + destination_metadata.st_dev, + destination_metadata.st_ino, + ) != (temporary_metadata.st_dev, temporary_metadata.st_ino): + raise PhysicalCommandLedgerCorrupt( + "physical command archive recovered destination identity changed" + ) + else: + assert destination_metadata is not None + if destination_payload != payload: + raise PhysicalCommandLedgerCorrupt( + "physical command archive content address contains different bytes" + ) + same_inode = ( + destination_metadata.st_dev, + destination_metadata.st_ino, + ) == (temporary_metadata.st_dev, temporary_metadata.st_ino) + if same_inode: + if destination_metadata.st_nlink != 2 or temporary_metadata.st_nlink != 2: + raise PhysicalCommandLedgerCorrupt( + "physical command archive publication alias count is unsafe" + ) + elif destination_metadata.st_nlink != 1 or temporary_metadata.st_nlink != 1: + raise PhysicalCommandLedgerCorrupt( + "physical command archive publication has an unrelated hard link" + ) + + try: + temporary.unlink() + _fsync_directory(archive_dir) + except OSError as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive temporary cleanup failed" + ) from exc + recovered = _read_private_file_bounded( + destination, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="archive segment", + ) + if recovered != payload: + raise PhysicalCommandLedgerCorrupt("physical command archive recovered different bytes") + + +def _load_private_archive_history( + archive_dir: Path, + *, + record: PhysicalCommandRecord, +) -> _PhysicalCommandArchiveHistory: + head = record.archive_head_sha256 + if head is None: + return _PhysicalCommandArchiveHistory() + try: + archive_metadata = archive_dir.lstat() + _require_private_directory_metadata( + archive_metadata, + label="archive directory", + ) + except (FileNotFoundError, OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive directory is missing or unsafe" + ) from exc + + operation_ids: set[str] = set() + reconciliation_ids: set[str] = set() + verification_ids: set[str] = set() + verified_control_session_ids: set[str] = set() + confirmation_ids: set[str] = set() + retirement_ids: set[str] = set() + reopening_ids: set[str] = set() + reconciliation_audits: dict[str, str] = {} + verification_audits: dict[str, str] = {} + confirmation_audits: dict[str, str] = {} + retirement_audits: dict[str, str] = {} + reopening_audits: dict[str, str] = {} + lineage_reconciliations_by_id: dict[str, PhysicalCommandReconciliation] = {} + lineage_classification_ids_by_operation: dict[str, str] = {} + lineage_confirmations_by_stop_operation: dict[ + str, PhysicalCommandOperatorConfirmation + ] = {} + lineage_attempt_sha256_by_operation: dict[str, str] = {} + archived_current_retirement_ids: set[str] = set() + archived_current_reopens: dict[str, str] = {} + visited: set[str] = set() + expected_sha256: str | None = head + expected_sequence = record.archive_segment_count + remaining_bytes = record.archive_total_bytes + child_revision = record.revision + head_record: PhysicalCommandRecord | None = None + + while expected_sequence > 0: + if expected_sha256 is None: + raise PhysicalCommandLedgerCorrupt( + "physical command archive chain ended before its declared count" + ) + _validate_sha256(expected_sha256, field_name="archive segment sha256") + if expected_sha256 in visited: + raise PhysicalCommandLedgerCorrupt("physical command archive replays a segment") + visited.add(expected_sha256) + segment_path = archive_dir / f"{expected_sha256}.json" + try: + payload = _read_private_file_bounded( + segment_path, + limit=PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES, + label="archive segment", + ) + except (FileNotFoundError, OSError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive segment is missing or unsafe" + ) from exc + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise PhysicalCommandLedgerCorrupt( + "physical command archive segment hash does not match its address" + ) + try: + document = json.loads( + payload.decode("utf-8"), + object_pairs_hook=_unique_json_object, + ) + if not isinstance(document, dict): + raise ValueError( + "physical command archive does not match the canonical schema" + ) + # The archive address and its embedded-record digest bind the exact + # historical bytes. Validate both against the source document + # before ``_archive_segment_from_mapping`` upgrades a legacy record + # to the current in-memory schema. Re-serializing that migrated + # object would otherwise falsely condemn an intact v4 segment when + # the current ledger schema advances to v5. + if _canonical_json_bytes(document) != payload: + raise ValueError( + "physical command archive segment encoding is not canonical" + ) + source_record = document.get("record") + if not isinstance(source_record, dict): + raise ValueError( + "physical command archive record does not match the canonical schema" + ) + source_record_sha256 = _sha256_document(source_record) + segment = _archive_segment_from_mapping(document) + except (UnicodeError, json.JSONDecodeError, TypeError, ValueError) as exc: + raise PhysicalCommandLedgerCorrupt( + "physical command archive segment is not canonical" + ) from exc + if segment.sequence != expected_sequence: + raise PhysicalCommandLedgerCorrupt( + "physical command archive sequence does not match the main ledger" + ) + if head_record is None: + head_record = segment.record + if segment.record_sha256 != source_record_sha256: + raise PhysicalCommandLedgerCorrupt( + "physical command archive record hash does not match" + ) + if segment.predecessor_sha256 != segment.record.archive_head_sha256: + raise PhysicalCommandLedgerCorrupt( + "physical command archive predecessor does not match its record" + ) + if segment.record.archive_segment_count != expected_sequence - 1: + raise PhysicalCommandLedgerCorrupt( + "physical command archive predecessor count is not contiguous" + ) + if remaining_bytes != segment.record.archive_total_bytes + len(payload): + raise PhysicalCommandLedgerCorrupt( + "physical command archive byte accounting is not contiguous" + ) + if segment.record.revision >= child_revision: + raise PhysicalCommandLedgerCorrupt( + "physical command archive revision did not precede its child" + ) + segment_current_retirement_ids = { + item.retirement_id + for item in segment.record.operator_retirements + if item.original_attempt.operation_id == record.operation_id + } + archived_current_retirement_ids.update(segment_current_retirement_ids) + for reopening in segment.record.operator_reconciliation_reopens: + if reopening.retirement_id not in segment_current_retirement_ids: + continue + prior_retirement_id = archived_current_reopens.setdefault( + reopening.reopening_id, + reopening.retirement_id, + ) + if prior_retirement_id != reopening.retirement_id: + raise PhysicalCommandLedgerCorrupt( + "physical command archive reopening changed retirement identity" + ) + _register_archive_record_audits( + segment.record, + reconciliation_audits=reconciliation_audits, + verification_audits=verification_audits, + confirmation_audits=confirmation_audits, + retirement_audits=retirement_audits, + reopening_audits=reopening_audits, + ) + _add_record_identities_to_archive_history( + segment.record, + operation_ids=operation_ids, + reconciliation_ids=reconciliation_ids, + verification_ids=verification_ids, + verified_control_session_ids=verified_control_session_ids, + confirmation_ids=confirmation_ids, + retirement_ids=retirement_ids, + reopening_ids=reopening_ids, + ) + _register_lineage_proof_indexes( + segment.record, + reconciliations_by_id=lineage_reconciliations_by_id, + classification_ids_by_operation=lineage_classification_ids_by_operation, + confirmations_by_stop_operation=lineage_confirmations_by_stop_operation, + attempt_sha256_by_operation=lineage_attempt_sha256_by_operation, + ) + remaining_bytes = segment.record.archive_total_bytes + child_revision = segment.record.revision + expected_sha256 = segment.predecessor_sha256 + expected_sequence -= 1 + + if expected_sha256 is not None or remaining_bytes != 0: + raise PhysicalCommandLedgerCorrupt( + "physical command archive chain exceeds its declared boundary" + ) + _require_prepared_stop_archive_gap_matches_audits( + record, + archived_retirement_ids=archived_current_retirement_ids, + archived_reopens=archived_current_reopens, + ) + _register_archive_record_audits( + record, + reconciliation_audits=reconciliation_audits, + verification_audits=verification_audits, + confirmation_audits=confirmation_audits, + retirement_audits=retirement_audits, + reopening_audits=reopening_audits, + ) + return _PhysicalCommandArchiveHistory( + head_sha256=head, + segment_count=record.archive_segment_count, + total_bytes=record.archive_total_bytes, + head_record=head_record, + operation_ids=frozenset(operation_ids), + reconciliation_ids=frozenset(reconciliation_ids), + verification_ids=frozenset(verification_ids), + verified_control_session_ids=frozenset(verified_control_session_ids), + confirmation_ids=frozenset(confirmation_ids), + retirement_ids=frozenset(retirement_ids), + reopening_ids=frozenset(reopening_ids), + lineage_reconciliations_by_id=tuple( + sorted(lineage_reconciliations_by_id.items()) + ), + lineage_classification_ids_by_operation=tuple( + sorted(lineage_classification_ids_by_operation.items()) + ), + lineage_confirmations_by_stop_operation=tuple( + sorted(lineage_confirmations_by_stop_operation.items()) + ), + lineage_attempt_sha256_by_operation=tuple( + sorted(lineage_attempt_sha256_by_operation.items()) + ), + ) + + +def _register_archive_record_audits( + record: PhysicalCommandRecord, + *, + reconciliation_audits: dict[str, str], + verification_audits: dict[str, str], + confirmation_audits: dict[str, str], + retirement_audits: dict[str, str], + reopening_audits: dict[str, str], +) -> None: + for reconciliation in record.reconciliations: + _register_immutable_archive_identity( + reconciliation_audits, + reconciliation.reconciliation_id, + _sha256_document(reconciliation.as_dict()), + label="reconciliation", + ) + _register_immutable_archive_identity( + verification_audits, + reconciliation.verified_binding.verification_id, + _sha256_document(reconciliation.verified_binding.as_dict()), + label="verification", + ) + for confirmation in record.operator_confirmations: + _register_immutable_archive_identity( + confirmation_audits, + confirmation.confirmation_id, + _sha256_document(confirmation.as_dict()), + label="confirmation", + ) + for retirement in record.operator_retirements: + _register_immutable_archive_identity( + retirement_audits, + retirement.retirement_id, + _sha256_document(retirement.as_dict()), + label="retirement", + ) + for reopening in record.operator_reconciliation_reopens: + _register_immutable_archive_identity( + reopening_audits, + reopening.reopening_id, + _sha256_document(reopening.as_dict()), + label="reopening", + ) + + +def _register_immutable_archive_identity( + index: dict[str, str], + identity: str, + audit_sha256: str, + *, + label: str, +) -> None: + prior = index.setdefault(identity, audit_sha256) + if prior != audit_sha256: + raise PhysicalCommandLedgerCorrupt( + f"physical command archive {label} identity changed between segments" + ) + + +def _archive_segment_from_mapping(value: object) -> PhysicalCommandArchiveSegment: + document = _exact_mapping( + value, + { + "schema_version", + "sequence", + "predecessor_sha256", + "record_sha256", + "record", + }, + label="archive segment", + ) + if document["schema_version"] != PHYSICAL_COMMAND_ARCHIVE_SCHEMA: + raise ValueError("unsupported physical command archive schema") + predecessor = _optional_string( + document["predecessor_sha256"], + field_name="predecessor_sha256", + ) + if predecessor is not None: + _validate_sha256(predecessor, field_name="predecessor_sha256") + record_sha256 = _required_string( + document["record_sha256"], + field_name="record_sha256", + ) + _validate_sha256(record_sha256, field_name="record_sha256") + return PhysicalCommandArchiveSegment( + sequence=_positive_int(document["sequence"], field_name="sequence"), + predecessor_sha256=predecessor, + record_sha256=record_sha256, + record=_record_from_mapping(document["record"]), + ) + + +def _read_private_file_bounded(path: Path, *, limit: int, label: str) -> bytes: + payload, _metadata = _read_private_file_bounded_with_links( + path, + limit=limit, + label=label, + allowed_link_counts=frozenset({1}), + ) + return payload + + +def _read_private_file_bounded_with_links( + path: Path, + *, + limit: int, + label: str, + allowed_link_counts: frozenset[int], +) -> tuple[bytes, os.stat_result]: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + before = os.fstat(descriptor) + _require_private_regular_file_with_links( + before, + label=label, + empty=False, + allowed_link_counts=allowed_link_counts, + ) + if before.st_size <= 0 or before.st_size > limit: + raise ValueError(f"physical command {label} exceeds its bounded size") + payload = _read_bounded(descriptor, limit) + after = os.fstat(descriptor) + path_metadata = path.lstat() + _require_private_regular_file_with_links( + after, + label=label, + empty=False, + allowed_link_counts=allowed_link_counts, + ) + if (after.st_dev, after.st_ino) != ( + path_metadata.st_dev, + path_metadata.st_ino, + ): + raise ValueError(f"physical command {label} identity changed while reading") + if ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_uid, + before.st_nlink, + before.st_size, + before.st_mtime_ns, + ) != ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_uid, + after.st_nlink, + after.st_size, + after.st_mtime_ns, + ): + raise ValueError(f"physical command {label} metadata changed while reading") + return payload, after + finally: + os.close(descriptor) + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = _canonical_json_bytes(payload) + if len(serialized) > PHYSICAL_COMMAND_LEDGER_MAX_BYTES: + raise ValueError("physical command ledger exceeds the bounded size") + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(path.parent, parents=True) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + temp_metadata = os.fstat(descriptor) + _require_private_regular_file(temp_metadata, label="temporary ledger", empty=False) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + published_identity = os.fstat(stream.fileno()) + os.replace(temp_path, path) + destination = path.lstat() + _require_private_regular_file(destination, label="published ledger", empty=False) + if (destination.st_dev, destination.st_ino) != ( + published_identity.st_dev, + published_identity.st_ino, + ): + raise OSError("physical command ledger atomic identity mismatch") + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + metadata = path.lstat() + _require_private_directory_metadata(metadata, label="ledger directory") + return True + _require_private_directory_metadata(metadata, label="ledger directory") + return False + + +def _require_private_directory_metadata(metadata: os.stat_result, *, label: str) -> None: + if ( + not stat.S_ISDIR(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o700 + or metadata.st_uid != os.getuid() + ): + raise ValueError(f"physical command {label} is not private and owner-controlled") + + +def _require_private_regular_file( + metadata: os.stat_result, + *, + label: str, + empty: bool, +) -> None: + _require_private_regular_file_with_links( + metadata, + label=label, + empty=empty, + allowed_link_counts=frozenset({1}), + ) + + +def _require_private_regular_file_with_links( + metadata: os.stat_result, + *, + label: str, + empty: bool, + allowed_link_counts: frozenset[int], +) -> None: + if ( + not stat.S_ISREG(metadata.st_mode) + or stat.S_IMODE(metadata.st_mode) != 0o600 + or metadata.st_uid != os.getuid() + or metadata.st_nlink not in allowed_link_counts + or (empty and metadata.st_size != 0) + ): + raise ValueError(f"physical command {label} is not a private regular file") + + +def _read_bounded(descriptor: int, limit: int) -> bytes: + chunks: list[bytes] = [] + remaining = limit + 1 + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 16 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + if len(payload) > limit: + raise ValueError("physical command ledger exceeds the bounded size") + return payload + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("physical command ledger contains duplicate fields") + document[key] = value + return document + + +def _sha256_document(value: Mapping[str, object]) -> str: + payload = json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _record_from_mapping(value: object) -> PhysicalCommandRecord: + if not isinstance(value, dict): + raise ValueError("physical command ledger does not match the canonical schema") + schema_version = value.get("schema_version") + if schema_version not in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V2_SCHEMA, + PHYSICAL_COMMAND_LEDGER_LEGACY_SCHEMA, + }: + raise ValueError("unsupported physical command ledger schema") + keys = { + "schema_version", + "revision", + "operation_id", + "parent_operation_id", + "acquisition_id", + "action", + "stage", + "identity", + "connection", + "compatibility_profile_id", + "payload_sha256", + "baseline_status", + "publish_call_returned", + "packet_id", + "qos2_completed", + "application_response", + "last_status", + "reconciliations", + "operator_confirmations", + "resolution", + "created_at_utc", + "updated_at_utc", + } + if schema_version in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V2_SCHEMA, + }: + keys.add("operator_retirements") + if schema_version in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + }: + keys.add("operator_reconciliation_reopens") + if schema_version in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + }: + keys.update( + { + "archive_head_sha256", + "archive_segment_count", + "archive_total_bytes", + } + ) + document = _exact_mapping( + value, + keys, + label="ledger", + ) + action_raw = _required_string(document["action"], field_name="action") + _validate_action(action_raw) + stage_raw = _required_string(document["stage"], field_name="stage") + if stage_raw not in _STAGES: + raise ValueError("unsupported physical command stage") + publish_raw = document["publish_call_returned"] + if publish_raw is not None and not isinstance(publish_raw, bool): + raise ValueError("publish_call_returned must be bool or null") + packet_raw = document["packet_id"] + packet_id = None if packet_raw is None else _positive_int(packet_raw, field_name="packet_id") + qos2_completed = _required_bool(document["qos2_completed"], field_name="qos2_completed") + response_raw = document["application_response"] + response = None if response_raw is None else _response_from_mapping(response_raw) + status_raw = document["last_status"] + last_status = None if status_raw is None else _status_from_mapping(status_raw) + reconciliations = _reconciliations_from_value(document["reconciliations"]) + if schema_version != PHYSICAL_COMMAND_LEDGER_SCHEMA and any( + item.kind == "prepared-stop-classification" for item in reconciliations + ): + raise ValueError("prepared STOP classification requires ledger schema v5") + operator_confirmations = _operator_confirmations_from_value(document["operator_confirmations"]) + operator_retirements = ( + _operator_retirements_from_value(document["operator_retirements"]) + if schema_version + in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V2_SCHEMA, + } + else () + ) + operator_reconciliation_reopens = ( + _operator_reconciliation_reopens_from_value(document["operator_reconciliation_reopens"]) + if schema_version + in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + } + else () + ) + resolution_raw = document["resolution"] + if resolution_raw is None: + resolution = None + else: + resolution_string = _required_string(resolution_raw, field_name="resolution") + _validate_resolution(resolution_string) + resolution = cast(PhysicalCommandResolution, resolution_string) + record = PhysicalCommandRecord( + schema_version=PHYSICAL_COMMAND_LEDGER_SCHEMA, + revision=_positive_int(document["revision"], field_name="revision"), + operation_id=_required_string(document["operation_id"], field_name="operation_id"), + parent_operation_id=_optional_string( + document["parent_operation_id"], + field_name="parent_operation_id", + ), + acquisition_id=_required_string( + document["acquisition_id"], + field_name="acquisition_id", + ), + action=cast(PhysicalCommandAction, action_raw), + stage=cast(PhysicalCommandStage, stage_raw), + identity=_identity_from_mapping(document["identity"]), + connection=_connection_from_mapping(document["connection"]), + compatibility_profile_id=_required_string( + document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ), + payload_sha256=_required_string( + document["payload_sha256"], + field_name="payload_sha256", + ), + baseline_status=_status_from_mapping(document["baseline_status"]), + publish_call_returned=publish_raw, + packet_id=packet_id, + qos2_completed=qos2_completed, + application_response=response, + last_status=last_status, + reconciliations=reconciliations, + operator_confirmations=operator_confirmations, + operator_retirements=operator_retirements, + resolution=resolution, + created_at_utc=_required_string( + document["created_at_utc"], + field_name="created_at_utc", + ), + updated_at_utc=_required_string( + document["updated_at_utc"], + field_name="updated_at_utc", + ), + operator_reconciliation_reopens=operator_reconciliation_reopens, + archive_head_sha256=( + _optional_string( + document["archive_head_sha256"], + field_name="archive_head_sha256", + ) + if schema_version + in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + } + else None + ), + archive_segment_count=( + _nonnegative_int( + document["archive_segment_count"], + field_name="archive_segment_count", + ) + if schema_version + in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + } + else 0 + ), + archive_total_bytes=( + _nonnegative_int( + document["archive_total_bytes"], + field_name="archive_total_bytes", + ) + if schema_version + in { + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + } + else 0 + ), + ) + _validate_record_semantics(record) + return record + + +def _identity_from_mapping(value: object) -> PhysicalCommandIdentity: + document = _exact_mapping( + value, + {"vendor_device_id_sha256", "device_serial_sha256"}, + label="identity", + ) + return PhysicalCommandIdentity( + vendor_device_id_sha256=_required_string( + document["vendor_device_id_sha256"], + field_name="vendor_device_id_sha256", + ), + device_serial_sha256=_required_string( + document["device_serial_sha256"], + field_name="device_serial_sha256", + ), + ) + + +def _connection_from_mapping(value: object) -> PhysicalCommandConnectionBinding: + document = _exact_mapping( + value, + { + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + }, + label="connection", + ) + mode_raw = _required_string(document["connection_mode"], field_name="connection_mode") + _validate_connection_mode(mode_raw) + return PhysicalCommandConnectionBinding( + intent_id=_required_string(document["intent_id"], field_name="intent_id"), + transport_ref=_required_string( + document["transport_ref"], + field_name="transport_ref", + ), + connection_mode=cast(PhysicalCommandConnectionMode, mode_raw), + target_ipv4=_required_string( + document["target_ipv4"], + field_name="target_ipv4", + ), + target_port=_positive_int(document["target_port"], field_name="target_port"), + host_path_epoch=_positive_int( + document["host_path_epoch"], + field_name="host_path_epoch", + ), + control_session_id=_required_string( + document["control_session_id"], + field_name="control_session_id", + ), + producer_generation=_positive_int( + document["producer_generation"], + field_name="producer_generation", + ), + ) + + +def _status_from_mapping(value: object) -> PhysicalCommandStatusEvidence: + document = _exact_mapping( + value, + { + "source", + "vendor_device_id_sha256", + "device_serial_sha256", + "control_session_id", + "host_path_epoch", + "producer_generation", + "session_state", + "session_state_code", + "project_bound", + "project_id_sha256", + "init_ready", + "status_message_sha256", + "mqtt_retained", + "observed_at_utc", + }, + label="status evidence", + ) + source_raw = _required_string(document["source"], field_name="source") + if source_raw not in _STATUS_SOURCES: + raise ValueError("unsupported physical status source") + state_raw = _required_string(document["session_state"], field_name="session_state") + if state_raw not in _SESSION_STATE_CODES: + raise ValueError("unsupported physical session state") + return PhysicalCommandStatusEvidence( + source=cast(PhysicalStatusSource, source_raw), + vendor_device_id_sha256=_required_string( + document["vendor_device_id_sha256"], + field_name="vendor_device_id_sha256", + ), + device_serial_sha256=_required_string( + document["device_serial_sha256"], + field_name="device_serial_sha256", + ), + control_session_id=_required_string( + document["control_session_id"], + field_name="control_session_id", + ), + host_path_epoch=_positive_int( + document["host_path_epoch"], + field_name="host_path_epoch", + ), + producer_generation=_positive_int( + document["producer_generation"], + field_name="producer_generation", + ), + session_state=cast(PhysicalSessionState, state_raw), + session_state_code=_nonnegative_int( + document["session_state_code"], + field_name="session_state_code", + ), + project_bound=_required_bool( + document["project_bound"], + field_name="project_bound", + ), + project_id_sha256=_optional_string( + document["project_id_sha256"], + field_name="project_id_sha256", + ), + init_ready=_required_bool(document["init_ready"], field_name="init_ready"), + status_message_sha256=_required_string( + document["status_message_sha256"], + field_name="status_message_sha256", + ), + mqtt_retained=_required_bool( + document["mqtt_retained"], + field_name="mqtt_retained", + ), + observed_at_utc=_required_string( + document["observed_at_utc"], + field_name="observed_at_utc", + ), + ) + + +def _response_from_mapping(value: object) -> PhysicalCommandApplicationResponse: + document = _exact_mapping( + value, + { + "operation_id", + "action", + "control_session_id", + "host_path_epoch", + "producer_generation", + "result_code", + "success", + "payload_sha256", + "observed_at_utc", + }, + label="application response", + ) + action_raw = _required_string(document["action"], field_name="action") + _validate_action(action_raw) + return PhysicalCommandApplicationResponse( + operation_id=_required_string( + document["operation_id"], + field_name="operation_id", + ), + action=cast(PhysicalCommandAction, action_raw), + control_session_id=_required_string( + document["control_session_id"], + field_name="control_session_id", + ), + host_path_epoch=_positive_int( + document["host_path_epoch"], + field_name="host_path_epoch", + ), + producer_generation=_positive_int( + document["producer_generation"], + field_name="producer_generation", + ), + result_code=_nonnegative_int(document["result_code"], field_name="result_code"), + success=_required_bool(document["success"], field_name="success"), + payload_sha256=_required_string( + document["payload_sha256"], + field_name="payload_sha256", + ), + observed_at_utc=_required_string( + document["observed_at_utc"], + field_name="observed_at_utc", + ), + ) + + +def _attempt_from_mapping(value: object) -> PhysicalCommandAttemptAudit: + document = _exact_mapping( + value, + { + "revision", + "operation_id", + "parent_operation_id", + "acquisition_id", + "action", + "stage", + "resolution", + "identity", + "connection", + "compatibility_profile_id", + "payload_sha256", + "baseline_status", + "publish_call_returned", + "packet_id", + "qos2_completed", + "application_response", + "last_status", + "created_at_utc", + "updated_at_utc", + }, + label="attempt audit", + ) + action_raw = _required_string(document["action"], field_name="action") + _validate_action(action_raw) + stage_raw = _required_string(document["stage"], field_name="stage") + if stage_raw not in {"prepared", "dispatching", "observing", "resolved"}: + raise ValueError("attempt audit has unsupported stage") + resolution_raw = document["resolution"] + if resolution_raw is None: + resolution = None + else: + resolution_string = _required_string( + resolution_raw, + field_name="attempt resolution", + ) + if resolution_string not in { + *_ORDINARY_RESOLUTIONS, + "physical-active-observed", + }: + raise ValueError("attempt audit has unsupported resolution") + resolution = cast(PhysicalCommandResolution, resolution_string) + publish_raw = document["publish_call_returned"] + if publish_raw is not None and not isinstance(publish_raw, bool): + raise ValueError("attempt publish_call_returned must be bool or null") + packet_raw = document["packet_id"] + response_raw = document["application_response"] + status_raw = document["last_status"] + return PhysicalCommandAttemptAudit( + revision=_positive_int(document["revision"], field_name="revision"), + operation_id=_required_string( + document["operation_id"], + field_name="operation_id", + ), + parent_operation_id=_optional_string( + document["parent_operation_id"], + field_name="parent_operation_id", + ), + acquisition_id=_required_string( + document["acquisition_id"], + field_name="acquisition_id", + ), + action=cast(PhysicalCommandAction, action_raw), + stage=cast(PhysicalCommandStage, stage_raw), + resolution=resolution, + identity=_identity_from_mapping(document["identity"]), + connection=_connection_from_mapping(document["connection"]), + compatibility_profile_id=_required_string( + document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ), + payload_sha256=_required_string( + document["payload_sha256"], + field_name="payload_sha256", + ), + baseline_status=_status_from_mapping(document["baseline_status"]), + publish_call_returned=publish_raw, + packet_id=( + None if packet_raw is None else _positive_int(packet_raw, field_name="packet_id") + ), + qos2_completed=_required_bool( + document["qos2_completed"], + field_name="qos2_completed", + ), + application_response=( + None if response_raw is None else _response_from_mapping(response_raw) + ), + last_status=(None if status_raw is None else _status_from_mapping(status_raw)), + created_at_utc=_required_string( + document["created_at_utc"], + field_name="created_at_utc", + ), + updated_at_utc=_required_string( + document["updated_at_utc"], + field_name="updated_at_utc", + ), + ) + + +def _verified_binding_from_mapping(value: object) -> PhysicalCommandVerifiedBinding: + document = _exact_mapping( + value, + { + "verification_id", + "identity", + "connection", + "device_info_message_sha256", + "verified_at_utc", + }, + label="verified binding", + ) + return PhysicalCommandVerifiedBinding( + verification_id=_required_string( + document["verification_id"], + field_name="verification_id", + ), + identity=_identity_from_mapping(document["identity"]), + connection=_connection_from_mapping(document["connection"]), + device_info_message_sha256=_required_string( + document["device_info_message_sha256"], + field_name="device_info_message_sha256", + ), + verified_at_utc=_required_string( + document["verified_at_utc"], + field_name="verified_at_utc", + ), + ) + + +def _reconciliation_from_mapping(value: object) -> PhysicalCommandReconciliation: + document = _exact_mapping( + value, + { + "reconciliation_id", + "kind", + "resolution", + "original_attempt", + "original_attempt_sha256", + "verified_binding", + "observation", + "reconciled_at_utc", + }, + label="reconciliation", + ) + resolution_raw = _required_string(document["resolution"], field_name="resolution") + if resolution_raw not in _RECONCILIATION_RESOLUTIONS: + raise ValueError("unsupported physical reconciliation resolution") + kind_raw = _required_string(document["kind"], field_name="kind") + if kind_raw not in _RECONCILIATION_KINDS: + raise ValueError("unsupported physical reconciliation kind") + return PhysicalCommandReconciliation( + reconciliation_id=_required_string( + document["reconciliation_id"], + field_name="reconciliation_id", + ), + kind=cast(PhysicalCommandReconciliationKind, kind_raw), + resolution=cast(PhysicalCommandReconciliationResolution, resolution_raw), + original_attempt=_attempt_from_mapping(document["original_attempt"]), + original_attempt_sha256=_required_string( + document["original_attempt_sha256"], + field_name="original_attempt_sha256", + ), + verified_binding=_verified_binding_from_mapping(document["verified_binding"]), + observation=_status_from_mapping(document["observation"]), + reconciled_at_utc=_required_string( + document["reconciled_at_utc"], + field_name="reconciled_at_utc", + ), + ) + + +def _operator_confirmation_from_mapping( + value: object, +) -> PhysicalCommandOperatorConfirmation: + document = _exact_mapping( + value, + { + "confirmation_id", + "reconciliation_id", + "stop_operation_id", + "confirmed_at_utc", + }, + label="operator confirmation", + ) + return PhysicalCommandOperatorConfirmation( + confirmation_id=_required_string( + document["confirmation_id"], + field_name="confirmation_id", + ), + reconciliation_id=_required_string( + document["reconciliation_id"], + field_name="reconciliation_id", + ), + stop_operation_id=_required_string( + document["stop_operation_id"], + field_name="stop_operation_id", + ), + confirmed_at_utc=_required_string( + document["confirmed_at_utc"], + field_name="confirmed_at_utc", + ), + ) + + +def _operator_retirement_from_mapping( + value: object, +) -> PhysicalCommandOperatorRetirement: + document = _exact_mapping( + value, + { + "retirement_id", + "reason", + "original_attempt", + "original_attempt_sha256", + "retired_transport_ref", + "retired_at_utc", + }, + label="operator retirement", + ) + reason = _required_string(document["reason"], field_name="reason") + if reason not in _PHYSICAL_COMMAND_OPERATOR_RETIREMENT_REASONS: + raise ValueError("unsupported physical operator-retirement reason") + return PhysicalCommandOperatorRetirement( + retirement_id=_required_string( + document["retirement_id"], + field_name="retirement_id", + ), + reason=cast(PhysicalCommandOperatorRetirementReason, reason), + original_attempt=_attempt_from_mapping(document["original_attempt"]), + original_attempt_sha256=_required_string( + document["original_attempt_sha256"], + field_name="original_attempt_sha256", + ), + retired_transport_ref=_required_string( + document["retired_transport_ref"], + field_name="retired_transport_ref", + ), + retired_at_utc=_required_string( + document["retired_at_utc"], + field_name="retired_at_utc", + ), + ) + + +def _operator_reconciliation_reopen_from_mapping( + value: object, +) -> PhysicalCommandOperatorReconciliationReopen: + document = _exact_mapping( + value, + { + "reopening_id", + "retirement_id", + "retirement_sha256", + "reason", + "retired_record_revision", + "reopened_transport_ref", + "discovery_generation", + "reopened_at_utc", + }, + label="operator reconciliation reopen", + ) + reason = _required_string(document["reason"], field_name="reason") + if reason not in _PHYSICAL_COMMAND_OPERATOR_RECONCILIATION_REOPEN_REASONS: + raise ValueError("unsupported physical operator-reconciliation reopen reason") + return PhysicalCommandOperatorReconciliationReopen( + reopening_id=_required_string( + document["reopening_id"], + field_name="reopening_id", + ), + retirement_id=_required_string( + document["retirement_id"], + field_name="retirement_id", + ), + retirement_sha256=_required_string( + document["retirement_sha256"], + field_name="retirement_sha256", + ), + reason=cast(PhysicalCommandOperatorReconciliationReopenReason, reason), + retired_record_revision=_positive_int( + document["retired_record_revision"], + field_name="retired_record_revision", + ), + reopened_transport_ref=_required_string( + document["reopened_transport_ref"], + field_name="reopened_transport_ref", + ), + discovery_generation=_positive_int( + document["discovery_generation"], + field_name="discovery_generation", + ), + reopened_at_utc=_required_string( + document["reopened_at_utc"], + field_name="reopened_at_utc", + ), + ) + + +def _reconciliations_from_value( + value: object, +) -> tuple[PhysicalCommandReconciliation, ...]: + if not isinstance(value, list) or len(value) > _MAX_RECONCILIATIONS: + raise ValueError("reconciliations must be a bounded array") + return tuple(_reconciliation_from_mapping(item) for item in value) + + +def _operator_confirmations_from_value( + value: object, +) -> tuple[PhysicalCommandOperatorConfirmation, ...]: + if not isinstance(value, list) or len(value) > _MAX_OPERATOR_CONFIRMATIONS: + raise ValueError("operator_confirmations must be a bounded array") + return tuple(_operator_confirmation_from_mapping(item) for item in value) + + +def _operator_retirements_from_value( + value: object, +) -> tuple[PhysicalCommandOperatorRetirement, ...]: + if not isinstance(value, list) or len(value) > _MAX_OPERATOR_RETIREMENTS: + raise ValueError("operator_retirements must be a bounded array") + return tuple(_operator_retirement_from_mapping(item) for item in value) + + +def _operator_reconciliation_reopens_from_value( + value: object, +) -> tuple[PhysicalCommandOperatorReconciliationReopen, ...]: + if not isinstance(value, list) or len(value) > _MAX_OPERATOR_RECONCILIATION_REOPENS: + raise ValueError("operator_reconciliation_reopens must be a bounded array") + return tuple(_operator_reconciliation_reopen_from_mapping(item) for item in value) + + +def _exact_mapping(value: object, keys: set[str], *, label: str) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"physical command {label} does not match the canonical schema") + return cast(Mapping[str, object], value) + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _validate_sha256(value: str, *, field_name: str) -> None: + if _SHA256.fullmatch(value) is None: + raise ValueError(f"{field_name} must be a lowercase SHA-256 digest") + + +def _validate_action(value: str) -> None: + if value not in _ACTIONS: + raise ValueError("unsupported physical command action") + + +def _validate_connection_mode(value: str) -> None: + if value not in _CONNECTION_MODES: + raise ValueError("unsupported physical command connection mode") + + +def _validate_resolution(value: str) -> None: + if value not in _RESOLUTIONS: + raise ValueError("unsupported physical command resolution") + + +def _validate_ipv4(value: str, *, field_name: str) -> None: + try: + parsed = ipaddress.ip_address(value) + except ValueError as exc: + raise ValueError(f"{field_name} must be an IPv4 address") from exc + if not isinstance(parsed, ipaddress.IPv4Address) or str(parsed) != value: + raise ValueError(f"{field_name} must be canonical IPv4") + + +def _validate_port(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= 65535: + raise ValueError(f"{field_name} must be within 1..65535") + + +def _validate_positive_int(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{field_name} must be a positive integer") + + +def _validate_nonnegative_int(value: int, *, field_name: str) -> None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a nonnegative integer") + + +def _positive_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{field_name} must be a positive integer") + return value + + +def _nonnegative_int(value: object, *, field_name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{field_name} must be a nonnegative integer") + return value + + +def _required_bool(value: object, *, field_name: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{field_name} must be bool") + return value + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _optional_string(value: object, *, field_name: str) -> str | None: + if value is None: + return None + return _required_string(value, field_name=field_name) + + +def _validated_timestamp(value: object, *, field_name: str) -> datetime: + raw = _required_string(value, field_name=field_name) + if not raw.endswith("Z"): + raise ValueError(f"{field_name} must be UTC") + try: + parsed = datetime.fromisoformat(raw.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise ValueError(f"{field_name} is invalid") from exc + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): + raise ValueError(f"{field_name} must be UTC") + parsed = parsed.astimezone(UTC) + if raw != _timestamp(parsed): + raise ValueError(f"{field_name} must use canonical millisecond UTC format") + return parsed + + +def _as_datetime(value: str) -> datetime: + return _validated_timestamp(value, field_name="observed_at_utc") + + +def _timestamp(value: datetime) -> str: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("physical command ledger clock must be timezone-aware") + return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _validate_retirement_request( + *, + retirement_id: str, + expected_operation_id: str, + expected_revision: int, + expected_transport_ref: str, + reason: PhysicalCommandOperatorRetirementReason, +) -> None: + _validate_identifier(retirement_id, field_name="retirement_id") + _validate_identifier(expected_operation_id, field_name="expected_operation_id") + _validate_positive_int(expected_revision, field_name="expected_revision") + _validate_identifier(expected_transport_ref, field_name="expected_transport_ref") + if reason not in _PHYSICAL_COMMAND_OPERATOR_RETIREMENT_REASONS: + raise ValueError("unsupported physical operator-retirement reason") + + +def _max_audit_floor(primary: str, secondary: str | None) -> str: + if secondary is None: + return primary + primary_value = _validated_timestamp(primary, field_name="primary audit floor") + secondary_value = _validated_timestamp(secondary, field_name="secondary audit floor") + return _timestamp(max(primary_value, secondary_value)) + + +def _nondecreasing_audit_timestamp(value: datetime, *, floor: str | None) -> str: + """Keep serialized audit metadata stable when the host wall clock rolls back. + + Durable revision/stage order, exact operation lineage, connection binding, + host-path epoch and producer generation authorize transitions. UTC is kept + for operators and forensics only, so a backward NTP/RTC adjustment clamps + the audit field instead of invalidating an otherwise valid state change. + """ + + candidate = _validated_timestamp(_timestamp(value), field_name="ledger clock") + if floor is None: + return _timestamp(candidate) + floor_value = _validated_timestamp(floor, field_name="audit timestamp floor") + return _timestamp(max(candidate, floor_value)) diff --git a/src/k1link/device_plugins/xgrids_k1/protocol/application_acceptance.py b/src/k1link/device_plugins/xgrids_k1/protocol/application_acceptance.py index b1064ce..9a470f4 100644 --- a/src/k1link/device_plugins/xgrids_k1/protocol/application_acceptance.py +++ b/src/k1link/device_plugins/xgrids_k1/protocol/application_acceptance.py @@ -21,6 +21,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( ) from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( MODELING_RESPONSE_TOPIC, + ApplicationMqttTransportError, ) from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( OneShotPublishEnvelope, @@ -65,6 +66,8 @@ class ApplicationBatchExchange(Protocol): envelopes: Sequence[OneShotPublishEnvelope], *, required_response_operation_keys: Collection[str], + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, ) -> dict[str, bytes]: ... def maintain_open_for( @@ -129,8 +132,31 @@ class OperatorDialogueCheckpoint: ) +class PhysicalAcceptancePermitReservation: + """One commit right that must still be fresh at dispatch admission.""" + + def __init__(self, permit: PhysicalAcceptancePermit, token: object) -> None: + self._permit = permit + self._token = token + self._finished = False + + def commit(self) -> None: + if self._finished: + raise ApplicationAcceptanceError( + "physical acceptance reservation was already finished" + ) + self._permit._commit_reservation(self._token) # noqa: SLF001 + self._finished = True + + def release(self) -> None: + if self._finished: + return + self._permit._release_reservation(self._token) # noqa: SLF001 + self._finished = True + + class PhysicalAcceptancePermit: - """Short, single-action capability that is consumed before MQTT publish.""" + """Short, single-action capability committed at physical dispatch admission.""" def __init__( self, @@ -152,20 +178,54 @@ class PhysicalAcceptancePermit: self._monotonic = monotonic self._expires_at = monotonic() + float(ttl_seconds) self._consumed = False + self._reservation: object | None = None @property def action(self) -> ModelingAction: return self._checklist.action def consume(self, action: ModelingAction) -> None: + reservation = self.reserve(action) + reservation.commit() + + def reserve( + self, + action: ModelingAction, + ) -> PhysicalAcceptancePermitReservation: with self._lock: if self._consumed: raise ApplicationAcceptanceError("physical acceptance permit was already consumed") + if self._reservation is not None: + raise ApplicationAcceptanceError("physical acceptance permit is already reserved") if self._monotonic() >= self._expires_at: raise ApplicationAcceptanceError("physical acceptance permit expired") if action is not self._checklist.action: raise ApplicationAcceptanceError("physical acceptance permit action mismatch") + token = object() + self._reservation = token + return PhysicalAcceptancePermitReservation(self, token) + + def _commit_reservation(self, token: object) -> None: + with self._lock: + if self._consumed or self._reservation is not token: + raise ApplicationAcceptanceError( + "physical acceptance reservation is no longer current" + ) + if self._monotonic() >= self._expires_at: + self._reservation = None + raise ApplicationMqttTransportError( + "physical acceptance permit expired before dispatch admission", + reason_code=( + "physical-acceptance-permit-expired-before-dispatch" + ), + ) self._consumed = True + self._reservation = None + + def _release_reservation(self, token: object) -> None: + with self._lock: + if self._reservation is token and not self._consumed: + self._reservation = None def snapshot(self) -> dict[str, object]: with self._lock: @@ -218,6 +278,7 @@ class PhysicalAcceptanceDialogueExecutor: self._command_complete = False self._dialogue_stage = "new" self._start_complete = False + self._active_session_adopted = False self._stop_attempted = False self._stop_complete = False self._active_authority: ApplicationControlAuthority | None = None @@ -246,13 +307,48 @@ class PhysicalAcceptanceDialogueExecutor: ) -> LiveDeviceControlBinding: """Emit retained ordinals 1-6 at control-session establishment.""" + binding = self.run_read_only_inspection_stage(orchestrator) + return self.complete_connection_stage(orchestrator, expected_binding=binding) + + def run_read_only_inspection_stage( + self, + orchestrator: ShadowApplicationBootstrapOrchestrator, + ) -> LiveDeviceControlBinding: + """Emit only ordinal-1 DeviceInfo for an explicit passive Verify. + + ModelingStatus and the time-setting DeviceConfig request belong to the + canonical preparation dialogue. They are deliberately excluded from + this stage so a recovery Verify cannot cross a device-mutation edge. + A later explicit workspace action may promote this same socket through + :meth:`complete_connection_stage`. + """ + if self._dialogue_stage != "new" or self._bootstrap_complete or self._command_complete: - raise ApplicationAcceptanceError("connection stage is not admissible now") - for expected_batch in (1, 2): - self._exchange_bootstrap_batch(orchestrator, expected_batch=expected_batch) + raise ApplicationAcceptanceError("inspection stage is not admissible now") + self._exchange_bootstrap_batch(orchestrator, expected_batch=1) binding = orchestrator.binding if binding is None: - raise ApplicationAcceptanceError("connection stage produced no live device binding") + raise ApplicationAcceptanceError("inspection stage produced no live device binding") + self._prepared_binding = binding + self._dialogue_stage = "inspection-ready" + return binding + + def complete_connection_stage( + self, + orchestrator: ShadowApplicationBootstrapOrchestrator, + *, + expected_binding: LiveDeviceControlBinding, + ) -> LiveDeviceControlBinding: + """Promote an ordinal-1 inspection into the canonical ordinals 2-6.""" + + if self._dialogue_stage != "inspection-ready": + raise ApplicationAcceptanceError("connection completion requires inspection-ready") + if self._prepared_binding != expected_binding: + raise ApplicationAcceptanceError("inspection binding changed before completion") + self._exchange_bootstrap_batch(orchestrator, expected_batch=2) + binding = orchestrator.binding + if binding is None or binding != expected_binding: + raise ApplicationAcceptanceError("connection stage changed the inspected identity") self._prepared_binding = binding self._dialogue_stage = "connection-ready" return binding @@ -261,10 +357,13 @@ class PhysicalAcceptanceDialogueExecutor: self, event: Literal["workspace-entered", "project-prompt-opened", "start-confirmed"], event_observed: Callable[[], bool], - ) -> OperatorDialogueCheckpoint: + *, + reconciled_active_observed: Callable[[], bool] | None = None, + ) -> OperatorDialogueCheckpoint | None: """Service the original socket until one exact operator UI event occurs.""" expected = { + "inspection-ready": "workspace-entered", "connection-ready": "workspace-entered", "workspace-ready": "project-prompt-opened", "project-ready": "start-confirmed", @@ -276,9 +375,19 @@ class PhysicalAcceptanceDialogueExecutor: binding = self._prepared_binding if binding is None: raise ApplicationAcceptanceError("canonical preparation binding is unavailable") - while not ( - self._transport.pre_start_ready(binding) and event_observed() - ): + if reconciled_active_observed is not None and event != "workspace-entered": + raise ApplicationAcceptanceError( + "active recovery may replace only the workspace-entry checkpoint" + ) + while True: + if reconciled_active_observed is not None and reconciled_active_observed(): + if not self._transport.scan_initialization_complete(binding): + raise ApplicationAcceptanceError( + "active recovery requires fresh bound SCANNING state" + ) + return None + if self._transport.pre_start_ready(binding) and event_observed(): + break self._transport.maintain_open_for( CONTROL_NETWORK_PUMP_QUANTUM_SECONDS, allowed_response_topics={MODELING_STATUS_RESPONSE_TOPIC}, @@ -291,16 +400,52 @@ class PhysicalAcceptanceDialogueExecutor: owner_token=self._checkpoint_owner, ) + def adopt_reconciled_scanning( + self, + *, + authority: ApplicationControlAuthority, + binding: LiveDeviceControlBinding, + ) -> None: + """Adopt externally observed SCANNING without inventing a new START. + + The caller has already committed an exact read-only physical-ledger + reconciliation for this control generation. This method performs no + publish; it only gives the existing socket enough local state to wait + for one later operator-confirmed STOP. + """ + + if self._dialogue_stage not in {"inspection-ready", "connection-ready"}: + raise ApplicationAcceptanceError( + "active recovery requires an inspected pre-START control session" + ) + if self._prepared_binding != binding: + raise ApplicationAcceptanceError("active recovery binding changed") + if self._command_complete or self._start_complete or self._stop_attempted: + raise ApplicationAcceptanceError("active recovery cannot replace a command attempt") + if not self._transport.scan_initialization_complete(binding): + raise ApplicationAcceptanceError( + "active recovery requires the bound K1 to report SCANNING" + ) + self._active_authority = authority + self._active_binding = binding + self._prepared_binding = None + self._active_session_adopted = True + self._dialogue_stage = "post-initialization-observed" + def run_workspace_entry_stage( self, orchestrator: ShadowApplicationBootstrapOrchestrator, checkpoint: OperatorDialogueCheckpoint, + *, + dispatch_guard: Callable[[], None] | None = None, ) -> LiveDeviceControlBinding: """Emit ordinal 7 only for the observed scan-workspace entry action.""" if self._dialogue_stage != "connection-ready": raise ApplicationAcceptanceError("workspace entry requires the connection stage") self._consume_checkpoint(checkpoint, expected="workspace-entered") + if dispatch_guard is not None: + dispatch_guard() self._exchange_bootstrap_batch(orchestrator, expected_batch=3) binding = orchestrator.binding if binding is None: @@ -312,12 +457,16 @@ class PhysicalAcceptanceDialogueExecutor: self, orchestrator: ShadowApplicationBootstrapOrchestrator, checkpoint: OperatorDialogueCheckpoint, + *, + dispatch_guard: Callable[[], None] | None = None, ) -> LiveDeviceControlBinding: """Emit ordinals 8-10 when the operator opens the project-name prompt.""" if self._dialogue_stage != "workspace-ready": raise ApplicationAcceptanceError("project prompt requires workspace entry") self._consume_checkpoint(checkpoint, expected="project-prompt-opened") + if dispatch_guard is not None: + dispatch_guard() self._exchange_bootstrap_batch(orchestrator, expected_batch=4) if not orchestrator.snapshot().bootstrap_complete: raise ApplicationAcceptanceError("project prompt did not complete the transcript") @@ -343,6 +492,7 @@ class PhysicalAcceptanceDialogueExecutor: binding: LiveDeviceControlBinding, permit: PhysicalAcceptancePermit, checkpoint: OperatorDialogueCheckpoint, + dispatch_guard: Callable[[], None] | None = None, ) -> ModelingResponse: """Execute retained operations 11-14 on one continuously serviced socket.""" @@ -363,6 +513,8 @@ class PhysicalAcceptanceDialogueExecutor: if permit.action is not ModelingAction.START: raise ApplicationAcceptanceError("canonical START requires a fresh START permit") + if dispatch_guard is not None: + dispatch_guard() permit.consume(ModelingAction.START) self._start_permit_snapshot = permit.snapshot() self._command_complete = True @@ -392,6 +544,8 @@ class PhysicalAcceptanceDialogueExecutor: ) from exc immediate = post_start.immediate_modeling_status + if dispatch_guard is not None: + dispatch_guard() self._transport.exchange_batch_once( [OneShotPublishEnvelope.from_dialogue_request(immediate)], required_response_operation_keys=(), @@ -414,6 +568,8 @@ class PhysicalAcceptanceDialogueExecutor: required_operations = { f"dialogue:{request.ordinal}:{request.message_type}" for request in refresh } + if dispatch_guard is not None: + dispatch_guard() refresh_responses = self._transport.exchange_batch_once( [OneShotPublishEnvelope.from_dialogue_request(request) for request in refresh], required_response_operation_keys=required_operations, @@ -469,7 +625,9 @@ class PhysicalAcceptanceDialogueExecutor: ) -> None: """Continuously service the original socket until the operator requests STOP.""" - if self._dialogue_stage != "post-initialization-observed" or not self._start_complete: + if self._dialogue_stage != "post-initialization-observed" or not ( + self._start_complete or self._active_session_adopted + ): raise ApplicationAcceptanceError( "active control ownership requires the complete post-START dialogue" ) @@ -491,12 +649,17 @@ class PhysicalAcceptanceDialogueExecutor: self, command: ShadowModelingCommand, permit: PhysicalAcceptancePermit, + *, + dispatch_guard: Callable[[], None] | None = None, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, ) -> ModelingResponse: """Emit retained STOP on the same socket and with a separate permit.""" if command.action is not ModelingAction.STOP: raise ApplicationAcceptanceError("canonical STOP executor requires STOP") - if self._dialogue_stage != "stop-requested" or not self._start_complete: + if self._dialogue_stage != "stop-requested" or not ( + self._start_complete or self._active_session_adopted + ): raise ApplicationAcceptanceError( "STOP requires continuous ownership from the canonical START session" ) @@ -507,21 +670,66 @@ class PhysicalAcceptanceDialogueExecutor: if authority is None or binding is None: raise ApplicationAcceptanceError("canonical START binding is no longer available") self._require_command_identity(command, authority=authority, binding=binding) - if not self._transport.scan_initialization_complete(binding): + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + scanning = self._transport.scan_initialization_complete(binding) + # The status projection can wait on the transport lock. Deadline + # admission is therefore sampled again before its result can advance + # the physical STOP dialogue. + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + if not scanning: raise ApplicationAcceptanceError( "canonical STOP requires the bound K1 to still report SCANNING" ) if permit.action is not ModelingAction.STOP: raise ApplicationAcceptanceError("canonical STOP requires a separate STOP permit") - permit.consume(ModelingAction.STOP) - self._stop_permit_snapshot = permit.snapshot() + if dispatch_guard is not None: + dispatch_guard() + # Route/control validation is read-only but may block. It cannot + # authorize a publish whose operation deadline elapsed meanwhile. + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + permit_reservation = permit.reserve(ModelingAction.STOP) self._stop_attempted = True self._dialogue_stage = "stop-attempted" - responses = self._transport.exchange_batch_once( - [OneShotPublishEnvelope.from_modeling_command(command)], - required_response_operation_keys={"modeling:stop"}, - ) + # Keep source-compatible test/integration transports on the legacy + # call shape unless a real operation deadline was supplied. + stop_envelope = OneShotPublishEnvelope.from_modeling_command(command) + try: + if dispatch_admission_deadline_reached is None: + responses = self._transport.exchange_batch_once( + [stop_envelope], + required_response_operation_keys={"modeling:stop"}, + dispatch_admission_commit=permit_reservation.commit, + ) + else: + responses = self._transport.exchange_batch_once( + [stop_envelope], + required_response_operation_keys={"modeling:stop"}, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + dispatch_admission_commit=permit_reservation.commit, + ) + except ApplicationMqttTransportError as exc: + if exc.reason_code in { + "physical-command-dispatch-deadline-expired", + "physical-acceptance-permit-expired-before-dispatch", + }: + # Atomic physical admission rejected either the operation + # deadline or the still-fresh permit before durable + # DISPATCHING. Preserve that stronger zero-attempt fact. + permit_reservation.release() + self._stop_attempted = False + self._dialogue_stage = "stop-requested" + raise + finally: + self._stop_permit_snapshot = permit.snapshot() payload = responses["modeling:stop"] self._record_response_evidence( phase="modeling", @@ -545,6 +753,19 @@ class PhysicalAcceptanceDialogueExecutor: self._dialogue_stage = "stop-acknowledged" return response + @staticmethod + def _require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached: Callable[[], bool] | None, + ) -> None: + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + def maintain_post_stop_until_standby(self) -> None: """Keep servicing control reports through save and protocol standby. @@ -577,6 +798,7 @@ class PhysicalAcceptanceDialogueExecutor: "command_complete": self._command_complete, "start_attempted": self._command_complete, "start_complete": self._start_complete, + "active_session_adopted": self._active_session_adopted, "stop_attempted": self._stop_attempted, "stop_complete": self._stop_complete, "dialogue_stage": self._dialogue_stage, diff --git a/src/k1link/device_plugins/xgrids_k1/protocol/application_authority.py b/src/k1link/device_plugins/xgrids_k1/protocol/application_authority.py index 5fc3e3a..d3baf88 100644 --- a/src/k1link/device_plugins/xgrids_k1/protocol/application_authority.py +++ b/src/k1link/device_plugins/xgrids_k1/protocol/application_authority.py @@ -17,11 +17,35 @@ KEYCHAIN_SERVICE = "NODEDC Mission Core XGRIDS K1 OpenAPI" KEYCHAIN_ACCOUNT = "lixelgo-application-fw-3.0.2" KEYCHAIN_TIMEOUT_SECONDS = 5.0 KEYCHAIN_INTERACTIVE_TIMEOUT_SECONDS = 300.0 +_ERR_SEC_USER_CANCELED = -128 +_ERR_SEC_AUTH_FAILED = -25293 +_ERR_SEC_INTERACTION_NOT_ALLOWED = -25308 class ApplicationAuthorityLoadError(RuntimeError): """The private application authority could not be loaded safely.""" + def __init__( + self, + message: str, + *, + reason_code: str = "application_authority_unavailable", + ) -> None: + super().__init__(message) + self.reason_code = reason_code + + +def _keychain_authority_reason_code(status: int) -> str: + """Reduce an OSStatus to a reviewed, secret-free operator class.""" + + if status == _ERR_SEC_INTERACTION_NOT_ALLOWED: + return "keychain-authorization-required" + if status == _ERR_SEC_AUTH_FAILED: + return "keychain-authorization-denied" + if status == _ERR_SEC_USER_CANCELED: + return "keychain-authorization-cancelled" + return "application_authority_unavailable" + class CommandRunner(Protocol): def __call__( @@ -97,8 +121,12 @@ def _read_keychain_secret_via_security_framework(*, service: str, account: str) if not isinstance(result, tuple) or len(result) != 2: raise ApplicationAuthorityLoadError("macOS Keychain authority lookup failed") status, secret_data = result - if int(status) != 0 or secret_data is None: - raise ApplicationAuthorityLoadError("macOS Keychain authority is unavailable") + status_code = int(status) + if status_code != 0 or secret_data is None: + raise ApplicationAuthorityLoadError( + "macOS Keychain authority is unavailable", + reason_code=_keychain_authority_reason_code(status_code), + ) try: return bytes(secret_data) except Exception as exc: diff --git a/src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py b/src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py index 56885d6..57449aa 100644 --- a/src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py +++ b/src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import hmac import math import secrets @@ -8,6 +9,8 @@ import time from collections import deque from collections.abc import Callable, Collection, Sequence from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Literal, Protocol import paho.mqtt.client as mqtt from paho.mqtt.enums import CallbackAPIVersion @@ -38,6 +41,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( OneShotPublishEnvelope, ) from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( + OPENAPI_SUCCESS, ModelingAction, ModelingProtocolError, decode_device_status_report, @@ -57,6 +61,14 @@ CONTROL_LOOP_INTERVAL_SECONDS = 0.05 MAX_CONTROL_MAINTAIN_SECONDS = 30.0 MAX_CONTROL_RESPONSE_BYTES = 64 * 1024 SYSTEM_ERROR_TOPIC = "lixel/application/report/system_error" +CONTROL_HEARTBEAT_TOPIC = "lixel/application/report/heartbeat" +# DeviceInfo proves identity only at one instant. Thereafter the unchanged +# control socket must keep producing reviewed MQTT evidence. Ninety seconds +# is deliberately longer than the retained 60-second MQTT keepalive, but still +# bounds a half-open control proof even when neither the OS route nor a TCP +# connect probe notices the break. Both clocks are checked so macOS suspend +# cannot preserve pre-sleep authority. +CONTROL_PROOF_TTL_SECONDS = 90.0 APPLICATION_RESPONSE_TOPIC_BY_REQUEST_TOPIC = { DEVICE_INFO_REQUEST_TOPIC: DEVICE_INFO_RESPONSE_TOPIC, @@ -108,7 +120,7 @@ CONTROL_SUBSCRIPTION_GROUPS: tuple[tuple[tuple[str, int], ...], ...] = ( ("SetAppConfigResponse", 2), ("GetMqttConfigResponse", 2), ("SetMqttConfigResponse", 2), - ("lixel/application/report/heartbeat", 2), + (CONTROL_HEARTBEAT_TOPIC, 2), ("lixel/calibration/response/lidar", 2), ("lixel/application/response/cameraConfig", 2), ("lixel/application/response/service", 2), @@ -196,6 +208,79 @@ class ApplicationControlDeviceFault(RuntimeError): self.reason_code = reason_code +class ApplicationControlProofStale(ApplicationControlDeviceFault): + """The bound MQTT session stopped producing fresh remote evidence.""" + + def __init__(self, message: str) -> None: + super().__init__(message, reason_code="control_proof_stale") + + +@dataclass(frozen=True, slots=True) +class ApplicationMqttPublishEvidence: + """Secret-free evidence for one exact MQTT publish attempt.""" + + operation_key: str + topic: str + payload_sha256: str + qos: int + retain: bool + packet_id: int | None + + +@dataclass(frozen=True, slots=True) +class ApplicationMqttResponseEvidence: + """Exactly correlated application-response evidence.""" + + operation_key: str + response_topic: str + payload_sha256: str + modeling_action: Literal["start", "stop"] | None + result_code: int | None + success: bool | None + observed_at_utc: str + + +@dataclass(frozen=True, slots=True) +class ApplicationMqttDeviceStatusEvidence: + """Decoded DeviceStatus plus the exact wire-message provenance.""" + + vendor_device_id_sha256: str | None + device_serial_sha256: str | None + session_state: str | None + session_state_code: int + project_bound: bool + project_id_sha256: str | None + init_ready: bool + status_message_sha256: str + mqtt_retained: bool + observed_at_utc: str + + +class ApplicationMqttEvidenceObserver(Protocol): + """Transport-only evidence port; policy and durable storage live above it.""" + + def publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: ... + + def publish_result( + self, + evidence: ApplicationMqttPublishEvidence, + *, + publish_call_returned: bool, + ) -> None: ... + + def qos2_completed(self, evidence: ApplicationMqttPublishEvidence) -> None: ... + + def application_response(self, evidence: ApplicationMqttResponseEvidence) -> None: ... + + def device_status(self, evidence: ApplicationMqttDeviceStatusEvidence) -> None: ... + + @dataclass(frozen=True, slots=True) class _ApplicationResponseExpectation: operation_key: str @@ -238,6 +323,8 @@ class ApplicationMqttTransportSnapshot: device_status_reports: int system_error_reports: int report_decode_errors: int + retained_control_reports_ignored: int + retained_application_responses_rejected: int latest_device_session_state: str | None latest_device_project_bound: bool | None latest_device_init_ready: bool | None @@ -247,6 +334,10 @@ class ApplicationMqttTransportSnapshot: last_loop_result_name: str | None last_loop_phase: str | None operation_keys_consumed: int + control_proof_revision: int + control_proof_source: str | None + control_proof_fresh: bool + control_proof_age_seconds: float | None clean_session: bool = False keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS automatic_reconnect: bool = False @@ -266,6 +357,10 @@ class ApplicationMqttTransportSnapshot: "device_status_reports": self.device_status_reports, "system_error_reports": self.system_error_reports, "report_decode_errors": self.report_decode_errors, + "retained_control_reports_ignored": self.retained_control_reports_ignored, + "retained_application_responses_rejected": ( + self.retained_application_responses_rejected + ), "latest_device_session_state": self.latest_device_session_state, "latest_device_project_bound": self.latest_device_project_bound, "latest_device_init_ready": self.latest_device_init_ready, @@ -275,6 +370,10 @@ class ApplicationMqttTransportSnapshot: "last_loop_result_name": self.last_loop_result_name, "last_loop_phase": self.last_loop_phase, "operation_keys_consumed": self.operation_keys_consumed, + "control_proof_revision": self.control_proof_revision, + "control_proof_source": self.control_proof_source, + "control_proof_fresh": self.control_proof_fresh, + "control_proof_age_seconds": self.control_proof_age_seconds, "clean_session": self.clean_session, "keepalive_seconds": self.keepalive_seconds, "automatic_reconnect": self.automatic_reconnect, @@ -301,6 +400,9 @@ class ReviewedApplicationMqttTransport: allow_device_ap: bool = False, client_factory: Callable[[], mqtt.Client] | None = None, monotonic: Callable[[], float] = time.monotonic, + suspend_aware_clock: Callable[[], float] = time.time, + utc_now: Callable[[], datetime] = lambda: datetime.now(UTC), + control_proof_ttl_seconds: float = CONTROL_PROOF_TTL_SECONDS, ) -> None: self._target_ipv4 = validate_private_ipv4(host) if self._target_ipv4 == AP_FALLBACK_IPV4 and not allow_device_ap: @@ -310,6 +412,7 @@ class ReviewedApplicationMqttTransport: for name, value in ( ("connect_timeout_seconds", connect_timeout_seconds), ("exchange_timeout_seconds", exchange_timeout_seconds), + ("control_proof_ttl_seconds", control_proof_ttl_seconds), ): if not math.isfinite(value) or value <= 0: raise ValueError(f"{name} must be finite and greater than zero") @@ -317,6 +420,9 @@ class ReviewedApplicationMqttTransport: self._connect_timeout_seconds = connect_timeout_seconds self._exchange_timeout_seconds = exchange_timeout_seconds self._monotonic = monotonic + self._suspend_aware_clock = suspend_aware_clock + self._utc_now = utc_now + self._control_proof_ttl_seconds = float(control_proof_ttl_seconds) self._client_factory = client_factory self._client: mqtt.Client | None = None self._lock = threading.Lock() @@ -327,7 +433,16 @@ class ReviewedApplicationMqttTransport: self._subscription_mid: int | None = None self._subscription_group_index = 0 self._completed_publish_mids: set[int] = set() - self._messages: deque[tuple[str, bytes]] = deque() + self._publish_evidence_by_mid: dict[int, ApplicationMqttPublishEvidence] = {} + self._qos2_evidence_notified_mids: set[int] = set() + self._publish_call_in_progress = False + self._unowned_publish_mids_during_call: set[int] = set() + self._messages: deque[tuple[str, bytes, bool]] = deque() + self._evidence_observer: ApplicationMqttEvidenceObserver | None = None + # The optional release callback keeps a process-local lifecycle fence + # held across the exact guard -> evidence -> client.publish window. + # A plain validator may still return None. + self._dispatch_guard: Callable[[], Callable[[], None] | None] | None = None self._callback_error: str | None = None self._consumed_operation_keys: set[str] = set() self._known_response_expectations: dict[str, _ApplicationResponseExpectation] = {} @@ -346,17 +461,62 @@ class ReviewedApplicationMqttTransport: self._device_status_reports = 0 self._system_error_reports = 0 self._report_decode_errors = 0 + self._retained_control_reports_ignored = 0 + self._retained_application_responses_rejected = 0 self._latest_device_session_state: str | None = None self._latest_device_project_bound: bool | None = None self._latest_device_init_ready: bool | None = None self._latest_device_id: str | None = None self._latest_device_serial: str | None = None self._latest_device_fault = False + self._latest_device_status_sequence = 0 + self._promoted_device_status_sequence = 0 + self._latest_device_status_observed_monotonic: float | None = None + self._latest_device_status_observed_suspend_aware: float | None = None self._latest_system_error_code: int | None = None self._latest_system_error_state: str | None = None self._last_loop_result_code: int | None = None self._last_loop_result_name: str | None = None self._last_loop_phase: str | None = None + self._control_proof_revision = 0 + self._control_proof_source: str | None = None + self._control_proof_observed_monotonic: float | None = None + self._control_proof_observed_suspend_aware: float | None = None + + def install_evidence_observer(self, observer: ApplicationMqttEvidenceObserver) -> None: + """Install the session-owned observer before this transport is opened.""" + + with self._lock: + if self._state != "new": + raise ApplicationMqttTransportError( + "control evidence observer must be installed before open", + reason_code="evidence_observer_install_too_late", + ) + if self._evidence_observer is not None: + raise ApplicationMqttTransportError( + "control evidence observer is already installed", + reason_code="evidence_observer_already_installed", + ) + self._evidence_observer = observer + + def install_dispatch_guard( + self, + guard: Callable[[], Callable[[], None] | None], + ) -> None: + """Install a session/path admission check evaluated before every publish.""" + + with self._lock: + if self._state != "new": + raise ApplicationMqttTransportError( + "control dispatch guard must be installed before open", + reason_code="dispatch_guard_install_too_late", + ) + if self._dispatch_guard is not None: + raise ApplicationMqttTransportError( + "control dispatch guard is already installed", + reason_code="dispatch_guard_already_installed", + ) + self._dispatch_guard = guard def open(self) -> ApplicationMqttTransportSnapshot: with self._lock: @@ -393,14 +553,15 @@ class ReviewedApplicationMqttTransport: envelopes: Sequence[OneShotPublishEnvelope], *, required_response_operation_keys: Collection[str], + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, ) -> dict[str, bytes]: batch = tuple(envelopes) required = frozenset(required_response_operation_keys) if not batch: raise ValueError("control exchange batch must not be empty") if not required and any( - envelope.topic != "lixel/application/request/modeling_status" - for envelope in batch + envelope.topic != "lixel/application/request/modeling_status" for envelope in batch ): raise ValueError("response-free exchange is limited to retained ModelingStatus reads") operation_keys = tuple(envelope.operation_key for envelope in batch) @@ -417,9 +578,14 @@ class ReviewedApplicationMqttTransport: if expectation.operation_key in required } if len(pending) != len(required): - raise ValueError( - "required responses do not identify one exact batch operation each" - ) + raise ValueError("required responses do not identify one exact batch operation each") + + # A physical operation deadline starts at public admission, not when + # this retained socket eventually reaches its publish loop. Refuse an + # already-expired STOP before servicing or consuming any batch state. + self._require_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) # Classify packets already delivered by the manual Paho loop before # consuming this batch. An exact response to an earlier issued @@ -433,6 +599,13 @@ class ReviewedApplicationMqttTransport: self._service_once(post_publish=True) self._drain_responses({}, {}) + # Draining callbacks and the zero-wait socket service above can still + # block on client/transport locks. Re-sample before reserving the + # operation key in this dialogue. + self._require_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + with self._lock: if self._consumed_operation_keys.intersection(operation_keys): raise ApplicationCommandOutcomeUnknown( @@ -482,14 +655,10 @@ class ReviewedApplicationMqttTransport: for operation_key in existing_operations if operation_key not in superseded_optional ] - self._optional_response_operations.difference_update( - superseded_optional - ) + self._optional_response_operations.difference_update(superseded_optional) for operation_key in superseded_optional: del self._known_response_expectations[operation_key] - self._known_response_expectations[ - expectation.operation_key - ] = expectation + self._known_response_expectations[expectation.operation_key] = expectation self._response_operations_by_correlation.setdefault( expectation.correlation_key, [], @@ -501,22 +670,164 @@ class ReviewedApplicationMqttTransport: client = self._require_client() publish_mids: set[int] = set() for envelope in batch: - with self._lock: - self._publish_attempts += 1 + dispatch_evidence = self._publish_evidence(envelope, packet_id=None) try: - info = client.publish( - envelope.topic, - payload=envelope.payload, - qos=envelope.qos, - retain=envelope.retain, + release_dispatch = self._run_dispatch_guard() + except Exception as exc: + if publish_mids: + self._fail_after_publish("control MQTT dispatch guard failed", exc) + # The guard is read-only admission. If the operation expired + # while it waited, preserve the stronger definite no-publish + # reason instead of reducing it to a generic route failure. + self._require_dispatch_deadline_open( + dispatch_admission_deadline_reached ) - except (OSError, RuntimeError, ValueError) as exc: - self._fail_after_publish("control MQTT publish call failed", exc) - if info.rc != mqtt.MQTT_ERR_SUCCESS or info.mid is None: + self._fail_before_publish("control MQTT dispatch guard failed", exc) + try: + with self._lock: + # The dispatch guard may block behind a facade-owned + # lifecycle fence. ``close()`` is allowed to run while it + # waits, so the readiness check performed when the batch + # was admitted is no longer authoritative here. Recheck + # after acquiring the exact publish lease and before both + # durable dispatch evidence and ``client.publish``. This + # makes a successful local close a no-future-publish proof + # for a worker that had only passed the earlier precheck. + if ( + self._closing + or self._state != "ready" + or not self._connected + or not self._subscribed + ): + raise ApplicationMqttTransportError( + "control transport closed before the publish fence", + reason_code="transport_closed_before_publish", + ) + # The observer below owns the final deadline check and + # durable DISPATCHING transition as one coordinator call. + # Do not sample the deadline here: that would create a + # reversible TOCTOU window before the ledger fsync starts. + if self._publish_call_in_progress or self._unowned_publish_mids_during_call: + raise ApplicationMqttTransportError( + "control MQTT publish evidence window is already occupied", + reason_code="publish_evidence_window_occupied", + ) + self._publish_call_in_progress = True + try: + dispatch_admission_committed = False + + def commit_dispatch_admission() -> None: + nonlocal dispatch_admission_committed + if dispatch_admission_commit is not None: + dispatch_admission_commit() + dispatch_admission_committed = True + + self._notify_publish_dispatching( + dispatch_evidence, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + dispatch_admission_commit=commit_dispatch_admission, + ) + except ApplicationMqttTransportError as exc: + with self._lock: + self._publish_call_in_progress = False + if ( + exc.reason_code + in { + "physical-command-dispatch-deadline-expired", + "physical-acceptance-permit-expired-before-dispatch", + } + and not dispatch_admission_committed + ): + # The coordinator checked while it still owned the + # PREPARED edge and did not start DISPATCHING. + with self._lock: + self._state = "failed" + self.close() + raise + if dispatch_admission_committed: + self._fail_after_publish( + "control MQTT dispatch evidence failed after physical admission", + exc, + ) + self._fail_before_publish( + "control MQTT dispatch evidence failed", + exc, + ) + except Exception as exc: + with self._lock: + self._publish_call_in_progress = False + if dispatch_admission_committed: + self._fail_after_publish( + "control MQTT dispatch evidence failed after physical admission", + exc, + ) + self._fail_before_publish("control MQTT dispatch evidence failed", exc) + with self._lock: + self._publish_attempts += 1 + try: + info = client.publish( + envelope.topic, + payload=envelope.payload, + qos=envelope.qos, + retain=envelope.retain, + ) + except (OSError, RuntimeError, ValueError) as exc: + with self._lock: + self._publish_call_in_progress = False + self._unowned_publish_mids_during_call.clear() + try: + self._notify_publish_result( + dispatch_evidence, + publish_call_returned=False, + ) + except Exception as observer_exc: + self._fail_after_publish( + "control MQTT publish-failure evidence failed", + observer_exc, + ) + self._fail_after_publish("control MQTT publish call failed", exc) + packet_id = info.mid if isinstance(info.mid, int) and info.mid > 0 else None + returned_evidence = self._publish_evidence(envelope, packet_id=packet_id) + try: + self._notify_publish_result( + returned_evidence, + publish_call_returned=True, + ) + except Exception as exc: + self._fail_after_publish("control MQTT publish-result evidence failed", exc) + finally: + if release_dispatch is not None: + release_dispatch() + if info.rc != mqtt.MQTT_ERR_SUCCESS or packet_id is None: self._fail_after_publish("control MQTT publish call returned an unsafe result") - if info.mid in publish_mids: + assert packet_id is not None + if packet_id in publish_mids: self._fail_after_publish("control MQTT reused a packet identifier") - publish_mids.add(info.mid) + publish_mids.add(packet_id) + notify_qos2 = False + unexpected_synchronous_mids: set[int] + with self._lock: + self._publish_evidence_by_mid[packet_id] = returned_evidence + unexpected_synchronous_mids = self._unowned_publish_mids_during_call - {packet_id} + if packet_id in self._unowned_publish_mids_during_call: + self._completed_publish_mids.add(packet_id) + self._qos2_completions += 1 + self._unowned_publish_mids_during_call.clear() + self._publish_call_in_progress = False + if ( + packet_id in self._completed_publish_mids + and packet_id not in self._qos2_evidence_notified_mids + ): + self._qos2_evidence_notified_mids.add(packet_id) + notify_qos2 = True + if unexpected_synchronous_mids: + self._fail_after_publish( + "control MQTT completed an unrelated packet during publish" + ) + if notify_qos2: + self._notify_qos2_completed_safely(returned_evidence) responses: dict[str, bytes] = {} deadline = self._monotonic() + self._exchange_timeout_seconds @@ -535,6 +846,10 @@ class ReviewedApplicationMqttTransport: self._drain_responses(pending, responses) with self._lock: self._correlated_responses += len(responses) + self._completed_publish_mids.difference_update(publish_mids) + self._qos2_evidence_notified_mids.difference_update(publish_mids) + for packet_id in publish_mids: + self._publish_evidence_by_mid.pop(packet_id, None) return responses def maintain_open_for( @@ -571,9 +886,7 @@ class ReviewedApplicationMqttTransport: except (OSError, RuntimeError, ValueError) as exc: self._fail_after_publish("control MQTT network loop failed", exc) if result != mqtt.MQTT_ERR_SUCCESS: - self._fail_after_publish( - self._record_loop_failure(result, phase="maintain-open") - ) + self._fail_after_publish(self._record_loop_failure(result, phase="maintain-open")) self._discard_allowed_responses(allowed) def close(self) -> None: @@ -623,6 +936,7 @@ class ReviewedApplicationMqttTransport: return ( self._latest_device_session_state == "ready" and self._latest_device_project_bound is False + and self._latest_device_init_ready is False ) def standby_complete(self, binding: LiveDeviceControlBinding) -> bool: @@ -633,6 +947,7 @@ class ReviewedApplicationMqttTransport: return ( self._latest_device_session_state == "ready" and self._latest_device_project_bound is False + and self._latest_device_init_ready is False ) def validate_bound_status(self, binding: LiveDeviceControlBinding) -> None: @@ -641,8 +956,23 @@ class ReviewedApplicationMqttTransport: with self._lock: self._require_healthy_bound_status_locked(binding) + def validate_control_proof(self, binding: LiveDeviceControlBinding) -> None: + """Require fresh remote evidence for the exact DeviceInfo-bound socket. + + This does not publish a probe and never retries. Freshness advances + only when the existing MQTT session receives either an exactly + correlated application response, a decoded bound DeviceStatus report, + or the vendor heartbeat already present in the retained subscription + profile. Repeated snapshots and successful local ``loop()`` calls do + not extend the proof. + """ + + with self._lock: + self._require_healthy_bound_status_locked(binding) + def snapshot(self) -> ApplicationMqttTransportSnapshot: with self._lock: + proof_age = self._control_proof_age_locked() return ApplicationMqttTransportSnapshot( state=self._state, connect_attempts=self._connect_attempts, @@ -655,6 +985,10 @@ class ReviewedApplicationMqttTransport: device_status_reports=self._device_status_reports, system_error_reports=self._system_error_reports, report_decode_errors=self._report_decode_errors, + retained_control_reports_ignored=self._retained_control_reports_ignored, + retained_application_responses_rejected=( + self._retained_application_responses_rejected + ), latest_device_session_state=self._latest_device_session_state, latest_device_project_bound=self._latest_device_project_bound, latest_device_init_ready=self._latest_device_init_ready, @@ -664,6 +998,14 @@ class ReviewedApplicationMqttTransport: last_loop_result_name=self._last_loop_result_name, last_loop_phase=self._last_loop_phase, operation_keys_consumed=len(self._consumed_operation_keys), + control_proof_revision=self._control_proof_revision, + control_proof_source=self._control_proof_source, + control_proof_fresh=( + self._control_proof_revision > 0 + and proof_age is not None + and proof_age <= self._control_proof_ttl_seconds + ), + control_proof_age_seconds=proof_age, ) def _new_client(self) -> mqtt.Client: @@ -730,9 +1072,31 @@ class ReviewedApplicationMqttTransport: if reason_code.is_failure: self._set_callback_error("control MQTT QoS2 transaction failed") return + evidence: ApplicationMqttPublishEvidence | None = None + unexpected = False with self._lock: - self._completed_publish_mids.add(mid) - self._qos2_completions += 1 + candidate = self._publish_evidence_by_mid.get(mid) + if candidate is None: + if self._publish_call_in_progress: + if mid in self._unowned_publish_mids_during_call: + unexpected = True + else: + self._unowned_publish_mids_during_call.add(mid) + else: + unexpected = True + elif mid in self._completed_publish_mids: + unexpected = True + else: + self._completed_publish_mids.add(mid) + self._qos2_completions += 1 + if mid not in self._qos2_evidence_notified_mids: + self._qos2_evidence_notified_mids.add(mid) + evidence = candidate + if unexpected: + self._set_callback_error("control MQTT received an unowned or duplicate PUBCOMP") + return + if evidence is not None: + self._notify_qos2_completed_safely(evidence) def on_message( _callback_client: mqtt.Client, @@ -747,13 +1111,18 @@ class ReviewedApplicationMqttTransport: if len(payload) > MAX_CONTROL_RESPONSE_BYTES: self._set_callback_error("control MQTT response exceeds the reviewed bound") return + mqtt_retained = bool(getattr(message, "retain", False)) if message.topic in CONTROL_REPORT_TOPICS: - self._observe_control_report(message.topic, payload) + self._observe_control_report( + message.topic, + payload, + mqtt_retained=mqtt_retained, + ) with self._lock: self._ignored_known_responses += 1 else: with self._lock: - self._messages.append((message.topic, payload)) + self._messages.append((message.topic, payload, mqtt_retained)) def on_disconnect( _callback_client: mqtt.Client, @@ -774,7 +1143,28 @@ class ReviewedApplicationMqttTransport: client.on_message = on_message client.on_disconnect = on_disconnect - def _observe_control_report(self, topic: str, payload: bytes) -> None: + def _observe_control_report( + self, + topic: str, + payload: bytes, + *, + mqtt_retained: bool, + ) -> None: + if mqtt_retained: + # Retained broker history is useful only as an audit fact. It can + # never prove current liveness, readiness, fault state, or a + # physical-command target transition. + with self._lock: + self._retained_control_reports_ignored += 1 + return + if topic == CONTROL_HEARTBEAT_TOPIC: + # The payload is intentionally opaque. Its value is not treated + # as identity or state; arrival on the already DeviceInfo-bound, + # non-reconnecting MQTT socket is only a liveness observation. + with self._lock: + if self._control_proof_revision > 0: + self._mark_control_proof_locked("mqtt-heartbeat") + return if topic == DEVICE_STATUS_TOPIC: try: device_report = decode_device_status_report(payload) @@ -782,6 +1172,35 @@ class ReviewedApplicationMqttTransport: with self._lock: self._report_decode_errors += 1 return + evidence = ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=( + self._sha256_text(device_report.header.device_id) + if device_report.header is not None + and device_report.header.device_id is not None + else None + ), + device_serial_sha256=( + self._sha256_text(device_report.device_sn) + if device_report.device_sn is not None + else None + ), + session_state=( + device_report.session_state.name.casefold() + if device_report.session_state is not None + else None + ), + session_state_code=device_report.modeling_state_code, + project_bound=bool(device_report.project_id), + project_id_sha256=( + self._sha256_text(device_report.project_id) + if device_report.project_id is not None + else None + ), + init_ready=device_report.init_ready, + status_message_sha256=hashlib.sha256(payload).hexdigest(), + mqtt_retained=mqtt_retained, + observed_at_utc=self._utc_now_text(), + ) with self._lock: self._device_status_reports += 1 self._latest_device_session_state = ( @@ -792,15 +1211,24 @@ class ReviewedApplicationMqttTransport: self._latest_device_project_bound = bool(device_report.project_id) self._latest_device_init_ready = device_report.init_ready self._latest_device_id = ( - device_report.header.device_id - if device_report.header is not None - else None + device_report.header.device_id if device_report.header is not None else None ) self._latest_device_serial = device_report.device_sn self._latest_device_fault = bool( - device_report.session_state is not None - and device_report.session_state.is_fault + device_report.session_state is not None and device_report.session_state.is_fault ) + self._latest_device_status_sequence += 1 + # Stamp wall/suspend-aware time first: if macOS sleeps between + # samples, this stays a pre-sleep observation. + self._latest_device_status_observed_suspend_aware = self._freshness_clock_value( + self._suspend_aware_clock, + "control proof suspend-aware clock", + ) + self._latest_device_status_observed_monotonic = self._freshness_clock_value( + self._monotonic, + "control proof monotonic clock", + ) + self._notify_device_status_safely(evidence) return if topic == SYSTEM_ERROR_TOPIC: try: @@ -831,19 +1259,108 @@ class ReviewedApplicationMqttTransport: raise ApplicationControlDeviceFault( f"K1 reported a control fault state: {state or 'unknown'}" ) - if self._device_status_reports == 0: - return - if self._latest_device_id is None or self._latest_device_serial is None: - raise ApplicationControlDeviceFault( - "K1 control status omitted the live device identity" + if self._device_status_reports: + if self._latest_device_id is None or self._latest_device_serial is None: + raise ApplicationControlDeviceFault( + "K1 control status omitted the live device identity" + ) + if ( + self._latest_device_id != binding.vendor_device_id + or self._latest_device_serial != binding.device_serial + ): + raise ApplicationControlDeviceFault( + "K1 control status identity drifted from the live DeviceInfo binding" + ) + if self._latest_device_status_sequence > self._promoted_device_status_sequence: + observed_monotonic = self._latest_device_status_observed_monotonic + observed_suspend_aware = self._latest_device_status_observed_suspend_aware + if observed_monotonic is None or observed_suspend_aware is None: + raise ApplicationControlDeviceFault( + "K1 control status freshness evidence is unavailable" + ) + self._mark_control_proof_locked( + "bound-device-status", + observed_monotonic=observed_monotonic, + observed_suspend_aware=observed_suspend_aware, + ) + self._promoted_device_status_sequence = self._latest_device_status_sequence + if not self._control_proof_fresh_locked(): + raise ApplicationControlProofStale( + "K1 control proof expired without fresh MQTT evidence" ) + + def _mark_control_proof_locked( + self, + source: str, + *, + observed_monotonic: float | None = None, + observed_suspend_aware: float | None = None, + ) -> None: + suspend_aware_now = ( + self._freshness_clock_value( + self._suspend_aware_clock, + "control proof suspend-aware clock", + ) + if observed_suspend_aware is None + else observed_suspend_aware + ) + monotonic_now = ( + self._freshness_clock_value( + self._monotonic, + "control proof monotonic clock", + ) + if observed_monotonic is None + else observed_monotonic + ) + self._control_proof_revision += 1 + self._control_proof_source = source + self._control_proof_observed_monotonic = monotonic_now + self._control_proof_observed_suspend_aware = suspend_aware_now + + def _control_proof_fresh_locked(self) -> bool: + age = self._control_proof_age_locked() + return bool( + self._control_proof_revision > 0 + and age is not None + and age <= self._control_proof_ttl_seconds + ) + + def _control_proof_age_locked(self) -> float | None: if ( - self._latest_device_id != binding.vendor_device_id - or self._latest_device_serial != binding.device_serial + self._control_proof_observed_monotonic is None + or self._control_proof_observed_suspend_aware is None ): - raise ApplicationControlDeviceFault( - "K1 control status identity drifted from the live DeviceInfo binding" + return None + monotonic_now = self._freshness_clock_value( + self._monotonic, + "control proof monotonic clock", + ) + suspend_aware_now = self._freshness_clock_value( + self._suspend_aware_clock, + "control proof suspend-aware clock", + ) + monotonic_elapsed = monotonic_now - self._control_proof_observed_monotonic + suspend_aware_elapsed = suspend_aware_now - self._control_proof_observed_suspend_aware + if monotonic_elapsed < 0 or suspend_aware_elapsed < 0: + return float("inf") + return max( + monotonic_elapsed, + suspend_aware_elapsed, + ) + + @staticmethod + def _freshness_clock_value(clock: Callable[[], float], name: str) -> float: + value = clock() + if ( + not isinstance(value, (int, float)) + or isinstance(value, bool) + or not math.isfinite(value) + ): + raise ApplicationMqttTransportError( + f"{name} returned an invalid value", + reason_code="control_proof_clock_invalid", ) + return float(value) def _subscribe_next_group(self, client: mqtt.Client) -> None: with self._lock: @@ -900,6 +1417,12 @@ class ReviewedApplicationMqttTransport: if post_publish: self._fail_after_publish(failure) self._fail_before_publish(failure) + with self._lock: + callback_error = self._callback_error + if callback_error is not None: + if post_publish: + self._fail_after_publish(callback_error) + self._fail_before_publish(callback_error) def _service_once(self, *, post_publish: bool) -> None: with self._lock: @@ -923,6 +1446,12 @@ class ReviewedApplicationMqttTransport: if post_publish: self._fail_after_publish(failure) self._fail_before_publish(failure) + with self._lock: + callback_error = self._callback_error + if callback_error is not None: + if post_publish: + self._fail_after_publish(callback_error) + self._fail_before_publish(callback_error) def _record_loop_failure( self, @@ -954,29 +1483,35 @@ class ReviewedApplicationMqttTransport: ) -> None: ambiguous_message: str | None = None ambiguous_reason_code: str | None = None + evidence_notifications: list[ApplicationMqttResponseEvidence] = [] with self._lock: while self._messages: - topic, payload = self._messages.popleft() + topic, payload, mqtt_retained = self._messages.popleft() + if mqtt_retained: + self._retained_application_responses_rejected += 1 + self._poison_locked("retained application response is not live evidence") + ambiguous_message = ( + "retained control application response cannot satisfy live correlation" + ) + ambiguous_reason_code = "retained_application_response" + break try: observed = decode_application_message_identity(payload) except ApplicationBootstrapError: self._poison_locked("response identity decoding failed") - ambiguous_message = ( - "control response identity could not be decoded safely" - ) + ambiguous_message = "control response identity could not be decoded safely" ambiguous_reason_code = "response_identity_decode_failed" break observed_modeling_action: ModelingAction | None = None + observed_result_code: int | None = None if topic == MODELING_RESPONSE_TOPIC: try: - observed_modeling_action = decode_modeling_response( - payload - ).action + modeling_response = decode_modeling_response(payload) + observed_modeling_action = modeling_response.action + observed_result_code = modeling_response.error.code except ModelingProtocolError: self._poison_locked("modeling response action decoding failed") - ambiguous_message = ( - "control modeling response could not be decoded safely" - ) + ambiguous_message = "control modeling response could not be decoded safely" ambiguous_reason_code = "modeling_response_decode_failed" break key = (topic, observed.session_id, observed_modeling_action) @@ -1000,10 +1535,28 @@ class ReviewedApplicationMqttTransport: ) if next_unobserved is not None: self._observed_response_operations.add(next_unobserved) - if ( - expectation is not None - and expectation.operation_key == next_unobserved - ): + self._mark_control_proof_locked("correlated-application-response") + modeling_action: Literal["start", "stop"] | None = None + if observed_modeling_action is ModelingAction.START: + modeling_action = "start" + elif observed_modeling_action is ModelingAction.STOP: + modeling_action = "stop" + evidence_notifications.append( + ApplicationMqttResponseEvidence( + operation_key=next_unobserved, + response_topic=topic, + payload_sha256=hashlib.sha256(payload).hexdigest(), + modeling_action=modeling_action, + result_code=observed_result_code, + success=( + observed_result_code == OPENAPI_SUCCESS + if observed_result_code is not None + else None + ), + observed_at_utc=self._utc_now_text(), + ) + ) + if expectation is not None and expectation.operation_key == next_unobserved: responses[next_unobserved] = payload continue self._ignored_known_responses += 1 @@ -1027,6 +1580,11 @@ class ReviewedApplicationMqttTransport: ) ambiguous_reason_code = "response_identity_mismatch" break + for evidence in evidence_notifications: + try: + self._notify_application_response(evidence) + except Exception as exc: + self._fail_after_publish("control MQTT response evidence failed", exc) if ambiguous_message is not None: self.close() raise ApplicationCommandOutcomeUnknown( @@ -1064,6 +1622,130 @@ class ReviewedApplicationMqttTransport: modeling_action=modeling_action, ) + @staticmethod + def _publish_evidence( + envelope: OneShotPublishEnvelope, + *, + packet_id: int | None, + ) -> ApplicationMqttPublishEvidence: + return ApplicationMqttPublishEvidence( + operation_key=envelope.operation_key, + topic=envelope.topic, + payload_sha256=envelope.payload_sha256, + qos=envelope.qos, + retain=envelope.retain, + packet_id=packet_id, + ) + + def _evidence_observer_snapshot(self) -> ApplicationMqttEvidenceObserver | None: + with self._lock: + return self._evidence_observer + + def _run_dispatch_guard(self) -> Callable[[], None] | None: + with self._lock: + guard = self._dispatch_guard + if guard is not None: + return guard() + return None + + @staticmethod + def _require_dispatch_deadline_open( + dispatch_admission_deadline_reached: Callable[[], bool] | None, + ) -> None: + """Check only before durable DISPATCHING; never reinterpret it after.""" + + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + + def _notify_publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None, + dispatch_admission_commit: Callable[[], None] | None, + ) -> None: + observer = self._evidence_observer_snapshot() + if observer is not None: + observer.publish_dispatching( + evidence, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + dispatch_admission_commit=dispatch_admission_commit, + ) + return + # Non-physical lower-level integrations have no durable coordinator. + # Keep their final deadline check at this same callback boundary. + self._require_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + if dispatch_admission_commit is not None: + dispatch_admission_commit() + + def _notify_publish_result( + self, + evidence: ApplicationMqttPublishEvidence, + *, + publish_call_returned: bool, + ) -> None: + observer = self._evidence_observer_snapshot() + if observer is not None: + observer.publish_result( + evidence, + publish_call_returned=publish_call_returned, + ) + + def _notify_qos2_completed_safely( + self, + evidence: ApplicationMqttPublishEvidence, + ) -> None: + observer = self._evidence_observer_snapshot() + if observer is None: + return + try: + observer.qos2_completed(evidence) + except Exception: + self._set_callback_error("control MQTT QoS2 evidence observer failed") + + def _notify_application_response( + self, + evidence: ApplicationMqttResponseEvidence, + ) -> None: + observer = self._evidence_observer_snapshot() + if observer is not None: + observer.application_response(evidence) + + def _notify_device_status_safely( + self, + evidence: ApplicationMqttDeviceStatusEvidence, + ) -> None: + observer = self._evidence_observer_snapshot() + if observer is None: + return + try: + observer.device_status(evidence) + except Exception: + self._set_callback_error("control MQTT DeviceStatus evidence observer failed") + + @staticmethod + def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + def _utc_now_text(self) -> str: + observed = self._utc_now() + if not isinstance(observed, datetime) or observed.tzinfo is None: + raise ApplicationMqttTransportError( + "control evidence UTC clock returned an invalid value", + reason_code="control_evidence_clock_invalid", + ) + return observed.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + def _set_callback_error(self, message: str) -> None: with self._lock: if self._callback_error is None: @@ -1073,9 +1755,15 @@ class ReviewedApplicationMqttTransport: with self._lock: self._state = "failed" self.close() + reason_code = self._transport_failure_reason_code(message) + if message == "control MQTT connect call failed": + if isinstance(cause, ConnectionRefusedError): + reason_code = "mqtt_connect_rejected" + elif isinstance(cause, TimeoutError): + reason_code = "mqtt_connection_timeout" error = ApplicationMqttTransportError( message, - reason_code=self._transport_failure_reason_code(message), + reason_code=reason_code, ) if cause is not None: raise error from cause diff --git a/src/k1link/device_plugins/xgrids_k1/protocol/application_session.py b/src/k1link/device_plugins/xgrids_k1/protocol/application_session.py index a185230..44282d1 100644 --- a/src/k1link/device_plugins/xgrids_k1/protocol/application_session.py +++ b/src/k1link/device_plugins/xgrids_k1/protocol/application_session.py @@ -1,12 +1,20 @@ from __future__ import annotations +import ipaddress import logging import threading import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Literal +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + PhysicalCommandCoordinator, + PhysicalCommandIntentContext, + PhysicalCommandPrepareSettlementError, + PhysicalCommandRuntimeBinding, + hash_physical_identity, +) from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( ApplicationAcceptanceError, PhysicalAcceptanceChecklist, @@ -14,6 +22,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( PhysicalAcceptancePermit, ) from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( + COMPATIBILITY_PROFILE_ID, ApplicationControlAuthority, LiveDeviceControlBinding, ShadowApplicationBootstrapOrchestrator, @@ -24,8 +33,12 @@ from k1link.device_plugins.xgrids_k1.protocol.application_execution import ( ) from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( ApplicationCommandOutcomeUnknown, + ApplicationMqttTransportError, ReviewedApplicationMqttTransport, ) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( CommandHeaderIdentity, ModelingAction, @@ -43,6 +56,7 @@ ApplicationControlPhase = Literal[ "idle", "connecting", "connection-ready", + "active-recovery-requested", "workspace-requested", "workspace-ready", "project-requested", @@ -63,6 +77,77 @@ logger = logging.getLogger(__name__) TransportFactory = Callable[[str], ReviewedApplicationMqttTransport] ScanningObserver = Callable[[], None] +StartPreparationCheckpointPhase = Literal["prepared", "resolved-not-dispatched"] +StartPreparationCheckpointObserver = Callable[ + [ + StartPreparationCheckpointPhase, + PhysicalCommandIntentContext, + OneShotPublishEnvelope, + ], + None, +] + + +class ApplicationConnectionBindingLost(ApplicationAcceptanceError): + """The live route no longer authorizes commands for this MQTT dialogue.""" + + reason_code = "application-connection-binding-lost" + + +class ApplicationControlProofExpired(ApplicationConnectionBindingLost): + """The unchanged MQTT session no longer has fresh remote liveness proof.""" + + reason_code = "control-proof-stale" + + +class ApplicationControlStateConflict(ApplicationAcceptanceError): + """A browser action belongs to an older control-session checkpoint.""" + + reason_code = "application-control-state-conflict" + + +class ApplicationStartCheckpointSettlementError(ApplicationAcceptanceError): + """A failed pre-dispatch START could not be fully sealed locally.""" + + reason_code = "application-start-checkpoint-settlement-failed" + + +@dataclass(frozen=True, slots=True) +class ApplicationConnectionBinding: + """Immutable network epoch for which DeviceInfo was requested. + + It is intentionally captured before the MQTT worker starts. A later + route/intent change cannot re-label an old DeviceInfo response as proof for + the new connection. + """ + + intent_id: str + transport_ref: str + host_path_epoch: int + target_ipv4: str + target_port: int + connection_mode: Literal["bridge", "quick-connect", "direct-connect"] + + def __post_init__(self) -> None: + if not self.intent_id.strip(): + raise ValueError("application connection binding requires an intent id") + if not self.transport_ref.strip(): + raise ValueError("application connection binding requires a transport ref") + if self.host_path_epoch < 1: + raise ValueError("application connection binding requires a host epoch") + parsed = ipaddress.ip_address(self.target_ipv4) + if not isinstance(parsed, ipaddress.IPv4Address) or str(parsed) != self.target_ipv4: + raise ValueError("application connection binding requires canonical IPv4") + if not 1 <= self.target_port <= 65_535: + raise ValueError("application connection binding target port is invalid") + + +ConnectionBindingValidator = Callable[[ApplicationConnectionBinding], bool | None] +ConnectionPathValidator = Callable[[ApplicationConnectionBinding], bool | None] +ConnectionDispatchLease = Callable[ + [ApplicationConnectionBinding, Callable[[], bool] | None], + Callable[[], None], +] @dataclass(frozen=True, slots=True) @@ -99,41 +184,94 @@ class InteractiveApplicationControlSession: transport_factory: TransportFactory = ReviewedApplicationMqttTransport, epoch_seconds: Callable[[], int] = lambda: int(time.time()), scanning_observer: ScanningObserver | None = None, + connection_path_validator: ConnectionPathValidator | None = None, + connection_binding_validator: ConnectionBindingValidator | None = None, + connection_dispatch_lease: ConnectionDispatchLease | None = None, + physical_command_coordinator: PhysicalCommandCoordinator | None = None, ) -> None: self._authority_loader = authority_loader self._transport_factory = transport_factory self._epoch_seconds = epoch_seconds self._scanning_observer = scanning_observer + # Before DeviceInfo arrives only the immutable intent/topology/host/TCP + # path can be checked. Reusing the full authority validator here would + # create a bootstrap cycle because that validator itself requires the + # DeviceInfo proof produced by this dialogue. + self._connection_path_validator = connection_path_validator or connection_binding_validator + self._connection_binding_validator = connection_binding_validator + self._connection_dispatch_lease = connection_dispatch_lease + self._physical_command_coordinator = physical_command_coordinator self._scanning_observer_errors = 0 self._lock = threading.RLock() + # Durable START preparation deliberately runs outside ``self._lock``. + # This narrow gate still admits only one prepare/callback/commit handoff + # while snapshots and STOP/close remain able to inspect the session. + self._start_prepare_gate = threading.Lock() + # The durable START->ACTIVE checkpoint handoff and STOP preparation + # share this short gate. In particular, ``scanning`` must never be + # observable while the physical ledger still has START at its head + # but the recovery checkpoint activation is merely pending: otherwise + # a concurrent STOP could replace that head and permanently strand the + # checkpoint in PREPARED. + self._scanning_transition_gate = threading.Lock() + self._scanning_observer_confirmed = scanning_observer is None self._phase: ApplicationControlPhase = "idle" self._host: str | None = None self._timezone_name: str | None = None self._thread: threading.Thread | None = None self._transport: ReviewedApplicationMqttTransport | None = None self._workspace_requested = threading.Event() + self._active_recovery_requested = threading.Event() self._project_requested = threading.Event() self._start_requested = threading.Event() self._stop_requested = threading.Event() self._cancel_requested = False self._start_confirmation: OperatorPresenceConfirmation | None = None self._stop_confirmation: OperatorPresenceConfirmation | None = None + self._stop_dispatch_admission_deadline_reached: Callable[[], bool] | None = None self._project_name: str | None = None + self._control_authority: ApplicationControlAuthority | None = None + self._prepared_start_command: ShadowModelingCommand | None = None + self._prepared_stop_command: ShadowModelingCommand | None = None self._failure: dict[str, object] | None = None self._dialogue_snapshot: dict[str, object] | None = None self._transport_snapshot: dict[str, object] | None = None + # This is emitted only after the canonical MQTT bootstrap has received + # and correlated DeviceInfo for the exact live peer. TCP reachability + # and a merely opened socket never populate it. + self._verified_control: dict[str, object] | None = None + self._connection_binding: ApplicationConnectionBinding | None = None + self._live_control_binding: LiveDeviceControlBinding | None = None self._outcome_unknown = False + self._inspection_only = False + self._inspection_promotion_allowed = False self._run_generation = 0 + # A generation identifies one MQTT dialogue. The revision identifies + # one exact operator checkpoint inside that dialogue. They are kept + # independent from wall clock and from API snapshot polling so a stale + # browser response cannot advance a later session that happens to be + # in the same named phase. + self._state_revision = 0 def open( self, *, host: str, timezone_name: str, - confirmation: OperatorPresenceConfirmation, + confirmation: OperatorPresenceConfirmation | None = None, + connection_binding: ApplicationConnectionBinding | None = None, + inspection_only: bool = False, ) -> dict[str, object]: - # Validate every explicit confirmation before creating a network owner. - confirmation.checklist(ModelingAction.START) + # Opening the MQTT dialogue begins with only the read-only DeviceInfo + # bootstrap. Durable identity admission must accept that proof before + # a normal session may continue its pre-START preparation; inspection + # sessions remain read-only until explicitly promoted. Physical + # acceptance belongs to the exact START/STOP dispatch boundaries below; + # a connection intent must never fabricate those confirmations merely + # to prove that the selected K1 is ready. + # ``confirmation`` remains accepted for wire compatibility with older + # local clients, but it is deliberately not consumed here. + del confirmation with self._lock: if self._phase != "idle" and self._can_open_locked(): self._reset_locked() @@ -151,9 +289,11 @@ class InteractiveApplicationControlSession: ) self._host = host self._timezone_name = timezone_name - self._set_phase_locked("connecting") + self._connection_binding = connection_binding + self._inspection_only = inspection_only self._run_generation += 1 generation = self._run_generation + self._set_phase_locked("connecting") self._thread = threading.Thread( target=self._run, args=(generation,), @@ -163,15 +303,167 @@ class InteractiveApplicationControlSession: self._thread.start() return self.snapshot() - def enter_workspace(self) -> dict[str, object]: + def adopt_reconciled_scanning( + self, + *, + reconciliation_id: str, + expected_session_generation: int, + expected_state_revision: int, + ) -> dict[str, object]: + """Adopt exact passive SCANNING proof for one later explicit STOP. + + This is a local checkpoint, not a device command. It is accepted only + after the physical coordinator exposes the exact latest reconciliation + for this DeviceInfo-bound control generation. + """ + + if not reconciliation_id.strip(): + raise ApplicationAcceptanceError("active recovery id is required") + self._validate_connection_binding("active-recovery-preflight") + coordinator = self._physical_command_coordinator + if coordinator is None: + raise ApplicationAcceptanceError( + "active recovery requires the durable physical coordinator" + ) + physical = coordinator.snapshot() with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) self._require_phase_locked("connection-ready") + self._require_exact_active_reconciliation_locked( + physical, + reconciliation_id=reconciliation_id, + ) + self._set_phase_locked("active-recovery-requested") + self._active_recovery_requested.set() + return self.snapshot() + + def enter_workspace( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("connection-ready") + if self._inspection_only and not self._inspection_promotion_allowed: + raise ApplicationAcceptanceError( + "read-only inspection has not completed its Verify boundary" + ) + self._validate_connection_binding("workspace-entry-operator-preflight") + with self._lock: + # The binding check deliberately runs outside the session lock. + # Re-check the exact browser checkpoint after it returns so a + # concurrent CLOSE (or any other accepted transition) cannot be + # followed by this now-stale continuation. + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("connection-ready") + if self._inspection_only and not self._inspection_promotion_allowed: + raise ApplicationAcceptanceError( + "read-only inspection has not completed its Verify boundary" + ) self._set_phase_locked("workspace-requested") self._workspace_requested.set() return self.snapshot() - def open_project_prompt(self) -> dict[str, object]: + def release_inspection_for_operator_dialogue( + self, + *, + expected_session_generation: int, + expected_state_revision: int, + ) -> dict[str, object]: + """End Verify's read-only boundary without sending another request.""" + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("connection-ready") + if not self._inspection_only: + return self.snapshot() + if not self._inspection_promotion_allowed: + self._inspection_promotion_allowed = True + self._state_revision += 1 + return self.snapshot() + + def validate_connection_binding(self) -> None: + """Fail closed when the DeviceInfo-bound route lost command authority.""" + + self._validate_connection_binding("operator-preflight") + + def validate_physical_reconciliation_binding(self) -> None: + """Validate read-only recovery proof without requiring command authority. + + An ambiguous durable START/STOP edge deliberately withholds normal + command authority until a fresh DeviceInfo plus a non-retained + DeviceStatus classifies the physical state. Reusing the normal + command validator here creates a circular gate: the proof needed to + resolve the edge is rejected because the edge is not resolved yet. + + This boundary remains fail closed. It requires the exact retained + MQTT control proof, re-samples the bound host route, and requires the + endpoint admission to remain on that exact host-path epoch. The + physical coordinator separately binds DeviceInfo identity and the + fresh DeviceStatus to this exact control generation before committing + any durable reconciliation. It never publishes a command. + """ + + with self._lock: + phase = self._phase + verified_control = self._verified_control + live_control_binding = self._live_control_binding + transport = self._transport + if phase != "connection-ready" or verified_control is None: + raise ApplicationConnectionBindingLost( + "physical reconciliation control proof is unavailable" + ) + if transport is not None: + if live_control_binding is None: + raise ApplicationConnectionBindingLost( + "physical reconciliation identity is unavailable" + ) + try: + transport.validate_control_proof(live_control_binding) + except ApplicationConnectionBindingLost: + raise + except Exception as exc: + if getattr(exc, "reason_code", None) == "control_proof_stale": + raise ApplicationControlProofExpired( + "physical reconciliation control proof is no longer fresh" + ) from exc + raise ApplicationConnectionBindingLost( + "physical reconciliation control proof is no longer fresh" + ) from exc + self._validate_connection_path("physical-reconciliation") + + def open_project_prompt( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("workspace-ready") + self._validate_connection_binding("project-prompt-operator-preflight") + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) self._require_phase_locked("workspace-ready") self._set_phase_locked("project-requested") self._project_requested.set() @@ -182,29 +474,226 @@ class InteractiveApplicationControlSession: *, project_name: str, confirmation: OperatorPresenceConfirmation, + command_context: PhysicalCommandIntentContext | None = None, + preparation_checkpoint_observer: ( + StartPreparationCheckpointObserver | None + ) = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, ) -> dict[str, object]: - with self._lock: - self._require_phase_locked("project-ready") - self._project_name = project_name - self._start_confirmation = confirmation - self._set_phase_locked("start-requested") - self._start_requested.set() + confirmation.checklist(ModelingAction.START) + if self._physical_command_coordinator is not None: + self._validate_connection_binding("start-prepare-preflight") + with self._start_prepare_gate: + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("project-ready") + authority = self._control_authority + binding = self._live_control_binding + if authority is None or binding is None: + raise ApplicationAcceptanceError( + "START durable preparation requires the live control identity" + ) + command = self._start_command(authority, binding, project_name) + coordinator = self._physical_command_coordinator + exact_generation = self._run_generation + exact_revision = self._state_revision + envelope = OneShotPublishEnvelope.from_modeling_command(command) + if coordinator is not None: + if command_context is None: + raise ApplicationAcceptanceError( + "START durable preparation requires operation lineage" + ) + if preparation_checkpoint_observer is None: + raise ApplicationAcceptanceError( + "START durable preparation requires a recovery-checkpoint observer" + ) + try: + coordinator.prepare( + command_context, + action="start", + envelope=envelope, + ) + except PhysicalCommandPrepareSettlementError as settlement_error: + raise ApplicationStartCheckpointSettlementError( + "failed START preparation has unresolved durable state" + ) from settlement_error + try: + # The physical PREPARED row is already durable. The + # observer must fsync the acquisition checkpoint before + # the worker-visible event can be released below. + preparation_checkpoint_observer( + "prepared", + command_context, + envelope, + ) + with self._lock: + if ( + self._run_generation != exact_generation + or self._state_revision != exact_revision + or self._phase != "project-ready" + or self._control_authority is not authority + or self._live_control_binding != binding + ): + raise ApplicationControlStateConflict( + "control session changed during durable START preparation" + ) + self._project_name = project_name + self._start_confirmation = confirmation + self._prepared_start_command = command + self._set_phase_locked("start-requested") + self._start_requested.set() + except BaseException as preparation_error: + try: + coordinator.resolve_prepared_not_dispatched("start") + preparation_checkpoint_observer( + "resolved-not-dispatched", + command_context, + envelope, + ) + except BaseException as settlement_error: + raise ApplicationStartCheckpointSettlementError( + "failed START remained locally unresolved after zero publish" + ) from settlement_error + raise preparation_error + else: + with self._lock: + if ( + self._run_generation != exact_generation + or self._state_revision != exact_revision + or self._phase != "project-ready" + or self._control_authority is not authority + or self._live_control_binding != binding + ): + raise ApplicationControlStateConflict( + "control session changed during START preparation" + ) + self._project_name = project_name + self._start_confirmation = confirmation + self._prepared_start_command = command + self._set_phase_locked("start-requested") + self._start_requested.set() return self.snapshot() def request_stop( self, *, confirmation: OperatorPresenceConfirmation, + command_context: PhysicalCommandIntentContext | None = None, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, ) -> dict[str, object]: - with self._lock: - self._require_phase_locked("scanning") - self._stop_confirmation = confirmation - self._set_phase_locked("stop-requested") - self._stop_requested.set() + confirmation.checklist(ModelingAction.STOP) + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + if self._physical_command_coordinator is not None: + self._validate_connection_binding("stop-prepare-preflight") + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + with self._scanning_transition_gate: + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + # A prior activation observer may have failed after physical START + # was resolved. Retry that storage-only handoff before allowing a + # STOP to replace the ledger head; never publish or prepare STOP + # against a still-PREPARED acquisition checkpoint. + if ( + self._physical_command_coordinator is not None + and not self._scanning_observer_confirmed + and not self._notify_scanning_observer() + ): + raise ApplicationAcceptanceError( + "STOP preparation is blocked until the active-acquisition " + "checkpoint is durably activated" + ) + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self._require_phase_locked("scanning") + authority = self._control_authority + binding = self._live_control_binding + if authority is None or binding is None: + raise ApplicationAcceptanceError( + "STOP durable preparation requires the live control identity" + ) + command = self._stop_command(authority, binding) + coordinator = self._physical_command_coordinator + exact_generation = self._run_generation + exact_revision = self._state_revision + if coordinator is not None: + if command_context is None: + raise ApplicationAcceptanceError( + "STOP durable preparation requires operation lineage" + ) + try: + coordinator.prepare( + command_context, + action="stop", + envelope=OneShotPublishEnvelope.from_modeling_command(command), + ) + except PhysicalCommandPrepareSettlementError as settlement_error: + raise ApplicationStartCheckpointSettlementError( + "failed STOP preparation has unresolved durable state" + ) from settlement_error + try: + # PREPARE may fsync. If the operation deadline elapsed while + # that durable write was in progress, seal this exact row as + # not-dispatched below instead of releasing the worker event. + self._require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached + ) + with self._lock: + if ( + self._run_generation != exact_generation + or self._state_revision != exact_revision + or self._phase != "scanning" + or self._control_authority is not authority + or self._live_control_binding != binding + ): + raise ApplicationControlStateConflict( + "control session changed during durable STOP preparation" + ) + self._stop_confirmation = confirmation + self._prepared_stop_command = command + self._stop_dispatch_admission_deadline_reached = ( + dispatch_admission_deadline_reached + ) + self._set_phase_locked("stop-requested") + self._stop_requested.set() + except BaseException as preparation_error: + if coordinator is not None: + try: + coordinator.resolve_prepared_not_dispatched("stop") + except BaseException as settlement_error: + raise ApplicationStartCheckpointSettlementError( + "failed STOP remained locally unresolved after zero publish" + ) from settlement_error + raise preparation_error return self.snapshot() - def close_prestart(self) -> dict[str, object]: + def close_prestart( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) if self._phase not in { "connection-ready", "workspace-ready", @@ -214,12 +703,23 @@ class InteractiveApplicationControlSession: "control session can be closed safely only between pre-START checkpoints" ) self._cancel_requested = True + # Closing is an accepted operator mutation even though the worker + # owns the eventual terminal phase. Advance the checkpoint now so + # another request that raced with CLOSE cannot continue from the + # same named phase while the socket is being retired. + self._state_revision += 1 transport = self._transport if transport is not None: transport.close() return self.snapshot() - def retire_for_network_change(self) -> dict[str, object]: + def retire_for_network_change( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + allow_terminal_failure: bool = False, + ) -> dict[str, object]: """Retire local control ownership for one new explicit network action. A correlated STOP followed by a network-loop loss in SCAN_STOPPING is @@ -231,13 +731,27 @@ class InteractiveApplicationControlSession: """ with self._lock: + self._require_checkpoint_locked( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) failure = self._failure failed_network_change_admissible = ( self._phase == "failed" and failure is not None and ( - failure.get("safe_to_retry") is True + allow_terminal_failure + # An explicit local reset or BLE scan may retire a dead + # worker even when the preceding physical edge remains + # unresolved. The durable coordinator is deliberately + # not reset here, so no START/STOP authority is invented. + or failure.get("safe_to_retry") is True or failure.get("network_change_admissible") is True + # The canonical socket owner has already proven that the + # local MQTT path is gone. Retiring this dead local worker + # permits a new network intent; it does not resolve or + # authorize any ambiguous physical START/STOP command. + or failure.get("reason_code") == "mqtt_network_loop_failed" ) ) if self._phase not in {"idle", "completed", "closed"} and not ( @@ -247,14 +761,10 @@ class InteractiveApplicationControlSession: "control session cannot be retired for a network change" ) if not self._worker_retired_locked(): - raise ApplicationAcceptanceError( - "control session worker is still retiring" - ) + raise ApplicationAcceptanceError("control session worker is still retiring") previous_phase = self._phase previous_reason = ( - self._json_string(failure.get("reason_code")) - if failure is not None - else None + self._json_string(failure.get("reason_code")) if failure is not None else None ) self._reset_locked() if previous_phase != "idle": @@ -286,7 +796,11 @@ class InteractiveApplicationControlSession: phase = self._phase return { "mode": "interactive-canonical", + "inspection_only": self._inspection_only, + "inspection_promotion_allowed": self._inspection_promotion_allowed, "state": phase, + "session_generation": self._run_generation, + "state_revision": self._state_revision, "control_socket_open": phase not in {"idle", "completed", "closed", "failed"}, "can_open": self._can_open_locked(), "can_enter_workspace": phase == "connection-ready", @@ -306,11 +820,18 @@ class InteractiveApplicationControlSession: dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None ), "transport": transport_snapshot, + "verified_control": (self._verified_control_snapshot_locked(transport_snapshot)), + "physical_command": ( + self._physical_command_coordinator.snapshot() + if self._physical_command_coordinator is not None + else None + ), } def _run(self, generation: int) -> None: executor: PhysicalAcceptanceDialogueExecutor | None = None transport: ReviewedApplicationMqttTransport | None = None + stop_publish_attempts_before_dispatch: int | None = None try: with self._lock: host = self._host @@ -320,6 +841,12 @@ class InteractiveApplicationControlSession: authority = self._authority_loader.load() transport = self._transport_factory(host) + coordinator = self._physical_command_coordinator + if coordinator is not None: + transport.install_evidence_observer(coordinator) + if self._connection_path_validator is not None: + self._validate_connection_path("control-open-preflight") + transport.install_dispatch_guard(self._acquire_connection_dispatch_lease) with self._lock: self._transport = transport transport.open() @@ -329,53 +856,202 @@ class InteractiveApplicationControlSession: timezone_name=timezone_name, ) executor = PhysicalAcceptanceDialogueExecutor(transport) - binding = executor.run_connection_stage(orchestrator) - self._set_phase("connection-ready") + with self._lock: + inspection_only = self._inspection_only + # DeviceInfo (ordinal 1) is the only bootstrap request that may + # cross the socket before durable physical-target admission. In + # particular, ordinal 4 mutates the K1 clock, so the legacy + # collapsed ordinals 1-6 path must never run before the coordinator + # can reject a retired identity discovered under a fresh BLE + # transport UUID. + binding = executor.run_read_only_inspection_stage(orchestrator) + control_session_id = f"application-control-{generation}-{time.monotonic_ns()}" + with self._lock: + connection_binding = self._connection_binding + if coordinator is not None: + if connection_binding is None: + raise ApplicationAcceptanceError( + "durable physical control requires an exact connection binding" + ) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=hash_physical_identity(binding.vendor_device_id), + device_serial_sha256=hash_physical_identity(binding.device_serial), + compatibility_profile_id=COMPATIBILITY_PROFILE_ID, + intent_id=connection_binding.intent_id, + transport_ref=connection_binding.transport_ref, + connection_mode=connection_binding.connection_mode, + target_ipv4=connection_binding.target_ipv4, + target_port=connection_binding.target_port, + host_path_epoch=connection_binding.host_path_epoch, + control_session_id=control_session_id, + producer_generation=generation, + ) + ) + if not inspection_only: + binding = executor.complete_connection_stage( + orchestrator, + expected_binding=binding, + ) + # Publish the identity proof and the phase under one lock so a + # consumer cannot observe connection-ready without its DeviceInfo + # evidence (or evidence while still claiming to be connecting). + with self._lock: + self._control_authority = authority + self._live_control_binding = binding + transport_snapshot = self._live_transport_snapshot_locked() + self._verified_control = { + "logical_device_id": binding.vendor_device_id, + "compatibility_profile_id": COMPATIBILITY_PROFILE_ID, + "control_session_id": control_session_id, + "producer_generation": generation, + "source": "mqtt-device-info", + **self._control_proof_fields(transport_snapshot), + **( + { + "intent_id": connection_binding.intent_id, + "transport_ref": connection_binding.transport_ref, + "host_path_epoch": connection_binding.host_path_epoch, + "target_ipv4": connection_binding.target_ipv4, + "target_port": connection_binding.target_port, + "connection_mode": connection_binding.connection_mode, + } + if connection_binding is not None + else {} + ), + } + self._set_phase_locked("connection-ready") workspace = executor.wait_for_operator_checkpoint( "workspace-entered", self._workspace_requested.is_set, + reconciled_active_observed=self._active_recovery_requested.is_set, ) - executor.run_workspace_entry_stage(orchestrator, workspace) - self._set_phase("workspace-ready") + if workspace is None: + self._validate_connection_binding("active-recovery-adoption") + executor.adopt_reconciled_scanning( + authority=authority, + binding=binding, + ) + with self._scanning_transition_gate: + # Recovery adopts an already-active physical acquisition. + # Its checkpoint deliberately retains an open transport + # gap until the post-Rerun PCL confirmation hook closes it; + # nevertheless an operator STOP must remain admissible in + # this interval and can cease that gap with terminal READY. + self._scanning_observer_confirmed = True + self._set_phase("scanning") + else: + if inspection_only: + self._validate_connection_binding( + "inspection-promotion-pre-dispatch" + ) + binding = executor.complete_connection_stage( + orchestrator, + expected_binding=binding, + ) + self._validate_connection_binding( + "inspection-promotion-post-response" + ) + self._validate_connection_binding("workspace-entry-pre-dispatch") + executor.run_workspace_entry_stage( + orchestrator, + workspace, + dispatch_guard=lambda: self._validate_connection_binding( + "workspace-entry-dispatch" + ), + ) + self._validate_connection_binding("workspace-entry-post-response") + self._set_phase("workspace-ready") - project = executor.wait_for_operator_checkpoint( - "project-prompt-opened", - self._project_requested.is_set, - ) - binding = executor.run_project_prompt_stage(orchestrator, project) - self._set_phase("project-ready") + project = executor.wait_for_operator_checkpoint( + "project-prompt-opened", + self._project_requested.is_set, + ) + assert project is not None + self._validate_connection_binding("project-prompt-pre-dispatch") + binding = executor.run_project_prompt_stage( + orchestrator, + project, + dispatch_guard=lambda: self._validate_connection_binding( + "project-prompt-dispatch" + ), + ) + self._validate_connection_binding("project-prompt-post-response") + self._set_phase("project-ready") - start_checkpoint = executor.wait_for_operator_checkpoint( - "start-confirmed", - self._start_requested.is_set, - ) - project_name, start_confirmation = self._start_request() - start_permit = PhysicalAcceptancePermit( - start_confirmation.checklist(ModelingAction.START) - ) - self._set_phase("initializing") - executor.execute_canonical_start( - self._start_command(authority, binding, project_name), - build_canonical_post_start_observation(authority, binding), - authority=authority, - binding=binding, - permit=start_permit, - checkpoint=start_checkpoint, - ) - self._set_phase("scanning") - self._notify_scanning_observer() + start_checkpoint = executor.wait_for_operator_checkpoint( + "start-confirmed", + self._start_requested.is_set, + ) + assert start_checkpoint is not None + start_command, start_confirmation = self._start_request() + start_permit = PhysicalAcceptancePermit( + start_confirmation.checklist(ModelingAction.START) + ) + self._validate_connection_binding("start-pre-dispatch") + self._set_phase("initializing") + executor.execute_canonical_start( + start_command, + build_canonical_post_start_observation(authority, binding), + authority=authority, + binding=binding, + permit=start_permit, + checkpoint=start_checkpoint, + dispatch_guard=lambda: self._validate_connection_binding( + "start-dispatch" + ), + ) + self._validate_connection_binding("start-post-response") + if coordinator is not None: + coordinator.resolve("start") + with self._scanning_transition_gate: + # Activate the exact durable checkpoint before publishing + # the STOP-admitting SCANNING phase. The observer performs + # storage only; it never sends a device command. + self._scanning_observer_confirmed = ( + self._notify_scanning_observer() + ) + self._set_phase("scanning") executor.maintain_active_until_stop_requested(self._stop_requested.is_set) - stop_confirmation = self._stop_request() + ( + stop_command, + stop_confirmation, + stop_dispatch_admission_deadline_reached, + ) = self._stop_request() stop_permit = PhysicalAcceptancePermit(stop_confirmation.checklist(ModelingAction.STOP)) + # Capture the exact transport counter before any remaining + # read-only validation. A deadline that expires during one of + # those checks is still deterministic zero-publish evidence. + stop_transport_before, stop_transport_before_available = ( + self._transport_snapshot_safely(transport) + ) + if stop_transport_before_available: + stop_publish_attempts_before_dispatch = self._json_int_or_none( + stop_transport_before.get("publish_attempts") + ) + self._require_stop_dispatch_deadline_open( + stop_dispatch_admission_deadline_reached + ) + self._validate_connection_binding("stop-pre-dispatch") + self._require_stop_dispatch_deadline_open( + stop_dispatch_admission_deadline_reached + ) self._set_phase("stopping") executor.execute_canonical_stop( - self._stop_command(authority, binding), + stop_command, stop_permit, + dispatch_guard=lambda: self._validate_connection_binding("stop-dispatch"), + dispatch_admission_deadline_reached=( + stop_dispatch_admission_deadline_reached + ), ) + self._validate_connection_binding("stop-post-response") self._set_phase("awaiting-standby-confirmation") executor.maintain_post_stop_until_standby() + if coordinator is not None: + coordinator.resolve("stop") self._set_phase("completed") except Exception as exc: dialogue_snapshot, dialogue_snapshot_available = self._executor_snapshot_safely( @@ -411,6 +1087,18 @@ class InteractiveApplicationControlSession: if dialogue_snapshot_available else (False if not dialogue_executor_created else None) ) + stop_command_attempted: bool | None = ( + dialogue_snapshot.get("stop_attempted") is True + if dialogue_snapshot_available + else (False if not dialogue_executor_created else None) + ) + stop_publish_attempts = ( + publish_attempts - stop_publish_attempts_before_dispatch + if publish_attempts is not None + and stop_publish_attempts_before_dispatch is not None + and publish_attempts >= stop_publish_attempts_before_dispatch + else None + ) diagnostic_snapshot_unavailable = [ component for component, created, available in ( @@ -438,15 +1126,36 @@ class InteractiveApplicationControlSession: "dialogue.modeling_command_attempted", dialogue_executor_created and modeling_command_attempted is None, ), + ( + "transport.stop_publish_attempts", + failure_reason_code + in { + "physical-command-dispatch-deadline-expired", + "physical-acceptance-permit-expired-before-dispatch", + } + and stop_publish_attempts is None, + ), ) if unavailable ] - outcome_unknown = ( + definite_stop_admission_rejected_before_publish = bool( + failure_reason_code + in { + "physical-command-dispatch-deadline-expired", + "physical-acceptance-permit-expired-before-dispatch", + } + and transport_snapshot_available + and stop_publish_attempts == 0 + and stop_command_attempted is False + and not diagnostic_evidence_unavailable + ) + outcome_unknown = not definite_stop_admission_rejected_before_publish and ( isinstance(exc, ApplicationCommandOutcomeUnknown) or modeling_command_attempted is True or bool(diagnostic_evidence_unavailable) or failed_phase in { + "active-recovery-requested", "scanning", "stop-requested", "stopping", @@ -462,9 +1171,7 @@ class InteractiveApplicationControlSession: failure_reason_code == "compatibility_profile_mismatch" and modeling_command_attempted is False and publish_attempts == 1 - and self._json_int_or_none( - transport_snapshot.get("correlated_responses") - ) + and self._json_int_or_none(transport_snapshot.get("correlated_responses")) == 1 ) status_reconciled_prestart_failure = ( @@ -472,15 +1179,12 @@ class InteractiveApplicationControlSession: and modeling_command_attempted is False and transport_snapshot_available and not diagnostic_evidence_unavailable - and self._json_int_or_none( - transport_snapshot.get("device_status_reports") - ) + and self._json_int_or_none(transport_snapshot.get("device_status_reports")) not in {None, 0} - and self._json_string( - transport_snapshot.get("latest_device_session_state") - ) + and self._json_string(transport_snapshot.get("latest_device_session_state")) == "ready" and transport_snapshot.get("latest_device_project_bound") is False + and transport_snapshot.get("latest_device_init_ready") is False and self._json_int_or_none( transport_snapshot.get("latest_system_error_code") ) @@ -488,7 +1192,19 @@ class InteractiveApplicationControlSession: ) stop_acknowledged_network_change = ( outcome_unknown - and failure_reason_code == "mqtt_network_loop_failed" + # A Mac Wi-Fi/route epoch change invalidates the exact + # control binding just as conclusively as the MQTT + # network loop terminating. Once the correlated STOP + # response and fresh SCAN_STOPPING status are both + # present, either transport-loss class may retire only + # the dead local socket for a later explicit read-only + # Verify. The durable STOP edge remains unresolved; + # this never authorizes replay or another START/STOP. + and failure_reason_code + in { + "mqtt_network_loop_failed", + "application-connection-binding-lost", + } and failed_phase == "awaiting-standby-confirmation" and dialogue_snapshot_available and transport_snapshot_available @@ -496,9 +1212,7 @@ class InteractiveApplicationControlSession: and dialogue_snapshot.get("stop_complete") is True and dialogue_stage == "stop-acknowledged" and modeling_command_attempted is True - and self._json_string( - transport_snapshot.get("latest_device_session_state") - ) + and self._json_string(transport_snapshot.get("latest_device_session_state")) == "scan_stopping" and transport_snapshot.get("latest_device_project_bound") is True and self._json_int_or_none( @@ -512,6 +1226,10 @@ class InteractiveApplicationControlSession: not transport_created or (transport_snapshot_available and publish_attempts == 0) or correlated_read_only_profile_mismatch + or ( + isinstance(exc, ApplicationConnectionBindingLost) + and modeling_command_attempted is False + ) ) ) self._failure = { @@ -522,6 +1240,7 @@ class InteractiveApplicationControlSession: "dialogue_stage": dialogue_stage, "transport_state": self._json_string(transport_snapshot.get("state")), "publish_attempts": publish_attempts, + "stop_publish_attempts": stop_publish_attempts, "qos2_completions": self._json_int_or_none( transport_snapshot.get("qos2_completions") ), @@ -535,6 +1254,7 @@ class InteractiveApplicationControlSession: transport_snapshot.get("late_known_responses") ), "modeling_command_attempted": modeling_command_attempted, + "stop_command_attempted": stop_command_attempted, "diagnostic_snapshot_unavailable": (diagnostic_snapshot_unavailable), "diagnostic_evidence_unavailable": (diagnostic_evidence_unavailable), "correlation_failure": ( @@ -618,10 +1338,9 @@ class InteractiveApplicationControlSession: "reason_code": failure_reason_code, "failed_phase": failed_phase, "dialogue_stage": dialogue_stage, - "transport_state": self._json_string( - transport_snapshot.get("state") - ), + "transport_state": self._json_string(transport_snapshot.get("state")), "publish_attempts": publish_attempts, + "stop_publish_attempts": stop_publish_attempts, "modeling_command_attempted": modeling_command_attempted, "outcome_unknown": outcome_unknown, "safe_to_retry": safe_to_retry, @@ -630,9 +1349,7 @@ class InteractiveApplicationControlSession: if status_reconciled_prestart_failure else None ), - "network_change_admissible": ( - stop_acknowledged_network_change - ), + "network_change_admissible": (stop_acknowledged_network_change), "network_change_reconciliation": ( "explicit-network-change-only-after-acknowledged-stop" if stop_acknowledged_network_change @@ -710,26 +1427,51 @@ class InteractiveApplicationControlSession: ) ) - def _start_request(self) -> tuple[str, OperatorPresenceConfirmation]: + def _start_request(self) -> tuple[ShadowModelingCommand, OperatorPresenceConfirmation]: with self._lock: - if self._project_name is None or self._start_confirmation is None: + if self._prepared_start_command is None or self._start_confirmation is None: raise ApplicationAcceptanceError("START request is incomplete") - return self._project_name, self._start_confirmation + return self._prepared_start_command, self._start_confirmation - def _stop_request(self) -> OperatorPresenceConfirmation: + def _stop_request( + self, + ) -> tuple[ + ShadowModelingCommand, + OperatorPresenceConfirmation, + Callable[[], bool] | None, + ]: with self._lock: - if self._stop_confirmation is None: + if self._prepared_stop_command is None or self._stop_confirmation is None: raise ApplicationAcceptanceError("STOP request is incomplete") - return self._stop_confirmation + return ( + self._prepared_stop_command, + self._stop_confirmation, + self._stop_dispatch_admission_deadline_reached, + ) + + @staticmethod + def _require_stop_dispatch_deadline_open( + dispatch_admission_deadline_reached: Callable[[], bool] | None, + ) -> None: + """Reject STOP only while its physical publish edge is still reversible.""" + + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) def _set_phase(self, phase: ApplicationControlPhase) -> None: with self._lock: self._set_phase_locked(phase) - def _notify_scanning_observer(self) -> None: + def _notify_scanning_observer(self) -> bool: observer = self._scanning_observer if observer is None: - return + return True try: observer() except Exception: @@ -742,6 +1484,105 @@ class InteractiveApplicationControlSession: "automatic_retry": False, }, ) + return False + return True + + def _validate_connection_binding(self, stage: str) -> None: + with self._lock: + binding = self._connection_binding + live_control_binding = self._live_control_binding + transport = self._transport + validator = self._connection_binding_validator + if transport is not None: + if live_control_binding is None: + raise ApplicationConnectionBindingLost( + f"application control identity is unavailable at {stage}" + ) + try: + transport.validate_control_proof(live_control_binding) + except ApplicationConnectionBindingLost: + raise + except Exception as exc: + if getattr(exc, "reason_code", None) == "control_proof_stale": + raise ApplicationControlProofExpired( + f"application control proof is no longer fresh at {stage}" + ) from exc + raise ApplicationConnectionBindingLost( + f"application control proof is no longer fresh at {stage}" + ) from exc + if validator is None: + # Unit integrations predating route supervision remain source + # compatible. Mission Core's real facade always supplies a guard. + return + if binding is None: + raise ApplicationConnectionBindingLost( + f"application control binding is unavailable at {stage}" + ) + try: + accepted = validator(binding) + except ApplicationConnectionBindingLost: + raise + except Exception as exc: + raise ApplicationConnectionBindingLost( + f"application control binding validation failed at {stage}" + ) from exc + if accepted is False: + raise ApplicationConnectionBindingLost( + f"application control binding is no longer authoritative at {stage}" + ) + + def _validate_connection_path(self, stage: str) -> None: + """Validate host-path authority before DeviceInfo identity is available.""" + + with self._lock: + binding = self._connection_binding + validator = self._connection_path_validator + if validator is None: + return + if binding is None: + raise ApplicationConnectionBindingLost( + f"application connection path is unavailable at {stage}" + ) + try: + accepted = validator(binding) + except ApplicationConnectionBindingLost: + raise + except Exception as exc: + raise ApplicationConnectionBindingLost( + f"application connection path validation failed at {stage}" + ) from exc + if accepted is False: + raise ApplicationConnectionBindingLost( + f"application connection path is no longer authoritative at {stage}" + ) + + def _acquire_connection_dispatch_lease(self) -> Callable[[], None] | None: + """Fence one exact MQTT publish against a concurrent network mutation.""" + + with self._lock: + binding = self._connection_binding + lease_factory = self._connection_dispatch_lease + dispatch_admission_deadline_reached = ( + self._stop_dispatch_admission_deadline_reached + ) + if binding is None: + raise ApplicationConnectionBindingLost( + "application connection path is unavailable at control-publish-dispatch" + ) + if lease_factory is None: + self._validate_connection_path("control-publish-dispatch") + return None + try: + return lease_factory( + binding, + dispatch_admission_deadline_reached, + ) + except ApplicationConnectionBindingLost: + raise + except Exception as exc: + raise ApplicationConnectionBindingLost( + "application dispatch lease is unavailable at control-publish-dispatch" + ) from exc @staticmethod def _executor_snapshot_safely( @@ -800,7 +1641,33 @@ class InteractiveApplicationControlSession: return "internal_control_error" def _set_phase_locked(self, phase: ApplicationControlPhase) -> None: + if self._phase == phase: + return self._phase = phase + self._state_revision += 1 + + def _require_checkpoint_locked( + self, + *, + expected_session_generation: int | None, + expected_state_revision: int | None, + ) -> None: + if expected_session_generation is None and expected_state_revision is None: + # Kept only for lower-level compatibility tests and integrations. + # Mission Core's public facade always supplies both values for an + # operator mutation. + return + if expected_session_generation is None or expected_state_revision is None: + raise ApplicationControlStateConflict( + "control checkpoint requires both session generation and state revision" + ) + if ( + expected_session_generation != self._run_generation + or expected_state_revision != self._state_revision + ): + raise ApplicationControlStateConflict( + "control session changed; refresh state before continuing" + ) def _require_phase_locked(self, expected: ApplicationControlPhase) -> None: if self._phase != expected: @@ -808,6 +1675,72 @@ class InteractiveApplicationControlSession: f"control action requires {expected}; current state is {self._phase}" ) + def _require_exact_active_reconciliation_locked( + self, + physical: Mapping[str, object], + *, + reconciliation_id: str, + ) -> None: + verified_control = self._verified_control + record = physical.get("record") + if verified_control is None or not isinstance(record, Mapping): + raise ApplicationAcceptanceError( + "active recovery lacks exact DeviceInfo or physical-ledger proof" + ) + reconciliations = record.get("reconciliations") + if not isinstance(reconciliations, list) or not reconciliations: + raise ApplicationAcceptanceError("active recovery reconciliation is unavailable") + latest = reconciliations[-1] + if not isinstance(latest, Mapping): + raise ApplicationAcceptanceError("active recovery reconciliation is malformed") + verified_binding = latest.get("verified_binding") + observation = latest.get("observation") + connection = ( + verified_binding.get("connection") + if isinstance(verified_binding, Mapping) + else None + ) + if not isinstance(connection, Mapping) or not isinstance(observation, Mapping): + raise ApplicationAcceptanceError("active recovery binding proof is incomplete") + exact_connection_fields = ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + if ( + latest.get("reconciliation_id") != reconciliation_id + or latest.get("kind") + not in { + "ambiguous-outcome", + "prepared-stop-classification", + "resolved-active-rebind", + } + or latest.get("resolution") != "physical-active-observed" + or any( + connection.get(field) != verified_control.get(field) + for field in exact_connection_fields + ) + or observation.get("source") != "explicit-read-only-reconciliation" + or observation.get("control_session_id") + != verified_control.get("control_session_id") + or observation.get("host_path_epoch") + != verified_control.get("host_path_epoch") + or observation.get("producer_generation") + != verified_control.get("producer_generation") + or observation.get("session_state") != "scanning" + or observation.get("project_bound") is not True + or observation.get("init_ready") is not True + or observation.get("mqtt_retained") is not False + ): + raise ApplicationAcceptanceError( + "active recovery proof does not match this exact SCANNING control generation" + ) + def _live_transport_snapshot_locked(self) -> dict[str, object]: if self._transport is not None: snapshot, available = self._transport_snapshot_safely(self._transport) @@ -829,9 +1762,36 @@ class InteractiveApplicationControlSession: } ) + def _verified_control_snapshot_locked( + self, + transport_snapshot: dict[str, object], + ) -> dict[str, object] | None: + if self._verified_control is None: + return None + return { + **self._verified_control, + **self._control_proof_fields(transport_snapshot), + } + + @staticmethod + def _control_proof_fields( + transport_snapshot: dict[str, object], + ) -> dict[str, object]: + revision = transport_snapshot.get("control_proof_revision") + source = transport_snapshot.get("control_proof_source") + fresh = transport_snapshot.get("control_proof_fresh") + return { + "control_proof_revision": ( + revision if isinstance(revision, int) and not isinstance(revision, bool) else 0 + ), + "control_proof_source": source if isinstance(source, str) else None, + "control_proof_fresh": fresh is True, + } + def _pending_operator_action_locked(self) -> str | None: return { "connection-ready": "enter-workspace", + "active-recovery-requested": "adopt-reconciled-scanning", "workspace-ready": "prepare-project", "project-ready": "start", "scanning": "stop", @@ -850,20 +1810,31 @@ class InteractiveApplicationControlSession: return (thread is None or not thread.is_alive()) and self._transport is None def _reset_locked(self) -> None: - self._phase = "idle" + self._set_phase_locked("idle") self._host = None self._timezone_name = None self._thread = None self._transport = None self._workspace_requested = threading.Event() + self._active_recovery_requested = threading.Event() self._project_requested = threading.Event() self._start_requested = threading.Event() self._stop_requested = threading.Event() self._cancel_requested = False self._start_confirmation = None self._stop_confirmation = None + self._stop_dispatch_admission_deadline_reached = None self._project_name = None + self._control_authority = None + self._prepared_start_command = None + self._prepared_stop_command = None self._failure = None self._dialogue_snapshot = None self._transport_snapshot = None + self._verified_control = None + self._connection_binding = None + self._live_control_binding = None self._outcome_unknown = False + self._inspection_only = False + self._inspection_promotion_allowed = False + self._scanning_observer_confirmed = self._scanning_observer is None diff --git a/src/k1link/device_plugins/xgrids_k1/semantic_topology_store.py b/src/k1link/device_plugins/xgrids_k1/semantic_topology_store.py new file mode 100644 index 0000000..b24d250 --- /dev/null +++ b/src/k1link/device_plugins/xgrids_k1/semantic_topology_store.py @@ -0,0 +1,667 @@ +from __future__ import annotations + +import fcntl +import ipaddress +import json +import os +import re +import stat +import tempfile +import threading +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import IO, Final, Literal, cast + +from k1link.sessions.store import resolve_missioncore_data_dir + +SEMANTIC_TOPOLOGY_SCHEMA: Final = "missioncore.xgrids-k1-semantic-topology/v1" +SEMANTIC_TOPOLOGY_FILENAME = "semantic-topology.json" +SEMANTIC_TOPOLOGY_LOCK_FILENAME = ".semantic-topology.lock" +SEMANTIC_TOPOLOGY_MAX_BYTES = 16 * 1024 +SEMANTIC_TOPOLOGY_MAX_REVISION = (1 << 63) - 1 + +TopologyConnectionMode = Literal["bridge", "quick-connect", "direct-connect"] +TopologyEvidenceSource = Literal["ble-post-write-status", "ble-read-only-status"] +SemanticTopologyStoreStatus = Literal["empty", "available", "corrupt"] + +_CONNECTION_MODES = frozenset({"bridge", "quick-connect", "direct-connect"}) +_EVIDENCE_SOURCES = frozenset({"ble-post-write-status", "ble-read-only-status"}) +_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$") +_SAFE_FIRMWARE_VERSION = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+() -]{0,63}$") + + +class SemanticTopologyStoreError(RuntimeError): + """Base error for durable, non-authoritative K1 topology evidence.""" + + reason_code = "semantic-topology-store-error" + + +class SemanticTopologyStoreCorrupt(SemanticTopologyStoreError): + """The on-disk evidence cannot be trusted and must not be adopted.""" + + reason_code = "semantic-topology-store-corrupt" + + +class StaleSemanticTopologyObservation(SemanticTopologyStoreError): + """An asynchronous writer no longer descends from the durable revision.""" + + reason_code = "semantic-topology-observation-stale" + + +@dataclass(frozen=True, slots=True) +class SemanticTopologyRecord: + """Last topology proved by an exact K1 BLE status read. + + This record proves only what the K1 reported at ``observed_at_utc``. It is + deliberately not a host-route, TCP, MQTT, connection, or acquisition lease. + A record loaded after restart is configured/offline evidence until all live + connection-supervisor gates are proved again. + """ + + schema_version: Literal["missioncore.xgrids-k1-semantic-topology/v1"] + revision: int + transport_ref: str + connection_mode: TopologyConnectionMode + ipv4: str + compatibility_profile_id: str + firmware_version: str + source: TopologyEvidenceSource + observed_at_utc: str + + @property + def live_connection_authority(self) -> Literal[False]: + return False + + def as_dict(self) -> dict[str, object]: + return { + "schema_version": self.schema_version, + "revision": self.revision, + "transport_ref": self.transport_ref, + "connection_mode": self.connection_mode, + "ipv4": self.ipv4, + "compatibility_profile_id": self.compatibility_profile_id, + "firmware_version": self.firmware_version, + "source": self.source, + "observed_at_utc": self.observed_at_utc, + } + + +@dataclass(frozen=True, slots=True) +class SemanticTopologySnapshot: + status: SemanticTopologyStoreStatus + record: SemanticTopologyRecord | None + reason_code: str | None + + @property + def configured_offline_evidence(self) -> bool: + return self.status == "available" and self.record is not None + + @property + def live_connection_authority(self) -> Literal[False]: + return False + + def as_dict(self) -> dict[str, object]: + """Project durable evidence without implying a live connection.""" + + return { + "schema_version": SEMANTIC_TOPOLOGY_SCHEMA, + "status": self.status, + "configured_offline_evidence": self.configured_offline_evidence, + "live_connection_authority": self.live_connection_authority, + "reason_code": self.reason_code, + "record": self.record.as_dict() if self.record is not None else None, + } + + +class SemanticTopologyStore: + """Private, atomic store for the last semantically proved K1 topology. + + The stable flock file serializes the complete reload/check/publish + transaction across backend processes. The store remains independent from + the network-mutation ledger: the ledger fences possible writes, while this + store retains only a successfully decoded, secret-free status observation. + """ + + def __init__(self, repository_root: Path) -> None: + data_dir = resolve_missioncore_data_dir(repository_root) + self.path = data_dir / "xgrids-k1" / SEMANTIC_TOPOLOGY_FILENAME + self._lock_path = data_dir / "xgrids-k1" / SEMANTIC_TOPOLOGY_LOCK_FILENAME + self._data_dir = data_dir + self._thread_lock = threading.RLock() + self._record: SemanticTopologyRecord | None = None + self._corrupt = False + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + + def snapshot(self) -> SemanticTopologySnapshot: + """Return configured/offline evidence; never return live authority.""" + + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + return SemanticTopologySnapshot( + status="corrupt", + record=None, + reason_code=SemanticTopologyStoreCorrupt.reason_code, + ) + if self._record is None: + return SemanticTopologySnapshot(status="empty", record=None, reason_code=None) + return SemanticTopologySnapshot( + status="available", + record=self._record, + reason_code=None, + ) + + def commit( + self, + *, + transport_ref: str, + connection_mode: TopologyConnectionMode, + ipv4: str, + compatibility_profile_id: str, + firmware_version: str, + source: TopologyEvidenceSource, + observed_at_utc: str, + predecessor_revision: int | None = None, + ) -> SemanticTopologyRecord: + """Atomically publish one exact status observation. + + Callers must invoke this only after the decoded BLE status has proved + the topology and after the selected transport's compatibility profile + has been attested. ``observed_at_utc`` is audit metadata and never + orders writes; serialized revision publication does. An asynchronous + caller that needs a stale-writer fence supplies the revision from which + its observation descends. No network mutation is performed here. + """ + + _validate_identifier(transport_ref, field_name="transport_ref") + _validate_connection_mode(connection_mode) + canonical_ipv4 = _canonical_ipv4(ipv4) + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + _validate_firmware_version(firmware_version) + _validate_source(source) + observed_at = _validated_timestamp(observed_at_utc, field_name="observed_at_utc") + canonical_observed_at = _timestamp(observed_at) + if predecessor_revision is not None: + _nonnegative_revision(predecessor_revision, field_name="predecessor_revision") + + with self._thread_lock, self._process_lock_locked(): + self._reload_locked() + if self._corrupt: + raise SemanticTopologyStoreCorrupt( + "semantic topology store is corrupt; persisted evidence was not replaced" + ) + current = self._record + current_revision = current.revision if current is not None else 0 + if ( + predecessor_revision is not None + and predecessor_revision != current_revision + ): + raise StaleSemanticTopologyObservation( + "semantic topology predecessor revision is no longer current" + ) + if current is not None and current.revision >= SEMANTIC_TOPOLOGY_MAX_REVISION: + raise SemanticTopologyStoreCorrupt( + "semantic topology revision is exhausted" + ) + revision = 1 if current is None else current.revision + 1 + record = SemanticTopologyRecord( + schema_version=SEMANTIC_TOPOLOGY_SCHEMA, + revision=revision, + transport_ref=transport_ref, + connection_mode=connection_mode, + ipv4=canonical_ipv4, + compatibility_profile_id=compatibility_profile_id, + firmware_version=firmware_version, + source=source, + observed_at_utc=canonical_observed_at, + ) + self._persist_locked(record) + return record + + @contextmanager + def _process_lock_locked(self) -> Iterator[None]: + data_dir_created = _ensure_private_directory(self._data_dir, parents=True) + if data_dir_created: + _fsync_directory(self._data_dir.parent) + store_dir_created = _ensure_private_directory(self.path.parent, parents=False) + if store_dir_created: + _fsync_directory(self._data_dir) + + flags = os.O_RDWR | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(self._lock_path, flags) + lock_created = False + except FileNotFoundError: + try: + descriptor = os.open( + self._lock_path, + flags | os.O_CREAT | os.O_EXCL, + 0o600, + ) + lock_created = True + except FileExistsError: + # A peer may win the create race. Open and validate exactly + # that stable inode instead of introducing a second lock. + try: + descriptor = os.open(self._lock_path, flags) + lock_created = False + except OSError as exc: + raise SemanticTopologyStoreCorrupt( + "semantic topology lock cannot be opened safely" + ) from exc + except OSError as exc: + raise SemanticTopologyStoreCorrupt( + "semantic topology lock cannot be created safely" + ) from exc + except OSError as exc: + raise SemanticTopologyStoreCorrupt( + "semantic topology lock cannot be opened safely" + ) from exc + stream: IO[bytes] | None = None + try: + try: + _validate_private_open_file( + descriptor, + self._lock_path, + label="semantic topology lock", + require_empty=True, + ) + except ValueError as exc: + raise SemanticTopologyStoreCorrupt( + "semantic topology lock is not a stable private file" + ) from exc + stream = os.fdopen(descriptor, "r+b", closefd=True) + descriptor = -1 + fcntl.flock(stream.fileno(), fcntl.LOCK_EX) + try: + try: + _validate_private_open_file( + stream.fileno(), + self._lock_path, + label="semantic topology lock", + require_empty=True, + ) + except ValueError as exc: + raise SemanticTopologyStoreCorrupt( + "semantic topology lock changed while being acquired" + ) from exc + if lock_created: + # The lock file is never replaced. Publish its first + # directory entry before relying on it after a restart. + _fsync_directory(self.path.parent) + yield + finally: + fcntl.flock(stream.fileno(), fcntl.LOCK_UN) + finally: + if stream is not None: + stream.close() + elif descriptor >= 0: + os.close(descriptor) + + def _persist_locked(self, record: SemanticTopologyRecord) -> None: + _write_private_json_atomic( + self.path, + record.as_dict(), + data_dir=self._data_dir, + ) + self._record = record + self._corrupt = False + + def _reload_locked(self) -> None: + try: + payload = _read_private_json(self.path) + except FileNotFoundError: + self._record = None + self._corrupt = False + return + except (OSError, UnicodeError, json.JSONDecodeError, TypeError, ValueError): + self._record = None + self._corrupt = True + return + try: + record = _record_from_mapping(payload) + except (TypeError, ValueError): + self._record = None + self._corrupt = True + return + self._record = record + self._corrupt = False + + +def _read_private_json(path: Path) -> object: + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except FileNotFoundError: + raise + except OSError as exc: + raise ValueError("semantic topology file cannot be opened safely") from exc + try: + metadata = _validate_private_open_file( + descriptor, + path, + label="semantic topology file", + require_empty=False, + ) + if metadata.st_size > SEMANTIC_TOPOLOGY_MAX_BYTES: + raise ValueError("semantic topology file exceeds the bounded size") + chunks: list[bytes] = [] + remaining = SEMANTIC_TOPOLOGY_MAX_BYTES + 1 + while remaining > 0: + chunk = os.read(descriptor, min(remaining, 64 * 1024)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + raw = b"".join(chunks) + if len(raw) > SEMANTIC_TOPOLOGY_MAX_BYTES: + raise ValueError("semantic topology file exceeds the bounded size") + finally: + os.close(descriptor) + return json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_json_object) + + +def _write_private_json_atomic( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, +) -> None: + serialized = ( + json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + "\n" + ).encode("utf-8") + if len(serialized) > SEMANTIC_TOPOLOGY_MAX_BYTES: + raise ValueError("semantic topology payload exceeds the bounded size") + + _ensure_private_directory(data_dir, parents=True) + _ensure_private_directory(path.parent, parents=False) + previous_identity = _existing_private_file_identity(path) + + descriptor, temp_name = tempfile.mkstemp( + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + ) + temp_path = Path(temp_name) + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(serialized) + stream.flush() + os.fsync(stream.fileno()) + _require_unchanged_existing_path(path, previous_identity) + os.replace(temp_path, path) + _fsync_directory(path.parent) + finally: + if descriptor >= 0: + os.close(descriptor) + temp_path.unlink(missing_ok=True) + + +def _existing_private_file_identity(path: Path) -> tuple[int, int] | None: + try: + metadata = path.lstat() + except FileNotFoundError: + return None + _validate_private_metadata(metadata, label="semantic topology file", require_empty=False) + return metadata.st_dev, metadata.st_ino + + +def _require_unchanged_existing_path( + path: Path, + expected: tuple[int, int] | None, +) -> None: + try: + metadata = path.lstat() + except FileNotFoundError: + if expected is None: + return + raise ValueError("semantic topology file disappeared during publication") from None + _validate_private_metadata(metadata, label="semantic topology file", require_empty=False) + observed = (metadata.st_dev, metadata.st_ino) + if expected is None or observed != expected: + raise ValueError("semantic topology file changed during publication") + + +def _validate_private_open_file( + descriptor: int, + path: Path, + *, + label: str, + require_empty: bool, +) -> os.stat_result: + metadata = os.fstat(descriptor) + _validate_private_metadata(metadata, label=label, require_empty=require_empty) + _validate_open_path_identity(descriptor, path, metadata, label=label) + return metadata + + +def _validate_open_path_identity( + descriptor: int, + path: Path, + metadata: os.stat_result, + *, + label: str, +) -> None: + del descriptor # metadata already came from this exact descriptor + try: + path_metadata = path.lstat() + except OSError as exc: + raise ValueError(f"{label} path cannot be verified") from exc + if (path_metadata.st_dev, path_metadata.st_ino) != (metadata.st_dev, metadata.st_ino): + raise ValueError(f"{label} path does not reference the opened inode") + _validate_private_metadata(path_metadata, label=label, require_empty=False) + + +def _validate_private_metadata( + metadata: os.stat_result, + *, + label: str, + require_empty: bool, +) -> None: + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"{label} is not a regular file") + if stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError(f"{label} is not private") + if metadata.st_nlink != 1: + raise ValueError(f"{label} has an unsafe hard link") + if require_empty and metadata.st_size != 0: + raise ValueError(f"{label} must remain empty") + + +def _ensure_private_directory(path: Path, *, parents: bool) -> bool: + try: + metadata = path.lstat() + except FileNotFoundError: + try: + path.mkdir(mode=0o700, parents=parents, exist_ok=False) + except FileExistsError: + metadata = path.lstat() + else: + path.chmod(0o700) + return True + if not stat.S_ISDIR(metadata.st_mode): + raise SemanticTopologyStoreCorrupt( + "semantic topology directory is not a regular private directory" + ) + if stat.S_IMODE(metadata.st_mode) != 0o700: + raise SemanticTopologyStoreCorrupt( + "semantic topology directory permissions are not private" + ) + return False + + +def _fsync_directory(path: Path) -> None: + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) + descriptor = os.open(path, flags) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + raise ValueError("semantic topology store contains duplicate fields") + document[key] = value + return document + + +def _record_from_mapping(value: object) -> SemanticTopologyRecord: + document = _exact_mapping( + value, + { + "schema_version", + "revision", + "transport_ref", + "connection_mode", + "ipv4", + "compatibility_profile_id", + "firmware_version", + "source", + "observed_at_utc", + }, + label="semantic topology", + ) + if document["schema_version"] != SEMANTIC_TOPOLOGY_SCHEMA: + raise ValueError("unsupported semantic topology schema") + revision = _positive_revision(document["revision"]) + transport_ref = _required_string(document["transport_ref"], field_name="transport_ref") + _validate_identifier(transport_ref, field_name="transport_ref") + mode_raw = _required_string(document["connection_mode"], field_name="connection_mode") + _validate_connection_mode(mode_raw) + source_raw = _required_string(document["source"], field_name="source") + _validate_source(source_raw) + compatibility_profile_id = _required_string( + document["compatibility_profile_id"], + field_name="compatibility_profile_id", + ) + _validate_identifier( + compatibility_profile_id, + field_name="compatibility_profile_id", + ) + firmware_version = _required_string( + document["firmware_version"], + field_name="firmware_version", + ) + _validate_firmware_version(firmware_version) + observed_at = _validated_timestamp( + document["observed_at_utc"], + field_name="observed_at_utc", + ) + observed_at_utc = _timestamp(observed_at) + if document["observed_at_utc"] != observed_at_utc: + raise ValueError("observed_at_utc is not canonical") + return SemanticTopologyRecord( + schema_version=SEMANTIC_TOPOLOGY_SCHEMA, + revision=revision, + transport_ref=transport_ref, + connection_mode=cast(TopologyConnectionMode, mode_raw), + ipv4=_canonical_ipv4(_required_string(document["ipv4"], field_name="ipv4")), + compatibility_profile_id=compatibility_profile_id, + firmware_version=firmware_version, + source=cast(TopologyEvidenceSource, source_raw), + observed_at_utc=observed_at_utc, + ) + + +def _exact_mapping(value: object, keys: set[str], *, label: str) -> Mapping[str, object]: + if not isinstance(value, dict) or set(value) != keys: + raise ValueError(f"{label} does not match the secret-free schema") + return cast(Mapping[str, object], value) + + +def _required_string(value: object, *, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{field_name} must be a non-empty string") + return value + + +def _validate_identifier(value: str, *, field_name: str) -> None: + if not isinstance(value, str) or _SAFE_IDENTIFIER.fullmatch(value) is None: + raise ValueError(f"{field_name} is outside the secret-free identifier schema") + + +def _validate_firmware_version(value: str) -> None: + if not isinstance(value, str) or _SAFE_FIRMWARE_VERSION.fullmatch(value) is None: + raise ValueError("firmware_version is outside the bounded version schema") + + +def _validate_connection_mode(value: str) -> None: + if not isinstance(value, str) or value not in _CONNECTION_MODES: + raise ValueError("unsupported semantic topology connection mode") + + +def _validate_source(value: str) -> None: + if not isinstance(value, str) or value not in _EVIDENCE_SOURCES: + raise ValueError("unsupported semantic topology evidence source") + + +def _canonical_ipv4(value: str) -> str: + if not isinstance(value, str) or len(value) > 15: + raise ValueError("semantic topology address must be canonical IPv4") + try: + address = ipaddress.ip_address(value) + except ValueError as exc: + raise ValueError("semantic topology address must be an IPv4 address") from exc + if not isinstance(address, ipaddress.IPv4Address) or str(address) != value: + raise ValueError("semantic topology address must be canonical IPv4") + return value + + +def _positive_revision(value: object) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + or value > SEMANTIC_TOPOLOGY_MAX_REVISION + ): + raise ValueError("revision must be a bounded positive integer") + return value + + +def _nonnegative_revision(value: object, *, field_name: str) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value < 0 + or value > SEMANTIC_TOPOLOGY_MAX_REVISION + ): + raise ValueError(f"{field_name} must be a bounded nonnegative integer") + return value + + +def _validated_timestamp(value: object, *, field_name: str) -> datetime: + raw = _required_string(value, field_name=field_name) + if len(raw) > 32 or not raw.endswith("Z"): + raise ValueError(f"{field_name} must be UTC") + try: + parsed = datetime.fromisoformat(raw.removesuffix("Z") + "+00:00") + except ValueError as exc: + raise ValueError(f"{field_name} is invalid") from exc + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): + raise ValueError(f"{field_name} must be UTC") + return parsed.astimezone(UTC) + + +def _timestamp(value: datetime) -> str: + return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") diff --git a/src/k1link/device_plugins/xgrids_k1/viewer/messages.py b/src/k1link/device_plugins/xgrids_k1/viewer/messages.py index 33642bc..7d5d3f4 100644 --- a/src/k1link/device_plugins/xgrids_k1/viewer/messages.py +++ b/src/k1link/device_plugins/xgrids_k1/viewer/messages.py @@ -13,3 +13,8 @@ class StreamMessage: received_at_epoch_ns: int received_monotonic_ns: int | None = None source: str = "replay" + # Assigned by VisualizationRuntime at the acquisition boundary. A raw + # replay/capture message may be unbound, but an observer must never treat a + # message from an older producer generation as evidence for the current + # device session. + producer_generation: int | None = None diff --git a/src/k1link/device_plugins/xgrids_k1/viewer/runtime.py b/src/k1link/device_plugins/xgrids_k1/viewer/runtime.py index 88722a2..d2c737d 100644 --- a/src/k1link/device_plugins/xgrids_k1/viewer/runtime.py +++ b/src/k1link/device_plugins/xgrids_k1/viewer/runtime.py @@ -1,11 +1,13 @@ from __future__ import annotations +import logging import math import queue import threading import time from collections.abc import Callable from contextlib import suppress +from dataclasses import replace from datetime import UTC, datetime from pathlib import Path from typing import Literal, Protocol, TypedDict @@ -15,8 +17,12 @@ from k1link.compute.live_perception import ( LivePerceptionResultFrame, decode_live_perception_result, ) -from k1link.data_plane import DecodedDataPlaneView, NormalizationError +from k1link.data_plane import DecodedDataPlaneView, DecodedPointCloudView, NormalizationError from k1link.device_plugins.xgrids_k1.mqtt import CapturedMqttMessage, CaptureError, capture_mqtt +from k1link.device_plugins.xgrids_k1.mqtt.capture import ( + RECOVERY_POINT_CLOUD_TOPICS, + RecoveryDecision, +) from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage from k1link.device_plugins.xgrids_k1.viewer.replay import iter_replay_messages from k1link.viewer.metrics import BridgeMetrics, MetricsSnapshot @@ -26,20 +32,29 @@ RuntimePhase = Literal[ "idle", "starting_live", "live", + "reconnecting", "replay", "stopping", "error", ] LIVE_CAPTURE_CLOCK_READY_TIMEOUT_SECONDS = 20.0 +LIVE_RERUN_PUBLISHER_JOIN_SECONDS = 0.5 SourceMode = Literal["idle", "live", "replay"] +RerunRecoveryState = Literal["inactive", "starting", "ready", "retrying", "stopped"] StateCallback = Callable[[], None] +ConnectionRecovery = Callable[[int], RecoveryDecision] +RecoveryConfirmer = Callable[[int], bool] +RecoveryPromotionCheckpoint = Callable[[DecodedPointCloudView, int, int, int], bool] BridgeFactory = Callable[..., RerunBridge] -# TODO: replace the mixed-modality FIFO with a latest point-cloud slot and a -# bounded pose queue. The compact queue protects acquisition from a slow -# visualizer, but under sustained pressure it can still evict pose messages. +PublishedEnvelopeObserver = Callable[[DecodedDataPlaneView, int], None] +# Live preview keeps point clouds in a separate latest-wins slot so pose/status +# traffic cannot evict the frame that proves visible-scene recovery. Replay +# deliberately retains its historical mixed FIFO and terminal bridge behavior. PREVIEW_QUEUE_SIZE = 4 PERCEPTION_PREVIEW_QUEUE_SIZE = 2 +logger = logging.getLogger(__name__) + class CanonicalNormalizer(Protocol): def __call__( @@ -59,11 +74,28 @@ class RuntimeSnapshot(TypedDict): message: str source_mode: SourceMode source_ready: bool + producer_generation: int foxglove_ws_url: str | None foxglove_viewer_url: str | None rerun_grpc_url: str | None viewer_settings: dict[str, object] metrics: MetricsSnapshot + connection_recovery: dict[str, object] + rerun_recovery: dict[str, object] + + +def _rerun_recovery_backoff_seconds(attempt: int) -> float: + """Return a capped retry delay without evaluating an unbounded exponent.""" + + if attempt <= 1: + return 0.5 + if attempt == 2: + return 1.0 + if attempt == 3: + return 2.0 + if attempt == 4: + return 4.0 + return 5.0 class VisualizationRuntime: @@ -77,15 +109,18 @@ class VisualizationRuntime: bridge_factory: BridgeFactory | None = None, normalizer: CanonicalNormalizer, message_observer: RawMessageObserver | None = None, + published_envelope_observer: PublishedEnvelopeObserver | None = None, + recovery_promotion_checkpoint: RecoveryPromotionCheckpoint | None = None, ) -> None: self._lock = threading.Lock() self._on_state_change = on_state_change self._thread: threading.Thread | None = None self._stop_event = threading.Event() self._phase: RuntimePhase = "idle" - self._message = "Готово. Включите устройство и начните с поиска по Bluetooth." + self._message = "Активного источника сейчас нет." self._source_mode: SourceMode = "idle" self._source_ready = False + self._producer_generation = 0 self._foxglove_ws_url: str | None = None self._foxglove_viewer_url: str | None = None self._rerun_grpc_url: str | None = None @@ -93,26 +128,81 @@ class VisualizationRuntime: self._bridge_factory = bridge_factory or RerunBridge self._normalizer = normalizer self._message_observer = message_observer + self._published_envelope_observer = published_envelope_observer + self._recovery_promotion_checkpoint = recovery_promotion_checkpoint self._bridge: RerunBridge | None = None + # At most one irrecoverably blocked live publisher may outlive its + # session. A later live session quarantines presentation until that + # exact daemon exits instead of allocating another native Rerun owner. + self._quarantined_live_publisher: threading.Thread | None = None self._closed = False self._scene_settings = RerunSceneSettings() self._metrics = BridgeMetrics() + self._connection_recovery_state: Literal[ + "inactive", "reconnecting", "blocked", "recovered", "standby", "fault" + ] = "inactive" + self._connection_recovery_attempt = 0 + self._connection_recovery_reason_code: str | None = None + self._connection_recovery_started_at_utc: str | None = None + self._connection_recovery_started_monotonic: float | None = None + self._connection_recovered_at_utc: str | None = None + self._connection_recovery_enabled = False + self._connection_recovery_request_generation: int | None = None + self._connection_recovery_request_reason: str | None = None + self._connection_recovery_candidate_generation: int | None = None + self._connection_recovery_candidate_attempt: int | None = None + self._connection_recovery_candidate_min_sequence: int | None = None + self._connection_recovery_confirmer_generation: int | None = None + self._connection_recovery_confirmer: RecoveryConfirmer | None = None + self._connection_recovery_confirmation_claim: tuple[int, int, int] | None = None + self._rerun_recovery_state: RerunRecoveryState = "inactive" + self._rerun_recovery_attempt = 0 + self._rerun_recovery_reason_code: str | None = None self._perception_messages: queue.Queue[LivePerceptionResultFrame] = queue.Queue( maxsize=PERCEPTION_PREVIEW_QUEUE_SIZE ) def snapshot(self) -> RuntimeSnapshot: with self._lock: + recovery_elapsed_ms = ( + int( + max( + time.monotonic() - self._connection_recovery_started_monotonic, + 0.0, + ) + * 1000 + ) + if self._connection_recovery_started_monotonic is not None + and self._connection_recovery_state in {"reconnecting", "blocked"} + else None + ) return { "phase": self._phase, "message": self._message, "source_mode": self._source_mode, "source_ready": self._source_ready, + "producer_generation": self._producer_generation, "foxglove_ws_url": self._foxglove_ws_url, "foxglove_viewer_url": self._foxglove_viewer_url, "rerun_grpc_url": self._rerun_grpc_url, "viewer_settings": self._scene_settings.as_dict(), "metrics": self._metrics.snapshot(), + "connection_recovery": { + "state": self._connection_recovery_state, + "attempt": self._connection_recovery_attempt, + "reason_code": self._connection_recovery_reason_code, + "started_at_utc": self._connection_recovery_started_at_utc, + "elapsed_ms": recovery_elapsed_ms, + "recovered_at_utc": self._connection_recovered_at_utc, + "automatic_command_retry": False, + "device_write_performed": False, + "network_mutation_performed": False, + }, + "rerun_recovery": { + "state": self._rerun_recovery_state, + "attempt": self._rerun_recovery_attempt, + "reason_code": self._rerun_recovery_reason_code, + }, } def update_scene_settings(self, settings: RerunSceneSettings) -> RuntimeSnapshot: @@ -124,7 +214,11 @@ class VisualizationRuntime: def publish_perception_result(self, encoded: bytes) -> bool: """Admit one validated latest-wins AI result without blocking acquisition.""" - frame = decode_live_perception_result(encoded) + return self.publish_perception_frame(decode_live_perception_result(encoded)) + + def publish_perception_frame(self, frame: LivePerceptionResultFrame) -> bool: + """Admit one already validated, acquisition-bound perception frame.""" + with self._lock: active = self._source_mode in {"live", "replay"} and not self._closed if not active: @@ -163,7 +257,13 @@ class VisualizationRuntime: source_mode="replay", phase="replay", message=f"Запускаем повтор записи: {resolved.name}", - target=lambda: self._run_replay(resolved, speed=speed, loop=loop), + connection_recovery_enabled=False, + target=lambda generation: self._run_replay( + resolved, + speed=speed, + loop=loop, + generation=generation, + ), ) def start_live( @@ -173,6 +273,7 @@ class VisualizationRuntime: *, duration_seconds: float | None = None, project_name: str, + recover_connection: ConnectionRecovery | None = None, ) -> None: if duration_seconds is not None and ( not math.isfinite(duration_seconds) or duration_seconds <= 0 @@ -183,12 +284,15 @@ class VisualizationRuntime: source_mode="live", phase="starting_live", message="Запускаем приём MQTT и локальный мост визуализации.", - target=lambda: self._run_live( + connection_recovery_enabled=recover_connection is not None, + target=lambda generation: self._run_live( host, out_dir.expanduser().resolve(), duration_seconds=duration_seconds, project_name=project_name, clock_established=clock_established, + generation=generation, + recover_connection=recover_connection, ), ) deadline = time.monotonic() + LIVE_CAPTURE_CLOCK_READY_TIMEOUT_SECONDS @@ -206,11 +310,15 @@ class VisualizationRuntime: def stop(self, *, wait_seconds: float = 5.0) -> None: notify_only = False with self._lock: + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None thread = self._thread if thread is None or not thread.is_alive(): self._phase = "idle" self._source_mode = "idle" self._source_ready = False + self._clear_connection_recovery_locked() + self._clear_rerun_recovery_locked() self._message = "Активного потока нет." notify_only = True else: @@ -229,6 +337,8 @@ class VisualizationRuntime: """Stop the active source and release its acquisition-scoped bridge.""" with self._lock: self._closed = True + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None thread = self._thread if thread is not None and thread.is_alive(): self._phase = "stopping" @@ -238,6 +348,8 @@ class VisualizationRuntime: self._phase = "idle" self._source_mode = "idle" self._source_ready = False + self._clear_connection_recovery_locked() + self._clear_rerun_recovery_locked() self._message = "Локальный поток завершён." self._notify() @@ -271,7 +383,8 @@ class VisualizationRuntime: source_mode: SourceMode, phase: RuntimePhase, message: str, - target: Callable[[], None], + connection_recovery_enabled: bool, + target: Callable[[int], None], ) -> None: with self._lock: if self._closed: @@ -279,15 +392,27 @@ class VisualizationRuntime: if self._thread is not None and self._thread.is_alive(): raise RuntimeError("поток уже запущен; сначала остановите текущую сессию") self._stop_event = threading.Event() + if ( + self._quarantined_live_publisher is not None + and not self._quarantined_live_publisher.is_alive() + ): + self._quarantined_live_publisher = None self._metrics = BridgeMetrics() + self._clear_connection_recovery_locked() + self._clear_rerun_recovery_locked() + self._connection_recovery_enabled = connection_recovery_enabled + self._producer_generation += 1 + generation = self._producer_generation self._phase = phase self._source_mode = source_mode self._source_ready = False + if source_mode == "live": + self._rerun_recovery_state = "starting" self._message = message self._foxglove_ws_url = None self._foxglove_viewer_url = None self._thread = threading.Thread( - target=lambda: self._run_target_safely(target, source_mode), + target=lambda: self._run_target_safely(target, source_mode, generation), name=f"k1-{source_mode}-session", daemon=True, ) @@ -296,21 +421,32 @@ class VisualizationRuntime: def _run_target_safely( self, - target: Callable[[], None], + target: Callable[[int], None], source_mode: SourceMode, + generation: int, ) -> None: """Convert setup failures before the pipeline into observable runtime state.""" try: - target() + target(generation) except BaseException as exc: with self._lock: closed = self._closed if closed: return - self._finish_error(f"Ошибка {source_mode}-источника: {type(exc).__name__}: {exc}") + self._finish_error( + f"Ошибка {source_mode}-источника: {type(exc).__name__}: {exc}", + generation=generation, + ) - def _run_replay(self, path: Path, *, speed: float, loop: bool) -> None: + def _run_replay( + self, + path: Path, + *, + speed: float, + loop: bool, + generation: int, + ) -> None: def produce(put: Callable[[StreamMessage], None]) -> str: while not self._stop_event.is_set(): first_source_ns: int | None = None @@ -337,7 +473,7 @@ class VisualizationRuntime: return f"Повтор завершён: обработано сообщений MQTT — {count}." return "Повтор записи остановлен." - self._run_pipeline(produce, running_phase="replay") + self._run_pipeline(produce, running_phase="replay", generation=generation) def _run_live( self, @@ -347,11 +483,21 @@ class VisualizationRuntime: duration_seconds: float | None, project_name: str, clock_established: threading.Event, + generation: int, + recover_connection: ConnectionRecovery | None, ) -> None: _write_live_session_preamble(out_dir, host, duration_seconds, project_name) def produce(put: Callable[[StreamMessage], None]) -> str: def on_message(message: CapturedMqttMessage) -> None: + if message.retain and message.topic in RECOVERY_POINT_CLOUD_TOPICS: + # The capture writer has already persisted this frame and + # its MQTT retain bit before invoking us. Keep that raw + # evidence, but never let a broker-retained point cloud + # become a visible frame or the post-publish proof that a + # restarted/reconnected receiver is fresh. Non-PCL report + # topics retain their existing viewer behavior. + return put( StreamMessage( sequence=message.sequence, @@ -370,15 +516,67 @@ class VisualizationRuntime: on_clock_established=clock_established.set, on_ready=lambda: self._set_running( "live", - "Приём запущен. Теперь дважды нажмите физическую кнопку устройства.", + "Локальный приём запущен. Ожидаем подтверждённое состояние K1.", + generation=generation, ), on_message_recorded=on_message, should_stop=self._stop_event.is_set, + on_connection_lost=( + ( + lambda reason: self._set_connection_reconnecting( + reason, + generation=generation, + ) + ) + if recover_connection is not None + else None + ), + consume_connection_recovery_request=( + ( + lambda: self._consume_connection_recovery_request( + generation=generation, + ) + ) + if recover_connection is not None + else None + ), + recover_connection=( + ( + lambda attempt: self._run_connection_recovery_attempt( + recover_connection, + attempt, + generation=generation, + ) + ) + if recover_connection is not None + else None + ), + on_recovery_point_cloud_candidate=( + lambda attempt, minimum_sequence: ( + self._arm_connection_recovery_point_cloud_candidate( + attempt, + minimum_sequence, + generation=generation, + ) + ) + if recover_connection is not None + else None + ), + on_recovery_confirmer_ready=( + ( + lambda confirmer: self._install_connection_recovery_confirmer( + confirmer, + generation=generation, + ) + ) + if recover_connection is not None + else None + ), ) return f"Приём остановлен. Сохранено сообщений: {summary['message_count']}." try: - self._run_pipeline(produce, running_phase="live") + self._run_pipeline(produce, running_phase="live", generation=generation) finally: _finalize_live_session(out_dir, self.snapshot()) @@ -387,8 +585,13 @@ class VisualizationRuntime: producer: Callable[[Callable[[StreamMessage], None]], str], *, running_phase: RuntimePhase, + generation: int, ) -> None: + live_supervised = running_phase == "live" messages: queue.Queue[StreamMessage] = queue.Queue(maxsize=PREVIEW_QUEUE_SIZE) + latest_live_point_cloud: list[StreamMessage] = [] + latest_live_point_cloud_lock = threading.Lock() + live_point_cloud_ready = threading.Event() source_done = threading.Event() publisher_ready = threading.Event() publisher_aborted = threading.Event() @@ -401,6 +604,10 @@ class VisualizationRuntime: break def enqueue(message: StreamMessage) -> None: + with self._lock: + if self._closed or generation != self._producer_generation: + return + message = replace(message, producer_generation=generation) # A plugin may consume non-visual status before the bounded preview # FIFO. This keeps control telemetry from evicting visual frames # without embedding a vendor decoder in the visual runtime. @@ -409,6 +616,15 @@ class VisualizationRuntime: ): self._notify() return + if live_supervised and message.topic in RECOVERY_POINT_CLOUD_TOPICS: + with latest_live_point_cloud_lock: + if latest_live_point_cloud: + latest_live_point_cloud[0] = message + self._metrics.preview_dropped() + else: + latest_live_point_cloud.append(message) + live_point_cloud_ready.set() + return try: messages.put_nowait(message) return @@ -425,11 +641,123 @@ class VisualizationRuntime: except queue.Full: self._metrics.preview_dropped() - def publish() -> None: + def pop_latest_live_point_cloud() -> StreamMessage | None: + with latest_live_point_cloud_lock: + if not latest_live_point_cloud: + live_point_cloud_ready.clear() + return None + message = latest_live_point_cloud.pop() + live_point_cloud_ready.clear() + return message + + def preview_empty() -> bool: + return ( + messages.empty() + and not live_point_cloud_ready.is_set() + and self._perception_messages.empty() + ) + + def publish_message(bridge: RerunBridge, message: StreamMessage) -> None: + processing_started_ns = time.monotonic_ns() + self._metrics.received(len(message.payload)) + try: + envelope = self._normalizer( + message, + processing_started_monotonic_ns=processing_started_ns, + ) + except NormalizationError: + self._metrics.decode_error() + else: + if envelope is not None: + # Recovery confirmation is deliberately downstream of this + # call. A failed Rerun frame is discarded and cannot promote + # MQTT recovery. + bridge.process(envelope) + self._confirm_connection_recovery_from_published_point_cloud( + envelope, + generation=generation, + ) + observer = self._published_envelope_observer + if observer is not None: + try: + observer(envelope, generation) + except Exception: + logger.exception( + "post-publication runtime observer failed", + extra={ + "event_code": ( + "visualization_post_publish_observer_failed" + ), + "producer_generation": generation, + }, + ) + if self._metrics.snapshot()["messages_received"] % 10 == 0: + self._notify() + + def publish_stream(bridge: RerunBridge) -> None: + serve_perception_next = False + while ( + not source_done.is_set() + or not preview_empty() + ): + if live_supervised and self._stop_event.is_set(): + return + if ( + self._stop_event.is_set() + and source_done.is_set() + and preview_empty() + ): + return + # A dedicated latest-PCL slot plus this fixed service order + # guarantees visible-scene progress even under an unbounded + # stream of perception results. Other MQTT preview traffic is + # then served before one perception result, so neither class can + # monopolize the single publisher thread. + live_point_cloud = pop_latest_live_point_cloud() + if live_point_cloud is not None: + publish_message(bridge, live_point_cloud) + continue + if serve_perception_next: + try: + perception = self._perception_messages.get_nowait() + except queue.Empty: + perception = None + if perception is not None: + try: + bridge.process_perception(perception) + finally: + self._perception_messages.task_done() + serve_perception_next = False + continue + try: + message = messages.get_nowait() + except queue.Empty: + message = None + if message is not None: + try: + publish_message(bridge, message) + finally: + messages.task_done() + serve_perception_next = True + continue + try: + perception = self._perception_messages.get_nowait() + except queue.Empty: + perception = None + if perception is not None: + try: + bridge.process_perception(perception) + finally: + self._perception_messages.task_done() + serve_perception_next = False + continue + live_point_cloud_ready.wait(timeout=0.05) + + def publish_terminal() -> None: bridge: RerunBridge | None = None try: with self._lock: - if self._closed: + if self._closed or generation != self._producer_generation: publisher_aborted.set() if publisher_aborted.is_set(): publisher_ready.set() @@ -441,7 +769,7 @@ class VisualizationRuntime: ) bridge = candidate with self._lock: - if self._closed: + if self._closed or generation != self._producer_generation: publisher_aborted.set() else: # Every acquisition owns a fresh Rerun recording/store. @@ -454,11 +782,16 @@ class VisualizationRuntime: assert bridge is not None bridge.begin_session(self._metrics) with self._lock: - if self._closed: + if self._closed or generation != self._producer_generation: publisher_aborted.set() else: self._rerun_grpc_url = bridge.grpc_url - if running_phase == "replay" and not self._closed and self._phase != "stopping": + if ( + running_phase == "replay" + and not self._closed + and generation == self._producer_generation + and self._phase != "stopping" + ): self._phase = running_phase self._source_ready = True self._message = "Локальный Rerun-мост готов; источник данных запущен." @@ -466,49 +799,7 @@ class VisualizationRuntime: self._notify() if publisher_aborted.is_set(): return - while ( - not source_done.is_set() - or not messages.empty() - or not self._perception_messages.empty() - ): - if ( - self._stop_event.is_set() - and source_done.is_set() - and messages.empty() - and self._perception_messages.empty() - ): - break - try: - perception = self._perception_messages.get_nowait() - except queue.Empty: - perception = None - if perception is not None: - try: - bridge.process_perception(perception) - finally: - self._perception_messages.task_done() - continue - try: - message = messages.get(timeout=0.05) - except queue.Empty: - continue - try: - processing_started_ns = time.monotonic_ns() - self._metrics.received(len(message.payload)) - try: - envelope = self._normalizer( - message, - processing_started_monotonic_ns=processing_started_ns, - ) - except NormalizationError: - self._metrics.decode_error() - else: - if envelope is not None: - bridge.process(envelope) - finally: - messages.task_done() - if self._metrics.snapshot()["messages_received"] % 10 == 0: - self._notify() + publish_stream(bridge) except BaseException as exc: with self._lock: closed = self._closed @@ -533,79 +824,718 @@ class VisualizationRuntime: publisher_error.append(exc) self._notify() - def join_publisher() -> None: + def live_generation_active() -> bool: + if self._stop_event.is_set() or source_done.is_set(): + return False + with self._lock: + return ( + not self._closed + and generation == self._producer_generation + and self._source_mode == "live" + and self._phase != "stopping" + ) + + def wait_live_retry(attempt: int) -> bool: + deadline = time.monotonic() + _rerun_recovery_backoff_seconds(attempt) + while live_generation_active(): + remaining = deadline - time.monotonic() + if remaining <= 0: + return True + if self._stop_event.wait(min(0.1, remaining)): + return False + return False + + def wait_for_quarantined_live_publisher() -> bool: + """Wait interruptibly without creating a second native Rerun owner.""" + + announced = False + while live_generation_active(): + with self._lock: + quarantined = self._quarantined_live_publisher + if quarantined is not None and not quarantined.is_alive(): + if self._quarantined_live_publisher is quarantined: + self._quarantined_live_publisher = None + quarantined = None + if quarantined is None: + return True + if not announced: + self._set_live_rerun_recovery( + "retrying", + attempt=0, + reason_code="publisher-quarantined", + generation=generation, + ) + announced = True + if self._stop_event.wait(0.1): + return False + return False + + def close_live_bridge( + bridge: RerunBridge, + *, + attempt: int, + active: bool, + reason_code: str | None, + ) -> str | None: + self._retire_live_rerun_bridge( + bridge, + generation=generation, + state="retrying" if active else "stopped", + attempt=attempt, + reason_code=reason_code, + ) + try: + bridge.close() + except BaseException: + reason_code = "bridge-close-failed" + self._set_live_rerun_recovery( + "retrying" if active else "stopped", + attempt=attempt, + reason_code=reason_code, + generation=generation, + ) + logger.exception( + "live Rerun bridge close failed; raw MQTT capture remains active", + extra={ + "event_code": "live_rerun_bridge_close_failed", + "producer_generation": generation, + "rerun_recovery_attempt": attempt, + }, + ) + return reason_code + + def publish_live_supervised() -> None: + if not wait_for_quarantined_live_publisher(): + return + attempt = 0 + last_reason: str | None = None + while live_generation_active(): + attempt += 1 + bridge: RerunBridge | None = None + stage = "bridge-constructor-failed" + try: + if not self._set_live_rerun_recovery( + "starting" if attempt == 1 else "retrying", + attempt=attempt, + reason_code=last_reason, + generation=generation, + ): + return + bridge = self._bridge_factory( + grpc_port=self._grpc_port, + metrics=self._metrics, + settings_provider=self._current_scene_settings, + ) + if not live_generation_active(): + close_live_bridge( + bridge, + attempt=attempt, + active=False, + reason_code=last_reason, + ) + return + stage = "bridge-begin-failed" + bridge.begin_session(self._metrics) + grpc_url = bridge.grpc_url + if not self._bind_live_rerun_bridge( + bridge, + grpc_url=grpc_url, + attempt=attempt, + generation=generation, + ): + close_live_bridge( + bridge, + attempt=attempt, + active=False, + reason_code=last_reason, + ) + return + stage = "bridge-process-failed" + publish_stream(bridge) + except BaseException: + active = live_generation_active() + last_reason = close_live_bridge( + bridge, + attempt=attempt, + active=active, + reason_code=stage, + ) if bridge is not None else stage + if bridge is None: + self._set_live_rerun_recovery( + "retrying" if active else "stopped", + attempt=attempt, + reason_code=last_reason, + generation=generation, + ) + logger.exception( + "live Rerun presentation failed; raw MQTT capture remains active", + extra={ + "event_code": "live_rerun_presentation_failed", + "producer_generation": generation, + "rerun_recovery_attempt": attempt, + "rerun_recovery_reason_code": last_reason, + }, + ) + if not active or not wait_live_retry(attempt): + return + continue + + close_live_bridge( + bridge, + attempt=attempt, + active=False, + reason_code=last_reason, + ) + return + + def join_terminal_publisher() -> None: # Never orphan a publisher: the session thread remains its owner. # On process shutdown both are daemon threads, so an irrecoverably # blocked native call cannot prevent the operating system from exit. while publisher.is_alive(): publisher.join(timeout=0.25) - publisher = threading.Thread(target=publish, name="k1-rerun-publisher", daemon=True) + publisher = threading.Thread( + target=publish_live_supervised if live_supervised else publish_terminal, + name=f"k1-rerun-publisher-{generation}", + daemon=True, + ) publisher.start() - if not publisher_ready.wait(timeout=15.0): - self._finish_error("Локальный мост визуализации не запустился за 15 секунд.") - source_done.set() - self._stop_event.set() - join_publisher() - return - if publisher_aborted.is_set(): - source_done.set() - self._stop_event.set() - join_publisher() - return - if publisher_error: - self._finish_error( - "Ошибка локального моста визуализации: " - f"{type(publisher_error[0]).__name__}: {publisher_error[0]}" - ) - source_done.set() - join_publisher() - return + if not live_supervised: + if not publisher_ready.wait(timeout=15.0): + self._finish_error( + "Локальный мост визуализации не запустился за 15 секунд.", + generation=generation, + ) + source_done.set() + self._stop_event.set() + join_terminal_publisher() + return + if publisher_aborted.is_set(): + source_done.set() + self._stop_event.set() + join_terminal_publisher() + return + if publisher_error: + self._finish_error( + "Ошибка локального моста визуализации: " + f"{type(publisher_error[0]).__name__}: {publisher_error[0]}", + generation=generation, + ) + source_done.set() + join_terminal_publisher() + return final_message = "Поток остановлен." try: final_message = producer(enqueue) except CaptureError as exc: - self._finish_error(f"Ошибка приёма MQTT: {exc}") + self._finish_error(f"Ошибка приёма MQTT: {exc}", generation=generation) except (OSError, RuntimeError, ValueError) as exc: - self._finish_error(f"Ошибка источника: {type(exc).__name__}: {exc}") + self._finish_error( + f"Ошибка источника: {type(exc).__name__}: {exc}", + generation=generation, + ) finally: source_done.set() - join_publisher() - if publisher_error: + if live_supervised: + publisher.join(timeout=LIVE_RERUN_PUBLISHER_JOIN_SECONDS) + if publisher.is_alive(): + quarantined = self._quarantine_live_rerun_publisher( + publisher, + generation=generation, + attempt=1, + reason_code="publisher-shutdown-timeout", + ) + logger.warning( + "live Rerun publisher did not stop within the bounded join; " + "presentation is quarantined until that daemon exits", + extra={ + "event_code": "live_rerun_publisher_detached", + "producer_generation": generation, + "quarantine_admitted": quarantined, + }, + ) + else: + join_terminal_publisher() + if not live_supervised and publisher_error: self._finish_error( - f"Ошибка публикации: {type(publisher_error[0]).__name__}: {publisher_error[0]}" + f"Ошибка публикации: {type(publisher_error[0]).__name__}: {publisher_error[0]}", + generation=generation, ) elif self.snapshot()["phase"] != "error": - self._finish_idle(final_message) + self._finish_idle(final_message, generation=generation) - def _set_running(self, phase: RuntimePhase, message: str) -> None: + def _set_live_rerun_recovery( + self, + state: RerunRecoveryState, + *, + attempt: int, + reason_code: str | None, + generation: int, + ) -> bool: with self._lock: - if self._phase != "stopping": + if generation != self._producer_generation or self._source_mode != "live": + return False + self._rerun_recovery_state = state + self._rerun_recovery_attempt = max(attempt, 0) + self._rerun_recovery_reason_code = reason_code + self._notify() + return True + + def _bind_live_rerun_bridge( + self, + bridge: RerunBridge, + *, + grpc_url: str, + attempt: int, + generation: int, + ) -> bool: + """Publish a live URL only for the still-authoritative producer generation.""" + + with self._lock: + if ( + self._closed + or self._stop_event.is_set() + or generation != self._producer_generation + or self._source_mode != "live" + or self._phase == "stopping" + or (self._bridge is not None and self._bridge is not bridge) + ): + return False + self._bridge = bridge + self._rerun_grpc_url = grpc_url + self._rerun_recovery_state = "ready" + self._rerun_recovery_attempt = attempt + self._rerun_recovery_reason_code = None + self._notify() + return True + + def _retire_live_rerun_bridge( + self, + bridge: RerunBridge, + *, + generation: int, + state: RerunRecoveryState, + attempt: int, + reason_code: str | None, + ) -> None: + notify = False + with self._lock: + if self._bridge is bridge: + self._bridge = None + self._rerun_grpc_url = None + notify = True + if generation == self._producer_generation and self._source_mode == "live": + self._rerun_recovery_state = state + self._rerun_recovery_attempt = max(attempt, 0) + self._rerun_recovery_reason_code = reason_code + notify = True + if notify: + self._notify() + + def _quarantine_live_rerun_publisher( + self, + publisher: threading.Thread, + *, + generation: int, + attempt: int, + reason_code: str, + ) -> bool: + with self._lock: + if generation != self._producer_generation or self._source_mode != "live": + return False + existing = self._quarantined_live_publisher + if existing is not None and existing.is_alive() and existing is not publisher: + # This should be unreachable because a generation must pass the + # quarantine gate before it can construct a publisher bridge. + self._rerun_recovery_state = "stopped" + self._rerun_recovery_reason_code = "publisher-quarantine-capacity-exhausted" + return False + self._quarantined_live_publisher = publisher + self._bridge = None + self._rerun_grpc_url = None + self._rerun_recovery_state = "stopped" + self._rerun_recovery_attempt = max(attempt, self._rerun_recovery_attempt, 1) + self._rerun_recovery_reason_code = reason_code + self._notify() + return True + + def _set_running(self, phase: RuntimePhase, message: str, *, generation: int) -> None: + with self._lock: + if generation == self._producer_generation and self._phase != "stopping": self._phase = phase self._source_ready = True self._message = message self._notify() - def _finish_idle(self, message: str) -> None: + def _set_connection_reconnecting(self, _reason: str, *, generation: int) -> None: + """Publish a neutral transport-loss state without terminalizing evidence.""" + with self._lock: + if ( + generation != self._producer_generation + or self._source_mode != "live" + or self._phase == "stopping" + ): + return + if self._connection_recovery_started_monotonic is None: + self._connection_recovery_started_monotonic = time.monotonic() + self._connection_recovery_started_at_utc = utc_now_iso() + already_reconnecting = self._phase == "reconnecting" + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None + self._clear_connection_recovery_candidate_locked() + self._connection_recovery_state = "reconnecting" + if not already_reconnecting: + self._connection_recovery_reason_code = "mqtt-report-transport-lost" + self._phase = "reconnecting" + self._source_ready = False + self._message = "Соединение с K1 потеряно. Восстанавливаем точную связь." + self._notify() + + def request_connection_recovery( + self, + reason_code: str, + *, + expected_generation: int, + ) -> bool: + """Wake the current live capture without waiting for MQTT keepalive. + + Admission belongs to the device facade. This method only applies its + exact producer-generation fence, publishes the neutral reconnecting + phase, and queues one coalescing signal for that producer's existing + capture loop. It never opens a socket or contacts the device itself. + """ + + normalized_reason = reason_code.strip() + if not normalized_reason: + raise ValueError("connection recovery reason must not be empty") + with self._lock: + if ( + expected_generation != self._producer_generation + or not self._connection_recovery_enabled + or self._source_mode != "live" + or self._phase not in {"live", "reconnecting"} + or self._stop_event.is_set() + ): + return False + if self._phase == "reconnecting": + return True + if self._connection_recovery_started_monotonic is None: + self._connection_recovery_started_monotonic = time.monotonic() + self._connection_recovery_started_at_utc = utc_now_iso() + self._connection_recovery_state = "reconnecting" + self._connection_recovery_reason_code = normalized_reason + self._connection_recovery_request_generation = expected_generation + self._connection_recovery_request_reason = normalized_reason + self._clear_connection_recovery_candidate_locked() + self._phase = "reconnecting" + self._source_ready = False + self._message = "Соединение с K1 потеряно. Восстанавливаем точную связь." + self._notify() + return True + + def _consume_connection_recovery_request(self, *, generation: int) -> str | None: + """Consume at most one owner wake for the exact running producer.""" + + with self._lock: + if self._connection_recovery_request_generation != generation: + return None + reason = self._connection_recovery_request_reason + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None + if ( + generation != self._producer_generation + or not self._connection_recovery_enabled + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + ): + return None + return reason + + def _run_connection_recovery_attempt( + self, + recover: ConnectionRecovery, + attempt: int, + *, + generation: int, + ) -> RecoveryDecision: + with self._lock: + if ( + generation != self._producer_generation + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + ): + return "blocked" + self._connection_recovery_attempt = attempt + self._connection_recovery_state = "reconnecting" + self._connection_recovery_reason_code = "read-only-rebind-in-progress" + self._clear_connection_recovery_candidate_locked() + self._message = "Восстанавливаем соединение с прежним K1." + self._notify() + decision = recover(attempt) + with self._lock: + if ( + generation != self._producer_generation + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + ): + return "blocked" + if decision == "blocked": + self._connection_recovery_state = "blocked" + self._connection_recovery_reason_code = "manual-recovery-required" + self._message = ( + "Автоматическая сверка остановлена. Завершите локальный приём " + "или выберите новое подключение." + ) + elif decision == "standby": + self._connection_recovery_state = "standby" + self._connection_recovery_reason_code = "device-reported-standby" + self._message = "K1 подтвердил завершение сканирования." + elif decision == "fault": + self._connection_recovery_state = "fault" + self._connection_recovery_reason_code = "device-recovery-state-unsafe" + self._message = "K1 вернул небезопасное состояние восстановления." + elif decision == "retry": + self._connection_recovery_reason_code = "endpoint-not-ready" + self._notify() + return decision + + def _arm_connection_recovery_point_cloud_candidate( + self, + attempt: int, + minimum_sequence: int, + *, + generation: int, + ) -> None: + """Fence one durable raw PCL candidate without declaring scene recovery.""" + + with self._lock: + if ( + generation != self._producer_generation + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + or attempt != self._connection_recovery_attempt + or minimum_sequence < 1 + ): + return + self._connection_recovery_candidate_generation = generation + self._connection_recovery_candidate_attempt = attempt + self._connection_recovery_candidate_min_sequence = minimum_sequence + + def _install_connection_recovery_confirmer( + self, + confirmer: RecoveryConfirmer, + *, + generation: int, + ) -> None: + """Install the capture-owned, attempt-fenced durable success edge.""" + + with self._lock: + if ( + generation != self._producer_generation + or self._source_mode != "live" + or not self._connection_recovery_enabled + or self._stop_event.is_set() + ): + return + self._connection_recovery_confirmer_generation = generation + self._connection_recovery_confirmer = confirmer + + def _confirm_connection_recovery_from_published_point_cloud( + self, + envelope: DecodedDataPlaneView, + *, + generation: int, + ) -> None: + """Promote once after a fresh, non-empty canonical PCL is in Rerun.""" + + if not isinstance(envelope, DecodedPointCloudView) or envelope.point_count < 1: + return + claim: tuple[int, int, int] + confirmer: RecoveryConfirmer + with self._lock: + attempt = self._connection_recovery_candidate_attempt + minimum_sequence = self._connection_recovery_candidate_min_sequence + if ( + generation != self._producer_generation + or generation != self._connection_recovery_candidate_generation + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + or attempt is None + or minimum_sequence is None + or envelope.context.sequence < minimum_sequence + ): + return + candidate_confirmer = self._connection_recovery_confirmer + if ( + candidate_confirmer is None + or generation != self._connection_recovery_confirmer_generation + ): + return + confirmer = candidate_confirmer + claim = (generation, attempt, minimum_sequence) + if self._connection_recovery_confirmation_claim is not None: + return + self._connection_recovery_confirmation_claim = claim + checkpoint = self._recovery_promotion_checkpoint + checkpoint_confirmed = True + if checkpoint is not None: + try: + checkpoint_confirmed = checkpoint( + envelope, + generation, + attempt, + minimum_sequence, + ) + except Exception: + logger.exception( + "active acquisition checkpoint recovery confirmation failed", + extra={ + "event_code": ( + "active_acquisition_checkpoint_recovery_confirmation_failed" + ), + "producer_generation": generation, + "recovery_attempt": attempt, + }, + ) + checkpoint_confirmed = False + if not checkpoint_confirmed: + with self._lock: + if self._connection_recovery_confirmation_claim == claim: + self._connection_recovery_confirmation_claim = None + return + # The checkpoint transition may block on its own fsync, but it never + # owns the runtime lock. Revalidate the complete claim before allowing + # the capture writer to close its gap. + with self._lock: + if ( + self._connection_recovery_confirmation_claim != claim + or generation != self._producer_generation + or generation != self._connection_recovery_candidate_generation + or generation != self._connection_recovery_confirmer_generation + or self._connection_recovery_confirmer is not confirmer + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + or attempt != self._connection_recovery_candidate_attempt + or minimum_sequence != self._connection_recovery_candidate_min_sequence + or envelope.context.sequence < minimum_sequence + ): + if self._connection_recovery_confirmation_claim == claim: + self._connection_recovery_confirmation_claim = None + return + try: + confirmed = confirmer(attempt) + except Exception: + logger.exception( + "capture recovery confirmation failed", + extra={"event_code": "capture_recovery_confirmation_failed"}, + ) + confirmed = False + with self._lock: + if self._connection_recovery_confirmation_claim == claim: + self._connection_recovery_confirmation_claim = None + if not confirmed: + return + if ( + generation != self._producer_generation + or generation != self._connection_recovery_candidate_generation + or generation != self._connection_recovery_confirmer_generation + or self._connection_recovery_confirmer is not confirmer + or self._source_mode != "live" + or self._phase != "reconnecting" + or self._stop_event.is_set() + or attempt != self._connection_recovery_candidate_attempt + or minimum_sequence != self._connection_recovery_candidate_min_sequence + or envelope.context.sequence < minimum_sequence + ): + return + self._clear_connection_recovery_candidate_locked() + self._connection_recovery_state = "recovered" + self._connection_recovery_attempt = attempt + self._connection_recovery_reason_code = None + self._connection_recovered_at_utc = utc_now_iso() + self._connection_recovery_started_monotonic = None + self._phase = "live" + self._source_ready = True + self._message = "Соединение с K1 восстановлено; локальный приём продолжен." + self._notify() + + def _clear_connection_recovery_candidate_locked(self) -> None: + self._connection_recovery_candidate_generation = None + self._connection_recovery_candidate_attempt = None + self._connection_recovery_candidate_min_sequence = None + self._connection_recovery_confirmation_claim = None + + def _finish_idle(self, message: str, *, generation: int) -> None: + with self._lock: + if generation != self._producer_generation: + return + live_source = self._source_mode == "live" + device_reported_standby = self._connection_recovery_state == "standby" self._phase = "idle" self._source_mode = "idle" self._source_ready = False - self._message = message + self._clear_connection_recovery_locked() + self._message = ( + "K1 подтвердил завершение сканирования; локальный приём остановлен." + if device_reported_standby + else message + ) self._foxglove_ws_url = None self._foxglove_viewer_url = None self._rerun_grpc_url = None + if live_source and self._rerun_recovery_state != "inactive": + self._rerun_recovery_state = "stopped" self._notify() - def _finish_error(self, message: str) -> None: + def _clear_connection_recovery_locked(self) -> None: + """Retire the current recovery contract while the runtime lock is held.""" + + self._connection_recovery_state = "inactive" + self._connection_recovery_attempt = 0 + self._connection_recovery_reason_code = None + self._connection_recovery_started_at_utc = None + self._connection_recovery_started_monotonic = None + self._connection_recovered_at_utc = None + self._connection_recovery_enabled = False + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None + self._connection_recovery_confirmer_generation = None + self._connection_recovery_confirmer = None + self._clear_connection_recovery_candidate_locked() + + def _clear_rerun_recovery_locked(self) -> None: + self._rerun_recovery_state = "inactive" + self._rerun_recovery_attempt = 0 + self._rerun_recovery_reason_code = None + + def _finish_error(self, message: str, *, generation: int | None = None) -> None: with self._lock: + if generation is not None and generation != self._producer_generation: + return + live_source = self._source_mode == "live" self._phase = "error" self._source_ready = False + self._connection_recovery_enabled = False + self._connection_recovery_request_generation = None + self._connection_recovery_request_reason = None + self._connection_recovery_confirmer_generation = None + self._connection_recovery_confirmer = None + self._clear_connection_recovery_candidate_locked() self._message = message self._foxglove_ws_url = None self._foxglove_viewer_url = None self._rerun_grpc_url = None + if live_source and self._rerun_recovery_state != "inactive": + self._rerun_recovery_state = "stopped" self._notify() def _current_scene_settings(self) -> RerunSceneSettings: diff --git a/src/k1link/host_network/__init__.py b/src/k1link/host_network/__init__.py index e3ee730..f33962c 100644 --- a/src/k1link/host_network/__init__.py +++ b/src/k1link/host_network/__init__.py @@ -1,10 +1,14 @@ """Host-owned network adapters used by device plugins.""" from k1link.host_network.wifi import ( + HostWifiAssociationIdentityProbe, + HostWifiAssociationIdentityResult, HostWifiCredentialMaterialAvailabilityResult, HostWifiCredentialMaterialStoreResult, HostWifiProfileError, + associate_with_ephemeral_wifi_credentials_once, associate_with_wifi_profile_once, + bind_route_fingerprint_to_wifi_association, check_wifi_credential_material, check_wifi_profile, ensure_wifi_profile_from_credential_source, @@ -12,10 +16,14 @@ from k1link.host_network.wifi import ( ) __all__ = [ + "HostWifiAssociationIdentityProbe", + "HostWifiAssociationIdentityResult", "HostWifiCredentialMaterialAvailabilityResult", "HostWifiCredentialMaterialStoreResult", "HostWifiProfileError", + "associate_with_ephemeral_wifi_credentials_once", "associate_with_wifi_profile_once", + "bind_route_fingerprint_to_wifi_association", "check_wifi_credential_material", "check_wifi_profile", "ensure_wifi_profile_from_credential_source", diff --git a/src/k1link/host_network/wifi.py b/src/k1link/host_network/wifi.py index 90ba8db..1d6433c 100644 --- a/src/k1link/host_network/wifi.py +++ b/src/k1link/host_network/wifi.py @@ -1,15 +1,17 @@ from __future__ import annotations import hashlib +import hmac import json import os -import stat +import secrets +import signal import subprocess import sys -import tempfile +import threading import time -from collections.abc import Callable, Iterator -from contextlib import contextmanager, suppress +from collections.abc import Callable +from contextlib import suppress from pathlib import Path from typing import Any, Literal, TypedDict @@ -35,6 +37,7 @@ class HostWifiProfileAvailabilityResult(TypedDict): schema_version: int adapter: str available: bool + credential_source: str | None class HostWifiProfileEnsureResult(TypedDict): @@ -57,6 +60,24 @@ class HostWifiCredentialMaterialAvailabilityResult(TypedDict): available: bool +class HostWifiAssociationIdentityResult(TypedDict): + """Secret-free continuity evidence for one host network interface. + + ``continuity_token`` is scoped to one probe instance and is safe to fold + into another local route fingerprint. It is not an SSID/BSSID digest that + can be compared between Mission Core processes. + """ + + schema_version: int + adapter: str + wifi_interface: bool | None + association_state: Literal["associated", "not-wifi", "unavailable"] + evidence_quality: Literal["ssid+bssid", "bssid-only", "not-wifi", "unavailable"] + continuity_proven: bool + continuity_token: str + reason_code: str | None + + _ERROR_MESSAGES = { "unsupported-platform": ( "для этой ОС ещё не установлен адаптер системных Wi-Fi-профилей" @@ -68,22 +89,28 @@ _ERROR_MESSAGES = { "credential provider точной версии прошивки не установлен на этом устройстве" ), "network-not-found": ( - "точка доступа выбранного K1 не найдена; проверьте режим K1 и питание" + "K1 подтвердил режим точки доступа, но macOS не увидела её Wi-Fi-сеть " + "за отведённое время" ), "profile-ssid-mismatch": "сохранённый профиль принадлежит другой точке доступа", - "credential-entry-cancelled": "ввод пароля точки доступа K1 отменён оператором", - "credential-invalid": "пароль точки доступа K1 имеет недопустимую длину", + "profile-credential-source-mismatch": ( + "сохранённый профиль K1 не связан с точной версией firmware provider" + ), "host-wifi-operation-timeout": ( "оператор не завершил системное подключение Wi-Fi за отведённое время" ), - "host-wifi-helper-build-timeout": ( - "локальный Wi-Fi helper не успел скомпилироваться за отведённое время" - ), - "host-wifi-helper-build-failed": "локальный Wi-Fi helper не удалось скомпилировать", - "host-wifi-helper-compiler-unavailable": "компилятор локального Wi-Fi helper недоступен", - "host-wifi-helper-cache-unavailable": "кэш локального Wi-Fi helper недоступен", "host-wifi-helper-missing": "исходный файл локального Wi-Fi helper не найден", "host-wifi-helper-unavailable": "локальный Wi-Fi helper недоступен", + "keychain-authorization-required": ( + "локальный профиль K1 требует отдельного разрешения связки ключей" + ), + "keychain-authorization-denied": "доступ к локальному профилю K1 запрещён", + "keychain-authorization-cancelled": "разрешение связки ключей отменено оператором", + "keychain-access-failed": "локальный профиль K1 недоступен в связке ключей", + "corewlan-authorization-denied": ( + "macOS не разрешила локальному сервису выполнять системную Wi-Fi-операцию" + ), + "wifi-interface-inactive": "системный Wi-Fi-интерфейс выключен или неактивен", "wifi-interface-unavailable": "системный Wi-Fi-интерфейс недоступен", "corewlan-error": "системный Wi-Fi не смог подключиться к точке доступа K1", } @@ -111,7 +138,11 @@ class HostWifiProfileError(RuntimeError): RunProcess = Callable[..., subprocess.CompletedProcess[bytes]] -DEFAULT_HELPER_BUILD_TIMEOUT_SECONDS = 120.0 +REVIEWED_MACOS_HELPER_COMMAND_PREFIX = ("/usr/bin/xcrun", "swift") +_ASSOCIATION_INSPECTION_PROFILE_ID = "host-association-inspection.v1" +_EPHEMERAL_ASSOCIATION_PROFILE_ID = "bridge-operation-memory.v1" +_ASSOCIATION_IDENTITY_HEX_LENGTH = 64 +MAX_ASSOCIATION_OBSERVATION_CACHE_AGE_SECONDS = 0.75 def _validate_profile_id(profile_id: str) -> None: @@ -124,232 +155,45 @@ def _validate_profile_id(profile_id: str) -> None: raise ValueError("host Wi-Fi profile id contains unsupported characters") -def _helper_cache_directory(helper_path: Path) -> Path: - """Resolve the process-local helper cache without an environment override.""" - - source = helper_path.expanduser().resolve() - for ancestor in source.parents: - if ancestor.name != "plugins": - continue - try: - relative = source.relative_to(ancestor) - except ValueError: # pragma: no cover - guarded by Path.parents - continue - if relative.parts[:2] == ("xgrids-k1", "macos"): - return ancestor.parent / ".runtime" / "mission-core" / "helpers" - # Tests and separately packaged adapters still get a stable cache beside - # their source tree. The repository layout above is the production path. - return source.parent / ".runtime" / "mission-core" / "helpers" - - -def _compiled_macos_helper_path(helper_path: Path) -> Path: - source = helper_path.expanduser().resolve() - source_sha256 = hashlib.sha256(source.read_bytes()).hexdigest() - return _helper_cache_directory(source) / f"{source.stem}-{source_sha256}" - - -def _is_ready_executable(path: Path) -> bool: - try: - metadata = path.lstat() - except OSError: - return False - return ( - stat.S_ISREG(metadata.st_mode) - and metadata.st_size > 0 - and metadata.st_mode & 0o111 != 0 - ) - - -@contextmanager -def _exclusive_helper_build_lock( - path: Path, - *, - timeout_seconds: float, -) -> Iterator[None]: - if timeout_seconds <= 0: - raise ValueError("timeout_seconds must be positive") - try: - import fcntl - except ImportError as exc: # pragma: no cover - production target is macOS - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - - flags = ( - os.O_RDWR - | os.O_CREAT - | getattr(os, "O_CLOEXEC", 0) - | getattr(os, "O_NOFOLLOW", 0) - ) - try: - descriptor = os.open(path, flags, 0o600) - except OSError as exc: - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - locked = False - try: - if not stat.S_ISREG(os.fstat(descriptor).st_mode): - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") - lock_started = time.monotonic() - while True: - try: - fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) - break - except BlockingIOError as exc: - elapsed_seconds = max(0.0, time.monotonic() - lock_started) - if elapsed_seconds >= timeout_seconds: - raise HostWifiProfileError( - "host-wifi-helper-build-timeout", - helper_stage="compile-lock", - helper_elapsed_ms=int(elapsed_seconds * 1000), - ) from exc - time.sleep(min(0.05, timeout_seconds - elapsed_seconds)) - except OSError as exc: - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - locked = True - yield - finally: - if locked: - with suppress(OSError): - fcntl.flock(descriptor, fcntl.LOCK_UN) - os.close(descriptor) - - -def _fsync_directory(path: Path) -> None: - try: - descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) - except OSError: - return - try: - os.fsync(descriptor) - finally: - os.close(descriptor) - - -def _ensure_compiled_macos_helper( - helper_path: Path, - *, - build_timeout_seconds: float, - runner: RunProcess, -) -> Path: - """Build the source-hash-addressed helper once and reuse it thereafter.""" - - if build_timeout_seconds <= 0: - raise ValueError("build_timeout_seconds must be positive") - source = helper_path.expanduser().resolve() - if not source.is_file(): - raise HostWifiProfileError("host-wifi-helper-missing") - try: - executable = _compiled_macos_helper_path(source) - executable.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - except OSError as exc: - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - if _is_ready_executable(executable): - return executable - - build_started_ns = time.monotonic_ns() - build_deadline = time.monotonic() + build_timeout_seconds - - def build_error( - reason_code: str, - *, - helper_stage: str = "compile", - ) -> HostWifiProfileError: - elapsed_ms = max(0, (time.monotonic_ns() - build_started_ns) // 1_000_000) - return HostWifiProfileError( - reason_code, - helper_stage=helper_stage, - helper_elapsed_ms=elapsed_ms, - ) - - lock_path = executable.with_name(f".{executable.name}.lock") - with _exclusive_helper_build_lock( - lock_path, - timeout_seconds=max(0.001, build_deadline - time.monotonic()), - ): - if _is_ready_executable(executable): - return executable - - try: - descriptor, staging_name = tempfile.mkstemp( - prefix=f".{executable.name}.", - suffix=".tmp", - dir=executable.parent, - ) - os.close(descriptor) - staging = Path(staging_name) - staging.unlink() - except OSError as exc: - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - - try: - remaining_build_seconds = build_deadline - time.monotonic() - if remaining_build_seconds <= 0: - raise build_error("host-wifi-helper-build-timeout") - - try: - completed = runner( - [ - "/usr/bin/xcrun", - "swiftc", - str(source), - "-o", - str(staging), - ], - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=remaining_build_seconds, - check=False, - ) - except subprocess.TimeoutExpired as exc: - raise build_error("host-wifi-helper-build-timeout") from exc - except OSError as exc: - raise build_error("host-wifi-helper-compiler-unavailable") from exc - - if completed.returncode != 0 or not _is_ready_executable(staging): - raise build_error("host-wifi-helper-build-failed") - try: - staging.chmod(0o700) - with staging.open("rb") as stream: - os.fsync(stream.fileno()) - os.replace(staging, executable) - _fsync_directory(executable.parent) - except OSError as exc: - raise HostWifiProfileError("host-wifi-helper-cache-unavailable") from exc - finally: - with suppress(OSError): - staging.unlink(missing_ok=True) - return executable - - def _run_macos_helper( helper_path: Path, request: dict[str, object], *, timeout_seconds: float, runner: RunProcess, - build_timeout_seconds: float = DEFAULT_HELPER_BUILD_TIMEOUT_SECONDS, + process_fence_descriptor_factory: Callable[[], int] | None = None, ) -> dict[str, Any]: if sys.platform != "darwin": raise HostWifiProfileError("unsupported-platform") + source = helper_path.expanduser().resolve() + if not source.is_file(): + raise HostWifiProfileError("host-wifi-helper-missing") if timeout_seconds <= 0: raise ValueError("timeout_seconds must be positive") - executable = _ensure_compiled_macos_helper( - helper_path, - build_timeout_seconds=build_timeout_seconds, - runner=runner, - ) - request_bytes = bytearray( json.dumps(request, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ) try: - completed = runner( - [str(executable)], - input=request_bytes, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout_seconds, - check=False, + command = [*REVIEWED_MACOS_HELPER_COMMAND_PREFIX, str(source)] + completed = ( + _run_macos_helper_with_process_fence( + command, + request_bytes=request_bytes, + timeout_seconds=timeout_seconds, + process_fence_descriptor_factory=( + process_fence_descriptor_factory + ), + ) + if process_fence_descriptor_factory is not None + else runner( + command, + input=request_bytes, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_seconds, + check=False, + ) ) except subprocess.TimeoutExpired as exc: raise HostWifiProfileError("host-wifi-operation-timeout") from exc @@ -389,22 +233,422 @@ def _run_macos_helper( return response +def _run_macos_helper_with_process_fence( + command: list[str], + *, + request_bytes: bytearray, + timeout_seconds: float, + process_fence_descriptor_factory: Callable[[], int], +) -> subprocess.CompletedProcess[bytes]: + """Run one mutating helper in a fenced, killable POSIX process group.""" + + if os.name != "posix": + raise HostWifiProfileError("host-wifi-helper-unavailable") + inherited_descriptor = process_fence_descriptor_factory() + process: subprocess.Popen[bytes] | None = None + try: + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=True, + pass_fds=(inherited_descriptor,), + ) + finally: + # Popen has either inherited the descriptor into the child or failed. + # The parent-side duplicate must never extend ownership on its own. + os.close(inherited_descriptor) + try: + stdout, stderr = process.communicate( + # subprocess accepts any bytes-like object at runtime. Keep the + # mutable buffer so the caller can zero credentials in-place. + input=request_bytes, # type: ignore[arg-type] + timeout=timeout_seconds, + ) + except BaseException: + _kill_and_reap_process_group(process) + raise + return subprocess.CompletedProcess( + args=command, + returncode=process.returncode, + stdout=stdout, + stderr=stderr, + ) + + +def _kill_and_reap_process_group(process: subprocess.Popen[bytes]) -> None: + """Prove the helper and every descendant are gone before dropping a fence.""" + + with suppress(ProcessLookupError, PermissionError): + os.killpg(process.pid, signal.SIGKILL) + try: + process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: + with suppress(OSError): + process.kill() + process.wait(timeout=5.0) + + +class HostWifiAssociationIdentityProbe: + """Observe macOS Wi-Fi association continuity without exporting its name. + + The Swift helper HMACs its in-process SSID/BSSID evidence with a random key + supplied over stdin. This object retains that key only for its own process + lifetime, so equal networks cannot be correlated across service restarts. + + When CoreWLAN cannot expose an association (for example because privacy + authorization hides both identifiers), observations for the same + interface/failure scope receive one process-scoped fallback token. The + token is not association proof and callers must keep that distinction, but + it prevents an unchanged kernel route from being misclassified as a new + network on every monitor poll. Route loss, interface/source changes, + recovery of exact BSSID evidence, or a service restart still changes the + combined route fingerprint. + """ + + def __init__( + self, + helper_path: Path, + *, + runner: RunProcess = subprocess.run, + continuity_key: bytes | None = None, + max_cache_age_seconds: float = MAX_ASSOCIATION_OBSERVATION_CACHE_AGE_SECONDS, + monotonic_clock: Callable[[], float] = time.monotonic, + wall_clock: Callable[[], float] = time.time, + ) -> None: + key = secrets.token_bytes(32) if continuity_key is None else bytes(continuity_key) + if len(key) != 32: + raise ValueError("host Wi-Fi continuity key must contain exactly 32 bytes") + if not 0 <= max_cache_age_seconds <= MAX_ASSOCIATION_OBSERVATION_CACHE_AGE_SECONDS: + raise ValueError( + "host Wi-Fi association cache age must be between 0 and 0.75 seconds" + ) + self._helper_path = helper_path + self._runner = runner + self._continuity_key = key + self._max_cache_age_seconds = max_cache_age_seconds + self._monotonic_clock = monotonic_clock + self._wall_clock = wall_clock + self._cached_observations: dict[ + str, + tuple[float, float, HostWifiAssociationIdentityResult], + ] = {} + self._last_interface_name: str | None = None + self._fallback_tokens: dict[str, str] = {} + self._lock = threading.Lock() + + def observe( + self, + interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + """Return one opaque association token, failing closed when uncertain.""" + + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + if interface_name is not None and ( + not 1 <= len(interface_name) <= 32 + or not all( + character.isascii() + and (character.isalnum() or character in "._-") + for character in interface_name + ) + ): + raise ValueError("host Wi-Fi interface name contains unsupported characters") + + deadline = time.monotonic() + timeout_seconds + if not self._lock.acquire(timeout=timeout_seconds): + return self._fallback_without_lock( + adapter="unavailable", + wifi_interface=None, + reason_code="host-wifi-operation-timeout", + scope=f"interface:{interface_name or 'none'}", + ) + try: + if interface_name is None: + self._invalidate_for_interface_transition_locked(None) + return self._fallback_locked( + adapter="unavailable", + wifi_interface=None, + reason_code="host-route-interface-unavailable", + scope="interface:none", + ) + self._invalidate_for_interface_transition_locked(interface_name) + cached = self._cached_observation_locked(interface_name) + if cached is not None: + return cached + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: + return self._remember_observation_locked( + interface_name, + self._fallback_locked( + adapter="unavailable", + wifi_interface=None, + reason_code="host-wifi-operation-timeout", + scope=f"interface:{interface_name}", + ), + ) + try: + response = _run_macos_helper( + self._helper_path, + { + "action": "inspect-association", + "profile_id": _ASSOCIATION_INSPECTION_PROFILE_ID, + "interface_name": interface_name, + "continuity_key_hex": self._continuity_key.hex(), + }, + timeout_seconds=remaining_seconds, + runner=self._runner, + ) + except HostWifiProfileError as exc: + return self._remember_observation_locked( + interface_name, + self._fallback_locked( + adapter="unavailable", + wifi_interface=None, + reason_code=exc.reason_code, + scope=f"interface:{interface_name}", + ), + ) + + adapter = response.get("adapter") + wifi_interface = response.get("wifi_interface") + association_identity = response.get("association_identity") + evidence_quality = response.get("association_evidence") + reason_code = response.get("reason_code") + if ( + not isinstance(adapter, str) + or not adapter + or not isinstance(wifi_interface, bool) + or evidence_quality + not in {"ssid+bssid", "bssid-only", "not-wifi", "unavailable"} + or (reason_code is not None and not isinstance(reason_code, str)) + ): + return self._remember_observation_locked( + interface_name, + self._fallback_locked( + adapter="unavailable", + wifi_interface=None, + reason_code="host-wifi-response-invalid", + scope=f"interface:{interface_name}", + ), + ) + + continuity_proven = evidence_quality in { + "ssid+bssid", + "bssid-only", + "not-wifi", + } + if not continuity_proven: + return self._remember_observation_locked( + interface_name, + self._fallback_locked( + adapter=adapter, + wifi_interface=wifi_interface, + reason_code=reason_code or "association-identity-unavailable", + scope=f"interface:{interface_name}", + ), + ) + if ( + not isinstance(association_identity, str) + or len(association_identity) != _ASSOCIATION_IDENTITY_HEX_LENGTH + or any(character not in "0123456789abcdef" for character in association_identity) + ): + return self._remember_observation_locked( + interface_name, + self._fallback_locked( + adapter="unavailable", + wifi_interface=wifi_interface, + reason_code="host-wifi-response-invalid", + scope=f"interface:{interface_name}", + ), + ) + return self._remember_observation_locked( + interface_name, + { + "schema_version": 1, + "adapter": adapter, + "wifi_interface": wifi_interface, + "association_state": ( + "not-wifi" if evidence_quality == "not-wifi" else "associated" + ), + "evidence_quality": evidence_quality, + "continuity_proven": True, + "continuity_token": association_identity, + "reason_code": None, + }, + ) + finally: + self._lock.release() + + def _invalidate_for_interface_transition_locked( + self, + interface_name: str | None, + ) -> None: + """Make an interface transition an immediate cache barrier.""" + + if interface_name == self._last_interface_name: + return + self._cached_observations.clear() + self._last_interface_name = interface_name + + def _cached_observation_locked( + self, + interface_name: str, + ) -> HostWifiAssociationIdentityResult | None: + """Reuse only a sub-second read-only association observation. + + Runtime compilation is deliberately not used here: the field-approved + credential/CoreWLAN path relies on Apple's signed Swift interpreter + identity. The cache is therefore limited to the read-only association + result, never a credential or mutation result. Both clocks must remain + monotonic within the 750 ms safety window; sleep, a wall-clock jump or + an interface transition forces a fresh source-runner observation before + the caller can publish. + """ + + cached = self._cached_observations.get(interface_name) + if cached is None or self._max_cache_age_seconds == 0: + return None + observed_monotonic, observed_wall, observation = cached + monotonic_age = self._monotonic_clock() - observed_monotonic + wall_age = self._wall_clock() - observed_wall + if ( + monotonic_age < 0 + or wall_age < 0 + or monotonic_age > self._max_cache_age_seconds + or wall_age > self._max_cache_age_seconds + ): + self._cached_observations.pop(interface_name, None) + return None + return observation.copy() + + def _remember_observation_locked( + self, + interface_name: str, + observation: HostWifiAssociationIdentityResult, + ) -> HostWifiAssociationIdentityResult: + if self._max_cache_age_seconds > 0: + self._cached_observations[interface_name] = ( + self._monotonic_clock(), + self._wall_clock(), + observation.copy(), + ) + return observation.copy() + + def _fallback_locked( + self, + *, + adapter: str, + wifi_interface: bool | None, + reason_code: str, + scope: str, + ) -> HostWifiAssociationIdentityResult: + token_scope = "\x1f".join( + ( + scope, + adapter, + "unknown" if wifi_interface is None else str(wifi_interface).lower(), + reason_code, + ) + ) + token = self._fallback_tokens.get(token_scope) + if token is None: + material = ( + "mission-core/host-wifi-association-fallback/v2\x1f" + f"{token_scope}" + ).encode() + token = hmac.new(self._continuity_key, material, hashlib.sha256).hexdigest() + self._fallback_tokens[token_scope] = token + return { + "schema_version": 1, + "adapter": adapter, + "wifi_interface": wifi_interface, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": token, + "reason_code": reason_code, + } + + def _fallback_without_lock( + self, + *, + adapter: str, + wifi_interface: bool | None, + reason_code: str, + scope: str, + ) -> HostWifiAssociationIdentityResult: + """Build the deterministic process-local fallback under lock contention.""" + + token_scope = "\x1f".join( + ( + scope, + adapter, + "unknown" if wifi_interface is None else str(wifi_interface).lower(), + reason_code, + ) + ) + material = ( + "mission-core/host-wifi-association-fallback/v2\x1f" + f"{token_scope}" + ).encode() + return { + "schema_version": 1, + "adapter": adapter, + "wifi_interface": wifi_interface, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": hmac.new( + self._continuity_key, + material, + hashlib.sha256, + ).hexdigest(), + "reason_code": reason_code, + } + + +def bind_route_fingerprint_to_wifi_association( + route_fingerprint: str, + association: HostWifiAssociationIdentityResult, +) -> str: + """Bind route facts to one opaque association observation.""" + + if not route_fingerprint: + raise ValueError("route fingerprint must be nonblank") + token = association["continuity_token"] + if ( + len(token) != _ASSOCIATION_IDENTITY_HEX_LENGTH + or any(character not in "0123456789abcdef" for character in token) + ): + raise ValueError("association continuity token is invalid") + material = ( + "mission-core/host-route+wifi-association/v1\x1f" + f"{route_fingerprint}\x1f{token}" + ) + return hashlib.sha256(material.encode()).hexdigest() + + def associate_with_wifi_profile_once( helper_path: Path, profile_id: str, expected_ssid: str, *, - scan_timeout_seconds: float = 15.0, + scan_timeout_seconds: float = 30.0, timeout_seconds: float = 180.0, runner: RunProcess = subprocess.run, + process_fence_descriptor_factory: Callable[[], int] | None = None, ) -> HostWifiProfileAssociationResult: """Associate through one device-scoped OS profile without exposing its secret. - The selected device supplies the expected, operator-visible AP SSID. The + The selected device supplies the expected, operator-visible AP SSID. The platform helper performs bounded exact-SSID discovery followed by at most - one association. It resolves a device-scoped secret from the OS credential - stores, or asks for it through a native secure prompt on first use. The - secret never crosses the helper boundary. + one association using the already materialized Mission Core profile. It + never falls back to another system credential or a post-write password + prompt, and the secret never crosses the helper boundary. """ _validate_profile_id(profile_id) @@ -422,6 +666,7 @@ def associate_with_wifi_profile_once( }, timeout_seconds=timeout_seconds, runner=runner, + process_fence_descriptor_factory=process_fence_descriptor_factory, ) already_associated = response.get("already_associated") profile_enrolled = response.get("profile_enrolled") @@ -439,12 +684,7 @@ def associate_with_wifi_profile_once( or type(scan_elapsed_ms) is not int or scan_elapsed_ms < 0 or not isinstance(credential_source, str) - or credential_source not in { - "mission-core-keychain", - "system-wifi-keychain", - "native-secure-prompt", - "exact-firmware-profile", - } + or credential_source != "exact-firmware-profile" ): raise HostWifiProfileError("host-wifi-response-invalid") return { @@ -459,6 +699,78 @@ def associate_with_wifi_profile_once( } +def associate_with_ephemeral_wifi_credentials_once( + helper_path: Path, + expected_ssid: str, + password: str, + *, + scan_timeout_seconds: float = 30.0, + timeout_seconds: float = 180.0, + runner: RunProcess = subprocess.run, + process_fence_descriptor_factory: Callable[[], int] | None = None, +) -> HostWifiProfileAssociationResult: + """Join one operator-selected Bridge network without persisting its secret. + + This host-side handoff is used after K1 has already applied the Bridge + topology and the Mac no longer has a direct route to it (most notably when + switching from the K1 access point back to the shared network). The + password crosses only the helper's stdin, is zeroed from the mutable Python + request buffer, and is neither stored in the Mission Core Keychain profile + store nor returned in evidence. + + The call performs at most one CoreWLAN association after a bounded exact + SSID scan. It never repeats the preceding BLE/device mutation. + """ + + if not 1 <= len(expected_ssid.encode("utf-8")) <= 32: + raise ValueError("SSID must contain between 1 and 32 UTF-8 bytes") + if not 1 <= len(password.encode("utf-8")) <= 64: + raise ValueError("Wi-Fi password must contain between 1 and 64 UTF-8 bytes") + if not 0 <= scan_timeout_seconds <= 60: + raise ValueError("scan_timeout_seconds must be between 0 and 60") + response = _run_macos_helper( + helper_path, + { + "action": "associate-ephemeral", + "profile_id": _EPHEMERAL_ASSOCIATION_PROFILE_ID, + "ssid": expected_ssid, + "password": password, + "scan_timeout_seconds": scan_timeout_seconds, + }, + timeout_seconds=timeout_seconds, + runner=runner, + process_fence_descriptor_factory=process_fence_descriptor_factory, + ) + already_associated = response.get("already_associated") + profile_enrolled = response.get("profile_enrolled") + scan_attempt_count = response.get("scan_attempt_count") + scan_elapsed_ms = response.get("scan_elapsed_ms") + credential_source = response.get("credential_source") + adapter = response.get("adapter") + if ( + not isinstance(already_associated, bool) + or profile_enrolled is not False + or not isinstance(adapter, str) + or not adapter + or type(scan_attempt_count) is not int + or scan_attempt_count < 1 + or type(scan_elapsed_ms) is not int + or scan_elapsed_ms < 0 + or credential_source != "operation-memory" + ): + raise HostWifiProfileError("host-wifi-response-invalid") + return { + "schema_version": 1, + "adapter": adapter, + "outcome": "already-associated" if already_associated else "associated", + "already_associated": already_associated, + "profile_enrolled": False, + "scan_attempt_count": scan_attempt_count, + "scan_elapsed_ms": scan_elapsed_ms, + "credential_source": "operation-memory", + } + + def check_wifi_profile( helper_path: Path, profile_id: str, @@ -483,13 +795,27 @@ def check_wifi_profile( runner=runner, ) available = response.get("profile_available") + credential_source = response.get("credential_source") adapter = response.get("adapter") - if not isinstance(available, bool) or not isinstance(adapter, str) or not adapter: + if ( + not isinstance(available, bool) + or not isinstance(adapter, str) + or not adapter + or ( + available + and credential_source != "exact-firmware-profile" + ) + or ( + not available + and credential_source is not None + ) + ): raise HostWifiProfileError("host-wifi-response-invalid") return { "schema_version": 1, "adapter": adapter, "available": available, + "credential_source": credential_source, } @@ -501,6 +827,7 @@ def ensure_wifi_profile_from_credential_source( *, timeout_seconds: float = 30.0, runner: RunProcess = subprocess.run, + process_fence_descriptor_factory: Callable[[], int] | None = None, ) -> HostWifiProfileEnsureResult: """Materialize one device profile from a firmware-scoped secure-store item. @@ -522,6 +849,7 @@ def ensure_wifi_profile_from_credential_source( }, timeout_seconds=timeout_seconds, runner=runner, + process_fence_descriptor_factory=process_fence_descriptor_factory, ) available = response.get("profile_available") profile_enrolled = response.get("profile_enrolled") diff --git a/src/k1link/sessions/active.py b/src/k1link/sessions/active.py index 7f40ab1..0b13c7f 100644 --- a/src/k1link/sessions/active.py +++ b/src/k1link/sessions/active.py @@ -30,6 +30,7 @@ class ActiveSessionLease: _descriptor: int _marker_identity: tuple[int, int] _released: bool = False + _marker_removed: bool = False @classmethod def acquire(cls, sessions_root: Path, session_root: Path) -> ActiveSessionLease: @@ -72,18 +73,21 @@ class ActiveSessionLease: if self._released: return marker = self.sessions_root / ACTIVE_SESSION_MARKER - try: + if not self._marker_removed: try: metadata = marker.lstat() except FileNotFoundError: metadata = None if metadata is not None and (metadata.st_dev, metadata.st_ino) == self._marker_identity: marker.unlink() - _fsync_directory(self.sessions_root) - finally: - _unlock_descriptor(self._descriptor) - os.close(self._descriptor) - self._released = True + self._marker_removed = True + # Keep the descriptor/lock owned until marker retirement is durably + # synchronized. If unlink or fsync fails, a later cleanup-only retry + # resumes this staged release without reopening command authority. + _fsync_directory_strict(self.sessions_root) + _unlock_descriptor(self._descriptor) + os.close(self._descriptor) + self._released = True def __enter__(self) -> ActiveSessionLease: return self @@ -170,3 +174,13 @@ def _fsync_directory(path: Path) -> None: os.fsync(descriptor) finally: os.close(descriptor) + + +def _fsync_directory_strict(path: Path) -> None: + """Synchronize a lease transition or fail while retaining its lock.""" + + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/src/k1link/viewer/rerun_bridge.py b/src/k1link/viewer/rerun_bridge.py index 905bde4..6bdc231 100644 --- a/src/k1link/viewer/rerun_bridge.py +++ b/src/k1link/viewer/rerun_bridge.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import logging import math import socket @@ -86,8 +87,19 @@ def _select_available_grpc_port( # Rerun binds all local interfaces. Probe the same address class # so a previous recording retained by a late viewer is detected. probe.bind(("0.0.0.0", candidate)) - except OSError: - continue + except OSError as exc: + if exc.errno == errno.EADDRINUSE: + continue + if exc.errno in {errno.EACCES, errno.EPERM}: + raise PermissionError( + exc.errno, + "Permission denied while probing local Rerun gRPC " + f"port {candidate}", + ) from exc + raise RuntimeError( + "Could not probe local Rerun gRPC " + f"port {candidate}: {type(exc).__name__}: {exc}" + ) from exc return candidate raise RuntimeError( "No local Rerun gRPC port is available in " diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index a14596f..64faf82 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -11,7 +11,6 @@ from typing import Any from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from fastapi.staticfiles import StaticFiles from pydantic import ValidationError from starlette.middleware.gzip import GZipMiddleware @@ -76,6 +75,7 @@ from k1link.web.e46j_raw_fisheye_realtime_api import ( ) from k1link.web.e47_semantic_slam_api import build_e47_semantic_slam_router from k1link.web.environment_api import build_environment_router +from k1link.web.frontend_assets import ControlStationStaticFiles, frontend_build_id from k1link.web.l3_pointpillars_visual_api import ( build_l3_pointpillars_visual_router, ) @@ -385,7 +385,7 @@ async def _recording_preparation_reconciler() -> None: async def app_lifespan(_: FastAPI) -> AsyncIterator[None]: reconciler: asyncio.Task[None] | None = None try: - configure_scanner_diagnostics(REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs") + configure_scanner_diagnostics(session_store.data_dir / "logs") session_recording_preparation_manager.start() # Recovery is intentionally a one-shot startup phase. The archive # helper owns a cross-process lease, while ordinary catalog requests @@ -480,11 +480,36 @@ async def invoke_device_plugin_action( except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except PluginExecutionError as exc: - raise HTTPException(status_code=502, detail=str(exc)) from exc + raise HTTPException(status_code=exc.http_status_code, detail=str(exc)) from exc except PluginRuntimeUnavailableError as exc: raise HTTPException(status_code=503, detail=str(exc)) from exc +_CLOSED_WEBSOCKET_SEND_ERRORS = frozenset( + { + "handler is closed", + 'Cannot call "send" once a close message has been sent.', + ( + "Unexpected ASGI message 'websocket.send', after sending " + "'websocket.close' or response already completed." + ), + } +) + + +def _is_closed_websocket_send_error(exc: RuntimeError) -> bool: + message = " ".join(str(exc).split()) + if message in _CLOSED_WEBSOCKET_SEND_ERRORS: + return True + # uvloop raises this after the browser has already torn down the TCP + # transport, before Starlette can translate the failed send into a + # WebSocketDisconnect. Match both stable parts so an unrelated RuntimeError + # containing only "closed" is never swallowed. + return message.startswith("unable to perform operation on ") and message.endswith( + "; the handler is closed" + ) + + @app.websocket("/api/v1/device-plugins/{plugin_id}/events") async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None: await websocket.accept() @@ -493,10 +518,17 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None: while True: state = await plugin_dispatcher.invoke(plugin_id, STATE_READ_ACTION_ID, {}) sequence += 1 - await websocket.send_json({"pluginId": plugin_id, "sequence": sequence, "state": state}) + try: + await websocket.send_json( + {"pluginId": plugin_id, "sequence": sequence, "state": state} + ) + except WebSocketDisconnect: + return + except RuntimeError as exc: + if _is_closed_websocket_send_error(exc): + return + raise await asyncio.sleep(0.5) - except WebSocketDisconnect: - return except (PluginNotFoundError, PluginActionNotFoundError): await websocket.close(code=1008, reason="Device plugin is not available") except (PluginExecutionError, PluginRuntimeUnavailableError): @@ -781,11 +813,7 @@ app.include_router( app.include_router( build_e47_semantic_slam_router( root_provider=lambda: ( - REPOSITORY_ROOT - / ".runtime" - / "compute-experiments" - / "e47" - / "semantic-slam-results" + REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e47" / "semantic-slam-results" ), ) ) @@ -1114,9 +1142,17 @@ app.include_router( root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system", ) ) -app.include_router(build_viewer_diagnostics_router()) - - frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist" +app.include_router( + build_viewer_diagnostics_router( + expected_ui_build_id=lambda: frontend_build_id(frontend_dist), + ) +) + + if frontend_dist.is_dir(): - app.mount("/", StaticFiles(directory=frontend_dist, html=True), name="frontend") + app.mount( + "/", + ControlStationStaticFiles(directory=frontend_dist, html=True), + name="frontend", + ) diff --git a/src/k1link/web/device_lifecycle.py b/src/k1link/web/device_lifecycle.py index 08088a2..41b265c 100644 --- a/src/k1link/web/device_lifecycle.py +++ b/src/k1link/web/device_lifecycle.py @@ -82,6 +82,8 @@ class OperationRecord: result: dict[str, Any] | None = None error: dict[str, Any] | None = None evidence_refs: tuple[str, ...] = () + context: dict[str, Any] = field(default_factory=dict) + events: list[dict[str, Any]] = field(default_factory=list) # A keyed, non-reversible digest supplied by the service. It is deliberately # excluded from API snapshots: callers only need mismatch detection, while # the journal must never retain action inputs or secret material. @@ -108,6 +110,8 @@ class OperationRecord: "result": dict(self.result) if self.result is not None else None, "error": dict(self.error) if self.error is not None else None, "evidence_refs": list(self.evidence_refs), + "context": dict(self.context), + "events": [dict(event) for event in self.events], } @@ -144,6 +148,7 @@ class OperationJournal: deadline_seconds: float | None = None, cancellable: bool = False, request_fingerprint: str | None = None, + context: Mapping[str, Any] | None = None, ) -> tuple[OperationRecord, bool]: action = action.strip() if not action: @@ -189,7 +194,9 @@ class OperationJournal: ), cancellable=cancellable, request_fingerprint=request_fingerprint, + context=dict(context or {}), ) + self._append_event_locked(record) self._records[resolved_id] = record self._order.append(resolved_id) if idempotency_key is not None: @@ -225,6 +232,7 @@ class OperationJournal: if status in TERMINAL_OPERATION_STATUSES: record.completed_at = self._clock() self._trim_locked() + self._append_event_locked(record) return record def request_cancel(self, operation_id: str) -> OperationRecord: @@ -239,6 +247,7 @@ class OperationJournal: record.state_revision += 1 record.stage_code = "cancellation-requested" record.message_code = "operation.cancellation_requested" + self._append_event_locked(record) return record def transition_if_pending( @@ -276,12 +285,32 @@ class OperationJournal: if status in TERMINAL_OPERATION_STATUSES: record.completed_at = self._clock() self._trim_locked() + self._append_event_locked(record) return record def get(self, operation_id: str) -> OperationRecord: with self._lock: return self._require_locked(operation_id) + def deadline_reached(self, operation_id: str | None) -> bool: + """Evaluate one operation deadline on the journal-owned server clock. + + Device lifecycle reducers use this instead of browser timers or a + second wall-clock source. Terminal records deliberately retain the + same answer so an idempotent local-cleanup retry can continue after the + operation outcome itself has already been sealed. + """ + + if operation_id is None: + return False + with self._lock: + record = self._records.get(operation_id) + return bool( + record is not None + and record.deadline_at is not None + and self._clock() >= record.deadline_at + ) + def latest(self) -> OperationRecord | None: with self._lock: return self._records[self._order[-1]] if self._order else None @@ -298,6 +327,32 @@ class OperationJournal: except KeyError as exc: raise KeyError(f"unknown operation: {operation_id}") from exc + def _append_event_locked(self, record: OperationRecord) -> None: + """Append one bounded, secret-free stage fact for operator diagnosis.""" + + error = record.error or {} + result = record.result or {} + side_effect_status = error.get("side_effect_status", result.get("side_effect_status")) + event: dict[str, Any] = { + "schema_version": "missioncore.operation-event/v1", + "sequence": record.sequence, + "status": record.status, + "stage_code": record.stage_code, + "message_code": record.message_code, + "observed_at": _iso(self._clock()), + "side_effect_status": ( + side_effect_status if isinstance(side_effect_status, str) else None + ), + "error_code": error.get("code") if isinstance(error.get("code"), str) else None, + "safe_to_retry": ( + error.get("safe_to_retry") + if isinstance(error.get("safe_to_retry"), bool) + else None + ), + "automatic_retry": False, + } + record.events.append(event) + def _trim_locked(self) -> None: while len(self._order) > self._max_records: oldest_id = next( diff --git a/src/k1link/web/frontend_assets.py b/src/k1link/web/frontend_assets.py new file mode 100644 index 0000000..426df53 --- /dev/null +++ b/src/k1link/web/frontend_assets.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import re +from pathlib import Path + +from fastapi.staticfiles import StaticFiles +from starlette.responses import Response +from starlette.types import Scope + +HTML_NO_STORE = "no-store" +HASHED_ASSET_IMMUTABLE = "public, max-age=31536000, immutable" + +_MODULE_SCRIPT = re.compile( + r']*\bsrc=["\'](?P/assets/[^"\']+)["\'][^>]*>', + re.IGNORECASE, +) +_HASHED_ASSET = re.compile( + r"^assets/(?:.+)-[A-Za-z0-9_-]{8,}\.[A-Za-z0-9]+$", +) + + +def frontend_build_id(frontend_root: Path) -> str | None: + """Return the exact content-hashed module loaded by the current index.""" + + try: + index = (frontend_root / "index.html").read_text(encoding="utf-8") + except OSError: + return None + for match in _MODULE_SCRIPT.finditer(index): + source = match.group("src") + if _HASHED_ASSET.fullmatch(source.removeprefix("/")): + return source + return None + + +class ControlStationStaticFiles(StaticFiles): + """Serve the SPA shell fresh while retaining immutable hashed assets.""" + + async def get_response(self, path: str, scope: Scope) -> Response: + response = await super().get_response(path, scope) + if response.status_code >= 400: + return response + content_type = response.headers.get("content-type", "").lower() + normalized = path.lstrip("/") + if content_type.startswith("text/html"): + response.headers["Cache-Control"] = HTML_NO_STORE + elif _HASHED_ASSET.fullmatch(normalized): + response.headers["Cache-Control"] = HASHED_ASSET_IMMUTABLE + return response diff --git a/src/k1link/web/plugin_runtime.py b/src/k1link/web/plugin_runtime.py index 493591a..c7347cc 100644 --- a/src/k1link/web/plugin_runtime.py +++ b/src/k1link/web/plugin_runtime.py @@ -68,6 +68,19 @@ class PluginActionNotFoundError(LookupError): class PluginExecutionError(RuntimeError): """A validated plugin action failed while talking to its device/runtime.""" + def __init__( + self, + message: str, + *, + http_status_code: int = 502, + reason_code: str | None = None, + ) -> None: + super().__init__(message) + if not 400 <= http_status_code <= 599: + raise ValueError("plugin execution HTTP status must be an error response") + self.http_status_code = http_status_code + self.reason_code = reason_code + class PluginRuntimeCompatibilityError(RuntimeError): """A plugin runtime cannot satisfy the reviewed manifest/host contract.""" @@ -124,9 +137,7 @@ class InProcessDevicePluginRuntime: raise PluginRuntimeCompatibilityError( "Runtime does not support the manifest host API version" ) - if frozenset(request.required_action_ids) != frozenset( - self.descriptor.action_ids - ): + if frozenset(request.required_action_ids) != frozenset(self.descriptor.action_ids): raise PluginRuntimeCompatibilityError( "Runtime handshake actions do not match the manifest" ) @@ -225,9 +236,7 @@ class DevicePluginDispatcher: @property def health_snapshots(self) -> dict[str, RuntimeHealthSnapshot]: - return { - plugin_id: runtime.health() for plugin_id, runtime in self._runtimes.items() - } + return {plugin_id: runtime.health() for plugin_id, runtime in self._runtimes.items()} async def invoke( self, diff --git a/src/k1link/web/runtime_diagnostics.py b/src/k1link/web/runtime_diagnostics.py index 3c69fcc..5abb8ba 100644 --- a/src/k1link/web/runtime_diagnostics.py +++ b/src/k1link/web/runtime_diagnostics.py @@ -55,6 +55,10 @@ _EXTRA_FIELDS: Final = ( "start_checkpoint_released", "stale_session_retired", "stream_id", + "ui_build_id", + "document_instance_id", + "viewer_instance_id", + "lifecycle_generation", "backend_activity_sequence", "viewer_range_max_ns", "stalled_for_ms", diff --git a/src/k1link/web/viewer_diagnostics_api.py b/src/k1link/web/viewer_diagnostics_api.py index dd259fc..75b5ff8 100644 --- a/src/k1link/web/viewer_diagnostics_api.py +++ b/src/k1link/web/viewer_diagnostics_api.py @@ -1,9 +1,11 @@ from __future__ import annotations import logging +from collections.abc import Callable from typing import Literal from fastapi import APIRouter, Response +from fastapi.responses import JSONResponse from pydantic import BaseModel, ConfigDict, Field logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver") @@ -23,12 +25,31 @@ LiveViewerFailureStage = Literal[ "receiver-stalled", ] +LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2" +LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA = "missioncore.live-viewer-client-contract/v1" +UI_BUILD_HEADER = "X-MissionCore-UI-Build" + +_INSTANCE_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" +_UI_BUILD_ID_PATTERN = r"^(?:development|/assets/[A-Za-z0-9._/-]+)$" + class LiveViewerDiagnosticEvent(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) - schema_version: Literal["missioncore.live-viewer-diagnostic/v1"] + schema_version: Literal["missioncore.live-viewer-diagnostic/v2"] event_code: LiveViewerEventCode + ui_build_id: str = Field(min_length=1, max_length=256, pattern=_UI_BUILD_ID_PATTERN) + document_instance_id: str = Field( + min_length=36, + max_length=36, + pattern=_INSTANCE_ID_PATTERN, + ) + viewer_instance_id: str = Field( + min_length=36, + max_length=36, + pattern=_INSTANCE_ID_PATTERN, + ) + lifecycle_generation: int = Field(ge=1) failure_stage: LiveViewerFailureStage | None = None stream_id: str | None = Field( default=None, @@ -42,17 +63,79 @@ class LiveViewerDiagnosticEvent(BaseModel): recovery_attempt: int | None = Field(default=None, ge=1, le=3) -def build_viewer_diagnostics_router() -> APIRouter: +def build_viewer_diagnostics_router( + *, + expected_ui_build_id: Callable[[], str | None] = lambda: "development", +) -> APIRouter: router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"]) + @router.get("/client-contract") + def get_live_viewer_client_contract() -> JSONResponse: + expected = expected_ui_build_id() + if expected is None: + return JSONResponse( + status_code=503, + content={ + "schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA, + "status": "frontend-build-unavailable", + }, + headers={"Cache-Control": "no-store"}, + ) + return JSONResponse( + content={ + "schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA, + "status": "ready", + "ui_build_id": expected, + "diagnostic_schema_version": LIVE_VIEWER_DIAGNOSTIC_SCHEMA, + }, + headers={ + "Cache-Control": "no-store", + UI_BUILD_HEADER: expected, + }, + ) + @router.post("/live-diagnostics", status_code=204) def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response: + expected = expected_ui_build_id() + if expected is None: + return JSONResponse( + status_code=503, + content={"detail": "frontend-build-unavailable"}, + headers={"Cache-Control": "no-store"}, + ) + # Vite's unhashed development entry intentionally has no deploy + # identity and the browser-side monitor is disabled for it. Keep local + # diagnostics usable without turning a dist build into a reload loop; + # every production document carries the exact hashed entry below. + if event.ui_build_id not in {expected, "development"}: + return JSONResponse( + status_code=409, + content={ + "detail": "stale-ui-build", + "expected_ui_build_id": expected, + }, + headers={ + "Cache-Control": "no-store", + UI_BUILD_HEADER: expected, + }, + ) logger.info( - "Mission Core live Rerun receiver diagnostic: event=%s stream=%s", + ( + "Mission Core live Rerun receiver diagnostic: event=%s stream=%s " + "document=%s viewer=%s lifecycle=%s build=%s" + ), event.event_code, event.stream_id, + event.document_instance_id, + event.viewer_instance_id, + event.lifecycle_generation, + event.ui_build_id, extra={ "event_code": event.event_code, + "ui_build_id": event.ui_build_id, + "document_instance_id": event.document_instance_id, + "viewer_instance_id": event.viewer_instance_id, + "lifecycle_generation": event.lifecycle_generation, "failure_stage": event.failure_stage, "stream_id": event.stream_id, "backend_activity_sequence": event.backend_activity_sequence, @@ -61,6 +144,6 @@ def build_viewer_diagnostics_router() -> APIRouter: "recovery_attempt": event.recovery_attempt, }, ) - return Response(status_code=204) + return Response(status_code=204, headers={UI_BUILD_HEADER: expected}) return router diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..2660660 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import atexit +import os +import shutil +import tempfile +from pathlib import Path + +# ``k1link.web.app`` builds the installed plugin environment at import time. +# Pytest imports test modules during collection and future tests may lazy-load +# it during execution or in a spawned child. Keep every mutable Mission Core +# store used by that composition inside one process-scoped test root for the +# entire pytest lifetime. Restoring the caller environment after collection +# would reopen the operator's real stores for lazy imports and child processes. +_ISOLATED_ENVIRONMENT_KEYS = ( + "MISSIONCORE_DATA_DIR", + "MISSIONCORE_EVIDENCE_DIR", + "MISSIONCORE_LEGACY_SESSIONS_DIR", +) +_ORIGINAL_ENVIRONMENT = { + key: os.environ.get(key) for key in _ISOLATED_ENVIRONMENT_KEYS +} +_TEST_RUNTIME_ROOT = Path( + tempfile.mkdtemp(prefix="mission-core-pytest-runtime-") +).resolve() +_TEST_RUNTIME_ROOT.chmod(0o700) + +_ISOLATED_ENVIRONMENT = { + "MISSIONCORE_DATA_DIR": _TEST_RUNTIME_ROOT / "data", + "MISSIONCORE_EVIDENCE_DIR": _TEST_RUNTIME_ROOT / "evidence", + "MISSIONCORE_LEGACY_SESSIONS_DIR": _TEST_RUNTIME_ROOT / "legacy-sessions", +} +for _key, _path in _ISOLATED_ENVIRONMENT.items(): + _path.mkdir(mode=0o700, parents=True, exist_ok=True) + _path.chmod(0o700) + os.environ[_key] = str(_path) + + +def _restore_environment() -> None: + for key, original_value in _ORIGINAL_ENVIRONMENT.items(): + if original_value is None: + os.environ.pop(key, None) + else: + os.environ[key] = original_value + + +@atexit.register +def _remove_test_runtime() -> None: + _restore_environment() + shutil.rmtree(_TEST_RUNTIME_ROOT, ignore_errors=True) diff --git a/tests/test_active_session_lease.py b/tests/test_active_session_lease.py index bbaa8da..7d49cf9 100644 --- a/tests/test_active_session_lease.py +++ b/tests/test_active_session_lease.py @@ -1,9 +1,13 @@ from __future__ import annotations +import os from pathlib import Path +import pytest + from k1link.device_plugins.xgrids_k1.archive import discover_legacy_viewer_sessions from k1link.sessions import ActiveSessionLease, recover_stale_active_session_marker +from k1link.sessions import active as active_session_module def test_active_session_lease_hides_live_evidence_until_release(tmp_path: Path) -> None: @@ -38,3 +42,65 @@ def test_startup_recovery_removes_only_an_unlocked_stale_marker(tmp_path: Path) assert recover_stale_active_session_marker(sessions) is True assert not marker.exists() assert recover_stale_active_session_marker(sessions) is False + + +def test_active_session_release_retries_unlink_without_dropping_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions = tmp_path / "sessions" + session = sessions / "20260812T231000Z_viewer_live" + lease = ActiveSessionLease.acquire(sessions, session) + marker = sessions / ".current_session" + original_unlink = Path.unlink + attempts = 0 + + def fail_once(path: Path, *args: object, **kwargs: object) -> None: + nonlocal attempts + if path == marker and attempts == 0: + attempts += 1 + raise OSError("injected marker unlink failure") + original_unlink(path, *args, **kwargs) + + monkeypatch.setattr(Path, "unlink", fail_once) + with pytest.raises(OSError, match="injected marker unlink failure"): + lease.release() + + assert marker.exists() + assert lease._released is False # noqa: SLF001 + os.fstat(lease._descriptor) # noqa: SLF001 + assert recover_stale_active_session_marker(sessions) is False + + lease.release() + assert lease._released is True # noqa: SLF001 + assert not marker.exists() + + +def test_active_session_release_retries_directory_fsync_after_unlink( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sessions = tmp_path / "sessions" + session = sessions / "20260812T231100Z_viewer_live" + lease = ActiveSessionLease.acquire(sessions, session) + original_fsync = active_session_module._fsync_directory_strict + attempts = 0 + + def fail_once(path: Path) -> None: + nonlocal attempts + if attempts == 0: + attempts += 1 + raise OSError("injected directory fsync failure") + original_fsync(path) + + monkeypatch.setattr(active_session_module, "_fsync_directory_strict", fail_once) + with pytest.raises(OSError, match="injected directory fsync failure"): + lease.release() + + assert lease._marker_removed is True # noqa: SLF001 + assert lease._released is False # noqa: SLF001 + os.fstat(lease._descriptor) # noqa: SLF001 + + lease.release() + lease.release() + assert lease._released is True # noqa: SLF001 diff --git a/tests/test_artifacts.py b/tests/test_artifacts.py index 5bc7387..e213269 100644 --- a/tests/test_artifacts.py +++ b/tests/test_artifacts.py @@ -1,4 +1,6 @@ import json +import os +import stat from pathlib import Path from k1link.artifacts import write_json_atomic @@ -9,3 +11,23 @@ def test_write_json_atomic(tmp_path: Path) -> None: write_json_atomic(output, {"value": "тест"}) assert json.loads(output.read_text(encoding="utf-8")) == {"value": "тест"} assert not list(output.parent.glob("*.tmp")) + + +def test_write_json_atomic_flushes_file_and_parent_directory( + tmp_path: Path, + monkeypatch, +) -> None: + output = tmp_path / "nested" / "artifact.json" + flushed_kinds: list[str] = [] + real_fsync = os.fsync + + def observing_fsync(descriptor: int) -> None: + mode = os.fstat(descriptor).st_mode + flushed_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(os, "fsync", observing_fsync) + + write_json_atomic(output, {"durable": True}) + + assert flushed_kinds == ["file", "directory"] diff --git a/tests/test_ble_scanner.py b/tests/test_ble_scanner.py index b2e2451..4d0c248 100644 --- a/tests/test_ble_scanner.py +++ b/tests/test_ble_scanner.py @@ -1,5 +1,7 @@ import asyncio from collections.abc import Iterator +from pathlib import Path +from typing import Literal import pytest from bleak.backends.device import BLEDevice @@ -7,25 +9,117 @@ from bleak.backends.scanner import AdvertisementData from pytest import MonkeyPatch import k1link.device_plugins.xgrids_k1.ble.scanner as scanner_module +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleRuntimeBusy, + bind_ble_runtime_owner_loop, + configure_ble_runtime_process_lease, + reset_ble_runtime_arbiter_for_tests, + run_ble_operation, +) from k1link.device_plugins.xgrids_k1.ble.scanner import ( advertisement_record, + capture_discovered_device, + connected_device_capture, + connected_device_recovery_name, + connected_device_recovery_snapshot, + demote_connected_device_handle_after_gatt_failure, + discover_known_device_capture_for_status_read, discovered_device, discovered_device_selection, + mark_captured_device_gatt_validated, + pin_connected_device_handle, + retrieve_connected_device_capture, + retrieve_known_device_capture_for_status_read, scan, ) +RecoveryOperationKind = Literal["status-read", "wifi-provision", "ap-enable"] + + +async def _retrieve_connected_inside_ble_lease( + macos_uuid: str, + *, + device_session_id: str, + operation_kind: RecoveryOperationKind = "status-read", +) -> scanner_module.CapturedDiscoveredDevice | None: + return await run_ble_operation( + operation_kind, + hard_timeout_seconds=2.0, + operation=lambda _progress: retrieve_connected_device_capture( + macos_uuid, + device_session_id=device_session_id, + ), + ) + @pytest.fixture(autouse=True) -def reset_runtime_handle_lease() -> Iterator[None]: - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 +def reset_runtime_handle_lease(tmp_path: Path) -> Iterator[None]: + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + configure_ble_runtime_process_lease(tmp_path) yield - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + + +class _FakeCoreBluetoothIdentifier: + def __init__(self, value: str) -> None: + self.value = value + + def UUIDString(self) -> str: # noqa: N802 - mirrors NSUUID + return self.value + + +class _FakeCoreBluetoothPeripheral: + def __init__(self, identifier: _FakeCoreBluetoothIdentifier, name: str) -> None: + self._identifier = identifier + self._name = name + + def identifier(self) -> _FakeCoreBluetoothIdentifier: + return self._identifier + + def name(self) -> str: + return self._name + + +class _FakeCentralManager: + def __init__( + self, + responses: list[list[_FakeCoreBluetoothPeripheral]], + ) -> None: + self.responses = responses + self.requests: list[list[object]] = [] + + def retrievePeripheralsWithIdentifiers_( # noqa: N802 - mirrors CoreBluetooth + self, + identifiers: list[object], + ) -> list[_FakeCoreBluetoothPeripheral]: + self.requests.append(identifiers) + return self.responses.pop(0) + + +class _FakeCoreBluetoothManager: + def __init__( + self, + loop: asyncio.AbstractEventLoop, + central_manager: _FakeCentralManager, + ) -> None: + self.event_loop = loop + self.central_manager = central_manager + self.ready_calls = 0 + + async def wait_until_ready(self) -> None: + self.ready_calls += 1 + + +def _active_status_read_runtime(owner_epoch: int = 7) -> dict[str, object]: + return { + "owner_epoch": owner_epoch, + "owner_loop_bound": True, + "active_operation_kind": "status-read", + "cleanup_pending": False, + "poisoned": False, + } def test_advertisement_record_marks_k1_candidate() -> None: @@ -85,11 +179,14 @@ def test_scan_retains_the_live_corebluetooth_handle( fake_discover, ) - result = asyncio.run(scan(1.0)) + async def scenario() -> object: + result = await scan(1.0) + assert result["devices"][0]["macos_uuid"] == "LIVE-UUID" + assert discovered_device("LIVE-UUID") is device + assert discovered_device_selection("LIVE-UUID").from_fresh_scan is True + return result - assert result["devices"][0]["macos_uuid"] == "LIVE-UUID" - assert discovered_device("LIVE-UUID") is device - assert discovered_device_selection("LIVE-UUID").from_fresh_scan is True + asyncio.run(scenario()) def test_scan_start_invalidates_previous_lease_and_failure_leaves_it_empty( @@ -142,13 +239,8 @@ def test_scan_start_invalidates_previous_lease_and_failure_leaves_it_empty( asyncio.run(scenario()) -@pytest.mark.parametrize("older_scan_fails", [False, True]) -def test_late_scan_generation_cannot_replace_or_clear_newer_lease( - monkeypatch: MonkeyPatch, - older_scan_fails: bool, -) -> None: +def test_process_arbiter_rejects_overlapping_scan(monkeypatch: MonkeyPatch) -> None: older_device = BLEDevice("OLDER-UUID", "XGR-OLDER", details=object()) - newer_device = BLEDevice("NEWER-UUID", "XGR-NEWER", details=object()) older_advertisement = AdvertisementData( local_name="XGR-OLDER", manufacturer_data={}, @@ -158,57 +250,95 @@ def test_late_scan_generation_cannot_replace_or_clear_newer_lease( rssi=-51, platform_data=(), ) - newer_advertisement = AdvertisementData( - local_name="XGR-NEWER", - manufacturer_data={}, - service_data={}, - service_uuids=[], - tx_power=0, - rssi=-31, - platform_data=(), - ) async def scenario() -> None: - call_count = 0 older_scan_started = asyncio.Event() release_older_scan = asyncio.Event() + busy_scan_admitted = False - async def overlapping_discover( + async def blocked_discover( **_kwargs: object, ) -> dict[str, tuple[BLEDevice, AdvertisementData]]: - nonlocal call_count - call_count += 1 - if call_count == 1: - older_scan_started.set() - await release_older_scan.wait() - if older_scan_fails: - raise RuntimeError("late older scan failed") - return {older_device.address: (older_device, older_advertisement)} - return {newer_device.address: (newer_device, newer_advertisement)} + older_scan_started.set() + await release_older_scan.wait() + return {older_device.address: (older_device, older_advertisement)} - monkeypatch.setattr(scanner_module.BleakScanner, "discover", overlapping_discover) + monkeypatch.setattr(scanner_module.BleakScanner, "discover", blocked_discover) older_task = asyncio.create_task(scan(1.0)) await older_scan_started.wait() - await scan(1.0) - assert discovered_device(newer_device.address) is newer_device - assert discovered_device(older_device.address) is None + def record_busy_admission() -> None: + nonlocal busy_scan_admitted + busy_scan_admitted = True + + with pytest.raises(BleRuntimeBusy) as raised: + await scan(1.0, on_admitted=record_busy_admission) + assert raised.value.reason_code == "ble-runtime-busy" + assert raised.value.active_operation_kind == "scan" + assert busy_scan_admitted is False release_older_scan.set() - if older_scan_fails: - with pytest.raises(RuntimeError, match="late older scan failed"): - await older_task - else: - await older_task - - # Neither a late success nor a late failure owns the current lease. - assert discovered_device(newer_device.address) is newer_device - assert discovered_device(older_device.address) is None + await older_task + assert discovered_device(older_device.address) is older_device asyncio.run(scenario()) -def test_runtime_handle_outlives_operator_candidate_boundary( +def test_owner_epoch_rebind_rejects_late_handle_publish() -> None: + old_device = BLEDevice("OLD-UUID", "XGR-OLD", details=object()) + late_device = BLEDevice("LATE-UUID", "XGR-LATE", details=object()) + first_loop = asyncio.new_event_loop() + try: + first_epoch, generation = first_loop.run_until_complete(_publish_owned_handle(old_device)) + finally: + first_loop.close() + + second_loop = asyncio.new_event_loop() + try: + second_epoch = second_loop.run_until_complete( + _rebind_and_reject_late_publish( + generation, + first_epoch, + old_device, + late_device, + ) + ) + assert second_epoch > first_epoch + assert scanner_module._runtime_handles == {} # noqa: SLF001 + finally: + second_loop.close() + + +async def _publish_owned_handle(device: BLEDevice) -> tuple[int, int]: + owner_epoch = bind_ble_runtime_owner_loop() + generation = scanner_module._begin_scan_generation(owner_epoch) # noqa: SLF001 + scanner_module._publish_scan_handles( # noqa: SLF001 + generation, + owner_epoch, + {device.address: device}, + ) + assert discovered_device(device.address) is device + return owner_epoch, generation + + +async def _rebind_and_reject_late_publish( + generation: int, + old_owner_epoch: int, + old_device: BLEDevice, + late_device: BLEDevice, +) -> int: + new_owner_epoch = bind_ble_runtime_owner_loop() + assert discovered_device(old_device.address) is None + scanner_module._publish_scan_handles( # noqa: SLF001 + generation, + old_owner_epoch, + {old_device.address: old_device, late_device.address: late_device}, + ) + assert discovered_device(late_device.address) is None + return new_owner_epoch + + +def test_runtime_handle_from_latest_scan_does_not_expire_by_age( monkeypatch: MonkeyPatch, ) -> None: clock = [100.0] @@ -221,4 +351,1243 @@ def test_runtime_handle_outlives_operator_candidate_boundary( assert discovered_device(handle.address) is handle clock[0] = 100.0 + scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 0.001 - assert discovered_device(handle.address) is None + assert discovered_device(handle.address) is handle + + +def test_connected_handle_is_session_scoped_and_survives_wall_clock_age( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + device = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + advertisement = AdvertisementData( + local_name="XGR-LIVE", + manufacturer_data={}, + service_data={}, + service_uuids=[], + tx_power=0, + rssi=-41, + platform_data=(), + ) + discoveries = [ + {device.address: (device, advertisement)}, + {}, + ] + + async def fake_discover(**_kwargs: object) -> dict[str, tuple[BLEDevice, AdvertisementData]]: + return discoveries.pop(0) + + monkeypatch.setattr(scanner_module.BleakScanner, "discover", fake_discover) + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + + async def scenario() -> None: + await scan(1.0) + captured = capture_discovered_device(device.address) + assert captured is not None + pin_connected_device_handle(captured, device_session_id="device-session-a") + + await scan(1.0) + assert discovered_device_selection( + device.address + ) == scanner_module.DiscoveredDeviceSelection( + device=None, + from_fresh_scan=True, + ) + retained_capture = connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + assert retained_capture is not None + assert retained_capture.device is captured.device + assert retained_capture.source == "retained-session" + owner_epoch_for_loop = scanner_module.ble_runtime_owner_epoch_for_current_loop + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: None, + ) + assert discovered_device_selection( + device.address + ) == scanner_module.DiscoveredDeviceSelection( + device=None, + from_fresh_scan=False, + ) + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + owner_epoch_for_loop, + ) + retained_capture = connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + assert retained_capture is not None + assert retained_capture.device is captured.device + assert ( + connected_device_capture( + device.address, + device_session_id="device-session-b", + ) + is None + ) + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + } + + monotonic_clock[0] += scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 0.001 + suspend_aware_clock[0] += ( + scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 0.001 + ) + assert connected_device_capture( + device.address, + device_session_id="device-session-a", + ) is not None + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + } + + asyncio.run(scenario()) + + +def test_scan_and_retained_handle_survive_macos_sleep_until_explicit_invalidation( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + device = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + advertisement = AdvertisementData( + local_name="XGR-LIVE", + manufacturer_data={}, + service_data={}, + service_uuids=[], + tx_power=0, + rssi=-41, + platform_data=(), + ) + + async def fake_discover( + **_kwargs: object, + ) -> dict[str, tuple[BLEDevice, AdvertisementData]]: + return {device.address: (device, advertisement)} + + monkeypatch.setattr(scanner_module.BleakScanner, "discover", fake_discover) + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + + async def scenario() -> None: + await scan(1.0) + captured = capture_discovered_device(device.address) + assert captured is not None + assert mark_captured_device_gatt_validated(captured) is True + pin_connected_device_handle(captured, device_session_id="device-session-a") + assert discovered_device(device.address) is device + retained_capture = connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + assert retained_capture is not None + assert retained_capture.device is captured.device + assert retained_capture.source == "retained-session" + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": True, + "gatt_validated_recently": True, + } + + # macOS monotonic time may stop while the laptop sleeps. Wall time is + # deliberately the only clock advanced here. GATT validation has the + # shortest lease, so it is revoked before the scan/retained handles. + suspend_aware_clock[0] += ( + scanner_module.BLE_GATT_VALIDATION_RECENCY_TTL_SECONDS + 0.001 + ) + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": True, + "gatt_validated_recently": False, + } + + suspend_aware_clock[0] += ( + scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + - scanner_module.BLE_GATT_VALIDATION_RECENCY_TTL_SECONDS + ) + + assert discovered_device(device.address) is device + assert scanner_module.captured_device_handle(captured) is device + assert connected_device_capture( + device.address, + device_session_id="device-session-a", + ) is not None + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": True, + "gatt_validated_recently": False, + } + + asyncio.run(scenario()) + + +def test_slow_gatt_handoff_refreshes_capture_and_retained_lease_together( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + device = BLEDevice("SLOW-UUID", "XGR-SLOW", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=7, + captured_at_monotonic=monotonic_clock[0], + captured_at_suspend_aware=suspend_aware_clock[0], + ) + + # A real connect/baseline can outlive the original scan lease. Its + # exact successful GATT result is the new freshness boundary. + elapsed = scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 2.0 + monotonic_clock[0] += elapsed + suspend_aware_clock[0] += elapsed + assert scanner_module.captured_device_handle(captured) is device + assert mark_captured_device_gatt_validated(captured) is True + pin_connected_device_handle(captured, device_session_id="device-session-a") + + retained = connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + assert retained is not None + assert retained.device is device + assert retained.source == "retained-session" + assert retained.captured_at_monotonic == monotonic_clock[0] + assert retained.captured_at_suspend_aware == suspend_aware_clock[0] + assert scanner_module.captured_device_handle(retained) is device + + asyncio.run(scenario()) + + +def test_validated_expired_retained_capture_repins_to_new_session( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + device = BLEDevice("ROTATE-UUID", "XGR-ROTATE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=11, + captured_at_monotonic=monotonic_clock[0], + captured_at_suspend_aware=suspend_aware_clock[0], + ) + pin_connected_device_handle(captured, device_session_id="old-session") + + elapsed = scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 2.0 + monotonic_clock[0] += elapsed + suspend_aware_clock[0] += elapsed + assert connected_device_capture( + device.address, + device_session_id="old-session", + ) is not None + + # The exact new GATT baseline, not the old scan timestamp, is the + # admission boundary for rotating this same transport into a new + # facade device session. + assert mark_captured_device_gatt_validated(captured) is True + pin_connected_device_handle(captured, device_session_id="new-session") + + assert connected_device_capture( + device.address, + device_session_id="old-session", + ) is None + repinned = connected_device_capture( + device.address, + device_session_id="new-session", + ) + assert repinned is not None + assert repinned.device is device + assert repinned.captured_at_monotonic == monotonic_clock[0] + assert repinned.captured_at_suspend_aware == suspend_aware_clock[0] + assert connected_device_recovery_snapshot( + device.address, + device_session_id="new-session", + )["gatt_validated_recently"] is True + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("source", ["retrieved-session", "retrieved-durable"]) +def test_retrieved_capture_cannot_be_pinned_before_exact_gatt_validation( + source: str, +) -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + device = BLEDevice("UNVALIDATED-UUID", "XGR-UNVALIDATED", details=object()) + now = scanner_module._freshness_now() # noqa: SLF001 + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source=source, # type: ignore[arg-type] + ) + + with pytest.raises(RuntimeError, match="requires exact GATT validation"): + pin_connected_device_handle(captured, device_session_id="new-session") + assert connected_device_recovery_snapshot( + device.address, + device_session_id="new-session", + )["status"] == "unavailable" + + asyncio.run(scenario()) + + +def test_gatt_handoff_token_retains_and_matches_exact_device_object() -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + validated_device = BLEDevice("HANDOFF-UUID", "XGR-HANDOFF", details=object()) + different_device = BLEDevice("HANDOFF-UUID", "XGR-HANDOFF", details=object()) + now = scanner_module._freshness_now() # noqa: SLF001 + validated = scanner_module.CapturedDiscoveredDevice( + device=validated_device, + macos_uuid=validated_device.address, + owner_epoch=owner_epoch, + scan_generation=13, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + impostor = scanner_module.CapturedDiscoveredDevice( + device=different_device, + macos_uuid=different_device.address, + owner_epoch=owner_epoch, + scan_generation=13, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + + assert mark_captured_device_gatt_validated(validated) is True + with pytest.raises(RuntimeError, match="requires exact GATT validation"): + pin_connected_device_handle(impostor, device_session_id="device-session-a") + + pin_connected_device_handle(validated, device_session_id="device-session-a") + retained = connected_device_capture( + validated_device.address, + device_session_id="device-session-a", + ) + assert retained is not None + assert retained.device is validated_device + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "operation_kind", + ["status-read", "wifi-provision", "ap-enable"], +) +def test_retained_handle_survives_age_and_can_be_explicitly_retrieved( + monkeypatch: MonkeyPatch, + operation_kind: RecoveryOperationKind, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + identifier = _FakeCoreBluetoothIdentifier("RETRIEVE-UUID") + original_peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-OLD") + retrieved_peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-NEW") + central = _FakeCentralManager([[retrieved_peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + original_device = BLEDevice( + identifier.value, + "XGR-OLD", + details=(original_peripheral, manager), + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=original_device, + macos_uuid=original_device.address, + owner_epoch=owner_epoch, + scan_generation=9, + captured_at_monotonic=monotonic_clock[0], + captured_at_suspend_aware=suspend_aware_clock[0], + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + + elapsed = scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 0.001 + monotonic_clock[0] += elapsed + suspend_aware_clock[0] += elapsed + assert scanner_module.captured_device_handle(captured) is original_device + assert connected_device_capture( + original_device.address, + device_session_id="device-session-a", + ) is not None + assert connected_device_recovery_snapshot( + original_device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + } + + retrieved = await _retrieve_connected_inside_ble_lease( + original_device.address, + device_session_id="device-session-a", + operation_kind=operation_kind, + ) + assert retrieved is not None + assert retrieved.source == "retrieved-session" + assert retrieved.device is not original_device + assert retrieved.device.details == (retrieved_peripheral, manager) + assert scanner_module.captured_device_handle(retrieved) is retrieved.device + assert central.requests == [[identifier]] + assert manager.ready_calls == 1 + assert connected_device_recovery_snapshot( + original_device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + } + + asyncio.run(scenario()) + + +def test_gatt_failure_invalidates_session_token_until_new_explicit_scan() -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + identifier = _FakeCoreBluetoothIdentifier("RETRY-UUID") + original_peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-OLD") + retrieved_peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-NEW") + central = _FakeCentralManager([[], [retrieved_peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + original_device = BLEDevice( + identifier.value, + "XGR-OLD", + details=(original_peripheral, manager), + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=original_device, + macos_uuid=original_device.address, + owner_epoch=owner_epoch, + scan_generation=3, + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + assert demote_connected_device_handle_after_gatt_failure(captured) is True + + assert ( + await _retrieve_connected_inside_ble_lease( + original_device.address, + device_session_id="device-session-a", + ) + is None + ) + assert connected_device_recovery_snapshot( + original_device.address, + device_session_id="device-session-a", + )["status"] == "unavailable" + assert central.requests == [] + + asyncio.run(scenario()) + + +def test_retrieval_rejects_uuid_session_and_owner_mismatch( + monkeypatch: MonkeyPatch, +) -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + identifier = _FakeCoreBluetoothIdentifier("EXACT-UUID") + peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-EXACT") + central = _FakeCentralManager([[peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + device = BLEDevice( + identifier.value, + "XGR-EXACT", + details=(peripheral, manager), + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + + assert ( + await _retrieve_connected_inside_ble_lease( + "OTHER-UUID", + device_session_id="device-session-a", + ) + is None + ) + assert ( + await _retrieve_connected_inside_ble_lease( + device.address, + device_session_id="device-session-b", + ) + is None + ) + owner_epoch_lookup = scanner_module.ble_runtime_owner_epoch_for_current_loop + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: owner_epoch + 1, + ) + assert ( + await _retrieve_connected_inside_ble_lease( + device.address, + device_session_id="device-session-a", + ) + is None + ) + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + owner_epoch_lookup, + ) + assert central.requests == [] + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["status"] == "retained" + + asyncio.run(scenario()) + + +def test_connected_retrieval_requires_active_owned_ble_lease() -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + identifier = _FakeCoreBluetoothIdentifier("LEASE-UUID") + peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-LEASE") + central = _FakeCentralManager([[peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + device = BLEDevice( + identifier.value, + "XGR-LEASE", + details=(peripheral, manager), + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + + assert ( + await retrieve_connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + is None + ) + assert central.requests == [] + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["status"] == "retained" + + asyncio.run(scenario()) + + +def test_connected_retrieval_rechecks_same_operation_kind_after_native_await( + monkeypatch: MonkeyPatch, +) -> None: + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + identifier = _FakeCoreBluetoothIdentifier("KIND-UUID") + peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-KIND") + central = _FakeCentralManager([[peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + device = BLEDevice(identifier.value, "XGR-KIND", details=(peripheral, manager)) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + runtime = { + "owner_epoch": owner_epoch, + "owner_loop_bound": True, + "active_operation_kind": "status-read", + "cleanup_pending": False, + "poisoned": False, + } + monkeypatch.setattr(scanner_module, "ble_runtime_snapshot", lambda: dict(runtime)) + + async def change_active_operation() -> None: + manager.ready_calls += 1 + runtime["active_operation_kind"] = "wifi-provision" + + manager.wait_until_ready = change_active_operation # type: ignore[method-assign] + + assert ( + await retrieve_connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + is None + ) + assert central.requests == [[identifier]] + # A changed lease never consumes or replaces the exact process token. + runtime["active_operation_kind"] = None + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["status"] == "retained" + + asyncio.run(scenario()) + + +def test_connected_device_recovery_name_is_exact_but_not_presence( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + device = BLEDevice("NAME-UUID", " XGR-EXACT ", details=object()) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + captured_at_monotonic=monotonic_clock[0], + captured_at_suspend_aware=suspend_aware_clock[0], + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + elapsed = scanner_module.BLE_RUNTIME_HANDLE_LEASE_TTL_SECONDS + 1.0 + monotonic_clock[0] += elapsed + suspend_aware_clock[0] += elapsed + + assert connected_device_capture( + device.address, + device_session_id="device-session-a", + ) is not None + assert connected_device_recovery_name( + device.address, + device_session_id="device-session-a", + ) == "XGR-EXACT" + assert connected_device_recovery_name( + device.address, + device_session_id="other-session", + ) is None + snapshot = connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) + assert snapshot["status"] == "retained" + assert snapshot["advertised_now"] is False + assert snapshot["gatt_validated_recently"] is False + + asyncio.run(scenario()) + + +def test_durable_uuid_retrieval_is_available_only_inside_explicit_status_read( + monkeypatch: MonkeyPatch, +) -> None: + macos_uuid = "11111111-2222-4333-8444-555555555555" + + async def scenario() -> None: + identifier = _FakeCoreBluetoothIdentifier(macos_uuid) + peripheral = _FakeCoreBluetoothPeripheral(identifier, "XGR-DURABLE") + central = _FakeCentralManager([[peripheral]]) + manager = _FakeCoreBluetoothManager(asyncio.get_running_loop(), central) + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: 7, + ) + runtime = _active_status_read_runtime() + monkeypatch.setattr(scanner_module, "ble_runtime_snapshot", lambda: runtime) + monkeypatch.setattr( + scanner_module, + "_new_corebluetooth_retrieval_context", + lambda requested: (manager, identifier) if requested == macos_uuid else None, + ) + + captured = await retrieve_known_device_capture_for_status_read(macos_uuid) + + assert captured is not None + assert captured.macos_uuid == macos_uuid + assert captured.source == "retrieved-durable" + assert captured.owner_epoch == 7 + assert captured.device.details == (peripheral, manager) + assert central.requests == [[identifier]] + assert manager.ready_calls == 1 + # Retrieval is only an opaque transport object. It is neither added to + # discovery nor pinned as a current device session by this helper. + assert discovered_device(macos_uuid) is None + assert connected_device_recovery_snapshot( + macos_uuid, + device_session_id="any-session", + )["status"] == "unavailable" + + asyncio.run(scenario()) + + +def test_durable_uuid_retrieval_rejects_non_status_operation_before_native_call( + monkeypatch: MonkeyPatch, +) -> None: + calls: list[str] = [] + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: 7, + ) + monkeypatch.setattr( + scanner_module, + "ble_runtime_snapshot", + lambda: { + **_active_status_read_runtime(), + "active_operation_kind": "wifi-provision", + }, + ) + monkeypatch.setattr( + scanner_module, + "_new_corebluetooth_retrieval_context", + lambda requested: calls.append(requested), + ) + + assert ( + asyncio.run( + retrieve_known_device_capture_for_status_read( + "11111111-2222-4333-8444-555555555555" + ) + ) + is None + ) + assert calls == [] + + +def test_exact_uuid_advertisement_scan_captures_without_publishing_discovery( + monkeypatch: MonkeyPatch, +) -> None: + macos_uuid = "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE" + native_details = object() + candidate = BLEDevice( + macos_uuid.lower(), + "XGR-ADVERTISEMENT", + details=native_details, + ) + calls: list[tuple[str, float]] = [] + + async def find_exact(address: str, *, timeout: float) -> BLEDevice: + calls.append((address, timeout)) + return candidate + + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: 7, + ) + monkeypatch.setattr( + scanner_module, + "ble_runtime_snapshot", + lambda: _active_status_read_runtime(), + ) + monkeypatch.setattr( + scanner_module.BleakScanner, + "find_device_by_address", + find_exact, + ) + + captured = asyncio.run( + discover_known_device_capture_for_status_read( + macos_uuid, + timeout_seconds=30.0, + ) + ) + + assert captured is not None + assert captured.macos_uuid == macos_uuid + assert captured.device.address == macos_uuid + assert captured.device.details is native_details + assert captured.source == "retrieved-durable" + assert calls == [(macos_uuid, 30.0)] + assert discovered_device(macos_uuid) is None + assert connected_device_recovery_snapshot( + macos_uuid, + device_session_id="any-session", + )["status"] == "unavailable" + + +def test_exact_uuid_advertisement_scan_rejects_wrong_uuid( + monkeypatch: MonkeyPatch, +) -> None: + macos_uuid = "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE" + wrong = BLEDevice( + "11111111-2222-4333-8444-555555555555", + "XGR-WRONG", + details=object(), + ) + + async def return_wrong(_address: str, *, timeout: float) -> BLEDevice: + assert timeout == 30.0 + return wrong + + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: 7, + ) + monkeypatch.setattr( + scanner_module, + "ble_runtime_snapshot", + lambda: _active_status_read_runtime(), + ) + monkeypatch.setattr( + scanner_module.BleakScanner, + "find_device_by_address", + return_wrong, + ) + + assert ( + asyncio.run( + discover_known_device_capture_for_status_read( + macos_uuid, + timeout_seconds=30.0, + ) + ) + is None + ) + assert discovered_device(macos_uuid) is None + + +def test_exact_uuid_advertisement_scan_rejects_timeout_and_owner_change( + monkeypatch: MonkeyPatch, +) -> None: + macos_uuid = "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE" + candidate = BLEDevice(macos_uuid, "XGR-LATE", details=object()) + runtime_snapshots = [ + _active_status_read_runtime(), + _active_status_read_runtime(owner_epoch=8), + ] + responses: list[BLEDevice | None] = [None, candidate] + + async def find_exact(_address: str, *, timeout: float) -> BLEDevice | None: + assert timeout == 30.0 + return responses.pop(0) + + monkeypatch.setattr( + scanner_module, + "ble_runtime_owner_epoch_for_current_loop", + lambda: 7, + ) + monkeypatch.setattr( + scanner_module.BleakScanner, + "find_device_by_address", + find_exact, + ) + + monkeypatch.setattr( + scanner_module, + "ble_runtime_snapshot", + lambda: _active_status_read_runtime(), + ) + assert ( + asyncio.run( + discover_known_device_capture_for_status_read( + macos_uuid, + timeout_seconds=30.0, + ) + ) + is None + ) + + monkeypatch.setattr( + scanner_module, + "ble_runtime_snapshot", + lambda: runtime_snapshots.pop(0), + ) + assert ( + asyncio.run( + discover_known_device_capture_for_status_read( + macos_uuid, + timeout_seconds=30.0, + ) + ) + is None + ) + assert responses == [] + assert discovered_device(macos_uuid) is None + + +def test_pending_gatt_handoff_expires_across_macos_sleep( + monkeypatch: MonkeyPatch, +) -> None: + monotonic_clock = [100.0] + suspend_aware_clock = [1_000.0] + monkeypatch.setattr(scanner_module, "monotonic", lambda: monotonic_clock[0]) + monkeypatch.setattr( + scanner_module, + "suspend_aware_time", + lambda: suspend_aware_clock[0], + ) + device = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=7, + captured_at_monotonic=monotonic_clock[0], + captured_at_suspend_aware=suspend_aware_clock[0], + ) + assert mark_captured_device_gatt_validated(captured) is True + + suspend_aware_clock[0] += ( + scanner_module.BLE_GATT_VALIDATION_HANDOFF_TTL_SECONDS + 0.001 + ) + pin_connected_device_handle(captured, device_session_id="device-session-a") + + snapshot = connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) + assert snapshot["status"] == "retained" + assert snapshot["gatt_validated_recently"] is False + + asyncio.run(scenario()) + + +def test_later_fresh_scan_does_not_replace_retained_corebluetooth_object( + monkeypatch: MonkeyPatch, +) -> None: + first = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + second = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + advertisement = AdvertisementData( + local_name="XGR-LIVE", + manufacturer_data={}, + service_data={}, + service_uuids=[], + tx_power=0, + rssi=-38, + platform_data=(), + ) + discoveries = [ + {first.address: (first, advertisement)}, + {second.address: (second, advertisement)}, + ] + + async def fake_discover(**_kwargs: object) -> dict[str, tuple[BLEDevice, AdvertisementData]]: + return discoveries.pop(0) + + monkeypatch.setattr(scanner_module.BleakScanner, "discover", fake_discover) + + async def scenario() -> None: + await scan(1.0) + captured = capture_discovered_device(first.address) + assert captured is not None + assert captured.device is first + pin_connected_device_handle(captured, device_session_id="device-session-a") + + await scan(1.0) + + retained = connected_device_capture( + first.address, + device_session_id="device-session-a", + ) + assert retained is not None + assert retained is not captured + assert retained.device is first + assert retained.scan_generation == captured.scan_generation + next_capture = capture_discovered_device(first.address) + assert next_capture is not None + assert next_capture.device is second + assert next_capture.scan_generation > captured.scan_generation + assert connected_device_recovery_snapshot( + first.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": True, + "gatt_validated_recently": False, + } + + asyncio.run(scenario()) + + +def test_exact_gatt_validation_handoff_marks_newly_pinned_recovery_token() -> None: + device = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=7, + captured_at_monotonic=100.0, + ) + + # The reviewed baseline read completes immediately before the facade + # creates its new device-session. Validation follows the exact object + # across only that bounded synchronous handoff. + assert mark_captured_device_gatt_validated(captured) is True + pin_connected_device_handle(captured, device_session_id="device-session-a") + + assert connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["gatt_validated_recently"] is True + + asyncio.run(scenario()) + + +def test_failed_gatt_demotes_only_the_exact_retained_corebluetooth_object() -> None: + first = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + replacement = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + stale_capture = scanner_module.CapturedDiscoveredDevice( + device=first, + macos_uuid=first.address, + owner_epoch=owner_epoch, + scan_generation=7, + ) + fresh_capture = scanner_module.CapturedDiscoveredDevice( + device=replacement, + macos_uuid=replacement.address, + owner_epoch=owner_epoch, + scan_generation=8, + ) + pin_connected_device_handle( + stale_capture, + device_session_id="device-session-a", + ) + + assert demote_connected_device_handle_after_gatt_failure(stale_capture) is True + assert ( + connected_device_capture( + first.address, + device_session_id="device-session-a", + ) + is None + ) + + # A late failure from the old generation must not revoke a replacement + # object already admitted for the same CoreBluetooth UUID/session. + pin_connected_device_handle( + fresh_capture, + device_session_id="device-session-a", + ) + assert demote_connected_device_handle_after_gatt_failure(stale_capture) is False + retained_replacement = connected_device_capture( + replacement.address, + device_session_id="device-session-a", + ) + assert retained_replacement is not None + assert retained_replacement.device is replacement + assert retained_replacement.scan_generation == fresh_capture.scan_generation + assert retained_replacement.source == "retained-session" + + asyncio.run(scenario()) + + +def test_failed_gatt_invalidates_unpinned_selection_until_new_scan_generation() -> None: + first = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + replacement = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + failed = scanner_module.CapturedDiscoveredDevice( + device=first, + macos_uuid=first.address, + owner_epoch=owner_epoch, + scan_generation=7, + ) + assert demote_connected_device_handle_after_gatt_failure(failed) is False + assert scanner_module.captured_device_handle(failed) is None + + next_scan = scanner_module.CapturedDiscoveredDevice( + device=replacement, + macos_uuid=replacement.address, + owner_epoch=owner_epoch, + scan_generation=8, + ) + assert scanner_module.captured_device_handle(next_scan) is replacement + + asyncio.run(scenario()) + + +def test_invalidated_visible_row_requires_a_new_exact_scan_object() -> None: + first = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + replacement = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + first_generation = scanner_module._begin_scan_generation(owner_epoch) # noqa: SLF001 + scanner_module._publish_scan_handles( # noqa: SLF001 + first_generation, + owner_epoch, + {first.address: first}, + ) + first_capture = capture_discovered_device(first.address) + assert first_capture is not None + assert first_capture.device is first + assert mark_captured_device_gatt_validated(first_capture) is True + pin_connected_device_handle( + first_capture, + device_session_id="verify-provisional-session", + ) + + # A losing read-only Verify retires its provisional device session but + # may leave the facade's row visible for product continuity. The native + # object behind that same generation must nevertheless stay revoked. + assert scanner_module.invalidate_connected_device_session( + first.address, + device_session_id="verify-provisional-session", + ) is True + invalidated_selection = discovered_device_selection(first.address) + assert invalidated_selection.from_fresh_scan is True + assert invalidated_selection.device is None + assert capture_discovered_device(first.address) is None + assert scanner_module.captured_device_handle(first_capture) is None + + second_generation = scanner_module._begin_scan_generation(owner_epoch) # noqa: SLF001 + scanner_module._publish_scan_handles( # noqa: SLF001 + second_generation, + owner_epoch, + {replacement.address: replacement}, + ) + replacement_capture = capture_discovered_device(replacement.address) + assert replacement_capture is not None + assert replacement_capture.device is replacement + assert replacement_capture.scan_generation == second_generation + assert scanner_module.captured_device_handle(replacement_capture) is replacement + + # Neither the old object nor a capture manufactured for another owner + # epoch gains authority from the new scan. + assert scanner_module.captured_device_handle(first_capture) is None + foreign_capture = scanner_module.CapturedDiscoveredDevice( + device=replacement, + macos_uuid=replacement.address, + owner_epoch=owner_epoch + 1, + scan_generation=second_generation, + ) + assert scanner_module.captured_device_handle(foreign_capture) is None + + asyncio.run(scenario()) + + +def test_exact_session_invalidation_does_not_clear_new_scan_handle() -> None: + retained = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + newly_scanned = BLEDevice("LIVE-UUID", "XGR-LIVE", details=object()) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=retained, + macos_uuid=retained.address, + owner_epoch=owner_epoch, + scan_generation=7, + ) + assert mark_captured_device_gatt_validated(captured) is True + pin_connected_device_handle(captured, device_session_id="device-session-a") + next_generation = scanner_module._begin_scan_generation(owner_epoch) # noqa: SLF001 + scanner_module._publish_scan_handles( # noqa: SLF001 + next_generation, + owner_epoch, + {newly_scanned.address: newly_scanned}, + ) + replacement_capture = scanner_module.capture_discovered_device(newly_scanned.address) + assert replacement_capture is not None + assert replacement_capture.device is newly_scanned + + assert scanner_module.invalidate_connected_device_session( + retained.address, + device_session_id="device-session-a", + ) is True + assert connected_device_capture( + retained.address, + device_session_id="device-session-a", + ) is None + assert scanner_module.captured_device_handle(captured) is None + with pytest.raises(RuntimeError, match="invalidated by session teardown"): + pin_connected_device_handle(captured, device_session_id="stale-repin") + with scanner_module._runtime_handle_lock: # noqa: SLF001 + assert ( # noqa: SLF001 + scanner_module._runtime_handles[newly_scanned.address] is newly_scanned + ) + assert scanner_module.captured_device_handle(replacement_capture) is newly_scanned + assert mark_captured_device_gatt_validated(replacement_capture) is True + pin_connected_device_handle( + replacement_capture, + device_session_id="device-session-b", + ) + assert connected_device_capture( + newly_scanned.address, + device_session_id="device-session-b", + ) is not None + + asyncio.run(scenario()) diff --git a/tests/test_canonical_pipeline.py b/tests/test_canonical_pipeline.py index 3c61cb2..e4ef463 100644 --- a/tests/test_canonical_pipeline.py +++ b/tests/test_canonical_pipeline.py @@ -12,6 +12,8 @@ import lz4.block import pytest import k1link.device_plugins.xgrids_k1.viewer.runtime as runtime_module +import k1link.viewer.rerun_bridge as rerun_bridge_module +from k1link.compute.live_perception import LivePerceptionResultFrame from k1link.data_plane import ( ConsumerFrameContext, DecodedDeviceStatusView, @@ -19,10 +21,15 @@ from k1link.data_plane import ( DecodedPoseView, NormalizationError, ) -from k1link.device_plugins.xgrids_k1.mqtt.capture import FRAME_HEADER, RAW_MAGIC +from k1link.device_plugins.xgrids_k1.mqtt.capture import ( + FRAME_HEADER, + RAW_MAGIC, + CapturedMqttMessage, +) from k1link.device_plugins.xgrids_k1.protocol.normalizer import normalize_k1_message from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage from k1link.device_plugins.xgrids_k1.viewer.runtime import VisualizationRuntime +from k1link.viewer.metrics import BridgeMetrics from k1link.viewer.rerun_bridge import RerunBridge @@ -49,6 +56,707 @@ class FakeRecording: return +@pytest.fixture +def socket_free_rerun_port_selector(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep FakeRecording tests independent of host TCP bind permission.""" + + monkeypatch.setattr( + rerun_bridge_module, + "_select_available_grpc_port", + lambda preferred_port, **_kwargs: preferred_port, + ) + + +class RuntimeBridgeStub: + """Socket-free bridge used for producer-generation lifecycle tests.""" + + grpc_url = "rerun+http://127.0.0.1:9876/proxy" + + def begin_session(self, _metrics: object) -> None: + return + + def process(self, _envelope: object) -> None: + return + + def process_perception(self, _frame: object) -> None: + return + + def close(self) -> None: + return + + +class MetricRuntimeBridgeStub(RuntimeBridgeStub): + """Prove post-publish observers run after the bridge's metric commit.""" + + def __init__(self, metrics: BridgeMetrics) -> None: + self._metrics = metrics + + def process(self, envelope: object) -> None: + if isinstance(envelope, DecodedPointCloudView): + self._metrics.published_pcl( + envelope.point_count, + time.monotonic_ns(), + 0.0, + ) + + +class FailingRuntimeBridgeStub(RuntimeBridgeStub): + def process(self, _envelope: object) -> None: + raise RuntimeError("synthetic rerun publish failure") + + +def test_visualization_runtime_initial_status_describes_only_its_data_source() -> None: + runtime = VisualizationRuntime(normalizer=normalize_k1_message) + + assert runtime.snapshot()["message"] == "Активного источника сейчас нет." + runtime.close() + + +def test_post_publish_observer_sees_only_normalized_frame_after_metric_commit( + tmp_path: Path, +) -> None: + topic = b"RealtimePointcloud" + payload = struct.pack(" None: + topic = b"RealtimePointcloud" + payload = struct.pack(" CapturedMqttMessage: + payload = struct.pack(" CapturedMqttMessage: + return CapturedMqttMessage( + sequence=sequence, + topic="RealtimePath", + payload=struct.pack( + " None: + raw_sequences: list[int] = [] + bridge_sequences: list[tuple[str, int]] = [] + observed_sequences: list[tuple[str, int]] = [] + retained_pose_published = threading.Event() + fresh_point_cloud_published = threading.Event() + + class TrackingBridge(MetricRuntimeBridgeStub): + def process(self, envelope: object) -> None: + if isinstance(envelope, DecodedPointCloudView): + bridge_sequences.append(("pointcloud", envelope.context.sequence)) + elif isinstance(envelope, DecodedPoseView): + bridge_sequences.append(("pose", envelope.context.sequence)) + super().process(envelope) + + def observe(envelope: object, _generation: int) -> None: + if isinstance(envelope, DecodedPointCloudView): + observed_sequences.append(("pointcloud", envelope.context.sequence)) + fresh_point_cloud_published.set() + elif isinstance(envelope, DecodedPoseView): + observed_sequences.append(("pose", envelope.context.sequence)) + retained_pose_published.set() + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + enqueue = callbacks["on_message_recorded"] + + retained_point_cloud = _captured_live_point_cloud(1, retain=True) + retained_pose = _captured_live_pose(2, retain=True) + fresh_point_cloud = _captured_live_point_cloud(3) + for message in (retained_point_cloud, retained_pose): + # `capture_mqtt` invokes this callback only after the raw writer + # has recorded the complete MQTT message, including `retain`. + raw_sequences.append(message.sequence) + enqueue(message) # type: ignore[operator] + + assert retained_pose_published.wait(timeout=2.0) + assert ("pointcloud", 1) not in bridge_sequences + assert ("pointcloud", 1) not in observed_sequences + + raw_sequences.append(fresh_point_cloud.sequence) + enqueue(fresh_point_cloud) # type: ignore[operator] + assert fresh_point_cloud_published.wait(timeout=2.0) + + should_stop = callbacks["should_stop"] + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": len(raw_sequences)} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=lambda **kwargs: TrackingBridge(kwargs["metrics"]), + normalizer=normalize_k1_message, + published_envelope_observer=observe, + ) + runtime.start_live( + "192.168.1.50", + tmp_path / "retained-pcl-freshness", + duration_seconds=None, + project_name="RETAINEDFRESH001", + ) + + assert fresh_point_cloud_published.wait(timeout=3.0) + assert raw_sequences == [1, 2, 3] + assert bridge_sequences == [("pose", 2), ("pointcloud", 3)] + assert observed_sequences == bridge_sequences + snapshot = runtime.snapshot() + assert snapshot["metrics"]["pcl_frames"] == 1 + runtime.stop(wait_seconds=2.0) + runtime.close() + + +def _perception_frame(index: int) -> LivePerceptionResultFrame: + return LivePerceptionResultFrame( + session_id="canonical-fairness-session", + session_generation=1, + frame_index=index, + source_frame_index=index, + session_seconds=float(index), + captured_at_epoch_ns=index, + image_jpeg=b"jpeg", + segmentation_mask=None, + objects=(), + delivery={}, + ) + + +def test_live_rerun_process_and_close_failure_preserve_capture_then_recover( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(runtime_module, "_rerun_recovery_backoff_seconds", lambda _attempt: 0.01) + first_process_failed = threading.Event() + second_bridge_entered = threading.Event() + allow_second_bridge = threading.Event() + second_bridge_ready = threading.Event() + recovery_confirmed = threading.Event() + raw_sequences: list[int] = [] + should_stop_before_explicit_stop: list[bool] = [] + confirmed_attempts: list[int] = [] + factory_calls = 0 + + class ProcessAndCloseFailingBridge(RuntimeBridgeStub): + def process(self, _envelope: object) -> None: + first_process_failed.set() + raise RuntimeError("synthetic live Rerun process failure") + + def close(self) -> None: + raise RuntimeError("synthetic live Rerun close failure") + + class RecoveredBridge(MetricRuntimeBridgeStub): + def begin_session(self, metrics: object) -> None: + super().begin_session(metrics) + second_bridge_entered.set() + assert allow_second_bridge.wait(timeout=2.0) + second_bridge_ready.set() + + def bridge_factory(**kwargs: object) -> RuntimeBridgeStub: + nonlocal factory_calls + factory_calls += 1 + if factory_calls == 1: + return ProcessAndCloseFailingBridge() + return RecoveredBridge(kwargs["metrics"]) # type: ignore[arg-type] + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + def confirm(attempt: int) -> bool: + confirmed_attempts.append(attempt) + recovery_confirmed.set() + return True + + callbacks["on_recovery_confirmer_ready"](confirm) # type: ignore[operator] + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + callbacks["on_connection_lost"]("synthetic MQTT loss") # type: ignore[operator] + assert callbacks["recover_connection"](1) == "resume" # type: ignore[operator] + callbacks["on_recovery_point_cloud_candidate"](1, 1) # type: ignore[operator] + + should_stop = callbacks["should_stop"] + enqueue = callbacks["on_message_recorded"] + should_stop_before_explicit_stop.append(should_stop()) # type: ignore[operator] + raw_sequences.append(1) + enqueue(_captured_live_point_cloud(1)) # type: ignore[operator] + assert first_process_failed.wait(timeout=2.0) + assert confirmed_attempts == [] + + # The failed presentation must not stop capture. Keep accepting raw + # reports while the bounded preview queue applies latest-wins drops. + for sequence in range(2, 10): + should_stop_before_explicit_stop.append(should_stop()) # type: ignore[operator] + raw_sequences.append(sequence) + enqueue(_captured_live_point_cloud(sequence)) # type: ignore[operator] + assert second_bridge_entered.wait(timeout=2.0) + allow_second_bridge.set() + assert second_bridge_ready.wait(timeout=2.0) + raw_sequences.append(10) + enqueue(_captured_live_point_cloud(10)) # type: ignore[operator] + assert recovery_confirmed.wait(timeout=2.0) + + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": len(raw_sequences)} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=bridge_factory, # type: ignore[arg-type] + normalizer=normalize_k1_message, + ) + runtime.start_live( + "192.168.1.50", + tmp_path / "live-rerun-recovery", + duration_seconds=None, + project_name="RERUNRECOVERY001", + recover_connection=lambda _attempt: "resume", + ) + + assert recovery_confirmed.wait(timeout=3.0) + snapshot = runtime.snapshot() + assert snapshot["phase"] == "live" + assert snapshot["source_ready"] is True + assert snapshot["connection_recovery"]["state"] == "recovered" + assert snapshot["rerun_recovery"] == { + "state": "ready", + "attempt": 2, + "reason_code": None, + } + assert snapshot["rerun_grpc_url"] == RuntimeBridgeStub.grpc_url + assert snapshot["metrics"]["preview_dropped"] >= 4 + assert raw_sequences == list(range(1, 11)) + assert should_stop_before_explicit_stop and not any(should_stop_before_explicit_stop) + assert confirmed_attempts == [1] + runtime.stop(wait_seconds=2.0) + assert runtime.snapshot()["phase"] == "idle" + runtime.close() + + +def test_live_latest_point_cloud_survives_pose_and_perception_pressure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + perception_started = threading.Event() + allow_perception_return = threading.Event() + recovery_confirmed = threading.Event() + published_point_cloud_sequences: list[int] = [] + confirmed_attempts: list[int] = [] + perception_calls = 0 + + class FairBridge(MetricRuntimeBridgeStub): + def process(self, envelope: object) -> None: + if isinstance(envelope, DecodedPointCloudView): + published_point_cloud_sequences.append(envelope.context.sequence) + super().process(envelope) + + def process_perception(self, _frame: object) -> None: + nonlocal perception_calls + perception_calls += 1 + perception_started.set() + runtime.publish_perception_frame(_perception_frame(perception_calls + 10)) + assert allow_perception_return.wait(timeout=2.0) + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + def confirm(attempt: int) -> bool: + confirmed_attempts.append(attempt) + recovery_confirmed.set() + return True + + callbacks["on_recovery_confirmer_ready"](confirm) # type: ignore[operator] + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + callbacks["on_connection_lost"]("synthetic MQTT loss") # type: ignore[operator] + assert callbacks["recover_connection"](1) == "resume" # type: ignore[operator] + callbacks["on_recovery_point_cloud_candidate"](1, 5) # type: ignore[operator] + + assert runtime.publish_perception_frame(_perception_frame(1)) + assert runtime.publish_perception_frame(_perception_frame(2)) + assert perception_started.wait(timeout=2.0) + + enqueue = callbacks["on_message_recorded"] + enqueue(_captured_live_point_cloud(5)) # type: ignore[operator] + enqueue(_captured_live_point_cloud(6)) # type: ignore[operator] + for sequence in range(7, 31): + enqueue(_captured_live_pose(sequence)) # type: ignore[operator] + allow_perception_return.set() + assert recovery_confirmed.wait(timeout=2.0) + + should_stop = callbacks["should_stop"] + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": 26} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=lambda **kwargs: FairBridge(kwargs["metrics"]), # type: ignore[arg-type] + normalizer=normalize_k1_message, + ) + runtime.start_live( + "192.168.1.50", + tmp_path / "fair-live-preview", + duration_seconds=None, + project_name="FAIRPREVIEW001", + recover_connection=lambda _attempt: "resume", + ) + + assert recovery_confirmed.wait(timeout=3.0) + snapshot = runtime.snapshot() + assert snapshot["phase"] == "live" + assert published_point_cloud_sequences == [6] + assert confirmed_attempts == [1] + assert perception_calls >= 1 + assert snapshot["metrics"]["preview_dropped"] >= 21 + runtime.stop(wait_seconds=2.0) + runtime.close() + + +@pytest.mark.parametrize("attempt", [5, 1025, 10**100]) +def test_live_rerun_recovery_backoff_saturates(attempt: int) -> None: + assert runtime_module._rerun_recovery_backoff_seconds(attempt) == 5.0 # noqa: SLF001 + + +def test_live_capture_clock_does_not_wait_for_initial_rerun_factory_recovery( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(runtime_module, "_rerun_recovery_backoff_seconds", lambda _attempt: 0.01) + capture_started = threading.Event() + bridge_ready = threading.Event() + factory_calls = 0 + + class ReadyBridge(RuntimeBridgeStub): + def begin_session(self, _metrics: object) -> None: + bridge_ready.set() + + def bridge_factory(**_kwargs: object) -> RuntimeBridgeStub: + nonlocal factory_calls + factory_calls += 1 + if factory_calls == 1: + raise RuntimeError("synthetic initial Rerun factory failure") + return ReadyBridge() + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + capture_started.set() + should_stop = callbacks["should_stop"] + assert should_stop() is False # type: ignore[operator] + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": 0} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=bridge_factory, # type: ignore[arg-type] + normalizer=normalize_k1_message, + ) + started = time.monotonic() + runtime.start_live( + "192.168.1.50", + tmp_path / "initial-rerun-failure", + duration_seconds=None, + project_name="RERUNFACTORY001", + ) + + assert time.monotonic() - started < 1.0 + assert capture_started.is_set() + assert bridge_ready.wait(timeout=2.0) + deadline = time.monotonic() + 2.0 + snapshot = runtime.snapshot() + while snapshot["rerun_recovery"]["state"] != "ready" and time.monotonic() < deadline: + time.sleep(0.005) + snapshot = runtime.snapshot() + assert snapshot["phase"] == "live" + assert snapshot["rerun_recovery"] == { + "state": "ready", + "attempt": 2, + "reason_code": None, + } + runtime.stop(wait_seconds=2.0) + runtime.close() + + +def test_live_quarantine_bounds_repeated_hangs_and_retries_after_publisher_exits( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + factory_entered = threading.Event() + release_factory = threading.Event() + late_bridge_closed = threading.Event() + recovered_bridge_ready = threading.Event() + capture_starts = 0 + factory_calls = 0 + + class LateBridge(RuntimeBridgeStub): + def close(self) -> None: + late_bridge_closed.set() + + class RecoveredBridge(RuntimeBridgeStub): + def begin_session(self, _metrics: object) -> None: + recovered_bridge_ready.set() + + def blocked_factory(**_kwargs: object) -> RuntimeBridgeStub: + nonlocal factory_calls + factory_calls += 1 + if factory_calls == 1: + factory_entered.set() + assert release_factory.wait(timeout=5.0) + return LateBridge() + return RecoveredBridge() + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + nonlocal capture_starts + capture_starts += 1 + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + should_stop = callbacks["should_stop"] + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": 0} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=blocked_factory, # type: ignore[arg-type] + normalizer=normalize_k1_message, + ) + runtime.start_live( + "192.168.1.50", + tmp_path / "blocked-rerun-factory", + duration_seconds=None, + project_name="RERUNBLOCKED001", + ) + assert factory_entered.wait(timeout=2.0) + + stopped_at = time.monotonic() + runtime.stop(wait_seconds=2.0) + assert time.monotonic() - stopped_at < 1.5 + stopped = runtime.snapshot() + assert stopped["phase"] == "idle" + assert stopped["rerun_grpc_url"] is None + assert stopped["rerun_recovery"]["state"] == "stopped" + assert stopped["rerun_recovery"]["reason_code"] == "publisher-shutdown-timeout" + quarantined = runtime._quarantined_live_publisher # noqa: SLF001 + assert quarantined is not None and quarantined.is_alive() + + # A second live session must still establish raw evidence, but it cannot + # allocate a second Rerun/native owner while the first publisher is hung. + runtime.start_live( + "192.168.1.50", + tmp_path / "quarantined-second-session", + duration_seconds=None, + project_name="RERUNQUARANTINE002", + ) + deadline = time.monotonic() + 2.0 + second = runtime.snapshot() + while ( + second["rerun_recovery"]["reason_code"] != "publisher-quarantined" + and time.monotonic() < deadline + ): + time.sleep(0.005) + second = runtime.snapshot() + assert second["phase"] == "live" + assert second["source_ready"] is True + assert second["rerun_recovery"] == { + "state": "retrying", + "attempt": 0, + "reason_code": "publisher-quarantined", + } + assert capture_starts == 2 + assert factory_calls == 1 + assert runtime._quarantined_live_publisher is quarantined # noqa: SLF001 + assert len( + [ + thread + for thread in threading.enumerate() + if thread.name.startswith("k1-rerun-publisher-") + ] + ) == 2 + + # Once the sole quarantined publisher exits, the waiting supervisor may + # allocate one new bridge and recover presentation for the current session. + release_factory.set() + assert late_bridge_closed.wait(timeout=2.0) + assert recovered_bridge_ready.wait(timeout=2.0) + deadline = time.monotonic() + 2.0 + recovered = runtime.snapshot() + while recovered["rerun_recovery"]["state"] != "ready" and time.monotonic() < deadline: + time.sleep(0.005) + recovered = runtime.snapshot() + assert recovered["rerun_recovery"] == { + "state": "ready", + "attempt": 1, + "reason_code": None, + } + assert factory_calls == 2 + assert runtime._quarantined_live_publisher is None # noqa: SLF001 + runtime.stop(wait_seconds=2.0) + + assert runtime.snapshot()["rerun_grpc_url"] is None + runtime.close() + + +def test_live_stop_wakes_long_rerun_retry_backoff( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(runtime_module, "_rerun_recovery_backoff_seconds", lambda _attempt: 60.0) + factory_failed = threading.Event() + + def unavailable_factory(**_kwargs: object) -> RuntimeBridgeStub: + factory_failed.set() + raise RuntimeError("synthetic unavailable Rerun factory") + + def fake_capture_mqtt( + _host: str, + _out_dir: Path, + **callbacks: object, + ) -> dict[str, object]: + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + should_stop = callbacks["should_stop"] + while not should_stop(): # type: ignore[operator] + time.sleep(0.005) + return {"message_count": 0} + + monkeypatch.setattr(runtime_module, "capture_mqtt", fake_capture_mqtt) + runtime = VisualizationRuntime( + bridge_factory=unavailable_factory, # type: ignore[arg-type] + normalizer=normalize_k1_message, + ) + runtime.start_live( + "192.168.1.50", + tmp_path / "rerun-long-backoff", + duration_seconds=None, + project_name="RERUNBACKOFF001", + ) + assert factory_failed.wait(timeout=2.0) + deadline = time.monotonic() + 2.0 + snapshot = runtime.snapshot() + while snapshot["rerun_recovery"]["state"] != "retrying" and time.monotonic() < deadline: + time.sleep(0.005) + snapshot = runtime.snapshot() + + stopped_at = time.monotonic() + runtime.stop(wait_seconds=1.0) + assert time.monotonic() - stopped_at < 0.5 + stopped = runtime.snapshot() + assert stopped["phase"] == "idle" + assert stopped["rerun_recovery"] == { + "state": "stopped", + "attempt": 1, + "reason_code": "bridge-constructor-failed", + } + runtime.close() + + def _message( topic: str, payload: bytes, @@ -137,6 +845,52 @@ def _context() -> ConsumerFrameContext: ) +def _published_point_cloud(sequence: int) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + +def test_runtime_binds_every_observed_message_to_its_producer_generation( + tmp_path: Path, +) -> None: + topic = b"RealtimePointcloud" + payload = b"observer-only" + capture = tmp_path / "mqtt.raw.k1mqtt" + capture.write_bytes(RAW_MAGIC + FRAME_HEADER.pack(len(topic), len(payload)) + topic + payload) + observed: list[int | None] = [] + + def observe(message: StreamMessage, _metrics: BridgeMetrics) -> bool: + observed.append(message.producer_generation) + return True + + runtime = VisualizationRuntime( + bridge_factory=lambda **_kwargs: RuntimeBridgeStub(), # type: ignore[arg-type] + normalizer=normalize_k1_message, + message_observer=observe, + ) + + for expected_generation in (1, 2): + runtime.start_replay(capture, speed=0.0) + deadline = time.monotonic() + 2.0 + while runtime.snapshot()["phase"] != "idle" and time.monotonic() < deadline: + time.sleep(0.01) + assert runtime.snapshot()["phase"] == "idle" + assert runtime.snapshot()["producer_generation"] == expected_generation + + assert observed == [1, 2] + runtime.close() + + def test_k1_normalizer_emits_transport_neutral_point_clouds() -> None: modern = normalize_k1_message( _message("lixel/application/report/lio_pcl", _lio_point_payload()), @@ -260,8 +1014,488 @@ def test_visual_runtime_has_no_implicit_vendor_normalizer() -> None: assert "normalizer: CanonicalNormalizer" in source +def test_runtime_recovery_state_is_generation_fenced_and_nonterminal() -> None: + runtime = VisualizationRuntime(normalizer=normalize_k1_message) + confirmed_attempts: list[int] = [] + with runtime._lock: # noqa: SLF001 - exact producer-generation fence test + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + lambda attempt: confirmed_attempts.append(attempt) or True, + generation=7, + ) + + runtime._set_connection_reconnecting( # noqa: SLF001 + "synthetic transport loss", + generation=7, + ) + reconnecting = runtime.snapshot() + assert reconnecting["phase"] == "reconnecting" + assert reconnecting["source_mode"] == "live" + assert reconnecting["source_ready"] is False + assert reconnecting["connection_recovery"]["state"] == "reconnecting" + assert reconnecting["connection_recovery"]["automatic_command_retry"] is False + assert reconnecting["connection_recovery"]["device_write_performed"] is False + assert reconnecting["connection_recovery"]["network_mutation_performed"] is False + + assert ( + runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda attempt: "resume" if attempt == 1 else "fault", + 1, + generation=7, + ) + == "resume" + ) + runtime._arm_connection_recovery_point_cloud_candidate( # noqa: SLF001 + 1, + 12, + generation=7, + ) + + def published_point_cloud(sequence: int, *, point_count: int = 1) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=tuple((float(index), 0.0, 0.0) for index in range(point_count)), + ) + + # A queued point cloud from before the reconnect-client sequence fence and + # a valid-but-empty cloud cannot recover the visible scene. + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + published_point_cloud(11), + generation=7, + ) + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + published_point_cloud(12, point_count=0), + generation=7, + ) + still_reconnecting = runtime.snapshot() + assert still_reconnecting["phase"] == "reconnecting" + assert still_reconnecting["source_ready"] is False + + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + published_point_cloud(13), + generation=7, + ) + recovered = runtime.snapshot() + assert recovered["phase"] == "live" + assert recovered["source_ready"] is True + assert recovered["producer_generation"] == 7 + assert recovered["connection_recovery"]["state"] == "recovered" + assert confirmed_attempts == [1] + + # The promoted candidate is one-shot. Replaying the same published + # envelope must not double-confirm the capture summary. + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + published_point_cloud(13), + generation=7, + ) + assert confirmed_attempts == [1] + + runtime._finish_idle("synthetic canonical STOP completion", generation=7) # noqa: SLF001 + idle = runtime.snapshot() + assert idle["phase"] == "idle" + assert idle["source_mode"] == "idle" + assert idle["source_ready"] is False + assert idle["connection_recovery"] == { + "state": "inactive", + "attempt": 0, + "reason_code": None, + "started_at_utc": None, + "elapsed_ms": None, + "recovered_at_utc": None, + "automatic_command_retry": False, + "device_write_performed": False, + "network_mutation_performed": False, + } + + with runtime._lock: # noqa: SLF001 + runtime._producer_generation = 8 # noqa: SLF001 + runtime._phase = "reconnecting" # noqa: SLF001 + runtime._source_ready = False # noqa: SLF001 + runtime._arm_connection_recovery_point_cloud_candidate( # noqa: SLF001 + 2, + 14, + generation=7, + ) + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + published_point_cloud(14), + generation=7, + ) + stale = runtime.snapshot() + assert stale["phase"] == "reconnecting" + assert stale["source_ready"] is False + assert stale["producer_generation"] == 8 + assert confirmed_attempts == [1] + + +def test_runtime_recovery_requires_capture_owned_durable_confirmer() -> None: + runtime = VisualizationRuntime(normalizer=normalize_k1_message) + with runtime._lock: # noqa: SLF001 - exact recovery admission seam + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + + runtime._set_connection_reconnecting("synthetic loss", generation=7) # noqa: SLF001 + assert ( # noqa: SLF001 + runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda _attempt: "resume", + 1, + generation=7, + ) + == "resume" + ) + runtime._arm_connection_recovery_point_cloud_candidate( # noqa: SLF001 + 1, + 12, + generation=7, + ) + + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + _published_point_cloud(12), + generation=7, + ) + + snapshot = runtime.snapshot() + assert snapshot["phase"] == "reconnecting" + assert snapshot["source_ready"] is False + assert snapshot["connection_recovery"]["state"] == "reconnecting" + runtime.stop() + + +def test_runtime_checkpoint_hook_precedes_capture_and_runtime_promotion() -> None: + events: list[str] = [] + + def checkpoint( + _envelope: DecodedPointCloudView, + generation: int, + attempt: int, + minimum_sequence: int, + ) -> bool: + snapshot = runtime.snapshot() + assert generation == 7 + assert attempt == 1 + assert minimum_sequence == 12 + assert snapshot["phase"] == "reconnecting" + assert snapshot["source_ready"] is False + events.append("checkpoint") + return True + + def capture_confirmer(attempt: int) -> bool: + assert attempt == 1 + assert events == ["checkpoint"] + assert runtime.snapshot()["phase"] == "reconnecting" + events.append("capture") + return True + + runtime = VisualizationRuntime( + normalizer=normalize_k1_message, + recovery_promotion_checkpoint=checkpoint, + ) + with runtime._lock: # noqa: SLF001 + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + capture_confirmer, + generation=7, + ) + runtime._set_connection_reconnecting("synthetic loss", generation=7) # noqa: SLF001 + assert runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda _attempt: "resume", + 1, + generation=7, + ) == "resume" + runtime._arm_connection_recovery_point_cloud_candidate(1, 12, generation=7) # noqa: SLF001 + + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + _published_point_cloud(12), + generation=7, + ) + + assert events == ["checkpoint", "capture"] + assert runtime.snapshot()["phase"] == "live" + runtime.stop() + + +def test_failed_checkpoint_hook_keeps_capture_gap_and_runtime_reconnecting() -> None: + capture_attempts: list[int] = [] + runtime = VisualizationRuntime( + normalizer=normalize_k1_message, + recovery_promotion_checkpoint=lambda *_args: False, + ) + with runtime._lock: # noqa: SLF001 + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + lambda attempt: capture_attempts.append(attempt) or True, + generation=7, + ) + runtime._set_connection_reconnecting("synthetic loss", generation=7) # noqa: SLF001 + assert runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda _attempt: "resume", + 1, + generation=7, + ) == "resume" + runtime._arm_connection_recovery_point_cloud_candidate(1, 12, generation=7) # noqa: SLF001 + + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + _published_point_cloud(12), + generation=7, + ) + + snapshot = runtime.snapshot() + assert capture_attempts == [] + assert snapshot["phase"] == "reconnecting" + assert snapshot["source_ready"] is False + runtime.stop() + + +def test_blocked_checkpoint_hook_revalidates_generation_before_capture() -> None: + entered = threading.Event() + release = threading.Event() + capture_attempts: list[int] = [] + + def checkpoint(*_args: object) -> bool: + entered.set() + assert release.wait(timeout=2.0) + return True + + runtime = VisualizationRuntime( + normalizer=normalize_k1_message, + recovery_promotion_checkpoint=checkpoint, # type: ignore[arg-type] + ) + with runtime._lock: # noqa: SLF001 + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + lambda attempt: capture_attempts.append(attempt) or True, + generation=7, + ) + runtime._set_connection_reconnecting("synthetic loss", generation=7) # noqa: SLF001 + assert runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda _attempt: "resume", + 1, + generation=7, + ) == "resume" + runtime._arm_connection_recovery_point_cloud_candidate(1, 12, generation=7) # noqa: SLF001 + + worker = threading.Thread( + target=lambda: runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + _published_point_cloud(12), + generation=7, + ) + ) + worker.start() + assert entered.wait(timeout=1.0) + with runtime._lock: # noqa: SLF001 + runtime._producer_generation = 8 # noqa: SLF001 + release.set() + worker.join(timeout=1.0) + + assert not worker.is_alive() + assert capture_attempts == [] + assert runtime.snapshot()["producer_generation"] == 8 + runtime.stop() + + +def test_blocked_durable_confirmer_does_not_block_stop_or_commit_stale_generation() -> None: + runtime = VisualizationRuntime(normalizer=normalize_k1_message) + confirmer_entered = threading.Event() + release_confirmer = threading.Event() + snapshot_returned = threading.Event() + stop_returned = threading.Event() + + def blocking_confirmer(_attempt: int) -> bool: + confirmer_entered.set() + assert release_confirmer.wait(timeout=3.0) + return True + + with runtime._lock: # noqa: SLF001 - exact recovery admission seam + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + blocking_confirmer, + generation=7, + ) + runtime._set_connection_reconnecting("synthetic loss", generation=7) # noqa: SLF001 + assert ( # noqa: SLF001 + runtime._run_connection_recovery_attempt( # noqa: SLF001 + lambda _attempt: "resume", + 1, + generation=7, + ) + == "resume" + ) + runtime._arm_connection_recovery_point_cloud_candidate( # noqa: SLF001 + 1, + 12, + generation=7, + ) + + confirmer_thread = threading.Thread( + target=lambda: runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + _published_point_cloud(12), + generation=7, + ) + ) + confirmer_thread.start() + assert confirmer_entered.wait(timeout=1.0) + + def read_snapshot() -> None: + runtime.snapshot() + snapshot_returned.set() + + snapshot_thread = threading.Thread(target=read_snapshot) + snapshot_thread.start() + assert snapshot_returned.wait(timeout=0.5) + snapshot_thread.join(timeout=1.0) + + def stop_runtime() -> None: + runtime.stop() + stop_returned.set() + + stop_thread = threading.Thread(target=stop_runtime) + stop_thread.start() + assert stop_returned.wait(timeout=0.5) + stop_thread.join(timeout=1.0) + + # Establish a new authoritative producer while the old generation's + # durable callback is still blocked. Its eventual success must not promote + # this replacement generation. + with runtime._lock: # noqa: SLF001 - synthetic replacement generation + runtime._producer_generation = 8 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "reconnecting" # noqa: SLF001 + runtime._source_ready = False # noqa: SLF001 + release_confirmer.set() + confirmer_thread.join(timeout=1.0) + assert not confirmer_thread.is_alive() + + stale = runtime.snapshot() + assert stale["producer_generation"] == 8 + assert stale["phase"] == "reconnecting" + assert stale["source_ready"] is False + assert stale["connection_recovery"]["state"] == "inactive" + runtime.stop() + + +def test_live_recovery_waits_for_fresh_nonempty_post_publish_point_cloud( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed: dict[str, object] = {} + confirmed_attempts: list[int] = [] + fake_capture_finished = threading.Event() + + def captured(sequence: int, payload: bytes) -> CapturedMqttMessage: + return CapturedMqttMessage( + sequence=sequence, + topic="RealtimePointcloud", + payload=payload, + qos=0, + retain=False, + dup=False, + received_at_utc="2026-08-12T00:00:00Z", + received_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + ) + + def wait_for(predicate: Callable[[dict[str, object]], bool]) -> dict[str, object]: + deadline = time.monotonic() + 2.0 + snapshot = runtime.snapshot() + while not predicate(snapshot) and time.monotonic() < deadline: + time.sleep(0.005) + snapshot = runtime.snapshot() + assert predicate(snapshot) + return snapshot + + def fake_capture_mqtt(_host: str, _out_dir: Path, **callbacks: object) -> dict[str, object]: + callbacks["on_recovery_confirmer_ready"]( # type: ignore[operator] + lambda attempt: confirmed_attempts.append(attempt) or True + ) + callbacks["on_clock_established"]() # type: ignore[operator] + callbacks["on_ready"]() # type: ignore[operator] + callbacks["on_connection_lost"]("synthetic loss") # type: ignore[operator] + assert callbacks["recover_connection"](1) == "resume" # type: ignore[operator] + + arm = callbacks["on_recovery_point_cloud_candidate"] + enqueue = callbacks["on_message_recorded"] + arm(1, 2) # type: ignore[operator] + enqueue(captured(2, b"malformed")) # type: ignore[operator] + malformed = wait_for( + lambda item: item["metrics"]["decode_errors"] == 1 # type: ignore[index] + ) + observed["malformed"] = malformed + + # Legacy RealtimePointcloud can normalize and publish a zero-point + # envelope; that is not visible scene recovery. + enqueue(captured(3, struct.pack(" None: topic = b"RealtimePointcloud" payload = b"short" @@ -285,7 +1519,7 @@ def test_runtime_counts_transport_and_normalization_failures_before_rerun( time.sleep(0.01) snapshot = runtime.snapshot() - assert snapshot["phase"] == "idle" + assert snapshot["phase"] == "idle", snapshot assert snapshot["metrics"]["messages_received"] == 1 assert snapshot["metrics"]["payload_bytes"] == len(payload) assert snapshot["metrics"]["decode_errors"] == 1 @@ -327,6 +1561,7 @@ def test_live_runtime_surfaces_preamble_failure_as_terminal_error( def test_live_start_waits_for_capture_clock_before_returning( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + socket_free_rerun_port_selector: None, ) -> None: entered_capture = threading.Event() allow_clock = threading.Event() diff --git a/tests/test_cli.py b/tests/test_cli.py index 3e3fef5..44944cc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,14 @@ +from __future__ import annotations + import json +import subprocess +import sys +from http.client import BadStatusLine from pathlib import Path +from types import SimpleNamespace from typing import Any +import pytest from typer.testing import CliRunner from k1link.device_plugins.xgrids_k1 import cli @@ -10,6 +17,36 @@ from k1link.device_plugins.xgrids_k1.cli import app runner = CliRunner() +class _FakeServeLease: + def __init__(self) -> None: + self.active = False + + def __enter__(self) -> _FakeServeLease: + assert self.active is False + self.active = True + return self + + def __exit__(self, *_: object) -> None: + assert self.active is True + self.active = False + + +def _install_fake_serve_lease( + monkeypatch: Any, + *, + error: BaseException | None = None, +) -> _FakeServeLease: + lease = _FakeServeLease() + + def acquire(_: Path) -> _FakeServeLease: + if error is not None: + raise error + return lease + + monkeypatch.setattr(cli, "_acquire_mission_core_serve_lease", acquire) + return lease + + def test_help() -> None: result = runner.invoke(app, ["--help"]) assert result.exit_code == 0 @@ -28,11 +65,14 @@ def test_doctor_json() -> None: def test_serve_resolves_frontend_from_repository_root(monkeypatch: Any) -> None: captured: dict[str, object] = {} + lease = _install_fake_serve_lease(monkeypatch) def fake_run(application: str, **kwargs: object) -> None: + assert lease.active is True captured.update({"application": application, **kwargs}) monkeypatch.setattr(cli.uvicorn, "run", fake_run) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "free") result = runner.invoke(app, ["serve"]) @@ -44,6 +84,315 @@ def test_serve_resolves_frontend_from_repository_root(monkeypatch: Any) -> None: "log_level": "info", "access_log": True, } + assert lease.active is False + + +def test_serve_releases_singleton_lease_when_uvicorn_fails(monkeypatch: Any) -> None: + lease = _install_fake_serve_lease(monkeypatch) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "free") + + def failed_run(*_: object, **__: object) -> None: + assert lease.active is True + raise RuntimeError("startup failed") + + monkeypatch.setattr(cli.uvicorn, "run", failed_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 1 + assert isinstance(result.exception, RuntimeError) + assert lease.active is False + + +def test_serve_reuses_an_existing_mission_core_backend(monkeypatch: Any) -> None: + lease = _install_fake_serve_lease(monkeypatch) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "mission-core") + + def unexpected_run(*_: object, **__: object) -> None: + raise AssertionError("a second backend must not be started") + + monkeypatch.setattr(cli.uvicorn, "run", unexpected_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 0 + assert "уже запущен" in result.stdout + assert "второй процесс не создан" in result.stdout + assert lease.active is False + + +def test_serve_rejects_an_unrelated_loopback_listener(monkeypatch: Any) -> None: + lease = _install_fake_serve_lease(monkeypatch) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "occupied") + + def unexpected_run(*_: object, **__: object) -> None: + raise AssertionError("an occupied port must fail before Uvicorn startup") + + monkeypatch.setattr(cli.uvicorn, "run", unexpected_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 2 + assert "уже занят" in result.stdout + assert "Mission Core не стал создавать второй backend" in result.stdout + assert "--port" not in result.stdout + assert lease.active is False + + +def test_serve_rejects_noncanonical_port_before_lock_or_health(monkeypatch: Any) -> None: + def unexpected_call(*_: object, **__: object) -> None: + raise AssertionError("noncanonical port must fail before lock or health inspection") + + monkeypatch.setattr(cli, "_acquire_mission_core_serve_lease", unexpected_call) + monkeypatch.setattr(cli, "_local_server_status", unexpected_call) + monkeypatch.setattr(cli.uvicorn, "run", unexpected_call) + + result = runner.invoke(app, ["serve", "--port", "8001"]) + + assert result.exit_code == 2 + assert "только на каноническом порту 8000" in result.stdout + assert "Другой локальный backend не создан" in result.stdout + + +def test_serve_busy_lock_reuses_confirmed_mission_core(monkeypatch: Any) -> None: + _install_fake_serve_lease( + monkeypatch, + error=cli._MissionCoreServeLeaseUnavailable("busy"), + ) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "mission-core") + + def unexpected_run(*_: object, **__: object) -> None: + raise AssertionError("busy singleton lease must not start Uvicorn") + + monkeypatch.setattr(cli.uvicorn, "run", unexpected_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 0 + assert "уже запущен" in result.stdout + assert "второй процесс не создан" in result.stdout + + +def test_serve_busy_lock_reports_in_progress_when_health_is_not_listening( + monkeypatch: Any, +) -> None: + _install_fake_serve_lease( + monkeypatch, + error=cli._MissionCoreServeLeaseUnavailable("busy"), + ) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "free") + + def unexpected_run(*_: object, **__: object) -> None: + raise AssertionError("busy singleton lease must not start Uvicorn") + + monkeypatch.setattr(cli.uvicorn, "run", unexpected_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 2 + assert "уже запускается или завершает работу" in result.stdout + assert "Второй backend не создан" in result.stdout + + +def test_serve_busy_lock_rejects_unconfirmed_health_listener(monkeypatch: Any) -> None: + _install_fake_serve_lease( + monkeypatch, + error=cli._MissionCoreServeLeaseUnavailable("busy"), + ) + monkeypatch.setattr(cli, "_local_server_status", lambda _: "occupied") + + def unexpected_run(*_: object, **__: object) -> None: + raise AssertionError("busy singleton lease must not start Uvicorn") + + monkeypatch.setattr(cli.uvicorn, "run", unexpected_run) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 2 + assert "канонический health endpoint" in result.stdout + assert "подтверждён" in result.stdout + assert "Второй backend не создан" in result.stdout + + +def test_serve_lock_integrity_error_fails_before_health_or_uvicorn(monkeypatch: Any) -> None: + _install_fake_serve_lease( + monkeypatch, + error=cli._MissionCoreServeLeaseError("unsafe lock"), + ) + + def unexpected_call(*_: object, **__: object) -> None: + raise AssertionError("an untrusted lock must block health and Uvicorn") + + monkeypatch.setattr(cli, "_local_server_status", unexpected_call) + monkeypatch.setattr(cli.uvicorn, "run", unexpected_call) + + result = runner.invoke(app, ["serve"]) + + assert result.exit_code == 2 + assert "безопасно получить блокировку" in result.stdout + assert "Второй backend не создан" in result.stdout + + +def test_serve_lock_is_stable_private_and_cross_process(tmp_path: Path) -> None: + expected_path = tmp_path / ".runtime" / "mission-core" / ".serve.lock" + + with cli._MissionCoreServeLease.acquire(tmp_path) as first: + assert first.path == expected_path + first_identity = (expected_path.stat().st_dev, expected_path.stat().st_ino) + assert expected_path.stat().st_mode & 0o777 == 0o600 + assert expected_path.parent.stat().st_mode & 0o777 == 0o700 + child = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; from pathlib import Path; " + "from k1link.device_plugins.xgrids_k1.cli import " + "_MissionCoreServeLease, _MissionCoreServeLeaseUnavailable; " + "root=Path(sys.argv[1]); " + "\ntry: lease=_MissionCoreServeLease.acquire(root)" + "\nexcept _MissionCoreServeLeaseUnavailable: raise SystemExit(0)" + "\nelse: lease.release(); raise SystemExit(1)" + ), + str(tmp_path), + ], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + assert child.returncode == 0, child.stderr + + assert expected_path.is_file() + with cli._MissionCoreServeLease.acquire(tmp_path) as second: + assert (second.path.stat().st_dev, second.path.stat().st_ino) == first_identity + + +def test_serve_lock_rejects_nonprivate_lock_directory(tmp_path: Path) -> None: + lock_dir = tmp_path / ".runtime" / "mission-core" + lock_dir.mkdir(mode=0o755, parents=True) + lock_dir.chmod(0o755) + + with pytest.raises(cli._MissionCoreServeLeaseError, match="not private"): + cli._MissionCoreServeLease.acquire(tmp_path) + + +def test_serve_lock_rejects_symlinked_lock_directory(tmp_path: Path) -> None: + runtime_dir = tmp_path / ".runtime" + runtime_dir.mkdir() + redirected = tmp_path / "redirected" + redirected.mkdir(mode=0o700) + (runtime_dir / "mission-core").symlink_to(redirected, target_is_directory=True) + + with pytest.raises(cli._MissionCoreServeLeaseError, match="not private"): + cli._MissionCoreServeLease.acquire(tmp_path) + + +class _FakeHealthResponse: + def __init__(self, *, status: int = 200, body: bytes = b"") -> None: + self.status = status + self.body = body + + def read(self, limit: int) -> bytes: + assert limit == 16_385 + return self.body[:limit] + + +class _FakeHealthConnection: + def __init__( + self, + *, + response: _FakeHealthResponse | None = None, + error: BaseException | None = None, + ) -> None: + self.response = response + self.error = error + self.request_call: tuple[str, str, dict[str, str]] | None = None + self.closed = False + + def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: + self.request_call = (method, path, headers) + if self.error is not None: + raise self.error + + def getresponse(self) -> _FakeHealthResponse: + assert self.response is not None + return self.response + + def close(self) -> None: + self.closed = True + + +def _install_fake_health_connection( + monkeypatch: Any, + connection: _FakeHealthConnection, +) -> None: + def fake_connection(host: str, port: int, *, timeout: float) -> _FakeHealthConnection: + assert (host, port, timeout) == ("127.0.0.1", 8000, 0.75) + return connection + + monkeypatch.setattr(cli.http.client, "HTTPConnection", fake_connection) + + +def test_local_server_status_recognizes_exact_mission_core_health(monkeypatch: Any) -> None: + connection = _FakeHealthConnection( + response=_FakeHealthResponse( + body=json.dumps({"service": "mission-core-control-plane"}).encode("utf-8") + ) + ) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "mission-core" + assert connection.request_call == ( + "GET", + "/api/health", + {"Accept": "application/json", "Connection": "close"}, + ) + assert connection.closed is True + + +def test_local_server_status_rejects_a_different_health_identity(monkeypatch: Any) -> None: + connection = _FakeHealthConnection( + response=_FakeHealthResponse(body=json.dumps({"service": "other"}).encode("utf-8")) + ) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "occupied" + assert connection.closed is True + + +def test_local_server_status_only_treats_explicit_refusal_as_free(monkeypatch: Any) -> None: + connection = _FakeHealthConnection(error=ConnectionRefusedError()) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "free" + assert connection.closed is True + + +def test_local_server_status_fails_closed_on_listener_timeout(monkeypatch: Any) -> None: + connection = _FakeHealthConnection(error=TimeoutError()) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "occupied" + assert connection.closed is True + + +def test_local_server_status_fails_closed_on_non_http_listener(monkeypatch: Any) -> None: + connection = _FakeHealthConnection(error=BadStatusLine("not HTTP")) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "occupied" + assert connection.closed is True + + +def test_local_server_status_rejects_oversized_health_payload(monkeypatch: Any) -> None: + connection = _FakeHealthConnection( + response=_FakeHealthResponse(body=b" " * 16_385) + ) + _install_fake_health_connection(monkeypatch, connection) + + assert cli._local_server_status(8000) == "occupied" + assert connection.closed is True def test_authority_provision_requires_explicit_reviewed_value_confirmation() -> None: @@ -144,3 +493,68 @@ def test_mqtt_capture_cli_uses_bounded_read_only_capture( } assert "messages: 2" in result.stdout assert "subscriptions active" in result.stdout + + +@pytest.mark.parametrize("ledger_status", ["resolved", "corrupt"]) +def test_cli_wifi_configure_blocks_retired_or_untrusted_audit_before_credentials_and_ble( + monkeypatch: Any, + tmp_path: Path, + ledger_status: str, +) -> None: + repository_root = tmp_path / "repository" + repository_root.mkdir() + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + monkeypatch.setattr(cli, "_configure_ble_process_lease", lambda: repository_root) + retired_transport_ref = "F89438FA-55ED-85AD-EED7-734AC84746D8" + record = ( + SimpleNamespace( + operator_retirements=( + SimpleNamespace( + retirement_id="retirement-cli-blocked-target", + retired_transport_ref=retired_transport_ref, + ), + ), + operator_reconciliation_reopens=(), + ) + if ledger_status == "resolved" + else None + ) + monkeypatch.setattr( + cli, + "PhysicalCommandLedger", + lambda _root: SimpleNamespace( + snapshot=lambda: SimpleNamespace(status=ledger_status, record=record) + ), + ) + calls: list[str] = [] + monkeypatch.setattr( + cli, + "prompt_wifi_credentials", + lambda: calls.append("credentials") or ("ssid", "secret"), + ) + + async def forbidden_provision(*_args: object, **_kwargs: object) -> dict[str, object]: + calls.append("ble-write") + raise AssertionError("retired CLI target must not reach BLE") + + monkeypatch.setattr(cli, "provision_wifi_once", forbidden_provision) + + result = runner.invoke( + app, + [ + "ble", + "wifi-configure", + "--device", + retired_transport_ref.lower(), + "--out", + str(tmp_path / "wifi-result.json"), + "--profile", + cli.PROFILE_ID, + "--confirm-write", + ], + ) + + assert result.exit_code == 2 + assert "blocked before credential or device access" in result.stdout + assert calls == [] + assert not (tmp_path / "wifi-result.json").exists() diff --git a/tests/test_connection_supervisor.py b/tests/test_connection_supervisor.py new file mode 100644 index 0000000..75e13af --- /dev/null +++ b/tests/test_connection_supervisor.py @@ -0,0 +1,2324 @@ +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import UTC, datetime + +import pytest + +from k1link.device_plugins.xgrids_k1.connection_supervisor import ( + ConnectionSupervisor, + ConnectionSupervisorClosed, + EndpointTarget, + HostPathProbeResult, + ReadOnlyConnectionMonitor, + RouteClass, + TcpReachabilityProbeResult, + VerifiedControlEvidence, +) + +NOW = datetime(2026, 8, 6, 12, 0, tzinfo=UTC) +TARGET = EndpointTarget("192.168.68.52") + + +def _clock() -> datetime: + return NOW + + +def _available_path(fingerprint: str = "en0:192.168.68.20:router-a") -> HostPathProbeResult: + return HostPathProbeResult( + available=True, + fingerprint=fingerprint, + interface="en0", + source_ipv4="192.168.68.20", + route_class="direct", + kernel_route_fingerprint=fingerprint, + ) + + +def _unavailable_path(reason: str = "route-unavailable") -> HostPathProbeResult: + return HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code=reason, + ) + + +def _apply_device_network( + supervisor: ConnectionSupervisor, + *, + intent_id: str, + connection_mode: str = "bridge", + target: EndpointTarget = TARGET, +) -> None: + assert supervisor.observe_device_network_applied( + intent_id=intent_id, + transport_ref="ble-k1-a", + connection_mode=connection_mode, # type: ignore[arg-type] + target=target, + source="ble-post-write-status", + ) + + +def _reachable_supervisor() -> tuple[ConnectionSupervisor, int]: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="bridge-1", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-1") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-session-a", + ) + ) + return supervisor, epoch + + +def _configured_unverified_supervisor() -> tuple[ConnectionSupervisor, int]: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="bridge-1", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-1") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.snapshot().lease.state == "configured-unverified" + return supervisor, epoch + + +def test_initial_snapshot_is_fail_closed_and_exposes_only_safe_actions() -> None: + snapshot = ConnectionSupervisor(clock=_clock).snapshot() + + assert snapshot.lease.state == "absent" + assert snapshot.authority.network_mutation_allowed is False + assert snapshot.authority.control_allowed is False + assert snapshot.authority.acquisition_start_allowed is False + assert snapshot.authority.data_ingest_authoritative is False + assert snapshot.authority.physical_motion_allowed is False + assert snapshot.diagnostics == () + assert snapshot.allowed_actions == ("select-connection-intent",) + assert snapshot.as_dict()["schema_version"] == ("missioncore.k1-connection-supervisor/v1") + + +def test_tcp_reachability_alone_never_promotes_identity_or_authority() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="quick-1", requested_mode="quick-connect") + epoch = supervisor.observe_host_path(_available_path()) + + accepted = supervisor.observe_endpoint( + target=TARGET, + intent_id="quick-1", + host_path_epoch=epoch, + reachable=True, + ) + snapshot = supervisor.snapshot() + + assert accepted is True + assert snapshot.endpoint.tcp_state == "reachable" + assert snapshot.device_identity.state == "unverified" + assert snapshot.lease.state == "absent" + assert snapshot.authority.control_allowed is False + assert "verify-control-device-info" not in snapshot.allowed_actions + assert "start-acquisition" not in snapshot.allowed_actions + + _apply_device_network( + supervisor, + intent_id="quick-1", + connection_mode="quick-connect", + ) + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="quick-1", + host_path_epoch=epoch, + reachable=True, + ) + configured = supervisor.snapshot() + assert configured.lease.state == "configured-unverified" + assert "verify-control-device-info" in configured.allowed_actions + + +def test_exact_device_info_and_control_evidence_promote_bounded_authority() -> None: + supervisor, epoch = _reachable_supervisor() + snapshot = supervisor.snapshot() + + assert snapshot.device_identity.state == "verified" + assert snapshot.control_plane.state == "healthy" + assert snapshot.lease.state == "reachable" + assert snapshot.lease.host_path_epoch == epoch + assert snapshot.authority.control_allowed is True + assert snapshot.authority.acquisition_start_allowed is True + assert snapshot.authority.data_ingest_authoritative is False + assert snapshot.authority.network_mutation_allowed is False + assert snapshot.authority.physical_motion_allowed is False + assert "start-acquisition" in snapshot.allowed_actions + assert snapshot.last_known is not None + assert snapshot.last_known.logical_device_id == "k1-device-a" + + +def test_identity_mismatch_is_observed_but_never_authoritative() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="bridge-1", + requested_mode="bridge", + expected_device_id="expected-k1", + ) + _apply_device_network(supervisor, intent_id="bridge-1") + epoch = supervisor.observe_host_path(_available_path()) + supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="different-k1", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="wrong-control", + ) + ) + snapshot = supervisor.snapshot() + + assert snapshot.device_identity.state == "mismatch" + assert snapshot.control_plane.state == "lost" + assert snapshot.lease.state == "configured-unverified" + assert snapshot.authority.control_allowed is False + assert "device-identity-mismatch" in snapshot.authority.reason_codes + + +def test_host_path_epoch_change_revokes_lease_but_preserves_last_known() -> None: + supervisor, first_epoch = _reachable_supervisor() + + second_epoch = supervisor.observe_host_path(_available_path("en0:192.168.77.20:router-b")) + snapshot = supervisor.snapshot() + + assert second_epoch == first_epoch + 1 + assert snapshot.host_path.epoch == second_epoch + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.device_identity.state == "stale" + assert snapshot.control_plane.state == "lost" + assert snapshot.lease.state == "configured-unverified" + assert snapshot.authority.control_allowed is False + assert snapshot.authority.acquisition_start_allowed is False + assert snapshot.last_known is not None + assert snapshot.last_known.target == TARGET + assert "probe-endpoint" in snapshot.allowed_actions + + +def test_host_path_failure_and_tcp_failure_each_revoke_authority() -> None: + host_failure, _ = _reachable_supervisor() + failed_epoch = host_failure.observe_host_path(_unavailable_path("wifi-interface-down")) + failed_snapshot = host_failure.snapshot() + + assert failed_epoch == 2 + assert failed_snapshot.lease.state == "configured-unverified" + assert failed_snapshot.authority.control_allowed is False + assert failed_snapshot.allowed_actions == ( + "select-connection-intent", + "inspect-host-network", + ) + + tcp_failure, epoch = _reachable_supervisor() + assert tcp_failure.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=False, + reason_code="mqtt-port-unreachable", + ) + tcp_snapshot = tcp_failure.snapshot() + assert tcp_snapshot.endpoint.tcp_state == "unreachable" + assert tcp_snapshot.lease.state == "configured-unverified" + assert tcp_snapshot.authority.control_allowed is False + assert "probe-endpoint" in tcp_snapshot.allowed_actions + + +def test_stale_async_probe_result_cannot_restore_a_new_host_epoch() -> None: + supervisor, first_epoch = _reachable_supervisor() + second_epoch = supervisor.observe_host_path(_available_path("en0:192.168.99.20:router-c")) + + accepted = supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=first_epoch, + reachable=True, + ) + snapshot = supervisor.snapshot() + + assert second_epoch == first_epoch + 1 + assert accepted is False + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.lease.state == "configured-unverified" + assert snapshot.authority.control_allowed is False + + +def test_quick_to_bridge_revokes_authority_with_same_target_and_host_epoch() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="quick-1", + requested_mode="quick-connect", + expected_device_id="k1-device-a", + ) + _apply_device_network( + supervisor, + intent_id="quick-1", + connection_mode="quick-connect", + ) + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="quick-1", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="quick-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="quick-connect", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="quick-control", + ) + ) + assert supervisor.snapshot().authority.control_allowed is True + + switched = supervisor.set_intent( + intent_id="bridge-2", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + + assert switched.host_path.epoch == epoch + assert switched.endpoint.target is None + assert switched.endpoint.intent_id == "bridge-2" + assert switched.device_identity.state == "unverified" + assert switched.control_plane.state == "lost" + assert switched.lease.state == "lost" + assert switched.last_known is not None + assert switched.last_known.target == TARGET + assert switched.authority.control_allowed is False + + _apply_device_network(supervisor, intent_id="bridge-2") + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-2", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-2", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="bridge-control", + ) + ) + restored = supervisor.snapshot() + assert restored.host_path.epoch == epoch + assert restored.lease.intent_id == "bridge-2" + assert restored.lease.connection_mode == "bridge" + assert restored.authority.control_allowed is True + + +def test_stale_old_intent_tcp_and_control_evidence_are_rejected() -> None: + supervisor, epoch = _reachable_supervisor() + supervisor.set_intent( + intent_id="bridge-2", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + + assert ( + supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + is False + ) + assert ( + supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="stale-control", + ) + ) + is False + ) + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-2" + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.device_identity.state == "unverified" + assert snapshot.lease.state == "lost" + assert snapshot.authority.control_allowed is False + + +def test_device_info_from_a_different_ble_transport_is_rejected() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="bridge-transport-bound", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-transport-bound") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-transport-bound", + host_path_epoch=epoch, + reachable=True, + ) + + accepted = supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-transport-bound", + transport_ref="ble-k1-b", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="wrong-transport-control", + ) + ) + + assert accepted is False + snapshot = supervisor.snapshot() + assert snapshot.device_network.transport_ref == "ble-k1-a" + assert snapshot.device_identity.state == "unverified" + assert snapshot.control_plane.state != "healthy" + assert snapshot.authority.control_allowed is False + + +@pytest.mark.parametrize("route_class", ["default", "tunnel"]) +def test_non_direct_route_never_grants_control_authority(route_class: RouteClass) -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent( + intent_id="bridge-1", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-1") + epoch = supervisor.observe_host_path( + HostPathProbeResult( + available=True, + fingerprint=f"en0:192.168.68.20:{route_class}", + interface="utun4" if route_class == "tunnel" else "en0", + source_ipv4="192.168.68.20", + route_class=route_class, + ) + ) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id=f"{route_class}-control", + ) + ) + + snapshot = supervisor.snapshot() + assert snapshot.device_identity.state == "verified" + assert snapshot.lease.state == "reachable" + assert snapshot.authority.control_allowed is False + assert snapshot.authority.acquisition_start_allowed is False + assert "host-route-not-direct" in snapshot.authority.reason_codes + + +def test_explicit_data_expiry_revokes_only_data_authority() -> None: + supervisor, epoch = _reachable_supervisor() + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-a", + ) + healthy = supervisor.snapshot() + assert healthy.authority.control_allowed is True + assert healthy.authority.data_ingest_authoritative is True + assert "stop-acquisition" in healthy.allowed_actions + + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="lost", + session_id="data-session-a", + reason_code="data-producer-observation-expired", + ) + data_lost = supervisor.snapshot() + assert data_lost.lease.state == "reachable" + assert data_lost.control_plane.state == "healthy" + assert data_lost.authority.control_allowed is True + assert data_lost.authority.acquisition_start_allowed is False + assert data_lost.authority.data_ingest_authoritative is False + assert "acknowledge-data-loss" in data_lost.allowed_actions + + +def test_mqtt_control_loss_preserves_fresh_data_session_as_evidence() -> None: + supervisor, epoch = _reachable_supervisor() + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-7", + ) + + assert supervisor.observe_control_loss( + intent_id="bridge-1", + host_path_epoch=epoch, + control_session_id="control-session-a", + reason_code="mqtt-control-loop-lost", + ) + control_lost = supervisor.snapshot() + assert control_lost.lease.state == "configured-unverified" + assert control_lost.authority.control_allowed is False + assert control_lost.authority.acquisition_start_allowed is False + assert control_lost.authority.data_ingest_authoritative is False + assert control_lost.data_plane.state == "healthy" + assert control_lost.data_plane.session_id == "data-session-generation-7" + assert control_lost.data_plane.host_path_epoch == epoch + assert [item.code for item in control_lost.diagnostics] == ["host.mqtt.transport-unavailable"] + + # A packet from the already-bound producer can refresh data evidence even + # though control is unavailable. It does not restore command authority. + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-7", + ) + refreshed = supervisor.snapshot() + assert refreshed.data_plane.state == "healthy" + assert refreshed.authority.control_allowed is False + assert refreshed.authority.data_ingest_authoritative is False + + +def test_k1_reboot_same_ip_rejects_phantom_control_snapshot_until_fresh_session() -> None: + supervisor, epoch = _reachable_supervisor() + stale_evidence = VerifiedControlEvidence( + intent_id="bridge-1", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-session-a", + control_proof_revision=1, + ) + + assert supervisor.observe_control_loss( + intent_id="bridge-1", + host_path_epoch=epoch, + control_session_id="control-session-a", + reason_code="k1-reboot-control-loop-lost", + ) + assert supervisor.observe_control_evidence(stale_evidence) is False + + rebooted = supervisor.snapshot() + assert rebooted.host_path.epoch == epoch + assert rebooted.endpoint.tcp_state == "reachable" + assert rebooted.control_plane.state == "lost" + assert rebooted.lease.state == "configured-unverified" + assert rebooted.authority.control_allowed is False + assert rebooted.authority.acquisition_start_allowed is False + + assert supervisor.observe_control_evidence( + replace( + stale_evidence, + control_session_id="control-session-after-reboot", + ) + ) + recovered = supervisor.snapshot() + assert recovered.control_plane.state == "healthy" + assert recovered.control_plane.session_id == "control-session-after-reboot" + assert recovered.authority.control_allowed is True + + +def test_host_route_loss_preserves_fresh_data_on_its_original_epoch() -> None: + supervisor, epoch = _reachable_supervisor() + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-8", + ) + + next_epoch = supervisor.observe_host_path(_unavailable_path("router-lost")) + lost = supervisor.snapshot() + + assert next_epoch == epoch + 1 + assert lost.host_path.available is False + assert lost.control_plane.state == "lost" + assert lost.lease.state == "configured-unverified" + assert lost.authority.control_allowed is False + assert lost.authority.acquisition_start_allowed is False + assert lost.authority.data_ingest_authoritative is False + assert lost.data_plane.state == "healthy" + assert lost.data_plane.session_id == "data-session-generation-8" + assert lost.data_plane.host_path_epoch == epoch + assert [item.code for item in lost.diagnostics] == ["host.route.unavailable"] + + # The old bounded session remains admissible as evidence, but cannot be + # rebound to the replacement host epoch by a late producer callback. + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-8", + ) + assert ( + supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=next_epoch, + state="healthy", + session_id="data-session-generation-8", + ) + is False + ) + + +def test_tcp_loss_preserves_fresh_data_session_as_evidence() -> None: + supervisor, epoch = _reachable_supervisor() + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-9", + ) + + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=False, + reason_code="tcp-route-lost", + ) + lost = supervisor.snapshot() + + assert lost.endpoint.tcp_state == "unreachable" + assert lost.control_plane.state == "lost" + assert lost.lease.state == "configured-unverified" + assert lost.authority.control_allowed is False + assert lost.authority.acquisition_start_allowed is False + assert lost.authority.data_ingest_authoritative is False + assert lost.data_plane.state == "healthy" + assert lost.data_plane.session_id == "data-session-generation-9" + assert lost.data_plane.host_path_epoch == epoch + assert [item.code for item in lost.diagnostics] == ["host.tcp.endpoint-unavailable"] + + +def test_late_control_loss_from_old_intent_or_session_cannot_revoke_new_control() -> None: + supervisor, epoch = _reachable_supervisor() + supervisor.set_intent( + intent_id="bridge-2", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-2") + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-2", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-2", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-session-b", + ) + ) + + assert ( + supervisor.observe_control_loss( + intent_id="bridge-1", + host_path_epoch=epoch, + control_session_id="control-session-a", + reason_code="late-old-worker-loss", + ) + is False + ) + assert ( + supervisor.observe_control_loss( + intent_id="bridge-2", + host_path_epoch=epoch, + control_session_id="control-session-a", + reason_code="late-old-session-loss", + ) + is False + ) + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-2" + assert snapshot.control_plane.state == "healthy" + assert snapshot.control_plane.session_id == "control-session-b" + assert snapshot.lease.state == "reachable" + assert snapshot.authority.control_allowed is True + + +def test_late_data_loss_from_old_session_cannot_poison_new_data_session() -> None: + supervisor, epoch = _reachable_supervisor() + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-old", + ) + assert supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-new", + ) + + assert ( + supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="lost", + session_id="data-session-old", + reason_code="late-old-data-worker-loss", + ) + is False + ) + snapshot = supervisor.snapshot() + assert snapshot.data_plane.state == "healthy" + assert snapshot.data_plane.session_id == "data-session-new" + assert snapshot.authority.control_allowed is True + assert snapshot.authority.data_ingest_authoritative is True + + +def test_monitor_injections_are_read_only_and_drop_delayed_tcp_result() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-1", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-1") + tcp_started = asyncio.Event() + release_tcp = asyncio.Event() + tcp_calls: list[EndpointTarget] = [] + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + return _available_path() + + async def tcp_probe(target: EndpointTarget) -> bool: + tcp_calls.append(target) + tcp_started.set() + await release_tcp.wait() + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + poll = asyncio.create_task(monitor.poll_once()) + await tcp_started.wait() + supervisor.observe_host_path(_available_path("en0:192.168.77.20:router-b")) + release_tcp.set() + await poll + + snapshot = supervisor.snapshot() + assert tcp_calls == [TARGET] + assert snapshot.host_path.epoch == 2 + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.authority.control_allowed is False + await monitor.close() + assert supervisor.snapshot().closed is True + with pytest.raises(ConnectionSupervisorClosed): + await monitor.poll_once() + + asyncio.run(scenario()) + + +def test_monitor_host_path_cas_rejects_switch_after_target_check() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-a", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-a") + next_target = EndpointTarget("192.168.68.77") + tcp_calls = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return _available_path() + + async def tcp_probe(_target: EndpointTarget) -> bool: + nonlocal tcp_calls + tcp_calls += 1 + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + def switch_after_check( + expected: EndpointTarget, + *, + expected_intent_id: str, + ) -> bool: + assert expected == TARGET + assert expected_intent_id == "bridge-a" + supervisor.set_intent(intent_id="bridge-b", requested_mode="bridge") + _apply_device_network( + supervisor, + intent_id="bridge-b", + target=next_target, + ) + # Reproduce the exact old boolean-check -> reducer-write gap. + return True + + monitor._target_is_current = switch_after_check # type: ignore[method-assign] # noqa: SLF001 + snapshot = await monitor.poll_once() + + assert tcp_calls == 0 + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-b" + assert snapshot.device_network.target == next_target + assert snapshot.host_path.epoch == 0 + assert snapshot.host_path.available is False + assert snapshot.host_path.reason_code == "host-path-not-observed" + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_failure_cas_rejects_switch_after_target_check() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-a", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-a") + next_target = EndpointTarget("192.168.68.77") + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + raise OSError("old-target host failure") + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("TCP must not run after a host failure") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + def switch_after_check( + expected: EndpointTarget, + *, + expected_intent_id: str, + ) -> bool: + assert expected == TARGET + assert expected_intent_id == "bridge-a" + supervisor.set_intent(intent_id="bridge-b", requested_mode="bridge") + _apply_device_network( + supervisor, + intent_id="bridge-b", + target=next_target, + ) + return True + + monitor._target_is_current = switch_after_check # type: ignore[method-assign] # noqa: SLF001 + snapshot = await monitor.poll_once() + + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-b" + assert snapshot.device_network.target == next_target + assert snapshot.host_path.epoch == 0 + assert snapshot.host_path.available is False + assert snapshot.host_path.reason_code == "host-path-not-observed" + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_target_provider_error_cannot_poison_a_new_same_target_intent() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-a", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-a") + target_checks = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + raise OSError("old-intent host failure") + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("TCP must not run after a host failure") + + def target_provider() -> EndpointTarget: + nonlocal target_checks + target_checks += 1 + if target_checks == 1: + return TARGET + # The next intent legitimately reuses the same host address. A + # provider error from A must still be rejected by the intent CAS. + supervisor.set_intent(intent_id="bridge-b", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-b") + raise RuntimeError("late target lookup failure from bridge-a") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=target_provider, + ) + snapshot = await monitor.poll_once() + + assert target_checks == 2 + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-b" + assert snapshot.device_network.target == TARGET + assert snapshot.host_path.epoch == 0 + assert snapshot.host_path.reason_code == "host-path-not-observed" + await monitor.close() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("changed_final_path", [False, True]) +def test_monitor_final_publication_cas_rejects_switch_after_target_check( + changed_final_path: bool, +) -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-a", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-a") + next_target = EndpointTarget("192.168.68.77") + first_path = _available_path() + final_path = ( + _available_path("en0:192.168.99.20:late-old-target-route") + if changed_final_path + else first_path + ) + paths = iter((first_path, final_path)) + target_checks = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return next(paths) + + async def tcp_probe(_target: EndpointTarget) -> bool: + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + def switch_on_final_check( + expected: EndpointTarget, + *, + expected_intent_id: str, + ) -> bool: + nonlocal target_checks + assert expected == TARGET + assert expected_intent_id == "bridge-a" + target_checks += 1 + if target_checks == 2: + supervisor.set_intent(intent_id="bridge-b", requested_mode="bridge") + _apply_device_network( + supervisor, + intent_id="bridge-b", + target=next_target, + ) + return True + + monitor._target_is_current = switch_on_final_check # type: ignore[method-assign] # noqa: SLF001 + snapshot = await monitor.poll_once() + + assert target_checks == 2 + assert snapshot.intent is not None + assert snapshot.intent.intent_id == "bridge-b" + assert snapshot.device_network.target == next_target + assert snapshot.host_path.epoch == 1 + assert snapshot.host_path.fingerprint == first_path.fingerprint + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.endpoint.target is None + assert snapshot.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_drops_changed_final_path_after_target_switch() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + next_target = EndpointTarget("192.168.68.77") + selected_target = TARGET + final_probe_started = asyncio.Event() + release_final_probe = asyncio.Event() + host_probe_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_probe_calls + assert target == TARGET + host_probe_calls += 1 + if host_probe_calls == 1: + return _available_path() + final_probe_started.set() + await release_final_probe.wait() + return _available_path("en0:192.168.77.20:late-old-target-route") + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: selected_target, + ) + poll = asyncio.create_task(monitor.poll_once()) + await final_probe_started.wait() + selected_target = next_target + release_final_probe.set() + snapshot = await poll + + assert host_probe_calls == 2 + assert snapshot.host_path.epoch == initial_epoch + assert snapshot.host_path.fingerprint == _available_path().fingerprint + assert snapshot.endpoint.target == TARGET + assert snapshot.endpoint.tcp_state == "reachable" + assert snapshot.authority.control_allowed is True + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_close_during_final_host_probe_drops_changed_path_without_error() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + final_probe_started = asyncio.Event() + release_final_probe = asyncio.Event() + host_probe_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_probe_calls + assert target == TARGET + host_probe_calls += 1 + if host_probe_calls == 1: + return _available_path() + final_probe_started.set() + await release_final_probe.wait() + return _available_path("en0:192.168.99.20:late-closed-route") + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + poll = asyncio.create_task(monitor.poll_once()) + await final_probe_started.wait() + await monitor.close() + release_final_probe.set() + snapshot = await poll + + assert host_probe_calls == 2 + assert snapshot.closed is True + assert snapshot.host_path.epoch == initial_epoch + assert snapshot.host_path.fingerprint == _available_path().fingerprint + assert snapshot.authority.control_allowed is False + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("failing_probe_call", [1, 2]) +def test_monitor_drops_host_probe_exception_after_target_switch( + failing_probe_call: int, +) -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + next_target = EndpointTarget("192.168.68.77") + selected_target = TARGET + failing_probe_started = asyncio.Event() + release_failing_probe = asyncio.Event() + host_probe_calls = 0 + tcp_probe_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_probe_calls + assert target == TARGET + host_probe_calls += 1 + if host_probe_calls != failing_probe_call: + return _available_path() + failing_probe_started.set() + await release_failing_probe.wait() + raise OSError("late host failure for old target") + + async def tcp_probe(target: EndpointTarget) -> bool: + nonlocal tcp_probe_calls + assert target == TARGET + tcp_probe_calls += 1 + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: selected_target, + ) + poll = asyncio.create_task(monitor.poll_once()) + await failing_probe_started.wait() + selected_target = next_target + release_failing_probe.set() + snapshot = await poll + + assert host_probe_calls == failing_probe_call + assert tcp_probe_calls == failing_probe_call - 1 + assert snapshot.host_path.epoch == initial_epoch + assert snapshot.host_path.available is True + assert snapshot.host_path.reason_code is None + assert snapshot.endpoint.target == TARGET + assert snapshot.endpoint.tcp_state == "reachable" + assert snapshot.authority.control_allowed is True + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_close_during_final_host_probe_exception_drops_failure() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + final_probe_started = asyncio.Event() + release_final_probe = asyncio.Event() + host_probe_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_probe_calls + assert target == TARGET + host_probe_calls += 1 + if host_probe_calls == 1: + return _available_path() + final_probe_started.set() + await release_final_probe.wait() + raise OSError("late final host failure after close") + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + poll = asyncio.create_task(monitor.poll_once()) + await final_probe_started.wait() + await monitor.close() + release_final_probe.set() + snapshot = await poll + + assert host_probe_calls == 2 + assert snapshot.closed is True + assert snapshot.host_path.epoch == initial_epoch + assert snapshot.host_path.available is True + assert snapshot.host_path.reason_code is None + assert snapshot.authority.control_allowed is False + + asyncio.run(scenario()) + + +def test_monitor_projects_typed_tcp_refusal_without_socket_details() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-refused", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-refused") + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return _available_path() + + async def tcp_probe(_target: EndpointTarget) -> TcpReachabilityProbeResult: + return TcpReachabilityProbeResult( + reachable=False, + reason_code="tcp-connection-refused", + ) + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + snapshot = await monitor.poll_once() + + assert snapshot.endpoint.tcp_state == "unreachable" + assert snapshot.endpoint.reason_code == "tcp-connection-refused" + assert [item.code for item in snapshot.diagnostics] == ["host.tcp.connection-refused"] + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_skips_tcp_when_route_probe_fails_and_closes_fail_closed() -> None: + async def scenario() -> None: + supervisor, _ = _reachable_supervisor() + tcp_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + return _unavailable_path("router-disappeared") + + async def tcp_probe(target: EndpointTarget) -> bool: + nonlocal tcp_calls + assert target == TARGET + tcp_calls += 1 + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + snapshot = await monitor.poll_once() + assert tcp_calls == 0 + assert snapshot.host_path.available is False + assert snapshot.lease.state == "configured-unverified" + assert snapshot.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_host_probe_exception_fails_closed_without_leaking_exception() -> None: + async def scenario() -> None: + supervisor, _ = _reachable_supervisor() + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + raise OSError("synthetic route diagnostic") + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + raise AssertionError("TCP must not run after a host-path failure") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + snapshot = await monitor.poll_once() + assert snapshot.host_path.reason_code == "host-path-probe-error" + assert [item.code for item in snapshot.diagnostics] == ["host.route.unavailable"] + assert snapshot.lease.state == "configured-unverified" + assert snapshot.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_monitor_close_during_external_probe_drops_the_late_result() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-1", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-1") + tcp_started = asyncio.Event() + release_tcp = asyncio.Event() + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + return _available_path() + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + tcp_started.set() + await release_tcp.wait() + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + poll = asyncio.create_task(monitor.poll_once()) + await tcp_started.wait() + await monitor.close() + release_tcp.set() + snapshot = await poll + + assert snapshot.closed is True + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.authority.control_allowed is False + + asyncio.run(scenario()) + + +def test_monitor_close_rejects_a_poll_queued_behind_an_inflight_probe() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-queued-close", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-queued-close") + first_probe_started = asyncio.Event() + release_first_probe = asyncio.Event() + host_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_calls + assert target == TARGET + host_calls += 1 + if host_calls > 1: + raise AssertionError("a queued poll contacted the host after close") + first_probe_started.set() + await release_first_probe.wait() + return _available_path() + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("close must supersede TCP contact") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + first_poll = asyncio.create_task(monitor.poll_once()) + await first_probe_started.wait() + queued_poll = asyncio.create_task(monitor.poll_once()) + # Let the queued poll pass the pre-lock closed check and block on the + # first poll's serialization lock before closing the monitor. + await asyncio.sleep(0) + await monitor.close() + release_first_probe.set() + + first_snapshot = await first_poll + assert first_snapshot.closed is True + with pytest.raises(ConnectionSupervisorClosed): + await queued_poll + assert host_calls == 1 + + asyncio.run(scenario()) + + +def test_monitor_loop_recovers_after_one_unexpected_iteration_failure() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-recovery", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-recovery") + recovered = asyncio.Event() + poll_calls = 0 + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + return _available_path() + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + interval_seconds=0.05, + ) + original_poll_once = monitor.poll_once + + async def fail_once_then_poll() -> object: + nonlocal poll_calls + poll_calls += 1 + if poll_calls == 1: + raise RuntimeError("synthetic monitor iteration failure") + snapshot = await original_poll_once() + recovered.set() + return snapshot + + monitor.poll_once = fail_once_then_poll # type: ignore[method-assign] + await monitor.start() + await asyncio.wait_for(recovered.wait(), timeout=1.0) + + snapshot = supervisor.snapshot() + assert poll_calls >= 2 + assert snapshot.host_path.available is True + assert snapshot.endpoint.tcp_state == "reachable" + await monitor.close() + + asyncio.run(scenario()) + + +def test_verified_control_survives_thirty_seconds_of_alternating_unproven_association( +) -> None: + async def scenario() -> None: + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + suspend_aware_clock=lambda: suspend_aware_now[0], + observation_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="bridge-deferred-timeout", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-deferred-timeout") + path = _available_path() + epoch = supervisor.observe_host_path(path) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-deferred-timeout", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-deferred-timeout", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-deferred-timeout", + ) + ) + initial = supervisor.snapshot() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + privacy_limited = HostPathProbeResult( + available=True, + fingerprint="privacy-fallback-must-not-replace-proven-identity", + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class=path.route_class, + reason_code="association-identity-unavailable", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + host_calls = 0 + tcp_calls = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_calls + host_calls += 1 + # Exercise both orderings inside a complete route/TCP/recheck + # pass: timeout -> privacy and privacy -> timeout. + return (technical_timeout, privacy_limited, privacy_limited, technical_timeout)[ + (host_calls - 1) % 4 + ] + + async def tcp_probe(_target: EndpointTarget) -> bool: + nonlocal tcp_calls + tcp_calls += 1 + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + refreshed = initial + for _ in range(31): + monotonic_now[0] += 1.0 + suspend_aware_now[0] += 1.0 + refreshed = await monitor.poll_once() + assert refreshed.host_path.available is True + assert refreshed.host_path.epoch == initial.host_path.epoch + assert refreshed.host_path.fingerprint == initial.host_path.fingerprint + assert refreshed.endpoint.tcp_state == "reachable" + assert refreshed.endpoint.host_path_epoch == initial.host_path.epoch + assert refreshed.device_identity.state == "verified" + assert refreshed.control_plane.state == "healthy" + assert refreshed.lease.state == "reachable" + assert refreshed.lease.generation == initial.lease.generation + assert refreshed.authority.control_allowed is True + + assert host_calls == 62 + assert tcp_calls == 31 + assert refreshed.revision == initial.revision + 62 + + # Retention refreshes only evidence actually sampled by the monitor. + # If the monitor itself goes silent, the ordinary transport TTL still + # fails closed instead of turning this continuity bridge into a lease. + monotonic_now[0] += 2.01 + suspend_aware_now[0] += 2.01 + expired = supervisor.snapshot() + assert expired.host_path.available is False + assert expired.host_path.reason_code == "host-path-observation-stale" + assert expired.authority.control_allowed is False + assert expired.lease.state != "reachable" + await monitor.close() + + asyncio.run(scenario()) + + +def test_unproven_association_retention_still_revokes_on_exact_tcp_loss() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + initial = supervisor.snapshot() + path = _available_path() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + host_calls = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_calls + host_calls += 1 + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> TcpReachabilityProbeResult: + return TcpReachabilityProbeResult( + reachable=False, + reason_code="tcp-connect-refused", + ) + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + lost = await monitor.poll_once() + + assert host_calls == 2 + assert lost.host_path.available is True + assert lost.host_path.epoch == initial_epoch + assert lost.host_path.fingerprint == initial.host_path.fingerprint + assert lost.endpoint.tcp_state == "unreachable" + assert lost.endpoint.reason_code == "tcp-connect-refused" + assert lost.control_plane.state == "lost" + assert lost.control_plane.reason_code == "tcp-connect-refused" + assert lost.lease.state != "reachable" + assert lost.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_proven_association_change_after_bridged_tcp_rotates_epoch_immediately() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + initial = supervisor.snapshot() + path = _available_path() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + changed_association = HostPathProbeResult( + available=True, + fingerprint="same-kernel-route:new-proven-bssid-token", + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class=path.route_class, + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + samples = iter((technical_timeout, changed_association)) + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return next(samples) + + async def tcp_probe(_target: EndpointTarget) -> bool: + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + changed = await monitor.poll_once() + + assert changed.host_path.available is True + assert changed.host_path.epoch == initial_epoch + 1 + assert changed.host_path.fingerprint == changed_association.fingerprint + assert changed.host_path.kernel_route_fingerprint == ( + initial.host_path.kernel_route_fingerprint + ) + assert changed.endpoint.tcp_state == "unknown" + assert changed.device_identity.state == "stale" + assert changed.control_plane.state == "lost" + assert changed.lease.state != "reachable" + assert changed.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_control_loss_during_unproven_recheck_cannot_be_restored_by_tcp_success() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _reachable_supervisor() + path = _available_path() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> bool: + assert supervisor.observe_control_loss( + intent_id="bridge-1", + host_path_epoch=initial_epoch, + control_session_id="control-session-a", + reason_code="mqtt-control-session-closed", + ) + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + lost = await monitor.poll_once() + + assert lost.host_path.epoch == initial_epoch + assert lost.control_plane.state == "lost" + assert lost.control_plane.reason_code == "mqtt-control-session-closed" + assert lost.lease.state != "reachable" + assert lost.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_control_proof_ttl_still_expires_during_healthy_transport_retention() -> None: + async def scenario() -> None: + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + suspend_aware_clock=lambda: suspend_aware_now[0], + observation_ttl_seconds=2.0, + control_proof_ttl_seconds=3.0, + ) + supervisor.set_intent( + intent_id="bridge-control-ttl", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-control-ttl") + path = _available_path() + epoch = supervisor.observe_host_path(path) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-control-ttl", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-control-ttl", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-ttl-during-retention", + ) + ) + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> bool: + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + for _ in range(3): + monotonic_now[0] += 0.9 + suspend_aware_now[0] += 0.9 + fresh = await monitor.poll_once() + assert fresh.host_path.epoch == epoch + assert fresh.endpoint.tcp_state == "reachable" + assert fresh.authority.control_allowed is True + + monotonic_now[0] += 0.31 + suspend_aware_now[0] += 0.31 + expired = await monitor.poll_once() + + assert expired.control_plane.state == "lost" + assert expired.control_plane.reason_code == "control-proof-observation-stale" + assert expired.lease.state != "reachable" + assert expired.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_same_path_positive_reducer_observation_resets_technical_failure_streak() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _configured_unverified_supervisor() + path = _available_path() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("technical failure must skip TCP") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + first = await monitor.poll_once() + second = await monitor.poll_once() + assert first.authority.control_allowed is False + assert second.authority.control_allowed is False + + refreshed_epoch = supervisor.observe_host_path(path) + assert refreshed_epoch == initial_epoch + external_positive = supervisor.snapshot() + + after_reset_first = await monitor.poll_once() + after_reset_second = await monitor.poll_once() + for deferred in (after_reset_first, after_reset_second): + assert deferred.revision == external_positive.revision + assert deferred.host_path == external_positive.host_path + assert deferred.authority.control_allowed is False + + confirmed = await monitor.poll_once() + assert confirmed.host_path.available is False + assert confirmed.host_path.reason_code == "host-wifi-operation-timeout" + assert confirmed.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_unrelated_supervisor_revisions_do_not_reset_technical_failure_streak() -> None: + async def scenario() -> None: + supervisor, epoch = _configured_unverified_supervisor() + path = _available_path() + initial = supervisor.snapshot() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("technical failure must skip TCP") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + + first = await monitor.poll_once() + assert first.revision == initial.revision + assert first.host_path is initial.host_path + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + endpoint_refresh = supervisor.snapshot() + assert endpoint_refresh.revision > first.revision + assert endpoint_refresh.host_path is initial.host_path + + second = await monitor.poll_once() + assert second.revision == endpoint_refresh.revision + assert second.host_path is initial.host_path + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-1", + host_path_epoch=epoch, + reachable=True, + ) + + confirmed = await monitor.poll_once() + assert confirmed.host_path.available is False + assert confirmed.host_path.reason_code == "host-wifi-operation-timeout" + assert confirmed.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_inflight_third_timeout_cannot_overwrite_a_concurrent_positive_observation() -> None: + async def scenario() -> None: + supervisor, initial_epoch = _configured_unverified_supervisor() + path = _available_path() + technical_timeout = HostPathProbeResult( + available=False, + fingerprint=None, + interface=path.interface, + source_ipv4=path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=path.kernel_route_fingerprint, + ) + third_probe_entered = asyncio.Event() + release_third_probe = asyncio.Event() + host_calls = 0 + + async def host_probe(_target: EndpointTarget) -> HostPathProbeResult: + nonlocal host_calls + host_calls += 1 + if host_calls == 3: + third_probe_entered.set() + await release_third_probe.wait() + return technical_timeout + + async def tcp_probe(_target: EndpointTarget) -> bool: + raise AssertionError("technical failure must skip TCP") + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + first = await monitor.poll_once() + second = await monitor.poll_once() + assert first.authority.control_allowed is False + assert second.authority.control_allowed is False + + inflight_third = asyncio.create_task(monitor.poll_once()) + await third_probe_entered.wait() + refreshed_epoch = supervisor.observe_host_path(path) + assert refreshed_epoch == initial_epoch + external_positive = supervisor.snapshot() + release_third_probe.set() + raced_timeout = await inflight_third + + assert raced_timeout.revision == external_positive.revision + assert raced_timeout.host_path is external_positive.host_path + assert raced_timeout.authority.control_allowed is False + + second_after_positive = await monitor.poll_once() + assert second_after_positive.revision == external_positive.revision + assert second_after_positive.host_path is external_positive.host_path + assert second_after_positive.authority.control_allowed is False + + confirmed = await monitor.poll_once() + assert confirmed.host_path.available is False + assert confirmed.host_path.reason_code == "host-wifi-operation-timeout" + assert confirmed.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) + + +def test_observation_ttl_revokes_authority_after_monitor_silence() -> None: + monotonic_now = [100.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + observation_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="bridge-ttl", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-ttl") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-ttl", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-ttl", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-ttl", + ) + ) + assert supervisor.observe_data_plane( + intent_id="bridge-ttl", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-route-ttl", + ) + assert supervisor.snapshot().authority.control_allowed is True + + monotonic_now[0] += 2.01 + expired = supervisor.snapshot() + + assert expired.host_path.epoch == epoch + 1 + assert expired.host_path.available is False + assert expired.host_path.reason_code == "host-path-observation-stale" + assert expired.endpoint.tcp_state == "unknown" + assert expired.device_identity.state == "stale" + assert expired.control_plane.state == "lost" + assert expired.lease.state == "configured-unverified" + assert expired.authority.control_allowed is False + assert expired.authority.data_ingest_authoritative is False + assert expired.data_plane.state == "healthy" + assert expired.data_plane.session_id == "data-session-route-ttl" + assert expired.data_plane.host_path_epoch == epoch + assert expired.last_known is not None + + +def test_endpoint_ttl_revokes_authority_even_when_route_observation_stays_fresh() -> None: + monotonic_now = [200.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + observation_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="quick-ttl", + requested_mode="quick-connect", + expected_device_id="k1-device-a", + ) + _apply_device_network( + supervisor, + intent_id="quick-ttl", + connection_mode="quick-connect", + ) + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="quick-ttl", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="quick-ttl", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="quick-connect", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="quick-control-ttl", + ) + ) + assert supervisor.observe_data_plane( + intent_id="quick-ttl", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-endpoint-ttl", + ) + + monotonic_now[0] += 1.5 + assert supervisor.observe_host_path(_available_path()) == epoch + monotonic_now[0] += 0.6 + expired = supervisor.snapshot() + + assert expired.host_path.available is True + assert expired.host_path.epoch == epoch + assert expired.endpoint.tcp_state == "unreachable" + assert expired.endpoint.reason_code == "endpoint-observation-stale" + assert expired.control_plane.state == "lost" + assert expired.lease.state == "configured-unverified" + assert expired.authority.control_allowed is False + assert expired.authority.data_ingest_authoritative is False + assert expired.data_plane.state == "healthy" + assert expired.data_plane.session_id == "data-session-endpoint-ttl" + assert expired.data_plane.host_path_epoch == epoch + + +def test_suspend_elapsed_time_revokes_authority_when_macos_monotonic_stops() -> None: + monotonic_now = [300.0] + suspend_aware_now = [1_000.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + suspend_aware_clock=lambda: suspend_aware_now[0], + observation_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="bridge-suspend", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-suspend") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-suspend", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-suspend", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-before-sleep", + ) + ) + + # mach_absolute_time/time.monotonic may remain unchanged while macOS is + # asleep. Wall elapsed time must still invalidate every pre-sleep proof. + suspend_aware_now[0] += 60.0 + expired = supervisor.snapshot() + + assert monotonic_now[0] == 300.0 + assert expired.host_path.epoch == epoch + 1 + assert expired.host_path.reason_code == "host-path-observation-stale" + assert expired.authority.control_allowed is False + assert expired.authority.acquisition_start_allowed is False + + +def test_control_proof_ttl_is_independent_from_fresh_route_and_tcp_observations() -> None: + monotonic_now = [400.0] + suspend_aware_now = [4_000.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + suspend_aware_clock=lambda: suspend_aware_now[0], + observation_ttl_seconds=300.0, + control_proof_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="bridge-control-proof-ttl", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-control-proof-ttl") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-control-proof-ttl", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-control-proof-ttl", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-proof-session", + control_proof_revision=4, + control_proof_source="correlated-application-response", + ) + ) + assert supervisor.observe_data_plane( + intent_id="bridge-control-proof-ttl", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-10", + ) + + # Replaying the same API snapshot is idempotent; it is not new remote + # evidence and therefore cannot extend authority. + monotonic_now[0] += 1.5 + suspend_aware_now[0] += 1.5 + assert ( + supervisor.refresh_control_evidence( + intent_id="bridge-control-proof-ttl", + host_path_epoch=epoch, + control_session_id="control-proof-session", + control_proof_revision=4, + control_proof_source="correlated-application-response", + ) + is False + ) + assert supervisor.observe_host_path(_available_path()) == epoch + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-control-proof-ttl", + host_path_epoch=epoch, + reachable=True, + ) + + monotonic_now[0] += 0.6 + suspend_aware_now[0] += 0.6 + expired = supervisor.snapshot() + + assert expired.host_path.available is True + assert expired.endpoint.tcp_state == "reachable" + assert expired.device_identity.state == "stale" + assert expired.control_plane.state == "lost" + assert expired.control_plane.reason_code == "control-proof-observation-stale" + assert expired.lease.state == "configured-unverified" + assert expired.authority.control_allowed is False + assert expired.authority.acquisition_start_allowed is False + assert expired.authority.data_ingest_authoritative is False + assert expired.data_plane.state == "healthy" + assert expired.data_plane.session_id == "data-session-generation-10" + assert expired.data_plane.host_path_epoch == epoch + assert [item.code for item in expired.diagnostics] == ["host.mqtt.transport-unavailable"] + + assert supervisor.observe_data_plane( + intent_id="bridge-control-proof-ttl", + host_path_epoch=epoch, + state="healthy", + session_id="data-session-generation-10", + ) + evidence_only = supervisor.snapshot() + assert evidence_only.control_plane.state == "lost" + assert evidence_only.data_plane.state == "healthy" + assert evidence_only.authority.control_allowed is False + assert evidence_only.authority.data_ingest_authoritative is False + + +def test_new_exact_control_proof_revision_refreshes_dual_clock_authority() -> None: + monotonic_now = [500.0] + suspend_aware_now = [5_000.0] + supervisor = ConnectionSupervisor( + clock=_clock, + monotonic_clock=lambda: monotonic_now[0], + suspend_aware_clock=lambda: suspend_aware_now[0], + observation_ttl_seconds=300.0, + control_proof_ttl_seconds=2.0, + ) + supervisor.set_intent( + intent_id="bridge-control-proof-refresh", + requested_mode="bridge", + expected_device_id="k1-device-a", + ) + _apply_device_network(supervisor, intent_id="bridge-control-proof-refresh") + epoch = supervisor.observe_host_path(_available_path()) + assert supervisor.observe_endpoint( + target=TARGET, + intent_id="bridge-control-proof-refresh", + host_path_epoch=epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="bridge-control-proof-refresh", + transport_ref="ble-k1-a", + host_path_epoch=epoch, + target=TARGET, + connection_mode="bridge", + logical_device_id="k1-device-a", + compatibility_profile_id="xgrids-k1-fw-3.0.2", + control_session_id="control-proof-refresh-session", + control_proof_revision=1, + ) + ) + + monotonic_now[0] += 1.8 + suspend_aware_now[0] += 1.8 + assert supervisor.refresh_control_evidence( + intent_id="bridge-control-proof-refresh", + host_path_epoch=epoch, + control_session_id="control-proof-refresh-session", + control_proof_revision=2, + control_proof_source="mqtt-heartbeat", + ) + + monotonic_now[0] += 1.8 + suspend_aware_now[0] += 1.8 + still_fresh = supervisor.snapshot() + assert still_fresh.authority.control_allowed is True + + # macOS monotonic can stop while asleep; wall elapsed time alone must + # revoke the refreshed proof without waiting for a route transition. + suspend_aware_now[0] += 60.0 + expired = supervisor.snapshot() + assert expired.host_path.available is True + assert expired.endpoint.tcp_state == "reachable" + assert expired.control_plane.reason_code == "control-proof-observation-stale" + assert expired.authority.control_allowed is False + + +def test_data_error_before_first_packet_is_an_ignored_observation() -> None: + supervisor, epoch = _reachable_supervisor() + + assert ( + supervisor.observe_data_plane( + intent_id="bridge-1", + host_path_epoch=epoch, + state="lost", + session_id=None, + reason_code="live-runtime-error", + ) + is False + ) + snapshot = supervisor.snapshot() + assert snapshot.data_plane.state == "idle" + assert snapshot.authority.control_allowed is True + + +def test_monitor_rechecks_route_after_tcp_and_rejects_success_from_old_wifi() -> None: + async def scenario() -> None: + supervisor = ConnectionSupervisor(clock=_clock) + supervisor.set_intent(intent_id="bridge-race", requested_mode="bridge") + _apply_device_network(supervisor, intent_id="bridge-race") + paths = iter( + ( + _available_path("en0:192.168.68.20:router-a"), + _available_path("en0:192.168.77.20:router-b"), + ) + ) + + async def host_probe(target: EndpointTarget) -> HostPathProbeResult: + assert target == TARGET + return next(paths) + + async def tcp_probe(target: EndpointTarget) -> bool: + assert target == TARGET + return True + + monitor = ReadOnlyConnectionMonitor( + supervisor, + host_path_probe=host_probe, + tcp_probe=tcp_probe, + target_provider=lambda: TARGET, + ) + snapshot = await monitor.poll_once() + + assert snapshot.host_path.fingerprint == "en0:192.168.77.20:router-b" + assert snapshot.host_path.epoch == 2 + assert snapshot.endpoint.tcp_state == "unknown" + assert snapshot.authority.control_allowed is False + await monitor.close() + + asyncio.run(scenario()) diff --git a/tests/test_device_lifecycle.py b/tests/test_device_lifecycle.py index 89fc04b..989ac81 100644 --- a/tests/test_device_lifecycle.py +++ b/tests/test_device_lifecycle.py @@ -37,7 +37,11 @@ def test_operation_journal_reuses_an_idempotent_request_without_storing_input() def test_operation_journal_records_ack_progress_and_terminal_result() -> None: journal = OperationJournal() - operation, _ = journal.begin("acquisition.prepare", cancellable=True) + operation, _ = journal.begin( + "acquisition.prepare", + cancellable=True, + context={"automatic_retry": False}, + ) journal.transition( operation.operation_id, @@ -61,6 +65,14 @@ def test_operation_journal_records_ack_progress_and_terminal_result() -> None: assert document["result"] == {"acquisition_id": "acq-test"} assert document["evidence_refs"] == ["evidence-manifest-test"] assert document["completed_at"] is not None + assert document["context"] == {"automatic_retry": False} + assert [event["stage_code"] for event in document["events"]] == [ + "accepted", + "compatibility-check", + "prepared", + ] + assert document["events"][-1]["status"] == "succeeded" + assert document["events"][-1]["automatic_retry"] is False with pytest.raises(ValueError, match="already terminal"): journal.transition( diff --git a/tests/test_frontend_assets.py b/tests/test_frontend_assets.py new file mode 100644 index 0000000..21ab28c --- /dev/null +++ b/tests/test_frontend_assets.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from k1link.web.frontend_assets import ( + HASHED_ASSET_IMMUTABLE, + HTML_NO_STORE, + ControlStationStaticFiles, + frontend_build_id, +) + + +def _frontend(tmp_path: Path) -> Path: + frontend = tmp_path / "dist" + assets = frontend / "assets" + assets.mkdir(parents=True) + (frontend / "index.html").write_text( + """ + + """, + encoding="utf-8", + ) + (assets / "index-dT7dN-y4.js").write_text("export {};", encoding="utf-8") + (assets / "runtime.js").write_text("export {};", encoding="utf-8") + return frontend + + +def test_frontend_build_id_is_exact_hashed_entry_module(tmp_path: Path) -> None: + frontend = _frontend(tmp_path) + + assert frontend_build_id(frontend) == "/assets/index-dT7dN-y4.js" + + (frontend / "index.html").write_text( + '', + encoding="utf-8", + ) + assert frontend_build_id(frontend) is None + + +def test_spa_shell_is_no_store_and_only_hashed_assets_are_immutable( + tmp_path: Path, +) -> None: + frontend = _frontend(tmp_path) + app = FastAPI() + app.mount("/", ControlStationStaticFiles(directory=frontend, html=True)) + client = TestClient(app) + + shell = client.get("/") + hashed = client.get("/assets/index-dT7dN-y4.js") + unhashed = client.get("/assets/runtime.js") + + assert shell.status_code == 200 + assert shell.headers["cache-control"] == HTML_NO_STORE + assert hashed.status_code == 200 + assert hashed.headers["cache-control"] == HASHED_ASSET_IMMUTABLE + assert unhashed.status_code == 200 + assert "immutable" not in unhashed.headers.get("cache-control", "") diff --git a/tests/test_frontend_plugin_boundaries.py b/tests/test_frontend_plugin_boundaries.py index 9679193..80227c2 100644 --- a/tests/test_frontend_plugin_boundaries.py +++ b/tests/test_frontend_plugin_boundaries.py @@ -104,8 +104,13 @@ def test_xgrids_frontend_uses_semantic_acquisition_actions_and_stable_identity() assert hook_source.index("xgridsK1Api.prepareAcquisition") < hook_source.index( "xgridsK1Api.startAcquisition" ) - assert "isSoftwareCommandedAcquisition(state)" in hook_source - assert '? "graceful" : "capture-only"' in hook_source + assert "isSoftwareCommandedAcquisition(currentState)" in hook_source + assert hook_source.index("isSoftwareCommandedAcquisition(currentState)") < ( + hook_source.index('mode: "graceful"') + ) + assert hook_source.index('mode: "graceful"') < hook_source.index( + 'mode: "capture-only"' + ) assert "state.device_ref" in runtime_source assert "instanceId: deviceRef.device_id" in runtime_source assert "acquisition?.acquisition_id" in runtime_source @@ -113,7 +118,7 @@ def test_xgrids_frontend_uses_semantic_acquisition_actions_and_stable_identity() assert 'id: "xgrids-k1-rerun-live"' not in runtime_source -def test_xgrids_live_copy_exposes_one_response_gated_launch_intent() -> None: +def test_xgrids_live_copy_exposes_response_gated_prepare_and_one_physical_start() -> None: repository_root = Path(__file__).resolve().parents[1] connection_source = ( repository_root @@ -125,16 +130,26 @@ def test_xgrids_live_copy_exposes_one_response_gated_launch_intent() -> None: / "K1AcquisitionPipeline.tsx" ).read_text("utf-8") - assert "Запустить сканирование и локальный приём" in connection_source - assert "Имя войдёт в единственный канонический START" in connection_source - assert "этапы идут строго по записанному порядку и только после ответов K1" in ( + assert "Запустить приём" in connection_source + assert "Запустить K1" not in connection_source + assert "Одно нажатие выполняет каноническую подготовку и один START" in ( connection_source ) + assert "Проверить условия и отправить START" not in connection_source + assert "Физический START отправляется только после отдельного финального окна" not in ( + connection_source + ) + assert "Имя войдёт в единственный канонический START" in connection_source + assert "один START после подтверждённого READY" in connection_source + assert "Автоматических повторов команд нет" in connection_source assert "Подключить управление K1" not in connection_source assert "Открыть рабочее пространство K1" not in connection_source assert "Сохранить проект и подготовить локальный приём" not in connection_source - assert "Остановить локальный приём" in connection_source - assert "Физическое состояние сканера остаётся неизвестным" in connection_source + assert "Аварийно завершить локальный приём" in connection_source + assert "Состояние сканирования остаётся неизвестным" in connection_source + assert "Физическое состояние сканера остаётся неизвестным" not in ( + connection_source + ) def test_xgrids_start_uses_atomic_automatic_source_transition() -> None: diff --git a/tests/test_k1_connection_acceptance_manifest.py b/tests/test_k1_connection_acceptance_manifest.py new file mode 100644 index 0000000..81fba08 --- /dev/null +++ b/tests/test_k1_connection_acceptance_manifest.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + + +def test_k1_connection_acceptance_manifest_covers_every_canonical_scenario() -> None: + repository_root = Path(__file__).resolve().parents[1] + canonical_path = repository_root / "docs" / "20_K1_CONNECTION_SUPERVISION_CANON.md" + manifest_path = repository_root / "docs" / "k1-connection-acceptance.manifest.json" + canonical_ids = set(re.findall(r"\bCONN-\d{2}\b", canonical_path.read_text("utf-8"))) + manifest = json.loads(manifest_path.read_text("utf-8")) + scenarios = manifest["scenarios"] + manifest_ids = [scenario["id"] for scenario in scenarios] + + assert manifest["schema_version"] == "missioncore.k1-connection-acceptance/v1" + assert len(manifest_ids) == len(set(manifest_ids)) + assert set(manifest_ids) == canonical_ids + assert all( + scenario["status"] in {"software-covered", "partial", "planned"} for scenario in scenarios + ) + for scenario in scenarios: + assert isinstance(scenario["remaining"], list) + if scenario["status"] in {"partial", "planned"}: + assert scenario["remaining"] + for relative_path in scenario["test_files"]: + assert (repository_root / relative_path).is_file(), ( + scenario["id"], + relative_path, + ) + + by_id = {scenario["id"]: scenario for scenario in scenarios} + assert by_id["CONN-39"] == { + "id": "CONN-39", + "status": "software-covered", + "test_files": ["tests/test_xgrids_network_mutation_ledger.py"], + "remaining": [], + } + assert by_id["CONN-67"] == { + "id": "CONN-67", + "status": "partial", + "test_files": ["tests/test_xgrids_acquisition_lifecycle.py"], + "remaining": [ + "real K1 repeated same-mode and cross-mode reconnect acceptance", + ], + } diff --git a/tests/test_live_perception.py b/tests/test_live_perception.py index 22b80ee..8645ac5 100644 --- a/tests/test_live_perception.py +++ b/tests/test_live_perception.py @@ -97,6 +97,8 @@ def test_live_ingress_wire_is_self_delimiting_and_explicitly_non_authoritative() header_bytes = struct.unpack("!I", encoded[:4])[0] header = json.loads(encoded[4 : 4 + header_bytes]) assert header["schema_version"] == LIVE_INGRESS_WIRE_SCHEMA + assert header["session_id"] == "session-1" + assert header["session_generation"] == 1 assert header["payload_bytes"] == 4 assert header["commands_enabled"] is False assert header["navigation_or_safety_accepted"] is False @@ -157,14 +159,45 @@ def test_live_ingress_new_session_discards_queued_events_from_previous_session() events.append(event) assert len(events) == 1 assert events[0].session_id == "session-2" + assert events[0].session_generation == 2 assert events[0].modality == "control" assert events[0].payload == b'{"event":"session-start"}' +def test_live_result_admission_is_bound_to_exact_session_generation() -> None: + ingress = LivePerceptionIngress() + ingress.begin_session("session-1") + first_generation = ingress.snapshot()["session_generation"] + ingress.end_session("session-1") + ingress.begin_session("session-2") + second_generation = ingress.snapshot()["session_generation"] + received: list[str] = [] + + assert ( + ingress.admit_result( + session_id="session-1", + session_generation=first_generation, + receiver=lambda: not received.append("stale"), + ) + is False + ) + assert ( + ingress.admit_result( + session_id="session-2", + session_generation=second_generation, + receiver=lambda: not received.append("current"), + ) + is True + ) + assert received == ["current"] + + def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> None: mask = np.zeros((600, 800), dtype=np.uint8) mask[100:120, 200:240] = 4 encoded = encode_live_perception_result( + session_id="session-1", + session_generation=1, frame_index=12, source_frame_index=44, session_seconds=1.25, @@ -188,6 +221,8 @@ def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> frame = decode_live_perception_result(encoded) assert frame.frame_index == 12 + assert frame.session_id == "session-1" + assert frame.session_generation == 1 assert frame.source_frame_index == 44 assert frame.image_jpeg == b"\xff\xd8test\xff\xd9" assert frame.segmentation_mask is not None @@ -200,6 +235,8 @@ def test_live_result_round_trip_keeps_video_mask_boxes_and_shadow_authority() -> def test_live_result_rejects_tampering_and_partial_cuboid() -> None: with pytest.raises(ValueError, match="cuboid is incomplete"): encode_live_perception_result( + session_id="session-1", + session_generation=1, frame_index=0, source_frame_index=0, session_seconds=0.0, @@ -218,6 +255,8 @@ def test_live_result_rejects_tampering_and_partial_cuboid() -> None: delivery={"health": "degraded"}, ) encoded = encode_live_perception_result( + session_id="session-1", + session_generation=1, frame_index=0, source_frame_index=0, session_seconds=0.0, diff --git a/tests/test_live_perception_shadow.py b/tests/test_live_perception_shadow.py index f6ed43e..7664858 100644 --- a/tests/test_live_perception_shadow.py +++ b/tests/test_live_perception_shadow.py @@ -7,18 +7,24 @@ import time from pathlib import Path import numpy as np +import pytest from k1link.compute.live_perception import ( LivePerceptionIngress, encode_live_perception_result, ) from k1link.device_plugins.xgrids_k1.live_perception_shadow import ( + build_live_perception_result_receiver, build_live_perception_shadow_router, ensure_live_shadow_token, ) -def test_shadow_token_is_stable_and_private(tmp_path: Path) -> None: +def test_shadow_token_is_stable_and_private( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("MISSIONCORE_DATA_DIR", raising=False) path, token = ensure_live_shadow_token(tmp_path) repeated_path, repeated_token = ensure_live_shadow_token(tmp_path) @@ -36,9 +42,7 @@ def test_shadow_router_exposes_only_the_exclusive_binary_stream() -> None: bearer_token="x" * 43, ) assert len(router.routes) == 1 - assert router.routes[0].path == ( - "/api/v1/device-plugins/test-plugin/live-perception-shadow" - ) + assert router.routes[0].path == ("/api/v1/device-plugins/test-plugin/live-perception-shadow") def test_shadow_router_accepts_only_validated_diagnostic_results_back() -> None: @@ -59,6 +63,8 @@ def test_shadow_router_accepts_only_validated_diagnostic_results_back() -> None: ) ingress.begin_session("session-1") encoded = encode_live_perception_result( + session_id="session-1", + session_generation=1, frame_index=0, source_frame_index=0, session_seconds=0.0, @@ -100,6 +106,95 @@ def test_shadow_router_accepts_only_validated_diagnostic_results_back() -> None: assert received == [encoded] +def test_shadow_result_receiver_rejects_previous_acquisition_generation() -> None: + ingress = LivePerceptionIngress() + published: list[tuple[str, int]] = [] + receiver = build_live_perception_result_receiver( + ingress, + lambda frame: not published.append((frame.session_id, frame.session_generation)), + ) + ingress.begin_session("session-1") + encoded = encode_live_perception_result( + session_id="session-1", + session_generation=1, + frame_index=0, + source_frame_index=0, + session_seconds=0.0, + captured_at_epoch_ns=1, + image_jpeg=bytes.fromhex("ffd878ffd9"), + segmentation_mask=None, + objects=[], + delivery={"health": "healthy"}, + ) + ingress.end_session("session-1") + ingress.begin_session("session-2") + + assert receiver(encoded) is False + assert published == [] + snapshot = ingress.snapshot() + assert snapshot["results_accepted"] == 0 + assert snapshot["results_rejected_stale"] == 1 + assert snapshot["results_rejected_receiver"] == 0 + + +def test_shadow_router_closes_a_worker_that_publishes_for_previous_session() -> None: + ingress = LivePerceptionIngress() + published: list[int] = [] + receiver = build_live_perception_result_receiver( + ingress, + lambda frame: not published.append(frame.frame_index), + ) + router = build_live_perception_shadow_router( + ingress, + "test-plugin", + bearer_token="x" * 43, + result_receiver=receiver, + ) + ingress.begin_session("session-1") + stale = encode_live_perception_result( + session_id="session-1", + session_generation=1, + frame_index=7, + source_frame_index=7, + session_seconds=0.0, + captured_at_epoch_ns=1, + image_jpeg=bytes.fromhex("ffd878ffd9"), + segmentation_mask=None, + objects=[], + delivery={"health": "healthy"}, + ) + ingress.end_session("session-1") + ingress.begin_session("session-2") + + class StaleResultWebSocket: + def __init__(self) -> None: + self.headers = {"authorization": f"Bearer {'x' * 43}"} + self.closed: list[dict[str, object]] = [] + + async def accept(self) -> None: + return + + async def receive(self) -> dict[str, object]: + await asyncio.sleep(0) + return {"type": "websocket.receive", "bytes": stale} + + async def send_bytes(self, _payload: bytes) -> None: + return + + async def close(self, **values: object) -> None: + self.closed.append(values) + + websocket = StaleResultWebSocket() + asyncio.run(router.routes[0].endpoint(websocket)) # type: ignore[attr-defined] + + assert published == [] + assert any( + item.get("code") == 1008 and item.get("reason") == "Shadow result session is stale" + for item in websocket.closed + ) + assert ingress.snapshot()["results_rejected_stale"] == 1 + + def test_shadow_router_does_not_discard_ingress_while_receiving_results() -> None: class SlowIngress(LivePerceptionIngress): def take_next( @@ -135,9 +230,7 @@ def test_shadow_router_does_not_discard_ingress_while_receiving_results() -> Non payload=modality.encode(), ) ingress.end_session("session-1") - expected_events = sum( - int(queue["depth"]) for queue in ingress.snapshot()["queues"].values() - ) + expected_events = sum(int(queue["depth"]) for queue in ingress.snapshot()["queues"].values()) class DuplexFakeWebSocket: def __init__(self) -> None: @@ -172,6 +265,4 @@ def test_shadow_router_does_not_discard_ingress_while_receiving_results() -> Non assert len(received) == 8 assert len(websocket.sent) == expected_events snapshot = ingress.snapshot() - assert sum( - int(queue["consumed"]) for queue in snapshot["queues"].values() - ) == expected_events + assert sum(int(queue["consumed"]) for queue in snapshot["queues"].values()) == expected_events diff --git a/tests/test_mqtt_capture.py b/tests/test_mqtt_capture.py index a0d07af..d3bf7cf 100644 --- a/tests/test_mqtt_capture.py +++ b/tests/test_mqtt_capture.py @@ -82,6 +82,99 @@ class FakeClient: return mqtt.MQTT_ERR_SUCCESS +class ConnectionLostAfterMessageClient(FakeClient): + """Emit one complete subscription/message cycle, then lose the socket.""" + + def loop(self, timeout: float) -> mqtt.MQTTErrorCode: + if self._step < 3: # noqa: SLF001 - deterministic MQTT test double + return super().loop(timeout) + assert timeout > 0 + self._step += 1 # noqa: SLF001 + return mqtt.MQTT_ERR_CONN_LOST + + +class SubscriptionUnavailableClient(FakeClient): + """Accept TCP/CONNACK, then fail the transient recovery subscription.""" + + def subscribe(self, topics: Any) -> tuple[mqtt.MQTTErrorCode, int]: + self.subscribe_calls.append(topics) + return mqtt.MQTT_ERR_NO_CONN, 0 + + +class SubscribedWithoutPointCloudClient(FakeClient): + """Reach SUBACK, emit only non-PCL reports, then lose the socket.""" + + def loop(self, timeout: float) -> mqtt.MQTTErrorCode: + if self._step < 2: # noqa: SLF001 - deterministic MQTT test double + return super().loop(timeout) + assert timeout > 0 + self._step += 1 # noqa: SLF001 + if self._step == 3: # noqa: SLF001 + assert self.on_message is not None + message = mqtt.MQTTMessage(topic=b"lixel/application/report/heartbeat") + message.payload = b"fresh-heartbeat" + message.qos = 0 + self.on_message(self, None, message) + return mqtt.MQTT_ERR_SUCCESS + if self._step == 4: # noqa: SLF001 + assert self.on_message is not None + message = mqtt.MQTTMessage(topic=b"lixel/application/report/device_status") + message.payload = b"fresh-status" + message.qos = 0 + self.on_message(self, None, message) + return mqtt.MQTT_ERR_SUCCESS + if self._step == 5: # noqa: SLF001 + assert self.on_message is not None + message = mqtt.MQTTMessage(topic=b"lixel/application/report/lio_pose") + message.payload = b"fresh-pose" + message.qos = 0 + self.on_message(self, None, message) + return mqtt.MQTT_ERR_SUCCESS + return mqtt.MQTT_ERR_CONN_LOST + + +class SpatialSequenceClient(FakeClient): + def __init__(self, messages: list[tuple[str, bytes, bool]]) -> None: + super().__init__() + self._messages = messages + + def loop(self, timeout: float) -> mqtt.MQTTErrorCode: + if self._step < 2: # noqa: SLF001 - deterministic MQTT test double + return super().loop(timeout) + assert timeout > 0 + self._step += 1 # noqa: SLF001 + message_index = self._step - 3 # noqa: SLF001 + if message_index < len(self._messages): + assert self.on_message is not None + topic, payload, retain = self._messages[message_index] + message = mqtt.MQTTMessage(topic=topic.encode()) + message.payload = payload + message.qos = 0 + message.retain = retain + self.on_message(self, None, message) + return mqtt.MQTT_ERR_SUCCESS + raise KeyboardInterrupt + + +class LateOldClientCallbackRecoveryClient(SpatialSequenceClient): + """Replay one late callback from the retired client before fresh data.""" + + def __init__(self, old_client: FakeClient) -> None: + super().__init__([("RealtimePointcloud", b"fresh-new-client-pcl", False)]) + self._old_client = old_client + + def loop(self, timeout: float) -> mqtt.MQTTErrorCode: + if self._step < 2: # noqa: SLF001 - deterministic MQTT test double + return super().loop(timeout) + if self._step == 2: # noqa: SLF001 + assert self._old_client.on_message is not None + late = mqtt.MQTTMessage(topic=b"RealtimePointcloud") + late.payload = b"late-old-client-pcl" + late.qos = 0 + self._old_client.on_message(self._old_client, None, late) + return super().loop(timeout) + + @pytest.mark.parametrize("address", ["10.0.0.1", "172.16.0.1", "172.31.255.254", "192.168.4.2"]) def test_validate_private_ipv4_accepts_only_rfc1918(address: str) -> None: assert validate_private_ipv4(address) == address @@ -96,6 +189,25 @@ def test_validate_private_ipv4_rejects_other_targets(address: str) -> None: validate_private_ipv4(address) +@pytest.mark.parametrize( + ("attempt", "expected"), + [ + (1, 0.5), + (2, 1.0), + (3, 2.0), + (4, 4.0), + (5, 5.0), + (1025, 5.0), + (10**100, 5.0), + ], +) +def test_recovery_backoff_saturates_without_unbounded_exponentiation( + attempt: int, + expected: float, +) -> None: + assert capture_module._recovery_backoff_seconds(attempt) == expected # noqa: SLF001 + + def test_capture_writes_verifiable_frames_metadata_and_summary(tmp_path: Path) -> None: fake = FakeClient() observed = [] @@ -176,6 +288,7 @@ def test_capture_writes_verifiable_frames_metadata_and_summary(tmp_path: Path) - for artifact_name in ( "mqtt.raw.k1mqtt", "mqtt.metadata.jsonl", + "mqtt.recovery.jsonl", "mqtt.timeline.origin.json", "mqtt.timeline.json", "mqtt.summary.json", @@ -264,6 +377,360 @@ def test_capture_can_be_stopped_by_owner_without_losing_artifacts(tmp_path: Path assert list(iter_capture_frames(tmp_path / "capture" / "mqtt.raw.k1mqtt"))[0].payload +def test_guarded_recovery_keeps_one_evidence_writer_and_never_publishes( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + second = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery") + clients = iter((first, second)) + losses: list[str] = [] + attempts: list[int] = [] + candidates: list[tuple[int, int]] = [] + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "recovering-capture", + duration_seconds=30, + on_connection_lost=losses.append, + recover_connection=lambda attempt: attempts.append(attempt) or "resume", + on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append( + (attempt, sequence) + ), + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + frames = list(iter_capture_frames(tmp_path / "recovering-capture" / "mqtt.raw.k1mqtt")) + assert [frame.payload for frame in frames] == [b"before-loss", b"after-recovery"] + assert losses and "network loop failed" in losses[0] + assert attempts == [1] + assert candidates == [(1, 2)] + assert summary["reconnect_enabled"] is True + assert summary["recovery_attempts"] == 1 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + assert summary["recovery_blocked"] is False + assert summary["publishing_enabled"] is False + assert first.subscribe_calls and second.subscribe_calls + + +def test_recovery_success_requires_one_exact_post_publish_confirmation( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + second = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery") + clients = iter((first, second)) + confirmers: list[Callable[[int], bool]] = [] + confirmation_results: list[bool] = [] + + def on_message_recorded(message: object) -> None: + if getattr(message, "payload", None) != b"after-recovery": + return + confirmer = confirmers[0] + confirmation_results.extend( + ( + confirmer(999), # stale/future attempt + confirmer(1), # exact post-publication edge + confirmer(1), # duplicate edge + ) + ) + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "confirmed-recovery", + duration_seconds=30, + on_message_recorded=on_message_recorded, + recover_connection=lambda _attempt: "resume", + on_recovery_confirmer_ready=confirmers.append, + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + assert confirmation_results == [False, True, False] + assert summary["recovery_attempts"] == 1 + assert summary["recovery_point_cloud_candidates"] == 1 + assert summary["successful_recoveries"] == 1 + assert len(summary["recovery_gaps"]) == 1 + gap = summary["recovery_gaps"][0] + assert gap["gap_index"] == 1 + assert gap["recovery_attempt"] == 1 + assert gap["outcome"] == "recovered" + assert gap["ended_monotonic_ns"] >= gap["started_monotonic_ns"] + assert gap["duration_seconds"] >= 0 + + journal_path = tmp_path / "confirmed-recovery" / "mqtt.recovery.jsonl" + journal = [json.loads(line) for line in journal_path.read_text().splitlines()] + assert [record["record_type"] for record in journal] == [ + "recovery_gap_started", + "recovery_gap_ended", + ] + assert journal[1]["gap_index"] == gap["gap_index"] + assert journal[1]["recovery_attempt"] == gap["recovery_attempt"] + assert journal[1]["outcome"] == gap["outcome"] + assert summary["artifact_hashes"]["recovery_gaps_jsonl_sha256"] == hashlib.sha256( + journal_path.read_bytes() + ).hexdigest() + + +@pytest.mark.parametrize("wake_step", [3, 4]) +def test_owner_wake_enters_same_single_recovery_loop_before_or_with_paho_loss( + tmp_path: Path, + wake_step: int, +) -> None: + first: FakeClient = ( + FakeClient(payload=b"before-owner-wake") + if wake_step == 3 + else ConnectionLostAfterMessageClient(payload=b"before-owner-wake") + ) + second = FakeClient(topic="RealtimePointcloud", payload=b"after-owner-wake") + clients = iter((first, second)) + wake_consumed = False + losses: list[str] = [] + attempts: list[int] = [] + + def consume_owner_wake() -> str | None: + nonlocal wake_consumed + if not wake_consumed and first._step >= wake_step: # noqa: SLF001 + wake_consumed = True + return "camera-source-ended" + return None + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / f"owner-wake-{wake_step}", + duration_seconds=30, + on_connection_lost=losses.append, + consume_connection_recovery_request=consume_owner_wake, + recover_connection=lambda attempt: attempts.append(attempt) or "resume", + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + frames = list( + iter_capture_frames(tmp_path / f"owner-wake-{wake_step}" / "mqtt.raw.k1mqtt") + ) + assert [frame.payload for frame in frames] == [ + b"before-owner-wake", + b"after-owner-wake", + ] + assert wake_consumed is True + assert len(losses) == 1 + assert attempts == [1] + assert first.disconnect_count == 1 + assert second.disconnect_count == 1 + assert summary["recovery_attempts"] == 1 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + assert summary["message_count"] == 2 + + +def test_guarded_recovery_retries_transient_subscription_handshake_failure( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + transient = SubscriptionUnavailableClient() + recovered_client = FakeClient(topic="RealtimePointcloud", payload=b"after-recovery") + clients = iter((first, transient, recovered_client)) + attempts: list[int] = [] + candidates: list[tuple[int, int]] = [] + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "recovering-subscription", + duration_seconds=30, + recover_connection=lambda attempt: attempts.append(attempt) or "resume", + on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append( + (attempt, sequence) + ), + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + frames = list( + iter_capture_frames(tmp_path / "recovering-subscription" / "mqtt.raw.k1mqtt") + ) + assert [frame.payload for frame in frames] == [b"before-loss", b"after-recovery"] + assert attempts == [1, 2] + assert candidates == [(2, 2)] + assert summary["recovery_attempts"] == 2 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + + +def test_guarded_recovery_suback_and_pose_without_point_cloud_stay_reconnecting( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + subscribed_only = SubscribedWithoutPointCloudClient() + recovered_client = SpatialSequenceClient( + [("lixel/application/report/lio_pcl", b"fresh-pcl", False)] + ) + clients = iter((first, subscribed_only, recovered_client)) + attempts: list[int] = [] + candidates: list[tuple[int, int]] = [] + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "suback-without-spatial-data", + duration_seconds=30, + recover_connection=lambda attempt: attempts.append(attempt) or "resume", + on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append( + (attempt, sequence) + ), + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + assert attempts == [1, 2] + assert candidates == [(2, 5)] + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + assert summary["message_count"] == 5 + + +def test_guarded_recovery_arms_candidate_before_enqueueing_first_fresh_point_cloud( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + recovered_client = SpatialSequenceClient( + [ + ("lixel/application/report/heartbeat", b"heartbeat", False), + ("lixel/application/report/lio_pcl", b"retained-pcl", True), + ("DeviceStatus", b"status", False), + ("RealtimePath", b"fresh-pose", False), + ("RealtimePointcloud", b"fresh-pcl", False), + ("RealtimePath", b"later-pose", False), + ] + ) + clients = iter((first, recovered_client)) + candidates: list[tuple[int, int]] = [] + events: list[str] = [] + + def record_candidate(attempt: int, sequence: int) -> None: + candidates.append((attempt, sequence)) + events.append(f"candidate:{attempt}:{sequence}") + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "fresh-point-cloud-recovery", + duration_seconds=30, + on_message_recorded=lambda message: events.append( + f"message:{message.payload.decode()}" + ), + recover_connection=lambda _attempt: "resume", + on_recovery_point_cloud_candidate=record_candidate, + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + assert candidates == [(1, 6)] + assert events.index("message:fresh-pose") < events.index("message:fresh-pcl") + assert events.index("candidate:1:6") < events.index("message:fresh-pcl") + assert events.count("candidate:1:6") == 1 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + assert summary["message_count"] == 7 + + +def test_guarded_recovery_ignores_late_spatial_callback_from_retired_client( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + recovered_client = LateOldClientCallbackRecoveryClient(first) + clients = iter((first, recovered_client)) + recorded_payloads: list[bytes] = [] + candidates: list[tuple[int, int]] = [] + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "late-old-client-callback", + duration_seconds=30, + on_message_recorded=lambda message: recorded_payloads.append(message.payload), + recover_connection=lambda _attempt: "resume", + on_recovery_point_cloud_candidate=lambda attempt, sequence: candidates.append( + (attempt, sequence) + ), + _client_factory=lambda: cast(mqtt.Client, next(clients)), + ) + + assert candidates == [(1, 2)] + assert recorded_payloads == [b"before-loss", b"fresh-new-client-pcl"] + assert summary["message_count"] == 2 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_point_cloud_candidates"] == 1 + + +def test_failed_resubscribe_consumes_resume_before_later_standby( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient(payload=b"before-loss") + transient = SubscriptionUnavailableClient() + clients = iter((first, transient)) + factory_calls = 0 + + def factory() -> mqtt.Client: + nonlocal factory_calls + factory_calls += 1 + return cast(mqtt.Client, next(clients)) + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "recovery-standby-after-resubscribe-failure", + duration_seconds=30, + recover_connection=lambda attempt: "resume" if attempt == 1 else "standby", + _client_factory=factory, + ) + + assert factory_calls == 2 + assert summary["stop_reason"] == "recovery_standby" + assert summary["recovery_attempts"] == 2 + assert summary["successful_recoveries"] == 0 + + +def test_guarded_recovery_can_stay_blocked_until_local_owner_finishes( + tmp_path: Path, +) -> None: + first = ConnectionLostAfterMessageClient() + factory_calls = 0 + stop_checks = 0 + + def factory() -> mqtt.Client: + nonlocal factory_calls + factory_calls += 1 + return cast(mqtt.Client, first) + + def should_stop() -> bool: + nonlocal stop_checks + stop_checks += 1 + return stop_checks >= 6 + + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "blocked-capture", + duration_seconds=30, + should_stop=should_stop, + recover_connection=lambda _attempt: "blocked", + _client_factory=factory, + ) + + assert factory_calls == 1 + assert summary["stop_reason"] == "external_stop" + assert summary["recovery_attempts"] == 1 + assert summary["successful_recoveries"] == 0 + assert summary["recovery_blocked"] is True + + +def test_guarded_recovery_standby_is_truthful_non_error_completion( + tmp_path: Path, +) -> None: + summary = capture_mqtt( + "192.168.1.50", + tmp_path / "standby-capture", + duration_seconds=30, + recover_connection=lambda _attempt: "standby", + _client_factory=lambda: cast(mqtt.Client, ConnectionLostAfterMessageClient()), + ) + + assert summary["stop_reason"] == "recovery_standby" + assert summary["message_count"] == 1 + assert summary["recovery_attempts"] == 1 + assert summary["successful_recoveries"] == 0 + + def test_owner_seals_session_clock_after_all_producers_stop(tmp_path: Path) -> None: capture_dir = tmp_path / "capture" summary = capture_mqtt( diff --git a/tests/test_plugin_catalog.py b/tests/test_plugin_catalog.py index 35e0dfd..014515c 100644 --- a/tests/test_plugin_catalog.py +++ b/tests/test_plugin_catalog.py @@ -88,7 +88,7 @@ def test_repository_catalog_exposes_xgrids_model() -> None: item for item in plugins if item["metadata"]["id"] == "nodedc.device.xgrids-lixelkity-k1" ) assert plugin["apiVersion"] == "missioncore.nodedc/v1alpha2" - assert plugin["metadata"]["version"] == "0.6.0" + assert plugin["metadata"]["version"] == "0.7.5" assert plugin["spec"]["hostApiRange"] == "v1alpha2" assert plugin["spec"]["compatibilityProfiles"] == [ { @@ -125,7 +125,7 @@ def test_repository_catalog_exposes_xgrids_model() -> None: } <= action_ids assert next(item for item in models if item["id"] == "xgrids.lixelkity-k1") == { "pluginId": "nodedc.device.xgrids-lixelkity-k1", - "pluginVersion": "0.6.0", + "pluginVersion": "0.7.5", "id": "xgrids.lixelkity-k1", "vendor": "XGRIDS", "displayName": "XGRIDS LixelKity K1", diff --git a/tests/test_plugin_runtime.py b/tests/test_plugin_runtime.py index fc317d9..98fc54b 100644 --- a/tests/test_plugin_runtime.py +++ b/tests/test_plugin_runtime.py @@ -20,19 +20,49 @@ from pydantic import ValidationError import k1link.web.device_plugin_composition as plugin_composition from k1link.device_plugins.xgrids_k1.facade import ( + ACTION_ACQUISITION_ABORT, + ACTION_ACQUISITION_PREPARE, + ACTION_ACQUISITION_START, + ACTION_ACQUISITION_STOP, + ACTION_APPLICATION_CONTROL_SESSION_CLOSE, + ACTION_APPLICATION_CONTROL_SESSION_OPEN, + ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER, + ACTION_CONFIGURED_ENDPOINT_PROBE, + ACTION_CONNECTION_MODE_SELECT, + ACTION_CONNECTION_RECONFIGURE_PREPARE, ACTION_CONNECTION_VERIFY, ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ, ACTION_DISCOVERY_SCAN, ACTION_NETWORK_PROVISION, + ACTION_PHYSICAL_COMMAND_RECONCILE, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, ACTION_STREAM_START_LIVE, ACTION_STREAM_START_REPLAY, ACTION_STREAM_STOP, ACTION_VIEWER_SETTINGS_UPDATE, XGRIDS_K1_PLUGIN_ID, XGRIDS_K1_PLUGIN_VERSION, + AbortAcquisitionRequest, + BleScanRequest, + CloseApplicationControlSessionRequest, CompatibilityAttestationRequest, + ConfiguredEndpointProbeRequest, + ConnectionVerificationError, ConnectionVerifyRequest, ConnectRequest, + DesiredConnectionModeRequest, + EnterApplicationWorkspaceRequest, + NetworkProvisioningConflict, + OpenApplicationControlSessionRequest, + PrepareAcquisitionRequest, + PrepareConnectionReconfigurationRequest, + ReconcilePhysicalCommandRequest, + ReopenRetiredPhysicalCommandReconciliationRequest, + RetireUnavailablePhysicalCommandRequest, + SnapshotRuntimeConflict, + StartAcquisitionRequest, + StopAcquisitionRequest, ViewerSettingsRequest, XgridsK1PluginFacade, ) @@ -57,6 +87,21 @@ from k1link.web.plugin_runtime import ( class FakeXgridsService: def __init__(self) -> None: self.calls: list[tuple[str, object]] = [] + self.scan_loop: asyncio.AbstractEventLoop | None = None + self.verify_loop: asyncio.AbstractEventLoop | None = None + self.snapshot_runtime_id = "snapshot-runtime-test" + self.bind_calls = 0 + + def require_snapshot_runtime_id(self, expected_snapshot_runtime_id: str) -> None: + if expected_snapshot_runtime_id != self.snapshot_runtime_id: + raise SnapshotRuntimeConflict() + + def bind_runtime_event_loop( + self, + loop: asyncio.AbstractEventLoop | None = None, + ) -> None: + del loop + self.bind_calls += 1 def state(self) -> dict[str, Any]: self.calls.append(("state", None)) @@ -66,21 +111,60 @@ class FakeXgridsService: self.calls.append(("calibration", None)) return {"status": "available", "snapshot_id": "fixture-snapshot"} - async def scan_ble(self, duration_seconds: float) -> dict[str, Any]: - self.calls.append(("scan", duration_seconds)) + async def scan_ble(self, request: BleScanRequest) -> dict[str, Any]: + self.scan_loop = asyncio.get_running_loop() + self.calls.append(("scan", request)) return {"phase": "idle", "devices": []} async def connect(self, request: ConnectRequest) -> dict[str, Any]: self.calls.append(("connect", request)) return {"phase": "connected", "k1_ip": "192.168.1.20"} - def verify_connection( + def select_connection_mode( + self, + request: DesiredConnectionModeRequest, + ) -> dict[str, Any]: + self.calls.append(("mode-select", request)) + return { + "phase": "idle", + "desired_connection_mode": request.connection_mode, + "desired_connection_mode_revision": request.expected_revision + 1, + } + + async def prepare_connection_reconfiguration( + self, + request: PrepareConnectionReconfigurationRequest, + ) -> dict[str, Any]: + self.calls.append(("reconfigure", request)) + return { + "phase": "idle", + "connection_reconfiguration": { + "revision": request.expected_reconfiguration_revision + 1, + "intent": request.intent, + }, + } + + async def verify_connection( self, request: ConnectionVerifyRequest | None = None, ) -> dict[str, Any]: + self.verify_loop = asyncio.get_running_loop() self.calls.append(("verify", request)) return {"phase": "connected", "k1_ip": "192.168.1.20"} + async def probe_configured_endpoint( + self, + request: ConfiguredEndpointProbeRequest | None = None, + ) -> dict[str, Any]: + self.calls.append(("endpoint-probe", request)) + return { + "phase": "idle", + "configured_endpoint_probe": { + "status": "reachable", + "ble_operation_performed": False, + }, + } + def start_live( self, project_name: str, @@ -101,10 +185,87 @@ class FakeXgridsService: self.calls.append(("stop", None)) return {"phase": "idle"} + def prepare_acquisition(self, request: PrepareAcquisitionRequest) -> dict[str, Any]: + self.calls.append(("prepare", request)) + return {"phase": "connected", "acquisition": {"state": "prepared"}} + + def start_acquisition(self, request: StartAcquisitionRequest) -> dict[str, Any]: + self.calls.append(("start", request)) + return {"phase": "starting_live"} + + def stop_acquisition(self, request: StopAcquisitionRequest) -> dict[str, Any]: + self.calls.append(("acquisition-stop", request)) + return {"phase": "stopping"} + + def abort_acquisition(self, request: AbortAcquisitionRequest) -> dict[str, Any]: + self.calls.append(("abort", request)) + return {"phase": "idle"} + + def open_application_control_session( + self, + request: OpenApplicationControlSessionRequest, + ) -> dict[str, Any]: + self.calls.append(("control-open", request)) + return {"phase": "connected"} + + def enter_application_workspace( + self, + request: EnterApplicationWorkspaceRequest, + ) -> dict[str, Any]: + self.calls.append(("control-enter", request)) + return {"phase": "connected"} + + def close_application_control_session( + self, + request: CloseApplicationControlSessionRequest, + ) -> dict[str, Any]: + self.calls.append(("control-close", request)) + return {"phase": "connected"} + + def reconcile_physical_command( + self, + request: ReconcilePhysicalCommandRequest, + ) -> dict[str, Any]: + self.calls.append(("physical-reconcile", request)) + return {"phase": "connected"} + def update_viewer_settings(self, request: ViewerSettingsRequest) -> dict[str, Any]: self.calls.append(("viewer", request)) return {"phase": "idle", "viewer_settings": request.model_dump()} + def retire_unavailable_physical_command( + self, + request: RetireUnavailablePhysicalCommandRequest, + ) -> dict[str, Any]: + self.calls.append(("physical-retire", request)) + return { + "phase": "idle", + "physical_command": { + "status": "resolved", + "physical_outcome": "unknown", + }, + } + + def reopen_retired_physical_command_reconciliation( + self, + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> dict[str, Any]: + self.calls.append(("physical-reopen", request)) + return { + "phase": "idle", + "physical_command": { + "status": "unresolved", + "requires_reconciliation": True, + }, + } + + +def _snapshot_fenced(payload: dict[str, Any] | None = None) -> dict[str, Any]: + return { + **(payload or {}), + "expected_snapshot_runtime_id": "snapshot-runtime-test", + } + def _in_process_runtime( adapter: Any, @@ -151,9 +312,7 @@ def test_manifest_and_runtime_facade_declare_identical_actions() -> None: def test_calibration_snapshot_action_calls_the_read_only_service_method() -> None: service = FakeXgridsService() - dispatcher = DevicePluginDispatcher( - [_in_process_runtime(XgridsK1PluginFacade(service))] - ) + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) result = asyncio.run( dispatcher.invoke( @@ -169,22 +328,21 @@ def test_calibration_snapshot_action_calls_the_read_only_service_method() -> Non def test_connection_verify_action_accepts_read_only_adoption_request() -> None: service = FakeXgridsService() - dispatcher = DevicePluginDispatcher( - [_in_process_runtime(XgridsK1PluginFacade(service))] - ) + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) result = asyncio.run( dispatcher.invoke( XGRIDS_K1_PLUGIN_ID, ACTION_CONNECTION_VERIFY, - { + _snapshot_fenced({ "device_id": "test-ble-transport", "compatibility_attestation": { "firmware_version": "3.0.2", "topology": "direct-lan", "verification": "live-device-info", }, - }, + "expected_discovery_generation": 0, + }), ) ) @@ -200,15 +358,13 @@ def test_connection_verify_action_accepts_read_only_adoption_request() -> None: def test_connection_verify_action_keeps_empty_refresh_request_compatible() -> None: service = FakeXgridsService() - dispatcher = DevicePluginDispatcher( - [_in_process_runtime(XgridsK1PluginFacade(service))] - ) + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) asyncio.run( dispatcher.invoke( XGRIDS_K1_PLUGIN_ID, ACTION_CONNECTION_VERIFY, - {}, + _snapshot_fenced(), ) ) @@ -219,6 +375,548 @@ def test_connection_verify_action_keeps_empty_refresh_request_compatible() -> No assert request.compatibility_attestation is None +def test_connection_mode_select_delegates_exact_cas_payload() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONNECTION_MODE_SELECT, + _snapshot_fenced({ + "connection_mode": "quick-connect", + "expected_revision": 7, + }), + ) + ) + + assert result["desired_connection_mode"] == "quick-connect" + assert result["desired_connection_mode_revision"] == 8 + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "mode-select" + assert isinstance(request, DesiredConnectionModeRequest) + assert request.connection_mode == "quick-connect" + assert request.expected_revision == 7 + + +def test_connection_scenario_reset_dispatches_before_runtime_loop_binding() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONNECTION_MODE_SELECT, + _snapshot_fenced( + { + "connection_mode": "bridge", + "expected_revision": 3, + "reset_scenario": True, + "reset_id": "op-reset-dispatch-local-only-01", + } + ), + ) + ) + + assert result["desired_connection_mode_revision"] == 4 + assert service.bind_calls == 0 + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "mode-select" + assert isinstance(request, DesiredConnectionModeRequest) + assert request.reset_scenario is True + assert request.reset_id == "op-reset-dispatch-local-only-01" + + +@pytest.mark.parametrize( + ("intent", "intent_id"), + [("select-device", None), ("change-network", None), ("cancel", "intent-9")], +) +def test_connection_reconfigure_action_delegates_exact_stable_cas_payload( + intent: str, + intent_id: str | None, +) -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + payload = { + "intent": intent, + "expected_reconfiguration_revision": 9, + "expected_reconfiguration_intent_id": intent_id, + "expected_desired_mode_revision": 4, + "expected_active_binding_key": "a" * 64 if intent_id is None else None, + } + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONNECTION_RECONFIGURE_PREPARE, + _snapshot_fenced(payload), + ) + ) + + assert result["connection_reconfiguration"] == { + "revision": 10, + "intent": intent, + } + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "reconfigure" + assert isinstance(request, PrepareConnectionReconfigurationRequest) + assert request.model_dump(mode="json") == payload + + +def test_configured_endpoint_probe_action_is_separate_from_ble_verify() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONFIGURED_ENDPOINT_PROBE, + _snapshot_fenced( + {"operation_id": "op-00000000-0000-4000-8000-000000000652"} + ), + ) + ) + + assert result["configured_endpoint_probe"] == { + "status": "reachable", + "ble_operation_performed": False, + } + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "endpoint-probe" + assert isinstance(request, ConfiguredEndpointProbeRequest) + assert request.operation_id == "op-00000000-0000-4000-8000-000000000652" + + +def test_physical_retirement_action_delegates_exact_confirmed_cas_payload() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + payload = { + "retirement_id": "retirement-browser-stable-id", + "expected_operation_id": "physical-stop-persisted", + "expected_revision": 17, + "expected_transport_ref": "F89438FA-55ED-85AD-EED7-734AC84746D8", + "operator_confirmed": True, + "reason": "device-permanently-unavailable-or-replaced", + } + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, + _snapshot_fenced(payload), + ) + ) + + assert result["physical_command"] == { + "status": "resolved", + "physical_outcome": "unknown", + } + assert service.bind_calls == 0 + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "physical-retire" + assert isinstance(request, RetireUnavailablePhysicalCommandRequest) + assert request.model_dump(mode="json") == payload + + +def test_physical_reopen_action_is_runtime_fenced_exact_and_never_binds_loop() -> None: + service = FakeXgridsService() + adapter = XgridsK1PluginFacade(service) + dispatcher = DevicePluginDispatcher([_in_process_runtime(adapter)]) + payload = { + "reopening_id": "reopening-browser-stable-id", + "expected_revision": 18, + "expected_retirement_id": "retirement-browser-stable-id", + "expected_transport_ref": "f89438fa-55ed-85ad-eed7-734ac84746d8", + "expected_discovery_generation": 7, + "expected_desired_mode": "bridge", + "expected_desired_mode_revision": 4, + "operator_confirmed": True, + "reason": "device-returned-for-explicit-reconciliation", + } + + result = asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + _snapshot_fenced(payload), + ) + ) + + assert result["physical_command"] == { + "status": "unresolved", + "requires_reconciliation": True, + } + assert service.bind_calls == 0 + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "physical-reopen" + assert isinstance(request, ReopenRetiredPhysicalCommandReconciliationRequest) + assert request.model_dump(mode="json") == payload + assert ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION in adapter.action_ids + + +@pytest.mark.parametrize( + "reason_code", + [ + "physical-command-reconciliation-reopen-stale-checkpoint", + "network-provisioning-idempotency-operation-mismatch", + "reconciliation-target-mode-mismatch", + "reconciliation-target-physical-recovery-mismatch", + "fresh-ble-candidate-required", + "physical-command-recovery-target-not-observed", + "network-provision-operation-active", + "control-local-retirement-pending", + "device-calibration-read-active", + "acquisition-active", + "acquisition-cleanup-pending", + "acquisition-start-operation-active", + "acquisition-stop-operation-active", + "local-runtime-active", + "control-session-not-admissible-for-network-change", + "k1-lifecycle-process-lease-control-owned", + "k1-lifecycle-process-lease-active", + ], +) +def test_physical_reopen_expected_conflicts_are_http_409(reason_code: str) -> None: + class ReopenConflictService(FakeXgridsService): + def reopen_retired_physical_command_reconciliation( + self, + request: ReopenRetiredPhysicalCommandReconciliationRequest, + ) -> dict[str, Any]: + del request + raise NetworkProvisioningConflict( + "reopen checkpoint is no longer executable", + reason_code=reason_code, + ) + + dispatcher = DevicePluginDispatcher( + [_in_process_runtime(XgridsK1PluginFacade(ReopenConflictService()))] + ) + payload = { + "reopening_id": "reopening-http-conflict", + "expected_revision": 18, + "expected_retirement_id": "retirement-http-conflict", + "expected_transport_ref": "f89438fa-55ed-85ad-eed7-734ac84746d8", + "expected_discovery_generation": 7, + "expected_desired_mode": "bridge", + "expected_desired_mode_revision": 4, + "operator_confirmed": True, + "reason": "device-returned-for-explicit-reconciliation", + } + + with pytest.raises(PluginExecutionError) as raised: + asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + _snapshot_fenced(payload), + ) + ) + + assert raised.value.http_status_code == 409 + assert raised.value.reason_code == reason_code + + +@pytest.mark.parametrize( + "action_id", + [ + ACTION_DISCOVERY_SCAN, + ACTION_NETWORK_PROVISION, + ACTION_CONNECTION_MODE_SELECT, + ACTION_CONNECTION_RECONFIGURE_PREPARE, + ACTION_CONNECTION_VERIFY, + ACTION_CONFIGURED_ENDPOINT_PROBE, + ACTION_ACQUISITION_PREPARE, + ACTION_ACQUISITION_START, + ACTION_ACQUISITION_STOP, + ACTION_ACQUISITION_ABORT, + ACTION_APPLICATION_CONTROL_SESSION_OPEN, + ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER, + ACTION_APPLICATION_CONTROL_SESSION_CLOSE, + ACTION_PHYSICAL_COMMAND_RECONCILE, + ACTION_STREAM_STOP, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + ], +) +@pytest.mark.parametrize( + "runtime_fence", + [None, "snapshot-runtime-stale-browser"], +) +def test_snapshot_fenced_actions_reject_stale_browser_before_service_or_io( + action_id: str, + runtime_fence: str | None, +) -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + payload = ( + {} + if runtime_fence is None + else {"expected_snapshot_runtime_id": runtime_fence} + ) + + with pytest.raises(PluginExecutionError) as raised: + asyncio.run(dispatcher.invoke(XGRIDS_K1_PLUGIN_ID, action_id, payload)) + + assert raised.value.http_status_code == 409 + assert raised.value.reason_code == "snapshot-runtime-conflict" + assert service.bind_calls == 0 + assert service.calls == [] + + +def test_lifecycle_ui_payload_contract_passes_backend_validation_exactly() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + compatibility = { + "firmware_version": "3.0.2", + "topology": "direct-lan", + "verification": "live-device-info", + } + payloads = ( + ( + ACTION_ACQUISITION_PREPARE, + { + "operation_id": "op-00000000-0000-4000-8000-000000000201", + "idempotency_key": ( + "acquisition.prepare:op-00000000-0000-4000-8000-000000000201" + ), + "project_name": "CONTRACT01", + "mount_type": "handheld", + "gnss_mode": "none", + "compatibility_attestation": compatibility, + "expected_control_session_generation": 7, + "expected_control_state_revision": 11, + }, + ), + ( + ACTION_ACQUISITION_START, + { + "operation_id": "op-00000000-0000-4000-8000-000000000202", + "idempotency_key": ( + "acquisition.start:op-00000000-0000-4000-8000-000000000202" + ), + "acquisition_id": "acquisition-contract", + "expected_control_session_generation": 7, + "expected_control_state_revision": 12, + }, + ), + ( + ACTION_ACQUISITION_STOP, + { + "operation_id": "op-00000000-0000-4000-8000-000000000203", + "idempotency_key": ( + "acquisition.stop:op-00000000-0000-4000-8000-000000000203" + ), + "acquisition_id": "acquisition-contract", + "mode": "graceful", + "expected_control_session_generation": 7, + "expected_control_state_revision": 13, + }, + ), + ( + ACTION_ACQUISITION_ABORT, + { + "operation_id": "op-00000000-0000-4000-8000-000000000204", + "idempotency_key": ( + "acquisition.abort:op-00000000-0000-4000-8000-000000000204" + ), + "acquisition_id": "acquisition-contract", + "expected_control_session_generation": 7, + "expected_control_state_revision": 14, + }, + ), + ( + ACTION_APPLICATION_CONTROL_SESSION_OPEN, + { + "operator_present": True, + "owner_controlled_device": True, + "lixelgo_closed": True, + "battery_storage_confirmed": True, + "expected_physical_state_confirmed": True, + "timezone_name": "Europe/Moscow", + }, + ), + ( + ACTION_APPLICATION_CONTROL_WORKSPACE_ENTER, + { + "operator_confirmed": True, + "expected_session_generation": 7, + "expected_state_revision": 15, + }, + ), + ( + ACTION_APPLICATION_CONTROL_SESSION_CLOSE, + { + "expected_session_generation": 7, + "expected_state_revision": 16, + }, + ), + ( + ACTION_PHYSICAL_COMMAND_RECONCILE, + { + "reconciliation_id": "reconciliation-contract", + "expected_session_generation": 7, + "expected_state_revision": 17, + }, + ), + ) + + for action_id, payload in payloads: + asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + action_id, + _snapshot_fenced(payload), + ) + ) + + assert [call[0] for call in service.calls] == [ + "prepare", + "start", + "acquisition-stop", + "abort", + "control-open", + "control-enter", + "control-close", + "physical-reconcile", + ] + for _, request in service.calls[:4]: + assert request.operation_id is not None + assert request.idempotency_key.endswith(request.operation_id) + + +def test_connection_verify_expected_state_is_not_reported_as_bad_gateway() -> None: + class AddressUnavailableService(FakeXgridsService): + async def verify_connection( + self, + request: ConnectionVerifyRequest | None = None, + ) -> dict[str, Any]: + del request + raise ConnectionVerificationError( + "K1 не сообщил адрес общей локальной сети", + reason_code="connection-verify-address-unavailable", + ) + + dispatcher = DevicePluginDispatcher( + [_in_process_runtime(XgridsK1PluginFacade(AddressUnavailableService()))] + ) + + with pytest.raises(PluginExecutionError) as raised: + asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONNECTION_VERIFY, + _snapshot_fenced(), + ) + ) + + assert raised.value.http_status_code == 409 + assert raised.value.reason_code == "connection-verify-address-unavailable" + + +@pytest.mark.parametrize( + ("reason_code", "expected_status"), + [ + ("ble-runtime-busy", 409), + ("ble-runtime-cleanup-pending", 409), + ("provisioning-already-running", 409), + ("connection-verify-mqtt-unreachable", 409), + ("connection-verify-lease-changed", 409), + ("connection-verify-resolved-apply-target-mismatch", 409), + ("configured-endpoint-topology-corrupt", 409), + ("connection-mode-selection-lifecycle-busy", 409), + ("connection-mode-selection-physical-state-unsafe", 409), + ("connection-mode-selection-control-state-unsafe", 409), + ("connection-mode-switch-acquisition-changed", 409), + ("connection-scenario-reset-pending", 409), + ("connection-scenario-reset-lifecycle-timeout", 409), + ("acquisition-start-lifecycle-busy", 409), + ("application-control-process-lease-unavailable", 409), + ("ble-runtime-owner-loop-conflict", 503), + ("ble-runtime-restart-required", 503), + ("connection-verify-exact-uuid-scan-timeout", 504), + ("ble-discovery-timeout", 504), + ("ble-status-read-timeout", 504), + ("ble-provisioning-timeout", 504), + ("ble-ap-enable-timeout", 504), + ("network-not-found", 504), + ("host-wifi-operation-timeout", 504), + ("physical-command-reconciliation-proof-timeout", 504), + ("physical-command-reconciliation-control-adoption-timeout", 504), + ("keychain-authorization-required", 409), + ("profile-unavailable", 409), + ("profile-credential-source-mismatch", 409), + ("wifi-interface-unavailable", 503), + ], +) +def test_ble_runtime_failure_preserves_actionable_http_class( + reason_code: str, + expected_status: int, +) -> None: + class ClassifiedRuntimeError(RuntimeError): + def __init__(self) -> None: + super().__init__("classified BLE failure") + self.reason_code = reason_code + + class FailingScanService(FakeXgridsService): + async def scan_ble(self, request: BleScanRequest) -> dict[str, Any]: + del request + raise ClassifiedRuntimeError() + + dispatcher = DevicePluginDispatcher( + [_in_process_runtime(XgridsK1PluginFacade(FailingScanService()))] + ) + + with pytest.raises(PluginExecutionError) as raised: + asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_DISCOVERY_SCAN, + _snapshot_fenced({"duration_seconds": 6}), + ) + ) + + assert raised.value.http_status_code == expected_status + assert raised.value.reason_code == reason_code + + +def test_connection_verify_reuses_the_discovery_event_loop() -> None: + service = FakeXgridsService() + dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) + + async def scenario() -> asyncio.AbstractEventLoop: + loop = asyncio.get_running_loop() + await dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_DISCOVERY_SCAN, + _snapshot_fenced({"duration_seconds": 6}), + ) + await dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_CONNECTION_VERIFY, + _snapshot_fenced({ + "device_id": "test-ble-transport", + "compatibility_attestation": { + "firmware_version": "3.0.2", + "topology": "direct-lan", + "verification": "live-device-info", + }, + "expected_discovery_generation": 0, + }), + ) + return loop + + dispatcher_loop = asyncio.run(scenario()) + + assert service.scan_loop is dispatcher_loop + assert service.verify_loop is dispatcher_loop + + def test_repository_runtime_composition_exactly_matches_catalog() -> None: repository_root = Path(__file__).resolve().parents[1] environment = load_installed_device_plugins(repository_root) @@ -437,12 +1135,16 @@ def test_dispatcher_routes_allowlisted_action_to_xgrids_facade() -> None: dispatcher.invoke( XGRIDS_K1_PLUGIN_ID, ACTION_DISCOVERY_SCAN, - {"duration_seconds": 6}, + _snapshot_fenced({"duration_seconds": 6}), ) ) assert state == {"phase": "idle", "devices": []} - assert service.calls == [("scan", 6.0)] + assert len(service.calls) == 1 + action, request = service.calls[0] + assert action == "scan" + assert isinstance(request, BleScanRequest) + assert request.duration_seconds == 6.0 def test_runtime_classifies_non_json_plugin_output_as_execution_failure( @@ -496,12 +1198,12 @@ def test_facade_validates_payload_before_calling_service() -> None: dispatcher.invoke( XGRIDS_K1_PLUGIN_ID, ACTION_NETWORK_PROVISION, - { + _snapshot_fenced({ "device_id": "id", "ssid": "network", "password": "x" * 24, "extra": True, - }, + }), ) ) @@ -529,7 +1231,7 @@ def test_facade_validates_payload_before_calling_service() -> None: {"path": "sessions/capture.k1mqtt", "speed": 1, "loop": False}, "replay", ), - (ACTION_STREAM_STOP, {}, "stop"), + (ACTION_STREAM_STOP, _snapshot_fenced(), "stop"), ( ACTION_VIEWER_SETTINGS_UPDATE, { @@ -571,7 +1273,13 @@ def test_sync_runtime_actions_run_outside_the_api_event_loop() -> None: dispatcher = DevicePluginDispatcher([_in_process_runtime(XgridsK1PluginFacade(service))]) event_loop_thread = threading.get_ident() - asyncio.run(dispatcher.invoke(XGRIDS_K1_PLUGIN_ID, ACTION_STREAM_STOP, {})) + asyncio.run( + dispatcher.invoke( + XGRIDS_K1_PLUGIN_ID, + ACTION_STREAM_STOP, + _snapshot_fenced(), + ) + ) assert service.thread_id is not None assert service.thread_id != event_loop_thread @@ -649,6 +1357,6 @@ def test_dispatcher_rejects_uncorrelated_transport_result() -> None: DevicePluginDispatcher([runtime]).invoke( XGRIDS_K1_PLUGIN_ID, ACTION_STREAM_STOP, - {}, + _snapshot_fenced(), ) ) diff --git a/tests/test_pytest_runtime_isolation.py b/tests/test_pytest_runtime_isolation.py new file mode 100644 index 0000000..8c686b9 --- /dev/null +++ b/tests/test_pytest_runtime_isolation.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def test_lazy_app_import_in_child_keeps_mutable_k1_state_out_of_repository() -> None: + repository_root = Path(__file__).resolve().parents[1] + data_root = Path(os.environ["MISSIONCORE_DATA_DIR"]).resolve() + evidence_root = Path(os.environ["MISSIONCORE_EVIDENCE_DIR"]).resolve() + legacy_root = Path(os.environ["MISSIONCORE_LEGACY_SESSIONS_DIR"]).resolve() + assert data_root != repository_root / ".runtime" / "mission-core" + assert evidence_root != repository_root / ".runtime" / "mission-core" / "evidence" + assert legacy_root != repository_root / "sessions" + + script = """ +import importlib +import json + +module = importlib.import_module("k1link.web.app") +contribution = module.plugin_environment._contributions[0] +runtime = contribution.runtime +service = runtime._adapter.service +print(json.dumps({ + "session_data": str(module.session_store.data_dir), + "service_evidence": str(service.evidence_root), + "shadow_token": str(service.live_perception_token_path), + "archive_roots": [str(item.root) for item in module.plugin_environment.observation_archives], +})) +module.plugin_environment.close() +""" + completed = subprocess.run( + [sys.executable, "-c", script], + cwd=repository_root, + check=True, + capture_output=True, + text=True, + timeout=30, + ) + document = json.loads(completed.stdout.strip().splitlines()[-1]) + + assert Path(document["session_data"]).resolve() == data_root + assert Path(document["service_evidence"]).resolve() == evidence_root + assert Path(document["shadow_token"]).resolve().is_relative_to(data_root) + assert {Path(item).resolve() for item in document["archive_roots"]} == { + legacy_root, + evidence_root, + } diff --git a/tests/test_rerun_bridge.py b/tests/test_rerun_bridge.py index 41946f7..8f92aef 100644 --- a/tests/test_rerun_bridge.py +++ b/tests/test_rerun_bridge.py @@ -1,9 +1,11 @@ from __future__ import annotations +import errno import json import logging import socket import struct +import sys import threading import time from pathlib import Path @@ -25,6 +27,8 @@ from k1link.viewer.rerun_bridge import ( _select_available_grpc_port, ) +rerun_bridge_module = sys.modules["k1link.viewer.rerun_bridge"] + class FakeRecording: def __init__(self) -> None: @@ -65,11 +69,73 @@ class DisconnectFailureRecording(FakeRecording): raise RuntimeError("synthetic disconnect failure") +@pytest.fixture +def socket_free_rerun_port_selector(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep FakeRecording tests independent of host TCP bind permission.""" + + monkeypatch.setattr( + rerun_bridge_module, + "_select_available_grpc_port", + lambda preferred_port, **_kwargs: preferred_port, + ) + + +def test_runtime_owner_recovery_wake_is_generation_fenced_and_coalesced() -> None: + runtime = VisualizationRuntime(normalizer=normalize_k1_message) + with runtime._lock: # noqa: SLF001 - bounded producer-state unit seam + runtime._producer_generation = 7 # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._source_ready = True # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + + assert ( + runtime.request_connection_recovery( + "camera-source-ended", + expected_generation=6, + ) + is False + ) + assert runtime.request_connection_recovery( + "camera-source-ended", + expected_generation=7, + ) + assert runtime.request_connection_recovery( + "mqtt_network_loop_failed", + expected_generation=7, + ) + snapshot = runtime.snapshot() + assert snapshot["phase"] == "reconnecting" + assert snapshot["connection_recovery"]["reason_code"] == "camera-source-ended" + assert runtime._consume_connection_recovery_request(generation=6) is None # noqa: SLF001 + assert ( # noqa: SLF001 + runtime._consume_connection_recovery_request(generation=7) + == "camera-source-ended" + ) + assert runtime._consume_connection_recovery_request(generation=7) is None # noqa: SLF001 + + with runtime._lock: # noqa: SLF001 + runtime._phase = "live" # noqa: SLF001 + runtime._source_ready = True # noqa: SLF001 + assert runtime.request_connection_recovery( + "host-route-unavailable", + expected_generation=7, + ) + runtime.stop() + assert runtime.snapshot()["phase"] == "idle" + assert runtime._consume_connection_recovery_request(generation=7) is None # noqa: SLF001 + + def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer( caplog: pytest.LogCaptureFixture, ) -> None: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as occupied: - occupied.bind(("0.0.0.0", 0)) + try: + occupied.bind(("0.0.0.0", 0)) + except OSError as exc: + if exc.errno in {errno.EACCES, errno.EPERM}: + pytest.skip(f"host sandbox denies TCP bind: errno={exc.errno}") + raise occupied.listen() preferred_port = int(occupied.getsockname()[1]) @@ -92,6 +158,82 @@ def test_rerun_port_selection_skips_a_recording_still_held_by_a_viewer( bridge.close() +def test_rerun_port_selection_retries_only_address_in_use( + monkeypatch: pytest.MonkeyPatch, +) -> None: + attempts: list[int] = [] + + class Probe: + def __enter__(self) -> Probe: + return self + + def __exit__(self, *_args: object) -> None: + return + + def bind(self, address: tuple[str, int]) -> None: + attempts.append(address[1]) + if len(attempts) < 3: + raise OSError(errno.EADDRINUSE, "synthetic address in use") + + monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe()) + + assert _select_available_grpc_port(9876, search_span=4) == 9878 + assert attempts == [9876, 9877, 9878] + + +@pytest.mark.parametrize("error_number", [errno.EPERM, errno.EACCES]) +def test_rerun_port_selection_reports_permission_denial_immediately( + monkeypatch: pytest.MonkeyPatch, + error_number: int, +) -> None: + attempts: list[int] = [] + + class Probe: + def __enter__(self) -> Probe: + return self + + def __exit__(self, *_args: object) -> None: + return + + def bind(self, address: tuple[str, int]) -> None: + attempts.append(address[1]) + raise OSError(error_number, "synthetic permission denial") + + monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe()) + + with pytest.raises(PermissionError, match="Permission denied.*9876") as error: + _select_available_grpc_port(9876, search_span=4) + + assert error.value.errno == error_number + assert attempts == [9876] + + +@pytest.mark.parametrize("error_number", [errno.EADDRNOTAVAIL, errno.EIO]) +def test_rerun_port_selection_does_not_misclassify_unexpected_socket_errors( + monkeypatch: pytest.MonkeyPatch, + error_number: int, +) -> None: + attempts: list[int] = [] + + class Probe: + def __enter__(self) -> Probe: + return self + + def __exit__(self, *_args: object) -> None: + return + + def bind(self, address: tuple[str, int]) -> None: + attempts.append(address[1]) + raise OSError(error_number, "synthetic unexpected bind failure") + + monkeypatch.setattr(rerun_bridge_module.socket, "socket", lambda *_args: Probe()) + + with pytest.raises(RuntimeError, match="Could not probe.*9876"): + _select_available_grpc_port(9876, search_span=4) + + assert attempts == [9876] + + def _message( topic: str, payload: bytes, @@ -129,7 +271,9 @@ def _envelope( return envelope -def test_legacy_points_and_pose_are_logged_to_rerun() -> None: +def test_legacy_points_and_pose_are_logged_to_rerun( + socket_free_rerun_port_selector: None, +) -> None: recording = FakeRecording() bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type] @@ -169,7 +313,9 @@ def test_live_blueprint_follows_stream_time_without_frontend_cursor_writes() -> assert panel.state == "hidden" -def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None: +def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid( + socket_free_rerun_port_selector: None, +) -> None: recording = FakeRecording() bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type] mask = np.zeros((600, 800), dtype=np.uint8) @@ -177,6 +323,8 @@ def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None: bridge.process_perception( LivePerceptionResultFrame( + session_id="test-live-perception-session", + session_generation=1, frame_index=3, source_frame_index=30, session_seconds=1.0, @@ -206,7 +354,9 @@ def test_live_perception_logs_original_mask_2d_distance_and_3d_cuboid() -> None: assert "/world/perception/boxes3d" in paths -def test_constructor_disconnects_recording_after_partial_setup_failure() -> None: +def test_constructor_disconnects_recording_after_partial_setup_failure( + socket_free_rerun_port_selector: None, +) -> None: recording = BlueprintFailureRecording() with pytest.raises(RuntimeError, match="synthetic blueprint failure"): @@ -215,7 +365,9 @@ def test_constructor_disconnects_recording_after_partial_setup_failure() -> None assert recording.disconnected is True -def test_fast_replay_trajectory_sampling_uses_source_time() -> None: +def test_fast_replay_trajectory_sampling_uses_source_time( + socket_free_rerun_port_selector: None, +) -> None: recording = FakeRecording() bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type] base_time_ns = 1_784_124_315_000_000_000 @@ -245,7 +397,9 @@ def test_fast_replay_trajectory_sampling_uses_source_time() -> None: bridge.close() -def test_bad_frame_is_rejected_before_rerun_without_publishing() -> None: +def test_bad_frame_is_rejected_before_rerun_without_publishing( + socket_free_rerun_port_selector: None, +) -> None: recording = FakeRecording() bridge = RerunBridge(recording_factory=lambda _: recording) # type: ignore[arg-type] @@ -296,7 +450,10 @@ def test_palettes_are_deterministic_and_custom_color_is_exact() -> None: assert custom_over_rgb.tolist() == [[16, 32, 48], [16, 32, 48]] -def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) -> None: +def test_runtime_owns_fresh_bridge_for_each_sequential_session( + tmp_path: Path, + socket_free_rerun_port_selector: None, +) -> None: capture = tmp_path / "mqtt.raw.k1mqtt" point_topic = "RealtimePointcloud" pose_topic = "RealtimePath" @@ -380,7 +537,10 @@ def test_runtime_owns_fresh_bridge_for_each_sequential_session(tmp_path: Path) - assert runtime.snapshot()["rerun_grpc_url"] is None -def test_runtime_reports_bridge_close_failure_instead_of_false_idle(tmp_path: Path) -> None: +def test_runtime_reports_bridge_close_failure_instead_of_false_idle( + tmp_path: Path, + socket_free_rerun_port_selector: None, +) -> None: capture = tmp_path / "mqtt.raw.k1mqtt" topic = b"RealtimePointcloud" payload = struct.pack(" None: +def test_close_during_blocked_factory_closes_the_late_bridge( + tmp_path: Path, + socket_free_rerun_port_selector: None, +) -> None: capture = tmp_path / "mqtt.raw.k1mqtt" topic = b"RealtimePointcloud" payload = struct.pack(" runtime.start_replay(capture, speed=0.0) -def test_stop_fails_closed_when_runtime_thread_misses_deadline(tmp_path: Path) -> None: +def test_stop_fails_closed_when_runtime_thread_misses_deadline( + tmp_path: Path, + socket_free_rerun_port_selector: None, +) -> None: capture = tmp_path / "mqtt.raw.k1mqtt" topic = b"RealtimePointcloud" payload = struct.pack(" None: repository = tmp_path / "repo" + monkeypatch.delenv("MISSIONCORE_EVIDENCE_DIR", raising=False) + monkeypatch.delenv("MISSIONCORE_DATA_DIR", raising=False) assert ( resolve_missioncore_evidence_dir(repository) == (repository / ".runtime" / "mission-core" / "evidence" / "sessions").resolve() diff --git a/tests/test_viewer_diagnostics_api.py b/tests/test_viewer_diagnostics_api.py index fe25136..b381a6a 100644 --- a/tests/test_viewer_diagnostics_api.py +++ b/tests/test_viewer_diagnostics_api.py @@ -130,11 +130,16 @@ def test_private_scanner_diagnostics_are_durable_structured_and_bounded( def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events( caplog: pytest.LogCaptureFixture, ) -> None: - router = build_viewer_diagnostics_router() + expected_build = "/assets/index-abcdefgh.js" + router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build) endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST") event = LiveViewerDiagnosticEvent( - schema_version="missioncore.live-viewer-diagnostic/v1", + schema_version="missioncore.live-viewer-diagnostic/v2", event_code="live_receiver_stalled", + ui_build_id=expected_build, + document_instance_id="00000000-0000-4000-8000-000000000001", + viewer_instance_id="00000000-0000-4000-8000-000000000002", + lifecycle_generation=4, failure_stage="receiver-stalled", stream_id="acquisition-123", backend_activity_sequence=8_572, @@ -150,8 +155,12 @@ def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events( response = endpoint(event) assert response.status_code == 204 + assert response.headers["x-missioncore-ui-build"] == expected_build assert "event=live_receiver_stalled" in caplog.text assert caplog.records[-1].failure_stage == "receiver-stalled" + assert caplog.records[-1].document_instance_id == event.document_instance_id + assert caplog.records[-1].viewer_instance_id == event.viewer_instance_id + assert caplog.records[-1].lifecycle_generation == 4 with pytest.raises(ValidationError): LiveViewerDiagnosticEvent.model_validate( { @@ -160,9 +169,91 @@ def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events( } ) fallback = LiveViewerDiagnosticEvent( - schema_version="missioncore.live-viewer-diagnostic/v1", + schema_version="missioncore.live-viewer-diagnostic/v2", event_code="live_receiver_active_store_admitted", + ui_build_id=expected_build, + document_instance_id="00000000-0000-4000-8000-000000000001", + viewer_instance_id="00000000-0000-4000-8000-000000000002", + lifecycle_generation=4, stream_id="acquisition-123", backend_activity_sequence=8_573, ) assert fallback.failure_stage is None + + +def test_live_viewer_diagnostic_rejects_stale_build_before_logging( + caplog: pytest.LogCaptureFixture, +) -> None: + expected_build = "/assets/index-ijklmnop.js" + router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build) + endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST") + event = LiveViewerDiagnosticEvent( + schema_version="missioncore.live-viewer-diagnostic/v2", + event_code="live_receiver_error", + ui_build_id="/assets/index-abcdefgh.js", + document_instance_id="00000000-0000-4000-8000-000000000001", + viewer_instance_id="00000000-0000-4000-8000-000000000002", + lifecycle_generation=1, + ) + + with caplog.at_level( + logging.INFO, + logger="k1link.device_plugins.xgrids_k1.viewer_receiver", + ): + response = endpoint(event) + + assert response.status_code == 409 + assert response.headers["x-missioncore-ui-build"] == expected_build + assert "Mission Core live Rerun receiver diagnostic" not in caplog.text + + +def test_live_viewer_client_contract_is_no_store_and_exact_build() -> None: + expected_build = "/assets/index-abcdefgh.js" + router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build) + endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET") + + response = endpoint() + + assert response.status_code == 200 + assert response.headers["cache-control"] == "no-store" + assert response.headers["x-missioncore-ui-build"] == expected_build + assert json.loads(response.body) == { + "schema_version": "missioncore.live-viewer-client-contract/v1", + "status": "ready", + "ui_build_id": expected_build, + "diagnostic_schema_version": "missioncore.live-viewer-diagnostic/v2", + } + + +def test_live_viewer_client_contract_no_dist_is_retryable_without_reload_header() -> None: + router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: None) + endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET") + + response = endpoint() + + assert response.status_code == 503 + assert response.headers["cache-control"] == "no-store" + assert "x-missioncore-ui-build" not in response.headers + assert json.loads(response.body) == { + "schema_version": "missioncore.live-viewer-client-contract/v1", + "status": "frontend-build-unavailable", + } + + +def test_development_viewer_diagnostics_do_not_reload_against_dist_build() -> None: + router = build_viewer_diagnostics_router( + expected_ui_build_id=lambda: "/assets/index-abcdefgh.js", + ) + endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST") + event = LiveViewerDiagnosticEvent( + schema_version="missioncore.live-viewer-diagnostic/v2", + event_code="live_receiver_active_store_admitted", + ui_build_id="development", + document_instance_id="00000000-0000-4000-8000-000000000001", + viewer_instance_id="00000000-0000-4000-8000-000000000002", + lifecycle_generation=1, + ) + + response = endpoint(event) + + assert response.status_code == 204 diff --git a/tests/test_web_console.py b/tests/test_web_console.py index fa8ded8..2560ba3 100644 --- a/tests/test_web_console.py +++ b/tests/test_web_console.py @@ -13,8 +13,14 @@ def test_ble_scan_exposes_every_device_and_only_labels_likely_k1( monkeypatch: MonkeyPatch, tmp_path: Path, ) -> None: - async def fake_scan(duration_seconds: float) -> dict[str, Any]: + async def fake_scan( + duration_seconds: float, + *, + on_admitted: object, + ) -> dict[str, Any]: assert duration_seconds == 6.0 + assert callable(on_admitted) + on_admitted() return { "devices": [ { diff --git a/tests/test_web_validation_security.py b/tests/test_web_validation_security.py index 00b5bf7..bac18ca 100644 --- a/tests/test_web_validation_security.py +++ b/tests/test_web_validation_security.py @@ -5,9 +5,39 @@ import json from typing import Any import pytest +from fastapi import WebSocketDisconnect +import k1link.web.app as app_module from k1link.device_plugins.xgrids_k1.facade import XGRIDS_K1_PLUGIN_ID from k1link.web.app import INVALID_REQUEST_DETAIL, app +from k1link.web.plugin_runtime import PluginExecutionError + + +class _StateDispatcher: + async def invoke( + self, + plugin_id: str, + action_id: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + del plugin_id, action_id, payload + return {"phase": "idle"} + + +class _FailingSendWebSocket: + def __init__(self, failure: Exception) -> None: + self.failure = failure + self.accepted = False + + async def accept(self) -> None: + self.accepted = True + + async def send_json(self, payload: dict[str, Any]) -> None: + del payload + raise self.failure + + async def close(self, *, code: int, reason: str) -> None: + del code, reason async def _post_json(path: str, payload: dict[str, Any]) -> tuple[int, str]: @@ -80,6 +110,21 @@ def test_validation_errors_do_not_echo_sensitive_request_values( "verification": "live-device-info", }, } + if wrap_input: + state_status, state_response = asyncio.run( + _post_json( + ( + f"/api/v1/device-plugins/{XGRIDS_K1_PLUGIN_ID}/actions/" + "state.read" + ), + {"input": {}}, + ) + ) + assert state_status == 200 + snapshot_runtime_id = json.loads(state_response)["state"][ + "snapshot_runtime_id" + ] + action_input["expected_snapshot_runtime_id"] = snapshot_runtime_id payload = {"input": action_input} if wrap_input else action_input status_code, response_text = asyncio.run(_post_json(path, payload)) @@ -89,3 +134,75 @@ def test_validation_errors_do_not_echo_sensitive_request_values( assert sensitive_value not in response_text assert sensitive_value[:32] not in response_text assert "input_value" not in response_text + + +def test_legacy_ble_scan_requires_an_explicit_snapshot_runtime_header() -> None: + status_code, response_text = asyncio.run( + _post_json("/api/ble/scan", {"duration_seconds": 1}) + ) + + assert status_code == 422 + assert json.loads(response_text) == {"detail": INVALID_REQUEST_DETAIL} + + +def test_plugin_expected_state_preserves_its_non_gateway_http_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class ExpectedStateDispatcher: + async def invoke( + self, + plugin_id: str, + action_id: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + del plugin_id, action_id, payload + raise PluginExecutionError( + "K1 не сообщил адрес общей локальной сети", + http_status_code=409, + reason_code="connection-verify-address-unavailable", + ) + + monkeypatch.setattr(app_module, "plugin_dispatcher", ExpectedStateDispatcher()) + + status_code, response_text = asyncio.run( + _post_json( + "/api/v1/device-plugins/test.plugin/actions/connection.verify", + {"input": {}}, + ) + ) + + assert status_code == 409 + assert json.loads(response_text) == {"detail": "K1 не сообщил адрес общей локальной сети"} + + +@pytest.mark.parametrize( + "failure", + [ + WebSocketDisconnect(code=1001), + RuntimeError("handler is closed"), + RuntimeError( + "unable to perform operation on ; " + "the handler is closed" + ), + ], +) +def test_device_plugin_events_treats_proven_transport_disconnect_as_completion( + monkeypatch: pytest.MonkeyPatch, + failure: Exception, +) -> None: + websocket = _FailingSendWebSocket(failure) + monkeypatch.setattr(app_module, "plugin_dispatcher", _StateDispatcher()) + + asyncio.run(app_module.device_plugin_events(websocket, "test.plugin")) + + assert websocket.accepted is True + + +def test_device_plugin_events_propagates_arbitrary_send_runtime_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + websocket = _FailingSendWebSocket(RuntimeError("plugin state serialization failed")) + monkeypatch.setattr(app_module, "plugin_dispatcher", _StateDispatcher()) + + with pytest.raises(RuntimeError, match="plugin state serialization failed"): + asyncio.run(app_module.device_plugin_events(websocket, "test.plugin")) diff --git a/tests/test_wifi_provisioning.py b/tests/test_wifi_provisioning.py index 4b23018..29839a3 100644 --- a/tests/test_wifi_provisioning.py +++ b/tests/test_wifi_provisioning.py @@ -1,13 +1,23 @@ import asyncio from collections.abc import Iterator +from pathlib import Path from types import SimpleNamespace from typing import Any import pytest -from bleak.exc import BleakDeviceNotFoundError, BleakGATTProtocolError +from bleak.backends.device import BLEDevice +from bleak.exc import BleakDeviceNotFoundError, BleakError, BleakGATTProtocolError import k1link.device_plugins.xgrids_k1.ble.scanner as scanner_module import k1link.device_plugins.xgrids_k1.ble.wifi_provisioning as wifi_module +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationHardTimeout, + bind_ble_runtime_owner_loop, + ble_runtime_snapshot, + configure_ble_runtime_process_lease, + reset_ble_runtime_arbiter_for_tests, + wait_for_ble_runtime_idle, +) from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( FRAME_LENGTH, build_wifi_provisioning_frame, @@ -18,16 +28,13 @@ from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import ( @pytest.fixture(autouse=True) -def reset_runtime_handle_lease() -> Iterator[None]: - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 +def reset_runtime_handle_lease(tmp_path: Path) -> Iterator[None]: + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + configure_ble_runtime_process_lease(tmp_path) yield - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() def _seed_scan_lease(handles: dict[str, object], *, observed_at: float) -> None: @@ -91,6 +98,7 @@ def test_parse_wifi_status_ap_baseline() -> None: assert parse_wifi_status(bytes(value)) == { "value_length": 54, "mode": "WIFI_AP", + "network_name": None, "ipv4": "192.168.56.1", "status_code": 1, "reserved": 0, @@ -98,6 +106,29 @@ def test_parse_wifi_status_ap_baseline() -> None: } +def test_parse_wifi_status_classifies_fw302_station_network_name() -> None: + """Regression for the redacted shape observed from the physical K1.""" + + network_name = b"LAB_NETWORK" + address = bytes((192, 168, 68, 51)) + value = bytearray(52) + value[0] = len(network_name) + value[1 : 1 + len(network_name)] = network_name + value[33] = len(address) + value[34 : 34 + len(address)] = address + value[50] = 1 + + assert parse_wifi_status(bytes(value)) == { + "value_length": 52, + "mode": "WIFI_CLIENT", + "network_name": "LAB_NETWORK", + "ipv4": "192.168.68.51", + "status_code": 1, + "reserved": 0, + "trailer_hex": "", + } + + def test_parse_wifi_status_rejects_short_frame() -> None: with pytest.raises(ValueError, match="at least 51 bytes"): parse_wifi_status(bytes(50)) @@ -112,7 +143,13 @@ def test_read_wifi_status_once_reads_only_and_returns_current_dhcp_address( value[33] = 4 value[34:38] = bytes((10, 255, 254, 77)) value[50] = 1 - characteristic = SimpleNamespace( + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write-without-response", "write"], + max_write_without_response_size=244, + ) + status_characteristic = SimpleNamespace( uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, service_uuid=wifi_module.SERVICE_UUID, properties=["read"], @@ -124,12 +161,17 @@ def test_read_wifi_status_once_reads_only_and_returns_current_dhcp_address( return service if uuid == wifi_module.SERVICE_UUID else None def get_characteristic(self, uuid: str) -> object | None: - return characteristic if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID else None + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None class FakeClient: def __init__(self, _device: object, **_kwargs: object) -> None: self.services = FakeServices() self.name = "XGR-K1" + self.mtu_size = 256 self.write_calls = 0 async def __aenter__(self) -> Any: @@ -160,6 +202,10 @@ def test_read_wifi_status_once_reads_only_and_returns_current_dhcp_address( assert result["operation"] == "single_reviewed_wifi_status_read" assert result["write_performed"] is False + assert result["write_characteristic_uuid"] == wifi_module.WRITE_CHARACTERISTIC_UUID + assert result["write_characteristic_properties"] == ["write", "write-without-response"] + assert result["max_write_without_response_size"] == 244 + assert result["mtu_size"] == 256 assert result["status"]["ipv4"] == "10.255.254.77" @@ -173,7 +219,12 @@ def test_read_wifi_status_recovery_keeps_fresh_retained_handle( value[34:38] = bytes((10, 255, 254, 77)) value[50] = 1 retained_handle = object() - characteristic = SimpleNamespace( + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, service_uuid=wifi_module.SERVICE_UUID, properties=["read"], @@ -185,7 +236,11 @@ def test_read_wifi_status_recovery_keeps_fresh_retained_handle( return service if uuid == wifi_module.SERVICE_UUID else None def get_characteristic(self, uuid: str) -> object | None: - return characteristic if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID else None + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None class FakeClient: def __init__(self, device: object, **_kwargs: object) -> None: @@ -224,6 +279,533 @@ def test_read_wifi_status_recovery_keeps_fresh_retained_handle( ) assert result["status"]["ipv4"] == "10.255.254.77" + assert result["max_write_without_response_size"] is None + assert result["mtu_size"] is None + + +def test_retrieved_capture_status_read_connects_and_validates_7f02_without_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device = BLEDevice("RETRIEVED-UUID", "XGR-RETRIEVED", details=object()) + value = bytearray(54) + value[0] = 11 + value[1:12] = b"WIFI_CLIENT" + value[33] = 4 + value[34:38] = bytes((192, 168, 68, 52)) + value[50] = 1 + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + events: list[str] = [] + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + class ReadOnlyClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + events.append("new-connect") + self.services = FakeServices() + self.name = "XGR-RETRIEVED" + + async def __aenter__(self) -> Any: + events.append("connected") + return self + + async def __aexit__(self, *_args: object) -> None: + events.append("disconnected") + + async def read_gatt_char(self, characteristic: object) -> bytes: + assert characteristic is status_characteristic + events.append("read-7f02") + return bytes(value) + + async def write_gatt_char(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("retrieved status validation must remain read-only") + + def forbidden_selection(_uuid: str) -> object: + raise AssertionError("an exact retrieved capture must not use scan selection") + + async def forbidden_lookup(*_args: object, **_kwargs: object) -> object: + raise AssertionError("an exact retrieved capture must not use UUID lookup") + + monkeypatch.setattr(wifi_module, "BleakClient", ReadOnlyClient) + monkeypatch.setattr(wifi_module, "discovered_device_selection", forbidden_selection) + monkeypatch.setattr( + wifi_module.BleakScanner, + "find_device_by_address", + forbidden_lookup, + ) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + now = scanner_module._freshness_now() # noqa: SLF001 + initial_capture = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=9, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="fresh-scan", + ) + scanner_module.pin_connected_device_handle( + initial_capture, + device_session_id="device-session-a", + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + scan_generation=9, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-session", + ) + + result = await read_wifi_status_once( + device.address, + timeout_seconds=1.0, + captured_device=captured, + ) + assert result["write_performed"] is False + assert result["status"]["ipv4"] == "192.168.68.52" + assert scanner_module.connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": True, + } + + asyncio.run(scenario()) + assert events == ["new-connect", "connected", "read-7f02", "disconnected"] + + +def test_durable_uuid_status_read_retrieves_then_connects_and_returns_capture( + monkeypatch: pytest.MonkeyPatch, +) -> None: + macos_uuid = "11111111-2222-4333-8444-555555555555" + device = BLEDevice(macos_uuid, "XGR-DURABLE", details=object()) + value = bytearray(54) + value[0] = 11 + value[1:12] = b"WIFI_CLIENT" + value[33] = 4 + value[34:38] = bytes((192, 168, 68, 52)) + value[50] = 1 + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + events: list[str] = [] + returned_captures: list[scanner_module.CapturedDiscoveredDevice] = [] + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + class ReadOnlyClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + self.services = FakeServices() + self.name = "XGR-DURABLE" + events.append("connect") + + async def __aenter__(self) -> Any: + return self + + async def __aexit__(self, *_args: object) -> None: + events.append("disconnect") + + async def read_gatt_char(self, characteristic: object) -> bytes: + assert characteristic is status_characteristic + events.append("read-7f02") + return bytes(value) + + async def write_gatt_char(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("durable UUID reconciliation must not write") + + async def retrieve(requested_uuid: str) -> scanner_module.CapturedDiscoveredDevice: + assert requested_uuid == macos_uuid + owner_epoch = scanner_module.ble_runtime_owner_epoch_for_current_loop() + assert owner_epoch is not None + now = scanner_module._freshness_now() # noqa: SLF001 + events.append("retrieve-known-uuid") + return scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + + monkeypatch.setattr( + wifi_module, + "discovered_device_selection", + lambda _uuid: SimpleNamespace(device=None, from_fresh_scan=False), + ) + monkeypatch.setattr( + wifi_module, + "retrieve_known_device_capture_for_status_read", + retrieve, + ) + monkeypatch.setattr( + wifi_module, + "discover_known_device_capture_for_status_read", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("cached retrieval must not start an advertisement scan") + ), + ) + monkeypatch.setattr( + wifi_module.BleakScanner, + "find_device_by_address", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("durable retrieval must not fall through to a scan") + ), + ) + monkeypatch.setattr(wifi_module, "BleakClient", ReadOnlyClient) + + result = asyncio.run( + read_wifi_status_once( + macos_uuid, + timeout_seconds=1.0, + allow_known_device_retrieval=True, + on_gatt_validated=returned_captures.append, + ) + ) + + assert result["write_performed"] is False + assert result["status"]["ipv4"] == "192.168.68.52" + assert len(returned_captures) == 1 + assert returned_captures[0].source == "retrieved-durable" + assert events == ["retrieve-known-uuid", "connect", "read-7f02", "disconnect"] + + +def test_durable_uuid_rediscover_scans_first_then_reads_once_without_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + macos_uuid = "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE" + device = BLEDevice(macos_uuid, "XGR-ADVERTISEMENT", details=object()) + value = bytearray(54) + value[0] = 11 + value[1:12] = b"WIFI_CLIENT" + value[33] = 4 + value[34:38] = bytes((192, 168, 68, 99)) + value[50] = 1 + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + events: list[str] = [] + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + class ReadOnlyClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + events.append("connect") + self.services = FakeServices() + self.name = "XGR-ADVERTISEMENT" + + async def __aenter__(self) -> Any: + return self + + async def __aexit__(self, *_args: object) -> None: + events.append("disconnect") + + async def read_gatt_char(self, characteristic: object) -> bytes: + assert characteristic is status_characteristic + events.append("read-7f02") + return bytes(value) + + async def write_gatt_char(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("advertisement recovery must remain read-only") + + async def exact_scan( + requested_uuid: str, + *, + timeout_seconds: float, + ) -> scanner_module.CapturedDiscoveredDevice: + assert requested_uuid == macos_uuid + assert timeout_seconds == 30.0 + owner_epoch = scanner_module.ble_runtime_owner_epoch_for_current_loop() + assert owner_epoch is not None + now = scanner_module._freshness_now() # noqa: SLF001 + events.append("exact-uuid-scan") + return scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=macos_uuid, + owner_epoch=owner_epoch, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-durable", + ) + + async def forbidden_retrieval( + _requested_uuid: str, + ) -> scanner_module.CapturedDiscoveredDevice | None: + raise AssertionError("rediscover must not connect a cached peripheral first") + + monkeypatch.setattr( + wifi_module, + "discover_known_device_capture_for_status_read", + exact_scan, + ) + monkeypatch.setattr( + wifi_module, + "retrieve_known_device_capture_for_status_read", + forbidden_retrieval, + ) + monkeypatch.setattr( + wifi_module, + "discovered_device_selection", + lambda _uuid: (_ for _ in ()).throw( + AssertionError("durable rediscovery must not consume public scan state") + ), + ) + monkeypatch.setattr(wifi_module, "BleakClient", ReadOnlyClient) + + result = asyncio.run( + read_wifi_status_once( + macos_uuid, + timeout_seconds=1.0, + exact_scan_timeout_seconds=30.0, + rediscover=True, + allow_known_device_retrieval=True, + ) + ) + + assert result["write_performed"] is False + assert result["status"]["ipv4"] == "192.168.68.99" + assert events == ["exact-uuid-scan", "connect", "read-7f02", "disconnect"] + + +def test_durable_uuid_rediscover_timeout_fails_without_cache_or_connect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + macos_uuid = "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE" + scan_calls: list[tuple[str, float]] = [] + + async def exact_scan( + requested_uuid: str, + *, + timeout_seconds: float, + ) -> scanner_module.CapturedDiscoveredDevice | None: + scan_calls.append((requested_uuid, timeout_seconds)) + return None + + async def forbidden_retrieval( + _requested_uuid: str, + ) -> scanner_module.CapturedDiscoveredDevice | None: + raise AssertionError("scan timeout must not fall back to cached retrieval") + + class ForbiddenClient: + def __init__(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("scan timeout must fail before GATT connect") + + monkeypatch.setattr( + wifi_module, + "discover_known_device_capture_for_status_read", + exact_scan, + ) + monkeypatch.setattr( + wifi_module, + "retrieve_known_device_capture_for_status_read", + forbidden_retrieval, + ) + monkeypatch.setattr(wifi_module, "BleakClient", ForbiddenClient) + + with pytest.raises(BleakDeviceNotFoundError) as raised: + asyncio.run( + read_wifi_status_once( + macos_uuid, + timeout_seconds=1.0, + exact_scan_timeout_seconds=30.0, + rediscover=True, + allow_known_device_retrieval=True, + ) + ) + + assert scan_calls == [(macos_uuid, 30.0)] + assert raised.value.operation_stage == "exact-uuid-scan" # type: ignore[attr-defined] + + +def test_durable_uuid_retrieval_exception_preserves_sanitized_resolution_stage( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def failing_retrieval( + _requested_uuid: str, + ) -> scanner_module.CapturedDiscoveredDevice | None: + raise BleakError("synthetic native retrieval detail") + + class ForbiddenClient: + def __init__(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("retrieval failure must occur before GATT connect") + + monkeypatch.setattr( + wifi_module, + "retrieve_known_device_capture_for_status_read", + failing_retrieval, + ) + monkeypatch.setattr(wifi_module, "BleakClient", ForbiddenClient) + + with pytest.raises(BleakError) as raised: + asyncio.run( + read_wifi_status_once( + "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE", + timeout_seconds=1.0, + allow_known_device_retrieval=True, + ) + ) + + assert raised.value.operation_stage == "resolution" # type: ignore[attr-defined] + + +def test_durable_uuid_rediscover_hard_timeout_includes_scan_and_gatt_budgets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + hard_timeouts: list[float] = [] + + async def capture_hard_timeout( + _operation_kind: str, + *, + hard_timeout_seconds: float, + operation: object, + progress: object, + ) -> dict[str, object]: + del operation, progress + hard_timeouts.append(hard_timeout_seconds) + return {} + + monkeypatch.setattr(wifi_module, "run_ble_operation", capture_hard_timeout) + + asyncio.run( + read_wifi_status_once( + "AAAAAAAA-BBBB-4CCC-8DDD-EEEEEEEEEEEE", + timeout_seconds=20.0, + exact_scan_timeout_seconds=30.0, + rediscover=True, + allow_known_device_retrieval=True, + ) + ) + + assert hard_timeouts == [55.0] + + +def test_failed_retrieved_status_read_invalidates_session_token( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device = BLEDevice("OFFLINE-RETRIEVED-UUID", "XGR-OFFLINE", details=object()) + + class OfflineClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + + async def __aenter__(self) -> Any: + raise BleakDeviceNotFoundError(device.address, "powered off") + + async def __aexit__(self, *_args: object) -> None: + return None + + monkeypatch.setattr(wifi_module, "BleakClient", OfflineClient) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + now = scanner_module._freshness_now() # noqa: SLF001 + initial_capture = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="fresh-scan", + ) + scanner_module.pin_connected_device_handle( + initial_capture, + device_session_id="device-session-a", + ) + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + captured_at_monotonic=now.monotonic, + captured_at_suspend_aware=now.suspend_aware, + source="retrieved-session", + ) + + with pytest.raises(BleakDeviceNotFoundError): + await read_wifi_status_once( + device.address, + timeout_seconds=1.0, + captured_device=captured, + ) + assert ( + scanner_module.connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + is None + ) + assert scanner_module.connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + ) == { + "status": "unavailable", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + } + + asyncio.run(scenario()) def test_provisioning_write_uses_retained_handle_without_rediscovery( @@ -305,7 +887,7 @@ def test_provisioning_write_does_not_fallback_when_fresh_scan_omits_device( assert caught.value.device_write_confirmed is False # type: ignore[attr-defined] -def test_provisioning_write_rediscovery_fallback_after_scan_lease_expires( +def test_provisioning_uses_latest_selected_handle_without_age_based_rediscovery( monkeypatch: pytest.MonkeyPatch, ) -> None: device_id = "synthetic-corebluetooth-uuid" @@ -343,8 +925,8 @@ def test_provisioning_write_rediscovery_fallback_after_scan_lease_expires( ) ) - assert rediscovery_calls == [(device_id, 1.0)] - assert client_calls == [rediscovered_handle] + assert rediscovery_calls == [] + assert client_calls == [expired_handle] assert scanner_module.discovered_device(device_id) is None @@ -410,10 +992,94 @@ def test_provisioning_baseline_error_keeps_type_and_adds_safe_gatt_facts( assert error.operation_stage == "baseline-read" # type: ignore[attr-defined] assert error.device_write_attempted is False # type: ignore[attr-defined] assert error.device_write_confirmed is False # type: ignore[attr-defined] + assert error.resolved_write_mode == "with_response" # type: ignore[attr-defined] + assert error.write_characteristic_properties == ("write",) # type: ignore[attr-defined] + assert error.max_write_without_response_size == 512 # type: ignore[attr-defined] + assert error.frame_length == FRAME_LENGTH # type: ignore[attr-defined] assert error.att_error_code == 0x0E # type: ignore[attr-defined] assert error.att_error_name == "UNLIKELY_ERROR" # type: ignore[attr-defined] +def test_provisioning_write_error_adds_only_safe_resolved_gatt_facts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device_id = "synthetic-corebluetooth-uuid" + retained_handle = object() + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + class FailingWriteClient: + def __init__(self, device: object, **_kwargs: object) -> None: + assert device is retained_handle + self.services = FakeServices() + self.name = "XGR-K1" + + async def __aenter__(self) -> Any: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def read_gatt_char(self, _characteristic: object) -> bytes: + return bytes(52) + + async def write_gatt_char( + self, + _characteristic: object, + _value: bytes, + *, + response: bool, + ) -> None: + assert response is True + raise BleakGATTProtocolError(0x03) + + monkeypatch.setattr(scanner_module, "monotonic", lambda: 100.0) + _seed_scan_lease({device_id: retained_handle}, observed_at=100.0) + monkeypatch.setattr(wifi_module, "BleakClient", FailingWriteClient) + + with pytest.raises(BleakGATTProtocolError) as caught: + asyncio.run( + provision_wifi_once( + device_id, + "LabNet", + "synthetic-password", + timeout_seconds=1.0, + ) + ) + + error = caught.value + assert error.operation_stage == "gatt-write" # type: ignore[attr-defined] + assert error.resolved_write_mode == "with_response" # type: ignore[attr-defined] + assert error.write_characteristic_properties == ("write",) # type: ignore[attr-defined] + assert error.max_write_without_response_size is None # type: ignore[attr-defined] + assert error.frame_length == FRAME_LENGTH # type: ignore[attr-defined] + diagnostic_text = repr(vars(error)) + assert "LabNet" not in diagnostic_text + assert "synthetic-password" not in diagnostic_text + assert "payload" not in vars(error) + assert "frame" not in vars(error) + + def test_provisioning_status_poll_error_reports_confirmed_write( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -485,5 +1151,253 @@ def test_provisioning_status_poll_error_reports_confirmed_write( assert error.operation_stage == "status-poll" # type: ignore[attr-defined] assert error.device_write_attempted is True # type: ignore[attr-defined] assert error.device_write_confirmed is True # type: ignore[attr-defined] + assert error.resolved_write_mode == "with_response" # type: ignore[attr-defined] + assert error.write_characteristic_properties == ("write",) # type: ignore[attr-defined] + assert error.max_write_without_response_size == 512 # type: ignore[attr-defined] + assert error.frame_length == FRAME_LENGTH # type: ignore[attr-defined] assert error.att_error_code == 0x12 # type: ignore[attr-defined] assert error.att_error_name == "DATABASE_OUT_OF_SYNC" # type: ignore[attr-defined] + + +def test_provision_deadline_during_gatt_write_reports_ambiguous_metadata( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device_id = "synthetic-corebluetooth-uuid" + retained_handle = object() + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write-without-response"], + max_write_without_response_size=512, + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + baseline = bytes(52) + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + async def scenario() -> None: + cancellation_observed = asyncio.Event() + cleanup_release = asyncio.Event() + + class StuckWriteClient: + def __init__(self, device: object, **_kwargs: object) -> None: + assert device is retained_handle + self.services = FakeServices() + self.name = "XGR-K1" + self.is_connected = True + + async def __aenter__(self) -> Any: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def read_gatt_char(self, _characteristic: object) -> bytes: + return baseline + + async def write_gatt_char(self, *_args: object, **_kwargs: object) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancellation_observed.set() + await cleanup_release.wait() + raise + + monkeypatch.setattr(scanner_module, "monotonic", lambda: 100.0) + _seed_scan_lease({device_id: retained_handle}, observed_at=100.0) + monkeypatch.setattr(wifi_module, "BleakClient", StuckWriteClient) + monkeypatch.setattr( + wifi_module, + "BLE_PROVISION_HARD_TIMEOUT_GRACE_SECONDS", + 0.01, + ) + + try: + with pytest.raises(BleOperationHardTimeout) as raised: + await provision_wifi_once( + device_id, + "LabNet", + "synthetic-password", + timeout_seconds=0.01, + ) + assert raised.value.reason_code == "ble-provisioning-timeout" + assert raised.value.operation_stage == "gatt-write" + assert raised.value.device_write_attempted is True + assert raised.value.device_write_confirmed is False + assert "LabNet" not in str(raised.value) + assert "synthetic-password" not in str(raised.value) + await cancellation_observed.wait() + assert ble_runtime_snapshot()["cleanup_pending"] is True + finally: + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_captured_recovery_handle_reads_7f02_before_exactly_one_station_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device = BLEDevice("RECOVERY-UUID", "XGR-RECOVERY", details=object()) + service = SimpleNamespace(uuid=wifi_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=wifi_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["write"], + ) + status_characteristic = SimpleNamespace( + uuid=wifi_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=wifi_module.SERVICE_UUID, + properties=["read"], + ) + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == wifi_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == wifi_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == wifi_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + baseline = bytearray(52) + baseline[0] = 7 + baseline[1:8] = b"WIFI_AP" + baseline[33] = 4 + baseline[34:38] = bytes((192, 168, 56, 1)) + connected = bytearray(52) + connected[0] = 11 + connected[1:12] = b"WIFI_CLIENT" + connected[33] = 4 + connected[34:38] = bytes((192, 168, 68, 50)) + events: list[str] = [] + + class RecordingClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + self.services = FakeServices() + self.name = "XGR-RECOVERY" + self.is_connected = True + + async def __aenter__(self) -> Any: + return self + + async def __aexit__(self, *_args: object) -> None: + return None + + async def read_gatt_char(self, _characteristic: object) -> bytes: + events.append("read-7f02") + return bytes(baseline if events.count("write-7f01") == 0 else connected) + + async def write_gatt_char( + self, + _characteristic: object, + _value: bytes, + *, + response: bool, + ) -> None: + assert response is True + events.append("write-7f01") + + monkeypatch.setattr(wifi_module, "BleakClient", RecordingClient) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + ) + scanner_module.pin_connected_device_handle( + captured, + device_session_id="device-session-a", + ) + result = await provision_wifi_once( + device.address, + "LabNet", + "synthetic-password", + timeout_seconds=1.0, + write_mode="with_response", + captured_device=captured, + ) + assert result["outcome"] == "lan_address_observed" + assert scanner_module.connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["gatt_validated_recently"] is True + + asyncio.run(scenario()) + assert events == ["read-7f02", "write-7f01", "read-7f02"] + + +def test_powered_off_recovery_handle_fails_before_station_write( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device = BLEDevice("OFFLINE-UUID", "XGR-OFFLINE", details=object()) + + class OfflineClient: + def __init__(self, selected: object, **_kwargs: object) -> None: + assert selected is device + + async def __aenter__(self) -> Any: + raise BleakDeviceNotFoundError(device.address, "powered off") + + async def __aexit__(self, *_args: object) -> None: + return None + + async def write_gatt_char(self, *_args: object, **_kwargs: object) -> None: + raise AssertionError("powered-off recovery must fail before 7f01") + + monkeypatch.setattr(wifi_module, "BleakClient", OfflineClient) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=device, + macos_uuid=device.address, + owner_epoch=owner_epoch, + ) + scanner_module.pin_connected_device_handle( + captured, + device_session_id="device-session-a", + ) + with pytest.raises(BleakDeviceNotFoundError) as raised: + await provision_wifi_once( + device.address, + "LabNet", + "synthetic-password", + timeout_seconds=1.0, + write_mode="with_response", + captured_device=captured, + ) + assert raised.value.operation_stage == "connect" # type: ignore[attr-defined] + assert raised.value.device_write_attempted is False # type: ignore[attr-defined] + assert ( + scanner_module.connected_device_capture( + device.address, + device_session_id="device-session-a", + ) + is None + ) + assert scanner_module.connected_device_recovery_snapshot( + device.address, + device_session_id="device-session-a", + )["status"] == "unavailable" + + asyncio.run(scenario()) diff --git a/tests/test_xgrids_acquisition_lifecycle.py b/tests/test_xgrids_acquisition_lifecycle.py index a8e48d0..3102866 100644 --- a/tests/test_xgrids_acquisition_lifecycle.py +++ b/tests/test_xgrids_acquisition_lifecycle.py @@ -1,37 +1,140 @@ from __future__ import annotations import asyncio +import fcntl +import hashlib import json import logging +import os +import subprocess import threading -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +import time +from collections.abc import AsyncIterator, Callable, Iterator, Mapping +from contextlib import asynccontextmanager, contextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime, timedelta +from itertools import count from pathlib import Path +from types import MethodType, SimpleNamespace from typing import Any import pytest +from missioncore_plugin_sdk.v0alpha2 import RuntimeActionInvocation from pydantic import SecretStr, ValidationError import k1link.device_plugins.xgrids_k1.facade as facade_module +import k1link.host_network.wifi as host_wifi_module +from k1link.data_plane import ConsumerFrameContext, DecodedPointCloudView +from k1link.device_plugins.xgrids_k1.active_acquisition_recovery_checkpoint import ( + ActiveAcquisitionRecoveryConnection, + ActiveAcquisitionRecoveryIdentity, + ActiveAcquisitionRecoveryTransportBinding, + active_acquisition_project_name_sha256, +) +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + ApplicationControlProcessLease, +) +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationHardTimeout, + BleOperationProgress, + BleRuntimeBusy, + bind_ble_runtime_owner_loop, + run_ble_operation, + wait_for_ble_runtime_idle, +) +from k1link.device_plugins.xgrids_k1.connection_supervisor import ( + DEFAULT_TRANSPORT_OBSERVATION_TTL_SECONDS, + EndpointTarget, + HostPathProbeResult, + VerifiedControlEvidence, +) +from k1link.device_plugins.xgrids_k1.device_identity_pin_store import ( + DeviceIdentityPinStoreCorrupt, +) from k1link.device_plugins.xgrids_k1.facade import ( + ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION, + ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, DEFAULT_LIVE_STREAMS, XGRIDS_K1_COMPATIBILITY_PROFILE_ID, AbortAcquisitionRequest, + BleScanRequest, CameraPreviewSelectRequest, CompatibilityAttestationRequest, + ConfiguredEndpointProbeRequest, ConnectionVerifyRequest, ConnectRequest, + DesiredConnectionModeRequest, + EnterApplicationWorkspaceRequest, + ForceFinishAcquisitionRequest, OpenApplicationControlSessionRequest, OperatorPresenceRequest, PrepareAcquisitionRequest, + PrepareConnectionReconfigurationRequest, + ReopenRetiredPhysicalCommandReconciliationRequest, + RetireUnavailablePhysicalCommandRequest, ShadowApplicationControlArmRequest, StartAcquisitionRequest, StopAcquisitionRequest, XgridsK1CompatibilityService, + XgridsK1PluginFacade, +) +from k1link.device_plugins.xgrids_k1.network_mutation_ledger import ( + NetworkConnectionMode, + NetworkMutationRecord, + NetworkStatusEvidence, + PreviousConnectionEvidence, +) +from k1link.device_plugins.xgrids_k1.network_provisioning_idempotency_journal import ( + NetworkProvisioningIdempotencyConflict, + derive_request_binding_sha256, +) +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + LedgerPhysicalCommandCoordinator, + PhysicalCommandRuntimeBinding, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + PhysicalCommandApplicationResponse, + PhysicalCommandBlocked, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandLedgerSnapshot, + PhysicalCommandStatusEvidence, + PhysicalCommandTransitionError, +) +from k1link.device_plugins.xgrids_k1.protocol import application_session as session_module +from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( + ApplicationAcceptanceError, ) from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( ApplicationControlAuthority, + LiveDeviceControlBinding, ) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, + ApplicationMqttTransportError, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) +from k1link.device_plugins.xgrids_k1.protocol.application_session import ( + ApplicationConnectionBinding, + ApplicationConnectionBindingLost, + InteractiveApplicationControlSession, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( + MODELING_STATE_BASE, +) +from k1link.device_plugins.xgrids_k1.semantic_topology_store import ( + SemanticTopologyStoreCorrupt, +) +from k1link.device_plugins.xgrids_k1.viewer.messages import StreamMessage +from k1link.host_network import HostWifiAssociationIdentityResult +from k1link.viewer.metrics import BridgeMetrics +from k1link.web.device_lifecycle import OperationJournal +from k1link.web.plugin_runtime import PluginExecutionError ATTESTATION = CompatibilityAttestationRequest( firmware_version="3.0.2", @@ -52,6 +155,347 @@ PRIMARY_TEST_CREDENTIAL = "x" * 24 SECONDARY_TEST_CREDENTIAL = "y" * 24 PROJECT_NAME = "K1 lifecycle test" PRIVATE_APPLICATION_AUTHORITY = "11111111-2222-3333-4444-555555555555" +DURABLE_K1_UUID = "F89438FA-55ED-85AD-EED7-734AC84746D8" +_CONNECT_INTENT_SEQUENCE = count(1) +_LIFECYCLE_INTENT_SEQUENCE = count(1) +_SYNTHETIC_SCAN_CAPTURES: dict[str, facade_module.CapturedDiscoveredDevice] = {} + + +@pytest.fixture(autouse=True) +def _synthetic_exact_scan_handoff(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Give facade-only lifecycle fixtures an exact selected scan object. + + Scanner ownership/object handoff itself is covered in ``test_ble_scanner``. + These service tests seed the facade's scan projection directly, so mirror + that synthetic scan with one object-bound capture instead of weakening the + production admission rule to accept a UUID without a CoreBluetooth object. + """ + + _SYNTHETIC_SCAN_CAPTURES.clear() + monkeypatch.setattr( + facade_module, + "_capture_network_intent_device", + lambda device_id: _SYNTHETIC_SCAN_CAPTURES.get(device_id), + ) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda *_args, **_kwargs: None, + ) + yield + _SYNTHETIC_SCAN_CAPTURES.clear() + + +def _connect_request(**values: Any) -> ConnectRequest: + """Give every physical network test an explicit, unique durable intent.""" + + values.setdefault( + "idempotency_key", + f"test-network-provision-{next(_CONNECT_INTENT_SEQUENCE)}", + ) + values.setdefault("expected_mode_revision", 0) + values.setdefault("expected_discovery_generation", 0) + return ConnectRequest(**values) + + +def _select_connection_mode( + service: XgridsK1CompatibilityService, + connection_mode: str, +) -> int: + """Select a process-local draft exactly as the product dropdown does.""" + + state = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode=connection_mode, # type: ignore[arg-type] + expected_revision=service.state()["desired_connection_mode_revision"], + ) + ) + return int(state["desired_connection_mode_revision"]) + + +def _lifecycle_intent_key(action: str) -> str: + return f"test-{action}-{next(_LIFECYCLE_INTENT_SEQUENCE)}" + + +def _prepare_request(**values: Any) -> PrepareAcquisitionRequest: + values.setdefault("idempotency_key", _lifecycle_intent_key("acquisition-prepare")) + values.setdefault("expected_control_session_generation", 1) + values.setdefault("expected_control_state_revision", 1) + return PrepareAcquisitionRequest(**values) + + +def _start_request(**values: Any) -> StartAcquisitionRequest: + values.setdefault("idempotency_key", _lifecycle_intent_key("acquisition-start")) + values.setdefault("expected_control_session_generation", 1) + values.setdefault("expected_control_state_revision", 2) + return StartAcquisitionRequest(**values) + + +def _stop_request(**values: Any) -> StopAcquisitionRequest: + values.setdefault("idempotency_key", _lifecycle_intent_key("acquisition-stop")) + values.setdefault("expected_control_session_generation", 1) + values.setdefault("expected_control_state_revision", 3) + return StopAcquisitionRequest(**values) + + +def _abort_request(**values: Any) -> AbortAcquisitionRequest: + values.setdefault("idempotency_key", _lifecycle_intent_key("acquisition-abort")) + values.setdefault("expected_control_session_generation", 1) + values.setdefault("expected_control_state_revision", 2) + return AbortAcquisitionRequest(**values) + + +def _force_finish_request(**values: Any) -> ForceFinishAcquisitionRequest: + values.setdefault( + "idempotency_key", + _lifecycle_intent_key("acquisition-force-finish-local"), + ) + values.setdefault("operator_confirmed", True) + return ForceFinishAcquisitionRequest(**values) + + +def _dispatch_test_network_write( + on_write_dispatch: Callable[[dict[str, Any], str], None] | None, +) -> None: + """Cross the same durable dispatch boundary as the production BLE helper.""" + + assert on_write_dispatch is not None + on_write_dispatch( + { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + }, + "with_response", + ) + + +async def _reachable_control_endpoint( + target: str, + *_: object, + endpoint_probe: Callable[[str], object] | None = None, + **__: object, +) -> object: + return endpoint_probe(target) if endpoint_probe is not None else True + + +async def _unreachable_control_endpoint( + target: str, + *_: object, + endpoint_probe: Callable[[str], object] | None = None, + **__: object, +) -> object: + return endpoint_probe(target) if endpoint_probe is not None else False + + +def _direct_host_path(target_ipv4: str, **_: object) -> HostPathProbeResult: + source_ipv4 = ( + "10.255.254.2" + if target_ipv4.startswith("10.") + else "192.168.56.2" + if target_ipv4.startswith("192.168.56.") + else "192.168.1.2" + ) + return HostPathProbeResult( + available=True, + fingerprint=f"test-route:{target_ipv4}", + interface="test0", + source_ipv4=source_ipv4, + route_class="direct", + kernel_route_fingerprint=f"test-route:{target_ipv4}", + ) + + +def _association_bound_direct_host_path(target_ipv4: str) -> HostPathProbeResult: + raw = _direct_host_path(target_ipv4) + assert raw.fingerprint is not None + return HostPathProbeResult( + available=True, + fingerprint=facade_module.bind_route_fingerprint_to_wifi_association( + raw.fingerprint, + DeterministicContendedAssociationProbe.proven_observation(), + ), + interface=raw.interface, + source_ipv4=raw.source_ipv4, + route_class=raw.route_class, + kernel_route_fingerprint=raw.fingerprint, + ) + + +def _tunnel_host_path(target_ipv4: str) -> HostPathProbeResult: + return HostPathProbeResult( + available=True, + fingerprint=f"test-tunnel:{target_ipv4}", + interface="utun-test", + source_ipv4="100.64.0.2", + route_class="tunnel", + ) + + +@pytest.mark.parametrize( + ("interface", "destination", "expected", "reason_code"), + [ + ("en0", "192.168.68.0", "direct", None), + ("bridge100", "192.168.68.0", "direct", None), + ("utun8", "192.168.68.0", "tunnel", "host-route-tunnel"), + ("tun0", "192.168.68.0", "tunnel", "host-route-tunnel"), + ("tap2", "192.168.68.0", "tunnel", "host-route-tunnel"), + ("wg0", "192.168.68.0", "tunnel", "host-route-tunnel"), + ("en0", "default", "default", "host-route-default"), + ( + "vendor-vpn0", + "192.168.68.0", + "unknown", + "host-route-interface-unreviewed", + ), + (None, "192.168.68.0", "unknown", "host-route-unclassified"), + ], +) +def test_host_route_classifier_is_fail_closed_for_virtual_and_unknown_interfaces( + interface: str | None, + destination: str, + expected: str, + reason_code: str | None, +) -> None: + assert facade_module._classify_host_route(interface, destination) == ( # noqa: SLF001 + expected, + reason_code, + ) + + +class FakeHostWifiAssociationProbe: + def __init__(self, *continuity_tokens: str) -> None: + self._continuity_tokens = continuity_tokens or ("a" * 64,) + self.interfaces: list[str | None] = [] + self.timeout_seconds: list[float] = [] + + def observe( + self, + interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + assert timeout_seconds > 0 + self.timeout_seconds.append(timeout_seconds) + token_index = min(len(self.interfaces), len(self._continuity_tokens) - 1) + self.interfaces.append(interface_name) + return { + "schema_version": 1, + "adapter": "test-host-association", + "wifi_interface": True, + "association_state": "associated", + "evidence_quality": "bssid-only", + "continuity_proven": True, + "continuity_token": self._continuity_tokens[token_index], + "reason_code": None, + } + + +class DeterministicContendedAssociationProbe: + """Real probe harness with fake lock/helper clocks and no wall-clock sleep.""" + + def __init__( + self, + *, + lock_wait_seconds: float, + helper_required_seconds: float, + on_first_lock_acquired: Callable[[], None] | None = None, + ) -> None: + self.monotonic = 100.0 + self.wall = 1_000.0 + self.lock_wait_seconds = lock_wait_seconds + self.helper_required_seconds = helper_required_seconds + self.on_first_lock_acquired = on_first_lock_acquired + self.lock_timeouts: list[float] = [] + self.helper_timeouts: list[float] = [] + self._lock_acquisitions = 0 + self._locked = False + + def advance(self, seconds: float) -> None: + self.monotonic += seconds + self.wall += seconds + + def acquire(self, *, timeout: float) -> bool: + self.lock_timeouts.append(timeout) + wait = self.lock_wait_seconds if self._lock_acquisitions == 0 else 0.0 + self._lock_acquisitions += 1 + if wait > timeout: + self.advance(timeout) + return False + self.advance(wait) + self._locked = True + if self._lock_acquisitions == 1 and self.on_first_lock_acquired is not None: + self.on_first_lock_acquired() + return True + + def release(self) -> None: + assert self._locked is True + self._locked = False + + def runner( + self, + argv: list[str], + **kwargs: object, + ) -> subprocess.CompletedProcess[bytes]: + timeout = kwargs.get("timeout") + assert isinstance(timeout, (int, float)) and not isinstance(timeout, bool) + timeout_seconds = float(timeout) + self.helper_timeouts.append(timeout_seconds) + if self.helper_required_seconds > timeout_seconds: + self.advance(timeout_seconds) + raise subprocess.TimeoutExpired(argv, timeout_seconds) + self.advance(self.helper_required_seconds) + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_identity":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + b'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","association_evidence":"bssid-only"}' + ), + stderr=b"", + ) + + def build( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> host_wifi_module.HostWifiAssociationIdentityProbe: + helper_path = tmp_path / "deterministic-association-helper.swift" + helper_path.write_text("// deterministic test helper\n", encoding="utf-8") + monkeypatch.setattr( + host_wifi_module, + "time", + SimpleNamespace(monotonic=lambda: self.monotonic), + ) + monkeypatch.setattr( + host_wifi_module, + "sys", + SimpleNamespace(platform="darwin"), + ) + probe = host_wifi_module.HostWifiAssociationIdentityProbe( + helper_path, + runner=self.runner, + continuity_key=b"c" * 32, + monotonic_clock=lambda: self.monotonic, + wall_clock=lambda: self.wall, + ) + probe._lock = self # type: ignore[assignment] # noqa: SLF001 + return probe + + @staticmethod + def proven_observation() -> HostWifiAssociationIdentityResult: + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "associated", + "evidence_quality": "bssid-only", + "continuity_proven": True, + "continuity_token": "a" * 64, + "reason_code": None, + } class FakeApplicationAuthorityLoader: @@ -68,10 +512,16 @@ class FakeVisualizationRuntime: self.phase = "idle" self.source_mode = "idle" self.source_ready = False + self.producer_generation = 0 self.pcl_frames = 0 self.start_calls: list[tuple[str, Path, float | None, str]] = [] self.stop_calls = 0 self.stop_error: Exception | None = None + self.recover_connection: Callable[[int], str] | None = None + self.recovery_requests: list[tuple[str, int]] = [] + self.recovery_request_pending = False + self.recovery_state = "inactive" + self.recovery_attempt = 0 def snapshot(self) -> dict[str, Any]: return { @@ -79,6 +529,7 @@ class FakeVisualizationRuntime: "message": "test runtime", "source_mode": self.source_mode, "source_ready": self.source_ready, + "producer_generation": self.producer_generation, "foxglove_ws_url": None, "foxglove_viewer_url": None, "rerun_grpc_url": None, @@ -100,6 +551,17 @@ class FakeVisualizationRuntime: "decode_publish_ms": None, "trajectory_poses": 0, }, + "connection_recovery": { + "state": self.recovery_state, + "attempt": self.recovery_attempt, + "reason_code": None, + "started_at_utc": None, + "elapsed_ms": None, + "recovered_at_utc": None, + "automatic_command_retry": False, + "device_write_performed": False, + "network_mutation_performed": False, + }, } def start_live( @@ -109,64 +571,253 @@ class FakeVisualizationRuntime: *, duration_seconds: float | None, project_name: str, + recover_connection: Callable[[int], str] | None = None, ) -> None: + self.producer_generation += 1 self.start_calls.append((host, out_dir, duration_seconds, project_name)) self.phase = "starting_live" self.source_mode = "live" + self.recover_connection = recover_connection def mark_ready(self) -> None: self.phase = "live" self.source_ready = True + def request_connection_recovery( + self, + reason_code: str, + *, + expected_generation: int, + ) -> bool: + if ( + expected_generation != self.producer_generation + or self.recover_connection is None + or self.source_mode != "live" + or self.phase not in {"live", "reconnecting"} + ): + return False + if self.phase == "reconnecting": + return True + self.phase = "reconnecting" + self.source_ready = False + self.recovery_state = "reconnecting" + self.recovery_requests.append((reason_code, expected_generation)) + self.recovery_request_pending = True + return True + def stop(self) -> None: self.stop_calls += 1 + self.recovery_request_pending = False if self.stop_error is not None: raise self.stop_error self.phase = "idle" self.source_mode = "idle" self.source_ready = False + self.recovery_state = "inactive" + self.recovery_attempt = 0 def close(self) -> None: self.stop() class FakeInteractiveControlSession: - def __init__(self) -> None: - self.state = "workspace-ready" + def __init__(self, *, initial_state: str = "workspace-ready") -> None: + self.state = initial_state + self.session_generation = 1 + self.state_revision = 1 + self.verified_control: dict[str, object] | None = None self.start_projects: list[str] = [] + self.start_contexts: list[Any] = [] self.stop_calls = 0 + self.stop_contexts: list[Any] = [] + self.stop_dispatch_deadlines: list[Callable[[], bool] | None] = [] + self.inspection_only = False + self.inspection_promotion_allowed = False + self.failure: dict[str, object] | None = None + self.outcome_unknown = False + self.transport_publish_attempts = 0 def snapshot(self) -> dict[str, object]: return { "state": self.state, + "session_generation": self.session_generation, + "state_revision": self.state_revision, + "inspection_only": self.inspection_only, + "inspection_promotion_allowed": self.inspection_promotion_allowed, + "can_open": self.state in {"idle", "completed", "closed", "failed"}, + "can_enter_workspace": self.state == "connection-ready", + "can_prepare_project": self.state == "workspace-ready", + "can_start": self.state == "project-ready", + "can_stop": self.state == "scanning", "can_confirm_standby": False, + "failure": self.failure, + "outcome_unknown": self.outcome_unknown, + "transport": { + "state": "failed" if self.state == "failed" else "ready", + "publish_attempts": self.transport_publish_attempts, + }, + "verified_control": self.verified_control, } - def open_project_prompt(self) -> dict[str, object]: - assert self.state == "workspace-ready" - self.state = "project-ready" + def open( + self, + *, + connection_binding: ApplicationConnectionBinding, + inspection_only: bool = False, + **_: object, + ) -> dict[str, object]: + assert self.state == "idle" + self.verified_control = _verified_control_for_binding(connection_binding) + self.inspection_only = inspection_only + self.inspection_promotion_allowed = not inspection_only + self.state = "connection-ready" + self.state_revision += 1 return self.snapshot() - def request_start(self, *, project_name: str, confirmation: object) -> dict[str, object]: - assert confirmation is not None - assert self.state == "project-ready" - self.start_projects.append(project_name) + def release_inspection_for_operator_dialogue( + self, + *, + expected_session_generation: int, + expected_state_revision: int, + ) -> dict[str, object]: + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + assert self.state == "connection-ready" + self.inspection_promotion_allowed = True + return self.snapshot() + + def adopt_reconciled_scanning( + self, + *, + reconciliation_id: str, + expected_session_generation: int, + expected_state_revision: int, + ) -> dict[str, object]: + assert reconciliation_id + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + assert self.state == "connection-ready" self.state = "scanning" return self.snapshot() - def request_stop(self, *, confirmation: object) -> dict[str, object]: + def enter_workspace( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert self.state == "connection-ready" + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self.state = "workspace-ready" + return self.snapshot() + + def _accept_checkpoint( + self, + *, + expected_session_generation: int | None, + expected_state_revision: int | None, + ) -> None: + assert expected_session_generation == self.session_generation + assert expected_state_revision == self.state_revision + self.state_revision += 1 + + def validate_connection_binding(self) -> None: + return None + + def validate_physical_reconciliation_binding(self) -> None: + return None + + def open_project_prompt( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert self.state == "workspace-ready" + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self.state = "project-ready" + return self.snapshot() + + def request_start( + self, + *, + project_name: str, + confirmation: object, + command_context: object, + preparation_checkpoint_observer: object | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + del preparation_checkpoint_observer assert confirmation is not None + assert command_context is not None + assert self.state == "project-ready" + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self.start_projects.append(project_name) + self.start_contexts.append(command_context) + self.state = "scanning" + return self.snapshot() + + def request_stop( + self, + *, + confirmation: object, + command_context: object, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert confirmation is not None + assert command_context is not None assert self.state == "scanning" + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) self.stop_calls += 1 + self.stop_contexts.append(command_context) + self.stop_dispatch_deadlines.append(dispatch_admission_deadline_reached) self.state = "awaiting-standby-confirmation" return self.snapshot() - def close_prestart(self) -> dict[str, object]: + def close_prestart( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + if expected_session_generation is not None or expected_state_revision is not None: + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) self.state = "closed" + self.verified_control = None + return self.snapshot() + + def retire_for_network_change(self, **_: object) -> dict[str, object]: + if self.state not in {"idle", "completed", "closed", "failed"}: + raise RuntimeError("control session cannot be retired") + self.state = "idle" + self.verified_control = None return self.snapshot() def close(self) -> None: self.state = "closed" + self.verified_control = None PHYSICAL_ACCEPTANCE = OperatorPresenceRequest( @@ -178,6 +829,73 @@ PHYSICAL_ACCEPTANCE = OperatorPresenceRequest( ) +def _exact_start_physical_proof( + *, + operation_id: str, + acquisition_id: str, + binding: ApplicationConnectionBinding, + resolved: bool, +) -> dict[str, object]: + session_state = "scanning" if resolved else "scan_starting" + return { + "status": "resolved" if resolved else "unresolved", + "reason_code": (None if resolved else "physical-command-reconciliation-required"), + "requires_reconciliation": not resolved, + "resolved_active_recovery_required": resolved, + "reconciled_physical_state": "active" if resolved else None, + "physical_active": resolved, + "automatic_replay_allowed": False, + "record": { + "revision": 7, + "operation_id": operation_id, + "acquisition_id": acquisition_id, + "action": "start", + "stage": "resolved" if resolved else "observing", + "resolution": "start-active-observed" if resolved else None, + "publish_call_returned": True, + "qos2_completed": True, + "packet_id": 14, + "application_response": { + "operation_id": operation_id, + "action": "start", + "success": True, + }, + "baseline_status": { + "control_session_id": "fake-control-session", + "host_path_epoch": binding.host_path_epoch, + "producer_generation": 1, + "session_state": "ready", + "project_bound": False, + "init_ready": False, + "mqtt_retained": False, + }, + "last_status": { + "session_state": session_state, + "project_bound": resolved, + "init_ready": resolved, + "mqtt_retained": False, + "system_error_code": None, + }, + "connection": { + "intent_id": binding.intent_id, + "transport_ref": binding.transport_ref, + "connection_mode": binding.connection_mode, + "target_ipv4": binding.target_ipv4, + "target_port": binding.target_port, + "host_path_epoch": binding.host_path_epoch, + "control_session_id": "fake-control-session", + "producer_generation": 1, + }, + "identity": { + "vendor_device_id_sha256": "a" * 64, + "device_serial_sha256": "b" * 64, + }, + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + "reconciliations": [], + }, + } + + def test_control_session_transport_metadata_does_not_leak_into_confirmation() -> None: request = OpenApplicationControlSessionRequest( operator_present=True, @@ -191,15 +909,2773 @@ def test_control_session_transport_metadata_does_not_leak_into_confirmation() -> assert request.confirmation() == PHYSICAL_ACCEPTANCE.confirmation() +async def _synthetic_prestart_control_bootstrap( + service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, +) -> None: + """Keep network lifecycle tests independent from a live MQTT/Keychain.""" + + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.intent is not None + assert supervisor.device_network.target is not None + assert supervisor.device_network.transport_ref is not None + operation, _ = service._operations.begin( # noqa: SLF001 + facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP, + device_id=service._device_id, # noqa: SLF001 + device_session_id=service._device_session_id, # noqa: SLF001 + deadline_seconds=1.0, + context={ + "connection_mode": connection_mode, + "parent_operation_id": parent_operation_id, + "automatic_retry": False, + }, + ) + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="mqtt-device-info", + message_code="connection.control_bootstrap.running", + ) + control = FakeInteractiveControlSession(initial_state="idle") + binding = ApplicationConnectionBinding( + intent_id=supervisor.intent.intent_id, + transport_ref=supervisor.device_network.transport_ref, + host_path_epoch=supervisor.host_path.epoch, + target_ipv4=supervisor.device_network.target.ipv4, + target_port=supervisor.device_network.target.port, + connection_mode=connection_mode, + ) + control.open( + connection_binding=binding, + inspection_only=inspection_only, + ) + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id=str( + supervisor.intent.expected_device_id or service._device_id # noqa: SLF001 + ), + control_session_id=( + f"fake-control-{parent_operation_id}" + if parent_operation_id is not None + else "fake-control-session" + ), + ) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + service._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + service.runtime.snapshot(), + ) + if not service._connection_supervisor.snapshot().authority.control_allowed: # noqa: SLF001 + failure = facade_module.ConnectionVerificationError( + "synthetic DeviceInfo authority was not admitted", + reason_code="control-bootstrap-device-identity-unverified", + ) + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "failed", + stage_code="device-info-failed", + message_code="connection.control_bootstrap.failed", + error={ + "category": "connection", + "code": failure.reason_code, + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + raise failure + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "succeeded", + stage_code="device-info-confirmed", + message_code="connection.control_bootstrap.completed", + result={ + "connection_mode": connection_mode, + "control_verified": True, + "device_write_performed": False, + "automatic_retry": False, + }, + ) + + def service_with_fake_runtime( tmp_path: Path, ) -> tuple[XgridsK1CompatibilityService, FakeVisualizationRuntime]: - service = XgridsK1CompatibilityService(tmp_path) + # Durable network/idempotency records are intentionally global in the real + # runtime. Each test needs its own private store while repeated service + # construction inside one test must still exercise restart semantics. + os.environ["MISSIONCORE_DATA_DIR"] = str(tmp_path / "private-data") + os.environ["MISSIONCORE_EVIDENCE_DIR"] = str(tmp_path / "private-evidence") + os.environ["MISSIONCORE_LEGACY_SESSIONS_DIR"] = str(tmp_path / "private-legacy-sessions") + service = XgridsK1CompatibilityService( + tmp_path, + host_wifi_association_probe=FakeHostWifiAssociationProbe(), + ) runtime = FakeVisualizationRuntime() service.runtime = runtime # type: ignore[assignment] + # The broad legacy lifecycle fixture replaces both the real application + # session and typed physical coordinator with mapping-only fakes. It has no + # durable PREPARED callback from which checkpoint v2 can be constructed. + # Keep those older tests scoped to their original lifecycle concern; the + # typed checkpoint ordering/CAS seams have dedicated integration tests. + service._active_acquisition_checkpoint_matches_current = MethodType( # type: ignore[method-assign] # noqa: SLF001 + lambda _service, **_kwargs: True, + service, + ) + legacy_start_operation_id: list[str | None] = [None] + + def checkpoint_start_operation_id( + _service: XgridsK1CompatibilityService, + *, + local_start_operation_id: str | None, + **_kwargs: object, + ) -> str | None: + if local_start_operation_id is not None: + legacy_start_operation_id[0] = local_start_operation_id + return legacy_start_operation_id[0] + + service._active_acquisition_checkpoint_start_operation_id = MethodType( # type: ignore[method-assign] # noqa: SLF001 + checkpoint_start_operation_id, + service, + ) + service._record_active_acquisition_checkpoint_gap = MethodType( # type: ignore[method-assign] # noqa: SLF001 + lambda _service, _lineage, **_kwargs: True, + service, + ) + service._confirm_active_acquisition_checkpoint_rebind = MethodType( # type: ignore[method-assign] # noqa: SLF001 + lambda _service, *_args, **_kwargs: True, + service, + ) + service._cease_active_acquisition_checkpoint_from_physical_head = MethodType( # type: ignore[method-assign] # noqa: SLF001 + lambda _service, **_kwargs: True, + service, + ) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + _synthetic_prestart_control_bootstrap, + service, + ) return service, runtime +def test_service_owns_one_wifi_association_probe_across_monitor_recreation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + association_probe = FakeHostWifiAssociationProbe() + constructed: list[Path] = [] + + def probe_factory(helper_path: Path) -> FakeHostWifiAssociationProbe: + constructed.append(helper_path) + return association_probe + + monkeypatch.setattr( + facade_module, + "HostWifiAssociationIdentityProbe", + probe_factory, + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + service = XgridsK1CompatibilityService(tmp_path) + _seed_supervised_connection(service, target_ipv4="192.168.1.20") + + first = service._sample_host_path("192.168.1.20") # noqa: SLF001 + service._connection_monitor = service._new_connection_monitor() # noqa: SLF001 + second = asyncio.run( # noqa: SLF001 + service._monitor_host_path(EndpointTarget("192.168.1.20")) + ) + + assert constructed == [tmp_path / "plugins" / "xgrids-k1" / "macos" / "associate_wifi.swift"] + assert association_probe.interfaces == ["test0", "test0"] + assert first.fingerprint == second.fingerprint + + +@pytest.mark.parametrize( + ("raw_paths", "association_tokens"), + [ + ( + ( + _direct_host_path("192.168.1.20"), + _direct_host_path("192.168.56.20"), + ), + ("a" * 64, "a" * 64, "a" * 64), + ), + ( + ( + _direct_host_path("192.168.1.20"), + _direct_host_path("192.168.1.20"), + ), + ("a" * 64, "a" * 64, "b" * 64), + ), + ], +) +def test_sync_tcp_success_cannot_cross_route_or_wifi_association_change( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + raw_paths: tuple[HostPathProbeResult, HostPathProbeResult], + association_tokens: tuple[str, str, str], +) -> None: + association_probe = FakeHostWifiAssociationProbe(*association_tokens) + service = XgridsK1CompatibilityService( + tmp_path, + host_wifi_association_probe=association_probe, + ) + service.runtime = FakeVisualizationRuntime() # type: ignore[assignment] + monkeypatch.setattr(facade_module, "_inspect_host_path", lambda _target: raw_paths[0]) + initial_path = service._sample_host_path("192.168.1.20") # noqa: SLF001 + _seed_supervised_connection(service, host_path=initial_path) + pending_paths = iter(raw_paths) + monkeypatch.setattr( + facade_module, + "_inspect_host_path", + lambda _target: next(pending_paths), + ) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) + + observation = service._probe_control_endpoint("192.168.1.20") # noqa: SLF001 + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + + assert observation.reachable is False + assert observation.reason_code == "host-path-changed-during-tcp-probe" + assert supervisor.host_path.fingerprint == observation.path.fingerprint + assert supervisor.endpoint.tcp_state == "unreachable" + assert supervisor.authority.control_allowed is False + assert supervisor.lease.state != "reachable" + + +def _seed_supervised_connection( + service: XgridsK1CompatibilityService, + *, + target_ipv4: str = "192.168.1.20", + connection_mode: facade_module.ConnectionMode = "bridge", + transport_ref: str = "test-ble-transport", + logical_device_id: str = "known-k1", + host_path: HostPathProbeResult | None = None, + endpoint_reachable: bool = True, + endpoint_reason: str | None = None, + with_control: bool = True, +) -> ApplicationConnectionBinding: + """Install one internally consistent test-only connection evidence chain. + + Older lifecycle fixtures used ``_k1_ip`` as if an address implied a live + connection. The production supervisor now deliberately requires four + independent facts: the K1-reported topology, the current host route, TCP + reachability and DeviceInfo bound to that exact intent/route epoch. Tests + that exercise already-open plugin control must seed all four explicitly; + tests for the pre-DeviceInfo state pass ``with_control=False``. + """ + + intent_id = f"test-intent-{connection_mode}-{transport_ref}-{target_ipv4}" + target = EndpointTarget(target_ipv4, facade_module.CONTROL_MQTT_PORT) + attestation = { + "bridge": ATTESTATION, + "quick-connect": QUICK_CONNECT_ATTESTATION, + "direct-connect": DIRECT_CONNECT_ATTESTATION, + }[connection_mode] + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref=transport_ref, + logical_device_id=logical_device_id, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + service._connection_supervisor.set_intent( # noqa: SLF001 + intent_id=intent_id, + requested_mode=connection_mode, + expected_device_id=logical_device_id, + ) + accepted = service._connection_supervisor.observe_device_network_applied( # noqa: SLF001 + intent_id=intent_id, + transport_ref=transport_ref, + connection_mode=connection_mode, + target=target, + source="ble-read-only-status", + ) + assert accepted is True + host_epoch = service._connection_supervisor.observe_host_path( # noqa: SLF001 + host_path or _direct_host_path(target_ipv4) + ) + endpoint_accepted = service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=target, + intent_id=intent_id, + host_path_epoch=host_epoch, + reachable=endpoint_reachable, + reason_code=( + None if endpoint_reachable else endpoint_reason or "test-endpoint-unreachable" + ), + ) + assert endpoint_accepted is True + binding = ApplicationConnectionBinding( + intent_id=intent_id, + transport_ref=transport_ref, + host_path_epoch=host_epoch, + target_ipv4=target_ipv4, + target_port=facade_module.CONTROL_MQTT_PORT, + connection_mode=connection_mode, + ) + if with_control: + assert endpoint_reachable is True + control_accepted = service._connection_supervisor.observe_control_evidence( # noqa: SLF001 + VerifiedControlEvidence( + intent_id=intent_id, + transport_ref=transport_ref, + host_path_epoch=host_epoch, + target=target, + connection_mode=connection_mode, + logical_device_id=logical_device_id, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id=f"test-control-{transport_ref}", + ) + ) + assert control_accepted is True + + with service._lock: # noqa: SLF001 + service._selected_device_id = transport_ref # noqa: SLF001 + service._connection_mode = connection_mode # noqa: SLF001 + service._k1_ip = target_ipv4 # noqa: SLF001 + service._device_id = logical_device_id # noqa: SLF001 + service._device_session_id = f"test-session-{transport_ref}" # noqa: SLF001 + service._device_session_opened_at = "2026-07-20T12:00:00Z" # noqa: SLF001 + service._compatibility_attestation = attestation.model_dump(mode="json") # noqa: SLF001 + service._connection_lease_generation = ( # noqa: SLF001 + service._connection_supervisor.snapshot().lease.generation # noqa: SLF001 + ) + control = service._application_control_session # noqa: SLF001 + if ( + with_control + and isinstance(control, FakeInteractiveControlSession) + and control.verified_control is None + ): + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id=logical_device_id, + ) + return binding + + +def _verified_control_for_binding( + binding: ApplicationConnectionBinding, + *, + logical_device_id: str = "known-k1", + control_session_id: str = "fake-control-session", + control_proof_revision: int = 1, + control_proof_source: str = "correlated-application-response", + control_proof_fresh: bool = True, + producer_generation: int = 1, +) -> dict[str, object]: + return { + "logical_device_id": logical_device_id, + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + "control_session_id": control_session_id, + "source": "mqtt-device-info", + "intent_id": binding.intent_id, + "transport_ref": binding.transport_ref, + "host_path_epoch": binding.host_path_epoch, + "target_ipv4": binding.target_ipv4, + "target_port": binding.target_port, + "connection_mode": binding.connection_mode, + "producer_generation": producer_generation, + "control_proof_revision": control_proof_revision, + "control_proof_source": control_proof_source, + "control_proof_fresh": control_proof_fresh, + } + + +def _seed_unresolved_physical_stop_for_retirement( + service: XgridsK1CompatibilityService, + binding: ApplicationConnectionBinding, +) -> RetireUnavailablePhysicalCommandRequest: + """Persist one accepted STOP whose READY outcome was never observed.""" + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=hashlib.sha256(b"retirement-vendor").hexdigest(), + device_serial_sha256=hashlib.sha256(b"retirement-serial").hexdigest(), + ) + connection = PhysicalCommandConnectionBinding( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode=binding.connection_mode, + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id="retirement-old-control-session", + producer_generation=11, + ) + + def status( + session_state: str, + *, + observed_at_utc: str, + ) -> PhysicalCommandStatusEvidence: + scanning = session_state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state=session_state, # type: ignore[arg-type] + session_state_code=302 if scanning else 300, + project_bound=scanning, + project_id_sha256=("c" * 64 if scanning else None), + init_ready=scanning, + status_message_sha256=hashlib.sha256( + f"{session_state}:{observed_at_utc}".encode() + ).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + start_operation_id = "physical-start-before-operator-retirement" + stop_operation_id = "physical-stop-before-operator-retirement" + acquisition_id = "acquisition-before-operator-retirement" + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="d" * 64, + baseline_status=status( + "ready", + observed_at_utc="2026-08-10T16:59:00.000Z", + ), + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing( + start_operation_id, + publish_call_returned=True, + packet_id=41, + ) + ledger.mark_qos2_completed(start_operation_id, packet_id=41) + ledger.record_application_response( + start_operation_id, + PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="e" * 64, + observed_at_utc="2026-08-10T16:59:01.000Z", + ), + ) + ledger.record_status_observation( + start_operation_id, + status("scanning", observed_at_utc="2026-08-10T16:59:02.000Z"), + ) + ledger.resolve(start_operation_id, resolution="start-active-observed") + ledger.prepare( + operation_id=stop_operation_id, + parent_operation_id=start_operation_id, + acquisition_id=acquisition_id, + action="stop", + identity=identity, + connection=connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="f" * 64, + baseline_status=status( + "scanning", + observed_at_utc="2026-08-10T17:00:00.000Z", + ), + ) + ledger.mark_dispatching(stop_operation_id) + ledger.mark_observing( + stop_operation_id, + publish_call_returned=True, + packet_id=42, + ) + ledger.mark_qos2_completed(stop_operation_id, packet_id=42) + ledger.record_application_response( + stop_operation_id, + PhysicalCommandApplicationResponse( + operation_id=stop_operation_id, + action="stop", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="1" * 64, + observed_at_utc="2026-08-10T17:00:01.000Z", + ), + ) + record = ledger.snapshot().record + assert record is not None and record.unresolved + return RetireUnavailablePhysicalCommandRequest( + retirement_id="operator-retirement-stable-request", + expected_operation_id=record.operation_id, + expected_revision=record.revision, + expected_transport_ref=record.connection.transport_ref, + operator_confirmed=True, + reason="device-permanently-unavailable-or-replaced", + ) + + +@pytest.mark.parametrize( + "failure_mode", + ["stale-visible-checkpoint", "cross-process-ledger-race"], +) +def test_physical_retirement_checkpoint_conflict_is_http_409_without_device_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_mode: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + service, + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ) + current_request = _seed_unresolved_physical_stop_for_retirement( + service, + binding, + ) + request = ( + current_request.model_copy( + update={ + "expected_operation_id": "physical-stop-different-current-checkpoint", + } + ) + if failure_mode == "stale-visible-checkpoint" + else current_request + ) + before = service.state() + before_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + before_operations = service._operations.snapshot() # noqa: SLF001 + io_calls: list[str] = [] + ledger_calls: list[str] = [] + + async def forbidden_async_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("ble-or-wifi") + raise AssertionError("retirement checkpoint conflict must not reach device I/O") + + def forbidden_sync_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("mqtt-tcp-monitor") + raise AssertionError("retirement checkpoint conflict must not probe the device") + + def final_ledger_cas(*_args: object, **_kwargs: object) -> object: + ledger_calls.append("retire") + if failure_mode == "cross-process-ledger-race": + raise PhysicalCommandTransitionError( + "physical command retirement used a stale record checkpoint" + ) + raise AssertionError("visible stale checkpoint must fail before the ledger CAS") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_async_io) + monkeypatch.setattr(service, "_probe_control_endpoint", forbidden_sync_io) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "retire_unavailable_target", + final_ledger_cas, + ) + + adapter = XgridsK1PluginFacade(service) + with pytest.raises(PluginExecutionError) as raised: + asyncio.run( + adapter.invoke( + RuntimeActionInvocation( + invocation_id=f"retirement-{failure_mode}", + plugin_id=adapter.plugin_id, + action_id=ACTION_PHYSICAL_COMMAND_RETIRE_UNAVAILABLE, + requested_at=datetime.now(UTC), + parameters={ + **request.model_dump(mode="json"), + "expected_snapshot_runtime_id": before["snapshot_runtime_id"], + }, + ) + ) + ) + + assert raised.value.http_status_code == 409 + assert raised.value.reason_code == "physical-command-retirement-stale-checkpoint" + assert ledger_calls == ([] if failure_mode == "stale-visible-checkpoint" else ["retire"]) + assert service._physical_command_ledger.snapshot().record == before_record # noqa: SLF001 + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert io_calls == [] + + +def test_operator_retirement_is_local_only_revokes_old_authority_and_keeps_mode_draft( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + _set_scanned_devices( + service, + [ + { + "device_id": "retired-k1", + "name": "Original K1", + "rssi": -40, + "address": None, + "connectable": True, + "likely_k1": True, + }, + { + "device_id": "replacement-k1", + "name": "Replacement K1", + "rssi": -45, + "address": None, + "connectable": True, + "likely_k1": True, + }, + ], + ) + binding = _seed_supervised_connection( + service, + transport_ref="RETIRED-K1", + connection_mode="bridge", + ) + request = _seed_unresolved_physical_stop_for_retirement(service, binding) + with service._lock: # noqa: SLF001 + # CoreBluetooth UUID aliases compare case-insensitively for fencing, + # while the durable retirement CAS keeps its original exact spelling. + service._selected_device_id = "retired-k1" # noqa: SLF001 + _seed_terminal_network_attempt(service) + before = service.state() + before_operations = service._operations.snapshot() # noqa: SLF001 + before_generation = service._ble_discovery_generation # noqa: SLF001 + before_devices = list(service._devices) # noqa: SLF001 + io_calls: list[str] = [] + + async def forbidden_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("device-or-network-io") + raise AssertionError("operator retirement must be local-only") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_io) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda *_args, **_kwargs: io_calls.append("tcp-probe"), + ) + + assert before["physical_command"]["operator_retirement"]["allowed"] is True + retirement_action = before["connection_policy"]["actions"]["retire-unavailable-physical-target"] + assert retirement_action == { + "allowed": True, + "reason_codes": [], + "target_source": "durable-physical-command", + "required_transport_ref": "RETIRED-K1", + "requires_live_gatt_validation": False, + "physical_command_allowed": False, + "physical_outcome": "unknown", + "device_write_performed": False, + "automatic_retry": False, + } + assert before["connection_policy"]["recommended_action"] == ( + "retire-unavailable-physical-target" + ) + assert before["connection_attempt"]["safe_next_action"] == ( + "retire-unavailable-physical-target" + ) + result = service.retire_unavailable_physical_command(request) + + physical = result["physical_command"] + assert physical["record"]["resolution"] == "operator-retired-outcome-unknown" + assert physical["operator_retirement"]["physical_outcome"] == "unknown" + assert physical["operator_retirement"]["allowed"] is False + assert result["desired_connection_mode"] == "quick-connect" + assert result["selected_device_id"] is None + assert result["k1_ip"] is None + assert result["connection_lifecycle"]["active_binding"] is None + assert result["connection_policy"]["facts"]["retired_transport_refs"] == ["RETIRED-K1"] + assert result["connection_policy"]["facts"]["eligible_fresh_transport_refs"] == [ + "replacement-k1" + ] + assert ( + result["connection_policy"]["actions"]["retire-unavailable-physical-target"]["allowed"] + is False + ) + assert result["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert result["connection_attempt"]["safe_next_action"] == ("scan-select-connect") + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.intent is not None + assert supervisor.intent.intent_id == ("physical-retirement-operator-retirement-stable-request") + assert supervisor.intent.requested_mode == "quick-connect" + assert supervisor.authority.control_allowed is False + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert service._ble_discovery_generation == before_generation # noqa: SLF001 + assert service._devices == before_devices # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert io_calls == [] + + +def test_retired_exact_fresh_candidate_reopens_locally_for_explicit_verify_only( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + service, + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ) + retirement = _seed_unresolved_physical_stop_for_retirement(service, binding) + service.retire_unavailable_physical_command(retirement) + before_operations = service._operations.snapshot() # noqa: SLF001 + before_runtime_starts = list(runtime.start_calls) + before_runtime_stops = runtime.stop_calls + + # A row retained from before retirement is not fresh enough to reopen it. + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + stale = service.state()["physical_command"]["operator_reconciliation_reopen"] + assert stale["allowed"] is False + assert "physical-command-reconciliation-reopen-target-not-observed" in stale["reason_codes"] + + with service._lock: # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + before = service.state() + projection = before["physical_command"]["operator_reconciliation_reopen"] + assert projection == { + "allowed": True, + "reason_codes": [], + "expected_revision": retirement.expected_revision + 1, + "expected_retirement_id": retirement.retirement_id, + "expected_transport_ref": DURABLE_K1_UUID.lower(), + "expected_discovery_generation": service._ble_discovery_generation, # noqa: SLF001 + "expected_desired_mode": "bridge", + "expected_desired_mode_revision": 0, + "device_io_performed": False, + "automatic_retry": False, + } + io_calls: list[str] = [] + + async def forbidden_async_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("ble-or-wifi") + raise AssertionError("reopen must remain local-only") + + def forbidden_sync_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("mqtt-tcp-monitor") + raise AssertionError("reopen must remain local-only") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_async_io) + monkeypatch.setattr(service, "_probe_control_endpoint", forbidden_sync_io) + reopen_request = ReopenRetiredPhysicalCommandReconciliationRequest( + reopening_id="reopening-explicit-fresh-row", + expected_revision=projection["expected_revision"], + expected_retirement_id=projection["expected_retirement_id"], + expected_transport_ref=projection["expected_transport_ref"], + expected_discovery_generation=projection["expected_discovery_generation"], + expected_desired_mode=projection["expected_desired_mode"], + expected_desired_mode_revision=projection["expected_desired_mode_revision"], + operator_confirmed=True, + reason="device-returned-for-explicit-reconciliation", + ) + result = service.reopen_retired_physical_command_reconciliation(reopen_request) + # A lost HTTP response is retried against the durable audit, not against + # the now-consumed projection. The retry must return usable state without + # appending another reopen or reacquiring BLE/network ownership. + retried = service.reopen_retired_physical_command_reconciliation(reopen_request) + + physical = result["physical_command"] + assert physical["status"] == "unresolved" + assert physical["requires_reconciliation"] is True + assert physical["record"]["stage"] == "observing" + assert physical["record"]["resolution"] is None + assert physical["record"]["operator_retirements"][0]["retirement_id"] == ( + retirement.retirement_id + ) + assert ( + physical["record"]["operator_reconciliation_reopens"][0]["reopening_id"] + == "reopening-explicit-fresh-row" + ) + assert len(retried["physical_command"]["record"]["operator_reconciliation_reopens"]) == 1 + assert ( + retried["connection_policy"]["actions"]["observe-fresh-device-network"]["allowed"] is True + ) + assert result["snapshot_runtime_id"] == before["snapshot_runtime_id"] + assert retried["snapshot_runtime_id"] == before["snapshot_runtime_id"] + assert result["snapshot_revision"] > before["snapshot_revision"] + assert ( + service._ble_discovery_generation + == projection[ # noqa: SLF001 + "expected_discovery_generation" + ] + ) + assert result["k1_lifecycle_process_lease"] == { + "held_by_current_service": False, + "holders": [], + "process_lease_quarantined": False, + "reason_code": None, + "restart_required": False, + } + observe_action = result["connection_policy"]["actions"]["observe-fresh-device-network"] + assert observe_action["allowed"] is True + assert observe_action["required_transport_ref"] == DURABLE_K1_UUID + assert observe_action["required_connection_mode"] == "bridge" + assert result["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + assert result["connection_policy"]["actions"]["start-acquisition"]["allowed"] is False + assert result["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is False + assert result["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert result["connection_policy"]["facts"]["retired_transport_refs"] == [] + assert ( + DURABLE_K1_UUID.lower() + in result["connection_policy"]["facts"]["eligible_fresh_transport_refs"] + ) + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert runtime.start_calls == before_runtime_starts + assert runtime.stop_calls == before_runtime_stops + assert io_calls == [] + + +@pytest.mark.parametrize("observed_session_state", ["ready", "scanning"]) +def test_casefolded_reopened_uuid_uses_actual_fresh_ref_and_verify_reconciles_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + observed_session_state: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + service, + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ) + retirement = _seed_unresolved_physical_stop_for_retirement(service, binding) + service.retire_unavailable_physical_command(retirement) + + # Retirement records the durable UUID exactly as originally observed. A + # later CoreBluetooth scan may return the same UUID with different case. + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + with service._lock: # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + reopened = service.reopen_retired_physical_command_reconciliation( + ReopenRetiredPhysicalCommandReconciliationRequest( + reopening_id="reopening-casefolded-end-to-end", + expected_revision=projection["expected_revision"], + expected_retirement_id=projection["expected_retirement_id"], + expected_transport_ref=projection["expected_transport_ref"], + expected_discovery_generation=projection["expected_discovery_generation"], + expected_desired_mode=projection["expected_desired_mode"], + expected_desired_mode_revision=projection["expected_desired_mode_revision"], + operator_confirmed=True, + reason="device-returned-for-explicit-reconciliation", + ) + ) + observation = reopened["connection_policy"]["actions"]["observe-fresh-device-network"] + assert observation["allowed"] is True + assert observation["required_transport_ref"] == DURABLE_K1_UUID + + actual_ref = DURABLE_K1_UUID.lower() + capture = _SYNTHETIC_SCAN_CAPTURES[actual_ref] + capture_calls: list[str] = [] + status_reads: list[str] = [] + forbidden_mutations: list[str] = [] + + def capture_actual(device_id: str) -> facade_module.CapturedDiscoveredDevice | None: + capture_calls.append(device_id) + return capture if device_id == actual_ref else None + + async def read_status( + device_id: str, + *, + captured_device: facade_module.CapturedDiscoveredDevice | None, + on_gatt_validated: Callable[[facade_module.CapturedDiscoveredDevice], None], + **_: object, + ) -> dict[str, Any]: + status_reads.append(device_id) + assert captured_device is capture + on_gatt_validated(capture) + return _wifi_status_read("10.255.254.77", device_id=device_id) + + async def forbidden_provision(*_: object, **__: object) -> object: + forbidden_mutations.append("wifi") + raise AssertionError("explicit Verify must not provision Wi-Fi") + + monkeypatch.setattr(facade_module, "capture_discovered_device", capture_actual) + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "direct-or-routed", + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + coordinator = service._physical_command_coordinator # noqa: SLF001 + + async def bootstrap_ready_proof( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:casefold:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="9" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-10T12:00:00.000Z", + ) + ) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=hashlib.sha256(b"retirement-vendor").hexdigest(), + device_serial_sha256=hashlib.sha256(b"retirement-serial").hexdigest(), + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=hashlib.sha256(b"retirement-vendor").hexdigest(), + device_serial_sha256=hashlib.sha256(b"retirement-serial").hexdigest(), + session_state=observed_session_state, + session_state_code=( + MODELING_STATE_BASE + (300 if observed_session_state == "ready" else 302) + ), + project_bound=observed_session_state == "scanning", + project_id_sha256=("5" * 64 if observed_session_state == "scanning" else None), + init_ready=observed_session_state == "scanning", + status_message_sha256="8" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-10T12:00:01.000Z", + ) + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_ready_proof, + service, + ) + result = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + # The frontend follows the server policy, whose durable + # physical recovery target retains the original uppercase + # spelling. Backend lookup must resolve that to the current + # lowercase advert before any native BLE call. + device_id=observation["required_transport_ref"], + source="fresh-scan", + compatibility_attestation=ATTESTATION, + operation_id="op-00000000-0000-4000-8000-000000001240", + expected_discovery_generation=projection["expected_discovery_generation"], + ) + ) + ) + + assert capture_calls == [actual_ref] + assert status_reads == [actual_ref] + assert result["selected_device_id"] == actual_ref + assert result["physical_command"]["record"]["resolution"] == ( + "physical-standby-observed" + if observed_session_state == "ready" + else "physical-active-observed" + ) + assert ( + result["physical_command"]["record"]["operator_retirements"][0]["retired_transport_ref"] + == DURABLE_K1_UUID + ) + assert ( + result["physical_command"]["record"]["operator_reconciliation_reopens"][0][ + "reopened_transport_ref" + ] + == actual_ref + ) + assert forbidden_mutations == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + if observed_session_state == "scanning": + assert result["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + assert result["connection_policy"]["actions"]["start-acquisition"]["allowed"] is False + assert result["application_control_session"]["can_stop"] is True + assert result["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is True, ( + result["connection_policy"]["actions"]["stop-acquisition"] + ) + assert result["acquisition"]["result"]["recovery_only"] is True + + +def _retired_fresh_reopen_request( + service: XgridsK1CompatibilityService, + *, + reopening_id: str, + connection_mode: facade_module.ConnectionMode = "bridge", +) -> ReopenRetiredPhysicalCommandReconciliationRequest: + if connection_mode != "bridge": + _select_connection_mode(service, connection_mode) + binding = _seed_supervised_connection( + service, + transport_ref=DURABLE_K1_UUID, + connection_mode=connection_mode, + ) + retirement = _seed_unresolved_physical_stop_for_retirement(service, binding) + service.retire_unavailable_physical_command(retirement) + with service._lock: # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is True + return ReopenRetiredPhysicalCommandReconciliationRequest( + reopening_id=reopening_id, + expected_revision=projection["expected_revision"], + expected_retirement_id=projection["expected_retirement_id"], + expected_transport_ref=projection["expected_transport_ref"], + expected_discovery_generation=projection["expected_discovery_generation"], + expected_desired_mode=projection["expected_desired_mode"], + expected_desired_mode_revision=projection["expected_desired_mode_revision"], + operator_confirmed=True, + reason="device-returned-for-explicit-reconciliation", + ) + + +def test_physical_reopen_final_checkpoint_conflict_is_http_409_without_device_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-cross-process-ledger-race", + ) + before = service.state() + before_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + before_operations = service._operations.snapshot() # noqa: SLF001 + io_calls: list[str] = [] + ledger_calls: list[str] = [] + + async def forbidden_async_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("ble-or-wifi") + raise AssertionError("reopen checkpoint conflict must not reach device I/O") + + def forbidden_sync_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("mqtt-tcp-monitor") + raise AssertionError("reopen checkpoint conflict must not probe the device") + + def conflicting_final_cas(*_args: object, **_kwargs: object) -> object: + ledger_calls.append("reopen") + raise PhysicalCommandTransitionError( + "physical command reconciliation reopen used a stale record revision" + ) + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_async_io) + monkeypatch.setattr(service, "_probe_control_endpoint", forbidden_sync_io) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reopen_retired_reconciliation", + conflicting_final_cas, + ) + + adapter = XgridsK1PluginFacade(service) + with pytest.raises(PluginExecutionError) as raised: + asyncio.run( + adapter.invoke( + RuntimeActionInvocation( + invocation_id="reopening-cross-process-ledger-race", + plugin_id=adapter.plugin_id, + action_id=(ACTION_PHYSICAL_COMMAND_REOPEN_RETIRED_RECONCILIATION), + requested_at=datetime.now(UTC), + parameters={ + **request.model_dump(mode="json"), + "expected_snapshot_runtime_id": before["snapshot_runtime_id"], + }, + ) + ) + ) + + assert raised.value.http_status_code == 409 + assert raised.value.reason_code == ("physical-command-reconciliation-reopen-stale-checkpoint") + assert ledger_calls == ["reopen"] + assert service._physical_command_ledger.snapshot().record == before_record # noqa: SLF001 + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert io_calls == [] + + +def test_reopen_projection_rejects_a_different_local_mode_draft_without_mutation( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-mode-mismatch", + ) + before_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + before_operations = service._operations.snapshot() # noqa: SLF001 + before_generation = service._ble_discovery_generation # noqa: SLF001 + + switched_revision = _select_connection_mode(service, "quick-connect") + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + + assert switched_revision == 1 + assert projection["allowed"] is False + assert projection["reason_codes"] == ["physical-command-reconciliation-reopen-mode-mismatch"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + + assert blocked.value.reason_code == ("physical-command-reconciliation-reopen-mode-mismatch") + assert service._physical_command_ledger.snapshot().record == before_record # noqa: SLF001 + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert service._ble_discovery_generation == before_generation # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_reopen_rejects_desired_mode_revision_aba_without_consuming_checkpoint( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + stale_request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-mode-revision-aba", + ) + before_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + before_operations = service._operations.snapshot() # noqa: SLF001 + + _select_connection_mode(service, "quick-connect") + current_revision = _select_connection_mode(service, "bridge") + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + + assert current_revision == 2 + assert projection["allowed"] is True + assert projection["expected_desired_mode"] == "bridge" + assert projection["expected_desired_mode_revision"] == 2 + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale: + service.reopen_retired_physical_command_reconciliation(stale_request) + + assert stale.value.reason_code == ("physical-command-reconciliation-reopen-stale-checkpoint") + assert service._physical_command_ledger.snapshot().record == before_record # noqa: SLF001 + assert service._operations.snapshot() == before_operations # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_reopen_freezes_mode_draft_until_physical_observation_terminalizes( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-freezes-mode-draft", + ) + reopened = service.reopen_retired_physical_command_reconciliation(request) + reopened_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + + assert reopened["physical_command"]["requires_reconciliation"] is True + assert reopened["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert ( + "connection-mode-selection-physical-state-unsafe" + in reopened["connection_lifecycle"]["mode_selection"]["reason_codes"] + ) + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + _select_connection_mode(service, "quick-connect") + + assert blocked.value.reason_code == ("connection-mode-selection-physical-state-unsafe") + assert service._physical_command_ledger.snapshot().record == reopened_record # noqa: SLF001 + assert service.state()["desired_connection_mode"] == "bridge" + assert service.state()["desired_connection_mode_revision"] == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_restarted_peer_can_align_only_to_pending_reopen_mode_without_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + owner, _ = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + owner, + reopening_id="reopening-quick-restart-mode-alignment", + connection_mode="quick-connect", + ) + owner.reopen_retired_physical_command_reconciliation(request) + + restarted, restarted_runtime = service_with_fake_runtime(tmp_path) + before_record = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + device_edges: list[str] = [] + monkeypatch.setattr( + facade_module, + "scan", + lambda *_args, **_kwargs: device_edges.append("scan"), + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: device_edges.append("network-write"), + ) + + before = restarted.state() + assert before["desired_connection_mode"] == "bridge" + assert before["connection_lifecycle"]["mode_selection"]["allowed"] is True + aligned = restarted.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=before["desired_connection_mode_revision"], + ) + ) + + assert aligned["desired_connection_mode"] == "quick-connect" + assert aligned["desired_connection_mode_revision"] == 1 + assert aligned["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert restarted._physical_command_ledger.snapshot().record == before_record # noqa: SLF001 + assert restarted_runtime.start_calls == [] + assert restarted_runtime.stop_calls == 0 + assert device_edges == [] + + +def _install_real_retirement_ready_bootstrap( + service: XgridsK1CompatibilityService, +) -> None: + """Feed exact fresh DeviceInfo/READY evidence to the real coordinator.""" + + coordinator = service._physical_command_coordinator # noqa: SLF001 + + async def bootstrap_ready_proof( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + vendor_hash = hashlib.sha256(b"retirement-vendor").hexdigest() + serial_hash = hashlib.sha256(b"retirement-serial").hexdigest() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:retirement:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="7" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-10T12:10:00.000Z", + ) + ) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=vendor_hash, + device_serial_sha256=serial_hash, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=vendor_hash, + device_serial_sha256=serial_hash, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="6" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-10T12:10:01.000Z", + ) + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_ready_proof, + service, + ) + + +def test_reopened_verify_network_unavailable_stays_unresolved_and_can_retry_exactly( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + reopen_request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-network-unavailable-retry", + ) + reopened = service.reopen_retired_physical_command_reconciliation(reopen_request) + observation = reopened["connection_policy"]["actions"]["observe-fresh-device-network"] + durable_ref = str(observation["required_transport_ref"]) + actual_ref = reopen_request.expected_transport_ref + capture = _SYNTHETIC_SCAN_CAPTURES[actual_ref] + observed_addresses: list[str | None] = [None] + status_reads: list[str] = [] + forbidden_mutations: list[str] = [] + + monkeypatch.setattr( + facade_module, + "capture_discovered_device", + lambda device_id: capture if device_id == actual_ref else None, + ) + + async def read_status( + device_id: str, + *, + captured_device: facade_module.CapturedDiscoveredDevice | None, + on_gatt_validated: Callable[[facade_module.CapturedDiscoveredDevice], None], + **_: object, + ) -> dict[str, Any]: + status_reads.append(device_id) + assert captured_device is capture + on_gatt_validated(capture) + return _wifi_status_read(observed_addresses[0], device_id=device_id) + + async def forbidden_provision(*_: object, **__: object) -> object: + forbidden_mutations.append("wifi") + raise AssertionError("network-unavailable Verify must not provision") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "direct-or-routed", + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + with pytest.raises(facade_module.ConnectionVerificationError) as unavailable: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id=durable_ref, + source="fresh-scan", + compatibility_attestation=ATTESTATION, + operation_id="op-00000000-0000-4000-8000-000000001241", + expected_discovery_generation=(reopen_request.expected_discovery_generation), + ) + ) + ) + + assert unavailable.value.reason_code == "connection-verify-address-unavailable" + failed = service.state() + assert failed["connection_verification"]["status"] == "unreachable" + assert failed["connection_verification"]["network_reachability"] == ("unreachable") + assert failed["connection_verification"]["reason_code"] == ( + "connection-verify-address-unavailable" + ) + assert failed["physical_command"]["status"] == "unresolved" + assert failed["physical_command"]["requires_reconciliation"] is True + assert failed["connection_policy"]["facts"]["retired_transport_refs"] == [] + assert failed["physical_command"]["operator_retirement"]["allowed"] is True + assert failed["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + assert failed["connection_policy"]["actions"]["start-acquisition"]["allowed"] is False + assert failed["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is False + # The disconnected selector was freely editable before this exact reopen. + # The admitted reopen now freezes that mode CAS until the immediately owned + # read-only observation terminalizes, so another tab cannot strand it. + assert failed["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert ( + "connection-mode-selection-physical-state-unsafe" + in failed["connection_lifecycle"]["mode_selection"]["reason_codes"] + ) + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + + # Restoring the old route does not replay anything. The operator can + # explicitly run the same server-bound observation against the still-fresh + # exact candidate and reconcile READY. + observed_addresses[0] = "10.255.254.77" + _install_real_retirement_ready_bootstrap(service) + recovered = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id=durable_ref, + source="fresh-scan", + compatibility_attestation=ATTESTATION, + operation_id="op-00000000-0000-4000-8000-000000001242", + expected_discovery_generation=(reopen_request.expected_discovery_generation), + ) + ) + ) + + assert status_reads == [actual_ref, actual_ref] + assert recovered["physical_command"]["record"]["resolution"] == ("physical-standby-observed") + assert forbidden_mutations == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + ("blocker", "reason_code"), + [ + ("network", "network-provision-operation-active"), + ("control", "control-session-not-admissible-for-network-change"), + ("lifecycle", "k1-lifecycle-process-lease-control-owned"), + ("ble", "ble-runtime-busy"), + ( + "nonconnectable", + "physical-command-reconciliation-reopen-target-not-connectable", + ), + ], +) +def test_reopen_projection_blocks_live_owners_before_durable_or_device_action( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + blocker: str, + reason_code: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id=f"reopening-blocked-{blocker}", + ) + before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert before is not None + native = dict(facade_module.ble_runtime_snapshot()) + if blocker == "network": + service._provisioning_active = True # noqa: SLF001 + elif blocker == "control": + service._application_control_session = FakeInteractiveControlSession( # type: ignore[assignment] # noqa: SLF001 + initial_state="workspace-ready" + ) + elif blocker == "lifecycle": + service._application_control_process_lease_holders.add("control") # noqa: SLF001 + elif blocker == "ble": + native["active_operation_kind"] = "scan" + monkeypatch.setattr(facade_module, "ble_runtime_snapshot", lambda: native) + elif blocker == "nonconnectable": + service._devices[0]["connectable"] = False # noqa: SLF001 + else: # pragma: no cover - exhaustive test table + raise AssertionError(blocker) + + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is False + assert reason_code in projection["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + + assert blocked.value.reason_code == reason_code + assert service._physical_command_ledger.snapshot().record == before # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize("safe_to_retry", [False, True]) +def test_reopen_uses_the_same_failed_control_admission_as_fresh_verify( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + safe_to_retry: bool, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id=f"reopening-failed-control-{safe_to_retry}", + ) + before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + control = FakeInteractiveControlSession(initial_state="failed") + base_snapshot = control.snapshot + + def failed_snapshot() -> dict[str, object]: + return { + **base_snapshot(), + "failure": { + "safe_to_retry": safe_to_retry, + "network_change_admissible": False, + }, + } + + monkeypatch.setattr(control, "snapshot", failed_snapshot) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + + if not safe_to_retry: + assert projection["allowed"] is False + assert "control-session-not-admissible-for-network-change" in projection["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + assert blocked.value.reason_code == ("control-session-not-admissible-for-network-change") + assert service._physical_command_ledger.snapshot().record == before # noqa: SLF001 + else: + assert projection["allowed"] is True + + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + ("blocker", "reason_code"), + [ + ("supervisor", "connection-supervisor-closed"), + ("idempotency-unavailable", "network-provisioning-idempotency-unavailable"), + ("idempotency-corrupt", "network-provisioning-idempotency-corrupt"), + ("topology-corrupt", "semantic-topology-store-corrupt"), + ("identity-corrupt", "device-identity-pin-store-corrupt"), + ("network-ledger-corrupt", "network-mutation-ledger-corrupt"), + ("foreign-reconciliation", "reconciliation-target-not-observed"), + ], +) +def test_reopen_rejects_unchanged_post_reopen_verify_blockers_before_commit( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + blocker: str, + reason_code: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id=f"reopening-environment-blocked-{blocker}", + ) + before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + if blocker == "supervisor": + monkeypatch.setattr( + service._connection_supervisor, # noqa: SLF001 + "snapshot", + lambda: SimpleNamespace(closed=True), + ) + elif blocker == "idempotency-unavailable": + service._network_provisioning_idempotency_journal = None # noqa: SLF001 + elif blocker == "idempotency-corrupt": + monkeypatch.setattr( + service, + "_network_provisioning_idempotency_public_snapshot", + lambda: {"status": "corrupt"}, + ) + elif blocker == "topology-corrupt": + monkeypatch.setattr( + service, + "_semantic_topology_public_snapshot", + lambda: {"status": "corrupt"}, + ) + elif blocker == "identity-corrupt": + monkeypatch.setattr( + service, + "_device_identity_pin_public_snapshot", + lambda: {"status": "corrupt"}, + ) + elif blocker == "network-ledger-corrupt": + monkeypatch.setattr( + service._network_mutation_ledger, # noqa: SLF001 + "snapshot", + lambda: SimpleNamespace(status="corrupt", record=None), + ) + elif blocker == "foreign-reconciliation": + monkeypatch.setattr( + service, + "_network_provisioning_idempotency_public_snapshot", + lambda: { + "status": "blocked", + "active_operation_id": "foreign-network-operation", + "active_action": facade_module.ACTION_NETWORK_PROVISION, + "active_stage": "unresolved", + }, + ) + monkeypatch.setattr( + service._network_mutation_ledger, # noqa: SLF001 + "snapshot", + lambda: SimpleNamespace( + status="unresolved", + record=SimpleNamespace( + operation_id="foreign-network-operation", + stage="dispatching", + resolution=None, + transport_ref="foreign-transport", + intended_mode="bridge", + ), + ), + ) + else: # pragma: no cover - exhaustive parameter table + raise AssertionError(blocker) + + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is False + assert reason_code in projection["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + + assert blocked.value.reason_code == reason_code + assert service._physical_command_ledger.snapshot().record == before # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + ("audit_shape", "reason_code"), + [ + ( + "different-unresolved-operation", + "network-provisioning-idempotency-operation-mismatch", + ), + ( + "unresolved-ledger-without-idempotency", + "network-provisioning-idempotency-operation-mismatch", + ), + ( + "active-idempotency-without-ledger", + "network-provisioning-idempotency-ledger-mismatch", + ), + ( + "same-operation-prepared-prepared", + "network-provisioning-idempotency-ledger-mismatch", + ), + ], +) +def test_reopen_rejects_unexecutable_network_audit_pairing_before_lease_or_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + audit_shape: str, + reason_code: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id=f"reopening-audit-blocked-{audit_shape}", + ) + operation_id = f"network-audit-{audit_shape}" + if audit_shape == "different-unresolved-operation": + _seed_unresolved_network_mutation( + service, + operation_id=operation_id, + idempotency_operation_id=f"{operation_id}-other", + transport_ref=request.expected_transport_ref, + ) + elif audit_shape == "unresolved-ledger-without-idempotency": + prepared = _prepare_network_mutation_for_reopen( + service, + operation_id=operation_id, + transport_ref=request.expected_transport_ref, + intended_mode="bridge", + ) + service._network_mutation_ledger.mark_dispatching( # noqa: SLF001 + operation_id, + expected_revision=prepared.revision, + ) + elif audit_shape == "active-idempotency-without-ledger": + _begin_network_idempotency_for_reopen( + service, + operation_id=operation_id, + mark_unresolved=True, + ) + elif audit_shape == "same-operation-prepared-prepared": + _prepare_network_mutation_for_reopen( + service, + operation_id=operation_id, + transport_ref=request.expected_transport_ref, + intended_mode="bridge", + ) + _begin_network_idempotency_for_reopen( + service, + operation_id=operation_id, + mark_unresolved=False, + ) + else: # pragma: no cover - exhaustive parameter table + raise AssertionError(audit_shape) + + physical_before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + network_before = service._network_mutation_ledger.snapshot() # noqa: SLF001 + idempotency_before = ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() + ) + operations_before = service._operations.snapshot() # noqa: SLF001 + lease_calls: list[str] = [] + io_calls: list[str] = [] + + def forbidden_lease(*_args: object, **_kwargs: object) -> None: + lease_calls.append("lease") + raise AssertionError("rejected reopen must not acquire the lifecycle lease") + + async def forbidden_async_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("device-io") + raise AssertionError("rejected reopen must not perform device I/O") + + monkeypatch.setattr(service, "_acquire_k1_lifecycle_process_lease", forbidden_lease) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_async_io) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda *_args, **_kwargs: io_calls.append("endpoint-io"), + ) + + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is False + assert reason_code in projection["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + + assert blocked.value.reason_code == reason_code + assert service._physical_command_ledger.snapshot().record == physical_before # noqa: SLF001 + assert service._network_mutation_ledger.snapshot() == network_before # noqa: SLF001 + assert ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() == idempotency_before + ) + assert service._operations.snapshot() == operations_before # noqa: SLF001 + assert lease_calls == [] + assert io_calls == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_reopen_rejects_unresolved_network_mode_conflict_before_lease_or_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-network-mode-conflict", + ) + _seed_unresolved_network_mutation( + service, + operation_id="network-mode-conflict", + transport_ref=request.expected_transport_ref.upper(), + intended_mode="quick-connect", + ) + physical_before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + network_before = service._network_mutation_ledger.snapshot() # noqa: SLF001 + idempotency_before = ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() + ) + lease_calls: list[str] = [] + io_calls: list[str] = [] + + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda *_args, **_kwargs: lease_calls.append("lease"), + ) + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + lambda *_args, **_kwargs: io_calls.append("gatt-read"), + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: io_calls.append("gatt-write"), + ) + + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is False + assert "reconciliation-target-mode-mismatch" in projection["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.reopen_retired_physical_command_reconciliation(request) + + assert blocked.value.reason_code == "reconciliation-target-mode-mismatch" + assert service._physical_command_ledger.snapshot().record == physical_before # noqa: SLF001 + assert service._network_mutation_ledger.snapshot() == network_before # noqa: SLF001 + assert ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() == idempotency_before + ) + assert lease_calls == [] + assert io_calls == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + "audit_shape", + [ + "prepared-without-idempotency", + "prepared-with-matching-unresolved-idempotency", + "resolved-not-dispatched-with-matching-unresolved-idempotency", + "resolved-target-observed-with-matching-unresolved-idempotency", + ], +) +def test_reopen_admits_network_audit_shapes_verify_can_repair_exactly( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + audit_shape: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id=f"reopening-repairable-{audit_shape}", + ) + operation_id = f"network-repairable-{audit_shape}" + prepared = _prepare_network_mutation_for_reopen( + service, + operation_id=operation_id, + transport_ref=request.expected_transport_ref, + # PREPARED is deterministically closed as not-dispatched before Verify + # chooses the physical recovery mode, so this difference is repairable. + intended_mode=("quick-connect" if audit_shape.startswith("prepared-") else "bridge"), + ) + if "matching-unresolved-idempotency" in audit_shape: + _begin_network_idempotency_for_reopen( + service, + operation_id=operation_id, + mark_unresolved=True, + ) + if audit_shape.startswith("resolved-not-dispatched"): + service._network_mutation_ledger.resolve( # noqa: SLF001 + operation_id, + expected_revision=prepared.revision, + resolution="not-dispatched", + ) + elif audit_shape.startswith("resolved-target-observed"): + dispatching = service._network_mutation_ledger.mark_dispatching( # noqa: SLF001 + operation_id, + expected_revision=prepared.revision, + ) + service._network_mutation_ledger.resolve( # noqa: SLF001 + operation_id, + expected_revision=dispatching.revision, + resolution="target-observed", + observation=NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.51", + status_code=1, + reserved=0, + ), + ) + + network_before = service._network_mutation_ledger.snapshot() # noqa: SLF001 + idempotency_before = ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() + ) + io_calls: list[str] = [] + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + lambda *_args, **_kwargs: io_calls.append("gatt-read"), + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: io_calls.append("gatt-write"), + ) + + projection = service._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert projection["allowed"] is True, projection + reopened = service.reopen_retired_physical_command_reconciliation(request) + + assert reopened["physical_command"]["requires_reconciliation"] is True + assert ( + reopened["connection_policy"]["actions"]["observe-fresh-device-network"]["allowed"] is True + ) + assert service._network_mutation_ledger.snapshot() == network_before # noqa: SLF001 + assert ( # noqa: SLF001 + service._require_network_provisioning_idempotency_journal().snapshot() == idempotency_before + ) + assert io_calls == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_fresh_observation_requires_network_and_physical_recovery_same_target( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices( + service, + [ + {"device_id": "physical-k1", "connectable": True}, + {"device_id": "network-k1", "connectable": True}, + ], + ) + _seed_unresolved_network_mutation( + service, + operation_id="network-other-fresh-target", + transport_ref="network-k1", + intended_mode="bridge", + ) + + policy = _project_connection_policy_for_test( + service, + physical_command=_resolved_active_physical_snapshot( + transport_ref="physical-k1", + connection_mode="bridge", + ), + ) + + observation = policy["actions"]["observe-fresh-device-network"] + assert observation["allowed"] is False + assert "reconciliation-target-physical-recovery-mismatch" in observation["reason_codes"] + assert observation["required_transport_ref"] == "physical-k1" + + +def test_reopen_rejects_discovery_generation_drift_before_durable_commit( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + service, + reopening_id="reopening-stale-discovery-generation", + ) + before = service._physical_command_ledger.snapshot().record # noqa: SLF001 + with service._lock: # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id=DURABLE_K1_UUID.lower()) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale: + service.reopen_retired_physical_command_reconciliation(request) + + assert stale.value.reason_code == ("physical-command-reconciliation-reopen-stale-checkpoint") + assert service._physical_command_ledger.snapshot().record == before # noqa: SLF001 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_reopen_two_tab_lost_response_retry_uses_one_durable_audit( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + first, + reopening_id="reopening-two-tab-lost-response", + ) + second, second_runtime = service_with_fake_runtime(tmp_path) + with second._lock: # noqa: SLF001 + second._ble_discovery_generation = request.expected_discovery_generation # noqa: SLF001 + _set_scanned_k1(second, device_id=request.expected_transport_ref) + + committed = first.reopen_retired_physical_command_reconciliation(request) + monkeypatch.setattr( + second, + "_acquire_k1_lifecycle_process_lease", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("durable lost-response retry must not reacquire the lease") + ), + ) + observed = second.reopen_retired_physical_command_reconciliation(request) + + assert committed["physical_command"]["status"] == "unresolved" + reopens = observed["physical_command"]["record"]["operator_reconciliation_reopens"] + assert [item["reopening_id"] for item in reopens] == ["reopening-two-tab-lost-response"] + assert ( + observed["connection_policy"]["actions"]["observe-fresh-device-network"]["allowed"] is True + ) + assert second_runtime.start_calls == [] + assert second_runtime.stop_calls == 0 + + +def test_reopen_quick_restart_retry_is_durable_and_does_not_require_local_draft( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + owner, _ = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + owner, + reopening_id="reopening-quick-restart-lost-response", + connection_mode="quick-connect", + ) + owner.reopen_retired_physical_command_reconciliation(request) + + restarted, restarted_runtime = service_with_fake_runtime(tmp_path) + assert restarted.state()["desired_connection_mode"] == "bridge" + monkeypatch.setattr( + restarted, + "_acquire_k1_lifecycle_process_lease", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("durable reopen retry must not reacquire the lease") + ), + ) + + observed = restarted.reopen_retired_physical_command_reconciliation(request) + after_release = restarted.state() + + assert observed["desired_connection_mode"] == "bridge" + assert after_release["connection_lifecycle"]["mode_selection"]["allowed"] is True + assert [ + item["reopening_id"] + for item in observed["physical_command"]["record"]["operator_reconciliation_reopens"] + ] == ["reopening-quick-restart-lost-response"] + assert restarted_runtime.start_calls == [] + assert restarted_runtime.stop_calls == 0 + + +def test_reopen_cross_process_lifecycle_owner_blocks_before_ledger_commit( + tmp_path: Path, +) -> None: + owner, _ = service_with_fake_runtime(tmp_path) + request = _retired_fresh_reopen_request( + owner, + reopening_id="reopening-cross-process-owner", + ) + actor, actor_runtime = service_with_fake_runtime(tmp_path) + with actor._lock: # noqa: SLF001 + actor._ble_discovery_generation = request.expected_discovery_generation # noqa: SLF001 + _set_scanned_k1(actor, device_id=request.expected_transport_ref) + assert ( + actor._physical_operator_reconciliation_reopen_projection()[ # noqa: SLF001 + "allowed" + ] + is False + ) + with actor._lock: # noqa: SLF001 + actor._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(actor, device_id=request.expected_transport_ref) + actor_projection = actor._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + assert actor_projection["allowed"] is True + actor_request = ReopenRetiredPhysicalCommandReconciliationRequest( + reopening_id=request.reopening_id, + expected_revision=actor_projection["expected_revision"], + expected_retirement_id=actor_projection["expected_retirement_id"], + expected_transport_ref=actor_projection["expected_transport_ref"], + expected_discovery_generation=actor_projection["expected_discovery_generation"], + expected_desired_mode=actor_projection["expected_desired_mode"], + expected_desired_mode_revision=actor_projection["expected_desired_mode_revision"], + operator_confirmed=True, + reason=request.reason, + ) + before = actor._physical_command_ledger.snapshot().record # noqa: SLF001 + + owner._acquire_k1_lifecycle_process_lease("control") # noqa: SLF001 + try: + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + actor.reopen_retired_physical_command_reconciliation(actor_request) + finally: + owner._release_k1_lifecycle_process_lease("control") # noqa: SLF001 + + assert actor._physical_command_ledger.snapshot().record == before # noqa: SLF001 + assert actor._application_control_process_lease_holders == set() # noqa: SLF001 + assert actor_runtime.start_calls == [] + assert actor_runtime.stop_calls == 0 + + +def test_peer_requires_a_new_scan_after_observing_shared_physical_retirement( + tmp_path: Path, +) -> None: + owner, owner_runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + owner, + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ) + retirement = _seed_unresolved_physical_stop_for_retirement(owner, binding) + + peer, peer_runtime = service_with_fake_runtime(tmp_path) + with peer._lock: # noqa: SLF001 + peer._ble_discovery_generation += 1 # noqa: SLF001 + stale_generation = peer._ble_discovery_generation # noqa: SLF001 + _set_scanned_k1(peer, device_id=DURABLE_K1_UUID.lower()) + + owner.retire_unavailable_physical_command(retirement) + retired_before = peer._physical_command_ledger.snapshot().record # noqa: SLF001 + stale = peer._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + + assert stale["allowed"] is False + assert "physical-command-reconciliation-reopen-target-not-observed" in stale["reason_codes"] + assert ( # noqa: SLF001 + peer._physical_retirement_reopen_generation_floors[retirement.retirement_id] + == stale_generation + ) + assert peer._physical_command_ledger.snapshot().record == retired_before # noqa: SLF001 + + with peer._lock: # noqa: SLF001 + peer._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(peer, device_id=DURABLE_K1_UUID.lower()) + fresh = peer._physical_operator_reconciliation_reopen_projection() # noqa: SLF001 + + assert fresh["allowed"] is True, fresh + reopened = peer.reopen_retired_physical_command_reconciliation( + ReopenRetiredPhysicalCommandReconciliationRequest( + reopening_id="reopening-after-peer-post-retirement-scan", + expected_revision=fresh["expected_revision"], + expected_retirement_id=fresh["expected_retirement_id"], + expected_transport_ref=fresh["expected_transport_ref"], + expected_discovery_generation=fresh["expected_discovery_generation"], + expected_desired_mode=fresh["expected_desired_mode"], + expected_desired_mode_revision=fresh["expected_desired_mode_revision"], + operator_confirmed=True, + reason="device-returned-for-explicit-reconciliation", + ) + ) + + assert reopened["physical_command"]["requires_reconciliation"] is True + assert owner_runtime.start_calls == [] + assert owner_runtime.stop_calls == 0 + assert peer_runtime.start_calls == [] + assert peer_runtime.stop_calls == 0 + + +def test_exact_retirement_retry_repairs_local_revocation_without_process_lease_or_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service, transport_ref="retired-k1") + request = _seed_unresolved_physical_stop_for_retirement(service, binding) + real_revoke = service._revoke_retired_physical_target_locally # noqa: SLF001 + + def fail_after_durable_commit(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("injected crash after ledger commit") + + monkeypatch.setattr( + service, + "_revoke_retired_physical_target_locally", + fail_after_durable_commit, + ) + with pytest.raises(RuntimeError, match="injected crash"): + service.retire_unavailable_physical_command(request) + committed = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert committed is not None + assert committed.resolution == "operator-retired-outcome-unknown" + assert service._selected_device_id == "retired-k1" # noqa: SLF001 + + # A replacement backend may legitimately advance the shared ledger before + # the original browser retries its lost retirement response. The carried + # retirement audit, not the current operation, is the idempotency key. + replacement_identity = PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ) + replacement_connection = PhysicalCommandConnectionBinding( + intent_id="replacement-intent", + transport_ref="replacement-k1", + connection_mode="bridge", + target_ipv4="192.168.1.21", + target_port=facade_module.CONTROL_MQTT_PORT, + host_path_epoch=21, + control_session_id="replacement-control-session", + producer_generation=22, + ) + replacement_status = PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=replacement_identity.vendor_device_id_sha256, + device_serial_sha256=replacement_identity.device_serial_sha256, + control_session_id=replacement_connection.control_session_id, + host_path_epoch=replacement_connection.host_path_epoch, + producer_generation=replacement_connection.producer_generation, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="9" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-10T17:01:00.000Z", + ) + replacement = service._physical_command_ledger.prepare( # noqa: SLF001 + operation_id="replacement-start-after-retirement", + parent_operation_id=committed.operation_id, + acquisition_id="replacement-acquisition-after-retirement", + action="start", + identity=replacement_identity, + connection=replacement_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="2" * 64, + baseline_status=replacement_status, + ) + advanced = service._physical_command_ledger.resolve( # noqa: SLF001 + replacement.operation_id, + resolution="not-dispatched", + ) + + monkeypatch.setattr(service, "_revoke_retired_physical_target_locally", real_revoke) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("exact retry must not acquire the shared process lease") + ), + ) + repaired = service.retire_unavailable_physical_command(request) + + persisted = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert persisted == advanced + assert repaired["selected_device_id"] is None + assert repaired["connection_lifecycle"]["active_binding"] is None + + +def test_stale_peer_blocks_every_retired_target_contact_path_before_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + owner, _ = service_with_fake_runtime(tmp_path) + owner_binding = _seed_supervised_connection(owner, transport_ref="RETIRED-K1") + request = _seed_unresolved_physical_stop_for_retirement(owner, owner_binding) + stale_peer, stale_runtime = service_with_fake_runtime(tmp_path) + _set_scanned_k1(stale_peer, device_id="retired-k1") + _seed_supervised_connection(stale_peer, transport_ref="retired-k1") + connect_request = _connect_request( + device_id="retired-k1", + ssid="DCCONSTRUCTIONS", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + + owner.retire_unavailable_physical_command(request) + io_calls: list[str] = [] + + async def forbidden_async_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("ble-or-network") + raise AssertionError("retired target must be rejected before I/O") + + def forbidden_sync_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("host-tcp-mqtt-http") + raise AssertionError("retired target must be rejected before I/O") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_async_io) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", forbidden_sync_io) + monkeypatch.setattr(stale_peer, "_sample_host_path", forbidden_sync_io) + monkeypatch.setattr(stale_peer, "_probe_control_endpoint", forbidden_sync_io) + monkeypatch.setattr(stale_peer, "_refresh_live_lan_address", forbidden_sync_io) + monkeypatch.setattr(stale_peer.camera_preview, "select", forbidden_sync_io) + monkeypatch.setattr( + stale_peer._calibration_snapshot_reader, # noqa: SLF001 + "capture", + forbidden_sync_io, + ) + monkeypatch.setattr( + SecretStr, + "get_secret_value", + lambda _self: (_ for _ in ()).throw( + AssertionError("retired Connect must not unwrap the credential") + ), + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as connect_error: + asyncio.run(stale_peer.connect(connect_request)) + assert connect_error.value.reason_code == "physical-command-target-retired" + with pytest.raises(facade_module.ConnectionVerificationError) as verify_error: + asyncio.run( + stale_peer.verify_connection( + ConnectionVerifyRequest( + device_id="retired-k1", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + assert verify_error.value.reason_code == "physical-command-target-retired" + with pytest.raises(facade_module.ConnectionVerificationError) as open_error: + stale_peer.open_application_control_session( + OpenApplicationControlSessionRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + timezone_name="Europe/Moscow", + ) + ) + assert open_error.value.reason_code == "physical-command-target-retired" + with pytest.raises(facade_module.ConfiguredEndpointProbeError) as probe_error: + asyncio.run(stale_peer.probe_configured_endpoint(ConfiguredEndpointProbeRequest())) + assert probe_error.value.reason_code == "physical-command-target-retired" + with pytest.raises(facade_module.ConnectionVerificationError) as calibration_error: + stale_peer.read_device_calibration_snapshot() + assert calibration_error.value.reason_code == "physical-command-target-retired" + assert stale_peer._connection_monitor_target() is None # noqa: SLF001 + asyncio.run(stale_peer._connection_monitor.poll_once()) # noqa: SLF001 + with pytest.raises(facade_module.ConnectionVerificationError): + asyncio.run( + stale_peer._monitor_control_endpoint( # noqa: SLF001 + EndpointTarget("192.168.1.20", facade_module.CONTROL_MQTT_PORT) + ) + ) + with pytest.raises(facade_module.ConnectionVerificationError): + stale_peer.prepare_acquisition( + _prepare_request( + project_name="RETIRED01", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + with pytest.raises(facade_module.ConnectionVerificationError): + stale_peer.start_live( + "RETIRED02", + "192.168.1.20", + None, + ATTESTATION, + ) + with pytest.raises(facade_module.ConnectionVerificationError) as camera_error: + stale_peer.select_camera_preview( + CameraPreviewSelectRequest( + source_id=facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + device_session_id="test-session-retired-k1", + ) + ) + assert camera_error.value.reason_code == "physical-command-target-retired" + + inspected = stale_peer.inspect_device() + assert inspected["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + assert stale_runtime.start_calls == [] + assert stale_runtime.stop_calls == 0 + assert io_calls == [] + + +def test_selected_camera_holds_retirement_fence_and_stale_generation_cannot_spawn( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + camera_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + camera_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + camera_service, + binding, + ) + camera_service.camera_preview._ffmpeg_path = ( # noqa: SLF001 + tmp_path / "synthetic-ffmpeg" + ) + + selected = camera_service.select_camera_preview( + CameraPreviewSelectRequest( + source_id=facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + device_session_id="test-session-retired-k1", + ) + ) + generation = selected["camera_preview"]["generation"] + assert isinstance(generation, int) + assert "camera" in ( # noqa: SLF001 + camera_service._application_control_process_lease_holders # noqa: SLF001 + ) + + retirement_service, _ = service_with_fake_runtime(tmp_path) + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + retirement_service.retire_unavailable_physical_command(request) + + stopped = camera_service.stop_camera_preview( + facade_module.CameraPreviewStopRequest( + generation=generation, + device_session_id="test-session-retired-k1", + ) + ) + assert stopped["camera_preview"]["active_source_id"] is None + assert "camera" not in ( # noqa: SLF001 + camera_service._application_control_process_lease_holders # noqa: SLF001 + ) + + retired = retirement_service.retire_unavailable_physical_command(request) + assert retired["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + + popen_calls: list[str] = [] + + def forbidden_popen(*_args: object, **_kwargs: object) -> object: + popen_calls.append("popen") + raise AssertionError("retired stale camera generation must not spawn FFmpeg") + + monkeypatch.setattr( + "k1link.device_plugins.xgrids_k1.camera.subprocess.Popen", + forbidden_popen, + ) + with pytest.raises(ValueError, match="generation не активно"): + camera_service.camera_preview.open_delivery(generation) + assert popen_calls == [] + + +def test_cancelled_target_probe_retains_retirement_fence_until_thread_finishes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + probe_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + probe_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + probe_service, + binding, + ) + retirement_service, _ = service_with_fake_runtime(tmp_path) + probe_entered = threading.Event() + release_probe = threading.Event() + + def blocked_host_path( + _target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + probe_entered.set() + assert release_probe.wait(timeout=5.0) + return _direct_host_path(_target) + + monkeypatch.setattr(probe_service, "_sample_host_path", blocked_host_path) + + async def scenario() -> None: + target = EndpointTarget(binding.target_ipv4, binding.target_port) + probe_task = asyncio.create_task( + probe_service._monitor_host_path(target) # noqa: SLF001 + ) + assert await asyncio.to_thread(probe_entered.wait, 5.0) + before = retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + + probe_task.cancel() + await asyncio.sleep(0) + probe_task.cancel() + await asyncio.sleep(0.01) + assert probe_task.done() is False + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + await asyncio.to_thread( + retirement_service.retire_unavailable_physical_command, + request, + ) + assert ( + retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + == before + ) + + release_probe.set() + with pytest.raises(asyncio.CancelledError): + await probe_task + retired = await asyncio.to_thread( + retirement_service.retire_unavailable_physical_command, + request, + ) + assert retired["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + + asyncio.run(scenario()) + + +def test_calibration_process_lease_blocks_cross_service_retirement_until_capture_finishes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calibration_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + calibration_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + calibration_service, + binding, + ) + retirement_service, _ = service_with_fake_runtime(tmp_path) + capture_entered = threading.Event() + release_capture = threading.Event() + outcome: dict[str, object] = {} + + class BlockingCalibrationReader: + def capture(self, **_kwargs: object) -> dict[str, object]: + capture_entered.set() + assert release_capture.wait(timeout=5) + return {"status": "available", "snapshot_id": "blocking-test"} + + calibration_service._calibration_snapshot_reader = ( # type: ignore[assignment] # noqa: SLF001 + BlockingCalibrationReader() + ) + monkeypatch.setattr( + calibration_service, + "_refresh_live_lan_address", + lambda: "192.168.1.20", + ) + + def capture() -> None: + try: + outcome["result"] = calibration_service.read_device_calibration_snapshot() + except BaseException as exc: # pragma: no cover - asserted below + outcome["error"] = exc + + worker = threading.Thread(target=capture, daemon=True) + worker.start() + assert capture_entered.wait(timeout=5) + projection = calibration_service.state()["physical_command"]["operator_retirement"] + assert projection["allowed"] is False + assert "device-calibration-read-active" in projection["reason_codes"] + before = retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + retirement_service.retire_unavailable_physical_command(request) + unchanged = retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert unchanged == before + + release_capture.set() + worker.join(timeout=5) + assert not worker.is_alive() + assert "error" not in outcome + assert outcome["result"] == { + "status": "available", + "snapshot_id": "blocking-test", + } + + +def test_calibration_hard_timeout_retains_retirement_fence_until_ble_cleanup( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + calibration_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + calibration_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + calibration_service, + binding, + ) + retirement_service, _ = service_with_fake_runtime(tmp_path) + + async def scenario() -> None: + cleanup_entered = asyncio.Event() + cleanup_release = asyncio.Event() + + async def detached_status_read( + _device_id: str, + **_kwargs: object, + ) -> dict[str, Any]: + async def stubborn_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cleanup_entered.set() + await cleanup_release.wait() + raise + + return await run_ble_operation( # type: ignore[return-value] + "status-read", + hard_timeout_seconds=0.01, + operation=stubborn_cleanup, + ) + + runtime_loop = asyncio.get_running_loop() + bind_ble_runtime_owner_loop(runtime_loop) + with calibration_service._lock: # noqa: SLF001 + calibration_service._runtime_event_loop = runtime_loop # noqa: SLF001 + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + detached_status_read, + ) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "default-route", + ) + + with pytest.raises(RuntimeError, match="BLE status-read не завершился"): + await asyncio.to_thread(calibration_service.read_device_calibration_snapshot) + await cleanup_entered.wait() + + with pytest.raises(facade_module.NetworkProvisioningConflict): + await asyncio.to_thread( + retirement_service.retire_unavailable_physical_command, + request, + ) + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + retired = await asyncio.to_thread( + retirement_service.retire_unavailable_physical_command, + request, + ) + assert retired["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + + asyncio.run(scenario()) + + +def test_operator_manual_live_retains_cross_process_fence_until_receiver_stops( + tmp_path: Path, +) -> None: + live_service, live_runtime = service_with_fake_runtime(tmp_path) + prepared = live_service.prepare_acquisition( + _prepare_request( + project_name="MANUAL01", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + + retirement_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + retirement_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + retirement_service, + binding, + ) + started = live_service.start_acquisition(_start_request(acquisition_id=acquisition_id)) + + assert started["acquisition"]["state"] == "starting" + assert live_runtime.start_calls + assert ( + live_service._operator_manual_acquisition_process_lease_id # noqa: SLF001 + == acquisition_id + ) + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + retirement_service.retire_unavailable_physical_command(request) + unresolved = retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert unresolved is not None and unresolved.unresolved + + stopped = live_service.stop_acquisition( + _stop_request( + acquisition_id=acquisition_id, + mode="capture-only", + ) + ) + assert stopped["acquisition"]["state"] == "completed" + assert live_service._operator_manual_acquisition_process_lease_id is None # noqa: SLF001 + + retired = retirement_service.retire_unavailable_physical_command(request) + assert retired["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + + +def test_ambiguous_manual_start_cleanup_keeps_retirement_fenced_until_retry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + live_service, live_runtime = service_with_fake_runtime(tmp_path) + prepared = live_service.prepare_acquisition( + _prepare_request( + project_name="MANUALFAIL", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + retirement_service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + retirement_service, + transport_ref="retired-k1", + ) + request = _seed_unresolved_physical_stop_for_retirement( + retirement_service, + binding, + ) + original_start_live = live_runtime.start_live + + def fail_after_receiver_start( + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Callable[[int], str] | None = None, + ) -> None: + original_start_live( + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + raise RuntimeError("injected receiver start failure") + + monkeypatch.setattr(live_runtime, "start_live", fail_after_receiver_start) + live_runtime.stop_error = RuntimeError("injected receiver cleanup failure") + camera_arm_calls: list[Path] = [] + monkeypatch.setattr( + live_service, + "_arm_camera_recording", + lambda out_dir, **_kwargs: camera_arm_calls.append(out_dir), + ) + + with pytest.raises(RuntimeError, match="receiver start failure"): + live_service.start_acquisition(_start_request(acquisition_id=acquisition_id)) + + assert live_runtime.start_calls + assert camera_arm_calls == [] + assert ( + live_service._operator_manual_acquisition_process_lease_id # noqa: SLF001 + == acquisition_id + ) + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + retirement_service.retire_unavailable_physical_command(request) + unresolved = retirement_service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert unresolved is not None and unresolved.unresolved + + live_runtime.stop_error = None + live_service._stop_acquisition_sources( # noqa: SLF001 + camera_status="failed", + camera_failure_code="cleanup-retry-after-start-failure", + ) + assert live_service._operator_manual_acquisition_process_lease_id is None # noqa: SLF001 + retirement_service.retire_unavailable_physical_command(request) + + +def test_state_snapshot_stamp_is_runtime_scoped_and_monotonic(tmp_path: Path) -> None: + service, _ = service_with_fake_runtime(tmp_path / "first-runtime") + + first = service.state() + second = service.state() + replacement, _ = service_with_fake_runtime(tmp_path / "replacement-runtime") + replacement_state = replacement.state() + + assert first["snapshot_runtime_id"].startswith("snapshot-runtime-") + assert first["snapshot_runtime_id"] == second["snapshot_runtime_id"] + assert first["snapshot_runtime_started_at_utc"] == (second["snapshot_runtime_started_at_utc"]) + assert ( + first["snapshot_runtime_started_monotonic_ns"] + == (second["snapshot_runtime_started_monotonic_ns"]) + ) + assert second["snapshot_revision"] == first["snapshot_revision"] + 1 + assert replacement_state["snapshot_runtime_id"] != first["snapshot_runtime_id"] + assert int(replacement_state["snapshot_runtime_started_monotonic_ns"]) > int( + first["snapshot_runtime_started_monotonic_ns"] + ) + assert replacement_state["snapshot_revision"] == 1 + + +def test_runtime_start_allocator_is_strictly_increasing_for_equal_clock_ticks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with facade_module._SNAPSHOT_RUNTIME_START_LOCK: # noqa: SLF001 + previous = facade_module._LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS # noqa: SLF001 + facade_module._LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS = 100 # noqa: SLF001 + try: + monkeypatch.setattr(facade_module.time, "monotonic_ns", lambda: 100) + first = facade_module._allocate_snapshot_runtime_started_monotonic_ns() # noqa: SLF001 + second = facade_module._allocate_snapshot_runtime_started_monotonic_ns() # noqa: SLF001 + assert (first, second) == ("101", "102") + finally: + with facade_module._SNAPSHOT_RUNTIME_START_LOCK: # noqa: SLF001 + facade_module._LAST_SNAPSHOT_RUNTIME_STARTED_MONOTONIC_NS = max( # noqa: SLF001 + previous, + 102, + ) + + def _wifi_status_read( ipv4: str | None, *, @@ -214,6 +3690,10 @@ def _wifi_status_read( "device_macos_uuid": device_id, "device_name": "XGR-K1", "service_uuid": "00007f00-0000-1000-8000-00805f9b34fb", + "write_characteristic_uuid": "00007f01-0000-1000-8000-00805f9b34fb", + "write_characteristic_properties": ["read", "write"], + "max_write_without_response_size": 253, + "mtu_size": 256, "status_characteristic_uuid": "00007f02-0000-1000-8000-00805f9b34fb", "operation": "single_reviewed_wifi_status_read", "write_performed": False, @@ -228,15 +3708,378 @@ def _wifi_status_read( } +def _durable_status_capture( + device_id: str = DURABLE_K1_UUID, +) -> facade_module.CapturedDiscoveredDevice: + return facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address=device_id), # type: ignore[arg-type] + macos_uuid=device_id, + owner_epoch=7, + source="retrieved-durable", + ) + + +def _ap_ready_wifi_status() -> dict[str, Any]: + return { + "value_length": 54, + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 1, + "trailer_hex": "", + } + + +@pytest.mark.parametrize("mode", [None, "UNKNOWN", "WIFI_AP"]) +def test_applied_bridge_target_requires_explicit_wifi_client_mode( + mode: str | None, +) -> None: + status = { + "mode": mode, + "ipv4": "192.168.68.52", + "status_code": 1, + "reserved": 0, + } + + assert facade_module._applied_network_target("bridge", status) is None # noqa: SLF001 + assert ( + facade_module._applied_network_target("direct-connect", status) # noqa: SLF001 + is None + ) + + +@pytest.mark.parametrize( + ("connection_mode", "baseline", "observed", "previous", "expected"), + [ + ( + "bridge", + NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.40", 1, 0), + {"mode": "WIFI_CLIENT", "ipv4": "192.168.68.40", "status_code": 1, "reserved": 0}, + None, + None, + ), + ( + "bridge", + NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.40", 1, 0), + {"mode": "WIFI_CLIENT", "ipv4": "192.168.68.99", "status_code": 1, "reserved": 0}, + None, + None, + ), + ( + "direct-connect", + NetworkStatusEvidence("WIFI_CLIENT", None, 1, 0), + {"mode": "WIFI_CLIENT", "ipv4": "172.20.10.2", "status_code": 1, "reserved": 0}, + None, + None, + ), + ( + "quick-connect", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_AP", "ipv4": "192.168.56.1", "status_code": 2, "reserved": 1}, + None, + None, + ), + ( + "bridge", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "192.168.68.50", "status_code": 1, "reserved": 0}, + None, + "192.168.68.50", + ), + ( + "direct-connect", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "172.20.10.2", "status_code": 1, "reserved": 0}, + None, + "172.20.10.2", + ), + ( + "quick-connect", + NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.50", 1, 0), + {"mode": "WIFI_AP", "ipv4": "192.168.56.1", "status_code": 1, "reserved": 1}, + None, + "192.168.56.1", + ), + ( + "quick-connect", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 0), + {"mode": "WIFI_AP", "ipv4": "192.168.56.1", "status_code": 1, "reserved": 1}, + None, + "192.168.56.1", + ), + ( + "quick-connect", + NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.50", 1, 0), + {"mode": "WIFI_AP", "ipv4": "192.168.56.1", "status_code": 1, "reserved": 1}, + PreviousConnectionEvidence("same-k1", "quick-connect", "192.168.56.1", "old-session"), + None, + ), + ( + "bridge", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "192.168.68.99", "status_code": 1, "reserved": 0}, + PreviousConnectionEvidence("same-k1", "bridge", "192.168.68.40", "old-session"), + None, + ), + ( + "direct-connect", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "172.20.10.2", "status_code": 1, "reserved": 0}, + PreviousConnectionEvidence("same-k1", "bridge", "172.20.10.2", "old-session"), + None, + ), + ( + "bridge", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "172.20.10.2", "status_code": 1, "reserved": 0}, + PreviousConnectionEvidence("same-k1", "direct-connect", "172.20.10.2", "old-session"), + None, + ), + ( + "bridge", + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + {"mode": "WIFI_CLIENT", "ipv4": "192.168.68.50", "status_code": 1, "reserved": 0}, + PreviousConnectionEvidence("other-k1", "bridge", "192.168.68.50", "old-session"), + "192.168.68.50", + ), + ], +) +def test_post_dispatch_target_requires_status_distinguishable_from_baseline( + connection_mode: facade_module.ConnectionMode, + baseline: NetworkStatusEvidence, + observed: dict[str, object], + previous: PreviousConnectionEvidence | None, + expected: str | None, +) -> None: + assert ( + facade_module._post_dispatch_network_target( # noqa: SLF001 + connection_mode, + observed, + transport_ref="same-k1", + baseline_status=baseline, + previous_connection=previous, + ) + == expected + ) + + +def test_post_dispatch_bridge_accepts_exact_fw302_network_name_when_baseline_is_same() -> None: + """An idempotent same-network Bridge request proves the requested target.""" + + status = { + "mode": "WIFI_CLIENT", + "network_name": "LAB_NETWORK", + "ipv4": "192.168.68.51", + "status_code": 1, + "reserved": 0, + } + + assert ( + facade_module._post_dispatch_network_target( # noqa: SLF001 + "bridge", + status, + transport_ref="same-k1", + baseline_status=NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.51", 1, 0), + previous_connection=PreviousConnectionEvidence( + "same-k1", + "bridge", + "192.168.68.51", + "old-session", + ), + expected_network_name="LAB_NETWORK", + ) + == "192.168.68.51" + ) + + +def test_post_dispatch_bridge_rejects_another_fw302_network_name() -> None: + status = { + "mode": "WIFI_CLIENT", + "network_name": "ANOTHER_NETWORK", + "ipv4": "192.168.68.51", + "status_code": 1, + "reserved": 0, + } + + assert ( + facade_module._post_dispatch_network_target( # noqa: SLF001 + "bridge", + status, + transport_ref="same-k1", + baseline_status=NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + expected_network_name="LAB_NETWORK", + ) + is None + ) + + +def _seed_unresolved_network_mutation( + service: XgridsK1CompatibilityService, + *, + operation_id: str = "ambiguous-operation", + idempotency_operation_id: str | None = None, + transport_ref: str = "test-ble-transport", + intended_mode: NetworkConnectionMode = "bridge", + baseline_status: NetworkStatusEvidence | None = None, + previous_connection: PreviousConnectionEvidence | None = None, +) -> dict[str, Any]: + resolved_idempotency_operation_id = idempotency_operation_id or operation_id + idempotency_key = f"test-reconciliation-{resolved_idempotency_operation_id}" + idempotency_journal = ( + service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + ) + idempotency_admission = idempotency_journal.begin( + idempotency_key=idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + operation_id=resolved_idempotency_operation_id, + request_binding_sha256=facade_module.derive_request_binding_sha256( + idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + canonical_request=b'{"test":"read-only-reconciliation"}', + ), + ) + ledger = service._network_mutation_ledger # noqa: SLF001 + prepared = ledger.prepare( + operation_id=operation_id, + transport_ref=transport_ref, + intended_mode=intended_mode, + write_mode="with_response", + baseline_status=baseline_status + or NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=1, + ), + previous_connection=previous_connection, + ) + idempotency_journal.mark_unresolved( + resolved_idempotency_operation_id, + expected_revision=idempotency_admission.record.revision, + ) + ledger.mark_dispatching( + operation_id, + expected_revision=prepared.revision, + ) + fence = service.state()["network_write_reconciliation"] + assert fence is not None + return fence + + +def _prepare_network_mutation_for_reopen( + service: XgridsK1CompatibilityService, + *, + operation_id: str, + transport_ref: str, + intended_mode: NetworkConnectionMode, +) -> NetworkMutationRecord: + return service._network_mutation_ledger.prepare( # noqa: SLF001 + operation_id=operation_id, + transport_ref=transport_ref, + intended_mode=intended_mode, + write_mode="with_response", + baseline_status=NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=1, + ), + ) + + +def _begin_network_idempotency_for_reopen( + service: XgridsK1CompatibilityService, + *, + operation_id: str, + mark_unresolved: bool, +) -> None: + idempotency_key = f"reopen-audit-{operation_id}" + journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + admission = journal.begin( + idempotency_key=idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + operation_id=operation_id, + request_binding_sha256=facade_module.derive_request_binding_sha256( + idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + canonical_request=b'{"test":"reopen-precommit"}', + ), + ) + if mark_unresolved: + journal.mark_unresolved( + operation_id, + expected_revision=admission.record.revision, + ) + + +def _seed_legacy_network_mutation_without_idempotency( + service: XgridsK1CompatibilityService, + *, + operation_id: str = "legacy-network-operation", + stage: str = "dispatching", +) -> None: + """Create the durable shape emitted before the idempotency journal existed.""" + + ledger = service._network_mutation_ledger # noqa: SLF001 + prepared = ledger.prepare( + operation_id=operation_id, + transport_ref="test-ble-transport", + intended_mode="bridge", + write_mode="with_response", + baseline_status=NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=1, + ), + ) + if stage == "prepared": + return + dispatching = ledger.mark_dispatching( + operation_id, + expected_revision=prepared.revision, + ) + if stage == "observing": + ledger.mark_observing( + operation_id, + expected_revision=dispatching.revision, + write_confirmed=True, + observation=NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.50", + status_code=1, + reserved=0, + ), + ) + elif stage != "dispatching": + raise ValueError("unsupported legacy test stage") + + def _set_scanned_devices( service: XgridsK1CompatibilityService, devices: list[dict[str, Any]], ) -> None: service._devices = devices # noqa: SLF001 + _SYNTHETIC_SCAN_CAPTURES.clear() + _SYNTHETIC_SCAN_CAPTURES.update( + { + str(item["device_id"]): facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address=str(item["device_id"])), # type: ignore[arg-type] + macos_uuid=str(item["device_id"]), + owner_epoch=1, + scan_generation=service._ble_discovery_generation, # noqa: SLF001 + ) + for item in devices + } + ) observed_monotonic = facade_module.time.monotonic() + observed_suspend_aware = facade_module.time.time() service._ble_device_last_seen_monotonic = { # noqa: SLF001 str(item["device_id"]): observed_monotonic for item in devices } + service._ble_device_last_seen_suspend_aware = { # noqa: SLF001 + str(item["device_id"]): observed_suspend_aware for item in devices + } def _set_scanned_k1( @@ -244,61 +4087,882 @@ def _set_scanned_k1( *, device_id: str = "test-ble-transport", ) -> None: - _set_scanned_devices(service, [ - { - "device_id": device_id, - "name": "XGR-K1", - "rssi": -44, - "address": None, - "connectable": True, - "likely_k1": True, + _set_scanned_devices( + service, + [ + { + "device_id": device_id, + "name": "XGR-K1", + "rssi": -44, + "address": None, + "connectable": True, + "likely_k1": True, + } + ], + ) + + +def _ble_scan_result(device_id: str) -> dict[str, Any]: + return { + "devices": [ + { + "macos_uuid": device_id, + "name": "XGR-K1", + "local_name": "XGR-K1", + "rssi": -44, + "k1_name_candidate": True, + } + ] + } + + +def _resolved_active_physical_snapshot( + *, + transport_ref: str = "k1-original", + connection_mode: str = "bridge", +) -> dict[str, object]: + return { + "status": "resolved", + "reason_code": None, + "requires_reconciliation": False, + "resolved_active_recovery_required": True, + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": ("explicit-read-only-deviceinfo-and-non-retained-devicestatus"), + "runtime_bound": False, + "reconciliation_ready": False, + "observed_session_state": None, + "active_operation_id": None, + "record": { + "operation_id": "persisted-start-active", + "action": "start", + "stage": "resolved", + "resolution": "start-active-observed", + "connection": { + "transport_ref": transport_ref, + "connection_mode": connection_mode, + }, + "reconciliations": [], + }, + } + + +def _pending_reopened_physical_snapshot( + *, + transport_ref: str = "k1-a", + connection_mode: str = "bridge", +) -> dict[str, object]: + """Project a classified STOP whose retired target was reopened for Verify.""" + + operation_id = "pending-reopened-prepared-stop" + retirement_id = "retirement-before-pending-reopen" + return { + "status": "resolved", + "reason_code": "physical-command-reconciliation-required", + "requires_reconciliation": False, + "resolved_active_recovery_required": True, + "reopened_physical_state_recovery_required": True, + "physical_active": False, + "reconciled_physical_state": None, + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": "explicit-read-only-reconciliation", + "runtime_bound": False, + "active_operation_id": None, + "record": { + "operation_id": operation_id, + "acquisition_id": "pending-reopened-acquisition", + "action": "stop", + "stage": "resolved", + "resolution": "not-dispatched", + "revision": 9, + "connection": { + "transport_ref": transport_ref, + "connection_mode": connection_mode, + }, + "operator_retirements": [ + { + "retirement_id": retirement_id, + "retired_transport_ref": transport_ref, + "original_attempt": {"operation_id": operation_id}, + } + ], + "operator_reconciliation_reopens": [ + { + "retirement_id": retirement_id, + "retired_record_revision": 8, + } + ], + "reconciliations": [], + }, + } + + +def test_physical_recovery_policy_pins_original_ref_and_mode_but_keeps_scan_read_only( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices( + service, + [ + { + "device_id": "k1-original", + "name": "Original K1", + "rssi": -40, + "address": None, + "connectable": True, + "likely_k1": True, + }, + { + "device_id": "k1-nearby", + "name": "Nearby K1", + "rssi": -35, + "address": None, + "connectable": True, + "likely_k1": True, + }, + ], + ) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: _resolved_active_physical_snapshot(), + ) + + state = service.state() + policy = state["connection_policy"]["actions"] + + assert policy["scan-ble"]["allowed"] is True + assert "physical-device-already-active" not in policy["scan-ble"]["reason_codes"] + assert policy["provision-fresh-device"]["allowed"] is False + assert "physical-device-already-active" in policy["provision-fresh-device"]["reason_codes"] + assert policy["observe-fresh-device-network"] == { + "allowed": True, + "reason_codes": [], + "target_source": "fresh-scan", + "required_transport_ref": "k1-original", + "required_connection_mode": "bridge", + "requires_live_gatt_validation": True, + "automatic_retry": False, + } + + +def test_physical_recovery_rejects_wrong_nearby_k1_before_topology_or_ble_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices( + service, + [ + { + "device_id": "k1-original", + "name": "Original K1", + "rssi": -40, + "address": None, + "connectable": True, + "likely_k1": True, + }, + { + "device_id": "k1-nearby", + "name": "Nearby K1", + "rssi": -35, + "address": None, + "connectable": True, + "likely_k1": True, + }, + ], + ) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: _resolved_active_physical_snapshot(), + ) + io_calls: list[str] = [] + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + lambda *_args, **_kwargs: io_calls.append("gatt-read"), + ) + monkeypatch.setattr( + service, + "_apply_read_only_device_topology", + lambda *_args, **_kwargs: io_calls.append("topology-commit"), + ) + semantic_before = service._semantic_topology_public_snapshot() # noqa: SLF001 + supervisor_before = service._connection_supervisor.snapshot() # noqa: SLF001 + network_ledger_before = service._network_mutation_ledger.snapshot() # noqa: SLF001 + + with pytest.raises(facade_module.ConnectionVerificationError) as rejected: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-nearby", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert rejected.value.reason_code == "physical-command-recovery-target-mismatch" + assert io_calls == [] + assert service._semantic_topology_public_snapshot() == semantic_before # noqa: SLF001 + assert service._connection_supervisor.snapshot() == supervisor_before # noqa: SLF001 + assert service._network_mutation_ledger.snapshot() == network_ledger_before # noqa: SLF001 + + +def test_connect_is_fenced_before_gatt_or_wifi_when_physical_state_is_active( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-original") + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: _resolved_active_physical_snapshot(), + ) + io_calls: list[str] = [] + + async def forbidden_network_io(*_args: object, **_kwargs: object) -> dict[str, Any]: + io_calls.append("network-io") + raise AssertionError("physical admission must fail before GATT/Wi-Fi") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_network_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_io) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as rejected: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-original", + ssid="DCCONSTRUCTIONS", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert rejected.value.reason_code == "physical-device-already-active" + assert io_calls == [] + + +def test_latest_scan_candidate_survives_wall_clock_age_but_requires_exact_capture( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + service._ble_device_last_seen_monotonic["k1-a"] = -1.0 # noqa: SLF001 + service._ble_device_last_seen_suspend_aware["k1-a"] = -1.0 # noqa: SLF001 + monkeypatch.setattr(facade_module, "_capture_network_intent_device", lambda _device_id: None) + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("a missing exact scan capture must fail before GATT write") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + state = service.state() + + assert [item["device_id"] for item in state["devices"]] == ["k1-a"] + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["connection_lifecycle"]["active_binding"] is None + assert state["connection_lifecycle"]["active_binding_key"] is None + assert state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-candidate-not-fresh" + + +def test_candidate_wall_clock_age_does_not_close_an_admitted_connection_session( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection(service, transport_ref="k1-a") + _set_scanned_k1(service, device_id="k1-a") + service._ble_device_last_seen_monotonic["k1-a"] = -1.0 # noqa: SLF001 + service._ble_device_last_seen_suspend_aware["k1-a"] = -1.0 # noqa: SLF001 + + state = service.state() + + assert [item["device_id"] for item in state["devices"]] == ["k1-a"] + assert state["selected_device_id"] == "k1-a" + assert state["device_session"] is not None + assert state["device_session"]["connectivity"] in {"connected", "degraded"} + + +def test_ble_scan_lists_twenty_candidates_without_selecting_or_opening_gatt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + low_level_calls: list[str] = [] + + async def twenty_candidate_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return { + "devices": [ + { + "macos_uuid": f"K1-CANDIDATE-{index:02d}", + "name": f"XGR-K1-{index:02d}", + "local_name": f"XGR-K1-{index:02d}", + "rssi": -40 - index, + "k1_name_candidate": True, + } + for index in range(20) + ] } - ]) + + async def forbidden_gatt(*_: object, **__: object) -> dict[str, Any]: + low_level_calls.append("gatt") + raise AssertionError("scan must not open GATT or write K1") + + monkeypatch.setattr(facade_module, "scan", twenty_candidate_scan) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_gatt) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_gatt) + + state = asyncio.run(service.scan_ble(1.0)) + + assert len(state["devices"]) == 20 + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["connection_mode"] is None + assert low_level_calls == [] -def test_ble_discovery_lease_expiry_hides_and_rejects_candidate( +def _attempt_competing_scan_from_sync_boundary( + service: XgridsK1CompatibilityService, + *, + operation_id: str, +) -> object: + outcomes: list[object] = [] + + def worker() -> None: + try: + outcomes.append( + asyncio.run( + service.scan_ble( + BleScanRequest( + duration_seconds=1.0, + operation_id=operation_id, + ) + ) + ) + ) + except BaseException as exc: + outcomes.append(exc) + + thread = threading.Thread(target=worker, daemon=True) + thread.start() + thread.join(timeout=1.0) + assert not thread.is_alive() + assert len(outcomes) == 1 + return outcomes[0] + + +def test_explicit_new_scan_invalidates_previous_candidate_authority( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_k1(service) - service._ble_device_last_seen_monotonic["test-ble-transport"] = ( # noqa: SLF001 - facade_module.time.monotonic() - - facade_module.BLE_DISCOVERY_LEASE_TTL_SECONDS - - 0.001 + + async def empty_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + return {"devices": []} + + monkeypatch.setattr(facade_module, "scan", empty_scan) + + state = asyncio.run(service.scan_ble(1.0)) + + assert state["devices"] == [] + assert state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + assert state["connection_policy"]["facts"]["fresh_transport_refs"] == [] + + +def test_uuid_without_exact_latest_scan_capture_cannot_retire_existing_session( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + service._selected_device_id = "k1-a" # noqa: SLF001 + service._connection_mode = "bridge" # noqa: SLF001 + service._device_session_id = "existing-session" # noqa: SLF001 + service._k1_ip = "192.168.68.50" # noqa: SLF001 + monkeypatch.setattr(facade_module, "_capture_network_intent_device", lambda _device_id: None) + monkeypatch.setattr( + service, + "_retire_application_control_for_network_change", + lambda: (_ for _ in ()).throw( + AssertionError("missing exact capture must fail before ownership handoff") + ), ) - async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: - raise AssertionError("an expired candidate must not be rediscovered by an action") - - async def forbidden_provision(*_: object, **__: object) -> dict[str, Any]: - raise AssertionError("an expired candidate must not reach the Wi-Fi write boundary") - - monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) - monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) - - stale_state = service.state() - assert stale_state["devices"] == [] - assert "устарели" in stale_state["message"] - with pytest.raises(ValueError, match="найдите и выберите"): - service.verify_connection( - ConnectionVerifyRequest( - device_id="test-ble-transport", - compatibility_attestation=ATTESTATION, - ) - ) - with pytest.raises(ValueError, match="найдите и выберите"): + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: asyncio.run( service.connect( - ConnectRequest( - device_id="test-ble-transport", - ssid="lab-network", + _connect_request( + device_id="k1-a", + ssid="lab-router", password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", compatibility_attestation=ATTESTATION, ) ) ) + assert raised.value.reason_code == "network-provision-candidate-not-fresh" + assert service._selected_device_id == "k1-a" # noqa: SLF001 + assert service._connection_mode == "bridge" # noqa: SLF001 + assert service._device_session_id == "existing-session" # noqa: SLF001 + assert service._k1_ip == "192.168.68.50" # noqa: SLF001 + + +def test_stale_runtime_generation_cannot_revive_live_data_plane( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + runtime.source_mode = "live" + runtime.phase = "live" + runtime.producer_generation = 2 + + consumed = service._observe_runtime_message( # noqa: SLF001 + StreamMessage( + sequence=7, + topic="RealtimePointcloud", + payload=b"stale", + received_at_epoch_ns=1, + received_monotonic_ns=1, + source="live_mqtt", + producer_generation=1, + ), + BridgeMetrics(), + ) + + assert consumed is True + assert service._last_live_data_monotonic is None # noqa: SLF001 + assert service._last_live_data_suspend_aware is None # noqa: SLF001 + assert service._last_live_data_session_id is None # noqa: SLF001 + + +@pytest.mark.parametrize("loss_boundary", ["control", "host"]) +def test_bound_live_data_session_expires_after_control_or_host_lease_loss( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + loss_boundary: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + with service._lock: # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + runtime.source_mode = "live" + runtime.phase = "live" + monotonic_now = [100.0] + monkeypatch.setattr(facade_module.time, "monotonic", lambda: monotonic_now[0]) + supervisor = service._connection_supervisor # noqa: SLF001 + data_session_id = "bound-live-data-session" + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id=data_session_id, + ) + with service._lock: # noqa: SLF001 + service._last_live_data_monotonic = monotonic_now[0] # noqa: SLF001 + service._last_live_data_session_id = data_session_id # noqa: SLF001 + + if loss_boundary == "control": + assert supervisor.observe_control_loss( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + control_session_id="test-control-test-ble-transport", + reason_code="test-control-loss", + ) + else: + supervisor.observe_host_path( + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="test-host-loss", + ) + ) + + revoked = supervisor.snapshot() + assert revoked.lease.state != "reachable" + assert revoked.data_plane.state == "healthy" + assert revoked.data_plane.host_path_epoch == binding.host_path_epoch + + monotonic_now[0] += facade_module.LIVE_DATA_PLANE_STALL_SECONDS + 0.01 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + stalled = supervisor.snapshot() + assert stalled.data_plane.state == "stalled" + assert stalled.data_plane.reason_code == "live-data-stalled" + assert stalled.data_plane.session_id == data_session_id + assert stalled.data_plane.host_path_epoch == binding.host_path_epoch + + monotonic_now[0] = 100.0 + facade_module.LIVE_DATA_PLANE_LOST_SECONDS + 0.01 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + lost = supervisor.snapshot() + assert lost.data_plane.state == "lost" + assert lost.data_plane.reason_code == "live-data-lost" + assert lost.data_plane.session_id == data_session_id + assert lost.data_plane.host_path_epoch == binding.host_path_epoch + + assert ( + supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id="new-data-session-without-current-lease", + ) + is False + ) + + +def test_bound_live_data_session_expires_across_host_suspend( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + runtime.source_mode = "live" + runtime.phase = "live" + monotonic_now = [300.0] + suspend_aware_now = [3_000.0] + monkeypatch.setattr(facade_module.time, "monotonic", lambda: monotonic_now[0]) + monkeypatch.setattr(facade_module.time, "time", lambda: suspend_aware_now[0]) + supervisor = service._connection_supervisor # noqa: SLF001 + data_session_id = "pre-suspend-data-session" + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id=data_session_id, + ) + with service._lock: # noqa: SLF001 + service._last_live_data_monotonic = monotonic_now[0] # noqa: SLF001 + service._last_live_data_suspend_aware = suspend_aware_now[0] # noqa: SLF001 + service._last_live_data_session_id = data_session_id # noqa: SLF001 + + supervisor.observe_host_path( + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-suspended", + ) + ) + suspend_aware_now[0] += facade_module.LIVE_DATA_PLANE_STALL_SECONDS + 0.01 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + assert supervisor.snapshot().data_plane.state == "stalled" + + suspend_aware_now[0] = 3_000.0 + facade_module.LIVE_DATA_PLANE_LOST_SECONDS + 0.01 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + lost = supervisor.snapshot() + assert lost.data_plane.state == "lost" + assert lost.data_plane.reason_code == "live-data-lost" + assert lost.data_plane.host_path_epoch == binding.host_path_epoch + + +def test_only_exact_bound_live_data_session_can_refresh_after_lease_loss( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + runtime.source_mode = "live" + runtime.phase = "live" + runtime.source_ready = True + runtime.producer_generation = 4 + monotonic_now = [200.0] + monkeypatch.setattr(facade_module.time, "monotonic", lambda: monotonic_now[0]) + + def published_point_cloud(sequence: int) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + runtime.pcl_frames = 1 + service._observe_published_runtime_envelope( # noqa: SLF001 + published_point_cloud(1), + runtime.producer_generation, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + bound = supervisor.snapshot().data_plane + assert bound.state == "healthy" + assert bound.session_id == "test-session-test-ble-transport" + assert bound.host_path_epoch == binding.host_path_epoch + + assert supervisor.observe_control_loss( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + control_session_id="test-control-test-ble-transport", + reason_code="test-control-loss", + ) + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="stalled", + session_id="test-session-test-ble-transport", + reason_code="test-data-stall", + ) + + monotonic_now[0] += 1.0 + runtime.pcl_frames = 2 + service._observe_published_runtime_envelope( # noqa: SLF001 + published_point_cloud(2), + runtime.producer_generation, + ) + refreshed = supervisor.snapshot() + assert refreshed.data_plane.state == "healthy" + assert refreshed.data_plane.session_id == "test-session-test-ble-transport" + assert refreshed.authority.control_allowed is False + assert refreshed.authority.data_ingest_authoritative is False + + with service._lock: # noqa: SLF001 + service._device_session_id = "new-session-without-current-lease" # noqa: SLF001 + monotonic_now[0] += 1.0 + runtime.pcl_frames = 3 + service._observe_published_runtime_envelope( # noqa: SLF001 + published_point_cloud(3), + runtime.producer_generation, + ) + rejected_new_session = supervisor.snapshot() + assert rejected_new_session.data_plane.session_id == ("test-session-test-ble-transport") + assert service._last_live_data_session_id == ( # noqa: SLF001 + "test-session-test-ble-transport" + ) + + +def test_postpublished_pcl_rolls_same_session_to_fresh_control_epoch_after_wifi_rebind( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + runtime.source_mode = "live" + runtime.phase = "live" + runtime.source_ready = True + runtime.producer_generation = 4 + + def published_point_cloud(sequence: int) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + runtime.pcl_frames = 1 + service._observe_published_runtime_envelope( # noqa: SLF001 + published_point_cloud(1), + runtime.producer_generation, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + before_loss = supervisor.snapshot() + data_session_id = "test-session-test-ble-transport" + assert before_loss.host_path.epoch == binding.host_path_epoch + assert before_loss.data_plane.session_id == data_session_id + assert before_loss.data_plane.host_path_epoch == binding.host_path_epoch + + supervisor.observe_host_path( + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + ) + rebound_epoch = supervisor.observe_host_path( + HostPathProbeResult( + available=True, + fingerprint="route-after-wifi-return", + interface="test0", + source_ipv4="192.168.1.2", + route_class="direct", + kernel_route_fingerprint="route-after-wifi-return", + ) + ) + target = EndpointTarget(binding.target_ipv4, binding.target_port) + assert supervisor.observe_endpoint( + target=target, + intent_id=binding.intent_id, + host_path_epoch=rebound_epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + host_path_epoch=rebound_epoch, + target=target, + connection_mode=binding.connection_mode, + logical_device_id="known-k1", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id="post-wifi-control", + control_proof_revision=2, + control_proof_source="correlated-application-response", + ) + ) + rebound = supervisor.snapshot() + assert rebound.host_path.epoch == rebound_epoch + assert rebound.authority.control_allowed is True + assert rebound.data_plane.session_id == data_session_id + assert rebound.data_plane.host_path_epoch == binding.host_path_epoch + assert rebound.authority.data_ingest_authoritative is False + + runtime.pcl_frames = 2 + service._observe_published_runtime_envelope( # noqa: SLF001 + published_point_cloud(2), + runtime.producer_generation, + ) + + recovered = supervisor.snapshot() + assert recovered.data_plane.state == "healthy" + assert recovered.data_plane.session_id == data_session_id + assert recovered.data_plane.host_path_epoch == rebound_epoch + assert recovered.data_plane.host_path_epoch == recovered.host_path.epoch + assert recovered.authority.data_ingest_authoritative is True + + +def test_pose_only_traffic_cannot_refresh_point_cloud_health_during_recovery( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + runtime.source_mode = "live" + runtime.phase = "live" + runtime.source_ready = True + runtime.producer_generation = 4 + now = [100.0] + monkeypatch.setattr(facade_module.time, "monotonic", lambda: now[0]) + monkeypatch.setattr(facade_module.time, "time", lambda: now[0]) + + def packet(sequence: int, topic: str) -> StreamMessage: + return StreamMessage( + sequence=sequence, + topic=topic, + payload=b"data-plane-evidence", + received_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + source="live_mqtt", + producer_generation=4, + ) + + baseline = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + runtime.pcl_frames = 1 + service._observe_published_runtime_envelope( # noqa: SLF001 + baseline, + runtime.producer_generation, + ) + assert service._last_live_data_monotonic == 100.0 # noqa: SLF001 + assert service._connection_supervisor.snapshot().data_plane.state == "healthy" # noqa: SLF001 + + runtime.phase = "reconnecting" + runtime.source_ready = False + runtime.recovery_state = "reconnecting" + + now[0] += facade_module.LIVE_DATA_PLANE_STALL_SECONDS + 0.01 + service._observe_runtime_message( # noqa: SLF001 + packet(2, "lixel/application/report/lio_pose"), + BridgeMetrics(), + ) + assert service._last_live_data_monotonic == 100.0 # noqa: SLF001 + service._observe_runtime_message( # noqa: SLF001 + packet(3, "RealtimePointcloud"), + BridgeMetrics(), + ) + assert service._last_live_data_monotonic == 100.0 # noqa: SLF001 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + assert service._connection_supervisor.snapshot().data_plane.state == "stalled" # noqa: SLF001 + + now[0] = 100.0 + facade_module.LIVE_DATA_PLANE_LOST_SECONDS + 0.01 + service._observe_runtime_message( # noqa: SLF001 + packet(4, "RealtimePath"), + BridgeMetrics(), + ) + assert service._last_live_data_monotonic == 100.0 # noqa: SLF001 + service._reconcile_connection_supervisor( # noqa: SLF001 + {"state": "unknown"}, + runtime.snapshot(), + ) + lost = service._connection_supervisor.snapshot().data_plane # noqa: SLF001 + assert lost.state == "lost" + assert lost.reason_code == "live-data-lost" + assert lost.host_path_epoch == binding.host_path_epoch + def test_new_ble_scan_generation_invalidates_old_candidates_before_io_and_on_failure( monkeypatch: pytest.MonkeyPatch, @@ -311,7 +4975,13 @@ def test_new_ble_scan_generation_invalidates_old_candidates_before_io_and_on_fai entered = asyncio.Event() release = asyncio.Event() - async def failing_scan(_duration_seconds: float) -> dict[str, Any]: + async def failing_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() entered.set() await release.wait() raise RuntimeError("synthetic BLE scan failure") @@ -326,65 +4996,291 @@ def test_new_ble_scan_generation_invalidates_old_candidates_before_io_and_on_fai asyncio.run(scenario()) - assert service.state()["devices"] == [] + failed_state = service.state() + assert failed_state["devices"] == [] assert service._devices == [] # noqa: SLF001 assert service._ble_device_last_seen_monotonic == {} # noqa: SLF001 + failed_operation = failed_state["last_operation"] + assert failed_operation["status"] == "failed" + assert failed_operation["error"] == { + "category": "transport", + "code": "RuntimeError", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + } -def test_older_ble_scan_cannot_replace_a_newer_generation( +def test_ble_scan_operation_id_is_exactly_once_and_request_bound( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - first_entered = asyncio.Event() - release_first = asyncio.Event() - call_count = 0 + transport_calls = 0 - def scan_result(device_id: str) -> dict[str, Any]: - return { - "devices": [ - { - "macos_uuid": device_id, - "name": "XGR-K1", - "local_name": "XGR-K1", - "rssi": -44, - "k1_name_candidate": True, - } - ] - } + async def successful_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + nonlocal transport_calls + if on_admitted is not None: + on_admitted() + transport_calls += 1 + return _ble_scan_result("exactly-once-device") - async def overlapping_scan(_duration_seconds: float) -> dict[str, Any]: - nonlocal call_count - call_count += 1 - if call_count == 1: - first_entered.set() - await release_first.wait() - return scan_result("older-generation") - return scan_result("newer-generation") + monkeypatch.setattr(facade_module, "scan", successful_scan) + operation_id = "op-00000000-0000-4000-8000-000000000001" + request = BleScanRequest(duration_seconds=6.0, operation_id=operation_id) - async def scenario() -> dict[str, Any]: - monkeypatch.setattr(facade_module, "scan", overlapping_scan) - older_task = asyncio.create_task(service.scan_ble(6.0)) - await asyncio.wait_for(first_entered.wait(), timeout=1.0) - newer_state = await service.scan_ble(6.0) - release_first.set() - await asyncio.wait_for(older_task, timeout=1.0) - return newer_state + first = asyncio.run(service.scan_ble(request)) + repeated = asyncio.run(service.scan_ble(request)) - newer_state = asyncio.run(scenario()) - - assert [item["device_id"] for item in newer_state["devices"]] == ["newer-generation"] - assert [item["device_id"] for item in service.state()["devices"]] == [ - "newer-generation" - ] + assert transport_calls == 1 + assert first["devices"][0]["connectable"] is None + assert first["last_operation"]["operation_id"] == operation_id + assert repeated["last_operation"]["operation_id"] == operation_id + with pytest.raises(ValueError, match="different request"): + asyncio.run( + service.scan_ble( + BleScanRequest( + duration_seconds=7.0, + operation_id=operation_id, + ) + ) + ) -def test_connect_stops_before_ble_write_when_a_new_scan_replaces_its_generation( +def test_ble_scan_maps_process_runtime_busy_to_operation_journal( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="active-scan-result") + service._operation_phase = "scanning" # noqa: SLF001 + + async def scenario() -> tuple[Any, dict[str, Any]]: + async def busy_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + del on_admitted + raise BleRuntimeBusy( + active_operation_kind="status-read", + cleanup_pending=False, + ) + + monkeypatch.setattr(facade_module, "scan", busy_scan) + with pytest.raises(BleRuntimeBusy, match="уже выполняется"): + await service.scan_ble(6.0) + assert service._operation_phase == "scanning" # noqa: SLF001 + rejected_state = service.state() + + async def recovered_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return _ble_scan_result("second-generation") + + monkeypatch.setattr(facade_module, "scan", recovered_scan) + recovered_state = await service.scan_ble(6.0) + return rejected_state, recovered_state + + rejected_state, recovered_state = asyncio.run(scenario()) + + assert [item["device_id"] for item in rejected_state["devices"]] == ["active-scan-result"] + rejected_operation = rejected_state["last_operation"] + assert rejected_operation["status"] == "failed" + assert rejected_operation["stage_code"] == "busy" + assert rejected_operation["message_code"] == "discovery.scan.already_running" + assert rejected_operation["error"] == { + "category": "conflict", + "code": "ble-runtime-busy", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + } + assert [item["device_id"] for item in recovered_state["devices"]] == ["second-generation"] + + +def test_ble_scan_does_not_block_owner_loop_on_sync_lifecycle_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + gate_held = threading.Event() + release_gate = threading.Event() + + def hold_sync_lifecycle_gate() -> None: + with service._acquisition_lifecycle_gate: # noqa: SLF001 + gate_held.set() + release_gate.wait(timeout=2.0) + + holder = threading.Thread(target=hold_sync_lifecycle_gate) + holder.start() + assert gate_held.wait(timeout=1.0) + + async def admitted_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + release_gate.set() + return _ble_scan_result("nonblocking-scan") + + monkeypatch.setattr(facade_module, "scan", admitted_scan) + started_at = time.monotonic() + started = asyncio.run(service.scan_ble(6.0)) + elapsed = time.monotonic() - started_at + holder.join(timeout=1.0) + + assert not holder.is_alive() + assert elapsed < 0.5 + assert [item["device_id"] for item in started["devices"]] == ["nonblocking-scan"] + + +def test_ble_scan_maps_hard_timeout_to_operation_journal( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + async def scenario() -> tuple[dict[str, Any], dict[str, Any]]: + async def timed_out_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + raise BleOperationHardTimeout( + "scan", + BleOperationProgress(operation_stage="discovery"), + ) + + monkeypatch.setattr(facade_module, "scan", timed_out_scan) + with pytest.raises(BleOperationHardTimeout, match="не завершилась"): + await service.scan_ble(6.0) + assert service._operation_phase is None # noqa: SLF001 + failed_state = service.state() + + async def recovered_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return _ble_scan_result("after-timeout") + + monkeypatch.setattr(facade_module, "scan", recovered_scan) + recovered_state = await service.scan_ble(6.0) + return failed_state, recovered_state + + failed_state, recovered_state = asyncio.run(scenario()) + + assert failed_state["devices"] == [] + failed_operation = failed_state["last_operation"] + assert failed_operation["status"] == "failed" + assert failed_operation["error"] == { + "category": "transport", + "code": "ble-discovery-timeout", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + "operation_stage": "discovery", + "device_write_attempted": False, + "device_write_confirmed": False, + "host_diagnostic": { + "schema_version": "missioncore.host-failure-diagnostic/v1", + "code": "host.bluetooth.operation-timeout", + "domain": "corebluetooth", + "impact": "discovery", + "operator_action": "explicit-retry", + "automatic_retry": False, + "redacted": True, + }, + } + assert [item["device_id"] for item in recovered_state["devices"]] == ["after-timeout"] + + +def test_cancelled_ble_scan_is_journaled_and_releases_process_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + async def scenario() -> tuple[dict[str, Any], dict[str, Any]]: + transport_entered = asyncio.Event() + transport_cancelled = asyncio.Event() + + async def blocked_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + transport_entered.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + transport_cancelled.set() + raise + + monkeypatch.setattr(facade_module, "scan", blocked_scan) + scan_task = asyncio.create_task(service.scan_ble(6.0)) + await asyncio.wait_for(transport_entered.wait(), timeout=1.0) + scan_task.cancel() + with pytest.raises(asyncio.CancelledError): + await scan_task + assert transport_cancelled.is_set() + assert service._operation_phase is None # noqa: SLF001 + cancelled_state = service.state() + + async def recovered_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return _ble_scan_result("after-cancellation") + + monkeypatch.setattr(facade_module, "scan", recovered_scan) + recovered_state = await service.scan_ble(6.0) + return cancelled_state, recovered_state + + cancelled_state, recovered_state = asyncio.run(scenario()) + + assert cancelled_state["devices"] == [] + cancelled_operation = cancelled_state["last_operation"] + assert cancelled_operation["status"] == "cancelled" + assert cancelled_operation["stage_code"] == "cancelled" + assert cancelled_operation["message_code"] == "discovery.scan.cancelled" + assert cancelled_operation["error"] == { + "category": "transport", + "code": "ble-discovery-cancelled", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + } + assert [item["device_id"] for item in recovered_state["devices"]] == ["after-cancellation"] + + +def test_connect_owns_lifecycle_while_safe_preflight_rejects_a_competing_scan( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_k1(service) + mode_revision = _select_connection_mode(service, "quick-connect") preflight_entered = threading.Event() release_preflight = threading.Event() @@ -395,12 +5291,18 @@ def test_connect_stops_before_ble_write_when_a_new_scan_replaces_its_generation( return { "schema_version": 1, "adapter": "macOS Keychain", - "available": True, + "available": False, "profile_enrolled": False, "credential_source": "exact-firmware-profile", } - async def replacement_scan(_duration_seconds: float) -> dict[str, Any]: + async def replacement_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() return { "devices": [ { @@ -432,10 +5334,11 @@ def test_connect_stops_before_ble_write_when_a_new_scan_replaces_its_generation( async def scenario() -> None: connect_task = asyncio.create_task( service.connect( - ConnectRequest( + _connect_request( device_id="test-ble-transport", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) @@ -444,12 +5347,14 @@ def test_connect_stops_before_ble_write_when_a_new_scan_replaces_its_generation( timeout=1.5, ) assert entered is True - replacement_state = await service.scan_ble(6.0) - assert [item["device_id"] for item in replacement_state["devices"]] == [ - "replacement-device" - ] + with pytest.raises( + facade_module.BleDiscoveryUnavailable, + match="настройки Wi-Fi", + ) as scan_failure: + await service.scan_ble(6.0) + assert scan_failure.value.reason_code == "ble-discovery-blocked-by-provisioning" release_preflight.set() - with pytest.raises(ValueError, match="изменились или устарели"): + with pytest.raises(facade_module.HostWifiProfileError): await asyncio.wait_for(connect_task, timeout=1.0) try: @@ -458,11 +5363,226 @@ def test_connect_stops_before_ble_write_when_a_new_scan_replaces_its_generation( release_preflight.set() +def test_provision_scan_is_rejected_after_atomic_fence_before_control_retirement( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + generation_before = service._ble_discovery_generation # noqa: SLF001 + scan_outcomes: list[object] = [] + provision_calls = 0 + + async def racing_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + return _ble_scan_result("racing-scan") + + original_retire = service._retire_application_control_for_network_change # noqa: SLF001 + + def retire_with_competing_scan() -> None: + assert service._provisioning_active is True # noqa: SLF001 + scan_outcomes.append( + _attempt_competing_scan_from_sync_boundary( + service, + operation_id="op-00000000-0000-4000-8000-000000000101", + ) + ) + original_retire() + + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal provision_calls + provision_calls += 1 + assert service._provisioning_active is True # noqa: SLF001 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-06T12:00:00Z", + "completed_at_utc": "2026-08-06T12:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + { + "status": { + "mode": "WIFI_CLIENT", + "ipv4": "192.168.68.50", + "status_code": 1, + "reserved": 0, + } + } + ], + } + + monkeypatch.setattr(facade_module, "scan", racing_scan) + monkeypatch.setattr( + service, + "_retire_application_control_for_network_change", + retire_with_competing_scan, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + + connected = asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert len(scan_outcomes) == 1 + scan_error = scan_outcomes[0] + assert isinstance(scan_error, facade_module.BleDiscoveryUnavailable) + assert scan_error.reason_code == "ble-discovery-blocked-by-provisioning" + assert service._ble_discovery_generation == generation_before # noqa: SLF001 + assert [item["device_id"] for item in connected["devices"]] == ["k1-a"] + assert provision_calls == 1 + assert connected["connection_mode"] == "bridge" + assert service._provisioning_active is False # noqa: SLF001 + scan_operation = next( + item + for item in connected["operations"] + if item["operation_id"] == "op-00000000-0000-4000-8000-000000000101" + ) + assert scan_operation["status"] == "failed" + assert scan_operation["error"]["code"] == "ble-discovery-blocked-by-provisioning" + + +def test_corrupt_identity_pin_store_fails_after_new_intent_retires_old_control( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + retire_calls = 0 + + def corrupt_identity_lookup(_transport_ref: str) -> None: + raise DeviceIdentityPinStoreCorrupt("synthetic corrupt pin store") + + def record_retirement() -> None: + nonlocal retire_calls + retire_calls += 1 + + async def forbidden_provision(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("identity preflight must fail before the BLE write boundary") + + monkeypatch.setattr(service, "_expected_vendor_device_id", corrupt_identity_lookup) + monkeypatch.setattr( + service, + "_retire_application_control_for_network_change", + record_retirement, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) + + with pytest.raises( + DeviceIdentityPinStoreCorrupt, + match="synthetic corrupt pin store", + ): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert retire_calls == 1 + assert service._provisioning_active is False # noqa: SLF001 + assert service._ble_discovery_generation == 0 # noqa: SLF001 + assert [item["device_id"] for item in service.state()["devices"]] == ["k1-a"] + + +def test_unavailable_semantic_topology_store_fails_after_new_intent_retirement_without_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + service._semantic_topology_store = None # noqa: SLF001 + retire_calls = 0 + + def record_retirement() -> None: + nonlocal retire_calls + retire_calls += 1 + + async def forbidden_provision(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("semantic store preflight must fail before the BLE write boundary") + + monkeypatch.setattr( + service, + "_retire_application_control_for_network_change", + record_retirement, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) + + with pytest.raises( + SemanticTopologyStoreCorrupt, + match="semantic topology store is unavailable", + ): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert retire_calls == 1 + assert service._provisioning_active is False # noqa: SLF001 + assert service._provisioning_gate.locked() is False # noqa: SLF001 + assert service._ble_discovery_generation == 0 # noqa: SLF001 + assert [item["device_id"] for item in service.state()["devices"]] == ["k1-a"] + + def test_verify_connection_adopts_scanned_existing_lan_without_device_write( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + # Ordinary topology adoption must still retire unrelated terminal local + # history. Only a later exact physical SCANNING reconciliation may + # materialize the bounded recovery-only STOP identity. + service._acquisition = facade_module.AcquisitionRecord( # noqa: SLF001 + acquisition_id="stale-terminal-acquisition", + device_id="stale-device", + device_session_id="stale-device-session", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_mode="plugin-commanded", + requested_streams=(), + target_host="192.168.68.99", + duration_seconds=None, + evidence_policy="disabled", + state="failed", + ) + service._acquisition_project_name = "STALE" # noqa: SLF001 _set_scanned_k1(service) status_reads: list[tuple[str, float, bool]] = [] @@ -471,6 +5591,7 @@ def test_verify_connection_adopts_scanned_existing_lan_without_device_write( *, timeout_seconds: float, rediscover: bool, + **_: object, ) -> dict[str, Any]: status_reads.append((device_id, timeout_seconds, rediscover)) return _wifi_status_read("10.255.254.77") @@ -483,15 +5604,19 @@ def test_verify_connection_adopts_scanned_existing_lan_without_device_write( monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) - state = service.verify_connection( - ConnectionVerifyRequest( - device_id="test-ble-transport", - compatibility_attestation=ATTESTATION, + state = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) ) ) - assert status_reads == [("test-ble-transport", 20.0, True)] + assert status_reads == [("test-ble-transport", 20.0, False)] assert state["selected_device_id"] == "test-ble-transport" assert state["k1_ip"] == "10.255.254.77" assert state["connection_mode"] == "bridge" @@ -499,9 +5624,9 @@ def test_verify_connection_adopts_scanned_existing_lan_without_device_write( assert state["device_session"]["connectivity"] == "connected" assert state["compatibility"]["attestation"]["topology"] == "direct-lan" assert state["connection_verification"] == { - "status": "adopted", + "status": "reachable", "lease_state": "reachable", - "lease_generation": 1, + "lease_generation": state["connection_supervisor"]["lease"]["generation"], "endpoint_validation": "ble-wifi-status-read+mqtt-tcp-connect", "network_reachability": "reachable", "host_route_class": "direct-or-routed", @@ -512,7 +5637,153 @@ def test_verify_connection_adopts_scanned_existing_lan_without_device_write( "previous_address_present": False, "write_performed": False, "observed_at": "2026-07-20T12:00:00Z", + "reason_code": None, + "supervisor_revision": state["connection_verification"]["supervisor_revision"], } + assert state["last_operation"]["action"] == "connection.verify" + assert state["last_operation"]["status"] == "succeeded" + assert state["last_operation"]["result"]["write_performed"] is False + assert state["last_operation"]["result"]["control_verified"] is True + assert state["connection_lifecycle"]["connection_ready"] is True + assert state["connection_lifecycle"]["active_mode"] == "bridge" + assert state["acquisition"] is None + assert service._acquisition_project_name is None # noqa: SLF001 + + +def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + generation_before = service._ble_discovery_generation # noqa: SLF001 + scan_outcomes: list[object] = [] + admission_calls = 0 + + async def racing_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + return _ble_scan_result("racing-scan") + + async def fake_status_read(*_: object, **__: object) -> dict[str, Any]: + return _wifi_status_read("10.255.254.77") + + original_retire = service._retire_application_control_for_network_change # noqa: SLF001 + original_apply = service._apply_read_only_device_topology # noqa: SLF001 + + def retire_with_competing_scan() -> None: + assert service._provisioning_active is True # noqa: SLF001 + scan_outcomes.append( + _attempt_competing_scan_from_sync_boundary( + service, + operation_id="op-00000000-0000-4000-8000-000000000102", + ) + ) + original_retire() + + def apply_with_fence_assertion(**kwargs: Any) -> str: + nonlocal admission_calls + admission_calls += 1 + assert service._provisioning_active is True # noqa: SLF001 + return original_apply(**kwargs) + + monkeypatch.setattr(facade_module, "scan", racing_scan) + monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read) + monkeypatch.setattr( + service, + "_retire_application_control_for_network_change", + retire_with_competing_scan, + ) + monkeypatch.setattr( + service, + "_apply_read_only_device_topology", + apply_with_fence_assertion, + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + verified = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert len(scan_outcomes) == 1 + scan_error = scan_outcomes[0] + assert isinstance(scan_error, facade_module.BleDiscoveryUnavailable) + assert scan_error.reason_code == "ble-discovery-blocked-by-provisioning" + assert service._ble_discovery_generation == generation_before # noqa: SLF001 + assert [item["device_id"] for item in verified["devices"]] == ["test-ble-transport"] + assert admission_calls == 1 + assert verified["connection_mode"] == "bridge" + assert verified["k1_ip"] == "10.255.254.77" + assert service._provisioning_active is False # noqa: SLF001 + scan_operation = next( + item + for item in verified["operations"] + if item["operation_id"] == "op-00000000-0000-4000-8000-000000000102" + ) + assert scan_operation["status"] == "failed" + assert scan_operation["error"]["code"] == "ble-discovery-blocked-by-provisioning" + + +def test_connection_verify_operation_id_is_exactly_once_and_request_bound( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + status_reads = 0 + + async def fake_status_read(*_: object, **__: object) -> dict[str, Any]: + nonlocal status_reads + status_reads += 1 + return _wifi_status_read("10.255.254.77") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + operation_id = "op-00000000-0000-4000-8000-000000000002" + request = ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + operation_id=operation_id, + expected_discovery_generation=0, + ) + + first = asyncio.run(service.verify_connection(request)) + repeated = asyncio.run(service.verify_connection(request)) + + assert status_reads == 1 + assert first["last_operation"]["operation_id"] == operation_id + assert repeated["last_operation"]["operation_id"] == operation_id + assert first["last_operation"]["result"]["lease_state"] == "reachable" + assert first["last_operation"]["result"]["lease_generation"] == ( + first["connection_supervisor"]["lease"]["generation"] + ) + with pytest.raises(ValueError, match="different request"): + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="different-k1", + compatibility_attestation=ATTESTATION, + operation_id=operation_id, + expected_discovery_generation=0, + ) + ) + ) def test_verify_connection_adoption_requires_current_scan_candidate( @@ -530,24 +5801,234 @@ def test_verify_connection_adoption_requires_current_scan_candidate( monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) - with pytest.raises(ValueError, match="найдите и выберите"): - service.verify_connection( - ConnectionVerifyRequest( - device_id="not-in-current-scan", - compatibility_attestation=ATTESTATION, + with pytest.raises( + facade_module.ConnectionVerificationError, + match="свежего кандидата", + ) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="not-in-current-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) ) ) + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + operation = service.state()["last_operation"] + assert operation["action"] == "connection.verify" + assert operation["status"] == "failed" + assert operation["error"] == { + "category": "connection", + "code": "connection-verify-candidate-not-fresh", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + } + + +def test_verify_connection_timeout_is_journaled_without_device_side_effect( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + + async def timed_out_status_read(*_: object, **__: object) -> dict[str, Any]: + failure = TimeoutError("synthetic private transport detail") + failure.operation_stage = "exact-uuid-scan" # type: ignore[attr-defined] + raise failure + + monkeypatch.setattr(facade_module, "read_wifi_status_once", timed_out_status_read) + + with ( + caplog.at_level(logging.INFO, logger=facade_module.__name__), + pytest.raises( + facade_module.ConnectionVerificationError, + match="не завершил ожидание точного", + ) as raised, + ): + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-exact-uuid-scan-timeout" + operation = service.state()["last_operation"] + assert operation["error"] == { + "category": "connection", + "code": "connection-verify-exact-uuid-scan-timeout", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + "operation_stage": "exact-uuid-scan", + } + accepted_at = datetime.fromisoformat(operation["accepted_at"]) + deadline_at = datetime.fromisoformat(operation["deadline_at"]) + assert (deadline_at - accepted_at).total_seconds() == ( + facade_module.CONNECTION_VERIFY_HARD_TIMEOUT_SECONDS + ) + failure_log = next( + record + for record in caplog.records + if getattr(record, "event_code", None) == "k1_connection_verify_failed" + ) + assert failure_log.operation_stage == "exact-uuid-scan" + assert failure_log.reason_code == "connection-verify-exact-uuid-scan-timeout" + assert failure_log.failure_category == "timeout" + assert failure_log.device_write_performed is False + assert failure_log.automatic_retry is False + assert "private" not in str(operation) + assert "private" not in caplog.text + + +def test_connection_lease_probe_race_is_an_expected_state_conflict() -> None: + failure = facade_module._connection_verification_error( # noqa: SLF001 + facade_module.ConnectionLeaseUnavailable( + "private concurrent lease mutation detail", + reason_code="connection_lease_changed_during_probe", + ) + ) + + assert failure.reason_code == "connection-verify-lease-changed" + assert str(failure) == "Подключение K1 изменилось во время read-only проверки" + assert "private" not in str(failure) + + +def test_read_only_gatt_contract_is_logged_before_missing_dhcp_rejection( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + + async def status_without_lan_address(*_: object, **__: object) -> dict[str, Any]: + return _wifi_status_read(None) + + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + status_without_lan_address, + ) + + with ( + caplog.at_level(logging.INFO, logger=facade_module.__name__), + pytest.raises(facade_module.ConnectionVerificationError) as raised, + ): + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-address-unavailable" + contract_log = next( + record + for record in caplog.records + if getattr(record, "event_code", None) == "k1_ble_gatt_contract_observed" + ) + assert contract_log.write_characteristic_properties == ["read", "write"] + assert contract_log.max_write_without_response_size == 253 + assert contract_log.mtu_size == 256 + assert contract_log.bridge_frame_length == 99 + assert contract_log.quick_connect_frame_length == 100 + assert contract_log.device_write_performed is False + assert contract_log.automatic_retry is False + assert "device_macos_uuid" not in contract_log.__dict__ + assert "write_characteristic_uuid" not in contract_log.__dict__ + @pytest.mark.parametrize( - ("ipv4", "is_local", "route_class", "endpoint_reachable", "message"), + ("cleanup_pending", "reason_code"), [ - (None, False, "direct-or-routed", True, "не сообщил актуальный DHCP-адрес"), - ("192.168.56.1", False, "device-ap", True, "не сообщил актуальный DHCP-адрес"), - ("10.255.254.77", True, "direct-or-routed", True, "этому компьютеру"), - ("10.255.254.77", False, "tunnel", True, "прямой локальный маршрут"), - ("10.255.254.77", False, "default-route", True, "прямой локальный маршрут"), - ("10.255.254.77", False, "direct-or-routed", False, "1883 недоступен"), + (False, "connection-verify-busy"), + (True, "connection-verify-cleanup-pending"), + ], +) +def test_verify_connection_preserves_process_ble_busy_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + cleanup_pending: bool, + reason_code: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + + async def busy_status_read(*_: object, **__: object) -> dict[str, Any]: + raise BleRuntimeBusy( + active_operation_kind="scan", + cleanup_pending=cleanup_pending, + ) + + monkeypatch.setattr(facade_module, "read_wifi_status_once", busy_status_read) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert raised.value.reason_code == reason_code + operation = service.state()["last_operation"] + assert operation["error"] == { + "category": "connection", + "code": reason_code, + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + } + + +@pytest.mark.parametrize( + ("ipv4", "is_local", "route_class", "endpoint_reachable", "reason_code"), + [ + (None, False, "direct-or-routed", True, "connection-verify-address-unavailable"), + ( + "192.168.56.1", + False, + "device-ap", + True, + "connection-verify-address-unavailable", + ), + ( + "10.255.254.77", + True, + "direct-or-routed", + True, + "connection-verify-local-address-conflict", + ), + ("10.255.254.77", False, "tunnel", True, "connection-verify-route-mismatch"), + ( + "10.255.254.77", + False, + "default-route", + True, + "connection-verify-route-mismatch", + ), + ( + "10.255.254.77", + False, + "direct-or-routed", + False, + "connection-verify-mqtt-unreachable", + ), ], ) def test_verify_connection_adoption_fails_closed_before_establishing_lease( @@ -557,7 +6038,7 @@ def test_verify_connection_adoption_fails_closed_before_establishing_lease( is_local: bool, route_class: str, endpoint_reachable: bool, - message: str, + reason_code: str, ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_k1(service) @@ -577,33 +6058,241 @@ def test_verify_connection_adoption_fails_closed_before_establishing_lease( "_control_endpoint_reachable", lambda _target: endpoint_reachable, ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) - with pytest.raises(RuntimeError, match=message): - service.verify_connection( - ConnectionVerifyRequest( - device_id="test-ble-transport", - compatibility_attestation=ATTESTATION, + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) ) ) + assert raised.value.reason_code == reason_code + state = service.state() + assert [item["device_id"] for item in state["devices"]] == ["test-ble-transport"] + # Fresh Bridge 7f02 evidence is provisional until route/TCP and exact + # DeviceInfo identity all succeed. Any failure before that point retires + # only the ephemeral projection, retains the fresh candidate for a later + # explicit choice, and never publishes untrusted durable topology. assert state["selected_device_id"] is None assert state["k1_ip"] is None assert state["connection_mode"] is None assert state["device_session"] is None + assert state["semantic_topology_store"]["status"] == "empty" + assert state["connection_supervisor"]["authority"]["control_allowed"] is False assert state["connection_verification"].get("write_performed") is not True -def test_connection_verify_request_requires_exact_bridge_attestation() -> None: - with pytest.raises(ValidationError, match="provided together"): +def test_apply_owned_rescan_after_unreachable_verify_dispatches_exactly_one_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + device_id = "k1-verify-then-apply" + _set_scanned_k1(service, device_id=device_id) + old_capture = _SYNTHETIC_SCAN_CAPTURES[device_id] + replacement_capture = facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address=device_id), # type: ignore[arg-type] + macos_uuid=device_id, + owner_epoch=old_capture.owner_epoch, + scan_generation=old_capture.scan_generation + 1, + ) + active_capture = old_capture + invalidated_capture_ids: set[int] = set() + pinned_by_session: dict[str, facade_module.CapturedDiscoveredDevice] = {} + + def exact_capture(requested_device_id: str) -> object | None: + assert requested_device_id == device_id + if id(active_capture) in invalidated_capture_ids: + return None + return active_capture + + def pin_exact_capture( + capture: facade_module.CapturedDiscoveredDevice, + *, + device_session_id: str, + ) -> None: + assert capture is active_capture + pinned_by_session[device_session_id] = capture + + def invalidate_exact_session( + requested_device_id: str, + *, + device_session_id: str, + ) -> bool: + assert requested_device_id == device_id + retired = pinned_by_session.pop(device_session_id, None) + if retired is None: + return False + invalidated_capture_ids.add(id(retired)) + return True + + async def status_on_unreachable_existing_network( + *_: object, + captured_device: facade_module.CapturedDiscoveredDevice | None = None, + on_gatt_validated: Callable[[facade_module.CapturedDiscoveredDevice], None] | None = None, + **__: object, + ) -> dict[str, Any]: + assert captured_device is old_capture + assert on_gatt_validated is not None + on_gatt_validated(old_capture) + return _wifi_status_read("192.168.1.20", device_id=device_id) + + monkeypatch.setattr(facade_module, "_capture_network_intent_device", exact_capture) + monkeypatch.setattr(facade_module, "capture_discovered_device", exact_capture) + monkeypatch.setattr(facade_module, "pin_connected_device_handle", pin_exact_capture) + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + invalidate_exact_session, + ) + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + status_on_unreachable_existing_network, + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "direct-or-routed", + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: False, + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as unavailable: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id=device_id, + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert unavailable.value.reason_code == "connection-verify-mqtt-unreachable" + after_verify = service.state() + assert [item["device_id"] for item in after_verify["devices"]] == [device_id] + assert after_verify["selected_device_id"] is None + assert invalidated_capture_ids == {id(old_capture)} + assert exact_capture(device_id) is None + + async def apply_owned_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + nonlocal active_capture + assert on_admitted is not None + on_admitted() + active_capture = replacement_capture + return _ble_scan_result(device_id) + + monkeypatch.setattr(facade_module, "scan", apply_owned_scan) + rescanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + assert rescanned["ble_discovery_generation"] == 1 + assert exact_capture(device_id) is replacement_capture + + dispatched_writes: list[facade_module.CapturedDiscoveredDevice] = [] + + async def provision_once( + *_: object, + captured_device: facade_module.CapturedDiscoveredDevice | None = None, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + assert captured_device is replacement_capture + _dispatch_test_network_write(on_write_dispatch) + dispatched_writes.append(captured_device) + return { + "started_at_utc": "2026-08-11T07:00:00Z", + "completed_at_utc": "2026-08-11T07:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": _wifi_status_read(None)["status"], + "observations": [{"status": _wifi_status_read("192.168.1.20")["status"]}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", provision_once) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + connected = asyncio.run( + service.connect( + _connect_request( + device_id=device_id, + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + expected_discovery_generation=rescanned["ble_discovery_generation"], + ) + ) + ) + + assert dispatched_writes == [replacement_capture] + assert connected["connection_lifecycle"]["connection_ready"] is True + network_operations = [ + item for item in connected["operations"] if item["action"] == "network.provision" + ] + assert len(network_operations) == 1 + assert network_operations[0]["status"] == "succeeded" + verify_operations = [ + item for item in connected["operations"] if item["action"] == "connection.verify" + ] + assert len(verify_operations) == 1 + assert verify_operations[0]["status"] == "failed" + assert verify_operations[0]["error"]["side_effect_status"] == "none" + + +def test_connection_verify_request_requires_paired_exact_firmware_attestation() -> None: + with pytest.raises(ValidationError, match="compatibility_attestation"): ConnectionVerifyRequest(device_id="test-ble-transport") - with pytest.raises(ValidationError, match="topology=direct-lan"): + with pytest.raises(ValidationError, match="device_id"): + ConnectionVerifyRequest(compatibility_attestation=ATTESTATION) + + with pytest.raises(ValidationError, match="3.0.2"): ConnectionVerifyRequest( device_id="test-ble-transport", - compatibility_attestation=QUICK_CONNECT_ATTESTATION, + compatibility_attestation={ + "firmware_version": "3.0.3", + "topology": "direct-lan", + "verification": "live-device-info", + }, ) + with pytest.raises(ValidationError, match="expected_discovery_generation"): + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + ) + + quick_connect = ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_discovery_generation=0, + ) + assert quick_connect.compatibility_attestation == QUICK_CONNECT_ATTESTATION + def test_verify_connection_refreshes_dynamic_dhcp_address_and_rotates_session( monkeypatch: pytest.MonkeyPatch, @@ -617,31 +6306,128 @@ def test_verify_connection_refreshes_dynamic_dhcp_address_and_rotates_session( service._device_session_id = "old-device-session" # noqa: SLF001 service._device_session_opened_at = "2026-07-20T10:00:00Z" # noqa: SLF001 service._device_calibration = {"status": "available"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + _set_scanned_k1(service) async def fake_status_read(*_: object, **__: object) -> dict[str, Any]: return _wifi_status_read("10.255.254.77") monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read) monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda target: target == "10.255.254.77", + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) - state = service.verify_connection() + state = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) assert state["k1_ip"] == "10.255.254.77" assert state["device_session"]["device_session_id"] != "old-device-session" assert state["connection_verification"] == { - "status": "live-address-observed", - "lease_state": "configured", - "lease_generation": 1, - "endpoint_validation": "ble-wifi-status-read", - "network_reachability": "not-probed", + "status": "reachable", + "lease_state": "reachable", + "lease_generation": state["connection_supervisor"]["lease"]["generation"], + "endpoint_validation": "ble-wifi-status-read+mqtt-tcp-connect", + "network_reachability": "reachable", + "host_route_class": "direct-or-routed", + "address_source": "ble-wifi-status-read", + "connection_origin": "external-existing-network", + "admission_source": "connection.verify", "address_changed": True, "previous_address_present": True, "write_performed": False, "observed_at": "2026-07-20T12:00:00Z", + "reason_code": None, + "supervisor_revision": state["connection_verification"]["supervisor_revision"], } assert state["device_calibration"]["status"] == "unavailable" +def test_bridge_read_only_verify_never_shortcuts_fresh_ble_status( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._selected_device_id = "test-ble-transport" # noqa: SLF001 + service._connection_mode = "bridge" # noqa: SLF001 + service._k1_ip = "10.255.254.54" # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + service._device_id = "known-k1" # noqa: SLF001 + service._device_session_id = "known-device-session" # noqa: SLF001 + _set_scanned_k1(service) + _seed_supervised_connection( + service, + target_ipv4="10.255.254.54", + with_control=False, + ) + service._device_session_id = "known-device-session" # noqa: SLF001 + + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + status_reads: list[tuple[str, float, bool]] = [] + + async def exact_status_read( + device_id: str, + *, + timeout_seconds: float, + rediscover: bool, + **_: object, + ) -> dict[str, Any]: + status_reads.append((device_id, timeout_seconds, rediscover)) + return _wifi_status_read("10.255.254.54") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", exact_status_read) + + state = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert status_reads == [("test-ble-transport", 20.0, False)] + assert state["k1_ip"] == "10.255.254.54" + # A fresh exact BLE observation creates a new intent/session fence even + # when DHCP returned the same address. Similar endpoint text is not + # continuity authority. + assert state["device_session"]["device_session_id"] != "known-device-session" + assert state["connection_verification"] == { + "status": "reachable", + "lease_state": "reachable", + "lease_generation": state["connection_supervisor"]["lease"]["generation"], + "endpoint_validation": "ble-wifi-status-read+mqtt-tcp-connect", + "network_reachability": "reachable", + "host_route_class": "direct-or-routed", + "address_source": "ble-wifi-status-read", + "connection_origin": "external-existing-network", + "admission_source": "connection.verify", + "address_changed": False, + "previous_address_present": True, + "write_performed": False, + "observed_at": "2026-07-20T12:00:00Z", + "reason_code": None, + "supervisor_revision": state["connection_verification"]["supervisor_revision"], + } + assert state["last_operation"]["status"] == "succeeded" + + def test_implicit_acquisition_target_uses_current_ble_dhcp_address( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -650,22 +6436,42 @@ def test_implicit_acquisition_target_uses_current_ble_dhcp_address( service._selected_device_id = "test-ble-transport" # noqa: SLF001 service._connection_mode = "bridge" # noqa: SLF001 service._k1_ip = "10.255.254.54" # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + status_loop: asyncio.AbstractEventLoop | None = None + runtime_loop: asyncio.AbstractEventLoop | None = None async def fake_status_read(*_: object, **__: object) -> dict[str, Any]: + nonlocal status_loop + status_loop = asyncio.get_running_loop() return _wifi_status_read("10.255.254.77") monkeypatch.setattr(facade_module, "read_wifi_status_once", fake_status_read) monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) - - state = service.prepare_acquisition( - PrepareAcquisitionRequest( - project_name=PROJECT_NAME, - compatibility_attestation=ATTESTATION, - ) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda target: target == "10.255.254.77", ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + async def scenario() -> dict[str, Any]: + nonlocal runtime_loop + runtime_loop = asyncio.get_running_loop() + service.bind_runtime_event_loop() + return await asyncio.to_thread( + service.prepare_acquisition, + _prepare_request( + project_name=PROJECT_NAME, + compatibility_attestation=ATTESTATION, + ), + ) + + state = asyncio.run(scenario()) assert state["k1_ip"] == "10.255.254.77" assert state["acquisition"]["target_host"] == "10.255.254.77" + assert status_loop is runtime_loop def test_control_session_reuses_reachable_process_owned_connection_lease( @@ -673,20 +6479,37 @@ def test_control_session_reuses_reachable_process_owned_connection_lease( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - service._selected_device_id = "test-ble-transport" # noqa: SLF001 - service._connection_mode = "bridge" # noqa: SLF001 - service._k1_ip = "10.255.254.54" # noqa: SLF001 - service._device_id = "known-k1" # noqa: SLF001 + _seed_supervised_connection( + service, + target_ipv4="10.255.254.54", + with_control=False, + ) service._device_session_id = "known-session" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 opened_hosts: list[str] = [] class FakeOpenControlSession: - def snapshot(self) -> dict[str, object]: - return {"state": "idle", "can_confirm_standby": False} + def __init__(self) -> None: + self.state = "idle" + self.verified_control: dict[str, object] | None = None - def open(self, *, host: str, **_: object) -> dict[str, object]: + def snapshot(self) -> dict[str, object]: + return { + "state": self.state, + "can_confirm_standby": False, + "verified_control": self.verified_control, + } + + def open( + self, + *, + host: str, + connection_binding: ApplicationConnectionBinding, + **_: object, + ) -> dict[str, object]: opened_hosts.append(host) + self.state = "connection-ready" + self.verified_control = _verified_control_for_binding(connection_binding) return self.snapshot() async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: @@ -695,6 +6518,8 @@ def test_control_session_reuses_reachable_process_owned_connection_lease( service._application_control_session = FakeOpenControlSession() # type: ignore[assignment] # noqa: SLF001 monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + admission_generation = service._connection_supervisor.snapshot().lease.generation # noqa: SLF001 state = service.open_application_control_session( OpenApplicationControlSessionRequest( @@ -718,12 +6543,13 @@ def test_control_session_reuses_reachable_process_owned_connection_lease( ) assert open_operation["status"] == "succeeded" assert open_operation["result"] == { - "lease_generation": 0, + "lease_generation": admission_generation, "connection_lease_reused": True, "recovery_performed": False, "address_changed": False, "device_write_performed": False, } + service._release_application_control_process_lease() # noqa: SLF001 def test_reachable_connection_lease_supports_repeated_independent_control_sessions( @@ -731,25 +6557,41 @@ def test_reachable_connection_lease_supports_repeated_independent_control_sessio tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - service._selected_device_id = "test-ble-transport" # noqa: SLF001 - service._connection_mode = "bridge" # noqa: SLF001 - service._k1_ip = "10.255.254.54" # noqa: SLF001 - service._device_id = "known-k1" # noqa: SLF001 + _seed_supervised_connection( + service, + target_ipv4="10.255.254.54", + with_control=False, + ) service._device_session_id = "known-session" # noqa: SLF001 - service._connection_lease_generation = 7 # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 opened_hosts: list[str] = [] class FakeCompletedControlSession: + def __init__(self) -> None: + self.state = "completed" + self.verified_control: dict[str, object] | None = None + def snapshot(self) -> dict[str, object]: return { - "state": "completed", + "state": self.state, "can_open": True, "can_confirm_standby": False, + "verified_control": self.verified_control, } - def open(self, *, host: str, **_: object) -> dict[str, object]: + def open( + self, + *, + host: str, + connection_binding: ApplicationConnectionBinding, + **_: object, + ) -> dict[str, object]: opened_hosts.append(host) + self.state = "connection-ready" + self.verified_control = _verified_control_for_binding( + connection_binding, + control_session_id=f"fake-control-session-{len(opened_hosts)}", + ) return self.snapshot() async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: @@ -758,6 +6600,7 @@ def test_reachable_connection_lease_supports_repeated_independent_control_sessio service._application_control_session = FakeCompletedControlSession() # type: ignore[assignment] # noqa: SLF001 monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) request = OpenApplicationControlSessionRequest( operator_present=True, owner_controlled_device=True, @@ -768,11 +6611,16 @@ def test_reachable_connection_lease_supports_repeated_independent_control_sessio ) service.open_application_control_session(request) + fake_session = service._application_control_session # noqa: SLF001 + fake_session.state = "completed" # type: ignore[attr-defined] state = service.open_application_control_session(request) assert opened_hosts == ["10.255.254.54", "10.255.254.54"] assert state["device_session"]["device_session_id"] == "known-session" - assert state["connection_verification"]["lease_generation"] == 7 + assert ( + state["connection_verification"]["lease_generation"] + == (state["connection_supervisor"]["lease"]["generation"]) + ) open_operations = [ operation for operation in state["operations"] @@ -783,6 +6631,242 @@ def test_reachable_connection_lease_supports_repeated_independent_control_sessio assert all( operation["result"]["connection_lease_reused"] is True for operation in open_operations ) + service._release_application_control_process_lease() # noqa: SLF001 + + +@pytest.mark.parametrize("continuity_change", ["host-epoch", "bridge-to-quick"]) +def test_old_device_info_binding_cannot_regain_authority_after_connection_change( + tmp_path: Path, + continuity_change: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + old_binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="bridge", + logical_device_id="known-k1", + with_control=False, + ) + old_control_snapshot = { + "state": "connection-ready", + "verified_control": _verified_control_for_binding(old_binding), + } + + service._reconcile_connection_supervisor( # noqa: SLF001 + old_control_snapshot, + runtime.snapshot(), + ) + assert service._connection_supervisor.snapshot().authority.control_allowed is True # noqa: SLF001 + + target = EndpointTarget( + facade_module.AP_FALLBACK_IPV4, + facade_module.CONTROL_MQTT_PORT, + ) + if continuity_change == "host-epoch": + new_epoch = service._connection_supervisor.observe_host_path( # noqa: SLF001 + HostPathProbeResult( + available=True, + fingerprint="test-route-after-wifi-reassociation", + interface="test1", + source_ipv4="192.168.56.3", + route_class="direct", + ) + ) + current_intent = service._connection_supervisor.snapshot().intent # noqa: SLF001 + assert current_intent is not None + assert service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=target, + intent_id=current_intent.intent_id, + host_path_epoch=new_epoch, + reachable=True, + ) + else: + new_intent_id = "test-intent-quick-after-bridge" + service._connection_supervisor.set_intent( # noqa: SLF001 + intent_id=new_intent_id, + requested_mode="quick-connect", + expected_device_id="known-k1", + ) + assert service._connection_supervisor.observe_device_network_applied( # noqa: SLF001 + intent_id=new_intent_id, + transport_ref="test-ble-transport", + connection_mode="quick-connect", + target=target, + source="ble-post-write-status", + ) + epoch = service._connection_supervisor.snapshot().host_path.epoch # noqa: SLF001 + assert service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=target, + intent_id=new_intent_id, + host_path_epoch=epoch, + reachable=True, + ) + + # The old application worker may still publish its old DeviceInfo snapshot. + # Reconciliation must compare the immutable binding, never relabel it with + # the new intent/route merely because the same TCP address answers. + service._reconcile_connection_supervisor( # noqa: SLF001 + old_control_snapshot, + runtime.snapshot(), + ) + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.endpoint.tcp_state == "reachable" + assert supervisor.authority.control_allowed is False + assert supervisor.authority.acquisition_start_allowed is False + assert supervisor.device_identity.state != "verified" + assert supervisor.lease.state == "configured-unverified" + + +def test_application_command_binding_guard_rejects_changed_host_epoch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + monkeypatch.setattr(service, "_sample_host_path", _direct_host_path) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "snapshot", + lambda: { + "state": "connection-ready", + "verified_control": _verified_control_for_binding( + binding, + control_session_id="test-control-test-ble-transport", + ), + }, + ) + + service._validate_application_connection_binding(binding) # noqa: SLF001 + service._connection_supervisor.observe_host_path( # noqa: SLF001 + HostPathProbeResult( + available=True, + fingerprint="route-after-mac-wifi-change", + interface="test1", + source_ipv4="192.168.56.3", + route_class="direct", + ) + ) + + with pytest.raises(ApplicationConnectionBindingLost) as caught: + service._validate_application_connection_binding(binding) # noqa: SLF001 + + assert caught.value.reason_code == "application-connection-binding-lost" + + +def test_explicit_stop_dispatch_waits_for_transient_local_lifecycle_holder( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + monkeypatch.setattr(service, "_sample_host_path", _direct_host_path) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "snapshot", + lambda: { + "state": "scanning", + "verified_control": _verified_control_for_binding(binding), + }, + ) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + entered = threading.Event() + completed = threading.Event() + release_callbacks: list[Callable[[], None]] = [] + failures: list[BaseException] = [] + + def acquire_stop_lease() -> None: + entered.set() + try: + release_callbacks.append( + service._acquire_application_dispatch_lease( # noqa: SLF001 + binding, + lambda: False, + ) + ) + except BaseException as exc: # pragma: no branch - asserted below + failures.append(exc) + finally: + completed.set() + + worker = threading.Thread(target=acquire_stop_lease) + worker.start() + assert entered.wait(1.0) + assert completed.wait(0.1) is False + + service._k1_command_dispatch_gate.release() # noqa: SLF001 + assert completed.wait(1.0) + worker.join(timeout=1.0) + + assert failures == [] + assert len(release_callbacks) == 1 + release_callbacks[0]() + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + +def test_explicit_stop_dispatch_contention_expires_without_crossing_publish_lease( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + expired = threading.Event() + completed = threading.Event() + failures: list[BaseException] = [] + + def acquire_stop_lease() -> None: + try: + service._acquire_application_dispatch_lease( # noqa: SLF001 + binding, + expired.is_set, + ) + except BaseException as exc: # pragma: no branch - asserted below + failures.append(exc) + finally: + completed.set() + + worker = threading.Thread(target=acquire_stop_lease) + worker.start() + assert completed.wait(0.1) is False + expired.set() + assert completed.wait(1.0) + worker.join(timeout=1.0) + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert len(failures) == 1 + assert isinstance(failures[0], ApplicationConnectionBindingLost) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + +def test_application_command_guard_projects_new_remote_proof_before_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + monkeypatch.setattr(service, "_sample_host_path", _direct_host_path) + control_session_id = "test-control-test-ble-transport" + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "snapshot", + lambda: { + "state": "connection-ready", + "verified_control": _verified_control_for_binding( + binding, + control_session_id=control_session_id, + control_proof_revision=2, + control_proof_source="mqtt-heartbeat", + ), + }, + ) + + service._validate_application_connection_binding(binding) # noqa: SLF001 + + supervisor = service._connection_supervisor # noqa: SLF001 + assert supervisor.snapshot().authority.control_allowed is True + assert supervisor._control_proof_revision == 2 # noqa: SLF001 + assert supervisor._control_proof_source == "mqtt-heartbeat" # noqa: SLF001 def test_control_session_recovers_changed_bridge_address_without_wifi_write( @@ -790,21 +6874,38 @@ def test_control_session_recovers_changed_bridge_address_without_wifi_write( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - service._selected_device_id = "test-ble-transport" # noqa: SLF001 - service._connection_mode = "bridge" # noqa: SLF001 - service._k1_ip = "10.255.254.54" # noqa: SLF001 - service._device_id = "known-k1" # noqa: SLF001 + _seed_supervised_connection( + service, + target_ipv4="10.255.254.54", + with_control=False, + ) service._device_session_id = "old-session" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 opened_hosts: list[str] = [] status_calls: list[dict[str, object]] = [] class FakeOpenControlSession: - def snapshot(self) -> dict[str, object]: - return {"state": "idle", "can_confirm_standby": False} + def __init__(self) -> None: + self.state = "idle" + self.verified_control: dict[str, object] | None = None - def open(self, *, host: str, **_: object) -> dict[str, object]: + def snapshot(self) -> dict[str, object]: + return { + "state": self.state, + "can_confirm_standby": False, + "verified_control": self.verified_control, + } + + def open( + self, + *, + host: str, + connection_binding: ApplicationConnectionBinding, + **_: object, + ) -> dict[str, object]: opened_hosts.append(host) + self.state = "connection-ready" + self.verified_control = _verified_control_for_binding(connection_binding) return self.snapshot() async def fake_status_read(*_: object, **kwargs: object) -> dict[str, Any]: @@ -820,23 +6921,29 @@ def test_control_session_recovers_changed_bridge_address_without_wifi_write( "_control_endpoint_reachable", lambda target: target == "10.255.254.77", ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) - state = service.open_application_control_session( - OpenApplicationControlSessionRequest( - operator_present=True, - owner_controlled_device=True, - lixelgo_closed=True, - battery_storage_confirmed=True, - expected_physical_state_confirmed=True, - timezone_name="Europe/Moscow", + async def scenario() -> dict[str, Any]: + service.bind_runtime_event_loop() + return await asyncio.to_thread( + service.open_application_control_session, + OpenApplicationControlSessionRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + timezone_name="Europe/Moscow", + ), ) - ) + + state = asyncio.run(scenario()) assert status_calls == [{"timeout_seconds": 20.0, "rediscover": True}] assert opened_hosts == ["10.255.254.77"] assert state["k1_ip"] == "10.255.254.77" assert state["device_session"]["device_session_id"] != "old-session" - assert state["connection_verification"]["status"] == "recovered" + assert state["connection_verification"]["status"] == "reachable" assert state["connection_verification"]["write_performed"] is False open_operation = next( operation @@ -853,12 +6960,14 @@ def test_control_session_prestart_failure_is_journaled_and_marks_lease_offline( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - service._selected_device_id = "test-ble-transport" # noqa: SLF001 - service._connection_mode = "bridge" # noqa: SLF001 - service._k1_ip = "10.255.254.54" # noqa: SLF001 - service._device_id = "known-k1" # noqa: SLF001 + _seed_supervised_connection( + service, + target_ipv4="10.255.254.54", + endpoint_reachable=False, + with_control=False, + ) service._device_session_id = "known-session" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 async def failed_status_read(*_: object, **__: object) -> dict[str, Any]: raise TimeoutError("synthetic BLE recovery timeout") @@ -885,7 +6994,7 @@ def test_control_session_prestart_failure_is_journaled_and_marks_lease_offline( state = service.state() assert state["application_control_session"]["state"] == "idle" assert state["device_session"]["connectivity"] == "offline" - assert state["connection_verification"]["lease_state"] == "disconnected" + assert state["connection_verification"]["lease_state"] == "configured-unverified" open_operation = next( operation for operation in state["operations"] @@ -901,24 +7010,28 @@ def test_control_session_prestart_failure_is_journaled_and_marks_lease_offline( } -def test_bridge_route_mismatch_stops_before_ble_recovery_and_vendor_commands( +def test_bridge_route_mismatch_persists_read_only_recovery_before_rejecting_control( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) - service._selected_device_id = "test-ble-transport" # noqa: SLF001 - service._connection_mode = "bridge" # noqa: SLF001 - service._k1_ip = "192.168.68.50" # noqa: SLF001 - service._device_id = "known-k1" # noqa: SLF001 + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + host_path=_tunnel_host_path("192.168.68.50"), + endpoint_reachable=False, + endpoint_reason="connection_lease_host_route_mismatch", + with_control=False, + ) service._device_session_id = "known-session" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 - - async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: - raise AssertionError("host route mismatch must stop before BLE recovery") + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False) monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel") - monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) + service._read_wifi_status_on_runtime_loop = ( # type: ignore[method-assign] # noqa: SLF001 + lambda *_args, **_kwargs: _wifi_status_read("192.168.68.77") + ) with pytest.raises( facade_module.ConnectionLeaseUnavailable, @@ -936,9 +7049,10 @@ def test_bridge_route_mismatch_stops_before_ble_recovery_and_vendor_commands( ) state = service.state() - assert state["connection_verification"]["reason_code"] == ( - "connection_lease_host_route_mismatch" - ) + assert state["k1_ip"] == "192.168.68.77" + assert state["semantic_topology_store"]["record"]["ipv4"] == "192.168.68.77" + assert state["device_session"]["device_session_id"] != "known-session" + assert state["connection_verification"]["lease_state"] == "configured-unverified" assert state["connection_verification"]["endpoint_validation"] == "host-route" assert state["connection_verification"]["host_route_class"] == "tunnel" assert state["application_control_session"]["state"] == "idle" @@ -950,7 +7064,7 @@ def test_prepare_creates_provisional_device_session_and_profiled_acquisition( service, runtime = service_with_fake_runtime(tmp_path) state = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -970,7 +7084,7 @@ def test_prepare_creates_provisional_device_session_and_profiled_acquisition( def test_project_name_is_normalized_and_control_characters_are_rejected() -> None: - request = PrepareAcquisitionRequest( + request = _prepare_request( project_name=" Ｋ１ Lab ", host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -979,7 +7093,7 @@ def test_project_name_is_normalized_and_control_characters_are_rejected() -> Non assert request.project_name == "K1 Lab" for invalid in (" ", "line\nbreak", "\ud800", "x" * 97): with pytest.raises(ValidationError): - PrepareAcquisitionRequest( + _prepare_request( project_name=invalid, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -987,12 +7101,12 @@ def test_project_name_is_normalized_and_control_characters_are_rejected() -> Non def test_acquisition_is_unbounded_by_default_and_accepts_ten_hour_hint() -> None: - unbounded = PrepareAcquisitionRequest( + unbounded = _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) - ten_hours = PrepareAcquisitionRequest( + ten_hours = _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=10 * 60 * 60, @@ -1004,8 +7118,15 @@ def test_acquisition_is_unbounded_by_default_and_accepts_ten_hour_hint() -> None def test_connection_modes_require_their_exact_topology_attestation() -> None: - assert ( + with pytest.raises(ValidationError, match="idempotency_key"): ConnectRequest( + device_id="synthetic-device", + ssid="synthetic-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + assert ( + _connect_request( device_id="synthetic-device", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, @@ -1013,7 +7134,7 @@ def test_connection_modes_require_their_exact_topology_attestation() -> None: == "quick-connect" ) assert ( - ConnectRequest( + _connect_request( device_id="synthetic-device", ssid="synthetic-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -1022,14 +7143,46 @@ def test_connection_modes_require_their_exact_topology_attestation() -> None: ).connection_mode == "direct-connect" ) + bridge = _connect_request( + device_id="synthetic-device", + ssid="synthetic-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + assert bridge.allow_host_wifi_switch is False + assert ( + _connect_request( + device_id="synthetic-device", + ssid="synthetic-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + allow_host_wifi_switch=True, + ).allow_host_wifi_switch + is True + ) + with pytest.raises(ValidationError, match="valid boolean"): + _connect_request( + device_id="synthetic-device", + ssid="synthetic-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + allow_host_wifi_switch="yes", + ) + with pytest.raises(ValidationError, match="only for Bridge"): + _connect_request( + device_id="synthetic-device", + connection_mode="quick-connect", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + allow_host_wifi_switch=True, + ) with pytest.raises(ValidationError): - ConnectRequest( + _connect_request( device_id="synthetic-device", connection_mode="quick-connect", compatibility_attestation=ATTESTATION, ) with pytest.raises(ValidationError, match="host Wi-Fi profile"): - ConnectRequest( + _connect_request( device_id="synthetic-device", ssid="synthetic-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -1037,14 +7190,14 @@ def test_connection_modes_require_their_exact_topology_attestation() -> None: compatibility_attestation=QUICK_CONNECT_ATTESTATION, ) with pytest.raises(ValidationError, match="32 UTF-8 bytes"): - ConnectRequest( + _connect_request( device_id="synthetic-device", ssid="🛰️" * 9, password=SecretStr(PRIMARY_TEST_CREDENTIAL), compatibility_attestation=ATTESTATION, ) with pytest.raises(ValidationError, match="64 UTF-8 bytes"): - ConnectRequest( + _connect_request( device_id="synthetic-device", ssid="synthetic-network", password=SecretStr("🔒" * 17), @@ -1054,14 +7207,14 @@ def test_connection_modes_require_their_exact_topology_attestation() -> None: def test_only_physically_accepted_mount_and_gnss_values_are_admitted() -> None: with pytest.raises(ValidationError): - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", mount_type="uav", # type: ignore[arg-type] compatibility_attestation=ATTESTATION, ) with pytest.raises(ValidationError): - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", gnss_mode="rtk", # type: ignore[arg-type] @@ -1147,7 +7300,7 @@ def test_prepare_acquisition_revokes_existing_shadow_authority_lease(tmp_path: P service.runtime = FakeVisualizationRuntime() # type: ignore[assignment] service._selected_device_id = "test-ble-transport" # noqa: SLF001 service._k1_ip = "192.168.1.20" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 service.arm_application_control_shadow( ShadowApplicationControlArmRequest( operator_confirmed=True, @@ -1156,7 +7309,7 @@ def test_prepare_acquisition_revokes_existing_shadow_authority_lease(tmp_path: P ) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -1171,7 +7324,7 @@ def test_prepare_acquisition_revokes_existing_shadow_authority_lease(tmp_path: P def test_operator_manual_start_is_confirmed_only_by_real_point_data(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -1180,7 +7333,7 @@ def test_operator_manual_start_is_confirmed_only_by_real_point_data(tmp_path: Pa ) acquisition_id = prepared["acquisition"]["acquisition_id"] - starting = service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + starting = service.start_acquisition(_start_request(acquisition_id=acquisition_id)) assert starting["acquisition"]["state"] == "starting" assert starting["last_operation"]["status"] == "running" @@ -1199,17 +7352,20 @@ def test_operator_manual_start_is_confirmed_only_by_real_point_data(tmp_path: Pa assert acquiring["last_operation"]["result"]["confirmation"] == "point-frame" +@pytest.mark.parametrize("proof_first", [False, True]) def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + proof_first: bool, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) control = FakeInteractiveControlSession() service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 - service._k1_ip = "192.168.1.20" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name="TEST001", host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1221,23 +7377,80 @@ def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions( assert runtime.start_calls == [] with pytest.raises(ValueError, match="подтверждения присутствия"): - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) assert control.start_projects == [] - service.start_acquisition( - StartAcquisitionRequest( + pre_pcl_state = service.start_acquisition( + _start_request( acquisition_id=acquisition_id, physical_acceptance=PHYSICAL_ACCEPTANCE, ) ) + pre_pcl_camera_streams = [ + stream + for stream in pre_pcl_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert pre_pcl_state["camera_preview"]["active_source_id"] is None + assert pre_pcl_state["camera_preview"]["delivery"] is None + assert pre_pcl_state["camera_preview"]["activation_admission"]["state"] == ( + "waiting-for-first-authoritative-pcl" + ) + assert all(stream["activation"]["selected"] is False for stream in pre_pcl_camera_streams) + assert all(stream["activation"]["controllable"] is False for stream in pre_pcl_camera_streams) + assert all(stream["delivery"] is None for stream in pre_pcl_camera_streams) + pre_pcl_camera = service.camera_preview.snapshot() + assert pre_pcl_camera["active_source_id"] is None + assert pre_pcl_camera["delivery"] is None + assert pre_pcl_camera["recording"]["active"] is False + assert pre_pcl_camera["recording"]["producer_alive"] is False assert control.start_projects == ["TEST001"] + assert control.stop_calls == 0 + assert control.start_projects == ["TEST001"] + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + monkeypatch.setattr( + service, + "_active_acquisition_checkpoint_start_operation_id", + lambda **_kwargs: start_operation_id, + ) + physical_proof = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=True, + ) runtime.mark_ready() + if proof_first: + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) + proof_without_frame = service.state() + assert proof_without_frame["acquisition"]["state"] == ("awaiting_external_start") + assert proof_without_frame["last_operation"]["status"] == "running" + else: + runtime.pcl_frames = 1 + frame_before_canonical_start = service.state() + assert frame_before_canonical_start["acquisition"]["state"] == ("awaiting_external_start") + assert frame_before_canonical_start["last_operation"]["status"] == "running" + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) runtime.pcl_frames = 1 acquiring = service.state() assert acquiring["acquisition"]["state"] == "acquiring" + assert acquiring["last_operation"]["status"] == "succeeded" + assert acquiring["last_operation"]["stage_code"] == ("canonical-start-and-first-point-frame") + assert acquiring["last_operation"]["result"]["confirmation"] == ( + "canonical-start-and-point-frame" + ) stopping = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, mode="graceful", physical_acceptance=PHYSICAL_ACCEPTANCE, @@ -1248,6 +7461,35 @@ def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions( assert stopping["last_operation"]["status"] == "running" control.state = "completed" + # This legacy mapping fixture does not let the fake request_stop replace + # the START projection with a typed durable STOP head. Bypass only its + # terminal checkpoint seam; dedicated integration tests cover real v5 + # STOP proof -> checkpoint CEASED ordering. + monkeypatch.setattr( + service, + "_cease_active_acquisition_checkpoint_from_physical_head", + lambda **_kwargs: True, + ) + # The production worker has also resolved the durable STOP ledger before + # publishing `completed`; this mapping-only fake still returns the old + # START document. Preserve this test's pre-checkpoint intent by omitting + # that stale mapping at the reducer boundary. + original_reconcile_acquisition = service._reconcile_acquisition # noqa: SLF001 + def reconcile_legacy_completed( + runtime_snapshot: Mapping[str, Any], + camera_snapshot: Mapping[str, Any], + control_snapshot: Mapping[str, Any], + **kwargs: Any, + ) -> None: + kwargs["physical_command_proof"] = None + original_reconcile_acquisition( + runtime_snapshot, + camera_snapshot, + control_snapshot, + **kwargs, + ) + + monkeypatch.setattr(service, "_reconcile_acquisition", reconcile_legacy_completed) completed = service.state() assert control.stop_calls == 1 assert completed["acquisition"]["state"] == "completed" @@ -1255,6 +7497,5740 @@ def test_plugin_commanded_acquisition_keeps_start_and_stop_as_explicit_actions( assert completed["last_operation"]["status"] == "succeeded" assert runtime.stop_calls == 1 assert completed["live_perception_shadow"]["active"] is False + assert completed["message"] == ( + "Приём и запись завершены. K1 остановлен и готов к новому запуску." + ) + + +def _activate_real_checkpoint_for_prepared_stop_fixture( + service: XgridsK1CompatibilityService, + *, + project_name: str, +) -> None: + """Back the specialized retained-receiver fixture with the real v5 CAS.""" + + store = service._active_acquisition_checkpoint # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert store is not None + assert acquisition is not None + assert out_dir is not None + assert record is not None and record.last_status is not None + binding = ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=service._snapshot_runtime_id, # noqa: SLF001 + intent_id=record.connection.intent_id, + transport_ref=record.connection.transport_ref, + connection_mode=record.connection.connection_mode, + target_ipv4=record.connection.target_ipv4, + target_port=record.connection.target_port, + host_path_epoch=record.connection.host_path_epoch, + control_session_id=record.connection.control_session_id, + producer_generation=record.connection.producer_generation, + logical_device_id=acquisition.device_id, + compatibility_profile_id=record.compatibility_profile_id, + vendor_device_id_sha256=record.identity.vendor_device_id_sha256, + device_serial_sha256=record.identity.device_serial_sha256, + ) + prepared = store.prepare( + transition_id=f"test-retained-prepare:{record.operation_id}", + predecessor_revision=0, + acquisition_id=acquisition.acquisition_id, + original_start_operation_id=record.operation_id, + start_payload_sha256=record.payload_sha256, + identity=ActiveAcquisitionRecoveryIdentity( + logical_device_id=acquisition.device_id, + vendor_device_id_sha256=record.identity.vendor_device_id_sha256, + device_serial_sha256=record.identity.device_serial_sha256, + ), + connection=ActiveAcquisitionRecoveryConnection( + transport_ref=record.connection.transport_ref, + connection_mode=record.connection.connection_mode, + target_ipv4=record.connection.target_ipv4, + target_port=record.connection.target_port, + ), + compatibility_profile_id=record.compatibility_profile_id, + project_name=project_name, + project_name_wire_sha256=active_acquisition_project_name_sha256( + project_name + ), + original_evidence_session_id=out_dir.name, + duration_seconds=acquisition.duration_seconds, + requested_streams=acquisition.requested_streams, + evidence_policy=acquisition.evidence_policy, + mount_type="handheld", + gnss_mode="none", + prepared_binding=binding, + ) + status_proof = service._checkpoint_status_proof( # noqa: SLF001 + status=record.last_status, + binding=binding, + evidence_session_id=out_dir.name, + ) + store.activate( + transition_id=f"test-retained-activate:{record.operation_id}", + expected_revision=prepared.revision, + expected_acquisition_id=prepared.acquisition_id, + expected_start_operation_id=prepared.original_start_operation_id, + status_proof=status_proof, + physical_proof=service._checkpoint_physical_proof( # noqa: SLF001 + record=record, + binding=binding, + checkpoint=prepared, + ), + ) + + +def _install_real_prepared_stop_dispatch_fixture( + service: XgridsK1CompatibilityService, + runtime: FakeVisualizationRuntime, + *, + confirm_first_pcl: bool = True, +) -> SimpleNamespace: + """Install one fully confirmed START and a STOP that only durably PREPAREs.""" + + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service, transport_ref="k1-stop-dispatch-race") + prepared = service.prepare_acquisition( + _prepare_request( + project_name="STOP_RACE", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = str(prepared["acquisition"]["acquisition_id"]) + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + ) + connection = PhysicalCommandConnectionBinding( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode=binding.connection_mode, + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id="fake-control-session", + producer_generation=1, + ) + + def status(session_state: str, observed_at_utc: str) -> PhysicalCommandStatusEvidence: + scanning = session_state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state=session_state, # type: ignore[arg-type] + session_state_code=302 if scanning else 300, + project_bound=scanning, + project_id_sha256="c" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256=hashlib.sha256(observed_at_utc.encode()).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="1" * 64, + baseline_status=status("ready", "2026-08-12T08:00:00.000Z"), + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing(start_operation_id, publish_call_returned=True, packet_id=71) + ledger.mark_qos2_completed(start_operation_id, packet_id=71) + ledger.record_application_response( + start_operation_id, + PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="2" * 64, + observed_at_utc="2026-08-12T08:00:01.000Z", + ), + ) + ledger.record_status_observation( + start_operation_id, + status("scanning", "2026-08-12T08:00:02.000Z"), + ) + ledger.resolve(start_operation_id, resolution="start-active-observed") + + coordinator = service._physical_command_coordinator # noqa: SLF001 + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:stop-race:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="3" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-12T08:00:03.000Z", + ) + ) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + host_path_epoch=connection.host_path_epoch, + control_session_id=connection.control_session_id, + producer_generation=connection.producer_generation, + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256="c" * 64, + init_ready=True, + status_message_sha256="4" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T08:00:04.000Z", + ) + ) + _activate_real_checkpoint_for_prepared_stop_fixture( + service, + project_name="STOP_RACE", + ) + runtime.mark_ready() + runtime.pcl_frames = 1 if confirm_first_pcl else 0 + assert service.state()["acquisition"]["state"] == ( + "acquiring" if confirm_first_pcl else "awaiting_external_start" + ) + + payload = b"exact-stop-dispatch-race" + envelope = OneShotPublishEnvelope( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload=payload, + payload_sha256=hashlib.sha256(payload).hexdigest(), + payload_bytes=len(payload), + qos=2, + retain=False, + ) + original_request_stop = control.request_stop + prepare_calls: list[str] = [] + + def request_stop_after_durable_prepare(**kwargs: object) -> dict[str, object]: + command_context = kwargs["command_context"] + coordinator.prepare(command_context, action="stop", envelope=envelope) # type: ignore[arg-type] + prepare_calls.append(str(command_context.operation_id)) # type: ignore[attr-defined] + result = original_request_stop(**kwargs) # type: ignore[arg-type] + control.state = "stop-requested" + return result + + control.request_stop = request_stop_after_durable_prepare # type: ignore[method-assign] + stop_operation_id = "op-00000000-0000-4000-8000-000000001401" + request = _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + idempotency_key="prepared-stop-dispatch-race", + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + return SimpleNamespace( + service=service, + runtime=runtime, + control=control, + binding=binding, + coordinator=coordinator, + ledger=ledger, + identity=identity, + connection=connection, + status=status, + envelope=envelope, + request=request, + start_operation_id=start_operation_id, + stop_operation_id=stop_operation_id, + prepare_calls=prepare_calls, + ) + + +def _observe_stop_race_route_loss(service: XgridsK1CompatibilityService) -> None: + service._connection_supervisor.observe_host_path( # noqa: SLF001 + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + ) + + +def _settle_real_prepared_stop_before_publish( + fixture: SimpleNamespace, +) -> facade_module._PreparedStopRecoveryOwner: + """Drive the real S0 row to truthful NONE while retaining its receiver.""" + + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + fixture.service.state() + owner = fixture.service._prepared_stop_recovery_owner # noqa: SLF001 + assert isinstance(owner, facade_module._PreparedStopRecoveryOwner) + operation = fixture.service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + return owner + + +def _classify_real_prepared_stop( + fixture: SimpleNamespace, + owner: facade_module._PreparedStopRecoveryOwner, + *, + session_state: str, + reconciliation_id: str, +) -> dict[str, object]: + """Install a fresh inspection binding and classify S0 with real coordinator audit.""" + + scanning = session_state == "scanning" + runtime_binding = PhysicalCommandRuntimeBinding( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=fixture.binding.intent_id, + transport_ref=fixture.binding.transport_ref, + connection_mode=fixture.binding.connection_mode, + target_ipv4=fixture.binding.target_ipv4, + target_port=fixture.binding.target_port, + host_path_epoch=fixture.binding.host_path_epoch + 1, + control_session_id=f"fresh-{reconciliation_id}", + producer_generation=2, + ) + fixture.coordinator.prepare_read_only_bootstrap() + fixture.coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=f"bootstrap:{reconciliation_id}:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="f" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-12T08:00:59.000Z", + ) + ) + fixture.coordinator.bind_control_session(runtime_binding) + fixture.coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=runtime_binding.vendor_device_id_sha256, + device_serial_sha256=runtime_binding.device_serial_sha256, + session_state=session_state, # type: ignore[arg-type] + session_state_code=MODELING_STATE_BASE + (302 if scanning else 300), + project_bound=scanning, + project_id_sha256="c" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256="e" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T08:01:00.000Z", + ) + ) + return dict( + fixture.service._classify_retained_prepared_stop_owned( # noqa: SLF001 + owner=owner, + reconciliation_id=reconciliation_id, + reconcile=lambda: fixture.coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ), + ) + ) + + +def _resolve_real_stop_ready(fixture: SimpleNamespace) -> None: + """Advance the fixture's exact S0 to canonical durable READY without polling.""" + + fixture.ledger.mark_dispatching(fixture.stop_operation_id) + fixture.ledger.mark_observing( + fixture.stop_operation_id, + publish_call_returned=True, + packet_id=72, + ) + fixture.ledger.mark_qos2_completed(fixture.stop_operation_id, packet_id=72) + fixture.ledger.record_application_response( + fixture.stop_operation_id, + PhysicalCommandApplicationResponse( + operation_id=fixture.stop_operation_id, + action="stop", + control_session_id=fixture.connection.control_session_id, + host_path_epoch=fixture.connection.host_path_epoch, + producer_generation=fixture.connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="5" * 64, + observed_at_utc="2026-08-12T08:00:05.000Z", + ), + ) + fixture.ledger.record_status_observation( + fixture.stop_operation_id, + fixture.status("ready", "2026-08-12T08:00:06.000Z"), + ) + fixture.ledger.resolve( + fixture.stop_operation_id, + resolution="stop-standby-observed", + ) + + +def test_prepared_stop_and_dispatch_gate_fence_route_loss_without_duplicate_publish( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + + stopping = service.stop_acquisition(fixture.request) + prepared_record = fixture.ledger.snapshot().record + assert prepared_record is not None + assert prepared_record.operation_id == fixture.stop_operation_id + assert prepared_record.stage == "prepared" + assert service._prepared_stop_dispatch_lineage_is_current() is True # noqa: SLF001 + assert stopping["selected_device_id"] == fixture.binding.transport_ref + + # A route-negative state reduction lands after durable PREPARE but before + # the asynchronous worker owns the command-dispatch gate. It must not + # tear down the only topology that can execute this already accepted STOP. + for _ in range(3): + _observe_stop_race_route_loss(service) + state = service.state() + assert state["selected_device_id"] == fixture.binding.transport_ref + assert fixture.prepare_calls == [fixture.stop_operation_id] + repeated = service.stop_acquisition(fixture.request) + assert repeated["selected_device_id"] == fixture.binding.transport_ref + assert fixture.prepare_calls == [fixture.stop_operation_id] + + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=fixture.envelope.topic, + payload_sha256=fixture.envelope.payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + published: list[str] = [] + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + fixture.coordinator.publish_dispatching(dispatch) + dispatching = fixture.ledger.snapshot().record + assert dispatching is not None and dispatching.stage == "dispatching" + # This is the exact mark_dispatching -> client.publish interleaving. + # The reducer sees confirmed loss but cannot cross the transport gate. + mid_dispatch = service.state() + assert mid_dispatch["selected_device_id"] == fixture.binding.transport_ref + published.append(fixture.envelope.payload_sha256) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert published == [fixture.envelope.payload_sha256] + assert fixture.prepare_calls == [fixture.stop_operation_id] + + # Once the dispatch fence is released, the same genuine route loss uses + # the normal fail-closed reducer; PREPARED lineage no longer applies. + retired = service.state() + assert retired["selected_device_id"] is None + assert retired["device_session"] is None + + +@pytest.mark.parametrize("terminal_reason", ["control-failed", "deadline"]) +def test_prepared_stop_barrier_expires_and_real_loss_retires_fail_closed( + tmp_path: Path, + terminal_reason: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + + if terminal_reason == "control-failed": + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + else: + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + operation.deadline_at = datetime.now(UTC) - timedelta(seconds=1) + for _ in range(3): + _observe_stop_race_route_loss(service) + + assert service._prepared_stop_dispatch_lineage_is_current() is False # noqa: SLF001 + retired = service.state() + if terminal_reason == "control-failed": + # A terminal pre-publish worker is no longer merely an expired dispatch + # barrier: it is exact NONE proof and retains the receiver/topology for + # the automatic read-only SCANNING/READY classification path. + assert retired["selected_device_id"] == fixture.binding.transport_ref + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert service._prepared_stop_recovery_owner is not None # noqa: SLF001 + else: + # A hung nonterminal worker whose bounded operation deadline elapsed has + # no terminal NONE proof. Genuine route loss therefore retires normally. + assert retired["selected_device_id"] is None + assert fixture.ledger.snapshot().record is not None + assert fixture.ledger.snapshot().record.stage == "prepared" # type: ignore[union-attr] + assert fixture.prepare_calls == [fixture.stop_operation_id] + + +def test_prepared_stop_pending_worker_outranks_runtime_error_then_settles_none( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + runtime.phase = "error" + + # Local producer symptoms cannot terminalize an accepted STOP while the + # exact durable command is still PREPARED and its worker is nonterminal. + pending = service.state() + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert pending["acquisition"]["state"] == "awaiting_external_stop" + assert operation.status == "running" + assert operation.error is None + assert fixture.ledger.snapshot().record is not None + assert fixture.ledger.snapshot().record.stage == "prepared" # type: ignore[union-attr] + + # The same row plus a terminal pre-publish worker is exact NONE proof. + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + settled = service.state() + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert settled["selected_device_id"] is None + + +def test_expired_stop_dispatch_admission_resolves_durable_no_dispatch( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + assert len(fixture.control.stop_dispatch_deadlines) == 1 + deadline_reached = fixture.control.stop_dispatch_deadlines[0] + assert deadline_reached is not None + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + operation.deadline_at = datetime.now(UTC) - timedelta(seconds=1) + assert deadline_reached() is True + + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "physical-command-dispatch-deadline-expired", + "modeling_command_attempted": True, + "stop_command_attempted": False, + "stop_publish_attempts": 0, + "diagnostic_evidence_unavailable": [], + "safe_to_retry": True, + } + fixture.control.outcome_unknown = False + fixture.control.transport_publish_attempts = 8 + + settled = service.state() + + record = fixture.ledger.snapshot().record + assert record is not None + assert record.stage == "resolved" + assert record.resolution == "not-dispatched" + assert record.resolved_unclassified_stop_recovery_required is True + assert fixture.coordinator.snapshot()["active_operation_id"] is None + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert operation.error["automatic_replay_allowed"] is False + assert service.camera_preview.snapshot()["recording"]["source_end_expected"] is False + assert service._prepared_stop_recovery_owner is not None # noqa: SLF001 + assert settled["physical_command"]["record"]["resolution"] == "not-dispatched" + + +def test_abort_rejects_exact_unresolved_stop_before_any_local_cleanup( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="исход PREPARED/DISPATCHING STOP", + ) as raised: + service.abort_acquisition( + _abort_request(acquisition_id=fixture.request.acquisition_id) + ) + + assert raised.value.reason_code == "acquisition-abort-physical-stop-unresolved" + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "awaiting_external_stop" # noqa: SLF001 + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert runtime.stop_calls == 0 + + +def test_capture_only_rejects_live_prepared_stop_before_operation_or_local_cleanup( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is not None + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + before_acquisition = ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + acquisition.result, + ) + + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="ещё владеет PREPARED STOP", + ) as raised: + service.stop_acquisition( + _stop_request( + acquisition_id=acquisition.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001450", + mode="capture-only", + ) + ) + + assert raised.value.reason_code == "acquisition-stop-worker-retirement-pending" + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + assert fixture.control.state == "stop-requested" + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + acquisition.result, + ) == before_acquisition + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert runtime.stop_calls == 0 + + +def test_capture_only_defers_terminal_prepared_stop_while_dispatch_gate_is_busy( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + fixture.control.state = "failed" + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="publish-переходом STOP", + ) as raised: + service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001451", + mode="capture-only", + ) + ) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert raised.value.reason_code == "acquisition-stop-dispatch-retirement-pending" + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert runtime.stop_calls == 0 + + +def test_capture_only_settles_terminal_prepared_stop_none_before_local_cleanup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + fixture.control.state = "failed" + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + completed = service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001452", + mode="capture-only", + ) + ) + + original_stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert original_stop.status == "failed" + assert original_stop.stage_code == "physical-stop-classified-not-dispatched" + assert original_stop.error is not None + assert original_stop.error["side_effect_status"] == "none" + assert original_stop.error["physical_command_sent"] is False + local_stop = completed["last_operation"] + assert local_stop["operation_id"] == "op-00000000-0000-4000-8000-000000001452" + assert local_stop["status"] == "succeeded" + assert completed["acquisition"]["state"] == "completed" + assert completed["acquisition"]["result"]["device_stop"] == "not-sent" + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert fixture.control.stop_calls == 1 + assert runtime.stop_calls == 1 + + +def test_capture_only_cleanup_failure_preserves_s0_and_confirmed_start_truth( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + service.stop_acquisition(fixture.request) + fixture.control.state = "failed" + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + runtime.stop_error = RuntimeError("synthetic capture-only cleanup timeout") + + with pytest.raises(RuntimeError, match="synthetic capture-only cleanup timeout"): + service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001457", + mode="capture-only", + ) + ) + + start = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + original_stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + local_stop = service._operations.get( # noqa: SLF001 + "op-00000000-0000-4000-8000-000000001457" + ) + assert start.status == "interrupted" + assert start.stage_code == "physical-start-active-local-retirement-before-point" + assert start.error is not None and start.error["side_effect_status"] == "succeeded" + assert original_stop.stage_code == "physical-stop-classified-not-dispatched" + assert original_stop.error is not None + assert original_stop.error["side_effect_status"] == "none" + assert local_stop.status == "failed" + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition_session_lease is not None # noqa: SLF001 + + +@pytest.mark.parametrize("stage", ["dispatching", "observing"]) +def test_capture_only_preserves_postpublish_stop_unknown_before_local_cleanup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stage: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=fixture.envelope.topic, + payload_sha256=fixture.envelope.payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + fixture.coordinator.publish_dispatching(dispatch) + if stage == "observing": + fixture.ledger.mark_observing( + fixture.stop_operation_id, + publish_call_returned=False, + packet_id=None, + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + completed = service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id=( + "op-00000000-0000-4000-8000-000000001453" + if stage == "dispatching" + else "op-00000000-0000-4000-8000-000000001454" + ), + mode="capture-only", + ) + ) + + original_stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert original_stop.status == "failed" + assert original_stop.stage_code == "physical-stop-outcome-unknown-local-retirement" + assert original_stop.error is not None + assert original_stop.error["side_effect_status"] == "unknown" + assert original_stop.error["automatic_replay_allowed"] is False + assert completed["last_operation"]["status"] == "succeeded" + assert completed["acquisition"]["result"]["device_stop"] == ( + "physical-outcome-unknown" + ) + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == stage + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert fixture.control.stop_calls == 1 + assert runtime.stop_calls == 1 + + +@pytest.mark.parametrize("action", ["capture-only", "abort", "force", "reset", "close"]) +def test_resolved_ready_preducer_dominates_destructive_action_and_pending_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + service.stop_acquisition(fixture.request) + _resolve_real_stop_ready(fixture) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + if action == "close": + monkeypatch.setattr(service.camera_preview, "close", lambda: None) + monkeypatch.setattr(runtime, "close", runtime.stop) + before_record = fixture.ledger.snapshot().record + before_prepare_calls = list(fixture.prepare_calls) + before_operation_ids = { + item["operation_id"] + for item in service._operations.snapshot() # noqa: SLF001 + } + + if action == "capture-only": + result = service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001455", + mode="capture-only", + ) + ) + assert result["acquisition"]["state"] == "completed" + elif action == "abort": + result = service.abort_acquisition( + _abort_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001456", + ) + ) + assert result["acquisition"]["state"] == "completed" + elif action == "force": + with service._acquisition_lifecycle_gate: # noqa: SLF001 + assert service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="test-resolved-ready-dominance", + require_recovery=False, + ) + elif action == "reset": + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service._desired_connection_mode_revision, # noqa: SLF001 + reset_scenario=True, + reset_id="resolved-ready-preducer-reset-0001", + ) + ) + else: + service.close() + + start = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert start.status == "interrupted" + assert start.stage_code == "physical-start-active-but-no-point-before-standby" + assert start.error is not None + assert start.error["side_effect_status"] == "succeeded" + assert start.error["automatic_replay_allowed"] is False + assert stop.status == "succeeded" + assert stop.stage_code == "device-standby-confirmed" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.result is not None # noqa: SLF001 + assert service._acquisition.result["device_state"] == "ready" # noqa: SLF001 + assert service._acquisition.result["device_stop"] == "protocol-confirmed" # noqa: SLF001 + assert fixture.ledger.snapshot().record == before_record + assert fixture.prepare_calls == before_prepare_calls + assert fixture.control.stop_calls == 1 + assert runtime.stop_calls == 1 + after_operation_ids = { + item["operation_id"] + for item in service._operations.snapshot() # noqa: SLF001 + } + assert after_operation_ids == before_operation_ids + + +def test_local_force_finish_fences_prepared_stop_then_settles_none_before_cleanup( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + with service._acquisition_lifecycle_gate: # noqa: SLF001 + assert service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="test-local-retirement", + require_recovery=False, + ) + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "interrupted" # noqa: SLF001 + assert service._acquisition.result is not None # noqa: SLF001 + assert service._acquisition.result["device_stop"] == "not-sent" # noqa: SLF001 + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert runtime.stop_calls == 1 + + +def test_local_force_finish_preserves_confirmed_start_truth_before_first_point( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + service.stop_acquisition(fixture.request) + start_before = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + assert start_before.status == "running" + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + with service._acquisition_lifecycle_gate: # noqa: SLF001 + assert service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="test-local-retirement", + require_recovery=False, + ) + + start = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + assert start.status == "interrupted" + assert start.stage_code == "physical-start-active-local-retirement-before-point" + assert start.error is not None + assert start.error["side_effect_status"] == "succeeded" + assert start.error["automatic_replay_allowed"] is False + stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert stop.stage_code == "physical-stop-classified-not-dispatched" + assert stop.error is not None and stop.error["side_effect_status"] == "none" + + +def test_local_force_finish_preserves_dispatching_stop_as_outcome_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=fixture.envelope.topic, + payload_sha256=fixture.envelope.payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + fixture.coordinator.publish_dispatching(dispatch) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + with service._acquisition_lifecycle_gate: # noqa: SLF001 + assert service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="test-local-retirement", + require_recovery=False, + ) + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-outcome-unknown-local-retirement" + assert operation.error is not None + assert operation.error["side_effect_status"] == "unknown" + assert operation.error["automatic_replay_allowed"] is False + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "dispatching" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "interrupted" # noqa: SLF001 + assert service._acquisition.result is not None # noqa: SLF001 + assert service._acquisition.result["device_stop"] == "physical-outcome-unknown" # noqa: SLF001 + assert "physical_command_sent" not in service._acquisition.result # noqa: SLF001 + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + + +def test_local_force_finish_defers_without_mutation_while_dispatch_gate_is_owned( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is not None + before_revision = acquisition.state_revision + before_recovery_generation = service._active_stream_recovery_generation # noqa: SLF001 + + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + with service._acquisition_lifecycle_gate, pytest.raises( # noqa: SLF001 + facade_module.LocalAcquisitionLifecycleError, + match="publish-переходом STOP", + ) as raised: + service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="test-local-retirement", + require_recovery=False, + ) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert raised.value.reason_code == "acquisition-stop-dispatch-retirement-pending" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition is acquisition # noqa: SLF001 + assert acquisition.state == "awaiting_external_stop" + assert acquisition.state_revision == before_revision + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert ( # noqa: SLF001 + service._active_stream_recovery_generation == before_recovery_generation + ) + assert fixture.control.state == "stop-requested" + assert runtime.stop_calls == 0 + + +def test_scenario_reset_fences_prepared_stop_before_local_cleanup_and_durable_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + reset = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service.state()["desired_connection_mode_revision"], + reset_scenario=True, + reset_id="prepared-stop-reset-0001", + ) + ) + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fixture.stop_operation_id + assert record.stage == "resolved" + assert record.resolution == "not-dispatched" + assert reset["connection_scenario_reset"]["physical_disposition"] == "not-dispatched" + assert reset["connection_scenario_reset"]["device_command_performed"] is False + assert reset["acquisition"]["state"] == "interrupted" + assert service._acquisition_stop_operation_id is None # noqa: SLF001 + assert runtime.stop_calls == 1 + + +def test_scenario_reset_dispatch_gate_defer_preserves_owner_then_exact_retry_converges( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + request = DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service.state()["desired_connection_mode_revision"], + reset_scenario=True, + reset_id="prepared-stop-reset-retry-0001", + ) + + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="publish-переходом STOP", + ) as raised: + service.select_connection_mode(request) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert raised.value.reason_code == "acquisition-stop-dispatch-retirement-pending" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "awaiting_external_stop" # noqa: SLF001 + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert runtime.stop_calls == 0 + + retried = service.select_connection_mode(request) + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + record = fixture.ledger.snapshot().record + assert record is not None and record.resolution == "not-dispatched" + assert retried["connection_scenario_reset"]["physical_disposition"] == "not-dispatched" + + +def test_scenario_reset_retires_dispatching_stop_as_outcome_unknown_never_cancelled( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=fixture.envelope.topic, + payload_sha256=fixture.envelope.payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + fixture.coordinator.publish_dispatching(dispatch) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + reset = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service.state()["desired_connection_mode_revision"], + reset_scenario=True, + reset_id="dispatching-stop-reset-0001", + ) + ) + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-outcome-unknown-local-retirement" + assert operation.error is not None + assert operation.error["side_effect_status"] == "unknown" + assert operation.error["automatic_replay_allowed"] is False + record = fixture.ledger.snapshot().record + assert record is not None + assert record.stage == "resolved" + assert record.resolution == "operator-retired-outcome-unknown" + assert reset["connection_scenario_reset"]["physical_disposition"] == ( + "operator-retired-outcome-unknown" + ) + assert service._acquisition_stop_operation_id is None # noqa: SLF001 + + +@pytest.mark.parametrize( + ("session_state", "expected_message", "expected_physical_state"), + [ + ( + "scanning", + "acquisition.recovery.scanning_adoption_pending", + "active", + ), + ( + "ready", + "acquisition.recovery.device_standby_observed", + "standby", + ), + ], +) +def test_retained_prepared_stop_classification_reserves_local_owner_from_durable_audit( + tmp_path: Path, + session_state: str, + expected_message: str, + expected_physical_state: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + original_acquisition = owner.acquisition + original_lease = owner.session_lease + original_out_dir = owner.out_dir + + reconciled = _classify_real_prepared_stop( + fixture, + owner, + session_state=session_state, + reconciliation_id=f"test-{session_state}-retained", + ) + + physical = fixture.coordinator.snapshot() + assert reconciled["operation_id"] == fixture.stop_operation_id + assert reconciled["stage"] == "resolved" + assert reconciled["resolution"] == "not-dispatched" + assert physical["reconciled_physical_state"] == expected_physical_state + assert service._acquisition is original_acquisition # noqa: SLF001 + assert service._acquisition_session_lease is original_lease # noqa: SLF001 + assert service._acquisition_out_dir == original_out_dir # noqa: SLF001 + assert service._acquisition.message_code == expected_message # noqa: SLF001 + assert runtime.stop_calls == 0 + assert fixture.control.stop_calls == 1 + stop_operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert stop_operation.stage_code == "physical-stop-classified-not-dispatched" + assert stop_operation.error is not None + assert stop_operation.error["side_effect_status"] == "none" + + +@pytest.mark.parametrize("session_state", ["ready", "scanning"]) +def test_resolved_unclassified_stop_compatibility_classifies_without_new_command( + tmp_path: Path, + session_state: str, +) -> None: + """A startup-resolved legacy STOP still takes the read-only classifier.""" + + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + _settle_real_prepared_stop_before_publish(fixture) + fixture.ledger.resolve( + fixture.stop_operation_id, + resolution="not-dispatched", + ) + fixture.coordinator = LedgerPhysicalCommandCoordinator(fixture.ledger) + service._physical_command_coordinator = fixture.coordinator # noqa: SLF001 + compatibility = fixture.coordinator.snapshot() + compatibility_record = compatibility["record"] + assert isinstance(compatibility_record, dict) + compatibility_revision = compatibility_record["revision"] + assert compatibility["status"] == "resolved" + assert compatibility["requires_reconciliation"] is True + assert compatibility["resolved_unclassified_stop_recovery_required"] is True + assert compatibility["reconciliation_ready"] is False + before_prepare_calls = list(fixture.prepare_calls) + before_control_stop_calls = fixture.control.stop_calls + + reconciliation_id = f"test-resolved-unclassified-{session_state}" + scanning = session_state == "scanning" + runtime_binding = PhysicalCommandRuntimeBinding( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=fixture.binding.intent_id, + transport_ref=fixture.binding.transport_ref, + connection_mode=fixture.binding.connection_mode, + target_ipv4=fixture.binding.target_ipv4, + target_port=fixture.binding.target_port, + host_path_epoch=fixture.binding.host_path_epoch + 1, + control_session_id=f"fresh-{reconciliation_id}", + producer_generation=2, + ) + fixture.coordinator.prepare_read_only_bootstrap() + fixture.coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=f"bootstrap:{reconciliation_id}:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="f" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-12T08:10:59.000Z", + ) + ) + fixture.coordinator.bind_control_session(runtime_binding) + fixture.coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=runtime_binding.vendor_device_id_sha256, + device_serial_sha256=runtime_binding.device_serial_sha256, + session_state=session_state, # type: ignore[arg-type] + session_state_code=MODELING_STATE_BASE + (302 if scanning else 300), + project_bound=scanning, + project_id_sha256="c" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256="e" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T08:11:00.000Z", + ) + ) + reconciled = fixture.coordinator.reconcile_unresolved( + reconciliation_id=reconciliation_id, + ) + + latest = reconciled["reconciliations"][-1] + original_attempt = latest["original_attempt"] + assert reconciled["revision"] == compatibility_revision + 1 + assert reconciled["operation_id"] == fixture.stop_operation_id + assert reconciled["stage"] == "resolved" + assert reconciled["resolution"] == "not-dispatched" + assert latest["kind"] == "prepared-stop-classification" + assert latest["resolution"] == ( + "physical-active-observed" + if session_state == "scanning" + else "physical-standby-observed" + ) + assert original_attempt["stage"] == "resolved" + assert original_attempt["resolution"] == "not-dispatched" + assert original_attempt["publish_call_returned"] is None + assert original_attempt["packet_id"] is None + assert original_attempt["qos2_completed"] is False + physical = fixture.coordinator.snapshot() + matcher = ( + service._matching_classified_prepared_stop_active # noqa: SLF001 + if scanning + else service._matching_classified_prepared_stop_standby # noqa: SLF001 + ) + assert matcher( + physical, + acquisition_id=reconciled["acquisition_id"], + stop_operation_id=fixture.stop_operation_id, + start_operation_id=fixture.start_operation_id, + verified_control=latest["verified_binding"]["connection"], + ) + assert fixture.prepare_calls == before_prepare_calls + assert fixture.control.stop_calls == before_control_stop_calls + assert runtime.stop_calls == 0 + + +def test_retained_prepared_stop_commit_and_local_reservation_are_one_lifecycle_transaction( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + fixture.coordinator.prepare_read_only_bootstrap() + fixture.coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:test-atomic:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="f" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-12T08:01:59.000Z", + ) + ) + binding = PhysicalCommandRuntimeBinding( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=fixture.binding.intent_id, + transport_ref=fixture.binding.transport_ref, + connection_mode=fixture.binding.connection_mode, + target_ipv4=fixture.binding.target_ipv4, + target_port=fixture.binding.target_port, + host_path_epoch=fixture.binding.host_path_epoch + 1, + control_session_id="fresh-atomic-classification", + producer_generation=2, + ) + fixture.coordinator.bind_control_session(binding) + fixture.coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256="c" * 64, + init_ready=True, + status_message_sha256="e" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T08:02:00.000Z", + ) + ) + committed = threading.Event() + allow_reservation = threading.Event() + state_finished = threading.Event() + errors: list[BaseException] = [] + + def reconcile_after_pause() -> dict[str, object]: + record = fixture.coordinator.reconcile_unresolved( + reconciliation_id="test-atomic-classification", + ) + committed.set() + assert allow_reservation.wait(timeout=5) + return record + + def classify() -> None: + try: + service._classify_retained_prepared_stop_owned( # noqa: SLF001 + owner=owner, + reconciliation_id="test-atomic-classification", + reconcile=reconcile_after_pause, + ) + except BaseException as exc: # pragma: no cover - assertion surface + errors.append(exc) + + def poll_state() -> None: + try: + service.state() + except BaseException as exc: # pragma: no cover - assertion surface + errors.append(exc) + finally: + state_finished.set() + + classify_thread = threading.Thread(target=classify) + classify_thread.start() + assert committed.wait(timeout=5) + poll_thread = threading.Thread(target=poll_state) + poll_thread.start() + assert state_finished.wait(timeout=0.05) is False + allow_reservation.set() + classify_thread.join(timeout=5) + poll_thread.join(timeout=5) + + assert not errors + assert state_finished.is_set() + assert service._acquisition is owner.acquisition # noqa: SLF001 + assert service._acquisition_session_lease is owner.session_lease # noqa: SLF001 + assert service._acquisition.message_code == ( # noqa: SLF001 + "acquisition.recovery.scanning_adoption_pending" + ) + + +def test_exact_old_stop_retry_is_idempotent_during_scanning_adoption_pending( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="scanning", + reconciliation_id="test-old-s0-retry-pending", + ) + before_record = fixture.ledger.snapshot().record + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + before_control_calls = fixture.control.stop_calls + before_prepare_calls = list(fixture.prepare_calls) + + repeated = service.stop_acquisition(fixture.request) + + assert repeated["acquisition"]["message_code"] == ( + "acquisition.recovery.scanning_adoption_pending" + ) + assert fixture.ledger.snapshot().record == before_record + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + assert fixture.control.stop_calls == before_control_calls + assert fixture.prepare_calls == before_prepare_calls + assert service._prepared_stop_recovery_owner is owner # noqa: SLF001 + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + "action", + ["abort", "capture-only", "force", "reset", "close"], +) +def test_ready_local_projection_pending_rejects_destructive_action_without_mutation( + tmp_path: Path, + action: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id=f"test-ready-pending-{action}", + ) + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is owner.acquisition + before_record = fixture.ledger.snapshot().record + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + before_state = ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + acquisition.result, + ) + before_stop_pointer = service._acquisition_stop_operation_id # noqa: SLF001 + before_start_pointer = service._acquisition_start_operation_id # noqa: SLF001 + before_control_calls = fixture.control.stop_calls + before_prepare_calls = list(fixture.prepare_calls) + + with pytest.raises(facade_module.LocalAcquisitionLifecycleError) as raised: + if action == "abort": + service.abort_acquisition( + _abort_request(acquisition_id=acquisition.acquisition_id) + ) + elif action == "capture-only": + service.stop_acquisition( + _stop_request( + acquisition_id=acquisition.acquisition_id, + operation_id=f"op-ready-pending-capture-{action}", + mode="capture-only", + ) + ) + elif action == "force": + service.force_finish_acquisition_locally( + _force_finish_request( + acquisition_id=acquisition.acquisition_id, + expected_state_revision=acquisition.state_revision, + expected_recovery_generation=( + service._active_stream_recovery_generation # noqa: SLF001 + ), + ) + ) + elif action == "reset": + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service._desired_connection_mode_revision, # noqa: SLF001 + reset_scenario=True, + reset_id="ready-pending-reset-0001", + ) + ) + else: + service.close() + + assert raised.value.reason_code == "acquisition-prepared-stop-adoption-pending" + assert fixture.ledger.snapshot().record == before_record + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + assert ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + acquisition.result, + ) == before_state + assert service._acquisition_stop_operation_id == before_stop_pointer # noqa: SLF001 + assert service._acquisition_start_operation_id == before_start_pointer # noqa: SLF001 + assert service._prepared_stop_recovery_owner is owner # noqa: SLF001 + assert fixture.control.stop_calls == before_control_calls + assert fixture.prepare_calls == before_prepare_calls + assert runtime.stop_calls == 0 + assert service._connection_scenario_reset_pending is None # noqa: SLF001 + assert service._service_close_requested.is_set() is False # noqa: SLF001 + + +def test_classified_scanning_adopts_same_receiver_and_allows_one_fresh_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + reconciled = _classify_real_prepared_stop( + fixture, + owner, + session_state="scanning", + reconciliation_id="test-scanning-in-place", + ) + latest = reconciled["reconciliations"][-1] + verified_binding = latest["verified_binding"]["connection"] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(verified_binding), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + old_device_session_id = owner.acquisition.device_session_id + service._device_session_id = "fresh-in-place-device-session" # noqa: SLF001 + runtime_start_calls = list(runtime.start_calls) + + service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner, + reconciliation_id="test-scanning-in-place", + reconciled_record=reconciled, + ) + + assert service._acquisition is owner.acquisition # noqa: SLF001 + assert service._acquisition_session_lease is owner.session_lease # noqa: SLF001 + assert service._acquisition_out_dir == owner.out_dir # noqa: SLF001 + assert service._acquisition.device_session_id != old_device_session_id # noqa: SLF001 + assert service._acquisition.device_session_id == ( # noqa: SLF001 + "fresh-in-place-device-session" + ) + assert service._acquisition.state == "acquiring" # noqa: SLF001 + assert service._acquisition_stop_operation_id is None # noqa: SLF001 + assert runtime.start_calls == runtime_start_calls + + # Hold the durable gap closure exactly where its fsync may block. A public + # STOP must publish priority without waiting for lifecycle ownership; once + # the store resumes, that priority prevents this already-published PCL from + # granting camera authority and the fresh STOP owns the next durable edge. + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + original_rebind_active = store.rebind_active + rebind_entered = threading.Event() + release_rebind = threading.Event() + + def blocked_rebind_active(**kwargs: Any) -> Any: + rebind_entered.set() + assert release_rebind.wait(timeout=2.0) + return original_rebind_active(**kwargs) + + monkeypatch.setattr(store, "rebind_active", blocked_rebind_active) + camera_calls: list[str] = [] + + def forbidden_camera_start(*_: object, **__: object) -> dict[str, object]: + camera_calls.append("camera-start") + return service.camera_preview.snapshot() + + monkeypatch.setattr( + service.camera_preview, + "activate_recording_producer", + forbidden_camera_start, + ) + monkeypatch.setattr( + service.camera_preview, + "retry_recording_producer", + forbidden_camera_start, + ) + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + runtime.phase = "live" + runtime.source_mode = "live" + runtime.source_ready = True + runtime.pcl_frames = 1 + publish_errors: list[BaseException] = [] + + def publish_first_pcl() -> None: + try: + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + except BaseException as exc: # pragma: no cover - asserted below + publish_errors.append(exc) + + fresh_stop_id = "op-00000000-0000-4000-8000-000000001402" + fresh_request = _stop_request( + acquisition_id=owner.lineage.acquisition_id, + operation_id=fresh_stop_id, + idempotency_key="classified-scanning-fresh-stop", + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=fixture.control.session_generation, + expected_control_state_revision=fixture.control.state_revision, + ) + stop_results: list[dict[str, Any]] = [] + stop_errors: list[BaseException] = [] + + def request_fresh_stop() -> None: + try: + stop_results.append(service.stop_acquisition(fresh_request)) + except BaseException as exc: # pragma: no cover - asserted below + stop_errors.append(exc) + + publish_thread = threading.Thread(target=publish_first_pcl) + stop_thread = threading.Thread(target=request_fresh_stop) + publish_thread.start() + try: + assert rebind_entered.wait(timeout=2.0) + stop_thread.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + if service._camera_stop_priority_counts.get( # noqa: SLF001 + owner.lineage.acquisition_id, + 0, + ): + break + time.sleep(0.005) + with service._lock: # noqa: SLF001 + assert service._camera_stop_priority_counts.get( # noqa: SLF001 + owner.lineage.acquisition_id, + 0, + ) == 1 + assert stop_thread.is_alive() + finally: + release_rebind.set() + publish_thread.join(timeout=2.0) + if stop_thread.ident is not None: + stop_thread.join(timeout=2.0) + + assert publish_thread.is_alive() is False + assert stop_thread.is_alive() is False + assert publish_errors == [] + assert stop_errors == [] + assert len(stop_results) == 1 + fresh = stop_results[0] + assert camera_calls == [] + with service._lock: # noqa: SLF001 + assert service._classified_stop_rebind_pending is None # noqa: SLF001 + assert service._classified_stop_rebind_inflight is None # noqa: SLF001 + + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fresh_stop_id + assert record.parent_operation_id == fixture.stop_operation_id + assert record.stage == "prepared" + assert fresh["acquisition"]["state"] == "awaiting_external_stop" + assert fixture.prepare_calls == [fixture.stop_operation_id, fresh_stop_id] + assert runtime.start_calls == runtime_start_calls + + +def test_blocked_checkpoint_fsync_starts_stop_deadline_and_expires_before_prepare( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + reconciled = _classify_real_prepared_stop( + fixture, + owner, + session_state="scanning", + reconciliation_id="test-scanning-stop-deadline-fsync", + ) + verified_binding = reconciled["reconciliations"][-1]["verified_binding"]["connection"] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(verified_binding), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + service._device_session_id = "fresh-stop-deadline-session" # noqa: SLF001 + service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner, + reconciliation_id="test-scanning-stop-deadline-fsync", + reconciled_record=reconciled, + ) + + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + original_rebind_active = store.rebind_active + rebind_entered = threading.Event() + release_rebind = threading.Event() + + def blocked_rebind_active(**kwargs: Any) -> Any: + rebind_entered.set() + assert release_rebind.wait(timeout=3.0) + return original_rebind_active(**kwargs) + + monkeypatch.setattr(store, "rebind_active", blocked_rebind_active) + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + runtime.phase = "live" + runtime.source_mode = "live" + runtime.source_ready = True + runtime.pcl_frames = 1 + publish_thread = threading.Thread( + target=service._observe_published_runtime_envelope, # noqa: SLF001 + kwargs={"envelope": frame, "producer_generation": runtime.producer_generation}, + daemon=True, + ) + publish_thread.start() + assert rebind_entered.wait(timeout=2.0) + + stop_id = "op-00000000-0000-4000-8000-000000001403" + request = _stop_request( + acquisition_id=owner.lineage.acquisition_id, + operation_id=stop_id, + idempotency_key="classified-scanning-fsync-expired-stop", + mode="graceful", + deadline_seconds=1.0, + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=fixture.control.session_generation, + expected_control_state_revision=fixture.control.state_revision, + ) + stop_errors: list[BaseException] = [] + def request_stop() -> None: + try: + service.stop_acquisition(request) + except BaseException as exc: # pragma: no cover - asserted below + stop_errors.append(exc) + + stop_thread = threading.Thread(target=request_stop, daemon=True) + stop_thread.start() + try: + priority_deadline = time.monotonic() + 0.5 + while time.monotonic() < priority_deadline: + operation = next( + ( + item + for item in service._operations.snapshot() # noqa: SLF001 + if item["operation_id"] == stop_id + ), + None, + ) + with service._lock: # noqa: SLF001 + priority = service._camera_stop_priority_counts.get( # noqa: SLF001 + owner.lineage.acquisition_id, + 0, + ) + if operation is not None and priority == 1: + break + time.sleep(0.005) + assert operation is not None + assert operation["status"] == "accepted" + assert priority == 1 + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert stop_thread.is_alive() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline and not service._operations.deadline_reached( # noqa: SLF001 + stop_id + ): + time.sleep(0.01) + assert service._operations.deadline_reached(stop_id) is True # noqa: SLF001 + finally: + release_rebind.set() + publish_thread.join(timeout=3.0) + stop_thread.join(timeout=3.0) + + assert publish_thread.is_alive() is False + assert stop_thread.is_alive() is False + assert len(stop_errors) == 1 + assert getattr(stop_errors[0], "reason_code", None) == ( + "physical-command-dispatch-deadline-expired" + ) + operation = service._operations.get(stop_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == ( + "physical-stop-dispatch-deadline-expired-before-prepare" + ) + assert operation.error is not None + assert operation.error["code"] == "physical-command-dispatch-deadline-expired" + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert operation.error["automatic_replay_allowed"] is False + assert fixture.prepare_calls == [fixture.stop_operation_id] + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fixture.stop_operation_id + assert record.action == "stop" + assert record.stage == "resolved" + assert fixture.control.stop_calls == 1 + + +def test_stop_deadline_after_durable_prepare_retains_exact_no_dispatch_owner( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + original_request_stop = fixture.control.request_stop + + def expire_after_durable_prepare(**kwargs: object) -> dict[str, object]: + command_context = kwargs["command_context"] + fixture.coordinator.prepare( + command_context, + action="stop", + envelope=fixture.envelope, + ) + fixture.prepare_calls.append(str(command_context.operation_id)) + fixture.coordinator.resolve_prepared_not_dispatched("stop") + raise ApplicationMqttTransportError( + "synthetic deadline after STOP PREPARE fsync", + reason_code="physical-command-dispatch-deadline-expired", + ) + + fixture.control.request_stop = expire_after_durable_prepare + try: + with pytest.raises( + ApplicationMqttTransportError, + match="synthetic deadline after STOP PREPARE fsync", + ): + service.stop_acquisition(fixture.request) + finally: + fixture.control.request_stop = original_request_stop + + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fixture.stop_operation_id + assert record.action == "stop" + assert record.stage == "resolved" + assert record.resolution == "not-dispatched" + assert record.publish_call_returned is None + assert record.packet_id is None + assert record.qos2_completed is False + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert operation.error["automatic_replay_allowed"] is False + assert fixture.prepare_calls == [fixture.stop_operation_id] + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + owner = service._prepared_stop_recovery_owner # noqa: SLF001 + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert acquisition is not None + assert acquisition.state == "awaiting_external_stop" + assert owner is not None + assert owner.acquisition is acquisition + assert owner.lineage.stop_operation_id == fixture.stop_operation_id + assert service.camera_preview.snapshot()["recording"]["source_end_expected"] is False + + # Exact retries observe the terminal journal identity and never prepare or + # publish another physical STOP, even after the original control socket is + # no longer admissible for a new command. + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + fixture.control.verified_control = None + retried = service.stop_acquisition(fixture.request) + assert retried["contract_version"] == "missioncore.device-plugin-state/v1alpha2" + assert retried["selected_device_id"] == fixture.binding.transport_ref + assert retried["acquisition"]["acquisition_id"] == fixture.request.acquisition_id + assert retried["acquisition"]["state"] in { + "awaiting_external_stop", + "interrupted", + } + assert retried["last_operation"]["operation_id"] == fixture.stop_operation_id + assert retried["last_operation"]["status"] == "failed" + assert fixture.prepare_calls == [fixture.stop_operation_id] + + with pytest.raises(ValueError, match="different request"): + service.stop_acquisition( + fixture.request.model_copy( + update={ + "idempotency_key": fixture.request.idempotency_key, + "expected_control_state_revision": ( + fixture.request.expected_control_state_revision + 1 + ), + } + ) + ) + assert fixture.prepare_calls == [fixture.stop_operation_id] + + +def test_corrupt_physical_snapshot_never_terminalizes_preadmitted_stop_as_none( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + original_snapshot = fixture.ledger.snapshot + original_owned = service._stop_acquisition_owned # noqa: SLF001 + entered_owned = threading.Event() + + def reject_before_prepare( + request: StopAcquisitionRequest, + preadmitted_stop: object | None = None, + ) -> dict[str, Any]: + assert preadmitted_stop is not None + entered_owned.set() + raise facade_module.LocalAcquisitionLifecycleError( + "synthetic admission rejection before physical PREPARE", + reason_code="synthetic-before-prepare", + ) + + monkeypatch.setattr(service, "_stop_acquisition_owned", reject_before_prepare) + monkeypatch.setattr( + fixture.ledger, + "snapshot", + lambda: PhysicalCommandLedgerSnapshot( + status="corrupt", + record=None, + reason_code="physical-command-ledger-corrupt", + ), + ) + fresh_request = fixture.request.model_copy( + update={ + "operation_id": "op-00000000-0000-4000-8000-000000001404", + "idempotency_key": "corrupt-ledger-preadmitted-stop", + } + ) + try: + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="synthetic admission rejection", + ): + service.stop_acquisition(fresh_request) + finally: + monkeypatch.setattr(fixture.ledger, "snapshot", original_snapshot) + monkeypatch.setattr(service, "_stop_acquisition_owned", original_owned) + + assert entered_owned.is_set() + operation = service._operations.get(fresh_request.operation_id) # noqa: SLF001 + assert operation.status == "accepted" + assert operation.error is None + assert fixture.prepare_calls == [] + assert original_snapshot().record is not None + assert original_snapshot().record.operation_id == fixture.start_operation_id # type: ignore[union-attr] + + +def test_classified_ready_finishes_same_receiver_and_pins_fresh_start_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id="test-ready-in-place", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + finished = service.state() + + stop_operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert finished["acquisition"]["state"] == "interrupted" + assert finished["acquisition"]["result"]["read_only_recovery"] is True + assert finished["acquisition"]["result"]["device_stop"] == "not-sent" + assert finished["acquisition"]["cleanup_pending"] is False + assert stop_operation.stage_code == "physical-stop-classified-not-dispatched" + assert stop_operation.error is not None + assert stop_operation.error["side_effect_status"] == "none" + assert service._acquisition_session_lease is None # noqa: SLF001 + assert service._prepared_stop_recovery_owner is None # noqa: SLF001 + assert runtime.stop_calls == 1 + + physical = fixture.coordinator.snapshot() + assert facade_module._physical_command_prepared_stop_ready_successor_pending( # noqa: SLF001 + physical + ) + with pytest.raises( + facade_module.NetworkProvisioningConflict, + ) as mutation: + service._require_physical_command_network_mutation_allowed() # noqa: SLF001 + assert mutation.value.reason_code == "physical-command-reconciliation-required" + policy = facade_module._connection_policy_projection( # noqa: SLF001 + supervisor=service._connection_supervisor.snapshot(), # noqa: SLF001 + ledger=service._network_mutation_ledger.snapshot(), # noqa: SLF001 + fresh_devices=[], + provisioning_active=False, + acquisition_active=False, + acquisition_cleanup_pending=False, + acquisition_state="interrupted", + runtime_active=False, + application_control_session=fixture.control.snapshot(), + physical_command=physical, + ble_runtime={ + "poisoned": False, + "cleanup_pending": False, + "active_operation_kind": None, + }, + lifecycle_process_lease_holders=(), + network_provisioning_idempotency={}, + network_provisioning_idempotency_available=True, + semantic_topology_store={"status": "available"}, + device_identity_pin_store={"status": "available"}, + current_device_recovery=None, + desired_connection_mode=service._desired_connection_mode, # noqa: SLF001 + active_connection_mode=service._connection_mode, # noqa: SLF001 + ) + assert "physical-command-reconciliation-required" not in policy["actions"][ + "start-acquisition" + ]["reason_codes"] + + # The classified READY binding remains pinned during an ordinary + # continuation, but an explicit clean-scenario reset must abandon that + # future START ownership. After the separately requested fresh Scan, the + # exact UUID is an ordinary one-click provisioning target again. + reset = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=finished["desired_connection_mode_revision"], + reset_scenario=True, + reset_id="classified-ready-clean-network-reset-0001", + ) + ) + assert reset["connection_scenario_reset"]["physical_disposition"] == ( + "operator-retired-outcome-unknown" + ) + reset_record = fixture.ledger.snapshot().record + assert reset_record is not None + assert reset_record.resolution == "operator-retired-outcome-unknown" + assert reset_record.original_command_outcome == "not-dispatched" + assert fixture.prepare_calls == [fixture.stop_operation_id] + + with service._lock: # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + generation = service._ble_discovery_generation # noqa: SLF001 + service._scenario_reset_successful_scan_generation = generation # noqa: SLF001 + assert service._connection_scenario_reset is not None # noqa: SLF001 + service._scenario_reset_successful_scan_reset_id = ( # noqa: SLF001 + service._connection_scenario_reset["reset_id"] + ) + service._connection_scenario_reset["active"] = False # noqa: SLF001 + service._connection_scenario_reset[ # noqa: SLF001 + "settled_by_discovery_generation" + ] = generation + _set_scanned_k1(service, device_id=fixture.binding.transport_ref) + + scanned = service.state() + assert scanned["connection_policy"]["facts"][ + "eligible_fresh_transport_refs" + ] == [fixture.binding.transport_ref] + assert scanned["connection_policy"]["actions"]["provision-fresh-device"][ + "allowed" + ] is True + assert scanned["connection_policy"]["actions"]["start-acquisition"][ + "allowed" + ] is False + + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + service, + "_schedule_control_bootstrap_continuation", + lambda **_kwargs: None, + ) + applied = asyncio.run( + service.connect( + _connect_request( + device_id=fixture.binding.transport_ref, + ssid="classified-ready-clean-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + operation_id="op-00000000-0000-4000-8000-000000001405", + expected_mode_revision=scanned[ + "desired_connection_mode_revision" + ], + expected_discovery_generation=generation, + expected_reconfiguration_revision=scanned[ + "connection_reconfiguration" + ]["revision"], + ) + ) + ) + assert writes == ["network-write"] + assert applied["connection_attempt"]["phase"] == "network_applied" + reopened = fixture.ledger.snapshot().record + assert reopened is not None + assert reopened.original_command_outcome == "not-dispatched" + assert reopened.reopened_physical_state_requires_reconciliation is True + assert len(reopened.operator_reconciliation_reopens) == 1 + assert reopened.operator_reconciliation_reopens[-1].reason == ( + "reset-network-intent-read-only-settlement" + ) + assert fixture.prepare_calls == [fixture.stop_operation_id] + + +def test_classified_ready_before_first_pcl_interrupts_pending_start_without_new_edge( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + start_before = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + assert start_before.status == "running" + assert service._acquisition_start_operation_id == fixture.start_operation_id # noqa: SLF001 + + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id="test-ready-before-first-pcl", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + finished = service.state() + + start = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + stop = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert finished["acquisition"]["state"] == "interrupted" + assert finished["acquisition"]["result"]["device_stop"] == "not-sent" + assert start.status == "interrupted" + assert start.stage_code == "physical-start-active-but-no-point-before-standby" + assert start.error is not None + assert start.error["side_effect_status"] == "succeeded" + assert start.error["automatic_replay_allowed"] is False + assert service._acquisition_start_operation_id is None # noqa: SLF001 + assert stop.stage_code == "physical-stop-classified-not-dispatched" + assert stop.error is not None + assert stop.error["side_effect_status"] == "none" + assert fixture.prepare_calls == [fixture.stop_operation_id] + assert fixture.control.stop_calls == 1 + assert runtime.stop_calls == 1 + + +@pytest.mark.parametrize("action", ["abort", "force", "reset", "close"]) +def test_resolved_ready_dominates_later_local_action_without_rewriting_s0_or_pending_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + action: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id=f"test-ready-dominates-{action}", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + completed = service.state() + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is not None + assert completed["acquisition"]["state"] == "interrupted" + assert completed["acquisition"]["result"]["device_stop"] == "not-sent" + + before_record = fixture.ledger.snapshot().record + before_acquisition = ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + dict(acquisition.result or {}), + ) + before_operations = { + item["operation_id"]: item + for item in service._operations.snapshot() # noqa: SLF001 + if item["operation_id"] + in {fixture.start_operation_id, fixture.stop_operation_id} + } + before_prepare_calls = list(fixture.prepare_calls) + before_control_stop_calls = fixture.control.stop_calls + + if action == "abort": + result = service.abort_acquisition( + _abort_request(acquisition_id=acquisition.acquisition_id) + ) + assert result["acquisition"]["state"] == "interrupted" + elif action == "force": + result = service.force_finish_acquisition_locally( + _force_finish_request( + acquisition_id=acquisition.acquisition_id, + expected_state_revision=acquisition.state_revision, + expected_recovery_generation=( + service._active_stream_recovery_generation # noqa: SLF001 + ), + ) + ) + assert result["acquisition"]["state"] == "interrupted" + elif action == "reset": + result = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=service._desired_connection_mode_revision, # noqa: SLF001 + reset_scenario=True, + reset_id="resolved-ready-reset-0001", + ) + ) + assert result["acquisition"]["state"] == "interrupted" + else: + service.close() + + after_operations = { + item["operation_id"]: item + for item in service._operations.snapshot() # noqa: SLF001 + if item["operation_id"] + in {fixture.start_operation_id, fixture.stop_operation_id} + } + after_record = fixture.ledger.snapshot().record + if action == "reset": + assert before_record is not None + assert after_record is not None + assert after_record.revision == before_record.revision + 1 + assert after_record.resolution == "operator-retired-outcome-unknown" + assert after_record.original_command_outcome == "not-dispatched" + assert after_record.reconciliations == before_record.reconciliations + assert after_record.operator_retirements[-1].reason == ( + "connection-scenario-reset-by-operator" + ) + assert after_record.operator_retirements[-1].original_attempt.operation_id == ( + before_record.operation_id + ) + assert after_record.operator_retirements[-1].original_attempt.revision == ( + before_record.revision + ) + else: + assert after_record == before_record + assert ( + acquisition.state, + acquisition.state_revision, + acquisition.message_code, + dict(acquisition.result or {}), + ) == before_acquisition + assert after_operations == before_operations + assert before_operations[fixture.start_operation_id]["status"] == "interrupted" + assert before_operations[fixture.start_operation_id]["stage_code"] == ( + "physical-start-active-but-no-point-before-standby" + ) + assert before_operations[fixture.stop_operation_id]["stage_code"] == ( + "physical-stop-classified-not-dispatched" + ) + assert before_operations[fixture.stop_operation_id]["error"]["side_effect_status"] == "none" + assert fixture.prepare_calls == before_prepare_calls + assert fixture.control.stop_calls == before_control_stop_calls + + +@pytest.mark.parametrize("camera_outcome", ["healthy", "dead", "clean-ended"]) +def test_classified_scanning_cancels_stop_eof_and_preserves_or_rearms_camera_epoch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + camera_outcome: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + reconciled = _classify_real_prepared_stop( + fixture, + owner, + session_state="scanning", + reconciliation_id=f"test-scanning-camera-{camera_outcome}", + ) + verified_binding = reconciled["reconciliations"][-1]["verified_binding"]["connection"] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(verified_binding), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + service._device_session_id = f"fresh-camera-{camera_outcome}-session" # noqa: SLF001 + # `_classify_real_prepared_stop` installs the fresh durable/control proof + # directly. Production Verify also publishes that same rotated route and + # DeviceInfo generation to the connection supervisor; mirror that portion + # here so the real camera activation admission remains exercised. + fresh_path = HostPathProbeResult( + available=True, + fingerprint=f"test-route:classified-camera:{camera_outcome}", + interface="test0", + source_ipv4="192.168.1.2", + route_class="direct", + kernel_route_fingerprint=(f"test-route:classified-camera:{camera_outcome}"), + ) + fresh_host_epoch = service._connection_supervisor.observe_host_path( # noqa: SLF001 + fresh_path + ) + assert fresh_host_epoch == verified_binding["host_path_epoch"] + fresh_target = EndpointTarget( + str(verified_binding["target_ipv4"]), + int(verified_binding["target_port"]), + ) + assert service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=fresh_target, + intent_id=str(verified_binding["intent_id"]), + host_path_epoch=fresh_host_epoch, + reachable=True, + ) + assert service._connection_supervisor.observe_control_evidence( # noqa: SLF001 + VerifiedControlEvidence( + intent_id=str(verified_binding["intent_id"]), + transport_ref=str(verified_binding["transport_ref"]), + host_path_epoch=fresh_host_epoch, + target=fresh_target, + connection_mode=verified_binding["connection_mode"], # type: ignore[arg-type] + logical_device_id="known-k1", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id=str(verified_binding["control_session_id"]), + ) + ) + + owner.out_dir.mkdir(parents=True, exist_ok=True) + initial_epoch = 5 if camera_outcome == "healthy" else None + camera_state: dict[str, object] = { + "phase": ( + "streaming" + if camera_outcome == "healthy" + else "error" + if camera_outcome == "dead" + else "idle" + ), + "generation": None if camera_outcome == "clean-ended" else 5, + "active_source_id": ( + None + if camera_outcome == "clean-ended" + else facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE + ), + # These are the real gateway shapes after producer detachment. The + # acquisition recording root stays active while both ended variants + # expose no active producer epoch. The dead case is the bounded + # pre-first-media decode failure which production admits for a fresh + # epoch; a generic source EOF must first finish its archive seal. + "recording": { + "active": True, + "session": owner.out_dir.name, + "active_epoch": initial_epoch, + "producer_alive": camera_outcome == "healthy", + "source_end_expected": False, + "committed_media_segment_count": 0, + }, + "error": ( + {"code": "invalid-fmp4", "message": "synthetic dead producer"} + if camera_outcome == "dead" + else None + ), + } + cancel_calls: list[str] = [] + select_calls: list[tuple[str, str]] = [] + start_recording_calls: list[Path] = [] + retry_calls: list[tuple[str, str, int, str]] = [] + + def snapshot_camera() -> dict[str, object]: + return { + **camera_state, + "recording": dict(camera_state["recording"]), # type: ignore[arg-type] + } + + def cancel_expected_source_end() -> dict[str, object]: + cancel_calls.append("cancel") + return snapshot_camera() + + def select_camera(source_id: str, target: str) -> dict[str, object]: + select_calls.append((source_id, target)) + assert camera_outcome == "clean-ended" + recording = camera_state["recording"] + assert isinstance(recording, dict) + camera_state.update( + { + "phase": "connecting", + "generation": 6, + "active_source_id": source_id, + "error": None, + } + ) + # A real select auto-spawns because the clean-ended gateway retained + # this acquisition's recording root. + recording.update({"active_epoch": 6, "producer_alive": True}) + return snapshot_camera() + + def start_recording(session_dir: Path) -> dict[str, object]: + start_recording_calls.append(session_dir) + assert session_dir == owner.out_dir + return snapshot_camera() + + def retry_recording( + source_id: str, + target: str, + *, + expected_generation: int, + expected_recording_session: str, + pre_retry_fence: Callable[[Callable[[], bool]], bool], + commit_fence: Callable[[Callable[[], bool]], bool] | None = None, + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + retry_calls.append( + (source_id, target, expected_generation, expected_recording_session) + ) + assert camera_outcome == "dead" + assert expected_generation == 5 + assert expected_recording_session == owner.out_dir.name + assert pre_retry_fence(lambda: True) is True + recording = camera_state["recording"] + assert isinstance(recording, dict) + recording.update({"active_epoch": 6, "producer_alive": True}) + camera_state.update( + { + "phase": "streaming", + "generation": 6, + "active_source_id": source_id, + "error": None, + } + ) + assert commit_fence is not None + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(snapshot_camera()) is None + ) is True + return snapshot_camera() + + def activate_recording( + source_id: str, + target: str, + session_dir: Path, + *, + commit_fence: Callable[[Callable[[], bool]], bool], + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + select_camera(source_id, target) + start_recording(session_dir) + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(snapshot_camera()) is None + ) is True + return snapshot_camera() + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot_camera) + monkeypatch.setattr( + service.camera_preview, + "cancel_expected_source_end", + cancel_expected_source_end, + ) + monkeypatch.setattr(service.camera_preview, "select", select_camera) + monkeypatch.setattr(service.camera_preview, "start_recording", start_recording) + monkeypatch.setattr( + service.camera_preview, + "activate_recording_producer", + activate_recording, + ) + monkeypatch.setattr( + service.camera_preview, + "retry_recording_producer", + retry_recording, + ) + activation_lineage = ( + owner.lineage.acquisition_id, + owner.out_dir.name, + owner.runtime_producer_generation, + ) + with service._lock: # noqa: SLF001 + service._camera_activation_lineage = activation_lineage # noqa: SLF001 + + service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner, + reconciliation_id=f"test-scanning-camera-{camera_outcome}", + reconciled_record=reconciled, + ) + + assert cancel_calls == ["cancel"] + assert snapshot_camera()["recording"]["source_end_expected"] is False # type: ignore[index] + assert select_calls == [] + assert start_recording_calls == [] + assert retry_calls == [] + with service._lock: # noqa: SLF001 + if camera_outcome == "healthy": + assert service._camera_activation_lineage == activation_lineage # noqa: SLF001 + assert service._camera_activation_retry_lineage is None # noqa: SLF001 + else: + assert service._camera_activation_lineage is None # noqa: SLF001 + assert service._camera_activation_retry_lineage == activation_lineage # noqa: SLF001 + assert snapshot_camera()["recording"]["active_epoch"] == initial_epoch # type: ignore[index] + + checkpoint_store = service._active_acquisition_checkpoint # noqa: SLF001 + assert checkpoint_store is not None + pending_before_pcl = service._classified_stop_rebind_pending # noqa: SLF001 + assert pending_before_pcl is not None + gap_snapshot = checkpoint_store.snapshot() + assert gap_snapshot.status == "active" + assert gap_snapshot.checkpoint is not None + assert gap_snapshot.checkpoint.last_gap_started_at_utc is not None + assert gap_snapshot.checkpoint.last_gap_recovered_at_utc is None + original_lease = service._acquisition_session_lease # noqa: SLF001 + original_out_dir = service._acquisition_out_dir # noqa: SLF001 + + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + runtime.phase = "live" + runtime.source_mode = "live" + runtime.source_ready = True + runtime.pcl_frames = 1 + stale_frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=2, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=False, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + empty_frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=0, + live=True, + ), + frame_id="map", + positions_xyz=(), + ) + service._observe_published_runtime_envelope( # noqa: SLF001 + stale_frame, + runtime.producer_generation, + ) + service._observe_published_runtime_envelope( # noqa: SLF001 + empty_frame, + runtime.producer_generation, + ) + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation + 1, + ) + with service._camera_restart_commit_gate, service._lock: # noqa: SLF001 + service._camera_stop_priority_counts[ # noqa: SLF001 + owner.lineage.acquisition_id + ] = 1 + try: + assert service._promote_classified_stop_rebind_from_pcl( # noqa: SLF001 + envelope=frame, + producer_generation=runtime.producer_generation, + ) is False + finally: + with service._camera_restart_commit_gate, service._lock: # noqa: SLF001 + service._camera_stop_priority_counts.pop( # noqa: SLF001 + owner.lineage.acquisition_id, + None, + ) + assert service._classified_stop_rebind_pending == pending_before_pcl # noqa: SLF001 + assert service._classified_stop_rebind_inflight is None # noqa: SLF001 + assert checkpoint_store.snapshot().checkpoint == gap_snapshot.checkpoint + assert select_calls == [] + assert start_recording_calls == [] + assert retry_calls == [] + + rebind_calls: list[str] = [] + original_rebind_active = checkpoint_store.rebind_active + + def observe_rebind_active(**kwargs: Any) -> Any: + rebind_calls.append(str(kwargs["transition_id"])) + committed = original_rebind_active(**kwargs) + if camera_outcome == "dead": + raise OSError("synthetic lost response after durable rebind") + return committed + + monkeypatch.setattr( + checkpoint_store, + "rebind_active", + observe_rebind_active, + ) + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + committed_snapshot = checkpoint_store.snapshot() + assert committed_snapshot.status == "active" + assert committed_snapshot.checkpoint is not None + assert committed_snapshot.checkpoint.revision == ( + gap_snapshot.checkpoint.revision + 1 + ) + assert committed_snapshot.checkpoint.last_gap_recovered_at_utc is not None + assert rebind_calls == [pending_before_pcl.transition_id] + assert service._classified_stop_rebind_pending is None # noqa: SLF001 + assert service._classified_stop_rebind_inflight is None # noqa: SLF001 + assert service._acquisition_session_lease is original_lease # noqa: SLF001 + assert service._acquisition_out_dir == original_out_dir # noqa: SLF001 + deadline = time.monotonic() + 2.0 + while ( + camera_outcome != "healthy" + and not (select_calls or retry_calls) + and time.monotonic() < deadline + ): + time.sleep(0.01) + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + time.sleep(0.05) + assert checkpoint_store.snapshot().checkpoint == committed_snapshot.checkpoint + assert rebind_calls == [pending_before_pcl.transition_id] + assert len(select_calls) + len(retry_calls) <= 1 + + if camera_outcome == "healthy": + assert select_calls == [] + assert start_recording_calls == [] + assert retry_calls == [] + assert snapshot_camera()["recording"]["active_epoch"] == initial_epoch # type: ignore[index] + elif camera_outcome == "clean-ended": + assert select_calls == [ + ( + facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + owner.lineage.target_ipv4, + ) + ] + assert start_recording_calls == [owner.out_dir] + assert retry_calls == [] + assert snapshot_camera()["recording"]["active_epoch"] == 6 # type: ignore[index] + else: + assert select_calls == [] + assert start_recording_calls == [] + assert retry_calls == [ + ( + facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + owner.lineage.target_ipv4, + 5, + owner.out_dir.name, + ) + ] + assert snapshot_camera()["recording"]["active_epoch"] == 6 # type: ignore[index] + + +def test_classified_ready_cleanup_failure_retains_owner_then_retries_locally( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id="test-ready-cleanup-retry", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + original_stop_current = service.camera_preview.stop_current + failure_pending = True + + def fail_once_stop_current() -> dict[str, object]: + nonlocal failure_pending + if failure_pending: + failure_pending = False + raise RuntimeError("synthetic camera current cleanup failure") + return original_stop_current() + + monkeypatch.setattr(service.camera_preview, "stop_current", fail_once_stop_current) + + with pytest.raises(RuntimeError, match="synthetic camera current cleanup failure"): + service.state() + failed = service._acquisition # noqa: SLF001 + assert failed is not None + assert failed.state == "failed" + assert failed.result is not None + assert failed.result["device_stop"] == "not-sent" + assert failed.result["local_cleanup_retry_pending"] is True + assert service._acquisition_session_lease is owner.session_lease # noqa: SLF001 + assert service._prepared_stop_recovery_owner is owner # noqa: SLF001 + assert runtime.stop_calls == 0 + + retried = service.state() + assert retried["acquisition"]["state"] == "failed" + assert retried["acquisition"]["cleanup_pending"] is False + assert retried["acquisition"]["result"]["device_stop"] == "not-sent" + assert retried["acquisition"]["result"]["receiver_stopped"] is True + assert "local_cleanup_retry_pending" not in retried["acquisition"]["result"] + assert retried["acquisition"]["message_code"] == ( + "acquisition.recovery.local_cleanup_completed" + ) + assert failed.result is not None + assert failed.result["receiver_stopped"] is True + assert "local_cleanup_retry_pending" not in failed.result + assert service._acquisition_session_lease is None # noqa: SLF001 + assert service._prepared_stop_recovery_owner is None # noqa: SLF001 + assert runtime.stop_calls == 1 + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fixture.stop_operation_id + assert record.resolution == "not-dispatched" + assert fixture.control.stop_calls == 1 + + +@pytest.mark.parametrize( + ("failure_stage", "expected_message_code"), + [ + ( + "camera-process-lease", + "acquisition.recovery.camera_fence_release_failed", + ), + ("runtime", "acquisition.recovery.runtime_cleanup_failed"), + ("capture-clock", "acquisition.recovery.capture_clock_failed"), + ( + "perception-ingress", + "acquisition.recovery.perception_ingress_cleanup_failed", + ), + ( + "evidence-lease", + "acquisition.recovery.evidence_lease_release_failed", + ), + ], +) +def test_classified_ready_staged_cleanup_failure_retains_exact_owner_for_local_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_stage: str, + expected_message_code: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id=f"test-ready-{failure_stage}", + ) + activation_lineage = ( + owner.lineage.acquisition_id, + owner.out_dir.name, + owner.runtime_producer_generation, + ) + with service._lock: # noqa: SLF001 + service._live_perception_camera_binding = ( # noqa: SLF001 + owner.out_dir.name, + facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + 1, + ) + service._camera_activation_lineage = activation_lineage # noqa: SLF001 + service._camera_activation_retry_lineage = activation_lineage # noqa: SLF001 + + failure_pending = True + process_lease = None + if failure_stage == "camera-process-lease": + assert service._ensure_camera_preview_process_lease() is True # noqa: SLF001 + process_lease = service._application_control_process_lease # noqa: SLF001 + assert process_lease is not None + original_release = type(process_lease).release + + def fail_once_process_release(candidate: object) -> object: + nonlocal failure_pending + if candidate is process_lease and failure_pending: + failure_pending = False + raise RuntimeError("synthetic camera process lease release failure") + return original_release(candidate) # type: ignore[arg-type] + + monkeypatch.setattr(type(process_lease), "release", fail_once_process_release) + elif failure_stage == "runtime": + runtime.stop_error = RuntimeError("synthetic recovery runtime stop failure") + elif failure_stage == "capture-clock": + original_seal = service._seal_acquisition_capture_clock # noqa: SLF001 + + def fail_once_clock() -> None: + nonlocal failure_pending + if failure_pending: + failure_pending = False + raise RuntimeError("synthetic recovery capture clock failure") + original_seal() + + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", fail_once_clock) + elif failure_stage == "perception-ingress": + original_end_session = service.live_perception_ingress.end_session + + def fail_once_ingress(session_id: str) -> None: + nonlocal failure_pending + if failure_pending: + failure_pending = False + raise RuntimeError("synthetic recovery ingress cleanup failure") + original_end_session(session_id) + + monkeypatch.setattr( + service.live_perception_ingress, + "end_session", + fail_once_ingress, + ) + elif failure_stage == "evidence-lease": + original_release_evidence = service._release_acquisition_session_lease # noqa: SLF001 + + def fail_once_evidence_release() -> None: + nonlocal failure_pending + if failure_pending: + failure_pending = False + raise RuntimeError("synthetic recovery evidence lease release failure") + original_release_evidence() + + monkeypatch.setattr( + service, + "_release_acquisition_session_lease", + fail_once_evidence_release, + ) + else: # pragma: no cover - exhaustive test table + raise AssertionError(failure_stage) + + with pytest.raises(RuntimeError, match="synthetic"): + service.state() + + failed = service._acquisition # noqa: SLF001 + assert failed is owner.acquisition + assert failed.state == "failed" + assert failed.message_code == expected_message_code + assert failed.result is not None + assert failed.result["device_stop"] == "not-sent" + assert failed.result["local_cleanup_retry_pending"] is True + assert failed.result["local_cleanup_stage"] == failure_stage + assert failed.result["local_cleanup_error_code"] == "RuntimeError" + assert service._acquisition_session_lease is owner.session_lease # noqa: SLF001 + assert service._prepared_stop_recovery_owner is owner # noqa: SLF001 + if failure_stage == "camera-process-lease": + assert runtime.stop_calls == 0 + assert service._application_control_process_lease is process_lease # noqa: SLF001 + assert service._application_control_process_lease_holders == { # noqa: SLF001 + "camera" + } + + runtime.stop_error = None + retried = service.state() + + assert retried["acquisition"]["state"] == "failed" + assert retried["acquisition"]["cleanup_pending"] is False + assert retried["acquisition"]["result"]["device_stop"] == "not-sent" + assert retried["acquisition"]["result"]["receiver_stopped"] is True + assert "local_cleanup_retry_pending" not in retried["acquisition"]["result"] + assert "local_cleanup_stage" not in retried["acquisition"]["result"] + assert retried["acquisition"]["message_code"] == ( + "acquisition.recovery.local_cleanup_completed" + ) + assert service._acquisition_session_lease is None # noqa: SLF001 + assert service._prepared_stop_recovery_owner is None # noqa: SLF001 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert service._application_control_process_lease is None # noqa: SLF001 + assert service._live_perception_camera_binding is None # noqa: SLF001 + assert service._camera_activation_lineage is None # noqa: SLF001 + assert service._camera_activation_retry_lineage is None # noqa: SLF001 + assert fixture.control.stop_calls == 1 + record = fixture.ledger.snapshot().record + assert record is not None + assert record.operation_id == fixture.stop_operation_id + assert record.resolution == "not-dispatched" + + +def test_classified_ready_ambiguous_process_fence_release_requires_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + owner = _settle_real_prepared_stop_before_publish(fixture) + _classify_real_prepared_stop( + fixture, + owner, + session_state="ready", + reconciliation_id="test-ready-ambiguous-process-fence", + ) + assert service._ensure_camera_preview_process_lease() is True # noqa: SLF001 + process_lease = service._application_control_process_lease # noqa: SLF001 + assert process_lease is not None + descriptor = process_lease._descriptor # noqa: SLF001 + original_flock = fcntl.flock + original_close = os.close + unlock_calls = 0 + close_calls = 0 + + def fail_unlock(candidate: int, operation: int) -> None: + nonlocal unlock_calls + if candidate == descriptor and operation == fcntl.LOCK_UN: + unlock_calls += 1 + raise OSError(5, "synthetic unlock failure") + original_flock(candidate, operation) + + def fail_close(candidate: int) -> None: + nonlocal close_calls + if candidate == descriptor: + close_calls += 1 + raise OSError(5, "synthetic close failure") + original_close(candidate) + + monkeypatch.setattr(fcntl, "flock", fail_unlock) + monkeypatch.setattr(os, "close", fail_close) + try: + with pytest.raises( + facade_module.ApplicationControlProcessLeaseReleaseAmbiguous + ): + service.state() + + failed = service._acquisition # noqa: SLF001 + assert failed is owner.acquisition + assert failed.state == "failed" + assert failed.result is not None + assert failed.result["local_cleanup_retry_pending"] is True + assert failed.result["local_cleanup_stage"] == "camera-process-lease" + assert service._acquisition_session_lease is owner.session_lease # noqa: SLF001 + assert service._prepared_stop_recovery_owner is owner # noqa: SLF001 + assert runtime.stop_calls == 0 + assert (unlock_calls, close_calls) == (1, 1) + + quarantined = service.state() + + assert quarantined["acquisition"]["cleanup_pending"] is True + assert quarantined["acquisition"]["result"][ + "local_cleanup_retry_pending" + ] is True + assert quarantined["k1_lifecycle_process_lease"] == { + "held_by_current_service": False, + "holders": [], + "process_lease_quarantined": True, + "reason_code": ( + "application-control-process-lease-release-ambiguous" + ), + "restart_required": True, + } + assert quarantined["connection_policy"]["allowed_actions"] == [] + assert quarantined["connection_policy"]["recommended_action"] == ( + "restart-mission-core" + ) + assert (unlock_calls, close_calls) == (1, 1) + assert runtime.stop_calls == 0 + with pytest.raises( + facade_module.ApplicationControlProcessLeaseUnavailable + ): + service._ensure_camera_preview_process_lease() # noqa: SLF001 + finally: + monkeypatch.setattr(fcntl, "flock", original_flock) + monkeypatch.setattr(os, "close", original_close) + original_flock(descriptor, fcntl.LOCK_UN) + original_close(descriptor) + + +def test_repeated_classified_stop_chain_preserves_pending_start_until_first_pcl( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture( + service, + runtime, + confirm_first_pcl=False, + ) + start_operation = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + assert start_operation.status == "running" + assert service._acquisition_start_operation_id == fixture.start_operation_id # noqa: SLF001 + + service.stop_acquisition(fixture.request) + owner0 = _settle_real_prepared_stop_before_publish(fixture) + reconciled0 = _classify_real_prepared_stop( + fixture, + owner0, + session_state="scanning", + reconciliation_id="test-chain-s0", + ) + binding0 = reconciled0["reconciliations"][-1]["verified_binding"]["connection"] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(binding0), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + service._device_session_id = "fresh-chain-s0-session" # noqa: SLF001 + service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner0, + reconciliation_id="test-chain-s0", + reconciled_record=reconciled0, + ) + assert service._acquisition.state == "awaiting_external_start" # noqa: SLF001 + assert service._acquisition_start_operation_id == fixture.start_operation_id # noqa: SLF001 + + stop1_id = "op-00000000-0000-4000-8000-000000001403" + service.stop_acquisition( + _stop_request( + acquisition_id=owner0.lineage.acquisition_id, + operation_id=stop1_id, + idempotency_key="classified-chain-s1", + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=fixture.control.session_generation, + expected_control_state_revision=fixture.control.state_revision, + ) + ) + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "application-connection-binding-lost", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + service.state() + owner1 = service._prepared_stop_recovery_owner # noqa: SLF001 + assert isinstance(owner1, facade_module._PreparedStopRecoveryOwner) + assert owner1.lineage.stop_operation_id == stop1_id + assert owner1.lineage.parent_physical_operation_id == fixture.stop_operation_id + assert owner1.start_operation_id == fixture.start_operation_id + + reconciled1 = _classify_real_prepared_stop( + fixture, + owner1, + session_state="scanning", + reconciliation_id="test-chain-s1", + ) + binding1 = reconciled1["reconciliations"][-1]["verified_binding"]["connection"] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(binding1), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + with service._lock: # noqa: SLF001 + service._selected_device_id = fixture.binding.transport_ref # noqa: SLF001 + service._connection_mode = fixture.binding.connection_mode # noqa: SLF001 + service._k1_ip = fixture.binding.target_ipv4 # noqa: SLF001 + service._device_session_id = "fresh-chain-s1-session" # noqa: SLF001 + service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner1, + reconciliation_id="test-chain-s1", + reconciled_record=reconciled1, + ) + assert service._matching_start_active_confirmed( # noqa: SLF001 + fixture.coordinator.snapshot(), + acquisition_id=owner1.lineage.acquisition_id, + start_operation_id=fixture.start_operation_id, + verified_control=fixture.control.verified_control, + ) + + repeated_s0 = service.stop_acquisition(fixture.request) + current_after_old_retry = fixture.ledger.snapshot().record + assert current_after_old_retry is not None + assert current_after_old_retry.operation_id == stop1_id + assert current_after_old_retry.stage == "resolved" + assert repeated_s0["acquisition"]["state"] == "awaiting_external_start" + assert fixture.prepare_calls == [fixture.stop_operation_id, stop1_id] + + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + runtime.pcl_frames = 1 + first_pcl = service.state() + start_operation = service._operations.get(fixture.start_operation_id) # noqa: SLF001 + assert first_pcl["acquisition"]["state"] == "acquiring" + assert start_operation.status == "succeeded" + assert start_operation.stage_code == "canonical-start-and-first-point-frame" + assert service._acquisition_start_operation_id is None # noqa: SLF001 + with service._lock: # noqa: SLF001 + service._active_stream_recovery_lineage = None # noqa: SLF001 + service._active_stream_recovery_state = "inactive" # noqa: SLF001 + runtime.phase = "reconnecting" + runtime.source_ready = False + runtime.recovery_state = "reconnecting" + second_disconnect = service._admit_active_stream_recovery_lineage( # noqa: SLF001 + runtime=runtime.snapshot(), + ) + assert second_disconnect is not None + assert second_disconnect.physical_operation_id == stop1_id + assert second_disconnect.start_operation_id == fixture.start_operation_id + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + current_physical = fixture.coordinator.snapshot() + current_record = current_physical["record"] + assert isinstance(current_record, dict) + assert service._physical_active_parent_proof( # noqa: SLF001 + current_physical, + acquisition_id=owner1.lineage.acquisition_id, + parent_operation_id=str(current_record["operation_id"]), + verified_control=fixture.control.verified_control, + ) is not None + + stop2_id = "op-00000000-0000-4000-8000-000000001404" + captured_stop2 = service._capture_prepared_stop_dispatch_lineage( # noqa: SLF001 + operation_id=stop2_id, + acquisition=owner1.acquisition, + ) + assert captured_stop2 is not None + service.stop_acquisition( + _stop_request( + acquisition_id=owner1.lineage.acquisition_id, + operation_id=stop2_id, + idempotency_key="classified-chain-s2", + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=fixture.control.session_generation, + expected_control_state_revision=fixture.control.state_revision, + ) + ) + current = fixture.ledger.snapshot().record + assert current is not None + assert current.operation_id == stop2_id + assert current.parent_operation_id == stop1_id + assert current.stage == "prepared" + + +def test_prepared_stop_gate_defer_preserves_terminal_truth_until_settlement_and_teardown( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + fixture.control.state = "failed" + fixture.control.state_revision += 1 + fixture.control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + + # Simulate the transport owning its global dispatch fence. Both truthful + # settlement and topology retirement defer; generic reducers must not + # convert the still-PREPARED STOP to UNKNOWN in the meantime. + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + deferred = service.state() + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + assert deferred["selected_device_id"] == fixture.binding.transport_ref + assert deferred["application_control_session"]["state"] == "failed" + + # On the next poll, settlement happens before proven-loss retirement could + # reset the failed control owner to idle. Exact NONE instead promotes the + # retained receiver into read-only classification recovery, so teardown is + # intentionally deferred rather than discarding the live evidence owner. + settled = service.state() + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert settled["selected_device_id"] == fixture.binding.transport_ref + assert service._prepared_stop_recovery_owner is not None # noqa: SLF001 + assert settled["application_control_session"]["state"] == "failed" + + +def test_stop_without_exact_parent_proof_fails_before_durable_prepare( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + parent = fixture.ledger.snapshot().record + assert parent is not None + original_snapshot = fixture.coordinator.snapshot + + def snapshot_without_parent_identity() -> dict[str, object]: + snapshot = original_snapshot() + record = snapshot.get("record") + assert isinstance(record, dict) + snapshot["record"] = {**record, "identity": None} + return snapshot + + monkeypatch.setattr(fixture.coordinator, "snapshot", snapshot_without_parent_identity) + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="physical lineage", + ) as raised: + service.stop_acquisition(fixture.request) + + assert raised.value.reason_code == "physical-stop-parent-proof-unavailable" + assert fixture.prepare_calls == [] + assert fixture.control.stop_calls == 0 + current = fixture.ledger.snapshot().record + assert current is not None + assert current.operation_id == parent.operation_id + assert current.stage == "resolved" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + + +@dataclass +class _ActualWorkerStopTransportSnapshot: + state: str + publish_attempts: int + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "publish_attempts": self.publish_attempts, + "qos2_completions": 0, + "correlated_responses": 1, + "ignored_known_responses": 0, + "late_known_responses": 0, + "latest_device_session_state": "scanning", + "latest_device_project_bound": True, + "latest_device_init_ready": True, + "automatic_retry": False, + "automatic_reconnect": False, + "control_proof_revision": 1, + "control_proof_source": "correlated-application-response", + "control_proof_fresh": True, + "control_proof_age_seconds": 0.0, + } + + +class _ActualWorkerStopRaceTransport: + """Fake MQTT client boundary driven by the real control-session worker.""" + + def __init__(self) -> None: + self.state = "new" + self.publish_attempts = 0 + self.fake_client_publish_calls: list[str] = [] + self.evidence_observer: object | None = None + self.dispatch_guard: Callable[[], Callable[[], None] | None] | None = None + self.before_transport_guard = threading.Event() + self.release_transport_guard = threading.Event() + self.after_mark_dispatching = threading.Event() + self.release_fake_publish = threading.Event() + + def install_evidence_observer(self, observer: object) -> None: + self.evidence_observer = observer + + def install_dispatch_guard( + self, + guard: Callable[[], Callable[[], None] | None], + ) -> None: + self.dispatch_guard = guard + + def open(self) -> _ActualWorkerStopTransportSnapshot: + self.state = "ready" + return self.snapshot() + + def close(self) -> None: + self.state = "closed" + + def snapshot(self) -> _ActualWorkerStopTransportSnapshot: + return _ActualWorkerStopTransportSnapshot(self.state, self.publish_attempts) + + def validate_control_proof(self, _binding: LiveDeviceControlBinding) -> None: + return None + + def scan_initialization_complete(self, _binding: LiveDeviceControlBinding) -> bool: + return True + + def exchange_batch_once( + self, + envelopes: Any, + *, + required_response_operation_keys: Any, + ) -> dict[str, bytes]: + envelope = tuple(envelopes)[0] + assert envelope.operation_key == "modeling:stop" + assert required_response_operation_keys == {"modeling:stop"} + assert self.dispatch_guard is not None + release = self.dispatch_guard() + try: + self.publish_attempts += 1 + observer = self.evidence_observer + assert observer is not None + observer.publish_dispatching( + ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=envelope.topic, + payload_sha256=envelope.payload_sha256, + qos=envelope.qos, + retain=envelope.retain, + packet_id=None, + ) + ) + self.after_mark_dispatching.set() + assert self.release_fake_publish.wait(3.0) + self.fake_client_publish_calls.append(envelope.payload_sha256) + raise RuntimeError("synthetic response omitted after fake client publish") + finally: + if release is not None: + release() + + +class _ActualWorkerStopRaceExecutor: + def __init__(self, transport: _ActualWorkerStopRaceTransport) -> None: + self.transport = transport + self.binding = LiveDeviceControlBinding( + vendor_device_id="actual-worker-device", + device_serial="ACTUAL-WORKER-SERIAL", + software_version="3.0.2", + system_version="3.0.2", + device_model="LixelKity K1", + device_type="A4", + is_activated=True, + ) + self.stage = "inspection" + + def run_read_only_inspection_stage(self, _orchestrator: object) -> LiveDeviceControlBinding: + observer = self.transport.evidence_observer + assert observer is not None + observer.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="8" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-12T09:00:00.000Z", + ) + ) + return self.binding + + def wait_for_operator_checkpoint( + self, + _event: str, + observed: Callable[[], bool], + *, + reconciled_active_observed: Callable[[], bool] | None = None, + ) -> None: + deadline = time.monotonic() + 3.0 + while not observed(): + if reconciled_active_observed is not None and reconciled_active_observed(): + return None + if time.monotonic() >= deadline: + raise TimeoutError("actual worker checkpoint was not released") + threading.Event().wait(0.005) + return None + + def adopt_reconciled_scanning(self, **_: object) -> None: + self.stage = "scanning" + + def maintain_active_until_stop_requested(self, observed: Callable[[], bool]) -> None: + while not observed(): + threading.Event().wait(0.005) + self.stage = "stop-requested" + + def execute_canonical_stop( + self, + command: object, + _permit: object, + *, + dispatch_guard: Callable[[], None] | None = None, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + ) -> object: + self.transport.before_transport_guard.set() + assert self.transport.release_transport_guard.wait(3.0) + if dispatch_guard is not None: + dispatch_guard() + assert dispatch_admission_deadline_reached is not None + assert dispatch_admission_deadline_reached() is False + envelope = OneShotPublishEnvelope.from_modeling_command(command) # type: ignore[arg-type] + return self.transport.exchange_batch_once( + [envelope], + required_response_operation_keys={"modeling:stop"}, + ) + + def maintain_post_stop_until_standby(self) -> None: + raise RuntimeError("synthetic standby omitted") + + def snapshot(self) -> dict[str, object]: + return { + "dialogue_stage": self.stage, + "start_attempted": True, + "start_complete": True, + "stop_attempted": self.transport.publish_attempts > 0, + "stop_complete": False, + "automatic_retry": False, + } + + +def _wait_control_state( + session: InteractiveApplicationControlSession, + expected: set[str], +) -> dict[str, object]: + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline: + snapshot = session.snapshot() + if str(snapshot.get("state")) in expected: + return snapshot + threading.Event().wait(0.005) + raise AssertionError(f"control did not reach {expected}: {session.snapshot()}") + + +def _install_actual_worker_recovered_stop_fixture( + service: XgridsK1CompatibilityService, + runtime: FakeVisualizationRuntime, + monkeypatch: pytest.MonkeyPatch, +) -> SimpleNamespace: + binding = _seed_supervised_connection( + service, + transport_ref="actual-worker-transport", + logical_device_id="actual-worker-device", + with_control=False, + ) + route_available = {"value": True} + monkeypatch.setattr( + service, + "_sample_host_path", + lambda _target, **_kwargs: ( + _direct_host_path(binding.target_ipv4) + if route_available["value"] + else HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + ), + ) + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=hashlib.sha256(b"actual-worker-device").hexdigest(), + device_serial_sha256=hashlib.sha256(b"ACTUAL-WORKER-SERIAL").hexdigest(), + ) + old_connection = PhysicalCommandConnectionBinding( + intent_id="old-active-intent", + transport_ref=binding.transport_ref, + connection_mode=binding.connection_mode, + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id="old-active-control", + producer_generation=1, + ) + + def physical_status(state: str, observed: str) -> PhysicalCommandStatusEvidence: + scanning = state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=old_connection.control_session_id, + host_path_epoch=old_connection.host_path_epoch, + producer_generation=old_connection.producer_generation, + session_state=state, # type: ignore[arg-type] + session_state_code=302 if scanning else 300, + project_bound=scanning, + project_id_sha256="a" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256=hashlib.sha256(observed.encode()).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + start_operation_id = "op-00000000-0000-4000-8000-000000001420" + acquisition_id = "acq-actual-worker-stop-race" + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=old_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="1" * 64, + baseline_status=physical_status("ready", "2026-08-12T09:00:01.000Z"), + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing(start_operation_id, publish_call_returned=True, packet_id=81) + ledger.mark_qos2_completed(start_operation_id, packet_id=81) + ledger.record_application_response( + start_operation_id, + PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=old_connection.control_session_id, + host_path_epoch=old_connection.host_path_epoch, + producer_generation=old_connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="2" * 64, + observed_at_utc="2026-08-12T09:00:02.000Z", + ), + ) + ledger.record_status_observation( + start_operation_id, + physical_status("scanning", "2026-08-12T09:00:03.000Z"), + ) + ledger.resolve(start_operation_id, resolution="start-active-observed") + + transport = _ActualWorkerStopRaceTransport() + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + _ActualWorkerStopRaceExecutor, + ) + coordinator = service._physical_command_coordinator # noqa: SLF001 + control = InteractiveApplicationControlSession( + FakeApplicationAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + connection_path_validator=service._validate_application_connection_path, # noqa: SLF001 + connection_binding_validator=service._validate_application_connection_binding, # noqa: SLF001 + connection_dispatch_lease=service._acquire_application_dispatch_lease, # noqa: SLF001 + physical_command_coordinator=coordinator, + ) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + control.open( + host=binding.target_ipv4, + timezone_name="Europe/Moscow", + connection_binding=binding, + inspection_only=True, + ) + ready = _wait_control_state(control, {"connection-ready"}) + verified = ready["verified_control"] + assert isinstance(verified, dict) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256="a" * 64, + init_ready=True, + status_message_sha256="3" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T09:00:04.000Z", + ) + ) + reconciliation_id = "actual-worker-rebind" + reconciled = coordinator.reconcile_resolved_active(reconciliation_id=reconciliation_id) + control.adopt_reconciled_scanning( + reconciliation_id=reconciliation_id, + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + scanning = _wait_control_state(control, {"scanning"}) + service._reconcile_connection_supervisor(scanning, runtime.snapshot()) # noqa: SLF001 + service._materialize_recovered_physical_stop_acquisition( # noqa: SLF001 + reconciliation_id=reconciliation_id, + reconciled_record=reconciled, + ) + stop_operation_id = "op-00000000-0000-4000-8000-000000001421" + request = _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + idempotency_key="actual-worker-stop-race", + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=scanning["session_generation"], + expected_control_state_revision=scanning["state_revision"], + ) + return SimpleNamespace( + binding=binding, + route_available=route_available, + ledger=ledger, + coordinator=coordinator, + control=control, + transport=transport, + request=request, + stop_operation_id=stop_operation_id, + ) + + +def test_capture_only_rejects_real_worker_queued_before_prepared_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_actual_worker_recovered_stop_fixture( + service, + runtime, + monkeypatch, + ) + service.stop_acquisition(fixture.request) + assert fixture.transport.before_transport_guard.wait(3.0) + before_operations = tuple(service._operations.snapshot()) # noqa: SLF001 + before_acquisition = service._acquisition # noqa: SLF001 + + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="ещё владеет PREPARED STOP", + ) as raised: + service.stop_acquisition( + _stop_request( + acquisition_id=fixture.request.acquisition_id, + operation_id="op-00000000-0000-4000-8000-000000001458", + mode="capture-only", + ) + ) + + assert raised.value.reason_code == "acquisition-stop-worker-retirement-pending" + assert tuple(service._operations.snapshot()) == before_operations # noqa: SLF001 + assert service._acquisition is before_acquisition # noqa: SLF001 + assert fixture.transport.fake_client_publish_calls == [] + assert fixture.transport.publish_attempts == 0 + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + assert runtime.stop_calls == 0 + + # Rejection performs no socket close or cancellation. The one accepted + # canonical S0 worker remains the sole owner and may cross its original + # publish edge exactly once after the test releases it. + fixture.transport.release_transport_guard.set() + assert fixture.transport.after_mark_dispatching.wait(3.0) + fixture.transport.release_fake_publish.set() + _wait_control_state(fixture.control, {"failed"}) + assert len(fixture.transport.fake_client_publish_calls) == 1 + assert fixture.transport.publish_attempts == 1 + record = fixture.ledger.snapshot().record + assert record is not None and record.stage in {"dispatching", "observing"} + + +def test_actual_worker_stop_survives_both_loss_reducer_seams_and_publishes_once( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_actual_worker_recovered_stop_fixture( + service, + runtime, + monkeypatch, + ) + service.stop_acquisition(fixture.request) + assert fixture.transport.before_transport_guard.wait(3.0) + + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.intent is not None and supervisor.device_network.target is not None + for _ in range(3): + assert service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=supervisor.device_network.target, + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + reachable=False, + reason_code="transient-control-port-negative", + ) + retained = service.state() + assert retained["selected_device_id"] == fixture.binding.transport_ref + assert fixture.transport.fake_client_publish_calls == [] + + # Restore the exact control-plane proof after the synthetic transient + # endpoint negatives; the first seam tested only PREPARED lineage. + service._reconcile_connection_supervisor( # noqa: SLF001 + fixture.control.snapshot(), + runtime.snapshot(), + ) + refreshed = fixture.control.snapshot() + refreshed_verified = refreshed.get("verified_control") + assert isinstance(refreshed_verified, dict) + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.intent is not None and supervisor.device_network.target is not None + assert service._connection_supervisor.observe_endpoint( # noqa: SLF001 + target=supervisor.device_network.target, + intent_id=supervisor.intent.intent_id, + host_path_epoch=supervisor.host_path.epoch, + reachable=True, + ) + assert service._connection_supervisor.observe_control_evidence( # noqa: SLF001 + VerifiedControlEvidence( + intent_id=supervisor.intent.intent_id, + transport_ref=fixture.binding.transport_ref, + host_path_epoch=supervisor.host_path.epoch, + target=supervisor.device_network.target, + connection_mode=fixture.binding.connection_mode, + logical_device_id="actual-worker-device", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id=str(refreshed_verified["control_session_id"]), + control_proof_revision=2, + control_proof_source="correlated-application-response", + ) + ) + fixture.transport.release_transport_guard.set() + assert fixture.transport.after_mark_dispatching.wait(3.0) + + # The real worker now owns the facade dispatch gate after mark_dispatching. + # A true route loss cannot tear topology out from under fake client.publish. + for _ in range(3): + _observe_stop_race_route_loss(service) + mid_publish = service.state() + assert mid_publish["selected_device_id"] == fixture.binding.transport_ref + assert fixture.transport.fake_client_publish_calls == [] + fixture.transport.release_fake_publish.set() + failed = _wait_control_state(fixture.control, {"failed"}) + assert failed["state"] == "failed" + assert len(fixture.transport.fake_client_publish_calls) == 1 + assert fixture.transport.publish_attempts == 1 + after_failure = service.state() + physical_record = fixture.ledger.snapshot().record + assert physical_record is not None + assert physical_record.stage in {"dispatching", "observing"} + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.stage_code != "physical-stop-classified-not-dispatched" + assert operation.error is None or operation.error.get("side_effect_status") != "none" + assert after_failure["physical_command"]["record"]["stage"] in { + "dispatching", + "observing", + } + + +def test_actual_worker_real_route_loss_fails_before_publish_and_settles_none( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_actual_worker_recovered_stop_fixture( + service, + runtime, + monkeypatch, + ) + service.stop_acquisition(fixture.request) + assert fixture.transport.before_transport_guard.wait(3.0) + fixture.route_available["value"] = False + for _ in range(3): + _observe_stop_race_route_loss(service) + retained = service.state() + assert retained["selected_device_id"] == fixture.binding.transport_ref + + fixture.transport.release_transport_guard.set() + failed = _wait_control_state(fixture.control, {"failed"}) + assert failed["failure"]["reason_code"] == "application-connection-binding-lost" # type: ignore[index] + assert fixture.transport.fake_client_publish_calls == [] + assert fixture.transport.publish_attempts == 0 + settled = service.state() + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert settled["selected_device_id"] is None + + +@pytest.mark.parametrize( + ("durable_start_resolved", "expected_side_effect_status"), + [(False, "unknown"), (True, "succeeded")], +) +@pytest.mark.parametrize("coincident_local_error", ["none", "camera", "runtime"]) +def test_plugin_start_requires_composite_active_proof_and_fails_closed_on_control_fault( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + durable_start_resolved: bool, + expected_side_effect_status: str, + coincident_local_error: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="START_GATE", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + pre_pcl_state = service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + pre_pcl_camera_streams = [ + stream + for stream in pre_pcl_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert pre_pcl_state["camera_preview"]["active_source_id"] is None + assert pre_pcl_state["camera_preview"]["delivery"] is None + assert pre_pcl_state["camera_preview"]["activation_admission"]["state"] == ( + "waiting-for-first-authoritative-pcl" + ) + assert all(stream["activation"]["selected"] is False for stream in pre_pcl_camera_streams) + assert all(stream["activation"]["controllable"] is False for stream in pre_pcl_camera_streams) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + physical_proof = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=durable_start_resolved, + ) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) + + # The live stream can begin during SCAN_STARTING. Neither frames alone nor + # even a durable edge observed after this browser reduction is enough while + # the current canonical control owner is not SCANNING/can_stop. + control.state = "initializing" + runtime.mark_ready() + runtime.pcl_frames = 30 + initializing = service.state() + assert initializing["acquisition"]["state"] == "awaiting_external_start" + assert initializing["last_operation"]["status"] == "running" + + original_control_snapshot = control.snapshot + + def failed_control_snapshot() -> dict[str, object]: + snapshot = original_control_snapshot() + snapshot["outcome_unknown"] = True + snapshot["failure"] = { + "code": "ApplicationControlDeviceFault", + "reason_code": "device_status_fault", + "failed_phase": "initializing", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + snapshot["transport"] = { + "latest_system_error_code": 0x32040133, + "latest_system_error_state": "algorithm_error", + } + return snapshot + + if coincident_local_error == "camera": + original_camera_snapshot = service.camera_preview.snapshot + + def failed_camera_snapshot() -> dict[str, Any]: + return { + **original_camera_snapshot(), + "phase": "error", + "recording": {"active": True}, + "error": {"code": "coincident-camera-fault"}, + } + + monkeypatch.setattr(service.camera_preview, "snapshot", failed_camera_snapshot) + elif coincident_local_error == "runtime": + runtime.phase = "error" + + monkeypatch.setattr(control, "snapshot", failed_control_snapshot) + control.state = "failed" + failed = service.state() + start_operation = next( + operation + for operation in failed["operations"] + if operation["operation_id"] == start_operation_id + ) + assert failed["acquisition"]["state"] == "failed" + assert failed["acquisition"]["cleanup_pending"] is True + assert failed["source_mode"] == "live" + assert start_operation["status"] == "failed" + assert start_operation["stage_code"] == "device-start-outcome-unknown" + assert start_operation["error"]["safe_to_retry"] is False + assert start_operation["error"]["side_effect_status"] == (expected_side_effect_status) + assert start_operation["error"]["automatic_replay_allowed"] is False + assert runtime.stop_calls == 0 + assert control.stop_calls == 0 + + runtime.pcl_frames += 1 + still_failed = service.state() + repeated_start_operation = next( + operation + for operation in still_failed["operations"] + if operation["operation_id"] == start_operation_id + ) + assert still_failed["acquisition"]["state"] == "failed" + assert still_failed["acquisition"]["cleanup_pending"] is True + assert still_failed["source_mode"] == "live" + assert repeated_start_operation == start_operation + assert runtime.stop_calls == 0 + assert control.stop_calls == 0 + + locally_stopped = service.stop_acquisition( + _stop_request(acquisition_id=acquisition_id, mode="capture-only") + ) + assert locally_stopped["source_mode"] == "idle" + assert locally_stopped["acquisition"]["cleanup_pending"] is False + assert locally_stopped["last_operation"]["result"]["device_stop"] == "unknown" + assert runtime.stop_calls == 1 + assert control.stop_calls == 0 + + +@pytest.mark.parametrize( + ("durable_start_resolved", "expected_side_effect_status"), + [(False, "unknown"), (True, "succeeded")], +) +@pytest.mark.parametrize("local_error", ["camera", "runtime"]) +def test_plugin_start_local_failure_preserves_post_checkpoint_physical_outcome( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + durable_start_resolved: bool, + expected_side_effect_status: str, + local_error: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="START_LOCAL_FAILURE", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + pre_pcl_state = service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + pre_pcl_camera_streams = [ + stream + for stream in pre_pcl_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert pre_pcl_state["camera_preview"]["active_source_id"] is None + assert pre_pcl_state["camera_preview"]["delivery"] is None + assert pre_pcl_state["camera_preview"]["activation_admission"]["state"] == ( + "waiting-for-first-authoritative-pcl" + ) + assert all(stream["activation"]["selected"] is False for stream in pre_pcl_camera_streams) + assert all(stream["activation"]["controllable"] is False for stream in pre_pcl_camera_streams) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + physical_proof = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=durable_start_resolved, + ) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) + runtime.mark_ready() + awaiting = service.state() + assert awaiting["acquisition"]["state"] == "awaiting_external_start" + assert awaiting["last_operation"]["status"] == "running" + + if local_error == "camera": + original_camera_snapshot = service.camera_preview.snapshot + + def failed_camera_snapshot() -> dict[str, Any]: + return { + **original_camera_snapshot(), + "phase": "error", + "recording": {"active": True}, + "error": {"code": "camera-failed-before-local-start-acceptance"}, + } + + monkeypatch.setattr(service.camera_preview, "snapshot", failed_camera_snapshot) + else: + runtime.phase = "error" + + failed = service.state() + start_operation = next( + operation + for operation in failed["operations"] + if operation["operation_id"] == start_operation_id + ) + assert failed["acquisition"]["state"] == "failed" + assert failed["acquisition"]["cleanup_pending"] is False + assert failed["source_mode"] == "idle" + assert start_operation["status"] == "failed" + assert start_operation["stage_code"] == "runtime-failed" + assert start_operation["error"]["safe_to_retry"] is False + assert start_operation["error"]["side_effect_status"] == (expected_side_effect_status) + assert start_operation["error"]["automatic_replay_allowed"] is False + assert runtime.stop_calls == 1 + assert control.stop_calls == 0 + + +@pytest.mark.parametrize("local_failure", ["none", "cleanup", "camera"]) +def test_durable_stop_ready_auto_cleans_local_receiver_after_control_loss( + tmp_path: Path, + local_failure: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A resolved exact STOP survives Wi-Fi/control retirement as completion truth.""" + + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="k1-stop-ready", + ) + control.verified_control = _verified_control_for_binding( + binding, + control_session_id="stop-ready-control", + ) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="STOP_READY", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + ) + physical_connection = PhysicalCommandConnectionBinding( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode="bridge", + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id="stop-ready-control", + producer_generation=1, + ) + + def status( + session_state: str, + *, + observed_at_utc: str, + ) -> PhysicalCommandStatusEvidence: + scanning = session_state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + session_state=session_state, # type: ignore[arg-type] + session_state_code=302 if scanning else 300, + project_bound=scanning, + project_id_sha256="c" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256=hashlib.sha256( + f"{session_state}:{observed_at_utc}".encode() + ).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=physical_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="1" * 64, + baseline_status=status( + "ready", + observed_at_utc="2026-08-10T08:01:30.000Z", + ), + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing( + start_operation_id, + publish_call_returned=True, + packet_id=94, + ) + ledger.mark_qos2_completed(start_operation_id, packet_id=94) + ledger.record_application_response( + start_operation_id, + PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="2" * 64, + observed_at_utc="2026-08-10T08:01:31.000Z", + ), + ) + ledger.record_status_observation( + start_operation_id, + status("scanning", observed_at_utc="2026-08-10T08:01:32.000Z"), + ) + ledger.resolve(start_operation_id, resolution="start-active-observed") + runtime.mark_ready() + runtime.pcl_frames = 1 + assert service.state()["acquisition"]["state"] == "acquiring" + + # Match an acquisition that resumed through the exact read-only recovery + # owner before the operator requested the one canonical STOP. + active_start_record = ledger.snapshot().record + assert active_start_record is not None + recovery_lineage = facade_module._ActiveStreamRecoveryLineage( # noqa: SLF001 + recovery_generation=service._active_stream_recovery_generation, # noqa: SLF001 + snapshot_runtime_id=service._snapshot_runtime_id, # noqa: SLF001 + acquisition_id=acquisition_id, + device_id=service._device_id, # type: ignore[arg-type] # noqa: SLF001 + device_session_id=service._device_session_id, # type: ignore[arg-type] # noqa: SLF001 + evidence_session_id=service._acquisition_out_dir.name, # type: ignore[union-attr] # noqa: SLF001 + runtime_producer_generation=runtime.producer_generation, + start_operation_id=start_operation_id, + physical_operation_id=start_operation_id, + physical_revision=active_start_record.revision, + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode="bridge", + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + ) + with service._lock: # noqa: SLF001 + service._active_stream_recovery_lineage = recovery_lineage # noqa: SLF001 + service._active_stream_recovery_state = "recovered" # noqa: SLF001 + service._active_stream_recovery_attempt = 7 # noqa: SLF001 + runtime.recovery_state = "recovered" + runtime.recovery_attempt = 7 + + stop_operation_id = "op-00000000-0000-4000-8000-000000001299" + stop_request = _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + stopping = service.stop_acquisition(stop_request) + assert stopping["acquisition"]["state"] == "awaiting_external_stop" + assert stopping["last_operation"]["status"] == "running" + assert control.stop_calls == 1 + assert runtime.source_mode == "live" + + # Wi-Fi/control disappears after the one STOP request. Model the exact + # production ordering: state() begins with a pending local retirement, + # turns the completed ephemeral control owner into idle, and only then + # reduces the durable READY edge. + control.state = "completed" + service._pending_local_control_retirement = True # noqa: SLF001 + ledger.prepare( + operation_id=stop_operation_id, + parent_operation_id=start_operation_id, + acquisition_id=acquisition_id, + action="stop", + identity=identity, + connection=physical_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="d" * 64, + baseline_status=status( + "scanning", + observed_at_utc="2026-08-10T08:01:40.000Z", + ), + ) + ledger.mark_dispatching(stop_operation_id) + ledger.mark_observing( + stop_operation_id, + publish_call_returned=True, + packet_id=95, + ) + ledger.mark_qos2_completed(stop_operation_id, packet_id=95) + ledger.record_application_response( + stop_operation_id, + PhysicalCommandApplicationResponse( + operation_id=stop_operation_id, + action="stop", + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="e" * 64, + observed_at_utc="2026-08-10T08:01:41.000Z", + ), + ) + ledger.record_status_observation( + stop_operation_id, + status("ready", observed_at_utc="2026-08-10T08:01:44.000Z"), + ) + ledger.resolve(stop_operation_id, resolution="stop-standby-observed") + + unintended_edges: list[str] = [] + + async def forbidden_device_edge(*_args: object, **_kwargs: object) -> object: + unintended_edges.append("device-or-network-io") + raise AssertionError("durable STOP READY reduction must be local-only") + + monkeypatch.setattr(facade_module, "scan", forbidden_device_edge) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_edge) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_edge) + + if local_failure == "cleanup": + runtime.stop_error = RuntimeError("synthetic local cleanup failure after READY") + elif local_failure == "camera": + original_camera_snapshot = service.camera_preview.snapshot + + def failed_camera_snapshot() -> dict[str, Any]: + snapshot = original_camera_snapshot() + return { + **snapshot, + "phase": "error", + "recording": {"active": True}, + "error": {"code": "synthetic-camera-producer-failed"}, + } + + monkeypatch.setattr(service.camera_preview, "snapshot", failed_camera_snapshot) + + completed = service.state() + operation = next( + item for item in completed["operations"] if item["operation_id"] == stop_operation_id + ) + + assert completed["application_control_session"]["state"] == "idle" + assert completed["physical_command"]["record"]["resolution"] == ("stop-standby-observed") + if local_failure == "cleanup": + assert completed["acquisition"]["state"] == "failed" + assert completed["acquisition"]["cleanup_pending"] is True + assert operation["status"] == "failed" + assert operation["stage_code"] == "local-finalization-failed" + assert operation["error"] == { + "category": "stream", + "code": "local-finalization-failed-after-device-standby", + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "succeeded", + } + assert completed["source_mode"] == "live" + assert completed["connection_policy"]["actions"]["scan-ble"]["allowed"] is False + + # Local cleanup alone is retried by polling. The physical STOP remains + # terminal and is never published again. + runtime.stop_error = None + completed = service.state() + operation = next( + item for item in completed["operations"] if item["operation_id"] == stop_operation_id + ) + assert completed["acquisition"]["state"] == "failed" + assert completed["acquisition"]["cleanup_pending"] is False + assert operation["status"] == "failed" + assert operation["stage_code"] == "local-finalization-failed" + assert runtime.stop_calls == 2 + elif local_failure == "camera": + assert completed["acquisition"]["state"] == "failed" + assert completed["acquisition"]["cleanup_pending"] is False + assert completed["acquisition"]["result"] == { + "receiver_stopped": True, + "device_state": "ready", + "device_stop": "protocol-confirmed", + "camera_failure_code": "synthetic-camera-producer-failed", + } + assert operation["status"] == "failed" + assert operation["stage_code"] == "local-finalization-failed" + assert operation["error"]["side_effect_status"] == "succeeded" + assert runtime.stop_calls == 1 + else: + assert completed["acquisition"]["state"] == "completed" + assert completed["acquisition"]["cleanup_pending"] is False + assert completed["acquisition"]["result"] == { + "receiver_stopped": True, + "device_state": "ready", + "device_stop": "protocol-confirmed", + } + assert operation["status"] == "succeeded" + assert operation["stage_code"] == "device-standby-confirmed" + assert runtime.stop_calls == 1 + assert completed["source_mode"] == "idle" + assert completed["live_perception_shadow"]["active"] is False + assert completed["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert unintended_edges == [] + assert control.stop_calls == 1 + if local_failure != "cleanup": + assert completed["connection_recovery"]["state"] == "inactive" + assert completed["connection_recovery"]["automatic_read_only_rebind"] is False + assert service._active_stream_recovery_lineage is None # noqa: SLF001 + + # Polling the same durable proof is a pure idempotent read: no second + # local teardown and, critically, no second physical STOP. + stop_call_count = runtime.stop_calls + repeated = service.state() + repeated_operation = next( + item for item in repeated["operations"] if item["operation_id"] == stop_operation_id + ) + assert repeated_operation["status"] == operation["status"] + assert repeated["acquisition"]["state"] == completed["acquisition"]["state"] + assert repeated["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert unintended_edges == [] + assert control.stop_calls == 1 + assert runtime.stop_calls == stop_call_count + + # A process restart retains only the resolved durable edge. There is no + # process-local receiver to recover and no physical action to replay. + restarted, restarted_runtime = service_with_fake_runtime(tmp_path) + restarted_state = restarted.state() + assert restarted_state["physical_command"]["record"]["operation_id"] == (stop_operation_id) + assert restarted_state["physical_command"]["record"]["resolution"] == ("stop-standby-observed") + assert restarted_state["acquisition"] is None + assert restarted_state["source_mode"] == "idle" + assert restarted_state["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert restarted_runtime.stop_calls == 0 + + +@pytest.mark.parametrize( + "mismatch", + [ + "operation", + "acquisition", + "response-operation", + "retained-ready", + "bound-ready", + "initialized-ready", + "missing-qos2", + ], +) +def test_durable_stop_ready_rejects_nonmatching_or_noncanonical_proof( + mismatch: str, +) -> None: + stop_operation_id = "op-00000000-0000-4000-8000-000000001399" + acquisition_id = "acquisition-stop-ready-proof" + proof: dict[str, Any] = { + "status": "resolved", + "requires_reconciliation": False, + "record": { + "operation_id": stop_operation_id, + "acquisition_id": acquisition_id, + "action": "stop", + "stage": "resolved", + "resolution": "stop-standby-observed", + "publish_call_returned": True, + "qos2_completed": True, + "packet_id": 91, + "application_response": { + "operation_id": stop_operation_id, + "action": "stop", + "success": True, + }, + "last_status": { + "session_state": "ready", + "project_bound": False, + "init_ready": False, + "mqtt_retained": False, + }, + }, + } + record = proof["record"] + if mismatch == "operation": + record["operation_id"] = "different-stop-operation" + elif mismatch == "acquisition": + record["acquisition_id"] = "different-acquisition" + elif mismatch == "response-operation": + record["application_response"]["operation_id"] = "different-stop-operation" + elif mismatch == "retained-ready": + record["last_status"]["mqtt_retained"] = True + elif mismatch == "bound-ready": + record["last_status"]["project_bound"] = True + elif mismatch == "initialized-ready": + record["last_status"]["init_ready"] = True + elif mismatch == "missing-qos2": + record["qos2_completed"] = False + + assert ( + XgridsK1CompatibilityService._matching_stop_standby_confirmed( # noqa: SLF001 + proof, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + is False + ) + + +@pytest.mark.parametrize( + "mismatch", + ["operation", "acquisition", "response-operation", "response-failed", "ready", "stopping"], +) +def test_stop_timeout_matcher_rejects_wrong_lineage_or_terminal_status( + mismatch: str, +) -> None: + stop_operation_id = "op-00000000-0000-4000-8000-000000001489" + acquisition_id = "acquisition-stop-timeout-proof" + proof: dict[str, Any] = { + "status": "unresolved", + "requires_reconciliation": True, + "record": { + "operation_id": stop_operation_id, + "acquisition_id": acquisition_id, + "action": "stop", + "stage": "observing", + "resolution": None, + # An exact application response may precede or outlive the local + # QoS2 callback. Timeout cleanup is local-only in either case. + "publish_call_returned": None, + "qos2_completed": False, + "packet_id": None, + "application_response": { + "operation_id": stop_operation_id, + "action": "stop", + "success": True, + "observed_at_utc": "2026-08-10T08:10:02.000Z", + }, + "last_status": None, + }, + } + record = proof["record"] + if mismatch == "operation": + record["operation_id"] = "different-stop-operation" + elif mismatch == "acquisition": + record["acquisition_id"] = "different-acquisition" + elif mismatch == "response-operation": + record["application_response"]["operation_id"] = "different-stop-operation" + elif mismatch == "response-failed": + record["application_response"]["success"] = False + elif mismatch in {"ready", "stopping"}: + record["last_status"] = { + "session_state": "ready" if mismatch == "ready" else "scan_stopping" + } + + assert ( + XgridsK1CompatibilityService._matching_stop_response_without_terminal_status( # noqa: SLF001 + proof, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + is False + ) + + +def test_stop_timeout_matcher_accepts_exact_response_without_qos2_callback() -> None: + stop_operation_id = "op-00000000-0000-4000-8000-000000001488" + acquisition_id = "acquisition-stop-timeout-no-qos" + proof = { + "status": "unresolved", + "requires_reconciliation": True, + "record": { + "operation_id": stop_operation_id, + "acquisition_id": acquisition_id, + "action": "stop", + "stage": "observing", + "resolution": None, + "publish_call_returned": None, + "qos2_completed": False, + "packet_id": None, + "application_response": { + "operation_id": stop_operation_id, + "action": "stop", + "success": True, + "observed_at_utc": "2026-08-10T08:10:02.000Z", + }, + "last_status": None, + }, + } + + assert XgridsK1CompatibilityService._matching_stop_response_without_terminal_status( # noqa: SLF001 + proof, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + ) + + +def _stop_response_without_terminal_status_fixture( + tmp_path: Path, + *, + qos2_completed: bool = True, +) -> SimpleNamespace: + clock_value = [datetime(2026, 8, 10, 8, 10, tzinfo=UTC)] + service, runtime = service_with_fake_runtime(tmp_path) + service._operations = OperationJournal(clock=lambda: clock_value[0]) # noqa: SLF001 + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="k1-stop-timeout", + ) + control.verified_control = _verified_control_for_binding( + binding, + control_session_id="stop-timeout-control", + ) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="STOP_TIMEOUT", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256="4" * 64, + device_serial_sha256="5" * 64, + ) + physical_connection = PhysicalCommandConnectionBinding( + intent_id=binding.intent_id, + transport_ref=binding.transport_ref, + connection_mode="bridge", + target_ipv4=binding.target_ipv4, + target_port=binding.target_port, + host_path_epoch=binding.host_path_epoch, + control_session_id="stop-timeout-control", + producer_generation=1, + ) + + def status( + session_state: str, + *, + observed_at_utc: str, + ) -> PhysicalCommandStatusEvidence: + scanning = session_state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + session_state=session_state, # type: ignore[arg-type] + session_state_code=302 if scanning else 300, + project_bound=scanning, + project_id_sha256="6" * 64 if scanning else None, + init_ready=scanning, + status_message_sha256=hashlib.sha256( + f"{session_state}:{observed_at_utc}".encode() + ).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=physical_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="7" * 64, + baseline_status=status( + "ready", + observed_at_utc="2026-08-10T08:09:55.000Z", + ), + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing( + start_operation_id, + publish_call_returned=True, + packet_id=81, + ) + ledger.mark_qos2_completed(start_operation_id, packet_id=81) + ledger.record_application_response( + start_operation_id, + PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="8" * 64, + observed_at_utc="2026-08-10T08:09:56.000Z", + ), + ) + ledger.record_status_observation( + start_operation_id, + status("scanning", observed_at_utc="2026-08-10T08:09:57.000Z"), + ) + ledger.resolve(start_operation_id, resolution="start-active-observed") + runtime.mark_ready() + runtime.pcl_frames = 1 + assert service.state()["acquisition"]["state"] == "acquiring" + + stop_operation_id = "op-00000000-0000-4000-8000-000000001499" + stop_request = _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + deadline_seconds=10.0, + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + stopping = service.stop_acquisition(stop_request) + assert stopping["acquisition"]["state"] == "awaiting_external_stop" + assert stopping["last_operation"]["status"] == "running" + + ledger.prepare( + operation_id=stop_operation_id, + parent_operation_id=start_operation_id, + acquisition_id=acquisition_id, + action="stop", + identity=identity, + connection=physical_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="9" * 64, + baseline_status=status( + "scanning", + observed_at_utc="2026-08-10T08:10:00.000Z", + ), + ) + ledger.mark_dispatching(stop_operation_id) + if qos2_completed: + ledger.mark_observing( + stop_operation_id, + publish_call_returned=True, + packet_id=82, + ) + ledger.mark_qos2_completed(stop_operation_id, packet_id=82) + ledger.record_application_response( + stop_operation_id, + PhysicalCommandApplicationResponse( + operation_id=stop_operation_id, + action="stop", + control_session_id=physical_connection.control_session_id, + host_path_epoch=physical_connection.host_path_epoch, + producer_generation=physical_connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="a" * 64, + observed_at_utc="2026-08-10T08:10:02.000Z", + ), + ) + return SimpleNamespace( + service=service, + runtime=runtime, + control=control, + ledger=ledger, + clock_value=clock_value, + acquisition_id=acquisition_id, + stop_operation_id=stop_operation_id, + status=status, + ) + + +@pytest.mark.parametrize("qos2_completed", [True, False]) +def test_stop_success_without_terminal_status_times_out_into_local_only_recovery( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + qos2_completed: bool, +) -> None: + fixture = _stop_response_without_terminal_status_fixture( + tmp_path, + qos2_completed=qos2_completed, + ) + service = fixture.service + runtime = fixture.runtime + control = fixture.control + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.deadline_at is not None + + fixture.clock_value[0] = operation.deadline_at - timedelta(milliseconds=1) + before_deadline = service.state() + assert before_deadline["acquisition"]["state"] == "awaiting_external_stop" + assert service._acquisition_session_lease is not None # noqa: SLF001 + assert before_deadline["source_mode"] == "live" + assert before_deadline["last_operation"]["status"] == "running" + assert runtime.stop_calls == 0 + assert control.stop_calls == 1 + + unintended_edges: list[str] = [] + + async def forbidden_device_edge(*_args: object, **_kwargs: object) -> object: + unintended_edges.append("device-or-network-io") + raise AssertionError("STOP timeout recovery must be local-only") + + monkeypatch.setattr(facade_module, "scan", forbidden_device_edge) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_edge) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_edge) + fixture.clock_value[0] = operation.deadline_at + timed_out = service.state() + timeout_operation = next( + item + for item in timed_out["operations"] + if item["operation_id"] == fixture.stop_operation_id + ) + + assert timed_out["application_control_session"]["state"] == "idle" + assert timed_out["acquisition"]["state"] == "failed" + assert timed_out["acquisition"]["cleanup_pending"] is False + assert timed_out["acquisition"]["message_code"] == ( + "acquisition.stop.accepted_physical_outcome_unknown" + ) + assert timed_out["acquisition"]["result"] == { + "receiver_stopped": True, + "device_state": "unknown", + "device_stop": "stop-accepted-physical-outcome-unknown", + "automatic_replay_allowed": False, + } + assert timeout_operation["status"] == "timed_out" + assert timeout_operation["stage_code"] == ("stop-accepted-physical-outcome-unknown") + assert timeout_operation["error"] == { + "category": "device", + "code": "stop-accepted-physical-outcome-unknown", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + } + assert timed_out["physical_command"]["status"] == "unresolved" + assert timed_out["physical_command"]["requires_reconciliation"] is True + assert timed_out["physical_command"]["record"]["resolution"] is None + assert timed_out["source_mode"] == "idle" + assert timed_out["live_perception_shadow"]["active"] is False + assert timed_out["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + for action in ("provision-fresh-device", "start-acquisition", "stop-acquisition"): + assert timed_out["connection_policy"]["actions"][action]["allowed"] is False + assert unintended_edges == [] + assert runtime.stop_calls == 1 + assert control.stop_calls == 1 + + repeated = service.state() + assert repeated["acquisition"]["state"] == "failed" + assert repeated["physical_command"]["status"] == "unresolved" + assert repeated["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert unintended_edges == [] + assert runtime.stop_calls == 1 + assert control.stop_calls == 1 + + restarted, restarted_runtime = service_with_fake_runtime(tmp_path) + restarted_state = restarted.state() + assert restarted_state["physical_command"]["status"] == "unresolved" + assert restarted_state["physical_command"]["record"]["operation_id"] == ( + fixture.stop_operation_id + ) + assert restarted_state["acquisition"] is None + assert restarted_state["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + for action in ("provision-fresh-device", "start-acquisition", "stop-acquisition"): + assert restarted_state["connection_policy"]["actions"][action]["allowed"] is False + assert restarted_runtime.stop_calls == 0 + + +def test_stop_timeout_local_cleanup_failure_retries_only_local_resources( + tmp_path: Path, +) -> None: + fixture = _stop_response_without_terminal_status_fixture(tmp_path) + operation = fixture.service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.deadline_at is not None + fixture.clock_value[0] = operation.deadline_at + fixture.runtime.stop_error = RuntimeError("synthetic timeout cleanup failure") + + failed_cleanup = fixture.service.state() + failed_operation = next( + item + for item in failed_cleanup["operations"] + if item["operation_id"] == fixture.stop_operation_id + ) + assert failed_cleanup["acquisition"]["state"] == "failed" + assert failed_cleanup["acquisition"]["cleanup_pending"] is True + assert failed_cleanup["source_mode"] == "live" + assert failed_operation["status"] == "failed" + assert failed_operation["stage_code"] == ("local-cleanup-failed-after-stop-outcome-unknown") + assert failed_operation["error"]["side_effect_status"] == "unknown" + assert failed_operation["error"]["safe_to_retry"] is False + assert failed_operation["error"]["automatic_replay_allowed"] is False + assert fixture.control.stop_calls == 1 + assert fixture.runtime.stop_calls == 1 + + fixture.runtime.stop_error = None + recovered = fixture.service.state() + assert recovered["acquisition"]["state"] == "failed" + assert recovered["acquisition"]["cleanup_pending"] is False + assert recovered["acquisition"]["result"]["receiver_stopped"] is True + assert "local_cleanup_retry_pending" not in recovered["acquisition"]["result"] + assert recovered["source_mode"] == "idle" + assert recovered["physical_command"]["status"] == "unresolved" + assert recovered["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert fixture.control.stop_calls == 1 + assert fixture.runtime.stop_calls == 2 + + repeated = fixture.service.state() + assert repeated["acquisition"]["cleanup_pending"] is False + assert fixture.control.stop_calls == 1 + assert fixture.runtime.stop_calls == 2 + + +def test_stop_response_preserves_precise_unknown_outcome_through_predeadline_loss( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _stop_response_without_terminal_status_fixture(tmp_path) + operation = fixture.service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.deadline_at is not None + original_snapshot = fixture.control.snapshot + + def failed_control_snapshot() -> dict[str, object]: + snapshot = original_snapshot() + if fixture.control.state == "failed": + snapshot["failure"] = { + "reason_code": "mqtt_network_loop_failed", + "safe_to_retry": False, + "modeling_command_attempted": True, + } + return snapshot + + monkeypatch.setattr(fixture.control, "snapshot", failed_control_snapshot) + fixture.control.state = "failed" + fixture.runtime.phase = "error" + fixture.clock_value[0] = operation.deadline_at - timedelta(milliseconds=1) + + before_deadline = fixture.service.state() + stop_operation = next( + item + for item in before_deadline["operations"] + if item["operation_id"] == fixture.stop_operation_id + ) + assert before_deadline["acquisition"]["state"] == "awaiting_external_stop" + assert before_deadline["acquisition"]["message_code"] == ("acquisition.stop.device_stopping") + assert stop_operation["status"] == "running" + assert before_deadline["physical_command"]["status"] == "unresolved" + assert fixture.runtime.stop_calls == 0 + assert fixture.control.stop_calls == 1 + + fixture.clock_value[0] = operation.deadline_at + timed_out = fixture.service.state() + stop_operation = next( + item + for item in timed_out["operations"] + if item["operation_id"] == fixture.stop_operation_id + ) + assert timed_out["acquisition"]["message_code"] == ( + "acquisition.stop.accepted_physical_outcome_unknown" + ) + assert stop_operation["status"] == "timed_out" + assert stop_operation["stage_code"] == "stop-accepted-physical-outcome-unknown" + assert timed_out["source_mode"] == "idle" + assert timed_out["physical_command"]["status"] == "unresolved" + assert fixture.runtime.stop_calls == 1 + assert fixture.control.stop_calls == 1 + + +def test_stop_ready_resolution_wins_when_it_arrives_at_timeout_deadline( + tmp_path: Path, +) -> None: + fixture = _stop_response_without_terminal_status_fixture(tmp_path) + operation = fixture.service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.deadline_at is not None + fixture.clock_value[0] = operation.deadline_at + fixture.ledger.record_status_observation( + fixture.stop_operation_id, + fixture.status("ready", observed_at_utc="2026-08-10T08:10:10.000Z"), + ) + fixture.ledger.resolve( + fixture.stop_operation_id, + resolution="stop-standby-observed", + ) + fixture.control.state = "idle" + fixture.control.verified_control = None + + completed = fixture.service.state() + completed_operation = next( + item + for item in completed["operations"] + if item["operation_id"] == fixture.stop_operation_id + ) + assert completed["acquisition"]["state"] == "completed" + assert completed["acquisition"]["result"]["device_state"] == "ready" + assert completed_operation["status"] == "succeeded" + assert completed["physical_command"]["record"]["resolution"] == ("stop-standby-observed") + assert fixture.control.stop_calls == 1 + assert fixture.runtime.stop_calls == 1 + + +def test_facade_wires_one_physical_command_coordinator_into_control_session( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + assert ( + service._application_control_session._physical_command_coordinator # noqa: SLF001 + is service._physical_command_coordinator # noqa: SLF001 + ) + physical_snapshot = service.state()["physical_command"] + assert physical_snapshot["automatic_replay_allowed"] is False + assert physical_snapshot["status"] == "empty" + + +@pytest.mark.parametrize( + ("stage", "action", "expected_resolution"), + [ + ( + "prepared", + "start", + [("op-00000000-0000-4000-8000-000000000111", "not-dispatched")], + ), + ("prepared", "stop", []), + ("dispatching", "start", []), + ("observing", "start", []), + ], +) +def test_facade_startup_auto_resolves_only_pre_dispatch_physical_attempt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + stage: str, + action: str, + expected_resolution: list[tuple[str, str]], +) -> None: + operation_id = "op-00000000-0000-4000-8000-000000000111" + + class StartupLedger: + def __init__(self) -> None: + self.resolutions: list[tuple[str, str]] = [] + + def snapshot(self) -> SimpleNamespace: + return SimpleNamespace( + status="unresolved", + record=SimpleNamespace( + operation_id=operation_id, + stage=stage, + action=action, + ), + ) + + def resolve(self, candidate: str, *, resolution: str) -> None: + self.resolutions.append((candidate, resolution)) + + ledger = StartupLedger() + monkeypatch.setattr(facade_module, "PhysicalCommandLedger", lambda _root: ledger) + + service = XgridsK1CompatibilityService( + tmp_path, + host_wifi_association_probe=FakeHostWifiAssociationProbe(), + ) + + assert service._physical_command_ledger is ledger # noqa: SLF001 + assert ledger.resolutions == expected_resolution + + +def test_facade_physical_context_chains_stop_to_durable_start_after_transient_id_clears( + tmp_path: Path, +) -> None: + start_operation_id = "op-00000000-0000-4000-8000-000000000101" + stop_operation_id = "op-00000000-0000-4000-8000-000000000102" + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + + class FacadeAdmissionLedger: + def __init__(self) -> None: + self.record: object | None = None + self.admission_calls = 0 + + def require_no_unresolved_attempt(self) -> None: + self.admission_calls += 1 + + def snapshot(self) -> SimpleNamespace: + return SimpleNamespace(record=self.record) + + ledger = FacadeAdmissionLedger() + service._physical_command_ledger = ledger # type: ignore[assignment] # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + operation_id=start_operation_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + start_context = control.start_contexts[-1] + assert start_context.operation_id == start_operation_id + assert start_context.parent_operation_id is None + assert start_context.acquisition_id == acquisition_id + assert start_context.operator_confirmation_id is None + assert start_context.operator_confirmed_at_utc is None + + ledger.record = SimpleNamespace(operation_id=start_context.operation_id) + service._physical_command_coordinator = SimpleNamespace( # type: ignore[assignment] # noqa: SLF001 + snapshot=lambda: _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=True, + ) + ) + runtime.mark_ready() + runtime.pcl_frames = 1 + service.state() + assert service._acquisition_start_operation_id is None # noqa: SLF001 + + service.stop_acquisition( + _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + stop_context = control.stop_contexts[-1] + assert stop_context.operation_id == stop_operation_id + assert stop_context.parent_operation_id == start_operation_id + assert stop_context.acquisition_id == acquisition_id + assert stop_context.operator_confirmation_id is None + assert stop_context.operator_confirmed_at_utc is None + assert ledger.admission_calls == 2 + + +def test_facade_blocks_unresolved_physical_start_before_local_receiver_side_effects( + tmp_path: Path, +) -> None: + operation_id = "op-00000000-0000-4000-8000-000000000103" + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + + class BlockedPhysicalLedger: + def require_no_unresolved_attempt(self) -> None: + raise PhysicalCommandBlocked("ambiguous previous physical edge") + + service._physical_command_ledger = BlockedPhysicalLedger() # type: ignore[assignment] # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + + with pytest.raises(PhysicalCommandBlocked): + service.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + operation_id=operation_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + + assert runtime.start_calls == [] + assert control.start_contexts == [] + assert service._operations.get(operation_id).status == "failed" # noqa: SLF001 + + +def test_plugin_commanded_prepare_rejects_stale_binding_before_creating_acquisition( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + class StaleInteractiveControlSession(FakeInteractiveControlSession): + def __init__(self) -> None: + super().__init__() + self.project_prompt_calls = 0 + self.close_calls = 0 + + def validate_connection_binding(self) -> None: + raise ApplicationConnectionBindingLost("test route changed") + + def open_project_prompt( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + self.project_prompt_calls += 1 + return super().open_project_prompt( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + def close_prestart(self) -> dict[str, object]: + self.close_calls += 1 + return super().close_prestart() + + control = StaleInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + + with pytest.raises(ApplicationConnectionBindingLost): + service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + + assert service._acquisition is None # noqa: SLF001 + assert control.project_prompt_calls == 0 + assert control.close_calls == 1 + assert control.state == "closed" + + +def test_workspace_entry_rejects_stale_binding_before_releasing_checkpoint( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + class StaleWorkspaceControlSession(FakeInteractiveControlSession): + def __init__(self) -> None: + super().__init__() + self.state = "connection-ready" + self.enter_calls = 0 + self.close_calls = 0 + + def validate_connection_binding(self) -> None: + raise ApplicationConnectionBindingLost("test route changed") + + def enter_workspace( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + # Production ``InteractiveApplicationControlSession`` owns this + # command-bound guard internally. The facade must not duplicate + # it ahead of the checkpoint. + self.validate_connection_binding() + del expected_session_generation, expected_state_revision + self.enter_calls += 1 + raise AssertionError("stale checkpoint must not be released") + + def close_prestart(self) -> dict[str, object]: + self.close_calls += 1 + return super().close_prestart() + + control = StaleWorkspaceControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection(service) + + with pytest.raises(ApplicationConnectionBindingLost): + service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=1, + expected_state_revision=1, + ) + ) + + assert control.enter_calls == 0 + assert control.close_calls == 1 + assert control.state == "closed" + + +class _BindingValidatingControlSession(FakeInteractiveControlSession): + """Test double with the production session's command-bound guards.""" + + def __init__( + self, + service: XgridsK1CompatibilityService, + binding: ApplicationConnectionBinding, + ) -> None: + super().__init__(initial_state="idle") + self._service = service + self._binding = binding + self.validation_calls = 0 + + def validate_connection_binding(self) -> None: + self.validation_calls += 1 + self._service._validate_application_connection_binding(self._binding) # noqa: SLF001 + + def enter_workspace( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + self.validate_connection_binding() + return super().enter_workspace( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + def open_project_prompt( + self, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + self.validate_connection_binding() + return super().open_project_prompt( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + def request_start( + self, + *, + project_name: str, + confirmation: object, + command_context: object, + preparation_checkpoint_observer: object | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + self.validate_connection_binding() + return super().request_start( + project_name=project_name, + confirmation=confirmation, + command_context=command_context, + preparation_checkpoint_observer=preparation_checkpoint_observer, + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + +def _install_binding_validating_ready_control( + service: XgridsK1CompatibilityService, + *, + host_path: HostPathProbeResult | None = None, +) -> tuple[ + _BindingValidatingControlSession, + ApplicationConnectionBinding, + dict[str, Any], +]: + binding = _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="ttl-bound-k1", + logical_device_id="ttl-bound-device", + host_path=host_path, + ) + control = _BindingValidatingControlSession(service, binding) + control.open(connection_binding=binding) + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id="ttl-bound-device", + control_session_id="test-control-ttl-bound-k1", + control_proof_revision=2, + ) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + service._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + service.runtime.snapshot(), + ) + service._acquire_application_control_process_lease() # noqa: SLF001 + state = service.state() + assert state["application_control_session"]["state"] == "connection-ready" + return control, binding, state + + +def test_public_commanded_workflow_refreshes_stable_route_before_ttl_reduction( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + control, binding, ready = _install_binding_validating_ready_control(service) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: stable_path) + target = EndpointTarget(binding.target_ipv4, binding.target_port) + + def advance_past_fifteen_seconds_with_fresh_endpoint() -> None: + monotonic_now[0] += 10.0 + suspend_aware_now[0] += 10.0 + assert supervisor.observe_endpoint( + target=target, + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=True, + ) + monotonic_now[0] += 5.01 + suspend_aware_now[0] += 5.01 + + advance_past_fifteen_seconds_with_fresh_endpoint() + polled = service.state() + assert polled["application_control_session"]["state"] == "connection-ready" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + workspace = service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=ready["application_control_session"]["session_generation"], + expected_state_revision=ready["application_control_session"]["state_revision"], + ) + ) + assert workspace["application_control_session"]["state"] == "workspace-ready" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert control.validation_calls == 1 + + # Retain a focused regression for the original 15-second reducer ordering + # even though the production silence lease now tolerates two monitor + # passes. Command-bound sampling must still precede stale reduction for + # any reviewed TTL configuration. + supervisor._observation_ttl_seconds = 15.0 # noqa: SLF001 + advance_past_fifteen_seconds_with_fresh_endpoint() + prepared = service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=workspace["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=workspace["application_control_session"][ + "state_revision" + ], + ) + ) + assert prepared["application_control_session"]["state"] == "project-ready" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert control.validation_calls == 3 + + advance_past_fifteen_seconds_with_fresh_endpoint() + started = service.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=prepared["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=prepared["application_control_session"][ + "state_revision" + ], + ) + ) + + assert started["acquisition"]["state"] == "starting" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert supervisor.snapshot().authority.control_allowed is True + assert control.start_projects == [PROJECT_NAME] + assert control.validation_calls == 5 + assert len(runtime.start_calls) == 1 + + +def test_existing_control_open_refreshes_before_ttl_reuse_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _runtime = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + supervisor._observation_ttl_seconds = 15.0 # noqa: SLF001 + control, binding, _ready = _install_binding_validating_ready_control(service) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: stable_path) + + monotonic_now[0] += 10.0 + suspend_aware_now[0] += 10.0 + assert supervisor.observe_endpoint( + target=EndpointTarget(binding.target_ipv4, binding.target_port), + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=True, + ) + monotonic_now[0] += 5.01 + suspend_aware_now[0] += 5.01 + + def forbid_reopen_or_recovery() -> tuple[str, dict[str, Any]]: + raise AssertionError("stable existing control must not reopen or enter BLE recovery") + + monkeypatch.setattr(service, "_reuse_or_recover_control_target", forbid_reopen_or_recovery) + reused = service.open_application_control_session( + OpenApplicationControlSessionRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + timezone_name="Europe/Moscow", + ) + ) + + assert reused["application_control_session"]["state"] == "connection-ready" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert supervisor.snapshot().authority.control_allowed is True + assert control.validation_calls == 1 + open_operation = next( + operation + for operation in reused["operations"] + if operation["action"] == "application-control.session.open" + ) + assert open_operation["result"]["connection_lease_reused"] is True + assert open_operation["result"]["device_write_performed"] is False + + +def test_existing_control_bootstrap_refreshes_before_ttl_reuse_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _runtime = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + supervisor._observation_ttl_seconds = 15.0 # noqa: SLF001 + control, binding, _ready = _install_binding_validating_ready_control(service) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned, # noqa: SLF001 + service, + ) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: stable_path) + + monotonic_now[0] += 10.0 + suspend_aware_now[0] += 10.0 + assert supervisor.observe_endpoint( + target=EndpointTarget(binding.target_ipv4, binding.target_port), + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=True, + ) + monotonic_now[0] += 5.01 + suspend_aware_now[0] += 5.01 + + async def forbid_retirement() -> None: + raise AssertionError("stable existing control must not be retired or reopened") + + monkeypatch.setattr( + service, + "_retire_prestart_control_for_mode_transition", + forbid_retirement, + ) + asyncio.run( + service._bootstrap_prestart_control_ready_owned( # noqa: SLF001 + parent_operation_id=None, + connection_mode="bridge", + ) + ) + + assert control.state == "connection-ready" + assert control.validation_calls == 1 + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert supervisor.snapshot().authority.control_allowed is True + bootstrap = [ + operation + for operation in service._operations.snapshot() # noqa: SLF001 + if operation["action"] == facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP + ] + assert len(bootstrap) == 1 + assert bootstrap[0]["status"] == "succeeded" + + +def test_workspace_route_change_still_fails_closed_before_checkpoint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + control, binding, ready = _install_binding_validating_ready_control(service) + changed_path = HostPathProbeResult( + available=True, + fingerprint="test-route:changed-before-workspace", + interface="test0", + source_ipv4="192.168.1.2", + route_class="direct", + kernel_route_fingerprint="test-route:changed-before-workspace", + ) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: changed_path) + + with pytest.raises(ApplicationConnectionBindingLost): + service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=ready["application_control_session"][ + "session_generation" + ], + expected_state_revision=ready["application_control_session"]["state_revision"], + ) + ) + + assert control.state == "closed" + assert supervisor.snapshot().host_path.epoch != binding.host_path_epoch + assert supervisor.snapshot().authority.control_allowed is False + assert runtime.start_calls == [] + assert control.start_projects == [] + + +def test_start_stale_endpoint_fails_before_receiver_or_device_checkpoint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_binding_validating_ready_control(service) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: stable_path) + workspace = service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=ready["application_control_session"]["session_generation"], + expected_state_revision=ready["application_control_session"]["state_revision"], + ) + ) + prepared = service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=workspace["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=workspace["application_control_session"][ + "state_revision" + ], + ) + ) + target = EndpointTarget(binding.target_ipv4, binding.target_port) + supervisor = service._connection_supervisor # noqa: SLF001 + assert supervisor.observe_endpoint( + target=target, + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=False, + reason_code="tcp-endpoint-unreachable", + ) + physical_before = service._physical_command_ledger.snapshot() # noqa: SLF001 + + with pytest.raises(ApplicationConnectionBindingLost): + service.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=prepared["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=prepared["application_control_session"][ + "state_revision" + ], + ) + ) + + assert prepared["acquisition"]["state"] == "prepared" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "prepared" # noqa: SLF001 + assert service._physical_command_ledger.snapshot() == physical_before # noqa: SLF001 + assert not any( + operation["action"] == facade_module.ACTION_ACQUISITION_START + for operation in service._operations.snapshot() # noqa: SLF001 + ) + assert runtime.start_calls == [] + assert control.start_projects == [] + + +def test_start_stale_control_proof_fails_before_receiver_or_device_checkpoint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_binding_validating_ready_control(service) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: stable_path) + workspace = service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=ready["application_control_session"]["session_generation"], + expected_state_revision=ready["application_control_session"]["state_revision"], + ) + ) + prepared = service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=workspace["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=workspace["application_control_session"][ + "state_revision" + ], + ) + ) + + def reject_stale_control_proof(_control: object) -> None: + raise ApplicationConnectionBindingLost("test MQTT control proof expired") + + control.validate_connection_binding = MethodType( # type: ignore[method-assign] + reject_stale_control_proof, + control, + ) + physical_before = service._physical_command_ledger.snapshot() # noqa: SLF001 + + with pytest.raises(ApplicationConnectionBindingLost): + service.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=prepared["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=prepared["application_control_session"][ + "state_revision" + ], + ) + ) + + assert prepared["acquisition"]["state"] == "prepared" + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "prepared" # noqa: SLF001 + assert service._physical_command_ledger.snapshot() == physical_before # noqa: SLF001 + assert not any( + operation["action"] == facade_module.ACTION_ACQUISITION_START + for operation in service._operations.snapshot() # noqa: SLF001 + ) + assert runtime.start_calls == [] + assert control.start_projects == [] def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start( @@ -1263,7 +13239,7 @@ def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start( service, runtime = service_with_fake_runtime(tmp_path) service.live_perception_ingress.begin_session("stale-completed-session") prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1271,7 +13247,7 @@ def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start( ) state = service.start_acquisition( - StartAcquisitionRequest(acquisition_id=prepared["acquisition"]["acquisition_id"]) + _start_request(acquisition_id=prepared["acquisition"]["acquisition_id"]) ) assert runtime.start_calls @@ -1280,78 +13256,3059 @@ def test_next_scan_retires_stale_terminal_live_perception_ingress_before_start( assert state["live_perception_shadow"]["session_id"] != "stale-completed-session" -def test_confirmed_scanning_activates_and_records_right_camera( +def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) control = FakeInteractiveControlSession() service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 - service._k1_ip = "192.168.1.20" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name="TEST001", host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition( - StartAcquisitionRequest( + pre_pcl_state = service.start_acquisition( + _start_request( acquisition_id=acquisition_id, physical_acceptance=PHYSICAL_ACCEPTANCE, ) ) + pre_pcl_camera_streams = [ + stream + for stream in pre_pcl_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert pre_pcl_state["camera_preview"]["active_source_id"] is None + assert pre_pcl_state["camera_preview"]["delivery"] is None + assert pre_pcl_state["camera_preview"]["activation_admission"]["state"] == ( + "waiting-for-first-authoritative-pcl" + ) + assert all(stream["activation"]["selected"] is False for stream in pre_pcl_camera_streams) + assert all(stream["activation"]["controllable"] is False for stream in pre_pcl_camera_streams) out_dir = service._acquisition_out_dir # noqa: SLF001 assert out_dir is not None out_dir.mkdir(parents=True) events: list[tuple[str, object]] = [] + activation_entered = threading.Event() + release_activation = threading.Event() camera_state: dict[str, object] = { "phase": "idle", "active_source_id": None, "recording": {"active": False}, } - def select_camera(source_id: str, target: str) -> dict[str, object]: + def activate_recording( + source_id: str, + target: str, + session_dir: Path, + *, + commit_fence: Callable[[Callable[[], bool]], bool], + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: events.append(("select", (source_id, target))) - camera_state["phase"] = "selected" camera_state["active_source_id"] = source_id - return dict(camera_state) - - def start_recording(session_dir: Path) -> dict[str, object]: events.append(("record", session_dir)) - camera_state["recording"] = {"active": True} - camera_state["phase"] = "connecting" - return dict(camera_state) + camera_state["recording"] = { + "active": True, + "session": session_dir.name, + "active_epoch": 1, + "producer_alive": True, + "last_segment_age_ms": 0, + "last_media_segment_age_ms": 0, + "media_ready": True, + "current_epoch": { + "generation": 1, + "init_committed": True, + "init_committed_age_ms": 0, + "first_media_committed": True, + "first_media_committed_age_ms": 0, + "committed_media_segment_count": 1, + "last_media_segment_age_ms": 0, + }, + } + camera_state["phase"] = "streaming" + committed = dict(camera_state) + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(committed) is None + ) is True + return committed - monkeypatch.setattr(service.camera_preview, "snapshot", lambda: dict(camera_state)) - monkeypatch.setattr(service.camera_preview, "select", select_camera) - monkeypatch.setattr(service.camera_preview, "start_recording", start_recording) + actual_camera_snapshot = service.camera_preview.snapshot() - service._activate_default_acquisition_camera() # noqa: SLF001 + def snapshot_camera() -> dict[str, object]: + return { + **actual_camera_snapshot, + **camera_state, + "recording": { + **actual_camera_snapshot["recording"], + **camera_state["recording"], # type: ignore[arg-type] + }, + } + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot_camera) + monkeypatch.setattr( + service.camera_preview, + "activate_recording_producer", + activate_recording, + ) + real_activate = service._activate_default_acquisition_camera # noqa: SLF001 + + def gated_activate(**kwargs: object) -> bool: + activation_entered.set() + assert release_activation.wait(timeout=2.0) + return real_activate(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(service, "_activate_default_acquisition_camera", gated_activate) + + runtime.mark_ready() + assert control.state == "scanning" + assert camera_state["active_source_id"] is None + assert events == [] + + with service._lock: # noqa: SLF001 + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert start_operation_id is not None + binding = _seed_supervised_connection(service) + physical = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=True, + ) + monkeypatch.setattr(service._physical_command_coordinator, "snapshot", lambda: physical) # noqa: SLF001 + runtime.pcl_frames = 1 + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + # The first PCL admits only the one automatic right-camera activation. + # Public manual controls stay closed until that worker proves an exact + # recording epoch, preventing a left/right selection race. + assert activation_entered.wait(timeout=2.0) + activating_state = service.state() + activating_camera_streams = [ + stream + for stream in activating_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert activating_state["camera_preview"]["activation_admission"]["state"] == "activating" + assert all( + stream["activation"]["controllable"] is False + for stream in activating_camera_streams + ) + with pytest.raises(RuntimeError, match="автоматического запуска правой камеры"): + service.select_camera_preview( + CameraPreviewSelectRequest( + device_session_id=service._device_session_id, # type: ignore[arg-type] # noqa: SLF001 + source_id="sensor.camera.left", + ) + ) + assert events == [] + release_activation.set() + deadline = time.monotonic() + 2.0 + while len(events) < 2 and time.monotonic() < deadline: + time.sleep(0.01) assert events == [ ("select", ("sensor.camera.right", "192.168.1.20")), ("record", out_dir), ] assert camera_state["active_source_id"] == "sensor.camera.right" - assert camera_state["recording"] == {"active": True} + assert camera_state["recording"] == { + "active": True, + "session": out_dir.name, + "active_epoch": 1, + "producer_alive": True, + "last_segment_age_ms": 0, + "last_media_segment_age_ms": 0, + "media_ready": True, + "current_epoch": { + "generation": 1, + "init_committed": True, + "init_committed_age_ms": 0, + "first_media_committed": True, + "first_media_committed_age_ms": 0, + "committed_media_segment_count": 1, + "last_media_segment_age_ms": 0, + }, + } assert runtime.source_mode == "live" + assert control.start_projects == ["TEST001"] + assert control.stop_calls == 0 + + admitted_state = service.state() + admitted_camera_streams = [ + stream + for stream in admitted_state["sensor_catalog"]["streams"] + if stream.get("semantic_channel_id") == "camera.preview.live" + ] + assert admitted_state["camera_preview"]["activation_admission"]["state"] == "admitted" + assert any( + stream["source_id"] == "sensor.camera.right" + and stream["activation"]["selected"] is True + and stream["activation"]["controllable"] is True + for stream in admitted_camera_streams + ) + + # Every later authoritative PCL for the same lineage is idempotent. + service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001 + time.sleep(0.05) + assert len(events) == 2 -def test_device_standby_retires_sources_after_terminal_local_stop_failure( +def test_stale_post_publish_pcl_cannot_activate_camera( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) control = FakeInteractiveControlSession() service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 - service._k1_ip = "192.168.1.20" # noqa: SLF001 - service._compatibility_attestation = {"profile": "exact"} # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="STALEPCL001", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + service.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + runtime.mark_ready() + runtime.pcl_frames = 1 + selected: list[object] = [] + monkeypatch.setattr( + service.camera_preview, + "select", + lambda *_args, **_kwargs: selected.append(object()), + ) + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + service._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation - 1, + ) + with service._lock: # noqa: SLF001 + service._acquisition.transition("interrupted", message_code="test.stale") # type: ignore[union-attr] # noqa: SLF001 + service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001 + time.sleep(0.05) + + assert selected == [] + assert service.camera_preview.snapshot()["active_source_id"] is None + + +def test_failed_post_pcl_camera_activation_retries_on_later_authoritative_frame( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 + prepared = service.prepare_acquisition( + _prepare_request( + project_name="RETRYPCL001", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + runtime.mark_ready() + runtime.pcl_frames = 1 + with service._lock: # noqa: SLF001 + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + assert start_operation_id is not None + assert out_dir is not None + physical = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=_seed_supervised_connection(service), + resolved=True, + ) + monkeypatch.setattr(service._physical_command_coordinator, "snapshot", lambda: physical) # noqa: SLF001 + out_dir.mkdir(parents=True) + actual_camera = service.camera_preview.snapshot() + camera_state: dict[str, object] = { + **actual_camera, + "phase": "idle", + "generation": None, + "active_source_id": None, + "recording": { + **actual_camera["recording"], + "active": False, + "session": None, + "active_epoch": None, + "producer_alive": False, + }, + "error": None, + } + select_calls: list[tuple[str, str]] = [] + recording_calls: list[Path] = [] + retry_calls: list[tuple[str, str, int, str]] = [] + + def snapshot_camera() -> dict[str, object]: + return { + **camera_state, + "recording": dict(camera_state["recording"]), # type: ignore[arg-type] + } + + def fail_first_activation( + source_id: str, + target: str, + session_dir: Path, + *, + commit_fence: Callable[[Callable[[], bool]], bool], + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + del commit_fence, committed_before_start + select_calls.append((source_id, target)) + recording_calls.append(session_dir) + camera_state.update( + { + "phase": "error", + "error": { + "code": "ffmpeg-start-failed", + "message": "synthetic first local spawn failure", + }, + "generation": 1, + "active_source_id": source_id, + "recording": { + **camera_state["recording"], # type: ignore[arg-type] + "active": True, + "session": session_dir.name, + "active_epoch": None, + "producer_alive": False, + }, + } + ) + raise RuntimeError("synthetic first local spawn failure") + + def retry_recording( + source_id: str, + target: str, + *, + expected_generation: int, + expected_recording_session: str, + pre_retry_fence: Callable[[Callable[[], bool]], bool], + commit_fence: Callable[[Callable[[], bool]], bool] | None = None, + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + retry_calls.append( + (source_id, target, expected_generation, expected_recording_session) + ) + assert camera_state["phase"] == "error" + assert camera_state["generation"] == expected_generation + assert pre_retry_fence(lambda: True) is True + recording = camera_state["recording"] + assert isinstance(recording, dict) + assert recording["session"] == expected_recording_session + camera_state.update( + { + "phase": "connecting", + "generation": expected_generation + 1, + "error": None, + "recording": { + **recording, + "active_epoch": expected_generation + 1, + "producer_alive": True, + "last_segment_age_ms": None, + "last_media_segment_age_ms": None, + "media_ready": False, + "current_epoch": { + "generation": expected_generation + 1, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + }, + }, + } + ) + committed = snapshot_camera() + assert commit_fence is not None + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(committed) is None + ) is True + return committed + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot_camera) + monkeypatch.setattr( + service.camera_preview, + "activate_recording_producer", + fail_first_activation, + ) + monkeypatch.setattr( + service.camera_preview, + "retry_recording_producer", + retry_recording, + ) + frame = DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=1, + captured_at_epoch_ns=1, + received_monotonic_ns=1, + processing_started_monotonic_ns=1, + encoded_size_bytes=16, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001 + deadline = time.monotonic() + 2.0 + while len(recording_calls) < 1 and time.monotonic() < deadline: + time.sleep(0.01) + assert recording_calls == [out_dir] + assert select_calls == [("sensor.camera.right", binding.target_ipv4)] + + # Normal UI polling between attempts must preserve PCL/Rerun and the + # scanner-owned session while this exact startup error is retryable. + provisional = service.state() + assert provisional["acquisition"]["state"] in { + "starting", + "awaiting_external_start", + "acquiring", + } + assert provisional["camera_preview"]["activation_admission"]["state"] == "activating" + assert runtime.source_mode == "live" + assert runtime.stop_calls == 0 + assert control.stop_calls == 0 + + # Frames inside the bounded retry window cannot create a worker storm. + service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001 + time.sleep(0.05) + assert retry_calls == [] + + with service._lock: # noqa: SLF001 + service._camera_activation_retry_not_before_monotonic = 0.0 # noqa: SLF001 + service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001 + deadline = time.monotonic() + 2.0 + while len(retry_calls) < 1 and time.monotonic() < deadline: + time.sleep(0.01) + + assert retry_calls == [ + ( + "sensor.camera.right", + binding.target_ipv4, + 1, + out_dir.name, + ) + ] + assert camera_state["generation"] == 2 + recording = camera_state["recording"] + assert isinstance(recording, dict) + assert recording["active_epoch"] == 2 + assert recording["producer_alive"] is True + with service._lock: # noqa: SLF001 + assert service._camera_activation_lineage == ( # noqa: SLF001 + acquisition_id, + out_dir.name, + runtime.producer_generation, + ) + assert control.start_projects == ["RETRYPCL001"] + assert control.stop_calls == 0 + admitted = service.state() + assert admitted["acquisition"]["state"] == "acquiring" + assert admitted["camera_preview"]["activation_admission"]["state"] == "activating" + assert admitted["camera_preview"]["recording"]["media_ready"] is False + assert admitted["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is True + + # The same shaped error without the exact retry claim stays terminal; the + # classifier never masks storage/archive failures or an unrelated lineage. + with service._lock: # noqa: SLF001 + service._camera_activation_lineage = None # noqa: SLF001 + service._camera_activation_retry_lineage = None # noqa: SLF001 + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + camera_state.update( + { + "phase": "error", + "error": { + "code": "camera-storage-failed", + "message": "synthetic durable archive failure", + }, + "recording": { + **recording, + "active_epoch": None, + "producer_alive": False, + }, + } + ) + terminal = service.state() + assert terminal["acquisition"]["state"] == "failed" + assert terminal["acquisition"]["result"]["camera_failure_code"] == ( + "camera-storage-failed" + ) + + +def _install_composite_active_recovery_fixture( + service: XgridsK1CompatibilityService, + runtime: FakeVisualizationRuntime, + monkeypatch: pytest.MonkeyPatch, + *, + runtime_phase: str = "reconnecting", + camera_phase: str = "error", +) -> tuple[FakeInteractiveControlSession, dict[str, object]]: + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="RECOVERY001", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = str(prepared["acquisition"]["acquisition_id"]) + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + assert acquisition is not None + assert start_operation_id is not None + assert out_dir is not None + acquisition.transition("acquiring", message_code="acquisition.acquiring") + out_dir.mkdir(parents=True, exist_ok=True) + runtime.phase = runtime_phase + runtime.source_mode = "live" + runtime.source_ready = runtime_phase == "live" + runtime.recovery_state = ( + "reconnecting" if runtime_phase == "reconnecting" else "inactive" + ) + runtime.recovery_request_pending = runtime_phase == "reconnecting" + runtime.pcl_frames = 4 + + physical: dict[str, object] = { + "status": "resolved", + "reason_code": None, + "requires_reconciliation": False, + "resolved_active_recovery_required": True, + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": ("explicit-read-only-deviceinfo-and-non-retained-devicestatus"), + "runtime_bound": True, + "reconciliation_ready": False, + "observed_session_state": None, + "active_operation_id": None, + "record": { + "revision": 7, + "operation_id": start_operation_id, + "acquisition_id": acquisition_id, + "action": "start", + "stage": "resolved", + "resolution": "start-active-observed", + "publish_call_returned": True, + "qos2_completed": True, + "packet_id": 41, + "application_response": { + "operation_id": start_operation_id, + "action": "start", + "success": True, + }, + "last_status": { + "session_state": "scanning", + "project_bound": True, + "init_ready": True, + "mqtt_retained": False, + }, + "connection": { + "intent_id": binding.intent_id, + "transport_ref": binding.transport_ref, + "connection_mode": binding.connection_mode, + "target_ipv4": binding.target_ipv4, + "target_port": binding.target_port, + }, + "reconciliations": [], + }, + } + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical, + ) + # START operation ownership is released after the normal application + # operation settles. The broad mapping-only fixture has no real durable + # checkpoint, so preserve its immutable original START through the same + # helper seam used by production checkpoint integration. + service._active_acquisition_checkpoint_start_operation_id = MethodType( # type: ignore[method-assign] # noqa: SLF001 + lambda _service, **_kwargs: start_operation_id, + service, + ) + camera = service.camera_preview.snapshot() + camera.update( + { + "phase": camera_phase, + "active_source_id": facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + "recording": { + "active": True, + "session": out_dir.name, + "active_epoch": 1 if camera_phase == "streaming" else None, + "producer_alive": camera_phase == "streaming", + "producer_age_ms": 1_000, + "last_segment_age_ms": 0 if camera_phase == "streaming" else None, + "completed_epochs": 0 if camera_phase == "streaming" else 1, + "last_summary": None, + "source_end_expected": False, + }, + "error": ( + { + "code": "camera-source-ended", + "message": "synthetic Wi-Fi loss", + } + if camera_phase == "error" + else None + ), + } + ) + monkeypatch.setattr(service.camera_preview, "snapshot", lambda: camera) + return control, physical + + +def test_camera_first_short_outage_is_observational_without_browser_side_effects( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="error", + ) + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + state = service.state() + + recovery = state["connection_recovery"] + assert runtime.recovery_requests == [] + assert runtime.recovery_request_pending is False + assert state["source_mode"] == "live" + assert recovery["state"] == "inactive" + assert recovery["automatic_read_only_rebind"] is False + assert state["acquisition"]["state"] == "acquiring" + assert state["selected_device_id"] == "test-ble-transport" + assert state["k1_ip"] == "192.168.1.20" + assert runtime.stop_calls == 0 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_camera_stall_snapshot_does_not_reconnect_live_mqtt_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + camera = service.camera_preview.snapshot() + recording = camera["recording"] + assert isinstance(recording, dict) + recording["last_segment_age_ms"] = ( + facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + ) + + state = service.state() + + assert runtime.recovery_requests == [] + assert runtime.recovery_request_pending is False + assert state["source_mode"] == "live" + assert state["acquisition"]["state"] == "acquiring" + assert runtime.stop_calls == 0 + assert control.stop_calls == 0 + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_backend_camera_watchdog_cas_restarts_only_camera_without_device_commands( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + camera = service.camera_preview.snapshot() + camera["generation"] = 1 + recording = camera["recording"] + assert isinstance(recording, dict) + recording.update( + { + "producer_age_ms": facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1, + "last_segment_age_ms": facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1, + "last_media_segment_age_ms": ( + facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1 + ), + "committed_media_segment_count": 10, + "media_ready": True, + "current_epoch": { + "generation": 1, + "init_committed": True, + "init_committed_age_ms": 20_000, + "first_media_committed": True, + "first_media_committed_age_ms": 20_000, + "committed_media_segment_count": 10, + "last_media_segment_age_ms": ( + facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1 + ), + }, + } + ) + restart_calls: list[tuple[str, str, int, str, int | None, int]] = [] + + def restart( + source_id: str, + target: str, + *, + expected_generation: int, + expected_recording_session: str, + expected_active_epoch: int | None, + expected_recording_media_segment_count: int, + pre_detach_fence: Callable[[Callable[[], bool]], bool], + commit_fence: Callable[[Callable[[], bool]], bool] | None = None, + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + restart_calls.append( + ( + source_id, + target, + expected_generation, + expected_recording_session, + expected_active_epoch, + expected_recording_media_segment_count, + ) + ) + assert pre_detach_fence(lambda: True) is True + camera.update( + { + "phase": "connecting", + "generation": 2, + "error": None, + "recording": { + **recording, + "active_epoch": 2, + "producer_alive": True, + "producer_age_ms": 0, + "last_segment_age_ms": None, + "last_media_segment_age_ms": None, + "media_ready": False, + "current_epoch": { + "generation": 2, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + }, + }, + } + ) + assert commit_fence is not None + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(dict(camera)) is None + ) is True + return dict(camera) + + monkeypatch.setattr(service.camera_preview, "restart_recording_producer", restart) + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + service._observe_camera_producer_stall("camera-stream-stalled", 1) # noqa: SLF001 + service._observe_camera_producer_stall("camera-stream-stalled", 1) # noqa: SLF001 + + assert restart_calls == [ + ( + facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + "192.168.1.20", + 1, + recording["session"], + 1, + 10, + ) + ] + assert runtime.phase == "live" + assert runtime.recovery_requests == [] + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + assert camera["recording"]["media_ready"] is False + projected = service.state() + assert projected["connection_recovery"]["state"] == "inactive" + assert projected["connection_recovery"]["camera_recovery"] == "owned" + assert projected["connection_recovery"]["camera_media_state"] == "pending-init" + assert projected["connection_recovery"]["camera_media_ready"] is False + assert projected["connection_recovery"]["camera_epoch"]["generation"] == 2 + assert projected["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is True + + +def test_stop_priority_overtakes_blocked_camera_watchdog_candidate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + camera = service.camera_preview.snapshot() + camera["generation"] = 1 + recording = camera["recording"] + assert isinstance(recording, dict) + recording.update( + { + "producer_age_ms": facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1, + "last_segment_age_ms": facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1, + "last_media_segment_age_ms": ( + facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1 + ), + "committed_media_segment_count": 10, + "media_ready": True, + "current_epoch": { + "generation": 1, + "init_committed": True, + "init_committed_age_ms": 20_000, + "first_media_committed": True, + "first_media_committed_age_ms": 20_000, + "committed_media_segment_count": 10, + "last_media_segment_age_ms": 20_000, + }, + } + ) + candidate_blocked = threading.Event() + release_candidate = threading.Event() + candidate_committed: list[bool] = [] + + def restart( + _source_id: str, + _target: str, + *, + expected_generation: int, + expected_recording_session: str, + expected_active_epoch: int | None, + expected_recording_media_segment_count: int, + pre_detach_fence: Callable[[Callable[[], bool]], bool], + commit_fence: Callable[[Callable[[], bool]], bool] | None = None, + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + del ( + expected_generation, + expected_recording_session, + expected_active_epoch, + expected_recording_media_segment_count, + ) + assert commit_fence is not None + del committed_before_start + candidate_blocked.set() + assert release_candidate.wait(3.0) + committed = commit_fence(lambda: candidate_committed.append(True) or True) + if not committed: + raise ValueError("camera recovery lineage устарела") + return dict(camera) + + monkeypatch.setattr(service.camera_preview, "restart_recording_producer", restart) + stop_owned_entered = threading.Event() + + def stop_owned( + _request: StopAcquisitionRequest, + _preadmitted_stop: object | None = None, + ) -> dict[str, object]: + stop_owned_entered.set() + control.request_stop( + confirmation=object(), + command_context=object(), + dispatch_admission_deadline_reached=lambda: False, + expected_session_generation=control.session_generation, + expected_state_revision=control.state_revision, + ) + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is not None + acquisition.transition( + "awaiting_external_stop", + message_code="acquisition.stop.device_stopping", + ) + service._acquisition_stop_operation_id = "priority-stop" # noqa: SLF001 + return {"stop": "prepared"} + + monkeypatch.setattr(service, "_stop_acquisition_owned", stop_owned) + observer = threading.Thread( + target=service._observe_camera_producer_stall, # noqa: SLF001 + args=("camera-stream-stalled", 1), + daemon=True, + ) + observer.start() + assert candidate_blocked.wait(3.0) + + request = _stop_request( + acquisition_id=None, + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + stop_result: list[dict[str, object]] = [] + stop_worker = threading.Thread( + target=lambda: stop_result.append(service.stop_acquisition(request)), + daemon=True, + ) + stop_worker.start() + assert stop_owned_entered.wait(3.0) + stop_worker.join(3.0) + + assert stop_worker.is_alive() is False + assert stop_result == [{"stop": "prepared"}] + assert control.stop_calls == 1 + assert candidate_committed == [] + + release_candidate.set() + observer.join(3.0) + assert observer.is_alive() is False + assert candidate_committed == [] + + +def test_stop_priority_overtakes_blocked_physical_snapshot_before_camera_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert acquisition is not None + assert out_dir is not None + assert start_operation_id is not None + + snapshot_blocked = threading.Event() + release_snapshot = threading.Event() + + def blocking_snapshot() -> dict[str, object]: + if threading.current_thread().name == "camera-commit-candidate": + snapshot_blocked.set() + assert release_snapshot.wait(3.0) + return physical + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + blocking_snapshot, + ) + camera_commit_calls: list[str] = [] + camera_commit_results: list[bool] = [] + camera_commit_worker = threading.Thread( + name="camera-commit-candidate", + target=lambda: camera_commit_results.append( + service._commit_camera_restart_if_still_active( # noqa: SLF001 + acquisition_id=acquisition.acquisition_id, + evidence_session_id=out_dir.name, + start_operation_id=start_operation_id, + runtime_producer_generation=runtime.producer_generation, + commit=lambda: camera_commit_calls.append("commit") or True, + ) + ), + daemon=True, + ) + camera_commit_worker.start() + assert snapshot_blocked.wait(3.0) + + stop_owned_entered = threading.Event() + + def stop_owned( + _request: StopAcquisitionRequest, + _preadmitted_stop: object | None = None, + ) -> dict[str, object]: + assert service._camera_stop_priority_counts == { # noqa: SLF001 + acquisition.acquisition_id: 1 + } + stop_owned_entered.set() + with service._lock: # noqa: SLF001 + acquisition.transition( + "awaiting_external_stop", + message_code="acquisition.stop.device_stopping", + ) + service._acquisition_stop_operation_id = "snapshot-priority-stop" # noqa: SLF001 + return {"stop": "prepared"} + + monkeypatch.setattr(service, "_stop_acquisition_owned", stop_owned) + stop_result: list[dict[str, object]] = [] + stop_worker = threading.Thread( + target=lambda: stop_result.append( + service.stop_acquisition( + _stop_request( + acquisition_id=None, + mode="graceful", + operator_confirmed=True, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + ), + daemon=True, + ) + stop_worker.start() + assert stop_owned_entered.wait(3.0) + stop_worker.join(3.0) + assert stop_worker.is_alive() is False + assert stop_result == [{"stop": "prepared"}] + + release_snapshot.set() + camera_commit_worker.join(3.0) + assert camera_commit_worker.is_alive() is False + assert camera_commit_results == [False] + assert camera_commit_calls == [] + + +def test_stop_priority_overtakes_blocked_initial_camera_popen( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + assert acquisition is not None + assert out_dir is not None + camera = service.camera_preview.snapshot() + camera.update( + { + "phase": "idle", + "generation": None, + "active_source_id": None, + "recording": {"active": False}, + "error": None, + } + ) + candidate_blocked = threading.Event() + release_candidate = threading.Event() + candidate_committed: list[bool] = [] + + def blocked_initial_popen( + _source_id: str, + _target: str, + _session_dir: Path, + *, + commit_fence: Callable[[Callable[[], bool]], bool], + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + del committed_before_start + candidate_blocked.set() + assert release_candidate.wait(3.0) + if not commit_fence(lambda: candidate_committed.append(True) or True): + raise ValueError("camera activation lineage устарела") + return dict(camera) + + monkeypatch.setattr( + service.camera_preview, + "activate_recording_producer", + blocked_initial_popen, + ) + monkeypatch.setattr(service, "_ensure_camera_preview_process_lease", lambda: False) + activation_result: list[bool] = [] + activation_errors: list[BaseException] = [] + + def activate() -> None: + try: + activation_result.append( + service._activate_default_acquisition_camera( # noqa: SLF001 + expected_acquisition_id=acquisition.acquisition_id, + expected_evidence_session_id=out_dir.name, + expected_runtime_generation=runtime.producer_generation, + ) + ) + except BaseException as exc: + activation_errors.append(exc) + + activation = threading.Thread( + target=activate, + daemon=True, + ) + activation.start() + assert candidate_blocked.wait(3.0) + + stop_result: list[dict[str, object]] = [] + stop_worker = threading.Thread( + target=lambda: stop_result.append( + service.stop_acquisition( + fixture.request.model_copy(update={"acquisition_id": None}) + ) + ), + daemon=True, + ) + stop_worker.start() + stop_worker.join(3.0) + assert stop_worker.is_alive() is False + assert len(stop_result) == 1 + assert stop_result[0]["acquisition"]["state"] == "awaiting_external_stop" + assert fixture.control.stop_calls == 1 + assert fixture.prepare_calls == [fixture.stop_operation_id] + prepared_record = fixture.ledger.snapshot().record + assert prepared_record is not None + assert prepared_record.operation_id == fixture.stop_operation_id + assert prepared_record.stage == "prepared" + + release_candidate.set() + activation.join(3.0) + assert activation.is_alive() is False + assert activation_result == [] + assert len(activation_errors) == 1 + assert isinstance(activation_errors[0], ValueError) + assert candidate_committed == [] + + +def test_nontransport_camera_failure_remains_terminal_without_recovery_wake( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + camera = service.camera_preview.snapshot() + camera["phase"] = "error" + camera["error"] = { + "code": "camera-artifact-write-failed", + "message": "synthetic local storage failure", + } + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + state = service.state() + + assert runtime.recovery_requests == [] + assert runtime.recovery_request_pending is False + assert state["acquisition"]["state"] == "failed" + assert state["acquisition"]["result"]["camera_failure_code"] == ( + "camera-artifact-write-failed" + ) + assert runtime.stop_calls == 1 + assert control.stop_calls == 0 + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_host_first_recovery_does_not_mask_later_camera_artifact_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + for _ in range(3): + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + reconnecting = service.state() + assert reconnecting["phase"] == "reconnecting" + assert runtime.recovery_request_pending is True + + camera = service.camera_preview.snapshot() + camera["phase"] = "error" + camera["error"] = { + "code": "camera-artifact-write-failed", + "message": "synthetic local artifact failure after host loss", + } + + failed = service.state() + + assert failed["acquisition"]["state"] == "failed" + assert failed["acquisition"]["result"]["camera_failure_code"] == ( + "camera-artifact-write-failed" + ) + assert failed["source_mode"] == "idle" + assert runtime.stop_calls == 1 + assert runtime.recovery_request_pending is False + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_control_first_loss_freezes_topology_before_ephemeral_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + state = service.state() + + assert runtime.recovery_requests == [ + ("mqtt_network_loop_failed", runtime.producer_generation), + ] + assert state["phase"] == "reconnecting" + assert state["connection_recovery"]["automatic_read_only_rebind"] is True + assert state["acquisition"]["state"] == "acquiring" + assert state["selected_device_id"] == "test-ble-transport" + assert state["device_session"] is not None + assert state["k1_ip"] == "192.168.1.20" + assert runtime.stop_calls == 0 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_incident_host_route_streak_wakes_recovery_before_binding_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="streaming", + ) + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + + states: list[dict[str, Any]] = [] + for _ in range(3): + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + states.append(service.state()) + + assert runtime.recovery_requests == [ + ("host-route-unavailable", runtime.producer_generation), + ] + assert all(state["selected_device_id"] == "test-ble-transport" for state in states) + assert states[-1]["phase"] == "reconnecting" + assert states[-1]["connection_recovery"]["automatic_read_only_rebind"] is True + assert states[-1]["acquisition"]["state"] == "acquiring" + assert control.state == "scanning" + assert runtime.stop_calls == 0 + + +@pytest.mark.parametrize("first_symptom", ["camera", "control"]) +def test_host_camera_and_late_mqtt_loss_share_one_recovery_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + first_symptom: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="live", + camera_phase="error" if first_symptom == "camera" else "streaming", + ) + + if first_symptom == "camera": + first = service.state() + assert first["connection_recovery"]["state"] == "inactive" + assert runtime.recovery_requests == [] + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + else: + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + first = service.state() + camera = service.camera_preview.snapshot() + camera["phase"] = "error" + camera["error"] = { + "code": "camera-source-ended", + "message": "synthetic later camera loss", + } + second = service.state() + generation = second["connection_recovery"]["generation"] + # A later Paho callback sees the already reconnecting runtime and therefore + # joins this owner; it cannot queue a second capture wake or lineage. + assert service._active_stream_recovery_admitted_for_runtime_loss() is True # noqa: SLF001 + + assert len(runtime.recovery_requests) == 1 + assert second["connection_recovery"]["generation"] == generation + assert second["connection_recovery"]["state"] == "reconnecting" + assert second["acquisition"]["state"] == "acquiring" + assert runtime.stop_calls == 0 + + +def test_active_stream_recovery_survives_old_control_keepalive_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + + admitted = service.state() + assert admitted["connection_recovery"]["state"] == "reconnecting" + with service._lock: # noqa: SLF001 + assert service._active_stream_recovery_started_monotonic is not None # noqa: SLF001 + service._active_stream_recovery_started_monotonic -= 60.0 # noqa: SLF001 + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + + after_keepalive = service.state() + + assert after_keepalive["phase"] == "reconnecting" + assert after_keepalive["connection_recovery"]["elapsed_ms"] >= 60_000 + assert after_keepalive["connection_recovery"]["automatic_command_retry"] is False + assert after_keepalive["acquisition"]["state"] == "acquiring" + assert after_keepalive["selected_device_id"] == "test-ble-transport" + assert after_keepalive["k1_ip"] == "192.168.1.20" + assert runtime.stop_calls == 0 + + +def test_recovered_projection_requires_a_live_ready_source_and_terminal_lease_retires_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + reconnecting = service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + assert reconnecting["connection_recovery"]["state"] == "reconnecting" + + runtime.phase = "idle" + runtime.source_mode = "idle" + runtime.source_ready = False + projected_reconnecting_idle = service._active_stream_recovery_projection( # noqa: SLF001 + runtime.snapshot() + ) + assert projected_reconnecting_idle["state"] == "inactive" + assert projected_reconnecting_idle["automatic_read_only_rebind"] is False + assert projected_reconnecting_idle["acquisition_id"] is None + + runtime.phase = "reconnecting" + runtime.source_mode = "live" + + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + runtime.recovery_attempt = 7 + recovered = service._active_stream_recovery_projection(runtime.snapshot()) # noqa: SLF001 + + assert recovered["state"] == "recovered" + assert recovered["attempt"] == 7 + assert recovered["automatic_read_only_rebind"] is True + + # An out-of-order runtime tail cannot publish recovered while the receiver + # is terminal. Durable physical-command history is owned by its ledger, not + # by this process-local recovery contract. + runtime.phase = "idle" + runtime.source_mode = "idle" + runtime.source_ready = False + projected_idle = service._active_stream_recovery_projection(runtime.snapshot()) # noqa: SLF001 + assert projected_idle["state"] == "inactive" + assert projected_idle["automatic_read_only_rebind"] is False + assert projected_idle["acquisition_id"] is None + assert projected_idle["camera_recovery"] == "inactive" + + service._release_acquisition_session_lease() # noqa: SLF001 + retired = service._active_stream_recovery_projection(runtime.snapshot()) # noqa: SLF001 + assert retired["state"] == "inactive" + assert retired["automatic_read_only_rebind"] is False + assert retired["acquisition_id"] is None + assert service._active_stream_recovery_lineage is None # noqa: SLF001 + + +def test_local_force_finish_cancels_generation_and_is_idempotent_for_mode_reset( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="reconnecting", + camera_phase="error", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + state = service.state() + acquisition = state["acquisition"] + recovery = state["connection_recovery"] + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + assert runtime.recovery_request_pending is True + + finished = service.force_finish_acquisition_locally( + _force_finish_request( + acquisition_id=acquisition["acquisition_id"], + expected_state_revision=acquisition["state_revision"], + expected_recovery_generation=recovery["generation"], + ) + ) + + assert finished["acquisition"]["state"] == "interrupted" + assert finished["acquisition"]["result"]["device_stop"] == "not-sent" + assert finished["acquisition"]["result"]["physical_command_sent"] is False + assert finished["connection_recovery"]["state"] == "force-finished" + assert runtime.stop_calls == 1 + assert runtime.recovery_request_pending is False + assert service._active_stream_recovery_lineage_is_current(lineage) is False # noqa: SLF001 + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + with service._acquisition_lifecycle_gate: # noqa: SLF001 + assert ( + service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="connection-mode-reset", + require_recovery=False, + ) + is False + ) + + +@pytest.mark.parametrize("cleanup_failure", ["runtime", "camera"]) +def test_local_force_finish_cleanup_failure_is_visible_retryable_and_fences_late_success( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cleanup_failure: str, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + before = service.state() + acquisition = before["acquisition"] + recovery = before["connection_recovery"] + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + original_stop_recording = service.camera_preview.stop_recording + camera_failure_active = cleanup_failure == "camera" + + def stop_recording(**kwargs: object) -> dict[str, object]: + nonlocal camera_failure_active + if camera_failure_active: + camera_failure_active = False + raise RuntimeError("synthetic camera local cleanup failure") + return original_stop_recording(**kwargs) # type: ignore[arg-type,return-value] + + if cleanup_failure == "runtime": + runtime.stop_error = RuntimeError("synthetic runtime local cleanup failure") + else: + monkeypatch.setattr(service.camera_preview, "stop_recording", stop_recording) + + with pytest.raises(RuntimeError, match=f"synthetic {cleanup_failure} local cleanup failure"): + service.force_finish_acquisition_locally( + _force_finish_request( + acquisition_id=acquisition["acquisition_id"], + expected_state_revision=acquisition["state_revision"], + expected_recovery_generation=recovery["generation"], + ) + ) + + failed = service.state() + failed_operation = failed["last_operation"] + assert failed["acquisition"]["state"] == "failed" + assert failed["acquisition"]["cleanup_pending"] is True + assert failed["acquisition"]["result"]["local_force_finish"] is True + assert failed["acquisition"]["result"]["device_stop"] == "not-sent" + assert failed["acquisition"]["result"]["physical_command_sent"] is False + assert failed_operation["action"] == facade_module.ACTION_ACQUISITION_FORCE_FINISH + assert failed_operation["status"] == "failed" + assert failed_operation["stage_code"] == "local-cleanup-failed" + assert failed_operation["error"]["retryable"] is True + assert failed_operation["error"]["safe_to_retry"] is True + assert failed_operation["error"]["side_effect_status"] == "none" + assert service._active_stream_recovery_lineage_is_current(lineage) is False # noqa: SLF001 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + # The retry is local finalization only. It seals the retained evidence + # lease through the compatibility stop seam and cannot revive recovery or + # publish a physical command. + runtime.stop_error = None + recovered = service.stop() + + assert recovered["acquisition"]["state"] == "failed" + assert recovered["acquisition"]["cleanup_pending"] is False + assert recovered["source_mode"] == "idle" + assert service._active_stream_recovery_lineage_is_current(lineage) is False # noqa: SLF001 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_connection_mode_reset_supersedes_queued_force_finish_without_state_revival( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="reconnecting", + camera_phase="error", + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + queued = service.state() + queued_acquisition = queued["acquisition"] + queued_recovery = queued["connection_recovery"] + assert runtime.recovery_request_pending is True + + with service._acquisition_lifecycle_gate: # noqa: SLF001 + reset_finished = service._force_finish_active_acquisition_locally( # noqa: SLF001 + reason_code="connection-mode-reset", + require_recovery=False, + ) + + assert reset_finished is True + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="acquisition changed", + ): + service.force_finish_acquisition_locally( + _force_finish_request( + acquisition_id=queued_acquisition["acquisition_id"], + expected_state_revision=queued_acquisition["state_revision"], + expected_recovery_generation=queued_recovery["generation"], + ) + ) + after = service.state() + assert after["acquisition"]["state"] == "interrupted" + assert after["connection_recovery"]["state"] == "force-finished" + assert runtime.stop_calls == 1 + assert runtime.recovery_request_pending is False + assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001 + + +def test_active_stream_recovery_scanning_resumes_same_physical_lineage_without_commands( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + physical["reconciliation_ready"] = True + physical["observed_session_state"] = "scanning" + control.state = "connection-ready" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + reconcile_ids: list[str] = [] + + def reconcile_resolved_active(*, reconciliation_id: str) -> dict[str, object]: + reconcile_ids.append(reconciliation_id) + record = physical["record"] + assert isinstance(record, dict) + record["revision"] = int(record["revision"]) + 1 + return record + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reconcile_resolved_active", + reconcile_resolved_active, + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("recovery must not repeat START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("recovery must not send STOP"), + ) + + decision = asyncio.run( + service._reconcile_active_stream_physical_state_owned( # noqa: SLF001 + lineage, + attempt=3, + ) + ) + + assert decision == "resume" + assert reconcile_ids == [ + facade_module._active_stream_reconciliation_id( # noqa: SLF001 + lineage, + attempt=3, + ) + ] + assert control.state == "scanning" + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + # The physical audit revision advanced before MQTT SUBACK. A failed first + # resubscribe must admit the same frozen lineage on attempt two. + assert service._admit_active_stream_recovery_lineage() is lineage # noqa: SLF001 + assert ( + physical["record"] + is service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + ) + + +def test_active_stream_control_adoption_gets_fresh_budget_after_slow_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + physical["reconciliation_ready"] = True + physical["observed_session_state"] = "scanning" + control.state = "connection-ready" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + original_adopt = control.adopt_reconciled_scanning + original_snapshot = control.snapshot + clock = [200.0] + adoption_pending = [False] + adoption_snapshots = [0] + + class FacadeTimeProxy: + def monotonic(self) -> float: + return clock[0] + + def __getattr__(self, name: str) -> Any: + return getattr(time, name) + + def reconcile_resolved_active(*, reconciliation_id: str) -> dict[str, object]: + assert reconciliation_id + record = physical["record"] + assert isinstance(record, dict) + record["revision"] = int(record["revision"]) + 1 + return record + + def delayed_adopt(**kwargs: object) -> dict[str, object]: + result = original_adopt(**kwargs) + adoption_pending[0] = True + control.state = "active-recovery-requested" + clock[0] = 205.01 + return result + + def delayed_snapshot() -> dict[str, object]: + if adoption_pending[0]: + adoption_snapshots[0] += 1 + if adoption_snapshots[0] >= 2: + control.state = "scanning" + clock[0] = 205.5 + return original_snapshot() + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reconcile_resolved_active", + reconcile_resolved_active, + ) + monkeypatch.setattr(control, "adopt_reconciled_scanning", delayed_adopt) + monkeypatch.setattr(control, "snapshot", delayed_snapshot) + monkeypatch.setattr(facade_module, "time", FacadeTimeProxy()) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("recovery must not repeat START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("recovery must not send STOP"), + ) + + decision = asyncio.run( + service._reconcile_active_stream_physical_state_owned( # noqa: SLF001 + lineage, + attempt=4, + ) + ) + + assert decision == "resume" + assert adoption_snapshots[0] >= 2 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +def test_epoch_one_to_three_rebind_namespaces_prior_acquisition_attempt_collision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regress the 2026-08-12 acq-cc9f5310 Wi-Fi recovery failure.""" + + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + with service._lock: # noqa: SLF001 + # Match the live process-local counter; another acquisition had + # already persisted auto-rebind:2:6 in the global ledger history. + service._active_stream_recovery_generation = 1 # noqa: SLF001 + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + assert lineage.recovery_generation == 2 + assert service._connection_supervisor.snapshot().host_path.epoch == 1 # noqa: SLF001 + + service._connection_supervisor.observe_host_path( # noqa: SLF001 + HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + ) + service._connection_supervisor.observe_host_path( # noqa: SLF001 + HostPathProbeResult( + available=True, + fingerprint="route-after-wifi-return", + interface="test0", + source_ipv4="192.168.1.100", + route_class="direct", + ) + ) + assert service._connection_supervisor.snapshot().host_path.epoch == 3 # noqa: SLF001 + assert service._active_stream_recovery_lineage_is_current(lineage) is True # noqa: SLF001 + + physical["reconciliation_ready"] = True + physical["observed_session_state"] = "scanning" + control.state = "connection-ready" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + historical_ids = {"auto-rebind:2:6", "auto-rebind:2:7"} + reconcile_ids: list[str] = [] + + def reconcile_resolved_active(*, reconciliation_id: str) -> dict[str, object]: + if reconciliation_id in historical_ids: + raise facade_module.PhysicalCommandTransitionError( + "physical reconciliation id has already been used" + ) + reconcile_ids.append(reconciliation_id) + record = physical["record"] + assert isinstance(record, dict) + record["revision"] = int(record["revision"]) + 1 + return record + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reconcile_resolved_active", + reconcile_resolved_active, + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("recovery must not replay START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("recovery must not publish STOP"), + ) + monkeypatch.setattr( + service, + "scan_ble", + lambda *_args, **_kwargs: pytest.fail("recovery must not touch BLE"), + ) + monkeypatch.setattr( + service, + "connect", + lambda *_args, **_kwargs: pytest.fail("recovery must not mutate network state"), + ) + + decision = asyncio.run( + service._reconcile_active_stream_physical_state_owned( # noqa: SLF001 + lineage, + attempt=6, + ) + ) + + expected_id = facade_module._active_stream_reconciliation_id( # noqa: SLF001 + lineage, + attempt=6, + ) + prior_process_lineage = facade_module.dataclass_replace( + lineage, + snapshot_runtime_id="snapshot-runtime-prior-process", + acquisition_id="acq-b0cc1fe4-b934-4a98-a3e5-ecab7afd17c4", + physical_operation_id="op-prior-acquisition-start", + ) + prior_process_id = facade_module._active_stream_reconciliation_id( # noqa: SLF001 + prior_process_lineage, + attempt=6, + ) + assert decision == "resume" + assert reconcile_ids == [expected_id] + assert expected_id not in historical_ids + assert expected_id != prior_process_id + assert expected_id == facade_module._active_stream_reconciliation_id( # noqa: SLF001 + lineage, + attempt=6, + ) + assert len(expected_id + ".device-info") <= 160 + # Physical rebind only authorizes transport recovery. Visible recovery is + # still fenced on the later non-empty post-published PCL. + assert runtime.snapshot()["phase"] == "reconnecting" + assert runtime.snapshot()["source_ready"] is False + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +@pytest.mark.parametrize( + ("observed_state", "expected_reason", "reconcile_expected"), + [ + ("ready", "device-reported-standby", True), + ("scan_over", "device-reported-scan-over", True), + ], +) +def test_active_stream_recovery_device_standby_never_restarts_scanner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + observed_state: str, + expected_reason: str, + reconcile_expected: bool, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + physical["reconciliation_ready"] = True + physical["observed_session_state"] = observed_state + control.state = "connection-ready" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + reconcile_ids: list[str] = [] + + def reconcile_resolved_active(*, reconciliation_id: str) -> dict[str, object]: + reconcile_ids.append(reconciliation_id) + record = physical["record"] + assert isinstance(record, dict) + record["revision"] = int(record["revision"]) + 1 + physical["resolved_active_recovery_required"] = False + physical["resolved_scan_over_recovery_required"] = observed_state == "scan_over" + physical["requires_reconciliation"] = observed_state == "scan_over" + return record + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reconcile_resolved_active", + reconcile_resolved_active, + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("standby recovery must not repeat START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("standby recovery must not send STOP"), + ) + + decision = asyncio.run( + service._reconcile_active_stream_physical_state_owned( # noqa: SLF001 + lineage, + attempt=4, + ) + ) + + assert decision == "standby" + assert bool(reconcile_ids) is reconcile_expected + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._active_stream_recovery_state == "standby" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == expected_reason # noqa: SLF001 + assert physical["resolved_active_recovery_required"] is False + assert physical["requires_reconciliation"] is (observed_state == "scan_over") + + +def test_active_stream_recovery_wrong_identity_blocks_without_retry_or_camera_reopen( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + async def monitor_quiescent() -> bool: + return True + + monkeypatch.setattr( + service, + "_await_connection_monitor_quiescence", + monitor_quiescent, + ) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path(lineage.target_ipv4), + reachable=True, + reason_code=None, + ), + ) + + async def reject_wrong_identity(**_kwargs: object) -> None: + raise facade_module.ConnectionVerificationError( + "same IP belongs to another K1", + reason_code="control-bootstrap-device-identity-unverified", + ) + + camera_reopens: list[str] = [] + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + reject_wrong_identity, + ) + monkeypatch.setattr( + service.camera_preview, + "select", + lambda *_args: camera_reopens.append("camera") or {}, + ) + + decision = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 1) # noqa: SLF001 + ) + + assert decision == "blocked" + assert service._active_stream_recovery_state == "blocked" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == ( # noqa: SLF001 + "control-bootstrap-device-identity-unverified" + ) + assert camera_reopens == [] + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + control.state = "idle" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + events: list[tuple[str, object]] = [] + + async def monitor_quiescent() -> bool: + events.append(("monitor", True)) + return True + + def probe_exact_target(target_ipv4: str) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001 + events.append(("probe", target_ipv4)) + return facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path(target_ipv4), + reachable=True, + reason_code=None, + ) + + async def inspection_only_bootstrap(**kwargs: object) -> None: + assert kwargs["connection_mode"] == lineage.connection_mode + assert kwargs["inspection_only"] is True + events.append(("device-info-status", kwargs["parent_operation_id"])) + + async def reconcile_same_lineage( + candidate: facade_module._ActiveStreamRecoveryLineage, # noqa: SLF001 + *, + attempt: int, + ) -> facade_module.RecoveryDecision: + assert candidate is lineage + events.append(("physical-read-only", attempt)) + return "resume" + + def recover_local_camera( + candidate: facade_module._ActiveStreamRecoveryLineage, # noqa: SLF001 + ) -> facade_module.RecoveryDecision: + assert candidate is lineage + events.append(("camera-local", candidate.evidence_session_id)) + return "resume" + + def forbidden_device_command(**_kwargs: object) -> object: + pytest.fail("active stream recovery must not publish START or STOP") + + async def forbidden_ble_or_network_mutation(*_args: object, **_kwargs: object) -> object: + pytest.fail("active stream recovery must not scan, write BLE, or provision Wi-Fi") + + monkeypatch.setattr(service, "_await_connection_monitor_quiescence", monitor_quiescent) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda holder: events.append(("lease-acquire", holder)), + ) + monkeypatch.setattr( + service, + "_release_k1_lifecycle_process_lease", + lambda holder: events.append(("lease-release", holder)), + ) + monkeypatch.setattr(service, "_probe_control_endpoint", probe_exact_target) + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + inspection_only_bootstrap, + ) + monkeypatch.setattr( + service, + "_reconcile_active_stream_physical_state_owned", + reconcile_same_lineage, + ) + monkeypatch.setattr(service, "_recover_active_stream_camera_owned", recover_local_camera) + monkeypatch.setattr(control, "request_start", forbidden_device_command) + monkeypatch.setattr(control, "request_stop", forbidden_device_command) + monkeypatch.setattr(service, "scan_ble", forbidden_ble_or_network_mutation) + monkeypatch.setattr(service, "connect", forbidden_ble_or_network_mutation) + + decision = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 5) # noqa: SLF001 + ) + + assert decision == "resume" + assert events == [ + ("monitor", True), + ("lease-acquire", "network"), + ("probe", lineage.target_ipv4), + ( + "device-info-status", + ( + f"active-stream-recovery:{lineage.acquisition_id}:" + f"{lineage.recovery_generation}" + ), + ), + ("physical-read-only", 5), + ("lease-release", "network"), + ] + # Camera recovery is now intentionally downstream of the post-Rerun + # checkpoint rebind. Control recovery alone cannot open/promote media. + assert ("camera-local", lineage.evidence_session_id) not in events + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + assert service._active_stream_recovery_lineage_is_current(lineage) is True # noqa: SLF001 + assert service._active_stream_recovery_reason_code == "mqtt-resubscribe-pending" # noqa: SLF001 + + +def test_incident_recovery_retires_epoch_one_control_and_retries_fresh_inspection_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regress the 2026-08-12 acq-209faf83 route-loss failure. + + The incident START/control owner was bound to host-path epoch 1. Wi-Fi + loss rotated that epoch, and the first fresh inspection raced one more + association edge. That expected local binding loss must keep retrying; + the next inspection must bind the current epoch without replaying any + physical or network command. + """ + + service, runtime = service_with_fake_runtime(tmp_path) + + class IncidentControl(FakeInteractiveControlSession): + def __init__(self) -> None: + super().__init__() + self.opened_bindings: list[ApplicationConnectionBinding] = [] + self.close_calls = 0 + self.retire_calls = 0 + self.binding_validation_calls = 0 + + def validate_connection_binding(self) -> None: + self.binding_validation_calls += 1 + raise AssertionError("recovery must not validate/reuse the epoch-1 owner") + + def close(self) -> None: + self.close_calls += 1 + super().close() + + def retire_for_network_change(self, **kwargs: object) -> dict[str, object]: + self.retire_calls += 1 + return super().retire_for_network_change(**kwargs) + + def open( + self, + *, + connection_binding: ApplicationConnectionBinding, + inspection_only: bool = False, + **kwargs: object, + ) -> dict[str, object]: + self.opened_bindings.append(connection_binding) + if len(self.opened_bindings) == 1: + # Match the live failure: the newly opened worker observed one + # more route/association epoch before its first MQTT publish. + service._connection_supervisor.observe_host_path( # noqa: SLF001 + recovered_path_b + ) + self.state = "failed" + self.failure = { + "reason_code": "application-connection-binding-lost", + "safe_to_retry": True, + "modeling_command_attempted": False, + } + return self.snapshot() + self.session_generation += 1 + super().open( + connection_binding=connection_binding, + inspection_only=inspection_only, + **kwargs, + ) + assert self.verified_control is not None + self.verified_control = { + **self.verified_control, + "control_session_id": ( + f"fake-recovery-control-{self.session_generation}" + ), + "producer_generation": self.session_generation, + } + return self.snapshot() + + control = IncidentControl() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + runtime_phase="reconnecting", + camera_phase="streaming", + ) + # The fixture installs its own fake session; replace it before freezing the + # recovery lineage while retaining the exact already-resolved START proof. + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + control.state = "scanning" + original_epoch = service._connection_supervisor.snapshot().host_path.epoch # noqa: SLF001 + assert original_epoch == 1 + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="host-route-unavailable", + ) + recovered_path_a = HostPathProbeResult( + available=True, + fingerprint="incident-route-after-wifi-return-a", + interface="test0", + source_ipv4="192.168.68.100", + route_class="direct", + ) + recovered_path_b = HostPathProbeResult( + available=True, + fingerprint="incident-route-after-wifi-return-b", + interface="test0", + source_ipv4="192.168.68.100", + route_class="direct", + ) + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + assert service._connection_supervisor.snapshot().host_path.epoch == 2 # noqa: SLF001 + + probe_paths = iter((recovered_path_a, recovered_path_b)) + + def probe_current_epoch( + target_ipv4: str, + ) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001 + path = next(probe_paths) + service._observe_connection_transport( # noqa: SLF001 + target_ipv4, + path=path, + reachable=True, + ) + return facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=path, + reachable=True, + reason_code=None, + ) + + async def monitor_quiescent() -> bool: + return True + + async def reconcile_same_lineage( + candidate: facade_module._ActiveStreamRecoveryLineage, # noqa: SLF001 + *, + attempt: int, + ) -> facade_module.RecoveryDecision: + assert candidate is lineage + assert attempt == 6 + return "resume" + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned, # noqa: SLF001 + service, + ) + monkeypatch.setattr(service, "_await_connection_monitor_quiescence", monitor_quiescent) + monkeypatch.setattr(service, "_probe_control_endpoint", probe_current_epoch) + monkeypatch.setattr(service, "_acquire_k1_lifecycle_process_lease", lambda _holder: None) + monkeypatch.setattr(service, "_release_k1_lifecycle_process_lease", lambda _holder: None) + monkeypatch.setattr(service, "_acquire_application_control_process_lease", lambda: None) + monkeypatch.setattr(service, "_reconcile_application_control_process_lease", lambda _s: None) + monkeypatch.setattr( + service, + "_reconcile_active_stream_physical_state_owned", + reconcile_same_lineage, + ) + monkeypatch.setattr( + service, + "_recover_active_stream_camera_owned", + lambda candidate: "resume" if candidate is lineage else "blocked", + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("recovery must not replay START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("recovery must not publish STOP"), + ) + monkeypatch.setattr( + service, + "scan_ble", + lambda *_args, **_kwargs: pytest.fail("recovery must not touch BLE"), + ) + monkeypatch.setattr( + service, + "connect", + lambda *_args, **_kwargs: pytest.fail("recovery must not mutate network state"), + ) + + first = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 5) # noqa: SLF001 + ) + + assert first == "retry" + assert service._active_stream_recovery_state == "reconnecting" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == ( # noqa: SLF001 + "application-connection-binding-lost" + ) + assert control.opened_bindings[0].host_path_epoch == 3 + assert control.close_calls == 1 + assert control.retire_calls == 2 + assert control.binding_validation_calls == 0 + + second = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 6) # noqa: SLF001 + ) + + assert second == "resume" + assert [binding.host_path_epoch for binding in control.opened_bindings] == [3, 4] + assert control.opened_bindings[-1].host_path_epoch == ( + service._connection_supervisor.snapshot().host_path.epoch # noqa: SLF001 + ) + assert control.inspection_only is True + assert control.state == "connection-ready" + assert service._active_stream_recovery_lineage_is_current(lineage) is True # noqa: SLF001 + assert service._active_stream_recovery_reason_code == "mqtt-resubscribe-pending" # noqa: SLF001 + + +def test_active_stream_recovery_waits_for_terminal_control_worker_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": True, + "safe_to_retry": False, + } + retire_attempts = 0 + + def retire_terminal_control(**_kwargs: object) -> dict[str, object]: + nonlocal retire_attempts + retire_attempts += 1 + if retire_attempts == 1: + raise ApplicationAcceptanceError("control session worker is still retiring") + control.state = "idle" + control.verified_control = None + return control.snapshot() + + async def monitor_quiescent() -> bool: + return True + + async def inspection_only_bootstrap(**_kwargs: object) -> None: + return None + + async def reconcile_same_lineage( + candidate: facade_module._ActiveStreamRecoveryLineage, # noqa: SLF001 + *, + attempt: int, + ) -> facade_module.RecoveryDecision: + assert candidate is lineage + assert attempt == 2 + return "resume" + + monkeypatch.setattr(control, "retire_for_network_change", retire_terminal_control) + monkeypatch.setattr(service, "_await_connection_monitor_quiescence", monitor_quiescent) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path(lineage.target_ipv4), + reachable=True, + reason_code=None, + ), + ) + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + inspection_only_bootstrap, + ) + monkeypatch.setattr( + service, + "_reconcile_active_stream_physical_state_owned", + reconcile_same_lineage, + ) + monkeypatch.setattr( + service, + "_recover_active_stream_camera_owned", + lambda candidate: "resume" if candidate is lineage else "blocked", + ) + + decision = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 2) # noqa: SLF001 + ) + + assert decision == "resume" + assert retire_attempts == 2 + assert service._active_stream_recovery_state == "reconnecting" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == "mqtt-resubscribe-pending" # noqa: SLF001 + + +def test_active_stream_recovery_exact_device_system_fault_is_terminal_without_commands( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + control.state = "idle" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + + async def monitor_quiescent() -> bool: + return True + + async def bootstrap_system_fault(**kwargs: object) -> None: + assert kwargs["inspection_only"] is True + control.state = "failed" + control.failure = { + "reason_code": "application-device-system-error", + "modeling_command_attempted": False, + "safe_to_retry": False, + } + + monkeypatch.setattr(service, "_await_connection_monitor_quiescence", monitor_quiescent) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path(lineage.target_ipv4), + reachable=True, + reason_code=None, + ), + ) + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + bootstrap_system_fault, + ) + monkeypatch.setattr( + service, + "_recover_active_stream_camera_owned", + lambda _candidate: pytest.fail("system fault must not reopen camera"), + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("system fault recovery must not repeat START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("system fault recovery must not send STOP"), + ) + + decision = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 6) # noqa: SLF001 + ) + + assert decision == "fault" + assert service._active_stream_recovery_state == "fault" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == ( # noqa: SLF001 + "active-stream-recovery-system-error" + ) + assert service._active_stream_recovery_terminal_outcome == "fault" # noqa: SLF001 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +def test_active_stream_recovery_bootstrap_system_error_is_normalized_terminal_fault( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + control.state = "idle" + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + async def monitor_quiescent() -> bool: + return True + + async def bootstrap_system_error(**kwargs: object) -> None: + assert kwargs["inspection_only"] is True + raise facade_module.ConnectionVerificationError( + "DeviceInfo reported a system fault", + reason_code="application-device-system-error", + ) + + monkeypatch.setattr(service, "_await_connection_monitor_quiescence", monitor_quiescent) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path(lineage.target_ipv4), + reachable=True, + reason_code=None, + ), + ) + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + bootstrap_system_error, + ) + monkeypatch.setattr( + service, + "_reconcile_active_stream_physical_state_owned", + lambda *_args, **_kwargs: pytest.fail( + "bootstrap system fault must stop before physical reconciliation" + ), + ) + monkeypatch.setattr( + service, + "_recover_active_stream_camera_owned", + lambda _candidate: pytest.fail("bootstrap system fault must not reopen camera"), + ) + monkeypatch.setattr( + control, + "request_start", + lambda **_kwargs: pytest.fail("system fault recovery must not repeat START"), + ) + monkeypatch.setattr( + control, + "request_stop", + lambda **_kwargs: pytest.fail("system fault recovery must not send STOP"), + ) + + decision = asyncio.run( + service._recover_active_stream_connection_owned(lineage, 7) # noqa: SLF001 + ) + + assert decision == "fault" + assert service._active_stream_recovery_state == "fault" # noqa: SLF001 + assert service._active_stream_recovery_reason_code == ( # noqa: SLF001 + "active-stream-recovery-system-error" + ) + assert service._active_stream_recovery_terminal_outcome == "fault" # noqa: SLF001 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + terminal = service.state() + + assert terminal["acquisition"]["state"] == "failed" + assert terminal["acquisition"]["result"]["device_state"] == "fault" + assert terminal["source_mode"] == "idle" + assert runtime.stop_calls == 1 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +def test_active_stream_recovery_never_restarts_nontransport_camera_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + camera = service.camera_preview.snapshot() + camera["phase"] = "error" + camera["error"] = { + "code": "camera-artifact-write-failed", + "message": "synthetic local artifact failure", + } + restart_calls: list[str] = [] + monkeypatch.setattr( + service.camera_preview, + "restart_recording_producer", + lambda *_args, **_kwargs: restart_calls.append("restart") or {}, + ) + + decision = service._recover_active_stream_camera_owned(lineage) # noqa: SLF001 + + assert decision == "blocked" + assert restart_calls == [] + + +@pytest.mark.parametrize( + ("camera_phase", "camera_error", "producer_alive", "active_epoch"), + [ + ("connecting", None, False, 7), + ( + "error", + { + "code": "invalid-fmp4", + "message": "synthetic pre-first-media camera failure", + }, + False, + 7, + ), + ( + "error", + { + "code": "invalid-fmp4", + "message": "synthetic detached pre-finalize camera failure", + }, + False, + None, + ), + ], +) +def test_active_stream_recovery_keeps_pre_first_media_camera_downstream_of_mqtt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + camera_phase: str, + camera_error: dict[str, str] | None, + producer_alive: bool, + active_epoch: int | None, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + camera_phase="streaming", + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + start_projects_before = list(control.start_projects) + stop_calls_before = control.stop_calls + camera = service.camera_preview.snapshot() + camera.update( + { + "phase": camera_phase, + "generation": 7, + "active_source_id": facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + "recording": { + "active": True, + "session": lineage.evidence_session_id, + "active_epoch": active_epoch, + "producer_alive": producer_alive, + "producer_age_ms": ( + facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1 + ), + "last_segment_age_ms": None, + "committed_media_segment_count": 0, + "completed_epochs": 0, + "last_summary": None, + }, + "error": camera_error, + } + ) + activation_lineage = ( + lineage.acquisition_id, + lineage.evidence_session_id, + lineage.runtime_producer_generation, + ) + with service._lock: # noqa: SLF001 + service._camera_activation_lineage = activation_lineage # noqa: SLF001 + restart_calls: list[str] = [] + monkeypatch.setattr( + service.camera_preview, + "restart_recording_producer", + lambda *_args, **_kwargs: restart_calls.append("restart") or {}, + ) + + decision = service._recover_active_stream_camera_owned(lineage) # noqa: SLF001 + + assert decision == "resume" + assert restart_calls == [] + with service._lock: # noqa: SLF001 + if camera_phase == "error" or producer_alive is False: + assert service._camera_activation_lineage is None # noqa: SLF001 + assert service._camera_activation_retry_lineage == activation_lineage # noqa: SLF001 + else: + assert service._camera_activation_lineage == activation_lineage # noqa: SLF001 + state = service.state() + assert state["phase"] == "reconnecting" + assert state["acquisition"]["state"] == "acquiring" + assert state["camera_preview"]["activation_admission"]["state"] == "activating" + assert runtime.stop_calls == 0 + assert control.start_projects == start_projects_before + assert control.stop_calls == stop_calls_before + + +def test_active_stream_recovery_never_retries_invalid_fmp4_after_media_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + camera_phase="streaming", + ) + service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + camera = service.camera_preview.snapshot() + camera.update( + { + "phase": "error", + "generation": 7, + "active_source_id": facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + "recording": { + "active": True, + "session": lineage.evidence_session_id, + "active_epoch": 7, + "committed_media_segment_count": 1, + "producer_alive": False, + "producer_age_ms": 1_000, + "last_segment_age_ms": 0, + "completed_epochs": 0, + "last_summary": None, + }, + "error": { + "code": "invalid-fmp4", + "message": "synthetic corruption after the first media commit", + }, + } + ) + activation_lineage = ( + lineage.acquisition_id, + lineage.evidence_session_id, + lineage.runtime_producer_generation, + ) + with service._lock: # noqa: SLF001 + service._camera_activation_lineage = activation_lineage # noqa: SLF001 + restart_calls: list[str] = [] + monkeypatch.setattr( + service.camera_preview, + "restart_recording_producer", + lambda *_args, **_kwargs: restart_calls.append("restart") or {}, + ) + monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None) + + assert service._recover_active_stream_camera_owned(lineage) == "blocked" # noqa: SLF001 + assert restart_calls == [] + with service._lock: # noqa: SLF001 + assert service._camera_activation_lineage == activation_lineage # noqa: SLF001 + assert service._camera_activation_retry_lineage is None # noqa: SLF001 + + terminal = service.state() + assert terminal["acquisition"]["state"] == "failed" + assert terminal["acquisition"]["result"]["camera_failure_code"] == "invalid-fmp4" + assert runtime.stop_calls == 1 + assert control.stop_calls == 0 + + +def test_active_stream_recovery_reopens_exact_camera_epoch_once_and_fences_stale_lineage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _control, _physical = _install_composite_active_recovery_fixture( + service, + runtime, + monkeypatch, + ) + state = service.state() + lineage = service._active_stream_recovery_lineage # noqa: SLF001 + assert lineage is not None + camera = dict(state["camera_preview"]) + camera["phase"] = "streaming" + camera["generation"] = 7 + camera["active_source_id"] = facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE + camera["recording"] = { + "active": True, + "session": lineage.evidence_session_id, + "active_epoch": 7, + "producer_alive": True, + "producer_age_ms": facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1, + "last_segment_age_ms": (facade_module.ACTIVE_STREAM_CAMERA_STALL_MILLISECONDS + 1), + "committed_media_segment_count": 4, + } + camera_calls: list[tuple[str, str, int, str, int | None, int]] = [] + perception_binds: list[tuple[str, int]] = [] + + def snapshot_camera() -> dict[str, object]: + return dict(camera) + + def reopen_camera( + source_id: str, + target: str, + *, + expected_generation: int, + expected_recording_session: str, + expected_active_epoch: int | None, + expected_recording_media_segment_count: int, + pre_detach_fence: Callable[[Callable[[], bool]], bool], + commit_fence: Callable[[Callable[[], bool]], bool] | None = None, + committed_before_start: Callable[[dict[str, object]], None] | None = None, + ) -> dict[str, object]: + camera_calls.append( + ( + source_id, + target, + expected_generation, + expected_recording_session, + expected_active_epoch, + expected_recording_media_segment_count, + ) + ) + assert pre_detach_fence(lambda: True) is True + camera.update( + { + "phase": "connecting", + "generation": expected_generation + 1, + "active_source_id": source_id, + "recording": { + "active": True, + "session": lineage.evidence_session_id, + "active_epoch": 8, + "producer_alive": True, + "producer_age_ms": 0, + "last_segment_age_ms": 0, + "last_media_segment_age_ms": None, + "committed_media_segment_count": 4, + "media_ready": False, + "current_epoch": { + "generation": 8, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + }, + }, + } + ) + assert commit_fence is not None + assert committed_before_start is not None + assert commit_fence( + lambda: committed_before_start(dict(camera)) is None + ) is True + return dict(camera) + + def bind_perception(session: str, reopened: dict[str, object]) -> None: + recording = reopened["recording"] + assert isinstance(recording, dict) + active_epoch = recording["active_epoch"] + assert isinstance(active_epoch, int) + perception_binds.append((session, active_epoch)) + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot_camera) + monkeypatch.setattr( + service.camera_preview, + "restart_recording_producer", + reopen_camera, + ) + monkeypatch.setattr( + service, + "_bind_live_perception_camera", + bind_perception, + ) + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + with service._lock: # noqa: SLF001 + service._active_stream_recovery_state = "recovered" # noqa: SLF001 + + assert ( # noqa: SLF001 + service._recover_active_stream_camera_owned( + lineage, + require_runtime_reconnecting=False, + ) + == "resume" + ) + assert ( # noqa: SLF001 + service._recover_active_stream_camera_owned( + lineage, + require_runtime_reconnecting=False, + ) + == "resume" + ) + assert camera_calls == [ + ( + facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + lineage.target_ipv4, + 7, + lineage.evidence_session_id, + 7, + 4, + ) + ] + assert perception_binds == [(lineage.evidence_session_id, 8)] + assert camera["recording"]["media_ready"] is False + recovery = service._active_stream_recovery_projection( # noqa: SLF001 + runtime.snapshot(), + camera=camera, + ) + assert recovery["camera_recovery"] == "owned" + assert recovery["camera_media_state"] == "pending-init" + assert recovery["camera_media_ready"] is False + assert recovery["camera_epoch"] == { + "generation": 8, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + } + + runtime.producer_generation += 1 + assert ( # noqa: SLF001 + service._recover_active_stream_camera_owned( + lineage, + require_runtime_reconnecting=False, + ) + == "blocked" + ) + assert len(camera_calls) == 1 + + +def test_device_standby_retires_sources_after_terminal_local_stop_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service) + service._compatibility_attestation = ATTESTATION.model_dump(mode="json") # noqa: SLF001 prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name="TEST001", host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1359,17 +16316,30 @@ def test_device_standby_retires_sources_after_terminal_local_stop_failure( ) acquisition_id = prepared["acquisition"]["acquisition_id"] service.start_acquisition( - StartAcquisitionRequest( + _start_request( acquisition_id=acquisition_id, physical_acceptance=PHYSICAL_ACCEPTANCE, ) ) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + physical_proof = _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=True, + ) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) runtime.mark_ready() runtime.pcl_frames = 1 service.state() stopping = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, mode="graceful", physical_acceptance=PHYSICAL_ACCEPTANCE, @@ -1410,17 +16380,17 @@ def test_device_standby_retires_sources_after_terminal_local_stop_failure( def test_second_start_conflict_does_not_tear_down_start_owner(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) with pytest.raises(RuntimeError, match="состояния starting"): - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) state = service.state() start_operations = [ @@ -1438,7 +16408,7 @@ def test_state_waits_for_atomic_start_handoff( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1461,7 +16431,7 @@ def test_state_waits_for_atomic_start_handoff( def start_worker() -> None: try: - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) except BaseException as exc: # pragma: no cover - asserted below worker_errors.append(exc) @@ -1500,7 +16470,7 @@ def test_abort_waits_for_start_handoff_then_stops_owned_producers( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1521,13 +16491,13 @@ def test_abort_waits_for_start_handoff_then_stops_owned_producers( def start_worker() -> None: try: - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) except BaseException as exc: # pragma: no cover - asserted below worker_errors.append(exc) def abort_worker() -> None: try: - service.abort_acquisition(AbortAcquisitionRequest(acquisition_id=acquisition_id)) + service.abort_acquisition(_abort_request(acquisition_id=acquisition_id)) except BaseException as exc: # pragma: no cover - asserted below worker_errors.append(exc) finally: @@ -1557,7 +16527,7 @@ def test_camera_selection_finishes_before_serialized_acquisition_stop( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1565,8 +16535,13 @@ def test_camera_selection_finishes_before_serialized_acquisition_stop( ) acquisition_id = prepared["acquisition"]["acquisition_id"] device_session_id = prepared["device_session"]["device_session_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) - service._k1_ip = "192.168.1.20" # noqa: SLF001 + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection(service) + service._device_session_id = device_session_id # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + assert out_dir is not None entered_camera_arm = threading.Event() release_camera_arm = threading.Event() stop_finished = threading.Event() @@ -1594,7 +16569,7 @@ def test_camera_selection_finishes_before_serialized_acquisition_stop( def stop_worker() -> None: try: service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, mode="capture-only", ) @@ -1604,11 +16579,41 @@ def test_camera_selection_finishes_before_serialized_acquisition_stop( finally: stop_finished.set() + camera_snapshot = service.camera_preview.snapshot() + camera_snapshot.update( + { + "phase": "streaming", + "generation": 1, + "active_source_id": facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE, + "recording": { + "active": True, + "session": out_dir.name, + "active_epoch": 1, + "media_ready": True, + "current_epoch": { + "generation": 1, + "init_committed": True, + "first_media_committed": True, + }, + }, + } + ) monkeypatch.setattr(service, "_arm_camera_recording", blocked_camera_arm) + monkeypatch.setattr( + service.camera_preview, + "snapshot", + lambda: camera_snapshot, + ) monkeypatch.setattr( service.camera_preview, "select", - lambda _source_id, _target: events.append("select"), + lambda source_id, _target: ( + events.append("select") + or { + "active_source_id": source_id, + "recording": {"active": False}, + } + ), ) monkeypatch.setattr( service.camera_preview, @@ -1632,13 +16637,13 @@ def test_camera_selection_finishes_before_serialized_acquisition_stop( assert service._acquisition_session_lease is None # noqa: SLF001 -def test_camera_arm_failure_seals_stopped_session_before_releasing_lease( +def test_receiver_start_failure_seals_stopped_session_before_releasing_lease( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1653,13 +16658,16 @@ def test_camera_arm_failure_seals_stopped_session_before_releasing_lease( *, duration_seconds: float | None, project_name: str, + recover_connection: object | None = None, ) -> None: assert duration_seconds is None assert project_name == PROJECT_NAME + assert recover_connection is None events.append("start") out_dir.mkdir(parents=True) runtime.phase = "starting_live" runtime.source_mode = "live" + raise RuntimeError("synthetic receiver start failure") def stop_runtime() -> None: events.append("runtime") @@ -1668,10 +16676,11 @@ def test_camera_arm_failure_seals_stopped_session_before_releasing_lease( monkeypatch.setattr(runtime, "start_live", start_live) monkeypatch.setattr(runtime, "stop", stop_runtime) + camera_arm_calls: list[Path] = [] monkeypatch.setattr( service, "_arm_camera_recording", - lambda _out_dir: (_ for _ in ()).throw(RuntimeError("synthetic camera arm failure")), + lambda out_dir, **_kwargs: camera_arm_calls.append(out_dir), ) monkeypatch.setattr( service.camera_preview, @@ -1684,13 +16693,14 @@ def test_camera_arm_failure_seals_stopped_session_before_releasing_lease( lambda _capture_root: events.append("seal"), ) - with pytest.raises(RuntimeError, match="synthetic camera arm failure"): - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + with pytest.raises(RuntimeError, match="synthetic receiver start failure"): + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) state = service.state() assert state["acquisition"]["state"] == "failed" assert state["last_operation"]["status"] == "failed" assert events == ["start", "camera", "runtime", "seal"] + assert camera_arm_calls == [] assert service._acquisition_session_lease is None # noqa: SLF001 @@ -1700,7 +16710,7 @@ def test_start_cleanup_failure_retains_lease_and_marks_side_effect_unknown( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1708,15 +16718,31 @@ def test_start_cleanup_failure_retains_lease_and_marks_side_effect_unknown( ) acquisition_id = prepared["acquisition"]["acquisition_id"] - def fail_camera_arm(out_dir: Path) -> None: + def fail_receiver_start( + _host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Callable[[int], str] | None = None, + ) -> None: + assert duration_seconds is None + assert project_name == PROJECT_NAME + assert recover_connection is None out_dir.mkdir() - raise RuntimeError("synthetic camera arm failure") + raise RuntimeError("synthetic receiver start failure") - monkeypatch.setattr(service, "_arm_camera_recording", fail_camera_arm) + monkeypatch.setattr(runtime, "start_live", fail_receiver_start) + camera_arm_calls: list[Path] = [] + monkeypatch.setattr( + service, + "_arm_camera_recording", + lambda out_dir, **_kwargs: camera_arm_calls.append(out_dir), + ) monkeypatch.setattr(facade_module, "seal_capture_clock", lambda _capture_root: None) runtime.stop_error = RuntimeError("synthetic cleanup timeout") - with pytest.raises(RuntimeError, match="synthetic camera arm failure"): - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + with pytest.raises(RuntimeError, match="synthetic receiver start failure"): + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) state = service.state() operation = next(item for item in state["operations"] if item["action"] == "acquisition.start") @@ -1724,6 +16750,7 @@ def test_start_cleanup_failure_retains_lease_and_marks_side_effect_unknown( assert operation["status"] == "failed" assert operation["error"]["side_effect_status"] == "unknown" assert service._acquisition_session_lease is not None # noqa: SLF001 + assert camera_arm_calls == [] runtime.stop_error = None service.stop() @@ -1733,7 +16760,7 @@ def test_start_cleanup_failure_retains_lease_and_marks_side_effect_unknown( def test_receiver_completion_without_point_data_fails_start_operation(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -1741,7 +16768,7 @@ def test_receiver_completion_without_point_data_fails_start_operation(tmp_path: ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.phase = "idle" runtime.source_mode = "idle" @@ -1755,7 +16782,7 @@ def test_receiver_completion_without_point_data_fails_start_operation(tmp_path: def test_capture_only_stop_never_claims_that_physical_k1_stopped(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -1763,10 +16790,10 @@ def test_capture_only_stop_never_claims_that_physical_k1_stopped(tmp_path: Path) ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) stopped = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="capture-only") + _stop_request(acquisition_id=acquisition_id, mode="capture-only") ) assert runtime.stop_calls == 1 @@ -1826,14 +16853,14 @@ def test_stop_timeout_retains_lease_and_blocks_replacement_acquisition( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) out_dir = service._acquisition_out_dir # noqa: SLF001 assert out_dir is not None out_dir.mkdir() @@ -1846,16 +16873,14 @@ def test_stop_timeout_retains_lease_and_blocks_replacement_acquisition( runtime.stop_error = RuntimeError("synthetic stop timeout") with pytest.raises(RuntimeError, match="synthetic stop timeout"): - service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="capture-only") - ) + service.stop_acquisition(_stop_request(acquisition_id=acquisition_id, mode="capture-only")) assert seal_calls == [] assert service._acquisition_session_lease is not None # noqa: SLF001 assert service.state()["acquisition"]["cleanup_pending"] is True with pytest.raises(RuntimeError, match="evidence-сессия"): service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name="replacement", host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1869,7 +16894,7 @@ def test_stop_timeout_retains_lease_and_blocks_replacement_acquisition( assert service._acquisition_session_lease is None # noqa: SLF001 assert retried["acquisition"]["cleanup_pending"] is False replacement = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name="replacement", host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -1883,19 +16908,17 @@ def test_replay_rejects_retained_failed_acquisition_lease( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.stop_error = RuntimeError("synthetic stop timeout") with pytest.raises(RuntimeError, match="synthetic stop timeout"): - service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="capture-only") - ) + service.stop_acquisition(_stop_request(acquisition_id=acquisition_id, mode="capture-only")) with pytest.raises(RuntimeError, match="не запечатана"): service.start_replay("sessions/fixture.k1mqtt", speed=1.0, loop=False) @@ -1910,14 +16933,14 @@ def test_new_explicit_stop_retries_retained_terminal_cleanup( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) out_dir = service._acquisition_out_dir # noqa: SLF001 assert out_dir is not None out_dir.mkdir() @@ -1929,13 +16952,11 @@ def test_new_explicit_stop_retries_retained_terminal_cleanup( ) runtime.stop_error = RuntimeError("synthetic stop timeout") with pytest.raises(RuntimeError, match="synthetic stop timeout"): - service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="capture-only") - ) + service.stop_acquisition(_stop_request(acquisition_id=acquisition_id, mode="capture-only")) runtime.stop_error = None recovered = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, mode="capture-only", idempotency_key="retry-retained-cleanup", @@ -1956,14 +16977,14 @@ def test_natural_receiver_completion_fails_acquisition_when_session_clock_cannot ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.pcl_frames = 1 service.state() out_dir = service._acquisition_out_dir # noqa: SLF001 @@ -1990,14 +17011,14 @@ def test_natural_receiver_completion_reserves_finalization_before_reentrant_call ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.pcl_frames = 1 service.state() out_dir = service._acquisition_out_dir # noqa: SLF001 @@ -2028,7 +17049,7 @@ def test_natural_receiver_completion_reserves_finalization_before_reentrant_call def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", duration_seconds=60, @@ -2036,7 +17057,7 @@ def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.mark_ready() service.state() runtime.pcl_frames = 1 @@ -2044,7 +17065,7 @@ def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) assert acquiring["acquisition"]["state"] == "acquiring" awaiting = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") + _stop_request(acquisition_id=acquisition_id, mode="graceful") ) operation_id = awaiting["last_operation"]["operation_id"] @@ -2053,7 +17074,7 @@ def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) assert awaiting["last_operation"]["status"] == "operator_action_required" retried = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, operation_id=operation_id, mode="graceful", @@ -2065,7 +17086,7 @@ def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) with pytest.raises(ValueError, match="исходную stop-operation"): service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, mode="graceful", operator_confirmed=True, @@ -2073,7 +17094,7 @@ def test_graceful_stop_waits_for_explicit_operator_confirmation(tmp_path: Path) ) completed = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, operation_id=operation_id, mode="graceful", @@ -2096,21 +17117,21 @@ def test_graceful_stop_retry_by_idempotency_key_reuses_original_operation( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.mark_ready() service.state() runtime.pcl_frames = 1 service.state() first = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, idempotency_key="graceful-stop-once", mode="graceful", @@ -2118,7 +17139,7 @@ def test_graceful_stop_retry_by_idempotency_key_reuses_original_operation( ) first_operation_id = first["last_operation"]["operation_id"] retried = service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, idempotency_key="graceful-stop-once", mode="graceful", @@ -2137,28 +17158,26 @@ def test_unrelated_graceful_stop_is_rejected_while_confirmation_is_pending( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.mark_ready() service.state() runtime.pcl_frames = 1 service.state() - first = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") - ) + first = service.stop_acquisition(_stop_request(acquisition_id=acquisition_id, mode="graceful")) expected_operation_id = first["last_operation"]["operation_id"] with pytest.raises(ValueError, match="уже ожидает"): service.stop_acquisition( - StopAcquisitionRequest( + _stop_request( acquisition_id=acquisition_id, - idempotency_key="unrelated-stop", + idempotency_key="unrelated-stop-request", mode="graceful", ) ) @@ -2176,19 +17195,17 @@ def test_graceful_stop_is_rejected_until_point_data_confirms_acquisition( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) with pytest.raises(ValueError, match="подтверждённого потока point cloud"): - service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") - ) + service.stop_acquisition(_stop_request(acquisition_id=acquisition_id, mode="graceful")) state = service.state() assert runtime.stop_calls == 0 @@ -2206,7 +17223,7 @@ def test_prepare_rejects_unsupported_evidence_policies( with pytest.raises(ValueError, match="evidence_policy=required"): service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", evidence_policy=evidence_policy, # type: ignore[arg-type] @@ -2232,7 +17249,7 @@ def test_prepare_rejects_stream_subsets_and_duplicates( with pytest.raises(ValueError, match="полный проверенный набор"): service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", requested_streams=requested_streams, # type: ignore[arg-type] @@ -2259,7 +17276,7 @@ def test_exact_profile_is_inactive_until_selected_for_live_device_info_verificat assert initial["device_calibration"]["compatibility_profile_id"] is None attested = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -2281,7 +17298,7 @@ def test_prepare_rejects_device_ap_without_completed_quick_connect(tmp_path: Pat with pytest.raises(ValueError, match="connection flow"): service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.56.1", compatibility_attestation=QUICK_CONNECT_ATTESTATION, @@ -2296,20 +17313,20 @@ def test_prepare_rejects_device_ap_without_completed_quick_connect(tmp_path: Pat def test_runtime_error_terminalizes_pending_graceful_stop(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.mark_ready() service.state() runtime.pcl_frames = 1 service.state() awaiting = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") + _stop_request(acquisition_id=acquisition_id, mode="graceful") ) stop_operation_id = awaiting["last_operation"]["operation_id"] @@ -2332,20 +17349,20 @@ def test_receiver_completion_terminalizes_unconfirmed_graceful_stop( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.mark_ready() service.state() runtime.pcl_frames = 1 service.state() awaiting = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") + _stop_request(acquisition_id=acquisition_id, mode="graceful") ) stop_operation_id = awaiting["last_operation"]["operation_id"] @@ -2372,18 +17389,18 @@ def test_unconfirmed_stop_operation_terminalizes_before_clock_seal_error( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.pcl_frames = 1 service.state() awaiting = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") + _stop_request(acquisition_id=acquisition_id, mode="graceful") ) stop_operation_id = awaiting["last_operation"]["operation_id"] out_dir = service._acquisition_out_dir # noqa: SLF001 @@ -2415,18 +17432,18 @@ def test_unconfirmed_stop_operation_terminalizes_before_clock_seal_error( def test_abort_failure_terminalizes_acquisition_and_pending_start(tmp_path: Path) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.stop_error = RuntimeError("synthetic receiver stop failure") with pytest.raises(RuntimeError, match="synthetic receiver stop failure"): - service.abort_acquisition(AbortAcquisitionRequest(acquisition_id=acquisition_id)) + service.abort_acquisition(_abort_request(acquisition_id=acquisition_id)) state = service.state() operations = {item["action"]: item for item in state["operations"]} @@ -2446,14 +17463,14 @@ def test_abort_reserves_stopping_before_reentrant_runtime_callback( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.pcl_frames = 1 service.state() out_dir = service._acquisition_out_dir # noqa: SLF001 @@ -2481,7 +17498,7 @@ def test_abort_reserves_stopping_before_reentrant_runtime_callback( lambda _capture_root: events.append("seal"), ) - aborted = service.abort_acquisition(AbortAcquisitionRequest(acquisition_id=acquisition_id)) + aborted = service.abort_acquisition(_abort_request(acquisition_id=acquisition_id)) assert aborted["acquisition"]["state"] == "aborted" assert events == ["camera", "runtime", "seal"] @@ -2494,14 +17511,14 @@ def test_clean_close_seals_and_cancels_pending_start_operation( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) out_dir = service._acquisition_out_dir # noqa: SLF001 assert out_dir is not None out_dir.mkdir() @@ -2542,18 +17559,18 @@ def test_clean_close_cancels_pending_external_stop_operation( ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) runtime.pcl_frames = 1 service.state() awaiting = service.stop_acquisition( - StopAcquisitionRequest(acquisition_id=acquisition_id, mode="graceful") + _stop_request(acquisition_id=acquisition_id, mode="graceful") ) stop_operation_id = awaiting["last_operation"]["operation_id"] monkeypatch.setattr(service.camera_preview, "close", lambda: None) @@ -2573,20 +17590,180 @@ def test_clean_close_cancels_pending_external_stop_operation( ) +@pytest.mark.parametrize("cleanup_failure", [False, True], ids=["clean", "cleanup-failed"]) +def test_close_settles_exact_prepared_stop_as_not_dispatched_after_worker_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cleanup_failure: bool, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + monkeypatch.setattr(service.camera_preview, "close", lambda: None) + if cleanup_failure: + monkeypatch.setattr( + runtime, + "close", + lambda: (_ for _ in ()).throw(RuntimeError("synthetic shutdown cleanup failure")), + ) + else: + monkeypatch.setattr(runtime, "close", runtime.stop) + + if cleanup_failure: + with pytest.raises(RuntimeError, match="synthetic shutdown cleanup failure"): + service.close() + else: + service.close() + + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-classified-not-dispatched" + assert operation.error is not None + assert operation.error["side_effect_status"] == "none" + assert operation.error["physical_command_sent"] is False + assert operation.error["automatic_replay_allowed"] is False + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + + +def test_close_never_classifies_stop_none_after_dispatch_boundary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=fixture.envelope.topic, + payload_sha256=fixture.envelope.payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + fixture.coordinator.publish_dispatching(dispatch) + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + monkeypatch.setattr(service.camera_preview, "close", lambda: None) + monkeypatch.setattr(runtime, "close", runtime.stop) + service.close() + + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "dispatching" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "failed" + assert operation.stage_code == "physical-stop-outcome-unknown-local-retirement" + assert operation.error is not None + assert operation.error["side_effect_status"] == "unknown" + assert operation.error["automatic_replay_allowed"] is False + + +def test_close_with_live_prepared_worker_and_busy_dispatch_gate_stays_conservative( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_real_prepared_stop_dispatch_fixture(service, runtime) + service.stop_acquisition(fixture.request) + monkeypatch.setattr(service.camera_preview, "close", lambda: None) + monkeypatch.setattr(runtime, "close", runtime.stop) + + assert service._k1_command_dispatch_gate.acquire(blocking=False) # noqa: SLF001 + try: + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="publish-переходом STOP", + ) as raised: + service.close() + finally: + service._k1_command_dispatch_gate.release() # noqa: SLF001 + + assert raised.value.reason_code == "acquisition-stop-dispatch-retirement-pending" + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.stage_code == "awaiting-external-stop" + assert operation.error is None + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "awaiting_external_stop" # noqa: SLF001 + assert runtime.stop_calls == 0 + + +def test_close_defers_real_worker_queued_before_dispatch_without_tearing_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + fixture = _install_actual_worker_recovered_stop_fixture( + service, + runtime, + monkeypatch, + ) + service.stop_acquisition(fixture.request) + assert fixture.transport.before_transport_guard.wait(3.0) + cleanup_calls: list[str] = [] + monkeypatch.setattr( + service.camera_preview, + "close", + lambda: cleanup_calls.append("camera"), + ) + monkeypatch.setattr(runtime, "close", lambda: cleanup_calls.append("runtime")) + + with pytest.raises( + facade_module.LocalAcquisitionLifecycleError, + match="не завершил PREPARED STOP", + ) as raised: + service.close() + + assert raised.value.reason_code == "acquisition-stop-worker-retirement-pending" + assert cleanup_calls == [] + assert fixture.transport.fake_client_publish_calls == [] + assert fixture.transport.publish_attempts == 0 + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.error is None + record = fixture.ledger.snapshot().record + assert record is not None and record.stage == "prepared" + assert service._acquisition_stop_operation_id == fixture.stop_operation_id # noqa: SLF001 + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "failed" # noqa: SLF001 + assert service._acquisition.result is not None # noqa: SLF001 + assert service._acquisition.result["recovery_only"] is True # noqa: SLF001 + assert runtime.stop_calls == 0 + + # The shutdown attempt explicitly failed, so the still-owned operation may + # resume. Its eventual publish is retained as ambiguous evidence, never + # retroactively labelled cancelled or not-dispatched. + fixture.transport.release_fake_publish.set() + fixture.transport.release_transport_guard.set() + closed = _wait_control_state(fixture.control, {"closed"}) + assert closed["state"] == "closed" + assert len(fixture.transport.fake_client_publish_calls) == 1 + record = fixture.ledger.snapshot().record + assert record is not None and record.stage in {"dispatching", "observing"} + operation = service._operations.get(fixture.stop_operation_id) # noqa: SLF001 + assert operation.status == "running" + assert operation.stage_code != "physical-stop-classified-not-dispatched" + + def test_close_failure_fails_pending_operation_and_retains_lease( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) acquisition_id = prepared["acquisition"]["acquisition_id"] - service.start_acquisition(StartAcquisitionRequest(acquisition_id=acquisition_id)) + service.start_acquisition(_start_request(acquisition_id=acquisition_id)) monkeypatch.setattr(service.camera_preview, "close", lambda: None) monkeypatch.setattr( runtime, @@ -2614,7 +17791,7 @@ def test_close_failure_fails_pending_operation_and_retains_lease( def test_replay_is_rejected_during_nonterminal_acquisition(tmp_path: Path) -> None: service, _ = service_with_fake_runtime(tmp_path) service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -2637,7 +17814,7 @@ def test_prepare_rejects_active_replay_without_stopping_or_replacing_it( with pytest.raises(RuntimeError, match="активного live/replay"): service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -2656,11 +17833,16 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) _set_scanned_devices( service, [{"device_id": "k1-a"}, {"device_id": "k1-b"}], ) - boundary_calls: list[tuple[str, str, str]] = [] + boundary_calls: list[tuple[str, str, str, str]] = [] async def scenario() -> dict[str, Any]: entered = asyncio.Event() @@ -2670,9 +17852,13 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b device_id: str, ssid: str, password: str, + *, + write_mode: str, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, **_: object, ) -> dict[str, Any]: - boundary_calls.append((device_id, ssid, password)) + boundary_calls.append((device_id, ssid, password, write_mode)) + _dispatch_test_network_write(on_write_dispatch) entered.set() await release.wait() return { @@ -2680,18 +17866,25 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b "completed_at_utc": "2026-07-16T12:00:01Z", "profile_id": "xgrids-k1-fw3-wifi-v1", "outcome": "lan_address_observed", - "observations": [{"status": {"ipv4": "192.168.1.20"}}], + "observations": [{"status": _wifi_status_read("192.168.1.20")["status"]}], } monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) monkeypatch.setattr( facade_module, "_host_route_class", lambda _target: "direct-or-routed", ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) first = asyncio.create_task( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -2700,9 +17893,9 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b ) ) await asyncio.wait_for(entered.wait(), timeout=1.0) - with pytest.raises(RuntimeError, match="уже выполняется"): + with pytest.raises(facade_module.ProvisioningAlreadyRunning): await service.connect( - ConnectRequest( + _connect_request( device_id="k1-b", ssid="other-network", password=SecretStr(SECONDARY_TEST_CREDENTIAL), @@ -2714,14 +17907,902 @@ def test_network_provisioning_is_single_flight_and_secret_is_unwrapped_only_at_b connected = asyncio.run(scenario()) - assert boundary_calls == [("k1-a", "lab-network", PRIMARY_TEST_CREDENTIAL)] + assert boundary_calls == [("k1-a", "lab-network", PRIMARY_TEST_CREDENTIAL, "with_response")] assert connected["k1_ip"] == "192.168.1.20" assert connected["connection_mode"] == "bridge" assert PRIMARY_TEST_CREDENTIAL not in str(connected) provision_operations = [ item for item in connected["operations"] if item["action"] == "network.provision" ] - assert {item["status"] for item in provision_operations} == {"succeeded", "failed"} + assert len(provision_operations) == 1 + assert provision_operations[0]["status"] == "succeeded" + attempt = connected["connection_attempt"] + assert attempt["attempt_id"] == provision_operations[0]["operation_id"] + assert attempt["connection_mode"] == "bridge" + assert attempt["side_effect_status"] == "applied" + assert attempt["safe_next_action"] == "start-acquisition" + assert [event["stage_code"] for event in attempt["timeline"]] == [ + "accepted", + "scan-selection-admitted", + "ble-provisioning-write", + "ble-write-dispatched", + "status-observing", + "device-topology-applied", + "host-wifi-switch-not-authorized", + "network-configured", + "accepted", + "host-route-and-control-endpoint", + "device-info-confirmed", + ] + diagnostic_bundle = attempt["diagnostic_bundle"] + assert diagnostic_bundle["redacted"] is True + assert diagnostic_bundle["automatic_retry"] is False + assert PRIMARY_TEST_CREDENTIAL not in json.dumps(diagnostic_bundle) + assert "lab-network" not in json.dumps(diagnostic_bundle) + + switched = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=connected["desired_connection_mode_revision"], + ) + ) + assert switched["connection_lifecycle"]["ready_to_start"] is False + assert switched["connection_attempt"]["status"] == "succeeded" + assert switched["connection_attempt"]["safe_next_action"] == "scan-select-connect" + assert ( + switched["connection_attempt"]["diagnostic_bundle"]["attempt"]["safe_next_action"] + == "scan-select-connect" + ) + + cancelled = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=switched["desired_connection_mode_revision"], + ) + ) + assert cancelled["connection_lifecycle"]["ready_to_start"] is True + assert cancelled["connection_attempt"]["safe_next_action"] == "start-acquisition" + assert ( + cancelled["connection_attempt"]["diagnostic_bundle"]["attempt"]["safe_next_action"] + == "start-acquisition" + ) + + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + disconnected = cancelled + for _ in range(3): + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + disconnected = service.state() + + assert invalidations == [True] + assert disconnected["connection_lifecycle"]["active_mode"] is None + assert disconnected["connection_lifecycle"]["ready_to_start"] is False + assert disconnected["connection_attempt"]["status"] == "succeeded" + assert disconnected["connection_attempt"]["safe_next_action"] == ("scan-select-connect") + assert ( + disconnected["connection_attempt"]["diagnostic_bundle"]["attempt"]["safe_next_action"] + == "scan-select-connect" + ) + + +def test_replay_cannot_start_while_network_transition_owns_lifecycle_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + + async def scenario() -> None: + entered = asyncio.Event() + release = asyncio.Event() + + async def held_network_write( + *_args: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + _dispatch_test_network_write(on_write_dispatch) + entered.set() + await release.wait() + return { + "started_at_utc": "2026-08-08T12:00:00Z", + "completed_at_utc": "2026-08-08T12:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", held_network_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + network_task = asyncio.create_task( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + await asyncio.wait_for(entered.wait(), timeout=1.0) + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + await asyncio.to_thread( + service.start_replay, + "sessions/fixture.k1mqtt", + 1.0, + False, + ) + assert runtime.phase == "idle" + assert runtime.source_mode == "idle" + release.set() + await asyncio.wait_for(network_task, timeout=1.0) + + asyncio.run(scenario()) + + +def test_network_transition_cannot_enter_while_replay_start_owns_lifecycle_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + replay_path = service.repository_root / "sessions" / "fixture.k1mqtt" + replay_path.parent.mkdir(parents=True, exist_ok=True) + replay_path.write_bytes(b"fixture") + entered = threading.Event() + release = threading.Event() + + def held_replay_start(_path: Path, *, speed: float, loop: bool) -> None: + assert speed == 1.0 + assert loop is False + runtime.phase = "replay" + runtime.source_mode = "replay" + entered.set() + assert release.wait(timeout=1.0) + + monkeypatch.setattr(runtime, "start_replay", held_replay_start, raising=False) + replay_errors: list[BaseException] = [] + + def replay_worker() -> None: + try: + service.start_replay(str(replay_path), speed=1.0, loop=False) + except BaseException as exc: # pragma: no cover - assertion captures it below + replay_errors.append(exc) + + thread = threading.Thread(target=replay_worker, daemon=True) + thread.start() + assert entered.wait(timeout=1.0) + try: + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-lifecycle-busy" + finally: + release.set() + thread.join(timeout=1.0) + + assert not thread.is_alive() + assert replay_errors == [] + assert runtime.phase == "replay" + assert runtime.source_mode == "replay" + + +def test_exact_network_idempotency_replay_precedes_expired_candidate_checks( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices(service, [{"device_id": "k1-a"}]) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "direct-or-routed", + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) + writes = 0 + + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-06T08:00:00Z", + "completed_at_utc": "2026-08-06T08:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [{"status": _wifi_status_read("192.168.1.20")["status"]}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + request = _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="one-browser-intent", + ) + asyncio.run(service.connect(request)) + journal_path = service._require_network_provisioning_idempotency_journal().path # noqa: SLF001 + journal_document = journal_path.read_text(encoding="utf-8") + assert "one-browser-intent" not in journal_document + assert "lab-network" not in journal_document + assert PRIMARY_TEST_CREDENTIAL not in journal_document + service._devices = [] # noqa: SLF001 + service._ble_device_last_seen_monotonic = {} # noqa: SLF001 + service._ble_device_last_seen_suspend_aware = {} # noqa: SLF001 + + replayed = asyncio.run(service.connect(request)) + + restarted_service, _ = service_with_fake_runtime(tmp_path) + restarted = asyncio.run(restarted_service.connect(request)) + + with pytest.raises(NetworkProvisioningIdempotencyConflict): + asyncio.run( + restarted_service.connect(request.model_copy(update={"ssid": "different-network"})) + ) + + with pytest.raises(NetworkProvisioningIdempotencyConflict): + asyncio.run( + restarted_service.connect(request.model_copy(update={"allow_host_wifi_switch": True})) + ) + + assert writes == 1 + operations = [item for item in replayed["operations"] if item["action"] == "network.provision"] + assert len(operations) == 1 + assert operations[0]["status"] == "succeeded" + restarted_operation = next( + item for item in restarted["operations"] if item["action"] == "network.provision" + ) + assert restarted_operation["operation_id"] == operations[0]["operation_id"] + assert restarted_operation["status"] == "succeeded" + assert restarted_operation["stage_code"] == "durable-terminal-replay" + assert restarted_operation["result"]["phase"] == "network_applied" + assert restarted_operation["result"]["control_state"] == "unknown" + assert restarted_operation["result"]["replay_binding_available"] is True + assert restarted_operation["result"]["snapshot_runtime_id"] == restarted["snapshot_runtime_id"] + assert restarted_operation["result"]["parent_intent_id"] == restarted_operation["operation_id"] + assert restarted_operation["result"]["transport_ref"] == "k1-a" + assert restarted_operation["result"]["connection_mode"] == "bridge" + assert restarted_operation["result"]["target_ipv4"] == "192.168.1.20" + assert restarted["connection_attempt"]["phase"] == "network_applied" + assert restarted["connection_attempt"]["control_state"] == "unknown" + + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda _target: facade_module.TcpReachabilityProbeResult(reachable=True), + ) + verified = asyncio.run(restarted_service.verify_connection(ConnectionVerifyRequest())) + + assert writes == 1 + assert verified["last_operation"]["action"] == "connection.verify" + assert verified["last_operation"]["status"] == "succeeded" + assert verified["connection_attempt"]["attempt_id"] == restarted_operation["operation_id"] + assert verified["connection_attempt"]["phase"] == "network_applied" + assert verified["connection_attempt"]["control_state"] == "ready" + assert verified["connection_attempt"]["safe_next_action"] == "start-acquisition" + + +def test_successful_session_requires_new_scan_generation_before_next_connect( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + targets = iter(["192.168.68.50", "192.168.68.51"]) + writes = 0 + + async def successful_write( + *_args: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + target = next(targets) + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-08T13:00:00Z", + "completed_at_utc": "2026-08-08T13:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [{"status": _wifi_status_read(target, device_id="k1-a")["status"]}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + first = asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + first_session = first["device_session"]["device_session_id"] + + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-candidate-not-fresh" + assert service._device_session_id == first_session # noqa: SLF001 + assert service._selected_device_id == "k1-a" # noqa: SLF001 + assert writes == 1 + + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + with pytest.raises(RuntimeError, match="не сообщило адрес"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + expected_discovery_generation=1, + ) + ) + ) + assert service._device_session_id is None # noqa: SLF001 + assert service._selected_device_id is None # noqa: SLF001 + assert writes == 2 + + +def test_operation_admission_failure_terminalizes_prepared_network_intent( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + def fail_operation_admission(*_args: object, **_kwargs: object) -> object: + raise RuntimeError("synthetic operation journal failure") + + monkeypatch.setattr(service._operations, "begin", fail_operation_admission) # noqa: SLF001 + request = _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="operation-admission-failure", + ) + + with pytest.raises(RuntimeError, match="operation journal failure"): + asyncio.run(service.connect(request)) + + snapshot = service._require_network_provisioning_idempotency_journal().snapshot() # noqa: SLF001 + assert snapshot.active_record is None + assert len(snapshot.records) == 1 + record = snapshot.records[0] + assert record.stage == "terminal" + assert record.terminal is not None + assert record.terminal.outcome == "failed" + assert record.terminal.outcome_code == "network.provision.admission_failed" + assert record.terminal.error_code == "operation-journal-admission-failed" + assert record.terminal.side_effect_status == "none" + assert record.terminal.safe_to_retry is True + + +def test_live_process_lease_prevents_false_restart_recovery_of_prepared_request( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + request = _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="live-owner-prepared-request", + ) + canonical_request = json.dumps( + { + "schema_version": "missioncore.xgrids-k1-network-provision-request/v1", + "device_id": request.device_id, + "ssid": request.ssid, + "password": PRIMARY_TEST_CREDENTIAL, + "connection_mode": request.connection_mode, + "expected_mode_revision": request.expected_mode_revision, + "expected_discovery_generation": request.expected_discovery_generation, + "expected_reconfiguration_revision": (request.expected_reconfiguration_revision), + "expected_reconfiguration_intent_id": (request.expected_reconfiguration_intent_id), + "compatibility_attestation": request.compatibility_attestation.model_dump(mode="json"), + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + owner_lease = ApplicationControlProcessLease.acquire(tmp_path) + try: + journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + operation_id = "op-11111111-2222-4333-8444-555555555555" + admitted = journal.begin( + idempotency_key=request.idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + operation_id=operation_id, + request_binding_sha256=derive_request_binding_sha256( + request.idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + canonical_request=canonical_request, + ), + ) + assert admitted.record.stage == "prepared" + + competing_service, _ = service_with_fake_runtime(tmp_path) + still_live = journal.snapshot().active_record + assert still_live is not None + assert still_live.operation_id == operation_id + assert still_live.stage == "prepared" + finally: + owner_lease.release() + + # The already-running competing facade may take over only after the OS + # lease proves that the original process is gone. It replays a durable, + # no-side-effect terminal result without touching BLE. + replayed = asyncio.run(competing_service.connect(request)) + terminal = journal.snapshot().records[-1] + operation = next( + item for item in replayed["operations"] if item["action"] == "network.provision" + ) + assert terminal.stage == "terminal" + assert terminal.terminal is not None + assert terminal.terminal.side_effect_status == "none" + assert terminal.terminal.safe_to_retry is True + assert operation["operation_id"] == operation_id + assert operation["status"] == "failed" + assert operation["stage_code"] == "durable-terminal-replay" + + +@pytest.mark.parametrize("legacy_stage", ["dispatching", "observing"]) +def test_restart_terminalizes_dispatched_legacy_ledger_as_interrupted_audit( + tmp_path: Path, + legacy_stage: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + operation_id = f"legacy-{legacy_stage}-operation" + _seed_legacy_network_mutation_without_idempotency( + service, + operation_id=operation_id, + stage=legacy_stage, + ) + journal_path = service._require_network_provisioning_idempotency_journal().path # noqa: SLF001 + assert not journal_path.exists() + + restarted, _ = service_with_fake_runtime(tmp_path) + idempotency = restarted._require_network_provisioning_idempotency_journal().snapshot() # noqa: SLF001 + + assert idempotency.status == "ready" + assert idempotency.active_record is None + terminal = idempotency.records[-1] + assert terminal.operation_id == operation_id + assert terminal.action == facade_module.ACTION_NETWORK_PROVISION + assert terminal.stage == "terminal" + assert terminal.terminal is not None + assert terminal.terminal.outcome_code == "network.provision.interrupted" + assert terminal.terminal.side_effect_status == "reconciled" + assert terminal.terminal.safe_to_retry is False + persisted = journal_path.read_text(encoding="utf-8") + assert "test-ble-transport" not in persisted + assert "WIFI_CLIENT" not in persisted + assert '"stage":"terminal"' in persisted + state = restarted.state() + assert state["network_mutation_ledger"]["stage"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_write_reconciliation"] is None + + +def test_restart_resolves_legacy_prepared_as_not_dispatched_without_adoption( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_legacy_network_mutation_without_idempotency( + service, + operation_id="legacy-prepared-operation", + stage="prepared", + ) + journal_path = service._require_network_provisioning_idempotency_journal().path # noqa: SLF001 + assert not journal_path.exists() + + restarted, _ = service_with_fake_runtime(tmp_path) + state = restarted.state() + + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "not-dispatched" + assert state["network_write_reconciliation"] is None + assert state["network_provisioning_idempotency"]["status"] == "empty" + assert state["network_provisioning_idempotency"]["active_operation_id"] is None + assert not journal_path.exists() + + +def test_restart_supersedes_cross_journal_mismatch_without_blocking_new_intent( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + service, + operation_id="legacy-ledger-operation", + idempotency_operation_id="other-idempotency-operation", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + state = restarted.state() + + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["operation_id"] == "legacy-ledger-operation" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_provisioning_idempotency"]["status"] == "ready" + assert state["network_provisioning_idempotency"]["reason_code"] is None + assert state["network_provisioning_idempotency"]["mutation_allowed"] is True + + +def test_new_explicit_intent_is_admitted_after_interrupted_dispatched_session( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + service, + operation_id="interrupted-operation", + idempotency_operation_id="interrupted-operation", + ) + journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + + recovered = facade_module._recover_network_provisioning_under_exclusive_process_lease( # noqa: SLF001 + network_ledger=service._network_mutation_ledger, # noqa: SLF001 + idempotency_journal=journal, + ) + admitted = journal.begin( + idempotency_key="new-explicit-operator-intent", + action=facade_module.ACTION_NETWORK_PROVISION, + operation_id="new-explicit-operation", + request_binding_sha256="a" * 64, + ) + + assert recovered.status == "resolved" + assert recovered.record is not None + assert recovered.record.resolution == "interrupted" + assert admitted.disposition == "admitted" + assert admitted.record.stage == "prepared" + + +def test_restart_legacy_adoption_preserves_corrupt_journal_and_fails_closed( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_legacy_network_mutation_without_idempotency(service) + journal_path = service._require_network_provisioning_idempotency_journal().path # noqa: SLF001 + corrupt = b'{"schema_version":"corrupt","records":[]}\n' + journal_path.write_bytes(corrupt) + journal_path.chmod(0o600) + + restarted, _ = service_with_fake_runtime(tmp_path) + state = restarted.state() + + assert journal_path.read_bytes() == corrupt + assert state["network_mutation_ledger"]["status"] == "unresolved" + assert state["network_provisioning_idempotency"]["status"] == "corrupt" + assert state["network_provisioning_idempotency"]["reason_code"] == ( + "network-provisioning-idempotency-corrupt" + ) + assert state["network_provisioning_idempotency"]["mutation_allowed"] is False + + +@pytest.mark.parametrize( + "corruption", + [ + "wrong-mode", + "oversize", + "symlink", + "ledger-wrong-mode", + "both-corrupt", + "journal-disappeared", + ], +) +def test_explicit_connect_quarantines_corrupt_network_audit_without_replay( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + corruption: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + path = journal.path + ledger_path = service._network_mutation_ledger.path # noqa: SLF001 + path.parent.mkdir(parents=True, exist_ok=True) + foreign_target = tmp_path / "foreign-audit-target" + if corruption in {"wrong-mode", "both-corrupt", "journal-disappeared"}: + path.write_text('{"schema_version":"wrong"}\n', encoding="utf-8") + path.chmod(0o644) + elif corruption == "oversize": + path.write_bytes(b"x" * (300 * 1024)) + path.chmod(0o600) + else: + if corruption == "symlink": + foreign_target.write_text("foreign", encoding="utf-8") + path.unlink(missing_ok=True) + path.symlink_to(foreign_target) + if corruption in {"ledger-wrong-mode", "both-corrupt"}: + ledger_path.write_text('{"schema_version":"wrong"}\n', encoding="utf-8") + ledger_path.chmod(0o644) + if corruption == "journal-disappeared": + service._network_provisioning_idempotency_journal = None # noqa: SLF001 + service._network_provisioning_idempotency_reason = ( # noqa: SLF001 + "network-provisioning-idempotency-corrupt" + ) + path.unlink() + _set_scanned_k1(service, device_id="k1-a") + + async def successful_write( + *_args: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-08T10:00:00Z", + "completed_at_utc": "2026-08-08T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": _wifi_status_read(None, device_id="k1-a")["status"], + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + + state = asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert state["connection_mode"] == "bridge" + assert state["network_provisioning_idempotency"]["status"] == "ready" + journal_quarantines = list(path.parent.glob(f"{path.name}.corrupt-*")) + ledger_quarantines = list(ledger_path.parent.glob(f"{ledger_path.name}.corrupt-*")) + assert len(journal_quarantines) == ( + 1 + if corruption + in { + "wrong-mode", + "oversize", + "symlink", + "both-corrupt", + } + else 0 + ) + assert len(ledger_quarantines) == ( + 1 + if corruption + in { + "ledger-wrong-mode", + "both-corrupt", + } + else 0 + ) + if corruption == "symlink": + assert foreign_target.read_text(encoding="utf-8") == "foreign" + + +def test_cancelled_bridge_write_is_audited_and_new_explicit_intent_writes_once( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices(service, [{"device_id": "k1-a"}]) + calls = 0 + + async def scenario() -> None: + entered = asyncio.Event() + + async def cancelled_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal calls + calls += 1 + assert on_write_dispatch is not None + on_write_dispatch( + _wifi_status_read(None, device_id="k1-a")["status"], + "with_response", + ) + entered.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError as exc: + exc.operation_stage = "gatt-write" # type: ignore[attr-defined] + exc.device_write_attempted = True # type: ignore[attr-defined] + exc.device_write_confirmed = True # type: ignore[attr-defined] + raise + + async def successful_second_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal calls + calls += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-08T10:00:00Z", + "completed_at_utc": "2026-08-08T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": _wifi_status_read(None, device_id="k1-a")["status"], + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", cancelled_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + first = asyncio.create_task( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="cancelled-write-1", + ) + ) + ) + await entered.wait() + first.cancel() + with pytest.raises(asyncio.CancelledError): + await first + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_second_write) + second = await service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="cancelled-write-2", + ) + ) + assert second["connection_mode"] == "bridge" + assert second["k1_ip"] == "192.168.68.50" + + asyncio.run(scenario()) + + state = service.state() + operation = next( + item for item in state["operations"] if item["idempotency_key"] == "cancelled-write-1" + ) + assert operation["status"] == "failed" + assert operation["error"]["code"] == "CancelledError" + assert operation["error"]["side_effect_status"] == "unknown" + assert operation["error"]["safe_to_retry"] is False + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_write_reconciliation"] is None + assert calls == 2 + + +def test_cancelled_host_mutation_finishes_before_ownership_is_released() -> None: + started = threading.Event() + release = threading.Event() + completed = threading.Event() + + def bounded_host_mutation() -> str: + started.set() + assert release.wait(timeout=2.0) + completed.set() + return "completed" + + async def scenario() -> None: + task = asyncio.create_task( + facade_module._run_blocking_operation_without_abandonment( # noqa: SLF001 + bounded_host_mutation + ) + ) + for _ in range(100): + if started.is_set(): + break + await asyncio.sleep(0.01) + assert started.is_set() + task.cancel() + await asyncio.sleep(0.01) + assert not task.done() + task.cancel() + await asyncio.sleep(0.01) + assert not task.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert completed.is_set() + + asyncio.run(scenario()) def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error( @@ -2729,16 +18810,21 @@ def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error( tmp_path: Path, ) -> None: service, runtime = service_with_fake_runtime(tmp_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, ) ) service.abort_acquisition( - AbortAcquisitionRequest( + _abort_request( acquisition_id=prepared["acquisition"]["acquisition_id"], ) ) @@ -2746,24 +18832,36 @@ def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error( runtime.source_mode = "live" stop_calls_before_connect = runtime.stop_calls - async def fake_provision(*_: object, **__: object) -> dict[str, Any]: + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + _dispatch_test_network_write(on_write_dispatch) return { "started_at_utc": "2026-07-28T18:11:37Z", "completed_at_utc": "2026-07-28T18:11:44Z", "profile_id": "xgrids-k1-fw3-wifi-v1", "outcome": "lan_address_observed", - "observations": [{"status": {"ipv4": "192.168.1.20"}}], + "observations": [{"status": _wifi_status_read("192.168.1.20")["status"]}], } monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) monkeypatch.setattr( facade_module, "_host_route_class", lambda _target: "direct-or-routed", ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) connected = asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -2778,6 +18876,9 @@ def test_bridge_network_change_retires_terminal_acquisition_and_receiver_error( assert connected["acquisition"] is None assert connected["source_mode"] == "idle" assert connected["phase"] == "connected" + assert connected["device_session"]["connectivity"] == "connected" + assert connected["connection_verification"]["lease_state"] == "reachable" + assert connected["connection_lifecycle"]["connection_ready"] is True assert runtime.stop_calls == stop_calls_before_connect + 1 @@ -2787,22 +18888,135 @@ def test_bridge_provisioning_reports_host_route_mismatch_without_hiding_device_s ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) + host_associations: list[tuple[str, str]] = [] - async def fake_provision(*_: object, **__: object) -> dict[str, Any]: + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + _dispatch_test_network_write(on_write_dispatch) return { "started_at_utc": "2026-07-28T19:43:03Z", "completed_at_utc": "2026-07-28T19:43:16Z", "profile_id": "xgrids-k1-fw3-wifi-v1", "outcome": "lan_address_observed", - "observations": [{"status": {"ipv4": "192.168.68.50"}}], + "observations": [{"status": _wifi_status_read("192.168.68.50")["status"]}], } monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel") + monkeypatch.setattr(facade_module, "_inspect_host_path", _tunnel_host_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _unreachable_control_endpoint, + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: False) + monkeypatch.setattr( + facade_module, + "associate_with_ephemeral_wifi_credentials_once", + lambda _helper, ssid, password, **_kwargs: ( + host_associations.append((ssid, password)) + or { + "adapter": "CoreWLAN", + "outcome": "associated", + "already_associated": False, + "scan_attempt_count": 1, + "scan_elapsed_ms": 20, + } + ), + ) - connected = asyncio.run( - service.connect( - ConnectRequest( + async def connect_and_join_read_only_continuation() -> dict[str, Any]: + await service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + allow_host_wifi_switch=True, + ) + ) + with service._lock: # noqa: SLF001 + task = service._control_bootstrap_continuation_task # noqa: SLF001 + if task is not None: + with pytest.raises(facade_module.ConnectionVerificationError): + await task + return service.state() + + connected = asyncio.run(connect_and_join_read_only_continuation()) + + assert host_associations == [("lab-router", PRIMARY_TEST_CREDENTIAL)] + assert connected["k1_ip"] == "192.168.68.50" + assert connected["phase"] == "device_selected" + assert connected["device_session"]["connectivity"] == "offline" + assert connected["connection_verification"] == { + "status": "endpoint-unreachable", + "lease_state": "configured-unverified", + "lease_generation": connected["connection_supervisor"]["lease"]["generation"], + "endpoint_validation": "host-route", + "network_reachability": "unreachable", + "reason_code": "tcp-endpoint-unreachable", + "host_route_class": "tunnel", + "write_performed": True, + "observed_at": connected["connection_verification"]["observed_at"], + "supervisor_revision": connected["connection_verification"]["supervisor_revision"], + } + operation = next( + item for item in connected["operations"] if item["action"] == "network.provision" + ) + assert operation["status"] == "succeeded" + assert operation["result"]["phase"] == "network_applied" + assert operation["result"]["host_wifi_switch_authorized"] is True + assert operation["result"]["host_route_ready"] is None + assert operation["result"]["host_route_class"] is None + assert connected["connection_attempt"]["phase"] == "network_applied" + assert connected["connection_attempt"]["control_state"] == "control_not_ready" + + +def test_bridge_provisioning_separates_address_from_reachable_control_endpoint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) + + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-06T10:00:00Z", + "completed_at_utc": "2026-08-06T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [{"status": _wifi_status_read("192.168.68.50")["status"]}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _unreachable_control_endpoint, + ) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: False, + ) + + async def connect_and_join_read_only_continuation() -> dict[str, Any]: + await service.connect( + _connect_request( device_id="k1-a", ssid="lab-router", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -2810,29 +19024,38 @@ def test_bridge_provisioning_reports_host_route_mismatch_without_hiding_device_s compatibility_attestation=ATTESTATION, ) ) - ) + with service._lock: # noqa: SLF001 + task = service._control_bootstrap_continuation_task # noqa: SLF001 + if task is not None: + with pytest.raises(facade_module.ConnectionVerificationError): + await task + return service.state() - assert connected["k1_ip"] == "192.168.68.50" - assert connected["phase"] == "device_selected" - assert "компьютер подключён к другой сети" in connected["message"] - assert connected["device_session"]["connectivity"] == "offline" - assert connected["connection_verification"] == { - "status": "host-route-mismatch", - "lease_state": "disconnected", - "lease_generation": 1, - "endpoint_validation": "host-route", + state = asyncio.run(connect_and_join_read_only_continuation()) + + assert state["k1_ip"] == "192.168.68.50" + assert state["phase"] == "device_selected" + assert state["device_session"]["connectivity"] == "offline" + assert state["connection_verification"] == { + "status": "endpoint-unreachable", + "lease_state": "configured-unverified", + "lease_generation": state["connection_supervisor"]["lease"]["generation"], + "endpoint_validation": "provisioning-status+mqtt-tcp-connect", "network_reachability": "unreachable", - "reason_code": "connection_lease_host_route_mismatch", - "host_route_class": "tunnel", + "reason_code": "tcp-endpoint-unreachable", + "host_route_class": "direct-or-routed", "write_performed": True, - "observed_at": connected["connection_verification"]["observed_at"], + "observed_at": state["connection_verification"]["observed_at"], + "supervisor_revision": state["connection_verification"]["supervisor_revision"], } - operation = next( - item for item in connected["operations"] if item["action"] == "network.provision" - ) + operation = next(item for item in state["operations"] if item["action"] == "network.provision") assert operation["status"] == "succeeded" - assert operation["result"]["host_route_ready"] is False - assert operation["result"]["host_route_class"] == "tunnel" + assert operation["stage_code"] == "network-configured" + assert operation["result"]["phase"] == "network_applied" + assert operation["result"]["control_state"] == "unknown" + assert operation["result"]["control_endpoint_reachable"] is None + assert state["connection_attempt"]["phase"] == "network_applied" + assert state["connection_attempt"]["control_state"] == "control_not_ready" def test_quick_connect_activates_the_device_ap_then_associates_the_host( @@ -2840,8 +19063,19 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) - activation_calls: list[str] = [] + activation_calls: list[tuple[str, str]] = [] association_calls: list[tuple[Path, str, str]] = [] ble_session_open = False @@ -2858,9 +19092,16 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( ) @asynccontextmanager - async def fake_activation_session(device_id: str, **_: object) -> AsyncIterator[dict[str, Any]]: + async def fake_activation_session( + device_id: str, + *, + write_mode: str, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_: object, + ) -> AsyncIterator[dict[str, Any]]: nonlocal ble_session_open - activation_calls.append(device_id) + activation_calls.append((device_id, write_mode)) + _dispatch_test_network_write(on_write_dispatch) ble_session_open = True try: yield { @@ -2871,7 +19112,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( "ready_observed": True, "write_performed": True, "write_mode": "with_response", - "observations": [{"status": {"mode": "WIFI_AP"}}], + "observations": [{"status": _ap_ready_wifi_status()}], } finally: ble_session_open = False @@ -2892,7 +19133,7 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( "profile_enrolled": True, "scan_attempt_count": 2, "scan_elapsed_ms": 900, - "credential_source": "system-wifi-keychain", + "credential_source": "exact-firmware-profile", } async def forbidden_provisioning_write(*_: object, **__: object) -> dict[str, Any]: @@ -2906,18 +19147,20 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( monkeypatch.setattr(facade_module, "associate_with_wifi_profile_once", fake_associate) monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provisioning_write) monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) state = asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) - assert activation_calls == ["k1-a"] + assert activation_calls == [("k1-a", "with_response")] assert not ble_session_open assert len(association_calls) == 1 assert association_calls[0][0].name == "associate_wifi.swift" @@ -2943,21 +19186,114 @@ def test_quick_connect_activates_the_device_ap_then_associates_the_host( ) == "192.168.56.1" ) + assert state["connection_lifecycle"]["connection_ready"] is True + control = state["application_control_session"] + workspace = service.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=control["session_generation"], + expected_state_revision=control["state_revision"], + ) + ) + workspace_control = workspace["application_control_session"] prepared = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_control_session_generation=workspace_control["session_generation"], + expected_control_state_revision=workspace_control["state_revision"], ) ) assert prepared["acquisition"]["target_host"] == "192.168.56.1" +def test_quick_connect_read_only_verify_requires_fresh_ble_status_without_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + _seed_supervised_connection( + service, + target_ipv4="192.168.56.1", + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-k1-a", + with_control=False, + ) + service._device_session_id = "session-k1-a" # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + supervisor_generation = service._connection_supervisor.snapshot().lease.generation # noqa: SLF001 + service._connection_lease_generation = supervisor_generation # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "status": "endpoint-unreachable", + "lease_state": "disconnected", + "lease_generation": supervisor_generation, + "endpoint_validation": "provisioning-status+mqtt-tcp-connect", + "network_reachability": "unreachable", + "reason_code": "connection_lease_endpoint_unreachable_after_provision", + "write_performed": True, + "observed_at": "2026-08-06T10:00:00Z", + } + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "device-ap") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + status_reads: list[tuple[str, float, bool]] = [] + + async def exact_ble_status( + device_id: str, + *, + timeout_seconds: float, + rediscover: bool, + **_: object, + ) -> dict[str, Any]: + status_reads.append((device_id, timeout_seconds, rediscover)) + status = _wifi_status_read("192.168.56.1", device_id=device_id) + status["status"] = _ap_ready_wifi_status() + return status + + monkeypatch.setattr(facade_module, "read_wifi_status_once", exact_ble_status) + + state = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-a", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert status_reads == [("k1-a", 20.0, False)] + assert state["connection_verification"]["lease_state"] == "reachable" + assert state["connection_verification"]["network_reachability"] == "reachable" + assert state["device_session"]["connectivity"] == "connected" + assert state["connection_verification"]["write_performed"] is False + assert state["connection_verification"]["endpoint_validation"] == ( + "ble-wifi-status-read+mqtt-tcp-connect" + ) + assert state["last_operation"]["action"] == "connection.verify" + assert state["last_operation"]["status"] == "succeeded" + + def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "direct-connect") + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) provisioning_calls: list[tuple[str, str, str]] = [] @@ -2965,15 +19301,18 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame( device_id: str, ssid: str, password: str, + *, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, **_: object, ) -> dict[str, Any]: provisioning_calls.append((device_id, ssid, password)) + _dispatch_test_network_write(on_write_dispatch) return { "started_at_utc": "2026-07-19T01:00:00Z", "completed_at_utc": "2026-07-19T01:00:01Z", "profile_id": "xgrids-k1-fw3-wifi-v1", "outcome": "lan_address_observed", - "observations": [{"status": {"ipv4": "172.20.10.2"}}], + "observations": [{"status": _wifi_status_read("172.20.10.2")["status"]}], } def forbidden_host_association(*_: object, **__: object) -> dict[str, Any]: @@ -2991,15 +19330,22 @@ def test_direct_connect_reuses_the_reviewed_ble_provisioning_frame( "_host_route_class", lambda _target: "direct-or-routed", ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) state = asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="controller-hotspot", password=SecretStr(PRIMARY_TEST_CREDENTIAL), connection_mode="direct-connect", compatibility_attestation=DIRECT_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) @@ -3015,6 +19361,7 @@ def test_quick_connect_missing_credential_provider_stops_before_ap_write( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) service._selected_device_id = "previous-k1" # noqa: SLF001 service._k1_ip = "192.168.1.20" # noqa: SLF001 @@ -3048,19 +19395,24 @@ def test_quick_connect_missing_credential_provider_stops_before_ap_write( with pytest.raises(RuntimeError, match="credential-source-unavailable"): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) state = service.state() - assert state["selected_device_id"] == "previous-k1" - assert state["k1_ip"] == "192.168.1.20" - assert state["connection_mode"] == "bridge" - assert not list(service.evidence_root.glob("*viewer_k1_ap_association*")) + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["k1_ip"] is None + assert state["connection_mode"] is None + failed_sessions = list(service.evidence_root.glob("*viewer_k1_ap_association*")) + assert len(failed_sessions) == 1 + assert (failed_sessions[0] / "operation-reservation.redacted.json").exists() + assert not (failed_sessions[0] / "ap-activation.redacted.json").exists() operation = next(item for item in state["operations"] if item["action"] == "network.provision") assert operation["status"] == "failed" assert operation["error"]["side_effect_status"] == "none" @@ -3072,6 +19424,7 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) monkeypatch.setattr( facade_module, @@ -3086,7 +19439,16 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag( ) @asynccontextmanager - async def not_ready_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]: + async def not_ready_session( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> AsyncIterator[dict[str, Any]]: + assert on_write_dispatch is not None + on_write_dispatch( + _wifi_status_read(None, device_id="k1-a")["status"], + "with_response", + ) yield { "profile_id": "xgrids-k1-fw3-quick-connect-ap-v1", "started_at_utc": "2026-07-19T15:00:00Z", @@ -3115,10 +19477,11 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag( with pytest.raises(RuntimeError, match="не подтвердил готовность"): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) @@ -3129,7 +19492,94 @@ def test_quick_connect_does_not_start_host_wifi_without_ap_ready_flag( assert not (quick_sessions[0] / "manifest.redacted.json").exists() reconciliation = service.state()["network_write_reconciliation"] assert reconciliation["transport_ref"] == "k1-a" - assert reconciliation["status"] == "device-state-unknown-after-write" + assert reconciliation["status"] == "previous-session-result-unknown" + assert reconciliation["reason_code"] == "network-mutation-audit-open" + assert reconciliation["blocks_new_explicit_intent"] is False + assert reconciliation["operation_stage"] == "observing" + assert reconciliation["scope"] == "durable-ledger" + assert reconciliation["ledger_revision"] == 3 + + +def test_quick_connect_unchanged_ready_baseline_does_not_associate_host_wifi( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") + _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) + monkeypatch.setattr( + facade_module, + "ensure_wifi_profile_from_credential_source", + lambda *_args, **_kwargs: { + "schema_version": 1, + "adapter": "macOS Keychain", + "available": True, + "profile_enrolled": True, + "credential_source": "exact-firmware-profile", + }, + ) + ready_status = _ap_ready_wifi_status() + + @asynccontextmanager + async def unchanged_ready_session( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> AsyncIterator[dict[str, Any]]: + assert on_write_dispatch is not None + on_write_dispatch(ready_status, "with_response") + yield { + "profile_id": "xgrids-k1-fw3-quick-connect-ap-v1", + "started_at_utc": "2026-08-06T10:00:00Z", + "completed_at_utc": "2026-08-06T10:00:01Z", + "outcome": "ap_ready_observed", + "baseline_status": ready_status, + "ready_observed": True, + "write_performed": True, + "write_mode": "with_response", + "observations": [{"status": ready_status}], + } + + def forbidden_association(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("baseline/previous AP-ready is not authority to mutate host Wi-Fi") + + monkeypatch.setattr( + facade_module, + "device_ap_activation_session", + unchanged_ready_session, + ) + monkeypatch.setattr( + facade_module, + "associate_with_wifi_profile_once", + forbidden_association, + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + connection_mode="quick-connect", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, + ) + ) + ) + + assert raised.value.reason_code == ( + "network-provision-target-not-distinguishable-from-baseline" + ) + state = service.state() + assert state["network_mutation_ledger"]["status"] == "unresolved" + assert state["network_mutation_ledger"]["stage"] == "observing" + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["error"]["code"] == ( + "network-provision-target-not-distinguishable-from-baseline" + ) + assert operation["error"]["side_effect_status"] == "unknown" + assert operation["error"]["safe_to_retry"] is False def test_failed_connection_change_revokes_the_previous_route( @@ -3137,6 +19587,7 @@ def test_failed_connection_change_revokes_the_previous_route( tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) service._selected_device_id = "previous-k1" # noqa: SLF001 service._k1_ip = "192.168.1.20" # noqa: SLF001 @@ -3154,7 +19605,16 @@ def test_failed_connection_change_revokes_the_previous_route( ) @asynccontextmanager - async def fake_activation_session(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]: + async def fake_activation_session( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> AsyncIterator[dict[str, Any]]: + assert on_write_dispatch is not None + on_write_dispatch( + _wifi_status_read(None, device_id="k1-a")["status"], + "with_response", + ) yield { "profile_id": "xgrids-k1-fw3-quick-connect-ap-v1", "started_at_utc": "2026-07-19T15:00:00Z", @@ -3163,7 +19623,7 @@ def test_failed_connection_change_revokes_the_previous_route( "ready_observed": True, "write_performed": True, "write_mode": "with_response", - "observations": [{"status": {"mode": "WIFI_AP"}}], + "observations": [{"status": _ap_ready_wifi_status()}], } def failed_association(*_: object, **__: object) -> dict[str, Any]: @@ -3183,19 +19643,28 @@ def test_failed_connection_change_revokes_the_previous_route( with pytest.raises(RuntimeError, match="network-not-found"): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) state = service.state() - assert state["selected_device_id"] is None - assert state["k1_ip"] is None - assert state["connection_mode"] is None - assert state["compatibility"]["attestation"] is None + # AP-ready is a semantic device-side commit. Host association may fail, + # but the old Bridge route must not be resurrected and the new Quick + # topology remains explicit, configured and offline. + assert state["selected_device_id"] == "k1-a" + assert state["k1_ip"] == facade_module.AP_FALLBACK_IPV4 + assert state["connection_mode"] == "quick-connect" + assert state["compatibility"]["attestation"]["topology"] == "device-ap" + assert state["device_session"]["connectivity"] == "offline" + assert state["connection_verification"]["lease_state"] == ("configured-unverified") + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["error"]["scan_attempt_count"] == 4 + assert operation["error"]["scan_elapsed_ms"] == 15014 quick_sessions = sorted(service.evidence_root.glob("*viewer_k1_ap_association*")) failure_evidence = json.loads( (quick_sessions[0] / "host-wifi-association.redacted.json").read_text(encoding="utf-8") @@ -3205,11 +19674,12 @@ def test_failed_connection_change_revokes_the_previous_route( assert failure_evidence["scan_elapsed_ms"] == 15014 -def test_quick_connect_helper_build_failure_after_ap_write_preserves_side_effect_facts( +def test_quick_connect_keychain_failure_after_ap_write_preserves_side_effect_facts( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, "quick-connect") _set_scanned_devices(service, [{"device_id": "k1-a", "name": "XGR-TEST-A"}]) monkeypatch.setattr( facade_module, @@ -3224,7 +19694,16 @@ def test_quick_connect_helper_build_failure_after_ap_write_preserves_side_effect ) @asynccontextmanager - async def ready_activation(*_: object, **__: object) -> AsyncIterator[dict[str, Any]]: + async def ready_activation( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> AsyncIterator[dict[str, Any]]: + assert on_write_dispatch is not None + on_write_dispatch( + _wifi_status_read(None, device_id="k1-a")["status"], + "with_response", + ) yield { "profile_id": "xgrids-k1-fw3-quick-connect-ap-v1", "started_at_utc": "2026-08-06T10:00:00Z", @@ -3233,33 +19712,32 @@ def test_quick_connect_helper_build_failure_after_ap_write_preserves_side_effect "ready_observed": True, "write_performed": True, "write_mode": "with_response", - "observations": [{"status": {"mode": "WIFI_AP"}}], + "observations": [{"status": _ap_ready_wifi_status()}], } - def failed_post_write_build(*_: object, **__: object) -> dict[str, Any]: + def failed_post_write_profile_read(*_: object, **__: object) -> dict[str, Any]: raise facade_module.HostWifiProfileError( - "host-wifi-helper-build-failed", - helper_stage="compile", - helper_elapsed_ms=21, + "keychain-authorization-required", ) monkeypatch.setattr(facade_module, "device_ap_activation_session", ready_activation) monkeypatch.setattr( facade_module, "associate_with_wifi_profile_once", - failed_post_write_build, + failed_post_write_profile_read, ) with pytest.raises( facade_module.HostWifiProfileError, - match="host-wifi-helper-build-failed", + match="keychain-authorization-required", ): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", connection_mode="quick-connect", compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=mode_revision, ) ) ) @@ -3267,10 +19745,11 @@ def test_quick_connect_helper_build_failure_after_ap_write_preserves_side_effect state = service.state() operation = next(item for item in state["operations"] if item["action"] == "network.provision") assert operation["stage_code"] == "host-wifi-association-failed" - assert operation["error"]["code"] == "host-wifi-helper-build-failed" + assert operation["error"]["code"] == "keychain-authorization-required" + assert operation["error"]["host_diagnostic"]["code"] == ("host.keychain.interaction-required") assert operation["error"]["side_effect_status"] == "confirmed" assert operation["error"]["safe_to_retry"] is False - assert operation["error"]["helper_stage"] == "compile" + assert "helper_stage" not in operation["error"] assert state["network_write_reconciliation"] is None @@ -3282,13 +19761,31 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host( service, _ = service_with_fake_runtime(tmp_path) _set_scanned_devices(service, [{"device_id": "k1-a"}]) - async def fake_provision(*_: object, **__: object) -> dict[str, Any]: + async def fake_provision( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + baseline_status = _ap_ready_wifi_status() + assert on_write_dispatch is not None + on_write_dispatch(baseline_status, "with_response") return { "started_at_utc": "2026-07-18T15:19:25Z", "completed_at_utc": "2026-07-18T15:19:26Z", "profile_id": "xgrids-k1-fw3-wifi-v1", "outcome": "lan_address_observed", - "observations": [{"status": {"ipv4": "10.255.254.51"}}], + "write_mode": "with_response", + "baseline_status": baseline_status, + "observations": [ + { + "status": { + "mode": "WIFI_CLIENT", + "ipv4": "10.255.254.51", + "status_code": 1, + "reserved": 0, + } + } + ], } monkeypatch.setattr(facade_module, "provision_wifi_once", fake_provision) @@ -3300,7 +19797,7 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host( ): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -3310,8 +19807,13 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host( ) state = service.state() - assert state["selected_device_id"] is None - assert state["k1_ip"] is None + # The K1 already confirmed its new device-side topology before the local + # address conflict was discovered. Preserve that truth while withholding + # all host/control authority. + assert state["selected_device_id"] == "k1-a" + assert state["k1_ip"] == "10.255.254.51" + assert state["connection_mode"] == "bridge" + assert state["device_session"]["connectivity"] == "offline" operation = next(item for item in state["operations"] if item["action"] == "network.provision") assert operation["status"] == "failed" assert operation["error"]["safe_to_retry"] is False @@ -3334,31 +19836,51 @@ def test_network_provisioning_rejects_an_ipv4_owned_by_the_local_host( assert "lab-network" not in caplog.text -def test_ambiguous_ble_write_blocks_new_network_mutation_until_reconciled( +def test_ambiguous_ble_write_is_audit_only_for_next_explicit_intent( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, + caplog: pytest.LogCaptureFixture, ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_devices(service, [{"device_id": "k1-a"}]) calls = 0 - async def ambiguous_write(*_: object, **__: object) -> dict[str, Any]: + async def ambiguous_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: nonlocal calls calls += 1 + assert on_write_dispatch is not None + on_write_dispatch( + _wifi_status_read(None, device_id="k1-a")["status"], + "with_response", + ) exc = RuntimeError("synthetic transport failure") exc.operation_stage = "gatt-write" # type: ignore[attr-defined] exc.device_write_attempted = True # type: ignore[attr-defined] exc.device_write_confirmed = False # type: ignore[attr-defined] exc.att_error_code = 4 # type: ignore[attr-defined] exc.att_error_name = "INVALID_PDU" # type: ignore[attr-defined] + exc.resolved_write_mode = "with_response" # type: ignore[attr-defined] + exc.write_characteristic_properties = ( # type: ignore[attr-defined] + "notify", + "write", + ) + exc.max_write_without_response_size = 253 # type: ignore[attr-defined] + exc.frame_length = 99 # type: ignore[attr-defined] raise exc monkeypatch.setattr(facade_module, "provision_wifi_once", ambiguous_write) - with pytest.raises(RuntimeError, match="synthetic transport failure"): + with ( + caplog.at_level(logging.ERROR, logger=facade_module.__name__), + pytest.raises(RuntimeError, match="synthetic transport failure"), + ): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -3381,35 +19903,36 @@ def test_ambiguous_ble_write_blocks_new_network_mutation_until_reconciled( "device_write_confirmed": False, "ble_att_error_code": 4, "ble_att_error_name": "INVALID_PDU", + "resolved_write_mode": "with_response", + "max_write_without_response_size": 253, + "frame_length": 99, + "write_characteristic_properties": ["notify", "write"], } - assert state["network_write_reconciliation"] == { - "status": "device-state-unknown-after-write", - "operation_id": operation["operation_id"], - "transport_ref": "k1-a", - "connection_mode": "bridge", - "operation_stage": "gatt-write", - "reason_code": "RuntimeError", - "device_write_confirmed": False, - "required_action": "explicit-read-only-ble-status-observation", - "scope": "process-runtime", - "observed_at": state["network_write_reconciliation"]["observed_at"], - } + audit = state["network_write_reconciliation"] + assert audit["status"] == "previous-session-result-unknown" + assert audit["operation_id"] == operation["operation_id"] + assert audit["reason_code"] == "network-mutation-audit-open" + assert audit["required_action"] == "none" + assert audit["blocks_new_explicit_intent"] is False + failure_log = next( + record + for record in caplog.records + if getattr(record, "event_code", None) == "k1_network_provision_failed" + ) + assert failure_log.resolved_write_mode == "with_response" + assert failure_log.write_characteristic_properties == ["notify", "write"] + assert failure_log.max_write_without_response_size == 253 + assert failure_log.frame_length == 99 - with pytest.raises( - facade_module.NetworkWriteReconciliationRequired, - match="новая запись заблокирована", - ): - asyncio.run( - service.connect( - ConnectRequest( - device_id="k1-a", - ssid="another-network", - password=SecretStr(SECONDARY_TEST_CREDENTIAL), - compatibility_attestation=ATTESTATION, - idempotency_key="ambiguous-write-2", - ) - ) - ) + restarted_service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices(restarted_service, [{"device_id": "k1-a"}]) + restarted_state = restarted_service.state() + assert restarted_state["network_write_reconciliation"] is None + assert restarted_state["network_mutation_ledger"]["mutation_allowed"] is True + assert restarted_state["network_mutation_ledger"]["resolution"] == "interrupted" + assert ( + restarted_state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + ) assert calls == 1 @@ -3420,9 +19943,15 @@ def test_unchanged_status_does_not_confirm_without_response_write( ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_devices(service, [{"device_id": "k1-a"}]) - unchanged_status = {"ipv4": None, "mode": "WIFI_CLIENT"} + unchanged_status = _wifi_status_read(None, device_id="k1-a")["status"] - async def unchanged_write(*_: object, **__: object) -> dict[str, Any]: + async def unchanged_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + on_write_dispatch(unchanged_status, "without_response") return { "started_at_utc": "2026-08-06T10:00:00Z", "completed_at_utc": "2026-08-06T10:00:01Z", @@ -3441,7 +19970,7 @@ def test_unchanged_status_does_not_confirm_without_response_write( ): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -3458,42 +19987,325 @@ def test_unchanged_status_does_not_confirm_without_response_write( assert record.device_write_attempted is True assert record.device_write_confirmed is False assert record.side_effect_status == "unknown" - assert service.state()["network_write_reconciliation"] is None + fence = service.state()["network_write_reconciliation"] + assert fence is not None + assert fence["transport_ref"] == "k1-a" + assert fence["connection_mode"] == "bridge" + assert fence["operation_stage"] == "observing" + assert fence["reason_code"] == "network-mutation-audit-open" + assert fence["required_action"] == "none" + assert fence["blocks_new_explicit_intent"] is False + assert fence["scope"] == "durable-ledger" + assert fence["ledger_revision"] == 3 + ledger = service.state()["network_mutation_ledger"] + assert ledger["status"] == "unresolved" + assert ledger["mutation_allowed"] is True + assert ledger["stage"] == "observing" + assert ledger["revision"] == 3 -def test_read_only_ble_status_clears_network_write_reconciliation_fence( +def test_same_bridge_baseline_does_not_confirm_with_response_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_devices(service, [{"device_id": "k1-a"}]) + unchanged_status = _wifi_status_read("192.168.68.50", device_id="k1-a")["status"] + + async def unchanged_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + on_write_dispatch(unchanged_status, "with_response") + return { + "started_at_utc": "2026-08-06T10:00:00Z", + "completed_at_utc": "2026-08-06T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "no_status_change_before_timeout", + "write_mode": "with_response", + "baseline_status": unchanged_status, + "observations": [{"status": unchanged_status}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", unchanged_write) + + with pytest.raises(RuntimeError, match="не сообщило адрес"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="replacement-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + + state = service.state() + assert state["network_mutation_ledger"]["status"] == "unresolved" + assert state["network_mutation_ledger"]["stage"] == "observing" + assert state["network_mutation_ledger"]["resolution"] is None + assert state["network_write_reconciliation"] is not None + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + + +@pytest.mark.parametrize( + ("connection_mode", "attestation", "observed_ipv4"), + [ + ("bridge", ATTESTATION, "192.168.68.99"), + ("direct-connect", DIRECT_CONNECT_ATTESTATION, "172.20.10.2"), + ], +) +def test_live_client_baseline_with_different_ipv4_keeps_write_ambiguous( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + connection_mode: NetworkConnectionMode, + attestation: CompatibilityAttestationRequest, + observed_ipv4: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + mode_revision = _select_connection_mode(service, connection_mode) + _set_scanned_devices(service, [{"device_id": "k1-a"}]) + baseline = _wifi_status_read("192.168.68.40", device_id="k1-a")["status"] + + async def changed_client_address( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + on_write_dispatch(baseline, "with_response") + return { + "started_at_utc": "2026-08-06T10:00:00Z", + "completed_at_utc": "2026-08-06T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": baseline, + "observations": [ + { + "status": _wifi_status_read( + observed_ipv4, + device_id="k1-a", + )["status"] + } + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", changed_client_address) + + with pytest.raises(RuntimeError, match="не сообщило адрес"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="replacement-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode=connection_mode, + compatibility_attestation=attestation, + expected_mode_revision=mode_revision, + ) + ) + ) + + state = service.state() + assert state["network_mutation_ledger"]["status"] == "unresolved" + assert state["network_mutation_ledger"]["stage"] == "observing" + assert state["network_mutation_ledger"]["resolution"] is None + assert state["network_write_reconciliation"] is not None + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + + +def test_read_only_reconciliation_terminalizes_the_matching_idempotency_edge( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_k1(service) - service._network_write_reconciliation = { # noqa: SLF001 - "status": "device-state-unknown-after-write", - "operation_id": "ambiguous-operation", - "transport_ref": "test-ble-transport", - "connection_mode": "bridge", - "operation_stage": "gatt-write", - "reason_code": "BleakGATTProtocolError", - "device_write_confirmed": False, - "required_action": "explicit-read-only-ble-status-observation", - "scope": "process-runtime", - "observed_at": "2026-08-06T10:00:00Z", + _seed_unresolved_network_mutation( + service, + operation_id="lost-response-operation", + ) + reads = 0 + + async def read_current_status(*_: object, **__: object) -> dict[str, Any]: + nonlocal reads + reads += 1 + return _wifi_status_read("192.168.68.50") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_status) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + + state = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert reads == 1 + assert state["connection_mode"] == "bridge" + assert state["k1_ip"] == "192.168.68.50" + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "target-observed" + assert state["network_provisioning_idempotency"] == { + "status": "ready", + "revision": state["network_provisioning_idempotency"]["revision"], + "reason_code": None, + "active_operation_id": None, + "active_action": None, + "active_stage": None, + "terminal_record_count": 1, + "mutation_allowed": True, } + idempotency_record = ( + service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + .snapshot() + .records[0] + ) + assert idempotency_record.operation_id == "lost-response-operation" + assert idempotency_record.stage == "terminal" + assert idempotency_record.terminal is not None + assert idempotency_record.terminal.outcome == "succeeded" + assert idempotency_record.terminal.outcome_code == "network.provision.reconciled" + assert idempotency_record.terminal.side_effect_status == "reconciled" + + +def test_restart_interrupts_legacy_edge_before_optional_read_only_verify( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + operation_id = "legacy-read-only-reconciliation" + _seed_legacy_network_mutation_without_idempotency( + service, + operation_id=operation_id, + stage="observing", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(restarted) + reads = 0 + + async def read_current_status(*_: object, **__: object) -> dict[str, Any]: + nonlocal reads + reads += 1 + return _wifi_status_read("192.168.68.50") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_status) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + + state = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert reads == 1 + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + idempotency = restarted._require_network_provisioning_idempotency_journal().snapshot() # noqa: SLF001 + assert idempotency.status == "ready" + assert idempotency.active_record is None + terminal = idempotency.records[-1] + assert terminal.operation_id == operation_id + assert terminal.stage == "terminal" + assert terminal.terminal is not None + assert terminal.terminal.outcome_code == "network.provision.interrupted" + assert terminal.terminal.side_effect_status == "reconciled" + + +def test_read_only_reconciliation_rejects_cross_journal_mismatch_before_gatt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + fence = _seed_unresolved_network_mutation( + service, + operation_id="network-ledger-operation", + idempotency_operation_id="idempotency-journal-operation", + ) + + async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("cross-journal mismatch must fail before GATT") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-idempotency-operation-mismatch" + state = service.state() + assert state["network_write_reconciliation"] == fence + assert state["network_mutation_ledger"]["status"] == "unresolved" + assert state["network_provisioning_idempotency"]["status"] == "blocked" + assert state["network_provisioning_idempotency"]["active_operation_id"] == ( + "idempotency-journal-operation" + ) + + +def test_read_only_addressless_status_keeps_network_write_reconciliation_fence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + _seed_unresolved_network_mutation(service) async def read_current_status(*_: object, **__: object) -> dict[str, Any]: return _wifi_status_read(None) monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_status) - with pytest.raises(RuntimeError, match="не сообщил актуальный DHCP-адрес"): - service.verify_connection( - ConnectionVerifyRequest( - device_id="test-ble-transport", - compatibility_attestation=ATTESTATION, + with pytest.raises( + facade_module.ConnectionVerificationError, + match="не сообщил адрес общей локальной сети", + ) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) ) ) + assert raised.value.reason_code == "connection-verify-address-unavailable" - assert service.state()["network_write_reconciliation"] is None + fence = service.state()["network_write_reconciliation"] + assert fence is not None + assert fence["operation_id"] == "ambiguous-operation" + assert fence["operation_stage"] == "dispatching" + assert fence["scope"] == "durable-ledger" + assert fence["ledger_revision"] == 2 def test_read_only_ble_status_for_another_transport_keeps_reconciliation_fence( @@ -3502,34 +20314,1047 @@ def test_read_only_ble_status_for_another_transport_keeps_reconciliation_fence( ) -> None: service, _ = service_with_fake_runtime(tmp_path) _set_scanned_k1(service, device_id="k1-b") - fence = { - "status": "device-state-unknown-after-write", - "operation_id": "ambiguous-operation", - "transport_ref": "k1-a", - "connection_mode": "bridge", - "operation_stage": "gatt-write", - "reason_code": "BleakGATTProtocolError", - "device_write_confirmed": False, - "required_action": "explicit-read-only-ble-status-observation", - "scope": "process-runtime", - "observed_at": "2026-08-06T10:00:00Z", - } - service._network_write_reconciliation = dict(fence) # noqa: SLF001 + fence = _seed_unresolved_network_mutation(service, transport_ref="k1-a") async def read_other_status(*_: object, **__: object) -> dict[str, Any]: - return _wifi_status_read(None, device_id="k1-b") + raise AssertionError("target mismatch must fail before a GATT read") monkeypatch.setattr(facade_module, "read_wifi_status_once", read_other_status) - with pytest.raises(RuntimeError, match="не сообщил актуальный DHCP-адрес"): - service.verify_connection( - ConnectionVerifyRequest( - device_id="k1-b", - compatibility_attestation=ATTESTATION, + with pytest.raises( + facade_module.ConnectionVerificationError, + match="относится к другому K1", + ) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-b", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=0, + ) + ) + ) + assert raised.value.reason_code == "connection-verify-reconciliation-target-mismatch" + + assert service.state()["network_write_reconciliation"] == fence + + +@pytest.mark.parametrize( + ("intended_mode", "attestation", "requested_mode"), + [ + ("bridge", DIRECT_CONNECT_ATTESTATION, "direct-connect"), + ("direct-connect", ATTESTATION, "bridge"), + ], +) +def test_read_only_reconciliation_requires_exact_requested_mode_despite_shared_status( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + intended_mode: NetworkConnectionMode, + attestation: CompatibilityAttestationRequest, + requested_mode: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + original_fence = _seed_unresolved_network_mutation( + service, + intended_mode=intended_mode, + ) + + async def read_current_status(*_: object, **__: object) -> dict[str, Any]: + raise AssertionError("mode mismatch must fail before a GATT read") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_status) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=attestation, + expected_discovery_generation=0, + ) ) ) - assert service.state()["network_write_reconciliation"] == fence + assert raised.value.reason_code == "connection-verify-reconciliation-target-mismatch" + state = service.state() + assert state["connection_mode"] is None + assert state["k1_ip"] is None + assert state["network_write_reconciliation"] == original_fence + ledger = state["network_mutation_ledger"] + assert ledger["status"] == "unresolved" + assert ledger["mutation_allowed"] is True + assert ledger["intended_mode"] == intended_mode + assert ledger["revision"] == 2 + assert ledger["resolution"] is None + verify_operation = next( + item for item in state["operations"] if item["action"] == "connection.verify" + ) + assert verify_operation["status"] == "failed" + + +def test_durable_restart_interrupts_old_attempt_without_implicit_gatt_recovery( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id="durable-restart-reconciliation", + transport_ref=DURABLE_K1_UUID, + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + captured = _durable_status_capture() + status_reads: list[tuple[str, bool, str | None]] = [] + pin_calls: list[tuple[object, str]] = [] + write_calls: list[str] = [] + + async def read_server_target( + device_id: str, + *, + captured_device: object | None = None, + recovery_device_session_id: str | None = None, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + status_reads.append((device_id, allow_known_device_retrieval, recovery_device_session_id)) + assert captured_device is None + assert on_gatt_validated is not None + on_gatt_validated(captured) + return _wifi_status_read("192.168.68.50", device_id=device_id) + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + write_calls.append("write") + raise AssertionError("durable reconciliation must not write to K1") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_server_target) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + + before = restarted.state() + assert before["devices"] == [] + assert before["selected_device_id"] is None + assert before["network_mutation_ledger"]["status"] == "resolved" + assert before["network_mutation_ledger"]["resolution"] == "interrupted" + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + state = restarted.state() + assert status_reads == [] + assert write_calls == [] + assert pin_calls == [] + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_provisioning_idempotency"]["status"] == "ready" + assert state["network_provisioning_idempotency"]["active_operation_id"] is None + assert state["network_provisioning_idempotency"]["terminal_record_count"] == 1 + terminal = ( + restarted._require_network_provisioning_idempotency_journal() # noqa: SLF001 + .snapshot() + .records[-1] + ) + assert terminal.operation_id == "durable-restart-reconciliation" + assert terminal.stage == "terminal" + assert terminal.terminal is not None + assert terminal.terminal.outcome_code == "network.provision.interrupted" + assert terminal.terminal.side_effect_status == "reconciled" + assert state["connection_supervisor"]["authority"]["control_allowed"] is False + + # Without an independently durable current/configured topology, startup's + # interrupted audit must not turn an arbitrary new advertisement into an + # implicit recovery continuation. + _set_scanned_k1(restarted, device_id="replacement-k1") + fresh_policy = restarted.state()["connection_policy"] + assert fresh_policy["actions"]["observe-fresh-device-network"]["allowed"] is True + assert fresh_policy["recommended_action"] == "select-connection-intent" + + +@pytest.mark.parametrize( + ("device_id", "attestation", "source", "reason_code"), + [ + ( + "AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE", + ATTESTATION, + "durable-configured-state", + "connection-verify-reconciliation-target-mismatch", + ), + ( + DURABLE_K1_UUID, + QUICK_CONNECT_ATTESTATION, + "durable-configured-state", + "connection-verify-reconciliation-target-mismatch", + ), + ( + DURABLE_K1_UUID, + ATTESTATION, + "fresh-scan", + "connection-verify-source-mismatch", + ), + ], +) +def test_durable_restart_rejects_request_target_mismatch_before_gatt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + device_id: str, + attestation: CompatibilityAttestationRequest, + source: str, + reason_code: str, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id="durable-request-mismatch", + transport_ref=DURABLE_K1_UUID, + ) + restarted, _ = service_with_fake_runtime(tmp_path) + reads: list[str] = [] + + async def forbidden_read(*_: object, **__: object) -> dict[str, Any]: + reads.append("read") + raise AssertionError("request mismatch must fail before GATT") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_read) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("request mismatch must not write") + ), + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=device_id, + source=source, # type: ignore[arg-type] + compatibility_attestation=attestation, + **({"expected_discovery_generation": 0} if source == "fresh-scan" else {}), + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + assert reads == [] + state = restarted.state() + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["selected_device_id"] is None + assert state["device_session"] is None + + +def test_durable_restart_same_bridge_baseline_cannot_resolve_ambiguous_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + baseline = NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.50", + status_code=1, + reserved=0, + ) + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id="durable-unchanged-bridge-baseline", + transport_ref=DURABLE_K1_UUID, + baseline_status=baseline, + ) + restarted, _ = service_with_fake_runtime(tmp_path) + captured = _durable_status_capture() + reads = 0 + pin_calls: list[tuple[object, str]] = [] + + async def read_unchanged_bridge_status( + device_id: str, + *, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + nonlocal reads + reads += 1 + assert device_id == DURABLE_K1_UUID + assert allow_known_device_retrieval is True + assert on_gatt_validated is not None + on_gatt_validated(captured) + return _wifi_status_read("192.168.68.50", device_id=device_id) + + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + read_unchanged_bridge_status, + ) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + assert reads == 0 + assert pin_calls == [] + state = restarted.state() + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_provisioning_idempotency"]["status"] == "ready" + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + assert state["device_session"] is None + + +@pytest.mark.parametrize( + ("intended_mode", "attestation", "baseline", "previous", "observed_status"), + [ + ( + "quick-connect", + QUICK_CONNECT_ATTESTATION, + NetworkStatusEvidence("WIFI_CLIENT", "192.168.68.50", 1, 0), + PreviousConnectionEvidence( + DURABLE_K1_UUID, + "quick-connect", + "192.168.56.1", + "previous-quick-session", + ), + _ap_ready_wifi_status(), + ), + ( + "bridge", + ATTESTATION, + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + PreviousConnectionEvidence( + DURABLE_K1_UUID, + "bridge", + "192.168.68.40", + "previous-bridge-session", + ), + _wifi_status_read("192.168.68.99")["status"], + ), + ( + "direct-connect", + DIRECT_CONNECT_ATTESTATION, + NetworkStatusEvidence("WIFI_AP", "192.168.56.1", 1, 1), + PreviousConnectionEvidence( + DURABLE_K1_UUID, + "direct-connect", + "172.20.10.3", + "previous-direct-session", + ), + _wifi_status_read("172.20.10.2")["status"], + ), + ], +) +def test_durable_restart_previous_topology_cannot_resolve_ambiguous_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + intended_mode: NetworkConnectionMode, + attestation: CompatibilityAttestationRequest, + baseline: NetworkStatusEvidence, + previous: PreviousConnectionEvidence, + observed_status: dict[str, Any], +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id=f"durable-previous-{intended_mode}", + transport_ref=DURABLE_K1_UUID, + intended_mode=intended_mode, + baseline_status=baseline, + previous_connection=previous, + ) + restarted, _ = service_with_fake_runtime(tmp_path) + captured = _durable_status_capture() + reads = 0 + pin_calls: list[tuple[object, str]] = [] + write_calls: list[str] = [] + + async def read_previous_topology( + device_id: str, + *, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + nonlocal reads + reads += 1 + assert device_id == DURABLE_K1_UUID + assert allow_known_device_retrieval is True + assert on_gatt_validated is not None + on_gatt_validated(captured) + status_read = _wifi_status_read(None, device_id=device_id) + status_read["status"] = observed_status + return status_read + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + write_calls.append("write") + raise AssertionError("read-only restart reconciliation must not write") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_previous_topology) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=attestation, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + assert reads == 0 + assert write_calls == [] + assert pin_calls == [] + state = restarted.state() + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_provisioning_idempotency"]["status"] == "ready" + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + assert state["device_session"] is None + + +@pytest.mark.parametrize("outcome", ["baseline", "wrong-mode", "read-failure"]) +def test_durable_restart_failed_status_keeps_ambiguity_and_does_not_pin( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + outcome: str, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id=f"durable-{outcome}", + transport_ref=DURABLE_K1_UUID, + ) + restarted, _ = service_with_fake_runtime(tmp_path) + captured = _durable_status_capture() + reads = 0 + pin_calls: list[tuple[object, str]] = [] + write_calls: list[str] = [] + + async def read_nonmatching_status( + device_id: str, + *, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + nonlocal reads + reads += 1 + assert device_id == DURABLE_K1_UUID + assert allow_known_device_retrieval is True + if outcome == "read-failure": + raise RuntimeError("synthetic durable GATT read failure") + assert on_gatt_validated is not None + on_gatt_validated(captured) + result = _wifi_status_read(None, device_id=device_id) + if outcome == "wrong-mode": + result["status"] = _ap_ready_wifi_status() + return result + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + write_calls.append("write") + raise AssertionError("failed status observation must not write") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_nonmatching_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + + with pytest.raises(facade_module.ConnectionVerificationError): + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert reads == 0 + assert write_calls == [] + assert pin_calls == [] + state = restarted.state() + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["network_provisioning_idempotency"]["status"] == "ready" + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + assert state["device_session"] is None + + +def test_semantic_only_durable_restart_uses_lan_and_mqtt_identity_not_ble( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + topology_store = first._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + _seed_unresolved_network_mutation( + first, + operation_id="semantic-restart-post-dispatch-unknown", + transport_ref=DURABLE_K1_UUID, + intended_mode="bridge", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + association_probe = FakeHostWifiAssociationProbe("a" * 64) + restarted._host_wifi_association_probe = association_probe # type: ignore[assignment] # noqa: SLF001 + reads = 0 + pin_calls: list[tuple[object, str]] = [] + binding_validation_epochs: list[tuple[int, int]] = [] + + async def forbidden_status_read(*_: object, **__: object) -> dict[str, Any]: + nonlocal reads + reads += 1 + raise AssertionError("resolved durable restart must not require BLE") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_status_read) + network_writes: list[str] = [] + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + network_writes.append("network-write") + raise AssertionError("semantic observation must not write") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda _target: facade_module.TcpReachabilityProbeResult(reachable=True), + ) + + async def bootstrap_after_exact_path_validation( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + supervisor = bound_service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.intent is not None + assert supervisor.device_network.target is not None + assert supervisor.device_network.transport_ref is not None + binding = ApplicationConnectionBinding( + intent_id=supervisor.intent.intent_id, + transport_ref=supervisor.device_network.transport_ref, + host_path_epoch=supervisor.host_path.epoch, + target_ipv4=supervisor.device_network.target.ipv4, + target_port=supervisor.device_network.target.port, + connection_mode=connection_mode, + ) + epoch_before = supervisor.host_path.epoch + bound_service._validate_application_connection_path(binding) # noqa: SLF001 + epoch_after = bound_service._connection_supervisor.snapshot().host_path.epoch # noqa: SLF001 + binding_validation_epochs.append((epoch_before, epoch_after)) + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + + restarted._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_after_exact_path_validation, + restarted, + ) + + before = restarted.state() + assert before["devices"] == [] + assert before["selected_device_id"] is None + assert before["connection_attempt"] is None + assert before["network_mutation_ledger"]["status"] == "resolved" + assert before["network_mutation_ledger"]["resolution"] == "interrupted" + assert before["semantic_topology_store"]["record"]["revision"] == 1 + assert ( + before["connection_policy"]["actions"]["observe-configured-device-network"][ + "requires_live_gatt_validation" + ] + is False + ) + assert ( + before["connection_policy"]["actions"]["observe-configured-device-network"]["allowed"] + is True + ) + assert before["connection_policy"]["recommended_action"] == ( + "observe-configured-device-network" + ) + assert before["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + state = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert reads == 0 + assert network_writes == [] + assert pin_calls == [] + assert state["semantic_topology_store"]["record"]["revision"] == 1 + assert state["connection_verification"]["endpoint_validation"] == ( + "durable-semantic-topology+mqtt-tcp-connect" + ) + assert state["connection_verification"]["lease_state"] == "reachable" + assert state["connection_supervisor"]["observed"]["device_network"]["source"] == ( + "durable-semantic-topology" + ) + assert state["connection_supervisor"]["observed"]["device_identity"]["state"] == "verified" + assert state["connection_supervisor"]["authority"]["control_allowed"] is True + assert state["connection_supervisor"]["authority"]["acquisition_start_allowed"] is True + assert binding_validation_epochs == [(1, 1)] + assert association_probe.interfaces == ["test0", "test0", "test0"] + assert state["connection_supervisor"]["observed"]["host_path"]["fingerprint"] != ( + "test-route:192.168.68.50" + ) + + +def test_semantic_durable_verify_refreshes_stale_dhcp_over_exact_read_only_ble( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + topology_store = first._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + association_probe = FakeHostWifiAssociationProbe("a" * 64) + restarted._host_wifi_association_probe = association_probe # type: ignore[assignment] # noqa: SLF001 + capture = _durable_status_capture() + status_reads: list[tuple[str, bool, str | None]] = [] + pin_calls: list[tuple[object, str]] = [] + network_writes: list[str] = [] + + async def read_current_dhcp_status( + device_id: str, + *, + captured_device: object | None = None, + recovery_device_session_id: str | None = None, + allow_known_device_retrieval: bool = False, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + status_reads.append( + (device_id, allow_known_device_retrieval, recovery_device_session_id) + ) + assert captured_device is None + assert allow_known_device_retrieval is True + assert on_gatt_validated is not None + on_gatt_validated(capture) + return _wifi_status_read("192.168.68.51", device_id=device_id) + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + network_writes.append("network-write") + raise AssertionError("stale DHCP recovery must not write to K1") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_dhcp_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_probe_configured_endpoint_host_only", + lambda target, **_kwargs: facade_module._ConfiguredEndpointHostObservation( # noqa: SLF001 + path=_direct_host_path(target), + reachable=False, + reason_code="tcp-connection-timeout", + ), + ) + + async def bootstrap_after_refresh( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + + restarted._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_after_refresh, + restarted, + ) + + state = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert status_reads == [(DURABLE_K1_UUID, True, None)] + assert network_writes == [] + assert len(pin_calls) == 1 + assert pin_calls[0][0] is capture + assert state["k1_ip"] == "192.168.68.51" + assert state["semantic_topology_store"]["record"]["revision"] == 2 + assert state["semantic_topology_store"]["record"]["ipv4"] == "192.168.68.51" + assert state["connection_verification"]["endpoint_validation"] == ( + "ble-wifi-status-read+mqtt-tcp-connect" + ) + assert state["connection_verification"]["address_source"] == "ble-wifi-status-read" + assert state["connection_verification"]["address_changed"] is True + assert state["connection_verification"]["write_performed"] is False + assert state["connection_supervisor"]["observed"]["device_identity"]["state"] == ( + "verified" + ) + assert state["connection_supervisor"]["authority"]["control_allowed"] is True + + +def test_semantic_durable_verify_rejects_association_change_during_tcp_probe( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + topology_store = first._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref=DURABLE_K1_UUID, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + association_probe = FakeHostWifiAssociationProbe("a" * 64, "b" * 64) + restarted._host_wifi_association_probe = association_probe # type: ignore[assignment] # noqa: SLF001 + device_edges: list[str] = [] + + async def forbidden_device_edge(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("device") + raise AssertionError("association race must fail before BLE or device write") + + async def forbidden_bootstrap(*_: object, **__: object) -> None: + device_edges.append("mqtt-bootstrap") + raise AssertionError("association race must fail before DeviceInfo bootstrap") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_edge) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_edge) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda _target: facade_module.TcpReachabilityProbeResult(reachable=True), + ) + restarted._bootstrap_prestart_control_ready_owned = forbidden_bootstrap # type: ignore[method-assign] # noqa: SLF001 + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-mqtt-unreachable" + assert device_edges == [] + assert association_probe.interfaces == ["test0", "test0"] + state = restarted.state() + assert state["selected_device_id"] is None + assert state["active_connection_mode"] is None + assert state["connection_supervisor"]["intent"] is None + assert state["connection_supervisor"]["observed"]["endpoint"]["target"] is None + assert state["semantic_topology_store"]["record"]["revision"] == 1 + + +@pytest.mark.parametrize("observed_session_state", ["ready", "scanning"]) +def test_public_physical_recovery_refreshes_stale_dhcp_then_classifies_without_writes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + observed_session_state: str, +) -> None: + original, _ = service_with_fake_runtime(tmp_path) + stop_operation_id, compatibility_revision = ( + _persist_resolved_unclassified_stop_for_restart(original) + ) + topology_store = original._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref="test-ble-transport", + connection_mode="bridge", + ipv4="192.168.68.52", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-12T13:28:19.000Z", + ) + + restarted, runtime = service_with_fake_runtime(tmp_path) + association_probe = FakeHostWifiAssociationProbe("a" * 64) + restarted._host_wifi_association_probe = association_probe # type: ignore[assignment] # noqa: SLF001 + capture = _durable_status_capture(device_id="test-ble-transport") + host_probes: list[str] = [] + status_reads: list[tuple[str, bool, bool, float | None]] = [] + command_edges: list[str] = [] + + def stale_host_probe( + target: str, + **_: object, + ) -> facade_module._ConfiguredEndpointHostObservation: # noqa: SLF001 + host_probes.append(target) + return facade_module._ConfiguredEndpointHostObservation( # noqa: SLF001 + path=_direct_host_path(target), + reachable=False, + reason_code="tcp-connection-timeout", + ) + + async def read_current_dhcp_status( + device_id: str, + *, + allow_known_device_retrieval: bool = False, + rediscover: bool = False, + exact_scan_timeout_seconds: float | None = None, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + status_reads.append( + ( + device_id, + allow_known_device_retrieval, + rediscover, + exact_scan_timeout_seconds, + ) + ) + assert on_gatt_validated is not None + on_gatt_validated(capture) + return _wifi_status_read("192.168.68.51", device_id=device_id) + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + command_edges.append("network-write") + raise AssertionError("physical recovery must not write K1 network state") + + monkeypatch.setattr( + facade_module, + "_probe_configured_endpoint_host_only", + stale_host_probe, + ) + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_current_dhcp_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr(facade_module, "pin_connected_device_handle", lambda *_a, **_k: None) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + _install_real_coordinator_bootstrap( + restarted, + observed_session_state=observed_session_state, + ) + + before = restarted.state() + configured_recovery = before["connection_policy"]["actions"][ + "observe-configured-device-network" + ] + assert configured_recovery["allowed"] is True + assert configured_recovery["required_transport_ref"] == "test-ble-transport" + assert configured_recovery["required_connection_mode"] == "bridge" + assert configured_recovery["requires_live_gatt_validation"] is True + + verified = asyncio.run(restarted.verify_connection(ConnectionVerifyRequest())) + + after = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert after is not None + assert after.operation_id == stop_operation_id + assert after.revision == compatibility_revision + 1 + assert after.resolved_unclassified_stop_recovery_required is False + assert after.reconciliations[-1].resolution == ( + "physical-active-observed" + if observed_session_state == "scanning" + else "physical-standby-observed" + ) + assert host_probes == [] + assert status_reads == [ + ( + "test-ble-transport", + True, + True, + facade_module.CONNECTION_VERIFY_EXACT_UUID_SCAN_TIMEOUT_SECONDS, + ) + ] + assert association_probe.timeout_seconds == [ + facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS, + facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS, + ] + assert command_edges == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert verified["k1_ip"] == "192.168.68.51" + assert verified["semantic_topology_store"]["record"]["ipv4"] == "192.168.68.51" + assert verified["last_operation"]["result"]["write_performed"] is False + assert verified["last_operation"]["result"]["physical_reconciliation"][ + "performed" + ] is True + + +def test_durable_restart_rejects_network_revision_race_before_commit_or_pin( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + operation_id = "durable-revision-race" + first, _ = service_with_fake_runtime(tmp_path) + _seed_unresolved_network_mutation( + first, + operation_id=operation_id, + transport_ref=DURABLE_K1_UUID, + ) + restarted, _ = service_with_fake_runtime(tmp_path) + captured = _durable_status_capture() + pin_calls: list[tuple[object, str]] = [] + + async def read_while_revision_changes( + device_id: str, + *, + on_gatt_validated: Callable[[object], None] | None = None, + **_: object, + ) -> dict[str, Any]: + assert on_gatt_validated is not None + on_gatt_validated(captured) + current_ledger = restarted._network_mutation_ledger.snapshot() # noqa: SLF001 + assert current_ledger.record is not None + restarted._network_mutation_ledger.mark_observing( # noqa: SLF001 + operation_id, + expected_revision=current_ledger.record.revision, + write_confirmed=False, + observation=NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.50", + status_code=1, + reserved=0, + ), + ) + return _wifi_status_read("192.168.68.50", device_id=device_id) + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_while_revision_changes) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("revision race must not write") + ), + ) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id=DURABLE_K1_UUID, + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + assert raised.value.reason_code == "connection-verify-candidate-not-fresh" + assert pin_calls == [] + state = restarted.state() + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == "interrupted" + assert state["semantic_topology_store"]["status"] == "empty" + assert state["selected_device_id"] is None + assert state["device_session"] is None def test_provisioning_cannot_switch_device_during_active_acquisition( @@ -3539,7 +21364,7 @@ def test_provisioning_cannot_switch_device_during_active_acquisition( service, _ = service_with_fake_runtime(tmp_path) _set_scanned_devices(service, [{"device_id": "k1-a"}]) service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -3557,7 +21382,7 @@ def test_provisioning_cannot_switch_device_during_active_acquisition( with pytest.raises(RuntimeError, match="активной acquisition-сессии"): asyncio.run( service.connect( - ConnectRequest( + _connect_request( device_id="k1-a", ssid="lab-network", password=SecretStr(PRIMARY_TEST_CREDENTIAL), @@ -3587,7 +21412,7 @@ def test_sensor_catalog_exposes_two_browser_adapter_cameras_after_profile_attest assert all(stream["availability"] == "unverified" for stream in initial_cameras) state = service.prepare_acquisition( - PrepareAcquisitionRequest( + _prepare_request( project_name=PROJECT_NAME, host="192.168.1.20", compatibility_attestation=ATTESTATION, @@ -3608,3 +21433,9885 @@ def test_sensor_catalog_exposes_two_browser_adapter_cameras_after_profile_attest assert state["connection_verification"]["network_reachability"] == "unknown" assert state["device_calibration"]["status"] == "unavailable" assert state["device_calibration"]["vehicle_extrinsics"] == ("host-domain-not-owned-by-plugin") + + +def test_quick_to_bridge_requires_new_explicit_scan_instead_of_retained_session( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-a", + ) + service._device_session_id = "quick-session-a" # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + supervisor_generation = service._connection_supervisor.snapshot().lease.generation # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "status": "control-transport-lost", + "lease_state": "disconnected", + "lease_generation": supervisor_generation, + "endpoint_validation": "terminal-mqtt-network-loop", + "network_reachability": "unreachable", + "reason_code": "connection_lease_control_transport_lost", + "observed_at": "2026-08-06T12:01:00Z", + } + captured = facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address="k1-a"), # type: ignore[arg-type] + macos_uuid="k1-a", + owner_epoch=7, + ) + provision_calls: list[tuple[object | None, str | None]] = [] + pin_calls: list[tuple[object, str]] = [] + + def recover(device_id: str, *, device_session_id: str) -> object | None: + assert (device_id, device_session_id) == ("k1-a", "quick-session-a") + return captured + + async def provision( + *_args: object, + captured_device: object | None = None, + recovery_device_session_id: str | None = None, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + provision_calls.append((captured_device, recovery_device_session_id)) + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-06T12:02:00Z", + "completed_at_utc": "2026-08-06T12:02:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": {"mode": "WIFI_AP", "ipv4": "192.168.56.1"}, + "observations": [{"status": _wifi_status_read("192.168.68.50")["status"]}], + } + + monkeypatch.setattr(facade_module, "connected_device_capture", recover) + monkeypatch.setattr( + facade_module, + "connected_device_recovery_snapshot", + lambda *_args, **_kwargs: { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + }, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", provision) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda handle, *, device_session_id: pin_calls.append((handle, device_session_id)), + ) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_control_endpoint_reachable", + lambda _target: True, + ) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-candidate-not-fresh" + assert provision_calls == [] + assert pin_calls == [] + + +def test_quick_to_bridge_does_not_retrieve_old_handle_without_new_scan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-a", + ) + service._device_session_id = "quick-session-a" # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + captured = facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address="k1-a"), # type: ignore[arg-type] + macos_uuid="k1-a", + owner_epoch=7, + source="retrieved-session", + ) + retained_lookups: list[tuple[str, str]] = [] + provision_calls: list[tuple[object | None, str | None]] = [] + + def retained_lookup(device_id: str, *, device_session_id: str) -> object | None: + retained_lookups.append((device_id, device_session_id)) + # Facade obtains the exact validated object only after the low-level + # operation resolved/retrieved it inside its BLE arbiter lease. + return captured + + async def provision( + *_args: object, + captured_device: object | None = None, + recovery_device_session_id: str | None = None, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + provision_calls.append((captured_device, recovery_device_session_id)) + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-06T12:02:00Z", + "completed_at_utc": "2026-08-06T12:02:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": {"mode": "WIFI_AP", "ipv4": "192.168.56.1"}, + "observations": [{"status": _wifi_status_read("192.168.68.50")["status"]}], + } + + monkeypatch.setattr(facade_module, "connected_device_capture", retained_lookup) + monkeypatch.setattr( + facade_module, + "connected_device_recovery_snapshot", + lambda *_args, **_kwargs: { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + }, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", provision) + monkeypatch.setattr(facade_module, "pin_connected_device_handle", lambda *_args, **_kw: None) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-candidate-not-fresh" + assert retained_lookups == [] + assert provision_calls == [] + + +def test_connection_policy_exposes_retained_recovery_without_inventing_presence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._selected_device_id = "k1-a" # noqa: SLF001 + service._connection_mode = "quick-connect" # noqa: SLF001 + service._device_id = "device-a" # noqa: SLF001 + service._device_session_id = "quick-session-a" # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + service._k1_ip = facade_module.AP_FALLBACK_IPV4 # noqa: SLF001 + monkeypatch.setattr( + facade_module, + "connected_device_recovery_snapshot", + lambda *_args, **_kwargs: { + "status": "retained", + "scope": "owner-epoch-device-session", + "gatt_validated_recently": True, + }, + ) + + state = service.state() + + assert state["devices"] == [] + assert state["current_device_recovery"]["handle_retained"] is True + assert state["current_device_recovery"]["advertised_now"] is False + policy = state["connection_policy"] + assert policy["facts"]["retained_context_is_presence"] is False + assert policy["actions"]["provision-fresh-device"]["allowed"] is False + assert policy["actions"]["recover-current-device-network"] == { + "allowed": True, + "reason_codes": [], + "target_source": "retained-current-process", + "required_transport_ref": "k1-a", + "requires_live_gatt_validation": True, + "automatic_retry": False, + } + assert policy["recommended_action"] == "recover-current-device-network" + + +def test_connection_policy_treats_open_durable_audit_as_nonblocking_context( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + _seed_unresolved_network_mutation(service, transport_ref="k1-a") + + policy = service.state()["connection_policy"] + + assert policy["facts"]["physical_network_state"] == "unknown" + assert policy["actions"]["provision-fresh-device"]["allowed"] is True + assert policy["actions"]["recover-current-device-network"]["allowed"] is False + assert policy["actions"]["observe-fresh-device-network"]["allowed"] is True + assert policy["actions"]["observe-fresh-device-network"]["required_transport_ref"] == "k1-a" + assert policy["recommended_action"] == "observe-fresh-device-network" + + +def _project_connection_policy_for_test( + service: XgridsK1CompatibilityService, + *, + idempotency: dict[str, object] | None = None, + idempotency_available: bool = True, + identity_pins: dict[str, object] | None = None, + physical_command: dict[str, object] | None = None, + ble_runtime: dict[str, object] | None = None, + lifecycle_holders: tuple[str, ...] = (), + current_device_recovery: dict[str, object] | None = None, + acquisition_active: bool = False, + acquisition_state: str | None = None, + acquisition_cleanup_pending: bool = False, + runtime_active: bool = False, + application_control_session: dict[str, object] | None = None, +) -> dict[str, Any]: + with service._lock: # noqa: SLF001 + fresh_devices = service._fresh_ble_devices_locked() # noqa: SLF001 + desired_connection_mode = service._desired_connection_mode # noqa: SLF001 + configured_connection_mode = service._connection_mode # noqa: SLF001 + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + active_connection_mode = ( + configured_connection_mode if supervisor.authority.control_allowed else None + ) + return facade_module._connection_policy_projection( # noqa: SLF001 + supervisor=supervisor, + ledger=service._network_mutation_ledger.snapshot(), # noqa: SLF001 + network_provisioning_idempotency=( + idempotency + if idempotency is not None + else service._network_provisioning_idempotency_public_snapshot() # noqa: SLF001 + ), + network_provisioning_idempotency_available=idempotency_available, + semantic_topology_store=service._semantic_topology_public_snapshot(), # noqa: SLF001 + device_identity_pin_store=( + identity_pins + if identity_pins is not None + else service._device_identity_pin_public_snapshot() # noqa: SLF001 + ), + physical_command=( + physical_command + if physical_command is not None + else service._physical_command_coordinator.snapshot() # noqa: SLF001 + ), + ble_runtime=( + ble_runtime + if ble_runtime is not None + else { + "owner_epoch": 1, + "owner_loop_bound": True, + "active_operation_kind": None, + "cleanup_pending": False, + "poisoned": False, + } + ), + lifecycle_process_lease_holders=lifecycle_holders, + fresh_devices=fresh_devices, + current_device_recovery=current_device_recovery, + provisioning_active=False, + acquisition_active=acquisition_active, + acquisition_state=acquisition_state, + acquisition_cleanup_pending=acquisition_cleanup_pending, + runtime_active=runtime_active, + application_control_session=( + application_control_session + if application_control_session is not None + else {"state": "idle", "failure": None} + ), + desired_connection_mode=desired_connection_mode, + active_connection_mode=active_connection_mode, + ) + + +@pytest.mark.parametrize( + ("available", "status", "expected_reason"), + [ + (False, "corrupt", "network-provisioning-idempotency-unavailable"), + (True, "corrupt", "network-provisioning-idempotency-corrupt"), + ], +) +def test_connection_policy_blocks_network_actions_when_idempotency_authority_is_untrusted( + tmp_path: Path, + available: bool, + status: str, + expected_reason: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + + policy = _project_connection_policy_for_test( + service, + idempotency={ + "status": status, + "revision": None, + "reason_code": expected_reason, + "active_operation_id": None, + "active_action": None, + "active_stage": None, + "terminal_record_count": 0, + "mutation_allowed": False, + }, + idempotency_available=available, + ) + + assert policy["actions"]["provision-fresh-device"]["allowed"] is False + assert expected_reason in policy["actions"]["provision-fresh-device"]["reason_codes"] + assert policy["actions"]["observe-fresh-device-network"]["allowed"] is False + assert expected_reason in policy["actions"]["observe-fresh-device-network"]["reason_codes"] + assert policy["actions"]["scan-ble"]["allowed"] is True + + +def test_connection_policy_blocks_cross_journal_mismatch_before_observation( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + _seed_unresolved_network_mutation( + service, + transport_ref="k1-a", + operation_id="network-ledger-operation", + ) + + policy = _project_connection_policy_for_test( + service, + idempotency={ + "status": "blocked", + "revision": 3, + "reason_code": "network-provisioning-idempotency-reconciliation-required", + "active_operation_id": "different-idempotency-operation", + "active_action": facade_module.ACTION_NETWORK_PROVISION, + "active_stage": "unresolved", + "terminal_record_count": 0, + "mutation_allowed": False, + }, + ) + + decision = policy["actions"]["observe-fresh-device-network"] + assert decision["allowed"] is False + assert "network-provisioning-idempotency-operation-mismatch" in decision["reason_codes"] + assert policy["facts"]["network_provisioning_active_operation_matches_ledger"] is False + + +def test_connection_policy_blocks_identity_dependent_network_control_and_start( + tmp_path: Path, +) -> None: + verify_service, _ = service_with_fake_runtime(tmp_path / "verify") + _set_scanned_k1(verify_service) + _seed_supervised_connection(verify_service, with_control=False) + corrupt_pins = { + "schema_version": "missioncore.xgrids-k1-device-identity-pins/v1", + "status": "corrupt", + "revision": None, + "pin_count": 0, + "reason_code": "device-identity-pin-store-corrupt", + } + + verify_policy = _project_connection_policy_for_test( + verify_service, + identity_pins=corrupt_pins, + ) + + for action in ( + "provision-fresh-device", + "observe-fresh-device-network", + "verify-control-device-info", + ): + assert verify_policy["actions"][action]["allowed"] is False + assert ( + "device-identity-pin-store-corrupt" in verify_policy["actions"][action]["reason_codes"] + ) + + start_service, _ = service_with_fake_runtime(tmp_path / "start") + _seed_supervised_connection(start_service) + start_policy = _project_connection_policy_for_test( + start_service, + identity_pins=corrupt_pins, + ) + + assert start_policy["actions"]["start-acquisition"]["allowed"] is False + assert ( + "device-identity-pin-store-corrupt" + in start_policy["actions"]["start-acquisition"]["reason_codes"] + ) + + +def test_connection_policy_blocks_network_observation_when_topology_store_is_corrupt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service) + monkeypatch.setattr( + service, + "_semantic_topology_public_snapshot", + lambda: { + "schema_version": "missioncore.xgrids-k1-semantic-topology/v1", + "status": "corrupt", + "configured_offline_evidence": False, + "live_connection_authority": False, + "reason_code": "semantic-topology-store-corrupt", + "record": None, + }, + ) + + policy = _project_connection_policy_for_test(service) + + assert policy["actions"]["observe-fresh-device-network"]["allowed"] is False + assert ( + "semantic-topology-store-corrupt" + in policy["actions"]["observe-fresh-device-network"]["reason_codes"] + ) + + +@pytest.mark.parametrize( + ("status", "expected_reason"), + [ + ("unresolved", "physical-command-reconciliation-required"), + ("corrupt", "physical-command-ledger-corrupt"), + ], +) +def test_connection_policy_blocks_physical_start_and_stop_on_untrusted_ledger( + tmp_path: Path, + status: str, + expected_reason: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + supervisor = service._connection_supervisor # noqa: SLF001 + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id="data-session-a", + ) + physical_command = { + "status": status, + "reason_code": expected_reason, + "requires_reconciliation": True, + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": "explicit-read-only-reconciliation", + "runtime_bound": False, + "active_operation_id": None, + "record": None, + } + + policy = _project_connection_policy_for_test( + service, + physical_command=physical_command, + ) + + for action in ("start-acquisition", "stop-acquisition"): + assert policy["actions"][action]["allowed"] is False + assert expected_reason in policy["actions"][action]["reason_codes"] + + +def test_pending_reopened_physical_state_allows_only_read_only_recovery_or_retirement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + physical = _pending_reopened_physical_snapshot() + policy = _project_connection_policy_for_test( + service, + physical_command=physical, + ) + + for action in ( + "provision-fresh-device", + "recover-current-device-network", + "start-acquisition", + "stop-acquisition", + ): + assert policy["actions"][action]["allowed"] is False + assert ( + "physical-command-reconciliation-required" + in policy["actions"][action]["reason_codes"] + ) + read_only = policy["actions"]["observe-fresh-device-network"] + assert read_only["allowed"] is True + assert read_only["required_transport_ref"] == "k1-a" + assert read_only["required_connection_mode"] == "bridge" + + # A scanning-shaped local snapshot is not STOP authority by itself. The + # reopened durable row must remain fenced unless the exact adopted worker + # explicitly exposes its STOP checkpoint. + no_stop_checkpoint = _project_connection_policy_for_test( + service, + physical_command=physical, + application_control_session={ + "state": "scanning", + "can_stop": False, + "failure": None, + }, + ) + assert no_stop_checkpoint["actions"]["stop-acquisition"]["allowed"] is False + assert ( + "physical-command-reconciliation-required" + in no_stop_checkpoint["actions"]["stop-acquisition"]["reason_codes"] + ) + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical, + ) + retirement = service._physical_operator_retirement_projection() # noqa: SLF001 + assert retirement["allowed"] is True + assert retirement["reason_codes"] == [] + + +@pytest.mark.parametrize( + ("ble_snapshot", "expected_reason"), + [ + ( + { + "owner_epoch": 4, + "owner_loop_bound": False, + "active_operation_kind": "scan", + "cleanup_pending": True, + "poisoned": False, + }, + "ble-runtime-cleanup-pending", + ), + ( + { + "owner_epoch": 5, + "owner_loop_bound": False, + "active_operation_kind": "status-read", + "cleanup_pending": True, + "poisoned": True, + }, + "ble-runtime-restart-required", + ), + ], +) +def test_connection_policy_blocks_control_and_physical_edges_on_ble_quarantine( + tmp_path: Path, + ble_snapshot: dict[str, object], + expected_reason: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection(service) + + policy = _project_connection_policy_for_test( + service, + ble_runtime=ble_snapshot, + ) + + assert policy["actions"]["start-acquisition"]["allowed"] is False + assert expected_reason in policy["actions"]["start-acquisition"]["reason_codes"] + + +def test_state_projects_ble_runtime_quarantine_into_composite_policy( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + poisoned = { + "owner_epoch": 8, + "owner_loop_bound": False, + "active_operation_kind": "status-read", + "cleanup_pending": True, + "poisoned": True, + } + monkeypatch.setattr(facade_module, "ble_runtime_snapshot", lambda: poisoned) + + state = service.state() + + assert state["ble_runtime"] == poisoned + assert state["connection_policy"]["facts"]["ble_runtime"] == { + "active_operation_kind": "status-read", + "cleanup_pending": True, + "poisoned": True, + } + assert ( + "ble-runtime-restart-required" + in state["connection_policy"]["actions"]["scan-ble"]["reason_codes"] + ) + + +def test_connection_policy_blocks_physical_edges_while_network_owns_process_lease( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + assert service._connection_supervisor.observe_data_plane( # noqa: SLF001 + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id="data-session-a", + ) + + policy = _project_connection_policy_for_test( + service, + lifecycle_holders=("network",), + ) + + for action in ("start-acquisition", "stop-acquisition"): + assert policy["actions"][action]["allowed"] is False + assert ( + "k1-lifecycle-process-lease-network-owned" in policy["actions"][action]["reason_codes"] + ) + + +def test_connection_policy_keeps_host_tcp_probe_independent_of_other_failures( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + with_control=False, + endpoint_reachable=False, + ) + + policy = _project_connection_policy_for_test( + service, + idempotency={ + "status": "corrupt", + "active_operation_id": None, + }, + idempotency_available=False, + identity_pins={"status": "corrupt"}, + physical_command={ + "status": "corrupt", + "requires_reconciliation": True, + }, + ble_runtime={ + "owner_epoch": 9, + "owner_loop_bound": False, + "active_operation_kind": "scan", + "cleanup_pending": True, + "poisoned": True, + }, + lifecycle_holders=("network",), + ) + + assert policy["actions"]["probe-endpoint"]["allowed"] is True + assert policy["actions"]["verify-control-device-info"]["allowed"] is False + assert policy["actions"]["start-acquisition"]["allowed"] is False + + +def test_connection_policy_data_loss_preserves_exact_stop_and_falls_back_to_local_only( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + supervisor = service._connection_supervisor # noqa: SLF001 + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="healthy", + session_id="data-session-a", + ) + assert supervisor.observe_data_plane( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + state="lost", + session_id="data-session-a", + reason_code="test-data-loss", + ) + + healthy_control = _project_connection_policy_for_test( + service, + acquisition_active=True, + runtime_active=True, + ) + + assert healthy_control["actions"]["acknowledge-data-loss"]["allowed"] is False + assert healthy_control["actions"]["stop-acquisition"]["allowed"] is True + assert healthy_control["actions"]["stop-local-receiver"]["allowed"] is True + assert healthy_control["actions"]["stop-local-receiver"]["physical_command_allowed"] is False + assert healthy_control["actions"]["stop-local-receiver"]["physical_outcome"] == ("unknown") + assert healthy_control["recommended_action"] == "stop-acquisition" + + assert supervisor.observe_control_loss( + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + control_session_id="test-control-test-ble-transport", + reason_code="test-control-loss", + ) + control_lost = _project_connection_policy_for_test( + service, + acquisition_active=True, + runtime_active=True, + ) + + assert control_lost["actions"]["stop-acquisition"]["allowed"] is False + assert ( + "physical-control-authority-unavailable" + in control_lost["actions"]["stop-acquisition"]["reason_codes"] + ) + assert control_lost["actions"]["stop-local-receiver"]["allowed"] is True + assert control_lost["actions"]["stop-local-receiver"]["execution_mode"] == ("capture-only") + assert control_lost["actions"]["stop-local-receiver"]["physical_command_allowed"] is False + assert control_lost["recommended_action"] == "stop-local-receiver" + + +@pytest.mark.parametrize( + ("acquisition_active", "cleanup_pending", "runtime_active", "expected_allowed"), + [ + (True, False, False, True), + (False, True, False, True), + (False, False, True, True), + (False, False, False, False), + ], +) +def test_connection_policy_local_receiver_cleanup_uses_independent_local_liveness( + tmp_path: Path, + acquisition_active: bool, + cleanup_pending: bool, + runtime_active: bool, + expected_allowed: bool, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection(service) + + policy = _project_connection_policy_for_test( + service, + acquisition_active=acquisition_active, + acquisition_cleanup_pending=cleanup_pending, + runtime_active=runtime_active, + ) + + local_cleanup = policy["actions"]["stop-local-receiver"] + assert local_cleanup["allowed"] is expected_allowed + assert local_cleanup["execution_mode"] == "capture-only" + assert local_cleanup["physical_command_allowed"] is False + assert local_cleanup["physical_outcome"] == "unknown" + if expected_allowed: + assert local_cleanup["reason_codes"] == [] + else: + assert local_cleanup["reason_codes"] == ["local-acquisition-receiver-not-active"] + + # Local cleanup eligibility never broadens the separately gated physical STOP. + assert policy["actions"]["stop-acquisition"]["allowed"] is False + + +def test_connection_policy_current_target_allows_host_probe_not_ble_refresh( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service) + assert service._connection_supervisor.observe_control_loss( # noqa: SLF001 + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + control_session_id="test-control-test-ble-transport", + reason_code="test-control-loss", + ) + + policy = _project_connection_policy_for_test( + service, + current_device_recovery=None, + ) + + assert service._connection_supervisor.snapshot().last_known is not None # noqa: SLF001 + assert policy["actions"]["inspect-configured-endpoint"]["allowed"] is True + assert ( + policy["actions"]["inspect-configured-endpoint"]["requires_live_gatt_validation"] is False + ) + assert policy["actions"]["inspect-configured-endpoint"]["target_source"] == ( + "configured-topology" + ) + assert policy["actions"]["inspect-configured-endpoint"]["required_transport_ref"] is None + assert policy["actions"]["observe-fresh-device-network"]["allowed"] is False + assert ( + "fresh-ble-candidate-required" + in policy["actions"]["observe-fresh-device-network"]["reason_codes"] + ) + + +def test_configured_endpoint_probe_uses_durable_topology_after_restart_without_ble( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + first, _ = service_with_fake_runtime(tmp_path) + topology_store = first._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref="durable-k1-transport", + connection_mode="bridge", + ipv4="192.168.1.20", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + + restarted, _ = service_with_fake_runtime(tmp_path) + ble_calls: list[str] = [] + + async def forbidden_ble(*_: object, **__: object) -> dict[str, Any]: + ble_calls.append("ble") + raise AssertionError("configured endpoint probe must not enter BLE") + + def forbidden_ble_lease(*_: object, **__: object) -> object: + ble_calls.append("ble-lease") + raise AssertionError("configured endpoint probe must not borrow the BLE lease") + + def forbidden_authoritative_probe(*_: object, **__: object) -> object: + raise AssertionError("configured endpoint probe must not update the supervisor") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_ble) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_ble) + monkeypatch.setattr( + facade_module, + "borrow_ble_runtime_process_lease", + forbidden_ble_lease, + ) + monkeypatch.setattr(restarted, "_probe_control_endpoint", forbidden_authoritative_probe) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: facade_module.TcpReachabilityProbeResult( + reachable=target == "192.168.1.20", + reason_code=(None if target == "192.168.1.20" else "tcp-endpoint-unreachable"), + ), + ) + + before = restarted.state() + assert before["selected_device_id"] is None + assert before["connection_policy"]["actions"]["inspect-configured-endpoint"]["allowed"] is True + assert ( + before["connection_policy"]["actions"]["inspect-configured-endpoint"]["target_source"] + == "configured-topology" + ) + assert ( + before["connection_policy"]["actions"]["inspect-configured-endpoint"][ + "required_transport_ref" + ] + is None + ) + assert ( + before["connection_policy"]["actions"]["observe-fresh-device-network"]["allowed"] is False + ) + + state = asyncio.run( + restarted.probe_configured_endpoint( + ConfiguredEndpointProbeRequest( + operation_id="op-00000000-0000-4000-8000-000000000651", + ) + ) + ) + + assert ble_calls == [] + assert state["selected_device_id"] is None + assert state["connection_verification"]["status"] == "not-probed" + assert state["configured_endpoint_probe"] == { + "schema_version": "missioncore.xgrids-k1-configured-endpoint-probe/v1", + "status": "reachable", + "target_source": "durable-semantic-topology", + "connection_mode": "bridge", + "endpoint": "192.168.1.20", + "transport_ref": "durable-k1-transport", + "intent_id": None, + "semantic_revision": 1, + "host_route_available": True, + "host_route_class": "direct", + "tcp_reachable": True, + "identity_validation": "not-performed", + "control_authority_granted": False, + "ble_operation_performed": False, + "network_mutation_performed": False, + "automatic_retry": False, + "observed_at": state["configured_endpoint_probe"]["observed_at"], + "reason_code": None, + } + assert state["connection_supervisor"]["authority"]["control_allowed"] is False + assert state["last_operation"]["action"] == "connection.endpoint-probe" + assert state["last_operation"]["status"] == "succeeded" + assert state["last_operation"]["result"]["identity_validation"] == "not-performed" + assert state["last_operation"]["result"]["network_mutation_performed"] is False + + +def test_configured_endpoint_probe_reports_unreachable_without_promoting_authority( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + topology_store = service._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref="durable-k1-transport", + connection_mode="quick-connect", + ipv4="192.168.56.1", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda _target: facade_module.TcpReachabilityProbeResult( + reachable=False, + reason_code="tcp-endpoint-unreachable", + ), + ) + + state = asyncio.run(service.probe_configured_endpoint()) + + probe = state["configured_endpoint_probe"] + assert probe["status"] == "endpoint-unreachable" + assert probe["host_route_available"] is True + assert probe["tcp_reachable"] is False + assert probe["reason_code"] == "tcp-endpoint-unreachable" + assert probe["control_authority_granted"] is False + assert state["connection_supervisor"]["authority"]["control_allowed"] is False + + +def test_configured_endpoint_probe_prefers_current_supervisor_target( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + topology_store = service._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + topology_store.commit( + transport_ref="older-durable-transport", + connection_mode="bridge", + ipv4="192.168.1.99", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:00:00Z", + ) + _seed_supervised_connection( + service, + target_ipv4="192.168.1.20", + with_control=False, + ) + probed_targets: list[str] = [] + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + probed_targets.append(target) + or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + state = asyncio.run(service.probe_configured_endpoint()) + + assert probed_targets == ["192.168.1.20"] + assert state["configured_endpoint_probe"]["target_source"] == "current-supervisor" + assert state["configured_endpoint_probe"]["endpoint"] == "192.168.1.20" + assert state["configured_endpoint_probe"]["transport_ref"] == "test-ble-transport" + assert state["configured_endpoint_probe"]["intent_id"] is not None + assert state["configured_endpoint_probe"]["semantic_revision"] is None + assert state["configured_endpoint_probe"]["identity_validation"] == "not-performed" + + +def test_configured_endpoint_probe_requires_current_or_durable_target( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + with pytest.raises( + facade_module.ConfiguredEndpointProbeError, + match="Нет сохранённого endpoint", + ) as raised: + asyncio.run(service.probe_configured_endpoint()) + + assert raised.value.reason_code == "configured-endpoint-unavailable" + operation = service.state()["last_operation"] + assert operation["action"] == "connection.endpoint-probe" + assert operation["status"] == "failed" + assert operation["error"]["side_effect_status"] == "none" + assert operation["error"]["safe_to_retry"] is True + + +def test_configured_endpoint_probe_maps_untrusted_store_to_action_scoped_error( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._semantic_topology_store = None # noqa: SLF001 + + with pytest.raises(facade_module.ConfiguredEndpointProbeError) as raised: + asyncio.run(service.probe_configured_endpoint()) + + assert raised.value.reason_code == "configured-endpoint-topology-corrupt" + operation = service.state()["last_operation"] + assert operation["action"] == "connection.endpoint-probe" + assert operation["status"] == "failed" + assert operation["error"]["side_effect_status"] == "none" + + +@pytest.mark.parametrize( + ( + "previous_mode", + "previous_session_id", + "previous_ipv4", + "requested_mode", + "requested_ssid", + "requested_password", + "requested_attestation", + ), + [ + pytest.param( + "quick-connect", + "quick-session-a", + facade_module.AP_FALLBACK_IPV4, + "bridge", + "lab-router", + SecretStr(PRIMARY_TEST_CREDENTIAL), + ATTESTATION, + id="quick-to-bridge", + ), + pytest.param( + "bridge", + "bridge-session-a", + "192.168.68.50", + "quick-connect", + None, + None, + QUICK_CONNECT_ATTESTATION, + id="bridge-to-quick-connect", + ), + ], +) +def test_fresh_cross_mode_prewrite_failure_leaves_clean_unselected_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + previous_mode: str, + previous_session_id: str, + previous_ipv4: str, + requested_mode: str, + requested_ssid: str | None, + requested_password: SecretStr | None, + requested_attestation: CompatibilityAttestationRequest, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._selected_device_id = "k1-a" # noqa: SLF001 + service._connection_mode = previous_mode # type: ignore[assignment] # noqa: SLF001 + service._device_id = "device-a" # noqa: SLF001 + service._device_session_id = previous_session_id # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + service._device_session_discovery_generation = 0 # noqa: SLF001 + service._k1_ip = previous_ipv4 # noqa: SLF001 + mode_revision = _select_connection_mode(service, requested_mode) + service._ble_discovery_generation += 1 # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + captured = facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address="k1-a"), # type: ignore[arg-type] + macos_uuid="k1-a", + owner_epoch=7, + scan_generation=service._ble_discovery_generation, # noqa: SLF001 + ) + prewrite_attempts: list[tuple[str, object | None]] = [] + + def prewrite_failure(captured_device: object | None) -> RuntimeError: + prewrite_attempts.append((requested_mode, captured_device)) + error = RuntimeError("powered off after fresh advertisement") + error.operation_stage = "connect" # type: ignore[attr-defined] + error.device_write_attempted = False # type: ignore[attr-defined] + error.device_write_confirmed = False # type: ignore[attr-defined] + return error + + async def bridge_powered_off_before_write( + *_args: object, + captured_device: object | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + raise prewrite_failure(captured_device) + + @asynccontextmanager + async def quick_powered_off_before_write( + *_args: object, + captured_device: object | None = None, + **_kwargs: object, + ) -> AsyncIterator[dict[str, Any]]: + raise prewrite_failure(captured_device) + yield {} + + monkeypatch.setattr( + facade_module, + "_capture_network_intent_device", + lambda _device_id: captured, + ) + monkeypatch.setattr( + facade_module, + "connected_device_capture", + lambda *_args, **_kwargs: None, + ) + if requested_mode == "quick-connect": + monkeypatch.setattr( + facade_module, + "ensure_wifi_profile_from_credential_source", + lambda *_args, **_kwargs: { + "schema_version": 1, + "adapter": "macOS Keychain", + "available": True, + "profile_enrolled": True, + "credential_source": "exact-firmware-profile", + }, + ) + monkeypatch.setattr( + facade_module, + "device_ap_activation_session", + quick_powered_off_before_write, + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("Quick Connect must not use the Bridge provisioning write") + ), + ) + else: + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + bridge_powered_off_before_write, + ) + monkeypatch.setattr( + facade_module, + "device_ap_activation_session", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("Bridge must not use the Quick Connect AP write") + ), + ) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("a failed pre-write recovery must not promote a new session") + ), + ) + + with pytest.raises(RuntimeError, match="powered off after fresh advertisement"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid=requested_ssid, + password=requested_password, + connection_mode=requested_mode, + compatibility_attestation=requested_attestation, + expected_mode_revision=mode_revision, + expected_discovery_generation=( + service._ble_discovery_generation # noqa: SLF001 + ), + ) + ) + ) + + assert prewrite_attempts == [(requested_mode, captured)] + assert service._selected_device_id is None # noqa: SLF001 + assert service._connection_mode is None # noqa: SLF001 + assert service._device_id == "device-a" # noqa: SLF001 + assert service._device_session_id is None # noqa: SLF001 + assert service._k1_ip is None # noqa: SLF001 + state = service.state() + assert state["current_device_recovery"] is None + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["error"]["safe_to_retry"] is True + assert operation["error"]["side_effect_status"] == "none" + assert operation["error"]["operation_stage"] == "connect" + assert state["network_mutation_ledger"]["status"] == "empty" + assert state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +def test_same_uuid_rediscovery_is_candidate_only_and_never_resurrects_session( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._selected_device_id = "k1-a" # noqa: SLF001 + service._connection_mode = "bridge" # noqa: SLF001 + service._device_id = "device-a" # noqa: SLF001 + service._device_session_id = "ended-session-a" # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + service._device_session_discovery_generation = 0 # noqa: SLF001 + service._k1_ip = "192.168.68.50" # noqa: SLF001 + invalidations: list[tuple[str, str]] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda device_id, *, device_session_id: ( + invalidations.append((device_id, device_session_id)) or True + ), + ) + + service._retire_ephemeral_device_binding_for_new_intent() # noqa: SLF001 + assert invalidations == [("k1-a", "ended-session-a")] + + discovery_calls: list[float] = [] + gatt_or_session_edges: list[str] = [] + + async def rediscover_same_uuid( + duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + discovery_calls.append(duration_seconds) + assert on_admitted is not None + on_admitted() + return _ble_scan_result("k1-a") + + def forbidden_edge(*_args: object, **_kwargs: object) -> None: + gatt_or_session_edges.append("unexpected") + raise AssertionError("rediscovery must remain candidate-only until explicit Connect") + + monkeypatch.setattr(facade_module, "scan", rediscover_same_uuid) + monkeypatch.setattr(facade_module, "_capture_network_intent_device", forbidden_edge) + monkeypatch.setattr(facade_module, "connected_device_capture", forbidden_edge) + monkeypatch.setattr(facade_module, "pin_connected_device_handle", forbidden_edge) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_edge) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_edge) + monkeypatch.setattr(facade_module, "device_ap_activation_session", forbidden_edge) + + rediscovered = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + + assert discovery_calls == [1.0] + assert gatt_or_session_edges == [] + assert rediscovered["devices"] == [ + { + "device_id": "k1-a", + "name": "XGR-K1", + "rssi": -44, + "address": None, + "connectable": None, + "likely_k1": True, + } + ] + assert rediscovered["selected_device_id"] is None + assert rediscovered["ble_discovery_generation"] == 1 + assert rediscovered["device_session"] is None + assert rediscovered["current_device_recovery"] is None + assert rediscovered["connection_mode"] is None + assert rediscovered["k1_ip"] is None + assert rediscovered["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +def test_quick_to_bridge_recovery_rejects_nonterminal_control_before_retirement( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._selected_device_id = "k1-a" # noqa: SLF001 + service._connection_mode = "quick-connect" # noqa: SLF001 + service._device_session_id = "quick-session-a" # noqa: SLF001 + service._k1_ip = facade_module.AP_FALLBACK_IPV4 # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + captured = facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address="k1-a"), # type: ignore[arg-type] + macos_uuid="k1-a", + owner_epoch=7, + ) + + class BusyControlSession: + def snapshot(self) -> dict[str, object]: + return {"state": "scanning", "failure": None} + + def retire_for_network_change(self, **_: object) -> dict[str, object]: + raise AssertionError("recovery admission must fail before retirement") + + service._application_control_session = BusyControlSession() # type: ignore[assignment] # noqa: SLF001 + monkeypatch.setattr( + facade_module, + "_capture_network_intent_device", + lambda *_args, **_kwargs: captured, + ) + monkeypatch.setattr( + facade_module, + "connected_device_capture", + lambda *_args, **_kwargs: captured, + ) + monkeypatch.setattr( + facade_module, + "connected_device_recovery_snapshot", + lambda *_args, **_kwargs: { + "status": "retained", + "scope": "owner-epoch-device-session", + "advertised_now": False, + "gatt_validated_recently": False, + }, + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("rejected recovery must not reach BLE") + ), + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as raised: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == "network-provision-recovery-control-not-admissible" + assert service._connection_mode == "quick-connect" # noqa: SLF001 + assert service._device_session_id == "quick-session-a" # noqa: SLF001 + + +@pytest.mark.parametrize( + "failure_reason", + ["mqtt_network_loop_failed", "application-connection-binding-lost"], +) +def test_terminal_control_loss_retires_ephemeral_device_session_without_retry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + failure_reason: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-a", + ) + service._device_session_id = "quick-session-a" # noqa: SLF001 + service._device_session_opened_at = "2026-08-06T12:00:00Z" # noqa: SLF001 + supervisor_generation = service._connection_supervisor.snapshot().lease.generation # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "status": "reachable", + "lease_state": "reachable", + "lease_generation": supervisor_generation, + "endpoint_validation": "provisioning-status+mqtt-tcp-connect", + "network_reachability": "reachable", + "write_performed": True, + "observed_at": "2026-08-06T12:00:00Z", + } + + class FailedControlSession: + retire_calls = 0 + retired = False + + def snapshot(self) -> dict[str, object]: + if self.retired: + return {"state": "idle", "can_open": True, "failure": None} + return { + "state": "failed", + "failure": { + "reason_code": failure_reason, + "failed_phase": "maintain-open", + }, + } + + def retire_for_network_change(self, **_: object) -> dict[str, object]: + self.retire_calls += 1 + if self.retire_calls == 1: + raise RuntimeError("control worker is still retiring") + self.retired = True + return {"state": "idle", "can_open": True} + + service._application_control_session = FailedControlSession() # type: ignore[assignment] # noqa: SLF001 + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + + state = service.state() + + assert invalidations == [True] + assert state["k1_ip"] is None + assert state["connection_mode"] is None + assert state["device_session"] is None + assert state["current_device_recovery"] is None + assert service._selected_device_id is None # noqa: SLF001 + assert state["connection_supervisor"]["authority"]["control_allowed"] is False + assert state["connection_supervisor"]["authority"]["acquisition_start_allowed"] is False + + _set_scanned_k1(service, device_id="k1-a") + fresh = service.state() + assert service._application_control_session.retire_calls == 2 # type: ignore[attr-defined] # noqa: SLF001 + assert fresh["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +def test_configured_unverified_endpoint_loss_retires_ephemeral_session( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + with_control=False, + ) + assert service._connection_supervisor.snapshot().last_known is None # noqa: SLF001 + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.endpoint.target is not None + + for attempt in range(3): + accepted = supervisor.observe_endpoint( + target=snapshot.endpoint.target, + intent_id=snapshot.intent.intent_id, + host_path_epoch=snapshot.host_path.epoch, + reachable=False, + reason_code="k1-control-port-unreachable", + ) + assert accepted is True + state = service.state() + if attempt < 2: + assert state["device_session"] is not None + + assert invalidations == [True] + assert state["device_session"] is None + assert state["connection_mode"] is None + assert state["k1_ip"] is None + _set_scanned_k1(service, device_id="k1-a") + fresh = service.state() + assert fresh["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +def test_endpoint_loss_preserves_same_generation_fresh_candidate_for_reprovision( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + with_control=False, + ) + _set_scanned_k1(service, device_id="k1-a") + discovery_generation = service._ble_discovery_generation # noqa: SLF001 + invalidations: list[tuple[str, str]] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda device_id, *, device_session_id: ( + invalidations.append((device_id, device_session_id)) or True + ), + ) + supervisor = service._connection_supervisor # noqa: SLF001 + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.endpoint.target is not None + + for _ in range(3): + accepted = supervisor.observe_endpoint( + target=snapshot.endpoint.target, + intent_id=snapshot.intent.intent_id, + host_path_epoch=snapshot.host_path.epoch, + reachable=False, + reason_code="k1-control-port-unreachable", + ) + assert accepted is True + state = service.state() + + assert invalidations == [("k1-a", "test-session-k1-a")] + assert state["device_session"] is None + assert state["selected_device_id"] is None + assert state["k1_ip"] is None + assert service._ble_discovery_generation == discovery_generation # noqa: SLF001 + assert [item["device_id"] for item in state["devices"]] == ["k1-a"] + assert state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + assert "свежий Bluetooth-кандидат можно настроить заново" in state["message"] + + +def test_three_proven_host_path_losses_retire_session_and_require_new_scan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + ) + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + first = service.state() + assert first["device_session"] is not None + assert first["connection_mode"] == "bridge" + assert invalidations == [] + + # Unrelated supervisor revisions must not count as additional host-path + # failures while the same negative observation remains current. + supervisor = service._connection_supervisor # noqa: SLF001 + current_intent = supervisor.snapshot().intent + assert current_intent is not None + for _ in range(2): + supervisor.set_intent( + intent_id=current_intent.intent_id, + requested_mode=current_intent.requested_mode, + expected_device_id=current_intent.expected_device_id, + ) + unchanged = service.state() + assert unchanged["device_session"] is not None + assert invalidations == [] + + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + second = service.state() + assert second["device_session"] is not None + assert invalidations == [] + + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + third = service.state() + assert invalidations == [True] + assert third["device_session"] is None + assert third["connection_mode"] is None + assert third["k1_ip"] is None + assert service._selected_device_id is None # noqa: SLF001 + assert service._connection_monitor_target() is None # noqa: SLF001 + assert third["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + + _set_scanned_k1(service, device_id="k1-a") + fresh = service.state() + assert fresh["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +@pytest.mark.parametrize("plane", ["host", "endpoint"]) +def test_positive_transport_edge_resets_loss_streak_between_state_polls( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + plane: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + with_control=False, + ) + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + + def observe(reachable: bool) -> None: + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.device_network.target is not None + if plane == "host": + supervisor.observe_host_path( + _direct_host_path("192.168.68.50") + if reachable + else HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + ) + else: + accepted = supervisor.observe_endpoint( + target=snapshot.device_network.target, + intent_id=snapshot.intent.intent_id, + host_path_epoch=snapshot.host_path.epoch, + reachable=reachable, + reason_code=None if reachable else "k1-control-port-unreachable", + ) + assert accepted is True + + observe(False) + assert service.state()["device_session"] is not None + observe(True) # The UI does not poll state() on this recovery edge. + observe(False) + assert service.state()["device_session"] is not None + observe(False) + assert service.state()["device_session"] is not None + assert invalidations == [] + observe(False) + assert service.state()["device_session"] is None + assert invalidations == [True] + + +@pytest.mark.parametrize("plane", ["host", "endpoint"]) +def test_positive_edge_between_loss_confirmation_and_teardown_cancels_retirement( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + plane: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + with_control=False, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + + def observe(reachable: bool) -> None: + snapshot = supervisor.snapshot() + assert snapshot.intent is not None + assert snapshot.device_network.target is not None + if plane == "host": + supervisor.observe_host_path( + _direct_host_path("192.168.68.50") + if reachable + else HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + ) + else: + accepted = supervisor.observe_endpoint( + target=snapshot.device_network.target, + intent_id=snapshot.intent.intent_id, + host_path_epoch=snapshot.host_path.epoch, + reachable=reachable, + reason_code=None if reachable else "k1-control-port-unreachable", + ) + assert accepted is True + + for _ in range(3): + observe(False) + + original_snapshot = supervisor.snapshot + snapshot_calls = 0 + + def snapshot_with_recovery_edge() -> Any: + nonlocal snapshot_calls + snapshot_calls += 1 + if snapshot_calls == 2: + before_recovery = original_snapshot() + assert before_recovery.intent is not None + assert before_recovery.device_network.target is not None + if plane == "host": + supervisor.observe_host_path(_direct_host_path("192.168.68.50")) + else: + accepted = supervisor.observe_endpoint( + target=before_recovery.device_network.target, + intent_id=before_recovery.intent.intent_id, + host_path_epoch=before_recovery.host_path.epoch, + reachable=True, + ) + assert accepted is True + return original_snapshot() + + monkeypatch.setattr(supervisor, "snapshot", snapshot_with_recovery_edge) + invalidations: list[bool] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda *_args, **_kwargs: invalidations.append(True) or True, + ) + + service._retire_ephemeral_connection_binding_on_proven_loss( # noqa: SLF001 + service._application_control_session.snapshot() # noqa: SLF001 + ) + + assert snapshot_calls == 2 + assert service._device_session_id is not None # noqa: SLF001 + assert service._selected_device_id == "k1-a" # noqa: SLF001 + assert invalidations == [] + + +def test_old_loss_teardown_preserves_candidate_from_new_scan_generation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + ) + service._device_session_discovery_generation = 4 # noqa: SLF001 + service._ble_discovery_generation = 4 # noqa: SLF001 + invalidations: list[tuple[str, str]] = [] + monkeypatch.setattr( + facade_module, + "invalidate_connected_device_session", + lambda device_id, *, device_session_id: ( + invalidations.append((device_id, device_session_id)) or True + ), + ) + lost_path = HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ) + + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + service.state() + # A new explicit scan generation completes while old-loss confirmation is + # still pending. Its selected candidate belongs to the next operator + # intent and must not be erased by teardown of the old device-session. + service._ble_discovery_generation = 5 # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + service.state() + service._connection_supervisor.observe_host_path(lost_path) # noqa: SLF001 + state = service.state() + + assert invalidations == [("k1-a", "test-session-k1-a")] + assert state["device_session"] is None + assert [item["device_id"] for item in state["devices"]] == ["k1-a"] + assert state["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + + +def test_connection_mode_and_discovery_cas_reject_stale_connect_before_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + initial = service.state() + assert initial["desired_connection_mode"] == "bridge" + assert initial["desired_connection_mode_revision"] == 0 + + selected = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=0, + ) + ) + assert selected["desired_connection_mode"] == "quick-connect" + assert selected["desired_connection_mode_revision"] == 1 + assert selected["configured_connection_mode"] is None + assert selected["active_connection_mode"] is None + assert selected["ble_discovery_generation"] == 0 + + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_select: + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="direct-connect", + expected_revision=0, + ) + ) + assert stale_select.value.reason_code == "connection-mode-draft-revision-conflict" + + io_calls: list[str] = [] + + def forbidden_preflight(*_: object, **__: object) -> dict[str, Any]: + io_calls.append("host-preflight") + raise AssertionError("stale mode/generation must fail before host or device I/O") + + async def forbidden_provision(*_: object, **__: object) -> dict[str, Any]: + io_calls.append("ble-write") + raise AssertionError("stale mode/generation must fail before BLE") + + monkeypatch.setattr( + facade_module, + "ensure_wifi_profile_from_credential_source", + forbidden_preflight, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_provision) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_mode: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + connection_mode="quick-connect", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=0, + ) + ) + ) + assert stale_mode.value.reason_code == "connection-mode-draft-revision-conflict" + + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_scan: + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + connection_mode="quick-connect", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_mode_revision=1, + expected_discovery_generation=1, + ) + ) + ) + assert stale_scan.value.reason_code == "network-provision-discovery-generation-conflict" + assert io_calls == [] + assert service.state()["operations"] == [] + + +def test_pending_mode_scan_retires_prestart_control_without_physical_edge( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-a", + ) + control = FakeInteractiveControlSession(initial_state="idle") + control.open(connection_binding=binding) + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id="device-a", + ) + close_calls: list[tuple[int | None, int | None]] = [] + original_close_prestart = control.close_prestart + + def counted_close_prestart( + _self: FakeInteractiveControlSession, + *, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + close_calls.append((expected_session_generation, expected_state_revision)) + return original_close_prestart( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + control.close_prestart = MethodType(counted_close_prestart, control) # type: ignore[method-assign] + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + service._acquire_application_control_process_lease() # noqa: SLF001 + ready = service.state() + assert ready["active_connection_mode"] == "quick-connect" + assert ready["connection_lifecycle"]["connection_ready"] is True + expected_close_checkpoint = ( + ready["application_control_session"]["session_generation"], + ready["application_control_session"]["state_revision"], + ) + + switched = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=ready["desired_connection_mode_revision"], + ) + ) + assert switched["connection_lifecycle"]["mode_change"] == { + "state": "switch-selected", + "from": "quick-connect", + "to": "bridge", + } + assert switched["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert switched["connection_policy"]["actions"]["start-acquisition"]["allowed"] is False + assert ( + "connection-mode-switch-pending" + in switched["connection_policy"]["actions"]["start-acquisition"]["reason_codes"] + ) + + device_edges: list[str] = [] + + async def fake_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert control.snapshot()["state"] == "idle" + assert service._connection_supervisor.snapshot().authority.control_allowed is False # noqa: SLF001 + assert service._selected_device_id is None # noqa: SLF001 + assert on_admitted is not None + on_admitted() + return _ble_scan_result("bridge-candidate") + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("mode selection/scan must not write device network state") + + monkeypatch.setattr(facade_module, "scan", fake_scan) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + scanned = asyncio.run(service.scan_ble(1.0)) + + assert close_calls == [expected_close_checkpoint] + assert device_edges == [] + assert scanned["desired_connection_mode"] == "bridge" + assert scanned["configured_connection_mode"] is None + assert scanned["active_connection_mode"] is None + assert scanned["connection_lifecycle"]["connection_ready"] is False + assert [item["device_id"] for item in scanned["devices"]] == ["bridge-candidate"] + repeated = service.state() + assert repeated["active_connection_mode"] is None + assert repeated["selected_device_id"] is None + + +def test_pending_reopened_physical_state_blocks_connect_and_mode_scan_before_device_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + device_edges: list[str] = [] + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("pending physical reopen must fail before GATT/Wi-Fi") + + async def forbidden_scan(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("ble-scan") + raise AssertionError("unsafe mode transition must fail before BLE scan") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + monkeypatch.setattr(facade_module, "scan", forbidden_scan) + + connect_service, _ = service_with_fake_runtime(tmp_path / "connect") + _set_scanned_k1(connect_service, device_id="k1-a") + pending_bridge = _pending_reopened_physical_snapshot() + monkeypatch.setattr( + connect_service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: pending_bridge, + ) + connect_topology_before = ( + connect_service._selected_device_id, # noqa: SLF001 + connect_service._k1_ip, # noqa: SLF001 + connect_service._connection_mode, # noqa: SLF001 + connect_service._connection_supervisor.snapshot(), # noqa: SLF001 + ) + + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked_connect: + asyncio.run( + connect_service.connect( + _connect_request( + device_id="k1-a", + ssid="synthetic-network", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert blocked_connect.value.reason_code == "physical-command-reconciliation-required" + assert ( + connect_service._selected_device_id, # noqa: SLF001 + connect_service._k1_ip, # noqa: SLF001 + connect_service._connection_mode, # noqa: SLF001 + connect_service._connection_supervisor.snapshot(), # noqa: SLF001 + ) == connect_topology_before + + mode_service, _ = service_with_fake_runtime(tmp_path / "mode") + _seed_supervised_connection( + mode_service, + transport_ref="k1-a", + connection_mode="bridge", + ) + pending_mode = _pending_reopened_physical_snapshot() + monkeypatch.setattr( + mode_service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: pending_mode, + ) + with mode_service._lock: # noqa: SLF001 + mode_service._desired_connection_mode = "quick-connect" # noqa: SLF001 + mode_service._desired_connection_mode_revision += 1 # noqa: SLF001 + mode_topology_before = ( + mode_service._selected_device_id, # noqa: SLF001 + mode_service._k1_ip, # noqa: SLF001 + mode_service._connection_mode, # noqa: SLF001 + mode_service._connection_supervisor.snapshot(), # noqa: SLF001 + ) + + with pytest.raises(facade_module.BleDiscoveryUnavailable) as blocked_scan: + asyncio.run(mode_service.scan_ble(1.0)) + assert blocked_scan.value.reason_code == "connection-mode-switch-physical-state-unsafe" + assert ( + mode_service._selected_device_id, # noqa: SLF001 + mode_service._k1_ip, # noqa: SLF001 + mode_service._connection_mode, # noqa: SLF001 + mode_service._connection_supervisor.snapshot(), # noqa: SLF001 + ) == mode_topology_before + assert device_edges == [] + + +def test_state_polling_auto_retires_terminal_prestart_control_once( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + + class FailedPrestartControl: + def __init__(self) -> None: + self.retire_calls = 0 + self.retired = False + + def snapshot(self) -> dict[str, object]: + if self.retired: + return {"state": "idle", "can_open": True, "failure": None} + return { + "state": "failed", + "can_open": True, + "failure": { + "reason_code": "mqtt-device-info-timeout", + "modeling_command_attempted": False, + "safe_to_retry": True, + }, + } + + def retire_for_network_change(self, **_: object) -> dict[str, object]: + self.retire_calls += 1 + self.retired = True + return self.snapshot() + + control = FailedPrestartControl() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + service._acquire_application_control_process_lease() # noqa: SLF001 + + first = service.state() + second = service.state() + + assert control.retire_calls == 1 + assert first["application_control_session"]["state"] == "idle" + assert second["application_control_session"]["state"] == "idle" + assert first["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + + +def _install_authoritative_ready_bridge_control( + service: XgridsK1CompatibilityService, + *, + transport_ref: str = "k1-a", +) -> tuple[FakeInteractiveControlSession, ApplicationConnectionBinding, dict[str, Any]]: + binding = _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref=transport_ref, + logical_device_id="device-a", + ) + control = FakeInteractiveControlSession(initial_state="idle") + control.open(connection_binding=binding) + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id="device-a", + control_session_id=f"test-control-{transport_ref}", + control_proof_revision=2, + ) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + service._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + service.runtime.snapshot(), + ) + service._acquire_application_control_process_lease() # noqa: SLF001 + state = service.state() + assert state["active_connection_mode"] == "bridge" + assert state["connection_lifecycle"]["active_binding_key"] is not None + return control, binding, state + + +def _reconfiguration_request( + state: dict[str, Any], + intent: str, +) -> PrepareConnectionReconfigurationRequest: + reconfiguration = state["connection_reconfiguration"] + lifecycle = state["connection_lifecycle"] + return PrepareConnectionReconfigurationRequest( + intent=intent, # type: ignore[arg-type] + expected_reconfiguration_revision=reconfiguration["revision"], + expected_reconfiguration_intent_id=reconfiguration["intent_id"], + expected_desired_mode_revision=state["desired_connection_mode_revision"], + expected_active_binding_key=lifecycle["active_binding_key"], + ) + + +def _seed_durable_bridge_topology( + service: XgridsK1CompatibilityService, + *, + transport_ref: str, + ipv4: str, +) -> None: + store = service._semantic_topology_store # noqa: SLF001 + assert store is not None + snapshot = store.snapshot() + store.commit( + transport_ref=transport_ref, + connection_mode="bridge", + ipv4=ipv4, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-10T12:00:00Z", + predecessor_revision=(snapshot.record.revision if snapshot.record is not None else 0), + ) + + +def _install_successful_bridge_verify_transport( + monkeypatch: pytest.MonkeyPatch, + *, + device_id: str, + ipv4: str, + writes: list[str], +) -> None: + async def existing_network_status(*_: object, **__: object) -> dict[str, Any]: + return { + **_wifi_status_read(ipv4), + "device_macos_uuid": device_id, + } + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + writes.append("network-write") + raise AssertionError("read-only Verify must not provision Wi-Fi") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", existing_network_status) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + facade_module, + "_host_route_class", + lambda _target: "direct-or-routed", + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + +def test_read_only_monitor_holder_does_not_flap_reconfiguration_policy( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, _binding, ready = _install_authoritative_ready_bridge_control(service) + ready_binding_key = ready["connection_lifecycle"]["active_binding_key"] + ready_host_epoch = ready["connection_supervisor"]["observed"]["host_path"]["epoch"] + + assert ready["connection_policy"]["actions"]["prepare-select-device"]["allowed"] is True + assert ready["connection_policy"]["actions"]["prepare-change-network"]["allowed"] is True + + service._acquire_k1_lifecycle_process_lease("monitor") # noqa: SLF001 + try: + during_probe = service.state() + finally: + service._release_k1_lifecycle_process_lease("monitor") # noqa: SLF001 + + assert "monitor" in during_probe["k1_lifecycle_process_lease"]["holders"] + assert during_probe["connection_lifecycle"]["active_binding_key"] == ready_binding_key + assert ( + during_probe["connection_supervisor"]["observed"]["host_path"]["epoch"] == ready_host_epoch + ) + for action in ("prepare-select-device", "prepare-change-network"): + decision = during_probe["connection_policy"]["actions"][action] + assert decision["allowed"] is True + assert "k1-lifecycle-process-lease-network-owned" not in decision["reason_codes"] + + +def test_monitor_transient_wifi_observer_failures_retain_verified_control_and_refresh_tcp( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + initial = service._connection_supervisor.snapshot() # noqa: SLF001 + assert initial.authority.control_allowed is True + tcp_calls: list[str] = [] + + def association_observation( + reason_code: str, + token: str, + ) -> HostWifiAssociationIdentityResult: + return { + "schema_version": 1, + "adapter": "unavailable", + "wifi_interface": None, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": token, + "reason_code": reason_code, + } + + association_calls = 0 + + def alternating_unproven_association( + _interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + nonlocal association_calls + assert timeout_seconds == (facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS) + association_calls += 1 + return ( + association_observation("host-wifi-operation-timeout", "a" * 64) + if association_calls % 2 + else association_observation("association-identity-unavailable", "b" * 64) + ) + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + alternating_unproven_association, + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_calls.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + async def three_mixed_cycles() -> tuple[ + facade_module.ConnectionSupervisorSnapshot, + facade_module.ConnectionSupervisorSnapshot, + facade_module.ConnectionSupervisorSnapshot, + ]: + first = await service._connection_monitor.poll_once() # noqa: SLF001 + second = await service._connection_monitor.poll_once() # noqa: SLF001 + third = await service._connection_monitor.poll_once() # noqa: SLF001 + return first, second, third + + retained_cycles = asyncio.run(three_mixed_cycles()) + + assert association_calls == 6 + assert tcp_calls == [binding.target_ipv4] * 3 + for retained in retained_cycles: + assert retained.revision > initial.revision + assert retained.host_path.available is True + assert retained.host_path.epoch == initial.host_path.epoch + assert retained.host_path.fingerprint == initial.host_path.fingerprint + assert retained.endpoint.tcp_state == "reachable" + assert retained.endpoint.host_path_epoch == initial.host_path.epoch + assert retained.device_identity.state == "verified" + assert retained.control_plane.state == "healthy" + assert retained.authority.control_allowed is True + assert retained.lease.state == "reachable" + assert retained.lease.generation == initial.lease.generation + assert service._connection_monitor_contact_gate.locked() is False # noqa: SLF001 + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + def privacy_limited_association( + _interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + assert timeout_seconds == (facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS) + return association_observation("association-identity-unavailable", "b" * 64) + + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + privacy_limited_association, + ) + privacy_path = service._sample_host_path( # noqa: SLF001 + binding.target_ipv4, + association_timeout_seconds=(facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS), + fail_closed_unproven_association=True, + ) + + assert privacy_path.available is True + assert privacy_path.reason_code is None + assert privacy_path.observation_failure_class == "association-observer" + assert privacy_path.fingerprint == initial.host_path.fingerprint + + +@pytest.mark.parametrize( + ("reason_code", "expected_failure_class"), + [ + ("host-wifi-operation-timeout", "association-observer"), + ("wifi-interface-inactive", "route"), + ], +) +def test_monitor_debounces_only_technical_association_timeout( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + reason_code: str, + expected_failure_class: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + def unavailable_association( + _interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + assert timeout_seconds == 3.0 + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "a" * 64, + "reason_code": reason_code, + } + + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + unavailable_association, + ) + + path = service._sample_host_path( # noqa: SLF001 + "192.168.68.51", + association_timeout_seconds=3.0, + fail_closed_unproven_association=True, + ) + + assert path.available is False + assert path.reason_code == reason_code + assert path.observation_failure_class == expected_failure_class + assert path.kernel_route_fingerprint == "test-route:192.168.68.51" + + +def test_monitor_association_sample_over_two_seconds_survives_four_full_cycles( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + host_samples: list[str] = [] + tcp_samples: list[str] = [] + + def slow_first_host_sample( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == 3.0 + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + host_samples.append(target) + if len(host_samples) % 2 == 1: + # Reproduce the field boundary that exceeded the former 2 s + # xcrun/CoreWLAN deadline without making the test depend on macOS. + time.sleep(2.05) + return _direct_host_path(target) + + monkeypatch.setattr(service, "_sample_host_path", slow_first_host_sample) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_samples.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + async def four_cycles() -> tuple[list[Any], list[float], list[dict[str, Any]]]: + snapshots: list[Any] = [] + elapsed_cycles: list[float] = [] + public_states: list[dict[str, Any]] = [] + for _ in range(4): + started = time.monotonic() + snapshots.append( + await service._connection_monitor.poll_once() # noqa: SLF001 + ) + elapsed_cycles.append(time.monotonic() - started) + public_states.append(service.state()) + return snapshots, elapsed_cycles, public_states + + refreshed_cycles, elapsed_cycles, public_states = asyncio.run(four_cycles()) + + assert all( + 2.0 < elapsed < DEFAULT_TRANSPORT_OBSERVATION_TTL_SECONDS for elapsed in elapsed_cycles + ) + assert facade_module.CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS == 5.0 + assert ( + facade_module.CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + < facade_module.CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS + ) + assert facade_module.CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS == 11.5 + assert ( + 2 * (facade_module.CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS + 1.0) + < DEFAULT_TRANSPORT_OBSERVATION_TTL_SECONDS + ) + assert host_samples == [binding.target_ipv4, binding.target_ipv4] * 4 + assert tcp_samples == [binding.target_ipv4] * 4 + initial_epoch = refreshed_cycles[0].host_path.epoch + initial_lease_generation = refreshed_cycles[0].lease.generation + initial_selected = public_states[0]["selected_device_id"] + initial_session = public_states[0]["device_session"]["device_session_id"] + for refreshed, public in zip(refreshed_cycles, public_states, strict=True): + assert refreshed.host_path.available is True + assert refreshed.host_path.epoch == initial_epoch + assert refreshed.endpoint.tcp_state == "reachable" + assert refreshed.lease.state == "reachable" + assert refreshed.lease.generation == initial_lease_generation + assert refreshed.authority.control_allowed is True + assert public["selected_device_id"] == initial_selected + assert public["device_session"]["device_session_id"] == initial_session + assert public["application_control_session"]["state"] == "connection-ready" + + +def test_monitor_refreshes_verified_binding_across_timeout_positive_timeout_cycles( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_authoritative_ready_bridge_control(service) + baseline_supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + baseline_path = baseline_supervisor.host_path + assert baseline_path.kernel_route_fingerprint is not None + selected_device_id = ready["selected_device_id"] + device_session_id = ready["device_session"]["device_session_id"] + active_binding_key = ready["connection_lifecycle"]["active_binding_key"] + tcp_samples: list[str] = [] + probe_mode = {"technical_failure": True} + + def sample_host(target: str, **_: object) -> HostPathProbeResult: + if not probe_mode["technical_failure"]: + return _direct_host_path(target) + return HostPathProbeResult( + available=False, + fingerprint=None, + interface=baseline_path.interface, + source_ipv4=baseline_path.source_ipv4, + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=baseline_path.kernel_route_fingerprint, + ) + + monkeypatch.setattr(service, "_sample_host_path", sample_host) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_samples.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + async def scenario() -> tuple[list[Any], list[dict[str, Any]]]: + snapshots: list[Any] = [] + states: list[dict[str, Any]] = [] + for _ in range(2): + snapshots.append( + await service._connection_monitor.poll_once() # noqa: SLF001 + ) + states.append(service.state()) + probe_mode["technical_failure"] = False + snapshots.append(await service._connection_monitor.poll_once()) # noqa: SLF001 + states.append(service.state()) + probe_mode["technical_failure"] = True + for _ in range(5): + snapshots.append( + await service._connection_monitor.poll_once() # noqa: SLF001 + ) + states.append(service.state()) + return snapshots, states + + monitor_states, public_states = asyncio.run(scenario()) + assert len(monitor_states) == 8 + assert tcp_samples == [binding.target_ipv4] * 8 + previous_revision = baseline_supervisor.revision + for retained, public in zip(monitor_states, public_states, strict=True): + assert retained.revision > previous_revision + previous_revision = retained.revision + assert retained.host_path.available is True + assert retained.host_path.epoch == baseline_path.epoch + assert retained.host_path.fingerprint == baseline_path.fingerprint + assert retained.host_path.reason_code is None + assert retained.host_path_negative_streak == 0 + assert retained.endpoint.tcp_state == "reachable" + assert retained.endpoint.host_path_epoch == baseline_path.epoch + assert retained.device_identity.state == "verified" + assert retained.control_plane.state == "healthy" + assert retained.lease.state == "reachable" + assert retained.lease.generation == baseline_supervisor.lease.generation + assert retained.authority.control_allowed is True + assert public["selected_device_id"] == selected_device_id + assert public["device_session"]["device_session_id"] == device_session_id + assert public["connection_lifecycle"]["active_binding_key"] == active_binding_key + assert public["application_control_session"]["state"] == "connection-ready" + assert public["k1_lifecycle_process_lease"]["holders"] == ["control"] + assert monitor_states[-1].host_path.observed_at != baseline_path.observed_at + assert control.state == "connection-ready" + + +def test_command_preflight_retains_exact_route_when_only_association_observer_times_out( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + association_calls = 0 + + def timed_out_association( + _interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + nonlocal association_calls + assert timeout_seconds == facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS + association_calls += 1 + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": f"{association_calls:064x}", + "reason_code": "host-wifi-operation-timeout", + } + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + timed_out_association, + ) + + for _ in range(5): + service._validate_application_connection_path(binding) # noqa: SLF001 + retained = service._connection_supervisor.snapshot() # noqa: SLF001 + assert retained.host_path.available is True + assert retained.host_path.epoch == baseline.host_path.epoch + assert retained.host_path.fingerprint == baseline.host_path.fingerprint + assert retained.endpoint.tcp_state == "reachable" + assert retained.device_identity.state == "verified" + assert retained.control_plane.state == "healthy" + assert retained.lease.state == "reachable" + assert retained.authority.control_allowed is True + + assert association_calls == 5 + + +def test_command_prepare_allows_contended_probe_plus_own_helper_within_eight_seconds( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + harness = DeterministicContendedAssociationProbe( + lock_wait_seconds=3.0, + helper_required_seconds=3.2, + ) + probe = harness.build(monkeypatch, tmp_path) + service, _ = service_with_fake_runtime(tmp_path) + service._host_wifi_association_probe = probe # type: ignore[assignment] # noqa: SLF001 + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: harness.monotonic # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: harness.wall # noqa: SLF001 + # The helper crosses this deliberately narrow regression TTL. A fresh + # exact same route/association result is itself the replacement sample and + # must refresh the original epoch before any snapshot reduction runs. + supervisor._observation_ttl_seconds = 5.0 # noqa: SLF001 + control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + target = EndpointTarget(binding.target_ipv4, binding.target_port) + harness.on_first_lock_acquired = lambda: supervisor.observe_endpoint( + target=target, + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=True, + ) + control.state = "workspace-ready" + control.state_revision += 1 + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + prepared = service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=control.session_generation, + expected_control_state_revision=control.state_revision, + ) + ) + + assert facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS == 3.0 + assert facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS == 8.0 + assert facade_module.CONNECTION_VERIFY_HARD_TIMEOUT_SECONDS == 125.0 + assert prepared["acquisition"]["state"] == "prepared" + assert prepared["application_control_session"]["state"] == "project-ready" + assert supervisor.snapshot().host_path.epoch == binding.host_path_epoch + assert supervisor.snapshot().authority.control_allowed is True + assert control.validation_calls == 2 + assert harness.lock_timeouts == [8.0, 8.0] + assert harness.helper_timeouts == [pytest.approx(5.0)] + assert harness.monotonic == pytest.approx(106.2) + + +def test_privacy_limited_monitor_keeps_prestart_authority_through_three_minutes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + timeouts: list[float] = [] + tcp_calls: list[str] = [] + + def privacy_limited_association( + _interface_name: str | None, + *, + timeout_seconds: float, + ) -> HostWifiAssociationIdentityResult: + timeouts.append(timeout_seconds) + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "d" * 64, + "reason_code": "association-identity-unavailable", + } + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + privacy_limited_association, + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_calls.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + async def monitor_for_three_minutes() -> None: + for proof_revision in range(3, 33): + observed = await service._connection_monitor.poll_once() # noqa: SLF001 + assert observed.host_path.epoch == binding.host_path_epoch + assert observed.authority.control_allowed is True + assert control.verified_control is not None + control.verified_control = { + **control.verified_control, + "control_proof_revision": proof_revision, + } + service._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + service.runtime.snapshot(), + ) + monotonic_now[0] += 6.1 + suspend_aware_now[0] += 6.1 + + asyncio.run(monitor_for_three_minutes()) + assert monotonic_now[0] > 280.0 + + control.state = "workspace-ready" + control.state_revision += 1 + prepared = service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=control.session_generation, + expected_control_state_revision=control.state_revision, + ) + ) + + final = supervisor.snapshot() + assert prepared["acquisition"]["state"] == "prepared" + assert final.host_path.epoch == binding.host_path_epoch + assert final.host_path.fingerprint == ( + _association_bound_direct_host_path(binding.target_ipv4).fingerprint + ) + assert final.endpoint.tcp_state == "reachable" + assert final.device_identity.state == "verified" + assert final.control_plane.state == "healthy" + assert final.authority.control_allowed is True + assert tcp_calls == [binding.target_ipv4] * 30 + assert timeouts.count(facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS) == 60 + assert timeouts.count(facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS) == 2 + + +def test_command_timeout_after_stale_epoch_fails_before_local_or_physical_mutation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + supervisor = service._connection_supervisor # noqa: SLF001 + + def expire_current_epoch() -> None: + stale = supervisor.snapshot() + assert stale.host_path.reason_code == "host-path-observation-stale" + + harness = DeterministicContendedAssociationProbe( + lock_wait_seconds=3.0, + helper_required_seconds=6.0, + on_first_lock_acquired=expire_current_epoch, + ) + probe = harness.build(monkeypatch, tmp_path) + service._host_wifi_association_probe = probe # type: ignore[assignment] # noqa: SLF001 + supervisor._monotonic_clock = lambda: harness.monotonic # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: harness.wall # noqa: SLF001 + supervisor._observation_ttl_seconds = 2.0 # noqa: SLF001 + control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + control.state = "workspace-ready" + control.state_revision += 1 + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + operations_before = service._operations.snapshot() # noqa: SLF001 + physical_before = service._physical_command_ledger.snapshot() # noqa: SLF001 + + with pytest.raises(ApplicationConnectionBindingLost): + service.prepare_acquisition( + _prepare_request( + project_name=PROJECT_NAME, + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=control.session_generation, + expected_control_state_revision=control.state_revision, + ) + ) + + stale = supervisor.snapshot() + assert harness.lock_timeouts == [8.0] + assert harness.helper_timeouts == [pytest.approx(5.0)] + assert harness.monotonic == pytest.approx(108.0) + assert stale.host_path.epoch == binding.host_path_epoch + 1 + assert stale.host_path.reason_code == "host-path-observation-stale" + assert service._acquisition is None # noqa: SLF001 + assert service._operations.snapshot() == operations_before # noqa: SLF001 + assert service._physical_command_ledger.snapshot() == physical_before # noqa: SLF001 + + +def test_command_privacy_bridge_never_retains_a_proven_association_change( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + lambda *_args, **_kwargs: { + **DeterministicContendedAssociationProbe.proven_observation(), + "continuity_token": "b" * 64, + }, + ) + + with pytest.raises(ApplicationConnectionBindingLost): + service._validate_application_connection_path(binding) # noqa: SLF001 + + changed = service._connection_supervisor.snapshot() # noqa: SLF001 + assert changed.host_path.epoch == baseline.host_path.epoch + 1 + assert changed.host_path.kernel_route_fingerprint == ( + baseline.host_path.kernel_route_fingerprint + ) + assert changed.host_path.fingerprint != baseline.host_path.fingerprint + assert changed.authority.control_allowed is False + + +@pytest.mark.parametrize( + "broken_fact", + [ + "configured-unverified", + "identity-stale", + "identity-intent", + "identity-mode", + "identity-epoch", + "control-lost", + "control-session", + "control-epoch", + "lease-lost", + "lease-intent", + "lease-mode", + "lease-target", + "lease-epoch", + "authority-denied", + ], +) +def test_command_privacy_bridge_requires_exact_verified_control_chain( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + broken_fact: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + supervisor = service._connection_supervisor # noqa: SLF001 + baseline = supervisor.snapshot() + candidate = baseline + if broken_fact == "configured-unverified": + candidate = replace( + baseline, + device_identity=replace(baseline.device_identity, state="unverified"), + control_plane=replace(baseline.control_plane, state="idle", session_id=None), + lease=replace(baseline.lease, state="configured-unverified"), + authority=replace( + baseline.authority, + control_allowed=False, + acquisition_start_allowed=False, + ), + ) + elif broken_fact.startswith("identity-"): + identity_changes: dict[str, object] = { + "identity-stale": {"state": "stale"}, + "identity-intent": {"intent_id": "other-intent"}, + "identity-mode": {"connection_mode": "quick-connect"}, + "identity-epoch": {"host_path_epoch": binding.host_path_epoch + 1}, + }[broken_fact] + candidate = replace( + baseline, + device_identity=replace(baseline.device_identity, **identity_changes), + ) + elif broken_fact.startswith("control-"): + control_changes: dict[str, object] = { + "control-lost": {"state": "lost"}, + "control-session": {"session_id": " "}, + "control-epoch": {"host_path_epoch": binding.host_path_epoch + 1}, + }[broken_fact] + candidate = replace( + baseline, + control_plane=replace(baseline.control_plane, **control_changes), + ) + elif broken_fact.startswith("lease-"): + lease_changes: dict[str, object] = { + "lease-lost": {"state": "lost"}, + "lease-intent": {"intent_id": "other-intent"}, + "lease-mode": {"connection_mode": "quick-connect"}, + "lease-target": { + "target": EndpointTarget("192.168.1.21", binding.target_port) + }, + "lease-epoch": {"host_path_epoch": binding.host_path_epoch + 1}, + }[broken_fact] + candidate = replace( + baseline, + lease=replace(baseline.lease, **lease_changes), + ) + else: + assert broken_fact == "authority-denied" + candidate = replace( + baseline, + authority=replace( + baseline.authority, + control_allowed=False, + acquisition_start_allowed=False, + ), + ) + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + lambda *_args, **_kwargs: { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "d" * 64, + "reason_code": "association-identity-unavailable", + }, + ) + monkeypatch.setattr( + supervisor, + "association_timeout_retention_candidate", + lambda *, expected_target: ( + candidate + if expected_target == EndpointTarget(binding.target_ipv4, binding.target_port) + else None + ), + ) + + sampled = service._sample_host_path(binding.target_ipv4) # noqa: SLF001 + + assert sampled.available is True + assert sampled.reason_code == "association-identity-unavailable" + assert sampled.fingerprint != baseline.host_path.fingerprint + + +@pytest.mark.parametrize("changed_raw_fact", ["kernel-route", "interface", "source-ipv4"]) +def test_command_privacy_bridge_never_retains_a_raw_route_change( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + changed_raw_fact: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_binding_validating_ready_control( + service, + host_path=_association_bound_direct_host_path("192.168.1.20"), + ) + supervisor = service._connection_supervisor # noqa: SLF001 + baseline = supervisor.snapshot() + assert baseline.host_path.kernel_route_fingerprint is not None + changed_path = HostPathProbeResult( + available=True, + fingerprint=( + "changed-kernel-route" + if changed_raw_fact == "kernel-route" + else baseline.host_path.kernel_route_fingerprint + ), + interface=( + "changed0" if changed_raw_fact == "interface" else baseline.host_path.interface + ), + source_ipv4=( + "192.168.99.2" + if changed_raw_fact == "source-ipv4" + else baseline.host_path.source_ipv4 + ), + route_class="direct", + kernel_route_fingerprint=( + "changed-kernel-route" + if changed_raw_fact == "kernel-route" + else baseline.host_path.kernel_route_fingerprint + ), + ) + monkeypatch.setattr(facade_module, "_inspect_host_path", lambda _target: changed_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + lambda *_args, **_kwargs: { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "d" * 64, + "reason_code": "association-identity-unavailable", + }, + ) + + with pytest.raises(ApplicationConnectionBindingLost): + service._validate_application_connection_path(binding) # noqa: SLF001 + + changed = supervisor.snapshot() + assert changed.host_path.epoch == binding.host_path_epoch + 1 + assert changed.authority.control_allowed is False + + +def test_command_timeout_crossing_host_ttl_refreshes_same_route_without_epoch_loss( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + supervisor._observation_ttl_seconds = 5.0 # noqa: SLF001 + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + baseline = supervisor.snapshot() + target = EndpointTarget(binding.target_ipv4, binding.target_port) + + monotonic_now[0] += 4.0 + suspend_aware_now[0] += 4.0 + assert supervisor.observe_endpoint( + target=target, + intent_id=binding.intent_id, + host_path_epoch=binding.host_path_epoch, + reachable=True, + ) + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + def timeout_after_crossing_host_ttl( + _interface_name: str | None, + *, + timeout_seconds: float, + ) -> HostWifiAssociationIdentityResult: + assert timeout_seconds == facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS + monotonic_now[0] += 2.0 + suspend_aware_now[0] += 2.0 + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "f" * 64, + "reason_code": "host-wifi-operation-timeout", + } + + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + timeout_after_crossing_host_ttl, + ) + + service._validate_application_connection_path(binding) # noqa: SLF001 + + retained = supervisor.snapshot() + assert retained.host_path.epoch == baseline.host_path.epoch + assert retained.host_path.fingerprint == baseline.host_path.fingerprint + assert retained.host_path.available is True + assert retained.endpoint.tcp_state == "reachable" + assert retained.device_identity.state == "verified" + assert retained.control_plane.state == "healthy" + assert retained.lease.state == "reachable" + assert retained.authority.control_allowed is True + + +def test_configured_unverified_monitor_keeps_same_route_without_promoting_authority( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection(service, with_control=False) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + association_calls = 0 + tcp_calls: list[str] = [] + + def timed_out_association( + _interface_name: str | None, + *, + timeout_seconds: float = 30.0, + ) -> HostWifiAssociationIdentityResult: + nonlocal association_calls + assert timeout_seconds == facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + association_calls += 1 + return { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": f"{association_calls:064x}", + "reason_code": "host-wifi-operation-timeout", + } + + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + timed_out_association, + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_calls.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + async def poll_five_times() -> list[facade_module.ConnectionSupervisorSnapshot]: + return [ + await service._connection_monitor.poll_once() # noqa: SLF001 + for _ in range(5) + ] + + snapshots = asyncio.run(poll_five_times()) + + assert association_calls == 10 + assert tcp_calls == [binding.target_ipv4] * 5 + for retained in snapshots: + assert retained.host_path.available is True + assert retained.host_path.epoch == baseline.host_path.epoch + assert retained.host_path.fingerprint == baseline.host_path.fingerprint + assert retained.endpoint.tcp_state == "reachable" + assert retained.device_identity.state == "unverified" + assert retained.control_plane.state != "healthy" + assert retained.lease.state == "configured-unverified" + assert retained.authority.control_allowed is False + assert retained.authority.acquisition_start_allowed is False + assert "verify-control-device-info" in retained.allowed_actions + + +def test_association_timeout_cannot_hide_a_real_kernel_route_change( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + changed_path = HostPathProbeResult( + available=True, + fingerprint="changed-kernel-route", + interface=baseline.host_path.interface, + source_ipv4=baseline.host_path.source_ipv4, + route_class="direct", + kernel_route_fingerprint="changed-kernel-route", + ) + + monkeypatch.setattr(facade_module, "_inspect_host_path", lambda _target: changed_path) + monkeypatch.setattr( + service._host_wifi_association_probe, # noqa: SLF001 + "observe", + lambda *_args, **_kwargs: { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "unavailable", + "evidence_quality": "unavailable", + "continuity_proven": False, + "continuity_token": "f" * 64, + "reason_code": "host-wifi-operation-timeout", + }, + ) + + with pytest.raises(ApplicationConnectionBindingLost): + service._validate_application_connection_path(binding) # noqa: SLF001 + + changed = service._connection_supervisor.snapshot() # noqa: SLF001 + assert changed.host_path.epoch == baseline.host_path.epoch + 1 + assert changed.host_path.kernel_route_fingerprint == "changed-kernel-route" + assert changed.authority.control_allowed is False + assert changed.lease.state != "reachable" + + +def test_command_preflight_drops_stale_sample_after_concurrent_route_change( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + changed_path = HostPathProbeResult( + available=True, + fingerprint="changed-combined-route", + interface="changed0", + source_ipv4="192.168.99.2", + route_class="direct", + kernel_route_fingerprint="changed-kernel-route", + ) + stale_path = HostPathProbeResult( + available=True, + fingerprint=baseline.host_path.fingerprint, + interface=baseline.host_path.interface, + source_ipv4=baseline.host_path.source_ipv4, + route_class="direct", + kernel_route_fingerprint=baseline.host_path.kernel_route_fingerprint, + ) + + def sample_after_newer_route_wins( + _target: str, + *, + association_timeout_seconds: float, + ) -> HostPathProbeResult: + assert ( + association_timeout_seconds + == facade_module.COMMAND_BOUND_ASSOCIATION_TIMEOUT_SECONDS + ) + observed_epoch = service._connection_supervisor.observe_host_path( # noqa: SLF001 + changed_path + ) + assert observed_epoch == baseline.host_path.epoch + 1 + return stale_path + + monkeypatch.setattr(service, "_sample_host_path", sample_after_newer_route_wins) + + with pytest.raises(ApplicationConnectionBindingLost): + service._validate_application_connection_path(binding) # noqa: SLF001 + + current = service._connection_supervisor.snapshot() # noqa: SLF001 + assert current.host_path.epoch == baseline.host_path.epoch + 1 + assert current.host_path.fingerprint == "changed-combined-route" + assert current.host_path.kernel_route_fingerprint == "changed-kernel-route" + assert current.host_path.interface == "changed0" + assert current.host_path.source_ipv4 == "192.168.99.2" + assert current.authority.control_allowed is False + + +@pytest.mark.parametrize( + "changed_route_fact", + ["fingerprint", "interface", "source-ipv4"], +) +def test_monitor_does_not_debounce_real_route_change_hidden_by_observer_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + changed_route_fact: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + baseline = service._connection_supervisor.snapshot() # noqa: SLF001 + baseline_path = baseline.host_path + assert baseline_path.kernel_route_fingerprint is not None + changed = HostPathProbeResult( + available=False, + fingerprint=None, + interface=("changed0" if changed_route_fact == "interface" else baseline_path.interface), + source_ipv4=( + "192.168.99.2" if changed_route_fact == "source-ipv4" else baseline_path.source_ipv4 + ), + route_class="unavailable", + reason_code="host-wifi-operation-timeout", + observation_failure_class="association-observer", + kernel_route_fingerprint=( + "changed-kernel-route" + if changed_route_fact == "fingerprint" + else baseline_path.kernel_route_fingerprint + ), + ) + monkeypatch.setattr(service, "_sample_host_path", lambda *_args, **_kwargs: changed) + tcp_samples: list[str] = [] + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_samples.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + observed = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert observed.revision > baseline.revision + assert observed.host_path.available is False + assert observed.host_path.reason_code == "host-wifi-operation-timeout" + assert observed.authority.control_allowed is False + assert observed.lease.state != "reachable" + assert tcp_samples == [] + + +def test_read_only_monitor_runs_alongside_steady_control_holder( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + initial = service._connection_supervisor.snapshot() # noqa: SLF001 + host_samples: list[str] = [] + tcp_samples: list[str] = [] + + def sample_host( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + host_samples.append(target) + return _direct_host_path(target) + + def sample_tcp(target: str) -> facade_module.TcpReachabilityProbeResult: + tcp_samples.append(target) + return facade_module.TcpReachabilityProbeResult(reachable=True) + + monkeypatch.setattr(service, "_sample_host_path", sample_host) + monkeypatch.setattr(facade_module, "_probe_control_endpoint_socket", sample_tcp) + + refreshed = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert host_samples == [binding.target_ipv4, binding.target_ipv4] + assert tcp_samples == [binding.target_ipv4] + assert refreshed.revision > initial.revision + assert refreshed.host_path.epoch == initial.host_path.epoch + assert refreshed.endpoint.tcp_state == "reachable" + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + +def test_read_only_monitor_keeps_ready_from_exact_durable_target_after_ephemeral_cleanup( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control( + service, + transport_ref="F89438FA-55ED-85AD-EED7-734AC84746D8", + ) + _seed_durable_bridge_topology( + service, + # CoreBluetooth UUID spelling is not durable identity. The monitor + # compares its physical key case-insensitively while retaining the + # exact supervisor spelling for current-process evidence. + transport_ref=binding.transport_ref.lower(), + ipv4=binding.target_ipv4, + ) + service._retire_ephemeral_device_binding_for_new_intent() # noqa: SLF001 + host_samples: list[str] = [] + tcp_samples: list[str] = [] + + def sample_host( + target: str, + **_: object, + ) -> HostPathProbeResult: + host_samples.append(target) + return _direct_host_path(target) + + def sample_tcp(target: str) -> facade_module.TcpReachabilityProbeResult: + tcp_samples.append(target) + return facade_module.TcpReachabilityProbeResult(reachable=True) + + monkeypatch.setattr(service, "_sample_host_path", sample_host) + monkeypatch.setattr(facade_module, "_probe_control_endpoint_socket", sample_tcp) + + refreshed = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert service._selected_device_id is None # noqa: SLF001 + assert service._device_session_id is None # noqa: SLF001 + assert host_samples == [binding.target_ipv4, binding.target_ipv4] + assert tcp_samples == [binding.target_ipv4] + assert refreshed.host_path.available is True + assert refreshed.endpoint.tcp_state == "reachable" + assert refreshed.device_identity.state == "verified" + assert refreshed.control_plane.state == "healthy" + assert refreshed.lease.state == "reachable" + assert refreshed.authority.control_allowed is True + assert refreshed.authority.acquisition_start_allowed is True + + +def test_read_only_monitor_durable_target_still_downgrades_on_real_route_loss( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + _seed_durable_bridge_topology( + service, + transport_ref=binding.transport_ref, + ipv4=binding.target_ipv4, + ) + service._retire_ephemeral_device_binding_for_new_intent() # noqa: SLF001 + tcp_samples: list[str] = [] + + monkeypatch.setattr( + service, + "_sample_host_path", + lambda _target, **_kwargs: HostPathProbeResult( + available=False, + fingerprint=None, + interface=None, + source_ipv4=None, + route_class="unavailable", + reason_code="router-link-lost", + ), + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_samples.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + lost = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert tcp_samples == [] + assert lost.host_path.available is False + assert lost.host_path.reason_code == "router-link-lost" + assert lost.endpoint.tcp_state == "unknown" + assert lost.lease.state == "configured-unverified" + assert lost.authority.control_allowed is False + assert lost.authority.acquisition_start_allowed is False + + +@pytest.mark.parametrize( + "invalid_context", + [ + "selected-without-session", + "session-without-selected", + "semantic-empty", + "semantic-store-missing", + "semantic-store-corrupt", + "semantic-ref-mismatch", + "semantic-mode-mismatch", + "semantic-ip-mismatch", + "semantic-profile-mismatch", + "physically-retired", + ], +) +def test_read_only_monitor_durable_fallback_rejects_inexact_or_unsafe_context( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + invalid_context: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + store = service._semantic_topology_store # noqa: SLF001 + assert store is not None + + contexts_with_exact_semantic = { + "selected-without-session", + "session-without-selected", + "semantic-store-missing", + "semantic-store-corrupt", + "physically-retired", + } + if invalid_context in contexts_with_exact_semantic: + _seed_durable_bridge_topology( + service, + transport_ref=binding.transport_ref, + ipv4=binding.target_ipv4, + ) + elif invalid_context.startswith("semantic-") and invalid_context != "semantic-empty": + snapshot = store.snapshot() + store.commit( + transport_ref=( + "different-k1" + if invalid_context == "semantic-ref-mismatch" + else binding.transport_ref + ), + connection_mode=( + "direct-connect" if invalid_context == "semantic-mode-mismatch" else "bridge" + ), + ipv4=( + "192.168.68.77" + if invalid_context == "semantic-ip-mismatch" + else binding.target_ipv4 + ), + compatibility_profile_id=( + "different.compatibility.profile" + if invalid_context == "semantic-profile-mismatch" + else XGRIDS_K1_COMPATIBILITY_PROFILE_ID + ), + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-11T08:00:00Z", + predecessor_revision=(snapshot.record.revision if snapshot.record is not None else 0), + ) + + service._retire_ephemeral_device_binding_for_new_intent() # noqa: SLF001 + if invalid_context == "selected-without-session": + with service._lock: # noqa: SLF001 + service._selected_device_id = binding.transport_ref # noqa: SLF001 + elif invalid_context == "session-without-selected": + with service._lock: # noqa: SLF001 + service._device_session_id = "orphaned-session" # noqa: SLF001 + elif invalid_context == "semantic-store-missing": + service._semantic_topology_store = None # noqa: SLF001 + elif invalid_context == "semantic-store-corrupt": + + def corrupt_snapshot() -> object: + raise facade_module.SemanticTopologyStoreCorrupt("synthetic corrupt store") + + monkeypatch.setattr(store, "snapshot", corrupt_snapshot) + elif invalid_context == "physically-retired": + monkeypatch.setattr( + service, + "_retired_physical_transport_refs", + lambda: {binding.transport_ref.casefold()}, + ) + + host_samples: list[str] = [] + tcp_samples: list[str] = [] + monkeypatch.setattr( + service, + "_sample_host_path", + lambda target, **_kwargs: host_samples.append(target) or _direct_host_path(target), + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda target: ( + tcp_samples.append(target) or facade_module.TcpReachabilityProbeResult(reachable=True) + ), + ) + + assert service._connection_monitor_target() is None # noqa: SLF001 + rejected = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert host_samples == [] + assert tcp_samples == [] + assert rejected.host_path.available is False + assert rejected.host_path.reason_code == "endpoint-target-unconfigured" + assert rejected.endpoint.tcp_state == "unknown" + assert rejected.lease.state == "configured-unverified" + assert rejected.authority.control_allowed is False + assert rejected.authority.acquisition_start_allowed is False + + +@pytest.mark.parametrize( + "durable_store_change", + ["missing", "corrupt", "replaced"], +) +def test_monitor_revalidates_durable_sourced_supervisor_even_with_ephemeral_binding( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + durable_store_change: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, binding, _ready = _install_authoritative_ready_bridge_control(service) + _seed_durable_bridge_topology( + service, + transport_ref=binding.transport_ref, + ipv4=binding.target_ipv4, + ) + supervisor = service._connection_supervisor # noqa: SLF001 + current = supervisor.snapshot() + assert current.intent is not None + assert current.device_network.target is not None + assert supervisor.observe_device_network_applied( + intent_id=current.intent.intent_id, + transport_ref=binding.transport_ref, + connection_mode="bridge", + target=current.device_network.target, + source="durable-semantic-topology", + ) + assert service._connection_monitor_target() == current.device_network.target # noqa: SLF001 + + store = service._semantic_topology_store # noqa: SLF001 + assert store is not None + if durable_store_change == "missing": + service._semantic_topology_store = None # noqa: SLF001 + elif durable_store_change == "corrupt": + + def corrupt_snapshot() -> object: + raise facade_module.SemanticTopologyStoreCorrupt("synthetic corrupt store") + + monkeypatch.setattr(store, "snapshot", corrupt_snapshot) + else: + semantic = store.snapshot() + assert semantic.record is not None + store.commit( + transport_ref=binding.transport_ref, + connection_mode="bridge", + ipv4="192.168.68.77", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-11T08:30:00Z", + predecessor_revision=semantic.record.revision, + ) + + host_samples: list[str] = [] + monkeypatch.setattr( + service, + "_sample_host_path", + lambda target, **_kwargs: host_samples.append(target) or _direct_host_path(target), + ) + + assert service._connection_monitor_target() is None # noqa: SLF001 + rejected = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + + assert host_samples == [] + assert rejected.host_path.available is False + assert rejected.host_path.reason_code == "endpoint-target-unconfigured" + assert rejected.authority.control_allowed is False + + +def test_monitor_network_admission_race_is_dropped_without_negative_evidence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + initial = service._connection_supervisor.snapshot() # noqa: SLF001 + original_acquire = service._acquire_k1_lifecycle_process_lease # noqa: SLF001 + network_injected = False + host_io: list[str] = [] + + def raced_acquire(holder: Any) -> None: + nonlocal network_injected + if holder == "monitor" and not network_injected: + original_acquire("network") + network_injected = True + original_acquire(holder) + + monkeypatch.setattr(service, "_acquire_k1_lifecycle_process_lease", raced_acquire) + monkeypatch.setattr( + service, + "_sample_host_path", + lambda *_args, **_kwargs: host_io.append("host") or _direct_host_path("10.0.0.1"), + ) + + try: + after_race = asyncio.run(service._connection_monitor.poll_once()) # noqa: SLF001 + finally: + if network_injected: + service._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + + assert network_injected is True + assert host_io == [] + assert after_race.revision == initial.revision + assert after_race.host_path == initial.host_path + assert after_race.endpoint == initial.endpoint + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + +def test_explicit_reconfiguration_waits_for_monitor_and_drops_stale_probe( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_authoritative_ready_bridge_control(service) + request = _reconfiguration_request(ready, "select-device") + initial_supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + probe_entered = threading.Event() + release_probe = threading.Event() + device_edges: list[str] = [] + + def blocked_host_path( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + probe_entered.set() + assert release_probe.wait(timeout=5.0) + assert target == binding.target_ipv4 + # If this superseded result were admitted after the operator won the + # transition, it would rotate the host epoch and revoke the old proof. + return _direct_host_path("192.168.56.20") + + async def forbidden_device_io(*_args: object, **_kwargs: object) -> object: + device_edges.append("ble-or-network") + raise AssertionError("local reconfiguration must not perform device I/O") + + monkeypatch.setattr(service, "_sample_host_path", blocked_host_path) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_io) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + async def scenario() -> tuple[dict[str, Any], dict[str, Any]]: + monitor_task = asyncio.create_task( + service._connection_monitor.poll_once() # noqa: SLF001 + ) + assert await asyncio.to_thread(probe_entered.wait, 5.0) + + during_probe = await asyncio.to_thread(service.state) + assert "monitor" in during_probe["k1_lifecycle_process_lease"]["holders"] + for action in ("prepare-select-device", "prepare-change-network"): + assert during_probe["connection_policy"]["actions"][action]["allowed"] is True + + prepare_task = asyncio.create_task(service.prepare_connection_reconfiguration(request)) + for _ in range(100): + if service._k1_lifecycle_transition_gate.locked(): # noqa: SLF001 + break + await asyncio.sleep(0.001) + assert service._k1_lifecycle_transition_gate.locked() # noqa: SLF001 + await asyncio.sleep(0.01) + assert prepare_task.done() is False + + release_probe.set() + monitor_state = await monitor_task + prepared = await prepare_task + return monitor_state, prepared + + monitor_state, prepared = asyncio.run(scenario()) + + assert monitor_state.host_path.epoch == initial_supervisor.host_path.epoch + assert prepared["connection_reconfiguration"]["intent"] == "select-device" + assert prepared["connection_reconfiguration"]["status"] == "awaiting-fresh-scan" + assert prepared["connection_supervisor"]["observed"]["host_path"]["epoch"] == ( + initial_supervisor.host_path.epoch + ) + assert "monitor" not in service._application_control_process_lease_holders # noqa: SLF001 + assert control.stop_calls == 0 + assert control.start_projects == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert device_edges == [] + assert binding.target_ipv4 == "192.168.1.20" + + +def test_cancelled_monitor_keeps_contact_owned_until_operator_can_commit( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_authoritative_ready_bridge_control(service) + request = _reconfiguration_request(ready, "select-device") + initial = service._connection_supervisor.snapshot() # noqa: SLF001 + probe_entered = threading.Event() + release_probe = threading.Event() + device_edges: list[str] = [] + + def blocked_host_path( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + probe_entered.set() + assert release_probe.wait(timeout=5.0) + assert target == binding.target_ipv4 + return _direct_host_path("192.168.56.20") + + async def forbidden_device_io(*_args: object, **_kwargs: object) -> object: + device_edges.append("ble-or-network") + raise AssertionError("cancelled monitor handoff must not perform device I/O") + + monkeypatch.setattr(service, "_sample_host_path", blocked_host_path) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_io) + + async def scenario() -> tuple[dict[str, Any], bool]: + monitor_task = asyncio.create_task( + service._connection_monitor.poll_once() # noqa: SLF001 + ) + assert await asyncio.to_thread(probe_entered.wait, 5.0) + monitor_task.cancel() + + prepare_task = asyncio.create_task(service.prepare_connection_reconfiguration(request)) + for _ in range(100): + if service._k1_lifecycle_transition_gate.locked(): # noqa: SLF001 + break + await asyncio.sleep(0.001) + assert service._k1_lifecycle_transition_gate.locked() # noqa: SLF001 + await asyncio.sleep(0.01) + assert prepare_task.done() is False + assert "network" not in service._application_control_process_lease_holders # noqa: SLF001 + + release_probe.set() + with pytest.raises(asyncio.CancelledError): + await monitor_task + prepared = await prepare_task + return prepared, service._connection_monitor_contact_gate.locked() # noqa: SLF001 + + prepared, contact_locked = asyncio.run(scenario()) + + assert contact_locked is False + assert service._k1_lifecycle_transition_gate.locked() is False # noqa: SLF001 + assert service._connection_reconfiguration_gate.locked() is False # noqa: SLF001 + assert prepared["connection_reconfiguration"]["status"] == "awaiting-fresh-scan" + assert prepared["connection_supervisor"]["observed"]["host_path"]["epoch"] == ( + initial.host_path.epoch + ) + assert "monitor" not in service._application_control_process_lease_holders # noqa: SLF001 + assert control.stop_calls == 0 + assert control.start_projects == [] + assert runtime.start_calls == [] + assert device_edges == [] + + +def test_monitor_quiescence_timeout_releases_operator_gates_and_drops_late_result( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, binding, ready = _install_authoritative_ready_bridge_control(service) + request = _reconfiguration_request(ready, "select-device") + initial = service._connection_supervisor.snapshot() # noqa: SLF001 + probe_entered = threading.Event() + release_probe = threading.Event() + device_edges: list[str] = [] + + def blocked_host_path( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + probe_entered.set() + assert release_probe.wait(timeout=5.0) + assert target == binding.target_ipv4 + return _direct_host_path("192.168.56.20") + + async def forbidden_device_io(*_args: object, **_kwargs: object) -> object: + device_edges.append("ble-or-network") + raise AssertionError("quiescence timeout must not perform device I/O") + + monkeypatch.setattr( + facade_module, + "CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS", + 0.01, + ) + monkeypatch.setattr(service, "_sample_host_path", blocked_host_path) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_io) + + async def scenario() -> None: + monitor_task = asyncio.create_task( + service._connection_monitor.poll_once() # noqa: SLF001 + ) + assert await asyncio.to_thread(probe_entered.wait, 5.0) + with pytest.raises(facade_module.NetworkProvisioningConflict) as conflict: + await service.prepare_connection_reconfiguration(request) + assert conflict.value.reason_code == "connection-reconfiguration-lifecycle-busy" + assert service._k1_lifecycle_transition_gate.locked() is False # noqa: SLF001 + assert service._connection_reconfiguration_gate.locked() is False # noqa: SLF001 + assert "network" not in service._application_control_process_lease_holders # noqa: SLF001 + + monitor_task.cancel() + release_probe.set() + with pytest.raises(asyncio.CancelledError): + await monitor_task + + asyncio.run(scenario()) + + after = service._connection_supervisor.snapshot() # noqa: SLF001 + assert after.revision == initial.revision + assert after.host_path == initial.host_path + assert service._connection_monitor_contact_gate.locked() is False # noqa: SLF001 + assert service._connection_reconfiguration_intent is None # noqa: SLF001 + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + assert control.stop_calls == 0 + assert control.start_projects == [] + assert runtime.start_calls == [] + assert device_edges == [] + + +def test_same_mode_scan_waits_for_monitor_instead_of_returning_busy( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _runtime = service_with_fake_runtime(tmp_path) + _control, binding, ready = _install_authoritative_ready_bridge_control(service) + initial_epoch = ready["connection_supervisor"]["observed"]["host_path"]["epoch"] + probe_entered = threading.Event() + release_probe = threading.Event() + scan_calls: list[float] = [] + tcp_calls: list[str] = [] + + def blocked_host_path( + target: str, + *, + association_timeout_seconds: float = 30.0, + fail_closed_unproven_association: bool = False, + ) -> HostPathProbeResult: + assert association_timeout_seconds == ( + facade_module.CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS + ) + assert fail_closed_unproven_association is True + probe_entered.set() + assert release_probe.wait(timeout=5.0) + assert target == binding.target_ipv4 + return _direct_host_path("192.168.56.20") + + async def fake_scan( + duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + scan_calls.append(duration_seconds) + assert on_admitted is not None + on_admitted() + return _ble_scan_result("fresh-k1") + + def forbidden_tcp(target: str) -> facade_module.TcpReachabilityProbeResult: + tcp_calls.append(target) + raise AssertionError("operator lifecycle gate must supersede the next monitor contact") + + monkeypatch.setattr(service, "_sample_host_path", blocked_host_path) + monkeypatch.setattr(facade_module, "_probe_control_endpoint_socket", forbidden_tcp) + monkeypatch.setattr(facade_module, "scan", fake_scan) + + async def scenario() -> tuple[dict[str, Any], int]: + monitor_task = asyncio.create_task( + service._connection_monitor.poll_once() # noqa: SLF001 + ) + assert await asyncio.to_thread(probe_entered.wait, 5.0) + scan_task = asyncio.create_task(service.scan_ble(0.1)) + for _ in range(100): + if service._k1_lifecycle_transition_gate.locked(): # noqa: SLF001 + break + await asyncio.sleep(0.001) + assert service._k1_lifecycle_transition_gate.locked() # noqa: SLF001 + await asyncio.sleep(0.01) + assert scan_task.done() is False + assert scan_calls == [] + assert "network" not in service._application_control_process_lease_holders # noqa: SLF001 + + release_probe.set() + monitor_state = await monitor_task + scanned = await scan_task + return scanned, monitor_state.host_path.epoch + + scanned, monitor_epoch = asyncio.run(scenario()) + + assert scan_calls == [0.1] + assert [item["device_id"] for item in scanned["devices"]] == ["fresh-k1"] + assert monitor_epoch == initial_epoch + assert scanned["connection_supervisor"]["observed"]["host_path"]["epoch"] == (initial_epoch) + assert service._connection_monitor_contact_gate.locked() is False # noqa: SLF001 + assert "monitor" not in service._application_control_process_lease_holders # noqa: SLF001 + assert tcp_calls == [] + + +def test_select_device_handoff_is_local_cancel_invalidates_candidates_and_stale_tabs( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _binding, ready = _install_authoritative_ready_bridge_control(service) + initial_request = _reconfiguration_request(ready, "select-device") + semantic_before = ready["semantic_topology_store"] + identity_before = ready["device_identity_pin_store"] + device_edges: list[str] = [] + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("local reconfiguration handoff must not write K1") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + prepared = asyncio.run(service.prepare_connection_reconfiguration(initial_request)) + + assert prepared["connection_reconfiguration"]["intent"] == "select-device" + assert prepared["connection_reconfiguration"]["status"] == "awaiting-fresh-scan" + assert prepared["connection_reconfiguration"]["required_connection_mode"] == "bridge" + assert prepared["connection_lifecycle"]["active_binding"] is None + assert prepared["selected_device_id"] is None + assert ( + prepared["connection_policy"]["actions"]["prepare-select-device"][ + "required_connection_mode" + ] + == "bridge" + ) + assert prepared["semantic_topology_store"] == semantic_before + assert prepared["device_identity_pin_store"] == identity_before + assert control.state == "idle" + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert device_edges == [] + + # The losing tab cannot repeat the original handoff after the stable + # operator-only revision/intent id advanced. + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_prepare: + asyncio.run(service.prepare_connection_reconfiguration(initial_request)) + assert stale_prepare.value.reason_code == "connection-reconfiguration-revision-conflict" + + minimum_generation = prepared["connection_reconfiguration"]["minimum_discovery_generation"] + service._ble_discovery_generation = minimum_generation # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-b") + fresh = service.state() + assert fresh["connection_reconfiguration"]["status"] == "fresh-scan-completed" + assert [item["device_id"] for item in fresh["devices"]] == ["k1-b"] + stale_connect_request = _connect_request( + device_id="k1-b", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + expected_mode_revision=fresh["desired_connection_mode_revision"], + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"]["revision"], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"]["intent_id"], + ) + stale_verify_request = ConnectionVerifyRequest( + device_id="k1-b", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"]["revision"], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"]["intent_id"], + ) + quick_verify_request = ConnectionVerifyRequest( + device_id="k1-b", + source="fresh-scan", + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"]["revision"], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"]["intent_id"], + ) + with pytest.raises(facade_module.NetworkProvisioningConflict) as quick_isolated: + asyncio.run(service.verify_connection(quick_verify_request)) + assert quick_isolated.value.reason_code == "connection-reconfiguration-target-mismatch" + cancel_request = _reconfiguration_request(fresh, "cancel") + + cancelled = asyncio.run(service.prepare_connection_reconfiguration(cancel_request)) + + assert cancelled["connection_reconfiguration"]["status"] == "idle" + assert cancelled["connection_reconfiguration"]["intent_id"] is None + assert cancelled["devices"] == [] + assert cancelled["ble_discovery_generation"] > minimum_generation + assert cancelled["semantic_topology_store"] == semantic_before + assert cancelled["device_identity_pin_store"] == identity_before + assert device_edges == [] + + password_reads: list[str] = [] + process_lease_calls: list[str] = [] + + def forbidden_password_read(_secret: SecretStr) -> str: + password_reads.append("unwrapped") + raise AssertionError("stale Connect must fail before password unwrap") + + def forbidden_process_lease(holder: str) -> None: + process_lease_calls.append(holder) + raise AssertionError("stale mutation must fail before process lease") + + monkeypatch.setattr(SecretStr, "get_secret_value", forbidden_password_read) + monkeypatch.setattr( + service, + "_acquire_k1_lifecycle_process_lease", + forbidden_process_lease, + ) + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_connect: + asyncio.run(service.connect(stale_connect_request)) + assert stale_connect.value.reason_code == "connection-reconfiguration-revision-conflict" + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_verify: + asyncio.run(service.verify_connection(stale_verify_request)) + assert stale_verify.value.reason_code == "connection-reconfiguration-revision-conflict" + assert password_reads == [] + assert process_lease_calls == [] + assert service._operations.snapshot() == [] # noqa: SLF001 + + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale_cancel: + asyncio.run(service.prepare_connection_reconfiguration(cancel_request)) + assert stale_cancel.value.reason_code == "connection-reconfiguration-revision-conflict" + + +def test_change_network_requires_authoritative_binding_before_any_local_teardown( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control, _binding, ready = _install_authoritative_ready_bridge_control(service) + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor_state = supervisor.snapshot() + assert supervisor_state.intent is not None + assert supervisor_state.device_network.target is not None + assert supervisor.observe_endpoint( + target=supervisor_state.device_network.target, + intent_id=supervisor_state.intent.intent_id, + host_path_epoch=supervisor_state.host_path.epoch, + reachable=False, + reason_code="tcp-endpoint-unreachable", + ) + degraded = service.state() + assert degraded["selected_device_id"] == "k1-a" + assert degraded["connection_lifecycle"]["active_binding"] is None + assert control.state == "connection-ready" + decision = degraded["connection_policy"]["actions"]["prepare-change-network"] + assert decision["allowed"] is False + assert "connection-reconfiguration-current-device-unavailable" in decision["reason_codes"] + + process_lease_calls: list[str] = [] + original_acquire = service._acquire_k1_lifecycle_process_lease # noqa: SLF001 + + def counted_acquire(holder: str) -> None: + process_lease_calls.append(holder) + original_acquire(holder) + + monkeypatch.setattr(service, "_acquire_k1_lifecycle_process_lease", counted_acquire) + before_control_revision = control.state_revision + with pytest.raises(facade_module.NetworkProvisioningConflict) as unavailable: + asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(degraded, "change-network") + ) + ) + + assert unavailable.value.reason_code == ( + "connection-reconfiguration-current-device-unavailable" + ) + assert process_lease_calls == [] + assert control.state == "connection-ready" + assert control.state_revision == before_control_revision + assert service._selected_device_id == "k1-a" # noqa: SLF001 + assert service._connection_reconfiguration_intent is None # noqa: SLF001 + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + +def test_bridge_reconfiguration_policy_rejects_quick_and_unresolved_physical_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + quick_service, _ = service_with_fake_runtime(tmp_path / "quick") + _select_connection_mode(quick_service, "quick-connect") + quick_binding = _seed_supervised_connection( + quick_service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-quick", + logical_device_id="device-quick", + ) + quick_control = FakeInteractiveControlSession(initial_state="idle") + quick_control.open(connection_binding=quick_binding) + quick_control.verified_control = _verified_control_for_binding( + quick_binding, + logical_device_id="device-quick", + control_session_id="test-control-k1-quick", + control_proof_revision=2, + ) + quick_service._application_control_session = quick_control # type: ignore[assignment] # noqa: SLF001 + quick_service._reconcile_connection_supervisor( # noqa: SLF001 + quick_control.snapshot(), + quick_service.runtime.snapshot(), + ) + quick_service._acquire_application_control_process_lease() # noqa: SLF001 + quick_state = quick_service.state() + quick_decision = quick_state["connection_policy"]["actions"]["prepare-select-device"] + assert quick_decision["allowed"] is False + assert "connection-reconfiguration-bridge-only" in quick_decision["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as quick_rejected: + asyncio.run( + quick_service.prepare_connection_reconfiguration( + _reconfiguration_request(quick_state, "select-device") + ) + ) + assert quick_rejected.value.reason_code == "connection-reconfiguration-bridge-only" + assert quick_control.state == "connection-ready" + + bridge_service, _ = service_with_fake_runtime(tmp_path / "physical") + bridge_control, _binding, _ready = _install_authoritative_ready_bridge_control(bridge_service) + monkeypatch.setattr( + bridge_service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: { + "status": "unresolved", + "requires_reconciliation": True, + "record": None, + }, + ) + unsafe = bridge_service.state() + unsafe_decision = unsafe["connection_policy"]["actions"]["prepare-select-device"] + assert unsafe_decision["allowed"] is False + assert "physical-command-reconciliation-required" in unsafe_decision["reason_codes"] + with pytest.raises(facade_module.NetworkProvisioningConflict) as physical_rejected: + asyncio.run( + bridge_service.prepare_connection_reconfiguration( + _reconfiguration_request(unsafe, "select-device") + ) + ) + assert physical_rejected.value.reason_code == ("physical-command-reconciliation-required") + assert bridge_control.state == "connection-ready" + assert bridge_service._connection_reconfiguration_intent is None # noqa: SLF001 + + +def test_change_network_handoff_pins_exact_active_bridge_transport_without_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _binding, ready = _install_authoritative_ready_bridge_control(service) + decision = ready["connection_policy"]["actions"]["prepare-change-network"] + assert decision["allowed"] is True + assert decision["required_transport_ref"] == "k1-a" + assert decision["required_connection_mode"] == "bridge" + device_edges: list[str] = [] + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("Prepare must remain local-only") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + prepared = asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(ready, "change-network") + ) + ) + + reconfiguration = prepared["connection_reconfiguration"] + assert reconfiguration["intent"] == "change-network" + assert reconfiguration["required_transport_ref"] == "k1-a" + assert reconfiguration["required_connection_mode"] == "bridge" + minimum_generation = reconfiguration["minimum_discovery_generation"] + service._ble_discovery_generation = minimum_generation # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-a") + observed = service.state() + assert observed["connection_reconfiguration"]["required_transport_observed"] is True + provision = observed["connection_policy"]["actions"]["provision-fresh-device"] + assert provision["required_transport_ref"] == "k1-a" + assert provision["required_connection_mode"] == "bridge" + wrong_target_request = _connect_request( + device_id="k1-b", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + expected_mode_revision=observed["desired_connection_mode_revision"], + expected_discovery_generation=observed["ble_discovery_generation"], + expected_reconfiguration_revision=observed["connection_reconfiguration"]["revision"], + expected_reconfiguration_intent_id=observed["connection_reconfiguration"]["intent_id"], + ) + password_reads: list[str] = [] + + def forbidden_password_read(_secret: SecretStr) -> str: + password_reads.append("unwrapped") + raise AssertionError("wrong pinned target must fail before password unwrap") + + monkeypatch.setattr(SecretStr, "get_secret_value", forbidden_password_read) + with pytest.raises(facade_module.NetworkProvisioningConflict) as wrong_target: + asyncio.run(service.connect(wrong_target_request)) + assert wrong_target.value.reason_code == "connection-reconfiguration-target-mismatch" + assert password_reads == [] + assert service._operations.snapshot() == [] # noqa: SLF001 + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert device_edges == [] + + +def test_select_device_fresh_verify_consumes_exact_reconfiguration_without_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_durable_bridge_topology( + service, + transport_ref="k1-a", + ipv4="10.255.254.20", + ) + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref="k1-a", + logical_device_id="device-a", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref="k1-new", + logical_device_id="device-b", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + semantic_before = service._semantic_topology_store.snapshot() # type: ignore[union-attr] # noqa: SLF001 + assert semantic_before.record is not None + assert semantic_before.record.transport_ref == "k1-a" + initial = service.state() + assert initial["connection_policy"]["actions"]["prepare-select-device"]["allowed"] is True + prepared = asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(initial, "select-device") + ) + ) + minimum_generation = prepared["connection_reconfiguration"]["minimum_discovery_generation"] + service._ble_discovery_generation = minimum_generation # noqa: SLF001 + _set_scanned_k1(service, device_id="k1-new") + fresh = service.state() + writes: list[str] = [] + _install_successful_bridge_verify_transport( + monkeypatch, + device_id="k1-new", + ipv4="10.255.254.77", + writes=writes, + ) + original_bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + semantic_during_bootstrap: list[str] = [] + + async def assert_semantic_commit_is_deferred(**kwargs: object) -> None: + before_device_info = service._semantic_topology_store.snapshot() # type: ignore[union-attr] # noqa: SLF001 + assert before_device_info.record is not None + semantic_during_bootstrap.append(before_device_info.record.transport_ref) + await original_bootstrap(**kwargs) # type: ignore[arg-type] + after_device_info = service._semantic_topology_store.snapshot() # type: ignore[union-attr] # noqa: SLF001 + assert after_device_info.record is not None + semantic_during_bootstrap.append(after_device_info.record.transport_ref) + + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + assert_semantic_commit_is_deferred, + ) + + verified = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-new", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"]["revision"], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"]["intent_id"], + ) + ) + ) + + assert writes == [] + assert verified["selected_device_id"] == "k1-new" + assert verified["active_connection_mode"] == "bridge" + assert verified["connection_reconfiguration"]["status"] == "idle" + assert verified["connection_reconfiguration"]["intent_id"] is None + assert semantic_during_bootstrap == ["k1-a", "k1-a"] + semantic_after = service._semantic_topology_store.snapshot() # type: ignore[union-attr] # noqa: SLF001 + assert semantic_after.record is not None + assert semantic_after.record.transport_ref == "k1-new" + assert semantic_after.record.revision == semantic_before.record.revision + 1 + + +def test_select_device_identity_mismatch_preserves_a_and_cancel_restart_recovers_a( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_durable_bridge_topology( + service, + transport_ref="k1-a", + ipv4="10.255.254.20", + ) + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref="k1-a", + logical_device_id="device-a", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref="k1-b", + logical_device_id="device-b", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + topology_store = service._semantic_topology_store # noqa: SLF001 + pin_store = service._device_identity_pin_store # noqa: SLF001 + assert topology_store is not None + assert pin_store is not None + semantic_before = topology_store.snapshot() + pins_before = pin_store.snapshot() + + prepared = asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(service.state(), "select-device") + ) + ) + service._ble_discovery_generation = prepared["connection_reconfiguration"][ # noqa: SLF001 + "minimum_discovery_generation" + ] + _set_scanned_k1(service, device_id="k1-b") + fresh = service.state() + writes: list[str] = [] + _install_successful_bridge_verify_transport( + monkeypatch, + device_id="k1-b", + ipv4="10.255.254.77", + writes=writes, + ) + # Force the synthetic DeviceInfo proof to present a different logical ID + # than B's immutable transport pin. The production reducer then rejects it. + monkeypatch.setattr(service, "_expected_vendor_device_id", lambda _ref: None) + + with pytest.raises(facade_module.ConnectionVerificationError) as mismatch: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-b", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"][ + "revision" + ], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"][ + "intent_id" + ], + ) + ) + ) + + assert mismatch.value.reason_code == "control-bootstrap-device-identity-unverified" + assert writes == [] + assert topology_store.snapshot() == semantic_before + assert pin_store.snapshot() == pins_before + failed = service.state() + assert failed["connection_reconfiguration"]["intent"] == "select-device" + assert failed["selected_device_id"] is None + assert failed["device_session"] is None + assert failed["connection_lifecycle"]["active_binding"] is None + assert failed["application_control_session"]["state"] == "idle" + assert [item["device_id"] for item in failed["devices"]] == ["k1-b"] + supervisor = service._connection_supervisor.snapshot() # noqa: SLF001 + assert supervisor.authority.control_allowed is False + assert not ( + supervisor.intent is not None + and supervisor.device_network.intent_id == supervisor.intent.intent_id + and supervisor.device_network.transport_ref == "k1-b" + ) + + cancelled = asyncio.run( + service.prepare_connection_reconfiguration(_reconfiguration_request(failed, "cancel")) + ) + assert cancelled["connection_reconfiguration"]["status"] == "idle" + assert topology_store.snapshot() == semantic_before + assert pin_store.snapshot() == pins_before + + restarted, _ = service_with_fake_runtime(tmp_path) + restarted_state = restarted.state() + restarted_record = restarted._semantic_topology_store.snapshot().record # type: ignore[union-attr] # noqa: SLF001 + assert restarted_record is not None + assert restarted_record.transport_ref == "k1-a" + assert restarted_state["selected_device_id"] is None + assert restarted._configured_endpoint_target().transport_ref == "k1-a" # noqa: SLF001 + + +def test_cold_fresh_bridge_identity_mismatch_keeps_semantic_empty_after_restart( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + service._pin_or_match_device_identity( # noqa: SLF001 + transport_ref="k1-b", + logical_device_id="device-b", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ) + topology_store = service._semantic_topology_store # noqa: SLF001 + pin_store = service._device_identity_pin_store # noqa: SLF001 + assert topology_store is not None + assert pin_store is not None + assert topology_store.snapshot().status == "empty" + pins_before = pin_store.snapshot() + _set_scanned_k1(service, device_id="k1-b") + fresh = service.state() + writes: list[str] = [] + _install_successful_bridge_verify_transport( + monkeypatch, + device_id="k1-b", + ipv4="10.255.254.77", + writes=writes, + ) + monkeypatch.setattr(service, "_expected_vendor_device_id", lambda _ref: None) + + with pytest.raises(facade_module.ConnectionVerificationError) as mismatch: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-b", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + ) + ) + ) + + assert mismatch.value.reason_code == "control-bootstrap-device-identity-unverified" + assert writes == [] + assert topology_store.snapshot().status == "empty" + assert pin_store.snapshot() == pins_before + failed = service.state() + assert failed["connection_reconfiguration"]["status"] == "idle" + assert failed["selected_device_id"] is None + assert failed["device_session"] is None + assert failed["connection_lifecycle"]["active_binding"] is None + assert failed["application_control_session"]["state"] == "idle" + assert [item["device_id"] for item in failed["devices"]] == ["k1-b"] + + restarted, _ = service_with_fake_runtime(tmp_path) + restarted_state = restarted.state() + assert restarted_state["semantic_topology_store"]["status"] == "empty" + assert restarted_state["selected_device_id"] is None + + +def test_cold_fresh_bridge_commits_semantic_only_after_device_info_pin( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + topology_store = service._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + assert topology_store.snapshot().status == "empty" + _set_scanned_k1(service, device_id="k1-cold") + fresh = service.state() + writes: list[str] = [] + _install_successful_bridge_verify_transport( + monkeypatch, + device_id="k1-cold", + ipv4="10.255.254.88", + writes=writes, + ) + original_bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + semantic_status_during_bootstrap: list[str] = [] + + async def assert_cold_commit_is_deferred(**kwargs: object) -> None: + semantic_status_during_bootstrap.append(topology_store.snapshot().status) + await original_bootstrap(**kwargs) # type: ignore[arg-type] + semantic_status_during_bootstrap.append(topology_store.snapshot().status) + + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + assert_cold_commit_is_deferred, + ) + + verified = asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-cold", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + ) + ) + ) + + assert writes == [] + assert semantic_status_during_bootstrap == ["empty", "empty"] + record = topology_store.snapshot().record + assert record is not None + assert record.transport_ref == "k1-cold" + assert record.ipv4 == "10.255.254.88" + assert record.revision == 1 + assert verified["selected_device_id"] == "k1-cold" + assert verified["connection_reconfiguration"]["status"] == "idle" + assert verified["device_identity_pin_store"]["pin_count"] == 1 + + +def test_select_device_cancel_during_device_info_discards_only_provisional_b( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_durable_bridge_topology( + service, + transport_ref="k1-a", + ipv4="10.255.254.20", + ) + topology_store = service._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + semantic_before = topology_store.snapshot() + prepared = asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(service.state(), "select-device") + ) + ) + service._ble_discovery_generation = prepared["connection_reconfiguration"][ # noqa: SLF001 + "minimum_discovery_generation" + ] + _set_scanned_k1(service, device_id="k1-b") + fresh = service.state() + writes: list[str] = [] + _install_successful_bridge_verify_transport( + monkeypatch, + device_id="k1-b", + ipv4="10.255.254.77", + writes=writes, + ) + + async def scenario() -> None: + bootstrap_entered = asyncio.Event() + + async def paused_device_info(**_: object) -> None: + bootstrap_entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr( + service, + "_bootstrap_prestart_control_ready_owned", + paused_device_info, + ) + verify_task = asyncio.create_task( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-b", + source="fresh-scan", + compatibility_attestation=ATTESTATION, + expected_discovery_generation=fresh["ble_discovery_generation"], + expected_reconfiguration_revision=fresh["connection_reconfiguration"][ + "revision" + ], + expected_reconfiguration_intent_id=fresh["connection_reconfiguration"][ + "intent_id" + ], + ) + ) + ) + await asyncio.wait_for(bootstrap_entered.wait(), timeout=1.0) + projected = service.state() + assert projected["selected_device_id"] == "k1-b" + assert topology_store.snapshot() == semantic_before + verify_task.cancel() + with pytest.raises(asyncio.CancelledError): + await verify_task + + asyncio.run(scenario()) + + assert writes == [] + assert topology_store.snapshot() == semantic_before + cancelled = service.state() + assert cancelled["connection_reconfiguration"]["intent"] == "select-device" + assert cancelled["selected_device_id"] is None + assert cancelled["device_session"] is None + assert cancelled["connection_lifecycle"]["active_binding"] is None + assert [item["device_id"] for item in cancelled["devices"]] == ["k1-b"] + assert cancelled["last_operation"]["status"] == "cancelled" + + +def test_reconfiguration_aborts_only_purely_local_prepare_without_start_or_stop( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + control.state = "workspace-ready" + control.state_revision = 1 + service._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + service.runtime.snapshot(), + ) + prepared_acquisition = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + assert prepared_acquisition["acquisition"]["state"] == "prepared" + assert control.state == "project-ready" + device_edges: list[str] = [] + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + handed_off = asyncio.run( + service.prepare_connection_reconfiguration( + _reconfiguration_request(prepared_acquisition, "select-device") + ) + ) + + assert handed_off["acquisition"]["state"] == "aborted" + assert handed_off["acquisition"]["result"] == { + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "superseded-by-connection-reconfiguration", + } + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert device_edges == [] + + +def test_orphaned_prestart_control_retires_on_host_epoch_loss_but_not_transient_tcp( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + supervisor = service._connection_supervisor # noqa: SLF001 + before = supervisor.snapshot() + changed_path = HostPathProbeResult( + available=True, + fingerprint="test-route:changed-after-sleep", + interface="test0", + source_ipv4="192.168.1.2", + route_class="direct", + ) + new_epoch = supervisor.observe_host_path(changed_path) + assert new_epoch != before.host_path.epoch + assert supervisor.snapshot().control_plane.reason_code == "host-path-epoch-changed" + + retired = service.state() + + assert control.state == "idle" + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert retired["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + transient_service, _ = service_with_fake_runtime(tmp_path / "transient") + transient_control, _binding, _ready = _install_authoritative_ready_bridge_control( + transient_service + ) + transient_supervisor = transient_service._connection_supervisor # noqa: SLF001 + transient = transient_supervisor.snapshot() + assert transient.intent is not None + assert transient.device_network.target is not None + assert transient_supervisor.observe_endpoint( + target=transient.device_network.target, + intent_id=transient.intent.intent_id, + host_path_epoch=transient.host_path.epoch, + reachable=False, + reason_code="tcp-endpoint-unreachable", + ) + + one_negative = transient_service.state() + + assert transient_control.state == "connection-ready" + assert transient_service._application_control_process_lease_holders == { # noqa: SLF001 + "control" + } + assert one_negative["connection_policy"]["actions"]["scan-ble"]["allowed"] is False + + +def test_default_transport_observation_ttl_outlives_one_slow_healthy_monitor_cycle( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + monotonic_now = [100.0] + suspend_aware_now = [1_000.0] + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001 + supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001 + control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + + # One production pass can consume 11.5 seconds plus its one-second + # interval. Observer silence through the former 15-second boundary must + # not tear down a verified pre-START owner or fabricate an epoch change. + monotonic_now[0] += 10.0 + suspend_aware_now[0] += 10.0 + delayed = service.state() + + assert control.state == "connection-ready" + assert delayed["active_connection_mode"] == "bridge" + assert delayed["connection_lifecycle"]["active_binding_key"] is not None + assert supervisor.snapshot().authority.control_allowed is True + + # The fallback now tolerates two complete bounded monitor passes. + monotonic_now[0] += 5.01 + suspend_aware_now[0] += 5.01 + former_boundary = service.state() + + assert control.state == "connection-ready" + assert former_boundary["active_connection_mode"] == "bridge" + assert supervisor.snapshot().authority.control_allowed is True + + # Complete silence beyond 30 seconds still fails closed. Existing explicit + # negative and fingerprint-change tests cover the faster loss paths. + monotonic_now[0] += 15.0 + suspend_aware_now[0] += 15.0 + expired = service.state() + + assert control.state == "idle" + assert expired["active_connection_mode"] is None + assert supervisor.snapshot().authority.control_allowed is False + + +def test_orphan_retirement_waits_for_inflight_lifecycle_transition(tmp_path: Path) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control, _binding, _ready = _install_authoritative_ready_bridge_control(service) + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor.observe_host_path( + HostPathProbeResult( + available=True, + fingerprint="test-route:bootstrap-race", + interface="test0", + source_ipv4="192.168.1.2", + route_class="direct", + ) + ) + assert service._k1_lifecycle_transition_gate.acquire(blocking=False) # noqa: SLF001 + try: + service.state() + assert control.state == "connection-ready" + finally: + service._k1_lifecycle_transition_gate.release() # noqa: SLF001 + + service.state() + assert control.state == "idle" + + +def test_mode_select_winner_fences_connect_before_password_or_durable_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + selection_entered = threading.Event() + release_selection = threading.Event() + selection_failures: list[BaseException] = [] + selection_results: list[dict[str, Any]] = [] + original_runtime_snapshot = runtime.snapshot + paused = False + + def paused_snapshot() -> dict[str, object]: + nonlocal paused + if threading.current_thread().name == "reconfigure-mode-winner" and not paused: + paused = True + selection_entered.set() + assert release_selection.wait(2.0) + return original_runtime_snapshot() + + monkeypatch.setattr(runtime, "snapshot", paused_snapshot) + + def run_selection() -> None: + try: + selection_results.append( + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=0, + ) + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + selection_failures.append(exc) + + selection_thread = threading.Thread( + target=run_selection, + name="reconfigure-mode-winner", + daemon=True, + ) + selection_thread.start() + assert selection_entered.wait(2.0) + password_reads: list[str] = [] + losing_request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + expected_mode_revision=0, + expected_discovery_generation=0, + compatibility_attestation=ATTESTATION, + ) + + def forbidden_password_read(_secret: SecretStr) -> str: + password_reads.append("unwrapped") + raise AssertionError("losing Connect must fail before password unwrap") + + monkeypatch.setattr(SecretStr, "get_secret_value", forbidden_password_read) + try: + with pytest.raises(facade_module.NetworkProvisioningConflict) as losing_connect: + asyncio.run(service.connect(losing_request)) + finally: + release_selection.set() + selection_thread.join(timeout=3.0) + + assert losing_connect.value.reason_code == "connection-reconfiguration-lifecycle-busy" + assert password_reads == [] + assert selection_thread.is_alive() is False + assert selection_failures == [] + assert selection_results[0]["desired_connection_mode"] == "quick-connect" + assert service._operations.snapshot() == [] # noqa: SLF001 + assert service._network_provisioning_idempotency_journal.snapshot().records == () # noqa: SLF001 + + +def test_bridge_host_association_is_one_shot_and_connection_finishes_ready( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = 0 + associations: list[tuple[str, str]] = [] + route_classes = iter(["default-route", "direct-or-routed"]) + + async def successful_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-09T10:00:00Z", + "completed_at_utc": "2026-08-09T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: next(route_classes)) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "associate_with_ephemeral_wifi_credentials_once", + lambda _helper, ssid, password, **_kwargs: ( + associations.append((ssid, password)) + or { + "adapter": "CoreWLAN", + "outcome": "associated", + "already_associated": False, + "scan_attempt_count": 1, + "scan_elapsed_ms": 25, + } + ), + ) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + + state = asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + allow_host_wifi_switch=True, + ) + ) + ) + + assert writes == 1 + assert associations == [("lab-router", PRIMARY_TEST_CREDENTIAL)] + assert state["connection_lifecycle"]["connection_ready"] is True + assert state["connection_lifecycle"]["active_mode"] == "bridge" + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["result"]["host_route_class"] is None + assert operation["result"]["control_endpoint_reachable"] is None + assert operation["result"]["host_wifi_switch_authorized"] is True + assert operation["result"]["host_wifi_association_performed"] is True + assert PRIMARY_TEST_CREDENTIAL not in str(state) + + +def test_bridge_host_wifi_switch_defaults_to_denied_and_does_not_replay_network_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = 0 + association_calls: list[str] = [] + + async def successful_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-09T10:30:00Z", + "completed_at_utc": "2026-08-09T10:30:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + def forbidden_host_association(*_: object, **__: object) -> dict[str, Any]: + association_calls.append("attempted") + raise AssertionError("Bridge must not switch host Wi-Fi without explicit opt-in") + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "tunnel") + monkeypatch.setattr(facade_module, "_inspect_host_path", _tunnel_host_path) + monkeypatch.setattr( + facade_module, + "associate_with_ephemeral_wifi_credentials_once", + forbidden_host_association, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="bridge-host-switch-default-deny", + ) + assert request.allow_host_wifi_switch is False + + first = asyncio.run(service.connect(request)) + replay = asyncio.run(service.connect(request)) + + assert writes == 1 + assert association_calls == [] + operation = next(item for item in first["operations"] if item["action"] == "network.provision") + assert operation["status"] == "succeeded" + assert operation["result"]["phase"] == "network_applied" + assert operation["result"]["host_wifi_switch_authorized"] is False + assert operation["result"]["host_wifi_association_performed"] is False + assert operation["result"]["host_wifi_association_outcome"] == "not-authorized" + assert first["connection_attempt"]["phase"] == "network_applied" + assert first["connection_attempt"]["control_state"] == "unknown" + assert first["connection_attempt"]["safe_next_action"] == "wait-for-current-attempt" + assert replay["connection_attempt"]["phase"] == "network_applied" + assert replay["connection_attempt"]["control_state"] == "unknown" + assert replay["connection_attempt"]["safe_next_action"] == "verify-control-read-only" + assert PRIMARY_TEST_CREDENTIAL not in str(first) + + +def test_applied_network_bootstrap_failure_recovers_by_verify_without_rewrite( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = 0 + bootstrap_attempts = 0 + + async def successful_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-09T11:00:00Z", + "completed_at_utc": "2026-08-09T11:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + async def failed_bootstrap( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + nonlocal bootstrap_attempts + assert inspection_only is True + bootstrap_attempts += 1 + operation, _ = bound_service._operations.begin( # noqa: SLF001 + facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP, + context={ + "connection_mode": connection_mode, + "parent_operation_id": parent_operation_id, + "automatic_retry": False, + }, + ) + bound_service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="mqtt-device-info", + message_code="connection.control_bootstrap.running", + ) + failure = facade_module.ConnectionVerificationError( + "synthetic DeviceInfo timeout", + reason_code="control-bootstrap-timeout", + ) + bound_service._operations.transition( # noqa: SLF001 + operation.operation_id, + "failed", + stage_code="device-info-failed", + message_code="connection.control_bootstrap.failed", + error={ + "category": "connection", + "code": failure.reason_code, + "retryable": True, + "safe_to_retry": True, + "side_effect_status": "none", + }, + ) + raise failure + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + failed_bootstrap, + service, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="applied-network-device-info-recovery", + ) + + failed = asyncio.run(service.connect(request)) + assert writes == 1 + assert bootstrap_attempts == 1 + assert failed["configured_connection_mode"] == "bridge" + assert failed["active_connection_mode"] is None + assert failed["connection_lifecycle"]["connection_ready"] is False + assert failed["connection_attempt"]["status"] == "failed" + assert failed["connection_attempt"]["phase"] == "network_applied" + assert failed["connection_attempt"]["control_state"] == "control_not_ready" + assert failed["connection_attempt"]["safe_next_action"] == "verify-control-read-only" + assert failed["network_mutation_ledger"]["resolution"] == "target-observed" + assert failed["network_provisioning_idempotency"]["active_operation_id"] is None + + replayed = asyncio.run(service.connect(request)) + assert writes == 1 + assert bootstrap_attempts == 1 + assert replayed["connection_lifecycle"]["connection_ready"] is False + assert replayed["connection_attempt"]["phase"] == "network_applied" + assert replayed["connection_attempt"]["control_state"] == "control_not_ready" + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + _synthetic_prestart_control_bootstrap, + service, + ) + monkeypatch.setattr( + facade_module, + "_probe_control_endpoint_socket", + lambda _target: facade_module.TcpReachabilityProbeResult(reachable=True), + ) + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + lambda *_args, **_kwargs: asyncio.sleep( + 0, + result=_wifi_status_read("192.168.68.50", device_id="k1-a"), + ), + ) + recovered = asyncio.run(service.verify_connection(ConnectionVerifyRequest())) + + assert writes == 1 + assert recovered["connection_lifecycle"]["connection_ready"] is True + assert recovered["connection_lifecycle"]["active_mode"] == "bridge" + assert recovered["last_operation"]["action"] == "connection.verify" + assert recovered["last_operation"]["status"] == "succeeded" + assert recovered["connection_attempt"]["status"] == "succeeded" + assert recovered["connection_attempt"]["phase"] == "network_applied" + assert recovered["connection_attempt"]["control_state"] == "ready" + assert recovered["connection_attempt"]["safe_next_action"] == "start-acquisition" + assert ( + recovered["connection_attempt"]["recovery_operation_id"] + == recovered["last_operation"]["operation_id"] + ) + + semantic_after_verify = service._semantic_topology_store.snapshot() # type: ignore[union-attr] # noqa: SLF001 + assert semantic_after_verify.record is not None + assert semantic_after_verify.record.source == "ble-read-only-status" + + restarted_service, _ = service_with_fake_runtime(tmp_path) + restarted_replay = asyncio.run(restarted_service.connect(request)) + restarted_network = next( + item + for item in restarted_replay["operations"] + if item["action"] == facade_module.ACTION_NETWORK_PROVISION + ) + assert writes == 1 + assert restarted_network["stage_code"] == "durable-terminal-replay" + assert restarted_network["result"]["replay_binding_available"] is True + assert restarted_network["result"]["target_ipv4"] == "192.168.68.50" + assert restarted_replay["connection_attempt"]["control_state"] == "unknown" + + # A same-UUID advertisement is incidental presence, not a reason to hide + # the exact resolved-Apply LAN recovery target or force another GATT read. + _set_scanned_k1(restarted_service, device_id="k1-a") + advertised_recovery = restarted_service.state() + configured_observation = advertised_recovery["connection_policy"]["actions"][ + "observe-configured-device-network" + ] + assert advertised_recovery["current_device_recovery"] is None + assert configured_observation["allowed"] is True + assert configured_observation["required_transport_ref"] == "k1-a" + assert configured_observation["required_connection_mode"] == "bridge" + assert configured_observation["requires_live_gatt_validation"] is False + assert ( + "fresh-candidate-supersedes-durable-recovery" not in configured_observation["reason_codes"] + ) + + durable_ble_reads = 0 + + async def forbidden_durable_ble_read(*_: object, **__: object) -> dict[str, Any]: + nonlocal durable_ble_reads + durable_ble_reads += 1 + raise AssertionError("resolved Apply recovery must stay LAN/MQTT-only") + + monkeypatch.setattr( + facade_module, + "read_wifi_status_once", + forbidden_durable_ble_read, + ) + + restarted_verified = asyncio.run( + restarted_service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-a", + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert writes == 1 + assert durable_ble_reads == 0 + assert restarted_verified["last_operation"]["action"] == "connection.verify" + assert restarted_verified["last_operation"]["status"] == "succeeded" + assert ( + restarted_verified["connection_attempt"]["attempt_id"] == restarted_network["operation_id"] + ) + assert restarted_verified["connection_attempt"]["control_state"] == "ready" + assert restarted_verified["connection_attempt"]["safe_next_action"] == ("start-acquisition") + + +@pytest.mark.parametrize( + ("mismatched_field", "mismatched_value"), + [ + ("transport_ref", "foreign-k1"), + ("connection_mode", "quick-connect"), + ("ipv4", "192.168.68.99"), + ("source", "foreign-evidence-source"), + ("compatibility_profile_id", "foreign.compatibility.profile"), + ], +) +def test_resolved_apply_durable_mismatch_blocks_policy_and_verify_without_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + mismatched_field: str, + mismatched_value: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + ledger = service._network_mutation_ledger # noqa: SLF001 + observation = NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.50", + status_code=1, + reserved=0, + ) + prepared = ledger.prepare( + operation_id="resolved-apply-mismatch", + transport_ref="k1-a", + intended_mode="bridge", + write_mode="with_response", + baseline_status=NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=1, + ), + ) + dispatching = ledger.mark_dispatching( + prepared.operation_id, + expected_revision=prepared.revision, + ) + observing = ledger.mark_observing( + dispatching.operation_id, + expected_revision=dispatching.revision, + write_confirmed=True, + observation=observation, + ) + ledger.resolve( + observing.operation_id, + expected_revision=observing.revision, + resolution="target-observed", + ) + + topology_store = service._semantic_topology_store # noqa: SLF001 + assert topology_store is not None + exact_record = topology_store.commit( + transport_ref="k1-a", + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + firmware_version=ATTESTATION.firmware_version, + source="ble-post-write-status", + observed_at_utc="2026-08-11T12:00:00Z", + ) + mismatched_record = exact_record.as_dict() + mismatched_record[mismatched_field] = mismatched_value + mismatched_snapshot_dict = { + "schema_version": "missioncore.xgrids-k1-semantic-topology/v1", + "status": "available", + "configured_offline_evidence": True, + "live_connection_authority": False, + "reason_code": None, + "record": mismatched_record, + } + mismatched_snapshot = SimpleNamespace( + status="available", + record=SimpleNamespace(**mismatched_record), + reason_code=None, + configured_offline_evidence=True, + live_connection_authority=False, + as_dict=lambda: mismatched_snapshot_dict, + ) + monkeypatch.setattr(topology_store, "snapshot", lambda: mismatched_snapshot) + + calls = {"host_probe": 0, "ble_read": 0, "network_write": 0, "control_open": 0} + + def forbidden_host_probe(*_: object, **__: object) -> None: + calls["host_probe"] += 1 + raise AssertionError("mismatched durable target must not probe the host") + + async def forbidden_ble_read(*_: object, **__: object) -> dict[str, Any]: + calls["ble_read"] += 1 + raise AssertionError("mismatched durable target must not read BLE") + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + calls["network_write"] += 1 + raise AssertionError("mismatched durable target must not write the network") + + def forbidden_control_open(*_: object, **__: object) -> dict[str, object]: + calls["control_open"] += 1 + raise AssertionError("mismatched durable target must not open control") + + monkeypatch.setattr( + facade_module, + "_probe_configured_endpoint_host_only", + forbidden_host_probe, + ) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_ble_read) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "open", + forbidden_control_open, + ) + + before = service.state() + decision = before["connection_policy"]["actions"]["observe-configured-device-network"] + assert decision["allowed"] is False + assert "resolved-apply-durable-target-mismatch" in decision["reason_codes"] + assert decision["required_transport_ref"] is None + assert decision.get("required_connection_mode") is None + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="k1-a", + source="durable-configured-state", + compatibility_attestation=ATTESTATION, + ) + ) + ) + assert raised.value.reason_code == ("connection-verify-resolved-apply-target-mismatch") + assert calls == { + "host_probe": 0, + "ble_read": 0, + "network_write": 0, + "control_open": 0, + } + after = ledger.snapshot() + assert after.record is not None + assert after.record.operation_id == "resolved-apply-mismatch" + assert after.record.write_confirmed is True + assert after.record.resolution == "target-observed" + + +def test_applied_network_bootstrap_cancellation_does_not_cancel_fast_ack_or_rewrite( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = 0 + + async def successful_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal writes + writes += 1 + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-09T11:30:00Z", + "completed_at_utc": "2026-08-09T11:30:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + def cancelled_control_open(**_: object) -> None: + raise asyncio.CancelledError + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned, # noqa: SLF001 + service, + ) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "open", + cancelled_control_open, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="applied-network-device-info-cancelled", + ) + + fast_ack = asyncio.run(service.connect(request)) + assert fast_ack["connection_attempt"]["phase"] == "network_applied" + + cancelled = service.state() + assert writes == 1 + network_operation = next( + item for item in cancelled["operations"] if item["action"] == "network.provision" + ) + assert network_operation["status"] == "succeeded" + assert network_operation["result"]["phase"] == "network_applied" + bootstrap_operation = next( + item + for item in service._operations.snapshot() # noqa: SLF001 + if item["action"] == facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP + ) + assert bootstrap_operation["status"] == "cancelled" + assert cancelled["connection_attempt"]["phase"] == "network_applied" + assert cancelled["connection_attempt"]["control_state"] == "unknown" + assert cancelled["connection_attempt"]["safe_next_action"] == ("verify-control-read-only") + + replayed = asyncio.run(service.connect(request)) + assert writes == 1 + assert replayed["connection_attempt"]["phase"] == "network_applied" + assert replayed["connection_attempt"]["control_state"] == "unknown" + + +def _install_successful_fast_ack_bridge_write( + monkeypatch: pytest.MonkeyPatch, +) -> list[str]: + writes: list[str] = [] + + async def successful_write( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + writes.append("network-write") + _dispatch_test_network_write(on_write_dispatch) + return { + "started_at_utc": "2026-08-11T10:00:00Z", + "completed_at_utc": "2026-08-11T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "observations": [ + {"status": _wifi_status_read("192.168.68.50", device_id="k1-a")["status"]} + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", successful_write) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + return writes + + +def test_connect_fast_ack_does_not_wait_for_blocked_endpoint_and_write_is_once( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-blocked-endpoint-once", + ) + + async def scenario() -> None: + endpoint_started = asyncio.Event() + release_endpoint = asyncio.Event() + + async def blocked_endpoint(*_: object, **__: object) -> object: + endpoint_started.set() + await release_endpoint.wait() + path = _direct_host_path("192.168.68.50") + service._observe_connection_transport( # noqa: SLF001 + "192.168.68.50", + path=path, + reachable=True, + ) + return facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=path, + reachable=True, + reason_code=None, + ) + + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + blocked_endpoint, + ) + fast_ack = await asyncio.wait_for(service.connect(request), timeout=1.0) + assert endpoint_started.is_set() + assert writes == ["network-write"] + network_operation = next( + item + for item in fast_ack["operations"] + if item["action"] == facade_module.ACTION_NETWORK_PROVISION + ) + assert network_operation["status"] == "succeeded" + assert network_operation["stage_code"] == "network-configured" + assert network_operation["result"]["phase"] == "network_applied" + assert network_operation["result"]["control_state"] == "unknown" + assert network_operation["result"]["host_route_ready"] is None + assert network_operation["result"]["control_endpoint_reachable"] is None + assert fast_ack["connection_attempt"]["control_state"] == "unknown" + assert fast_ack["connection_attempt"]["safe_next_action"] == ("wait-for-current-attempt") + + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + assert continuation_task is not None + assert not continuation_task.done() + release_endpoint.set() + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + ready = service.state() + assert ready["connection_attempt"]["control_state"] == "ready" + assert ready["connection_attempt"]["safe_next_action"] == "start-acquisition" + + replay = await service.connect(request) + assert writes == ["network-write"] + assert replay["connection_attempt"]["phase"] == "network_applied" + + asyncio.run(scenario()) + + +def test_connect_fast_ack_does_not_wait_for_blocked_device_info( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + synthetic_bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-blocked-device-info", + ) + + async def scenario() -> None: + device_info_started = asyncio.Event() + release_device_info = asyncio.Event() + + async def blocked_bootstrap( + _service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + device_info_started.set() + await release_device_info.wait() + await synthetic_bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + blocked_bootstrap, + service, + ) + fast_ack = await asyncio.wait_for(service.connect(request), timeout=1.0) + assert device_info_started.is_set() + assert writes == ["network-write"] + assert fast_ack["connection_attempt"]["status"] == "running" + assert fast_ack["connection_attempt"]["control_state"] == "unknown" + assert fast_ack["connection_attempt"]["safe_next_action"] == ("wait-for-current-attempt") + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + assert continuation_task is not None + release_device_info.set() + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + assert service.state()["connection_attempt"]["control_state"] == "ready" + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("observed_state", ["ready", "scanning"]) +def test_reset_retired_apply_continuation_settles_without_receiver_rehydrate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + observed_state: str, +) -> None: + """A reset-owned Apply settles physical truth without reviving capture.""" + + service, runtime = service_with_fake_runtime(tmp_path) + old_binding = _seed_supervised_connection( + service, + transport_ref="k1-a", + logical_device_id="k1-a", + ) + _seed_unresolved_physical_stop_for_retirement(service, old_binding) + reset = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=service.state()["desired_connection_mode_revision"], + reset_scenario=True, + reset_id="reset-retired-apply-scanning-stop-only-0001", + ) + ) + + async def fresh_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + return _ble_scan_result("k1-a") + + monkeypatch.setattr(facade_module, "scan", fresh_scan) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + synthetic_bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + coordinator = service._physical_command_coordinator # noqa: SLF001 + inspection_modes: list[bool] = [] + + async def physical_bootstrap( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + inspection_modes.append(inspection_only) + assert inspection_only is True + coordinator.prepare_read_only_bootstrap() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=f"bootstrap:{parent_operation_id}:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="7" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-14T08:30:00.000Z", + ) + ) + await synthetic_bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + reopened = bound_service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert reopened is not None + assert reopened.stage == "observing" + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=( + reopened.identity.vendor_device_id_sha256 + ), + device_serial_sha256=reopened.identity.device_serial_sha256, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=( + reopened.identity.vendor_device_id_sha256 + ), + device_serial_sha256=reopened.identity.device_serial_sha256, + session_state=observed_state, # type: ignore[arg-type] + session_state_code=( + MODELING_STATE_BASE + (302 if observed_state == "scanning" else 300) + ), + project_bound=observed_state == "scanning", + project_id_sha256=( + "8" * 64 if observed_state == "scanning" else None + ), + init_ready=observed_state == "scanning", + status_message_sha256="9" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-14T08:30:01.000Z", + ) + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + async def forbidden_receiver_rehydrate(**_kwargs: object) -> None: + pytest.fail("reset-owned SCANNING settlement must not recreate a receiver") + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + physical_bootstrap, + service, + ) + monkeypatch.setattr( + service, + "_rehydrate_active_acquisition_after_restart", + forbidden_receiver_rehydrate, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="reset-retired-apply-scanning-stop-only", + operation_id="op-00000000-0000-4000-8000-000000001401", + expected_mode_revision=reset["desired_connection_mode_revision"], + expected_discovery_generation=scanned["ble_discovery_generation"], + ) + + async def scenario() -> tuple[ + dict[str, Any], + dict[str, Any], + dict[str, Any] | None, + ]: + applied = await service.connect(request) + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + if continuation_task is not None: + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + settled = service.state() + await service.connect(request.model_copy(update={"operation_id": None})) + successor: dict[str, Any] | None = None + if observed_state == "ready": + with pytest.raises(facade_module.NetworkProvisioningConflict) as stale: + await service.connect( + request.model_copy( + update={ + "operation_id": None, + "ssid": "changed-without-reconfiguration", + } + ) + ) + assert stale.value.reason_code == "physical-command-target-retired" + prepared = await service.prepare_connection_reconfiguration( + _reconfiguration_request(settled, "change-network") + ) + fresh = await service.scan_ble(BleScanRequest(duration_seconds=1.0)) + _set_scanned_k1(service, device_id="k1-a") + monkeypatch.setattr( + service, + "_schedule_control_bootstrap_continuation", + lambda **_kwargs: None, + ) + successor = await service.connect( + _connect_request( + device_id="k1-a", + ssid="new-lab-router", + password=SecretStr(SECONDARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="reset-ready-explicit-change-network-successor", + operation_id="op-00000000-0000-4000-8000-000000001402", + expected_mode_revision=prepared[ + "desired_connection_mode_revision" + ], + expected_discovery_generation=fresh[ + "ble_discovery_generation" + ], + expected_reconfiguration_revision=fresh[ + "connection_reconfiguration" + ]["revision"], + expected_reconfiguration_intent_id=fresh[ + "connection_reconfiguration" + ]["intent_id"], + ) + ) + return applied, settled, successor + + applied, settled, successor = asyncio.run(scenario()) + physical = settled["physical_command"] + control = settled["application_control_session"] + acquisition = settled["acquisition"] + + assert applied["connection_attempt"]["phase"] == "network_applied" + assert inspection_modes == [True] + assert writes == ( + ["network-write", "network-write"] + if observed_state == "ready" + else ["network-write"] + ) + expected_resolution = ( + "physical-active-observed" + if observed_state == "scanning" + else "physical-standby-observed" + ) + assert physical["record"]["resolution"] == expected_resolution + assert physical["requires_reconciliation"] is False + assert physical["resolved_active_recovery_required"] is ( + observed_state == "scanning" + ) + if observed_state == "scanning": + assert control["state"] == "scanning" + assert control["inspection_only"] is True + assert control["can_stop"] is True + assert control["can_start"] is False + assert acquisition["state"] == "failed" + assert acquisition["requested_streams"] == [] + assert acquisition["result"]["recovery_only"] is True + assert settled["connection_attempt"]["safe_next_action"] == "stop-acquisition" + assert settled["connection_policy"]["actions"]["stop-acquisition"]["allowed"] is True + assert settled["connection_policy"]["actions"]["start-acquisition"]["allowed"] is False + authority = service._connection_scenario_reset_retired_transport_authority # noqa: SLF001 + assert authority is not None + synthetic_standby = { + **physical, + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": False, + "reconciled_physical_state": "standby", + "physical_active": False, + "observed_session_state": "ready", + "record": { + **physical["record"], + "resolution": "stop-standby-observed", + }, + } + with service._lock: # noqa: SLF001 + assert ( + service._consumed_reset_authority_has_safe_reconfiguration_successor_locked( # noqa: SLF001 + authority=authority, + physical=physical, + request=request, + ) + is False + ) + # Even after a later STOP proves standby, the old request itself + # still lacks a separately committed Change Network generation. + assert ( + service._consumed_reset_authority_has_safe_reconfiguration_successor_locked( # noqa: SLF001 + authority=authority, + physical=synthetic_standby, + request=request, + ) + is False + ) + service._connection_reconfiguration_revision += 1 # noqa: SLF001 + service._connection_reconfiguration_intent_id = ( # noqa: SLF001 + "change-network-after-explicit-stop" + ) + service._connection_reconfiguration_intent = "change-network" # noqa: SLF001 + service._connection_reconfiguration_required_transport_ref = "k1-a" # noqa: SLF001 + service._connection_reconfiguration_required_connection_mode = "bridge" # noqa: SLF001 + service._ble_discovery_generation += 1 # noqa: SLF001 + service._connection_reconfiguration_minimum_discovery_generation = ( # noqa: SLF001 + service._ble_discovery_generation # noqa: SLF001 + ) + successor_request = request.model_copy( + update={ + "operation_id": None, + "ssid": "new-network-after-explicit-stop", + "idempotency_key": "new-key-after-explicit-stop", + "expected_discovery_generation": ( + service._ble_discovery_generation # noqa: SLF001 + ), + "expected_reconfiguration_revision": ( + service._connection_reconfiguration_revision # noqa: SLF001 + ), + "expected_reconfiguration_intent_id": ( + service._connection_reconfiguration_intent_id # noqa: SLF001 + ), + } + ) + assert service._consumed_reset_authority_has_safe_reconfiguration_successor_locked( # noqa: SLF001 + authority=authority, + physical=synthetic_standby, + request=successor_request, + ) + else: + assert control["state"] == "connection-ready" + assert control["inspection_only"] is True + assert control["inspection_promotion_allowed"] is True + assert settled["connection_attempt"]["safe_next_action"] == ( + "start-acquisition" + ) + assert successor is not None + assert successor["last_operation"]["status"] == "succeeded" + assert service._connection_scenario_reset_retired_transport_authority is None # noqa: SLF001 + assert runtime.phase == "idle" + assert runtime.source_mode == "idle" + assert runtime.start_calls == [] + assert service._acquisition_session_lease is None # noqa: SLF001 + assert service.camera_preview.snapshot()["phase"] == "idle" + assert service._live_perception_camera_binding is None # noqa: SLF001 + + +@pytest.mark.parametrize("supersession", ["intent", "runtime"]) +def test_control_bootstrap_continuation_cannot_publish_stale_parent_proof( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + supersession: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key=f"fast-ack-stale-{supersession}", + ) + + async def scenario() -> None: + endpoint_started = asyncio.Event() + release_endpoint = asyncio.Event() + + async def blocked_endpoint(*_: object, **__: object) -> object: + endpoint_started.set() + await release_endpoint.wait() + return facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=_direct_host_path("192.168.68.50"), + reachable=True, + reason_code=None, + ) + + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + blocked_endpoint, + ) + fast_ack = await service.connect(request) + assert endpoint_started.is_set() + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + if supersession == "runtime": + service._snapshot_runtime_id = "snapshot-runtime-superseded" # noqa: SLF001 + assert continuation_task is not None + if supersession == "intent": + service._connection_supervisor.set_intent( # noqa: SLF001 + intent_id="explicit-new-intent", + requested_mode="bridge", + ) + release_endpoint.set() + with pytest.raises(facade_module.ConnectionVerificationError): + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + state = service.state() + assert writes == ["network-write"] + assert fast_ack["connection_attempt"]["phase"] == "network_applied" + assert state["connection_attempt"]["phase"] == "network_applied" + assert state["connection_attempt"]["control_state"] == "control_not_ready" + assert state["connection_attempt"]["safe_next_action"] == ("verify-control-read-only") + assert state["network_mutation_ledger"]["resolution"] == "target-observed" + + asyncio.run(scenario()) + + +def test_service_close_cancels_owned_control_bootstrap_without_rewriting_network( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-service-close-owns-child", + ) + + async def scenario() -> None: + endpoint_started = asyncio.Event() + + async def blocked_endpoint(*_: object, **__: object) -> object: + endpoint_started.set() + await asyncio.Event().wait() + + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + blocked_endpoint, + ) + fast_ack = await service.connect(request) + assert endpoint_started.is_set() + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + continuation = service._control_bootstrap_continuation # noqa: SLF001 + assert continuation_task is not None + assert continuation is not None + service.close() + with pytest.raises(asyncio.CancelledError): + await continuation_task + child = service._operations.get(continuation.operation_id).as_dict() # noqa: SLF001 + network = service._operations.get(continuation.parent_operation_id).as_dict() # noqa: SLF001 + assert child["status"] == "cancelled" + assert network["status"] == "succeeded" + assert network["result"]["phase"] == "network_applied" + assert fast_ack["connection_attempt"]["phase"] == "network_applied" + assert writes == ["network-write"] + + asyncio.run(scenario()) + + +def test_control_bootstrap_final_fence_rejects_supersession_during_device_info( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + control = FakeInteractiveControlSession(initial_state="idle") + original_open = control.open + + def open_with_exact_identity( + *, + connection_binding: ApplicationConnectionBinding, + **kwargs: object, + ) -> dict[str, object]: + snapshot = original_open(connection_binding=connection_binding, **kwargs) + intent = service._connection_supervisor.snapshot().intent # noqa: SLF001 + assert intent is not None + control.verified_control = _verified_control_for_binding( + connection_binding, + logical_device_id=str(intent.expected_device_id or service._device_id), # noqa: SLF001 + ) + return snapshot + + monkeypatch.setattr(control, "open", open_with_exact_identity) + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + production_bootstrap = MethodType( + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned, # noqa: SLF001 + service, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-device-info-final-fence", + ) + + async def scenario() -> None: + device_info_ready = asyncio.Event() + release_device_info = asyncio.Event() + + async def ready_then_block( + _service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await production_bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + device_info_ready.set() + await release_device_info.wait() + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + ready_then_block, + service, + ) + fast_ack = await service.connect(request) + await asyncio.wait_for(device_info_ready.wait(), timeout=1.0) + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + continuation = service._control_bootstrap_continuation # noqa: SLF001 + assert continuation_task is not None + assert continuation is not None + child_before_release = service._operations.get( # noqa: SLF001 + continuation.operation_id + ).as_dict() + assert child_before_release["status"] == "running" + + service._connection_supervisor.record_monitor_failure( # noqa: SLF001 + "test-route-lost-during-device-info" + ) + release_device_info.set() + with pytest.raises(facade_module.ConnectionVerificationError): + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + + child = service._operations.get(continuation.operation_id).as_dict() # noqa: SLF001 + network = service._operations.get(continuation.parent_operation_id).as_dict() # noqa: SLF001 + state = service.state() + assert child["status"] == "failed" + assert network["status"] == "succeeded" + assert network["result"]["phase"] == "network_applied" + assert fast_ack["connection_attempt"]["phase"] == "network_applied" + assert state["connection_attempt"]["control_state"] == "control_not_ready" + assert state["connection_attempt"]["safe_next_action"] != "start-acquisition" + assert control.state in {"closed", "idle"} + assert writes == ["network-write"] + + asyncio.run(scenario()) + + +def test_historical_bootstrap_success_loses_ready_projection_with_live_route( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-live-authority-loss", + ) + + async def scenario() -> None: + await service.connect(request) + with service._lock: # noqa: SLF001 + continuation_task = service._control_bootstrap_continuation_task # noqa: SLF001 + if continuation_task is not None: + await asyncio.wait_for(asyncio.shield(continuation_task), timeout=1.0) + ready = service.state() + assert ready["connection_attempt"]["control_state"] == "ready" + assert ready["connection_attempt"]["safe_next_action"] == "start-acquisition" + + service._connection_supervisor.record_monitor_failure( # noqa: SLF001 + "test-route-loss-after-bootstrap-success" + ) + lost = service.state() + child = next( + item + for item in service._operations.snapshot() # noqa: SLF001 + if item["action"] == facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP + and item.get("context", {}).get("ownership") == "service-owned-apply-continuation" + ) + assert child["status"] == "succeeded" + assert lost["connection_attempt"]["control_state"] == "control_not_ready" + assert lost["connection_attempt"]["safe_next_action"] != "start-acquisition" + assert lost["connection_lifecycle"]["ready_to_start"] is False + assert writes == ["network-write"] + + asyncio.run(scenario()) + + +def test_control_bootstrap_cancelled_before_first_step_is_never_left_accepted( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + writes = _install_successful_fast_ack_bridge_write(monkeypatch) + monkeypatch.setattr( + facade_module, + "_await_control_endpoint_reachable", + _reachable_control_endpoint, + ) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="fast-ack-cancel-before-first-step", + ) + + async def scenario() -> None: + connected = await service.connect(request) + with service._lock: # noqa: SLF001 + first_task = service._control_bootstrap_continuation_task # noqa: SLF001 + if first_task is not None: + await asyncio.wait_for(asyncio.shield(first_task), timeout=1.0) + await asyncio.sleep(0) + network = next( + item + for item in connected["operations"] + if item["action"] == facade_module.ACTION_NETWORK_PROVISION + ) + result = network["result"] + service._schedule_control_bootstrap_continuation( # noqa: SLF001 + parent_operation_id=network["operation_id"], + connection_mode=result["connection_mode"], + transport_ref=result["transport_ref"], + target=facade_module.EndpointTarget( + result["target_ipv4"], + result["target_port"], + ), + device_session_id=result["device_session_id"], + ) + with service._lock: # noqa: SLF001 + cancelled_task = service._control_bootstrap_continuation_task # noqa: SLF001 + cancelled_continuation = service._control_bootstrap_continuation # noqa: SLF001 + assert cancelled_task is not None + assert cancelled_continuation is not None + # No await has occurred since create_task: the coroutine has not entered. + service._request_control_bootstrap_continuation_close() # noqa: SLF001 + with pytest.raises(asyncio.CancelledError): + await cancelled_task + await asyncio.sleep(0) + child = service._operations.get( # noqa: SLF001 + cancelled_continuation.operation_id + ).as_dict() + assert child["status"] == "cancelled" + assert child["stage_code"] == "control-bootstrap-cancelled" + with service._lock: # noqa: SLF001 + assert service._control_bootstrap_continuation is None # noqa: SLF001 + assert service._control_bootstrap_continuation_task is None # noqa: SLF001 + assert service._control_bootstrap_continuation_loop is None # noqa: SLF001 + assert writes == ["network-write"] + + asyncio.run(scenario()) + + +def _seed_terminal_network_attempt( + service: XgridsK1CompatibilityService, +) -> None: + operation, created = service._operations.begin( # noqa: SLF001 + facade_module.ACTION_NETWORK_PROVISION, + operation_id="op-00000000-0000-4000-8000-000000009901", + context={ + "snapshot_runtime_id": service.state()["snapshot_runtime_id"], + "transport_ref": "terminal-attempt-k1", + "connection_mode": "bridge", + }, + ) + assert created is True + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="ble-provisioning-write", + message_code="connection.network.running", + ) + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "succeeded", + stage_code="network-configured", + message_code="connection.network.succeeded", + result={ + "phase": "network_applied", + "side_effect_status": "applied", + "automatic_retry": False, + }, + ) + + +def test_terminal_connection_attempt_projects_current_local_cleanup_not_wait( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _seed_terminal_network_attempt(service) + runtime.source_mode = "live" + runtime.phase = "error" + + state = service.state() + + assert state["connection_attempt"]["status"] == "succeeded" + assert state["connection_policy"]["actions"]["stop-local-receiver"]["allowed"] is True + assert state["connection_attempt"]["safe_next_action"] == ("stop-local-receiver") + assert ( + state["connection_attempt"]["diagnostic_bundle"]["attempt"]["safe_next_action"] + == "stop-local-receiver" + ) + + +def test_terminal_connection_attempt_without_allowed_action_never_claims_wait_or_scan( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + _seed_terminal_network_attempt(service) + monkeypatch.setattr( + facade_module, + "ble_runtime_snapshot", + lambda: { + "owner_epoch": 9, + "owner_loop_bound": False, + "active_operation_kind": "status-read", + "cleanup_pending": True, + "poisoned": True, + }, + ) + + state = service.state() + + assert state["connection_attempt"]["status"] == "succeeded" + assert state["connection_policy"]["actions"]["scan-ble"]["allowed"] is False + assert state["connection_policy"]["actions"]["stop-local-receiver"]["allowed"] is False + assert state["connection_attempt"]["safe_next_action"] == ("manual-recovery-required") + + +def test_connection_attempt_projection_never_starts_without_ready_control() -> None: + runtime_id = "snapshot-runtime-test" + network_operation_id = "op-00000000-0000-4000-8000-000000000001" + network_operation = { + "operation_id": network_operation_id, + "action": facade_module.ACTION_NETWORK_PROVISION, + "status": "succeeded", + "stage_code": "network-configured", + "context": { + "snapshot_runtime_id": runtime_id, + "transport_ref": "k1-a", + "connection_mode": "bridge", + }, + "result": { + "phase": "network_applied", + "control_state": "unknown", + "snapshot_runtime_id": runtime_id, + "parent_intent_id": network_operation_id, + "transport_ref": "k1-a", + "connection_mode": "bridge", + "target_ipv4": "192.168.68.50", + "target_port": facade_module.CONTROL_MQTT_PORT, + }, + "events": [], + } + without_child = facade_module._connection_attempt_projection([network_operation]) # noqa: SLF001 + assert without_child is not None + assert without_child["control_state"] == "unknown" + assert without_child["safe_next_action"] == "verify-control-read-only" + + foreign_child = { + "operation_id": "op-00000000-0000-4000-8000-000000000002", + "action": facade_module.ACTION_CONNECTION_CONTROL_BOOTSTRAP, + "status": "running", + "stage_code": "mqtt-device-info", + "context": { + "parent_operation_id": network_operation["operation_id"], + "connection_mode": "bridge", + }, + "events": [], + } + foreign = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, foreign_child] + ) + assert foreign is not None + assert foreign["safe_next_action"] == "verify-control-read-only" + + owned_child = { + **foreign_child, + "context": { + "ownership": "service-owned-apply-continuation", + "snapshot_runtime_id": runtime_id, + "parent_operation_id": network_operation["operation_id"], + "parent_intent_id": network_operation["operation_id"], + "connection_mode": "bridge", + "transport_ref": "k1-a", + "target_ipv4": "192.168.68.50", + "target_port": facade_module.CONTROL_MQTT_PORT, + "device_session_id": "device-session-test", + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + } + owned = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, owned_child] + ) + assert owned is not None + assert owned["control_state"] == "unknown" + assert owned["safe_next_action"] == "wait-for-current-attempt" + + ready_child = { + **owned_child, + "status": "succeeded", + "result": { + "connection_mode": "bridge", + "control_verified": True, + "network_mutation_performed": False, + "ble_operation_performed": False, + "automatic_retry": False, + }, + } + ready = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, ready_child] + ) + assert ready is not None + assert ready["control_state"] == "ready" + assert ready["safe_next_action"] == "start-acquisition" + + scanning_child = { + **ready_child, + "result": { + **ready_child["result"], + "physical_reconciliation": { + "performed": True, + "resolution": "physical-active-observed", + "observed_session_state": "scanning", + "device_write_performed": False, + "automatic_retry": False, + }, + }, + } + scanning = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, scanning_child] + ) + assert scanning is not None + assert scanning["control_state"] == "unknown" + assert scanning["safe_next_action"] == "stop-acquisition" + assert scanning["safe_next_action"] != "start-acquisition" + assert scanning["physical_reconciliation"] == ( + scanning_child["result"]["physical_reconciliation"] + ) + + foreign_verify = { + "operation_id": "op-00000000-0000-4000-8000-000000000003", + "action": facade_module.ACTION_CONNECTION_VERIFY, + "status": "succeeded", + "stage_code": "device-info-confirmed", + "context": { + "snapshot_runtime_id": runtime_id, + "requested_transport_ref": "other-k1", + "requested_connection_mode": "quick-connect", + }, + "result": { + "write_performed": False, + "control_verified": True, + "verified_binding": { + "snapshot_runtime_id": runtime_id, + "intent_id": "foreign-intent", + "transport_ref": "other-k1", + "connection_mode": "quick-connect", + "target_ipv4": "192.168.43.1", + "target_port": facade_module.CONTROL_MQTT_PORT, + "host_path_epoch": 2, + }, + }, + "events": [], + } + foreign_verify_projection = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, foreign_verify] + ) + assert foreign_verify_projection is not None + assert foreign_verify_projection["control_state"] == "unknown" + assert foreign_verify_projection["safe_next_action"] == "verify-control-read-only" + + exact_verify_context = { + "snapshot_runtime_id": runtime_id, + "recovery_parent_operation_id": network_operation_id, + "recovery_parent_intent_id": network_operation_id, + "recovery_transport_ref": "k1-a", + "recovery_connection_mode": "bridge", + "recovery_target_ipv4": "192.168.68.50", + "recovery_target_port": facade_module.CONTROL_MQTT_PORT, + } + exact_verify = { + **foreign_verify, + "status": "running", + "context": exact_verify_context, + "result": None, + } + exact_pending = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, exact_verify] + ) + assert exact_pending is not None + assert exact_pending["control_state"] == "unknown" + assert exact_pending["safe_next_action"] == "wait-for-current-attempt" + + wrong_verify_binding = { + **exact_verify, + "status": "succeeded", + "result": foreign_verify["result"], + } + wrong_verify = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, wrong_verify_binding] + ) + assert wrong_verify is not None + assert wrong_verify["control_state"] == "unknown" + assert wrong_verify["safe_next_action"] == "verify-control-read-only" + + exact_verify_success = { + **exact_verify, + "status": "succeeded", + "result": { + "write_performed": False, + "control_verified": True, + "verified_binding": { + "snapshot_runtime_id": runtime_id, + "intent_id": "durable-read-only-verify-intent", + "transport_ref": "k1-a", + "connection_mode": "bridge", + "target_ipv4": "192.168.68.50", + "target_port": facade_module.CONTROL_MQTT_PORT, + "host_path_epoch": 2, + }, + }, + } + exact_ready = facade_module._connection_attempt_projection( # noqa: SLF001 + [network_operation, exact_verify_success] + ) + assert exact_ready is not None + assert exact_ready["control_state"] == "ready" + assert exact_ready["safe_next_action"] == "start-acquisition" + + +def test_connect_reconfiguration_admission_fences_mode_selection_before_transition_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Connect's pre-secret semantic admission is one deterministic winner.""" + + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + borrow_entered = threading.Event() + release_borrow = threading.Event() + connect_failures: list[BaseException] = [] + device_writes: list[str] = [] + + @contextmanager + def paused_network_ble_borrow() -> Iterator[object]: + # Connect has passed its semantic admission and retains the + # reconfiguration gate across durable journal creation. Mode Select + # cannot publish a stale draft in this pre-transition interval. + borrow_entered.set() + assert release_borrow.wait(2.0) + raise RuntimeError("synthetic admitted Connect ended before BLE") + yield object() # pragma: no cover - contextmanager generator marker + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + device_writes.append("gatt-write") + raise AssertionError("stale mode Connect crossed the GATT boundary") + + monkeypatch.setattr( + service, + "_borrow_network_ble_process_lease", + paused_network_ble_borrow, + ) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="connect-admission-wins-before-mode-selection", + ) + + def run_connect() -> None: + try: + asyncio.run(service.connect(request)) + except BaseException as exc: # pragma: no branch - expected contract edge + connect_failures.append(exc) + + connect_thread = threading.Thread(target=run_connect, daemon=True) + connect_thread.start() + assert borrow_entered.wait(2.0) + try: + with pytest.raises(facade_module.NetworkProvisioningConflict) as blocked: + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=0, + ) + ) + assert blocked.value.reason_code == "connection-mode-selection-lifecycle-busy" + finally: + release_borrow.set() + connect_thread.join(timeout=3.0) + + assert connect_thread.is_alive() is False + assert len(connect_failures) == 1 + assert isinstance(connect_failures[0], RuntimeError) + assert str(connect_failures[0]) == "synthetic admitted Connect ended before BLE" + assert device_writes == [] + state = service.state() + assert state["desired_connection_mode"] == "bridge" + assert state["desired_connection_mode_revision"] == 0 + assert state["configured_connection_mode"] is None + assert state["active_connection_mode"] is None + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["status"] == "failed" + assert operation["error"]["side_effect_status"] == "none" + terminal_record = next( + record + for record in ( + service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + .snapshot() + .records + ) + if record.operation_id == operation["operation_id"] + ) + terminal = terminal_record.terminal + assert terminal is not None + assert terminal.side_effect_status == "none" + assert terminal.safe_to_retry is True + + +def test_mode_selection_is_rejected_while_connect_owns_transition_gate( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Once Connect owns lifecycle admission its exact mode cannot change.""" + + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + capture_entered = threading.Event() + release_capture = threading.Event() + connect_failures: list[BaseException] = [] + device_writes: list[str] = [] + + def paused_capture(_device_id: str) -> None: + # Capture is the first candidate-bound step after Connect acquired the + # lifecycle gate and repeated its exact mode/discovery CAS check. + capture_entered.set() + assert release_capture.wait(2.0) + return None + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + device_writes.append("gatt-write") + raise AssertionError("test precondition must fail before GATT") + + monkeypatch.setattr(facade_module, "_capture_network_intent_device", paused_capture) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + request = _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + idempotency_key="mode-cas-connect-owns-transition-gate", + ) + + def run_connect() -> None: + try: + asyncio.run(service.connect(request)) + except BaseException as exc: # pragma: no branch - expected contract edge + connect_failures.append(exc) + + connect_thread = threading.Thread(target=run_connect, daemon=True) + connect_thread.start() + assert capture_entered.wait(2.0) + try: + with pytest.raises(facade_module.NetworkProvisioningConflict) as busy_select: + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=0, + ) + ) + assert busy_select.value.reason_code == "connection-mode-selection-lifecycle-busy" + during_connect = service.state() + assert during_connect["desired_connection_mode"] == "bridge" + assert during_connect["desired_connection_mode_revision"] == 0 + finally: + release_capture.set() + connect_thread.join(timeout=3.0) + + assert connect_thread.is_alive() is False + assert len(connect_failures) == 1 + assert isinstance(connect_failures[0], facade_module.NetworkProvisioningConflict) + assert connect_failures[0].reason_code == "network-provision-candidate-not-fresh" + assert device_writes == [] + state = service.state() + assert state["desired_connection_mode"] == "bridge" + assert state["desired_connection_mode_revision"] == 0 + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert operation["status"] == "failed" + assert operation["error"]["side_effect_status"] == "none" + + +def test_mode_selection_winning_before_start_final_admission_publishes_no_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A newer prepared-mode draft wins before START owns its final fence.""" + + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host=facade_module.AP_FALLBACK_IPV4, + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + fingerprint_entered = threading.Event() + release_fingerprint = threading.Event() + start_failures: list[BaseException] = [] + selection_results: list[dict[str, Any]] = [] + selection_failures: list[BaseException] = [] + original_fingerprint = service._request_fingerprint # noqa: SLF001 + + def paused_fingerprint(action: str, payload: dict[str, Any]) -> str: + if action == facade_module.ACTION_ACQUISITION_START: + fingerprint_entered.set() + assert release_fingerprint.wait(2.0) + return original_fingerprint(action, payload) + + monkeypatch.setattr(service, "_request_fingerprint", paused_fingerprint) + + def run_start() -> None: + try: + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + except BaseException as exc: # pragma: no branch - expected race loser + start_failures.append(exc) + + start_thread = threading.Thread(target=run_start, daemon=True) + start_thread.start() + assert fingerprint_entered.wait(2.0) + + def run_selection() -> None: + try: + selection_results.append( + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=1, + ) + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + selection_failures.append(exc) + + selection_thread = threading.Thread(target=run_selection, daemon=True) + selection_thread.start() + try: + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + if service._desired_connection_mode_revision == 2: # noqa: SLF001 + break + time.sleep(0.01) + with service._lock: # noqa: SLF001 + assert service._desired_connection_mode == "bridge" # noqa: SLF001 + assert service._desired_connection_mode_revision == 2 # noqa: SLF001 + finally: + release_fingerprint.set() + start_thread.join(timeout=3.0) + selection_thread.join(timeout=3.0) + + assert start_thread.is_alive() is False + assert selection_thread.is_alive() is False + assert selection_failures == [] + assert selection_results[0]["desired_connection_mode"] == "bridge" + assert selection_results[0]["desired_connection_mode_revision"] == 2 + assert len(start_failures) == 1 + assert isinstance(start_failures[0], facade_module.NetworkProvisioningConflict) + assert start_failures[0].reason_code == "connection-mode-switch-pending" + assert control.start_projects == [] + assert runtime.start_calls == [] + state = service.state() + assert state["acquisition"]["state"] == "prepared" + start_operation = next( + item for item in state["operations"] if item["action"] == "acquisition.start" + ) + assert start_operation["status"] == "failed" + assert start_operation["error"]["side_effect_status"] == "none" + + +def test_start_owned_and_queued_states_reject_mode_selection_until_dispatch_resolves( + tmp_path: Path, +) -> None: + """Queued START remains a mode-selection fence after the HTTP call returns.""" + + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + request_start_entered = threading.Event() + release_request_start = threading.Event() + + class QueuedStartControl(FakeInteractiveControlSession): + def request_start( + self, + *, + project_name: str, + confirmation: object, + command_context: object, + preparation_checkpoint_observer: object | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + del preparation_checkpoint_observer + assert confirmation is not None + assert command_context is not None + assert self.state == "project-ready" + self._accept_checkpoint( + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + self.start_projects.append(project_name) + self.start_contexts.append(command_context) + self.state = "start-requested" + request_start_entered.set() + assert release_request_start.wait(2.0) + # The real worker has only been released here; modeling:start has + # deliberately not been published in this deterministic fixture. + return self.snapshot() + + control = QueuedStartControl() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host=facade_module.AP_FALLBACK_IPV4, + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + start_failures: list[BaseException] = [] + + def run_start() -> None: + try: + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + start_failures.append(exc) + + start_thread = threading.Thread(target=run_start, daemon=True) + start_thread.start() + assert request_start_entered.wait(2.0) + try: + with pytest.raises(facade_module.NetworkProvisioningConflict) as owned: + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=1, + ) + ) + assert owned.value.reason_code == "connection-mode-selection-lifecycle-busy" + finally: + release_request_start.set() + start_thread.join(timeout=3.0) + + assert start_thread.is_alive() is False + assert start_failures == [] + queued = service.state() + assert queued["acquisition"]["state"] == "starting" + assert queued["desired_connection_mode"] == "quick-connect" + assert queued["desired_connection_mode_revision"] == 1 + assert queued["connection_lifecycle"]["mode_selection"] == { + "allowed": False, + "reason_codes": [ + "connection-mode-selection-physical-state-unsafe", + "connection-mode-selection-control-state-unsafe", + ], + "automatic_retry": False, + } + assert "select-connection-mode" not in queued["connection_lifecycle"]["allowed_actions"] + assert runtime.start_calls + with pytest.raises(facade_module.NetworkProvisioningConflict) as queued_select: + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=1, + ) + ) + assert queued_select.value.reason_code == "connection-mode-selection-physical-state-unsafe" + assert service.state()["desired_connection_mode_revision"] == 1 + + +@pytest.mark.parametrize( + ("source_mode", "target_mode", "target_ipv4", "source_attestation"), + [ + ( + "quick-connect", + "bridge", + facade_module.AP_FALLBACK_IPV4, + QUICK_CONNECT_ATTESTATION, + ), + ( + "bridge", + "quick-connect", + "192.168.1.20", + ATTESTATION, + ), + ], + ids=["quick-to-bridge", "bridge-to-quick"], +) +def test_pending_mode_scan_aborts_purely_local_prepared_acquisition_without_stop( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + source_mode: facade_module.ConnectionMode, + target_mode: facade_module.ConnectionMode, + target_ipv4: str, + source_attestation: CompatibilityAttestationRequest, +) -> None: + """PREPARE without START is locally reversible at the explicit Scan commit.""" + + service, runtime = service_with_fake_runtime(tmp_path) + if source_mode != "bridge": + _select_connection_mode(service, source_mode) + source_revision = service.state()["desired_connection_mode_revision"] + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=target_ipv4, + connection_mode=source_mode, + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host=target_ipv4, + compatibility_attestation=source_attestation, + ) + ) + assert prepared["acquisition"]["state"] == "prepared" + assert control.state == "project-ready" + switched = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode=target_mode, + expected_revision=source_revision, + ) + ) + assert switched["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert switched["connection_lifecycle"]["mode_selection"]["allowed"] is True + assert "select-connection-mode" in switched["connection_lifecycle"]["allowed_actions"] + + physical_or_network_edges: list[str] = [] + + async def fake_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "aborted" # noqa: SLF001 + assert control.state == "idle" + assert on_admitted is not None + on_admitted() + return _ble_scan_result(f"{target_mode}-candidate") + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + physical_or_network_edges.append("network-write") + raise AssertionError("prepared mode switch must not write K1 network state") + + monkeypatch.setattr(facade_module, "scan", fake_scan) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: physical_or_network_edges.append("physical-command"), + ) + + scanned = asyncio.run(service.scan_ble(1.0)) + + assert physical_or_network_edges == [] + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert scanned["acquisition"]["state"] == "aborted" + assert scanned["acquisition"]["result"] == { + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "superseded-by-connection-mode-switch", + } + assert scanned["configured_connection_mode"] is None + assert scanned["active_connection_mode"] is None + assert [item["device_id"] for item in scanned["devices"]] == [f"{target_mode}-candidate"] + + +def test_mode_draft_can_be_cancelled_before_scan_without_touching_active_binding( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Quick -> Bridge draft -> Quick is two local CAS writes and no I/O.""" + + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + control = FakeInteractiveControlSession(initial_state="project-ready") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + before = service.state() + before_supervisor_revision = service._connection_supervisor.snapshot().revision # noqa: SLF001 + device_edges: list[str] = [] + + async def forbidden_scan(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("ble-scan") + raise AssertionError("draft cancellation must not scan") + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("draft cancellation must not write network state") + + monkeypatch.setattr(facade_module, "scan", forbidden_scan) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + switched = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=before["desired_connection_mode_revision"], + ) + ) + cancelled = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=switched["desired_connection_mode_revision"], + ) + ) + + assert device_edges == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert control.state == "project-ready" + assert control.start_projects == [] + assert control.stop_calls == 0 + assert cancelled["desired_connection_mode_revision"] == ( + before["desired_connection_mode_revision"] + 2 + ) + assert cancelled["desired_connection_mode"] == "quick-connect" + assert cancelled["configured_connection_mode"] == "quick-connect" + assert cancelled["active_connection_mode"] == "quick-connect" + assert ( + cancelled["connection_lifecycle"]["active_binding_key"] + == before["connection_lifecycle"]["active_binding_key"] + ) + assert ( + cancelled["connection_lifecycle"]["active_binding"] + == before["connection_lifecycle"]["active_binding"] + ) + assert cancelled["connection_lifecycle"]["mode_change"]["state"] == "ready" + assert cancelled["connection_lifecycle"]["connection_ready"] is True + assert cancelled["connection_lifecycle"]["ready_to_start"] is True + assert cancelled["connection_lifecycle"]["mode_selection"]["allowed"] is True + assert service._connection_supervisor.snapshot().revision == before_supervisor_revision # noqa: SLF001 + + +def test_lifecycle_mode_selection_is_disallowed_while_acquiring( + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + control = FakeInteractiveControlSession() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host=facade_module.AP_FALLBACK_IPV4, + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + ) + ) + with service._lock: # noqa: SLF001 + assert service._acquisition is not None # noqa: SLF001 + service._acquisition.transition( # noqa: SLF001 + "starting", + message_code="acquisition.start.waiting_receiver_ready", + ) + service._acquisition.transition( # noqa: SLF001 + "acquiring", + message_code="acquisition.acquiring", + ) + service._acquisition_start_operation_id = "test-start-operation" # noqa: SLF001 + runtime.source_mode = "live" + runtime.phase = "live" + control.state = "scanning" + + state = service.state() + + assert state["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert ( + "connection-mode-selection-physical-state-unsafe" + in state["connection_lifecycle"]["mode_selection"]["reason_codes"] + ) + assert ( + "connection-mode-selection-control-state-unsafe" + in state["connection_lifecycle"]["mode_selection"]["reason_codes"] + ) + assert "select-connection-mode" not in state["connection_lifecycle"]["allowed_actions"] + + +def test_lifecycle_mode_selection_is_disallowed_by_unsafe_physical_ledger( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession(initial_state="connection-ready") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection(service, connection_mode="bridge") + control.verified_control = _verified_control_for_binding(binding) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: { + "status": "unresolved", + "requires_reconciliation": True, + "record": None, + }, + ) + + state = service.state() + + assert state["connection_lifecycle"]["mode_selection"] == { + "allowed": False, + "reason_codes": ["connection-mode-selection-physical-state-unsafe"], + "automatic_retry": False, + } + assert "select-connection-mode" not in state["connection_lifecycle"]["allowed_actions"] + + +def test_disconnected_mode_draft_ignores_old_physical_history_without_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A cold selector is local; the old ledger still fences later actions.""" + + service, runtime = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession(initial_state="idle") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + physical_snapshot = { + "status": "unresolved", + "requires_reconciliation": True, + "resolved_active_recovery_required": False, + "record": { + "operation_id": "old-stop", + "action": "stop", + "stage": "observing", + "resolution": None, + }, + } + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_snapshot, + ) + device_edges: list[str] = [] + monkeypatch.setattr( + facade_module, + "scan", + lambda *_args, **_kwargs: device_edges.append("scan"), + ) + monkeypatch.setattr( + facade_module, + "provision_wifi_once", + lambda *_args, **_kwargs: device_edges.append("network-write"), + ) + + before = service.state() + assert before["connection_lifecycle"]["mode_selection"]["allowed"] is True + + selected = service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=before["desired_connection_mode_revision"], + ) + ) + + assert selected["desired_connection_mode"] == "quick-connect" + assert selected["desired_connection_mode_revision"] == 1 + assert selected["physical_command"]["requires_reconciliation"] is True + assert selected["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is False + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert control.state == "idle" + assert device_edges == [] + + +def test_carried_old_reopen_audit_does_not_freeze_a_successor_mode_draft() -> None: + physical_snapshot: dict[str, object] = { + "status": "unresolved", + "requires_reconciliation": True, + "record": { + "operation_id": "successor-stop", + "revision": 41, + "stage": "observing", + "resolution": None, + "operator_retirements": [ + { + "retirement_id": "old-retirement", + "original_attempt": {"operation_id": "old-stop"}, + } + ], + "operator_reconciliation_reopens": [ + { + "retirement_id": "old-retirement", + "retired_record_revision": 12, + } + ], + }, + } + + assert ( + facade_module._physical_command_pending_reopened_reconciliation_mode( # noqa: SLF001 + physical_snapshot + ) + is None + ) + assert ( + facade_module._connection_mode_selection_reason_codes( # noqa: SLF001 + acquisition_state=None, + acquisition_lease_retained=False, + acquisition_start_operation_id=None, + runtime={"source_mode": "idle"}, + physical_command=physical_snapshot, + control_state="idle", + desired_connection_mode="bridge", + ) + == [] + ) + + +_RECOVERY_VENDOR_HASH = "a" * 64 +_RECOVERY_SERIAL_HASH = "b" * 64 +_RECOVERY_PROJECT_HASH = "c" * 64 +_RECOVERY_ACQUISITION_ID = "acquisition-persisted-active-k1" +_RECOVERY_START_OPERATION_ID = "physical-start-persisted-active-k1" + + +def _persist_resolved_active_start_for_restart( + service: XgridsK1CompatibilityService, +) -> None: + """Persist one successful START without retaining process-local acquisition.""" + + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + ) + connection = PhysicalCommandConnectionBinding( + intent_id="old-start-intent", + transport_ref="test-ble-transport", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=facade_module.CONTROL_MQTT_PORT, + host_path_epoch=1, + control_session_id="old-start-control", + producer_generation=1, + ) + + def status( + session_state: str, + *, + observed_at_utc: str, + ) -> PhysicalCommandStatusEvidence: + project_bound = session_state == "scanning" + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state=session_state, # type: ignore[arg-type] + session_state_code=302 if session_state == "scanning" else 300, + project_bound=project_bound, + project_id_sha256=_RECOVERY_PROJECT_HASH if project_bound else None, + init_ready=project_bound, + status_message_sha256=hashlib.sha256( + f"{session_state}:{observed_at_utc}".encode() + ).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + ledger = service._physical_command_ledger # noqa: SLF001 + ledger.prepare( + operation_id=_RECOVERY_START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=_RECOVERY_ACQUISITION_ID, + action="start", + identity=identity, + connection=connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="d" * 64, + baseline_status=status( + "ready", + observed_at_utc="2026-08-09T12:00:00.000Z", + ), + ) + ledger.mark_dispatching(_RECOVERY_START_OPERATION_ID) + ledger.mark_observing( + _RECOVERY_START_OPERATION_ID, + publish_call_returned=True, + packet_id=41, + ) + ledger.mark_qos2_completed(_RECOVERY_START_OPERATION_ID, packet_id=41) + ledger.record_application_response( + _RECOVERY_START_OPERATION_ID, + PhysicalCommandApplicationResponse( + operation_id=_RECOVERY_START_OPERATION_ID, + action="start", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="e" * 64, + observed_at_utc="2026-08-09T12:00:01.000Z", + ), + ) + ledger.record_status_observation( + _RECOVERY_START_OPERATION_ID, + status("scanning", observed_at_utc="2026-08-09T12:00:02.000Z"), + ) + ledger.resolve( + _RECOVERY_START_OPERATION_ID, + resolution="start-active-observed", + ) + + +def _persist_resolved_unclassified_stop_for_restart( + service: XgridsK1CompatibilityService, +) -> tuple[str, int]: + """Persist the exact legacy startup shape without process-local owners.""" + + _persist_resolved_active_start_for_restart(service) + ledger = service._physical_command_ledger # noqa: SLF001 + start = ledger.snapshot().record + assert start is not None + assert start.last_status is not None + stop_operation_id = "physical-stop-resolved-unclassified-k1" + ledger.prepare( + operation_id=stop_operation_id, + parent_operation_id=start.operation_id, + acquisition_id=start.acquisition_id, + action="stop", + identity=start.identity, + connection=start.connection, + compatibility_profile_id=start.compatibility_profile_id, + payload_sha256="6" * 64, + baseline_status=start.last_status, + ) + resolved = ledger.resolve( + stop_operation_id, + resolution="not-dispatched", + ) + assert resolved.resolved_unclassified_stop_recovery_required is True + return stop_operation_id, resolved.revision + + +def _persist_classified_ready_stop_for_restart( + service: XgridsK1CompatibilityService, +) -> tuple[str, PhysicalCommandConnectionBinding]: + """Persist S0 plus one read-only READY proof on an obsolete C1 binding.""" + + stop_operation_id, _ = _persist_resolved_unclassified_stop_for_restart(service) + coordinator = service._physical_command_coordinator # noqa: SLF001 + observed_at_utc = datetime.now(UTC).isoformat(timespec="milliseconds").replace( + "+00:00", + "Z", + ) + c1 = PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id="classified-ready-c1-intent", + transport_ref="test-ble-transport", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=facade_module.CONTROL_MQTT_PORT, + host_path_epoch=7, + control_session_id="classified-ready-c1-control", + producer_generation=7, + ) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:classified-ready-c1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="4" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc=observed_at_utc, + ) + ) + coordinator.bind_control_session(c1) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="5" * 64, + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + ) + classified = coordinator.reconcile_unresolved( + reconciliation_id="classified-ready-c1", + ) + assert classified["operation_id"] == stop_operation_id + assert classified["resolution"] == "not-dispatched" + return stop_operation_id, PhysicalCommandConnectionBinding( + intent_id=c1.intent_id, + transport_ref=c1.transport_ref, + connection_mode=c1.connection_mode, + target_ipv4=c1.target_ipv4, + target_port=c1.target_port, + host_path_epoch=c1.host_path_epoch, + control_session_id=c1.control_session_id, + producer_generation=c1.producer_generation, + ) + + +def test_resolved_active_power_loss_exposes_local_retirement_and_revokes_target( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + _persist_resolved_active_start_for_restart(service) + _seed_terminal_network_attempt(service) + before = service.state() + checkpoint = before["physical_command"]["operator_retirement"] + action = before["connection_policy"]["actions"]["retire-unavailable-physical-target"] + io_calls: list[str] = [] + + async def forbidden_io(*_args: object, **_kwargs: object) -> object: + io_calls.append("device-or-network-io") + raise AssertionError("resolved-active retirement must stay local-only") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_io) + monkeypatch.setattr( + service, + "_probe_control_endpoint", + lambda *_args, **_kwargs: io_calls.append("tcp-probe"), + ) + + assert before["physical_command"]["resolved_active_recovery_required"] is True + assert checkpoint["allowed"] is True + assert action["allowed"] is True + assert action["target_source"] == "durable-physical-command" + assert action["required_transport_ref"] == "test-ble-transport" + assert before["connection_policy"]["recommended_action"] == ( + "retire-unavailable-physical-target" + ) + assert before["connection_attempt"]["safe_next_action"] == ( + "retire-unavailable-physical-target" + ) + + retired = service.retire_unavailable_physical_command( + RetireUnavailablePhysicalCommandRequest( + retirement_id="retirement-resolved-active-power-loss", + expected_operation_id=checkpoint["expected_operation_id"], + expected_revision=checkpoint["expected_revision"], + expected_transport_ref=checkpoint["expected_transport_ref"], + operator_confirmed=True, + reason="device-permanently-unavailable-or-replaced", + ) + ) + + assert retired["physical_command"]["record"]["resolution"] == ( + "operator-retired-outcome-unknown" + ) + assert retired["physical_command"]["operator_retirement"]["allowed"] is False + assert retired["selected_device_id"] is None + assert retired["k1_ip"] is None + assert retired["connection_policy"]["facts"]["retired_transport_refs"] == ["test-ble-transport"] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert io_calls == [] + + +class _VerifyPhysicalRecoveryCoordinator: + """Small exact-proof coordinator double for facade Verify recovery tests.""" + + def __init__( + self, + *, + observed_session_state: str | None, + reconciliation_ready: bool, + prepared_stop_operation_id: str | None = None, + ) -> None: + self.observed_session_state = observed_session_state + self.reconciliation_ready = reconciliation_ready + self.resolution: str | None = None + self.reconciliation_ids: list[str] = [] + self.verified_control: dict[str, object] | None = None + self.prepared_stop_operation_id = prepared_stop_operation_id + + def _resolved_record(self, reconciliation_id: str) -> dict[str, object]: + assert self.verified_control is not None + connection = { + field: self.verified_control[field] + for field in ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + } + return { + "operation_id": self.prepared_stop_operation_id or "persisted-start-observing", + "acquisition_id": "acq-persisted-start", + "action": "stop" if self.prepared_stop_operation_id is not None else "start", + "stage": "resolved", + "resolution": ( + "not-dispatched" + if self.prepared_stop_operation_id is not None + else self.resolution + ), + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + "connection": { + "transport_ref": "test-ble-transport", + "connection_mode": "bridge", + }, + "reconciliations": [ + { + "reconciliation_id": reconciliation_id, + "kind": ( + "prepared-stop-classification" + if self.prepared_stop_operation_id is not None + else "ambiguous-outcome" + ), + "resolution": "physical-active-observed", + "original_attempt": { + "operation_id": ( + self.prepared_stop_operation_id or "persisted-start-observing" + ), + "action": ( + "stop" if self.prepared_stop_operation_id is not None else "start" + ), + }, + "verified_binding": {"connection": connection}, + "observation": { + "source": "explicit-read-only-reconciliation", + "control_session_id": self.verified_control["control_session_id"], + "host_path_epoch": self.verified_control["host_path_epoch"], + "producer_generation": self.verified_control["producer_generation"], + "session_state": "scanning", + "project_bound": True, + "project_id_sha256": "8" * 64, + "init_ready": True, + "mqtt_retained": False, + }, + } + ], + } + + def snapshot(self) -> dict[str, object]: + resolved_active_recovery_required = self.resolution == "physical-active-observed" + reconciliations = [{"resolution": self.resolution}] if self.resolution is not None else [] + return { + "status": "resolved" if self.resolution is not None else "unresolved", + "reason_code": ( + None if self.resolution is not None else "physical-command-reconciliation-required" + ), + "requires_reconciliation": self.resolution is None, + "resolved_active_recovery_required": resolved_active_recovery_required, + "automatic_replay_allowed": False, + "normal_session_recovery_supported": False, + "recovery_requirement": ( + "explicit-read-only-deviceinfo-and-non-retained-devicestatus" + if self.resolution is None or resolved_active_recovery_required + else None + ), + "runtime_bound": True, + "reconciliation_ready": ( + (self.resolution is None or resolved_active_recovery_required) + and self.reconciliation_ready + ), + "observed_session_state": self.observed_session_state, + "active_operation_id": None, + "record": { + "operation_id": self.prepared_stop_operation_id or "persisted-start-observing", + "action": "stop" if self.prepared_stop_operation_id is not None else "start", + "stage": "resolved" if self.resolution is not None else "observing", + "resolution": self.resolution, + "connection": { + "transport_ref": "test-ble-transport", + "connection_mode": "bridge", + }, + "reconciliations": reconciliations, + }, + } + + def reconcile_unresolved(self, *, reconciliation_id: str) -> dict[str, object]: + assert self.reconciliation_ready is True + assert self.observed_session_state in {"ready", "scanning"} + self.reconciliation_ids.append(reconciliation_id) + self.resolution = ( + "physical-standby-observed" + if self.observed_session_state == "ready" + else "physical-active-observed" + ) + return self._resolved_record(reconciliation_id) + + def reconcile_resolved_active( + self, + *, + reconciliation_id: str, + ) -> dict[str, object]: + assert self.resolution == "physical-active-observed" + assert self.reconciliation_ready is True + assert self.observed_session_state in {"ready", "scanning"} + self.reconciliation_ids.append(reconciliation_id) + if self.observed_session_state == "ready": + self.resolution = "physical-standby-observed" + return self._resolved_record(reconciliation_id) + + +def _install_synthetic_verify_recovery( + service: XgridsK1CompatibilityService, + coordinator: _VerifyPhysicalRecoveryCoordinator, +) -> None: + """Install a no-I/O topology+DeviceInfo path while keeping real gates.""" + + _seed_supervised_connection(service, connection_mode="bridge", with_control=False) + with service._lock: # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "lease_generation": service._connection_lease_generation, # noqa: SLF001 + "lease_state": "reachable", + } + service._physical_command_coordinator = coordinator # type: ignore[assignment] # noqa: SLF001 + + async def verify_topology_without_io( + bound_service: XgridsK1CompatibilityService, + request: ConnectionVerifyRequest, + ) -> tuple[str, bool, None]: + operation, created = bound_service._operations.begin( # noqa: SLF001 + facade_module.ACTION_CONNECTION_VERIFY, + operation_id=request.operation_id, + deadline_seconds=10.0, + ) + if created: + bound_service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="topology-confirmed-device-info-pending", + message_code="connection.verify.device_info_pending", + ) + return operation.operation_id, created, None + + async def bootstrap_with_process_holder( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + coordinator.verified_control = dict(verified) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + service._verify_connection_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + verify_topology_without_io, + service, + ) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_with_process_holder, + service, + ) + + +def _install_real_coordinator_bootstrap( + service: XgridsK1CompatibilityService, + *, + observed_session_state: str, +) -> None: + """Install exact DeviceInfo/status evidence on the facade's real topology path.""" + + assert observed_session_state in {"ready", "scanning"} + coordinator = service._physical_command_coordinator # noqa: SLF001 + + async def bootstrap_with_real_coordinator( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + coordinator.prepare_read_only_bootstrap() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=( + f"bootstrap:{parent_operation_id}:DeviceInfoRequest" + ), + response_topic="lixel/application/response/device_info", + payload_sha256="7" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-13T08:00:00.000Z", + ) + ) + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + scanning = observed_session_state == "scanning" + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + session_state=observed_session_state, # type: ignore[arg-type] + session_state_code=MODELING_STATE_BASE + (302 if scanning else 300), + project_bound=scanning, + project_id_sha256=_RECOVERY_PROJECT_HASH if scanning else None, + init_ready=scanning, + status_message_sha256="8" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T08:00:01.000Z", + ) + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_with_real_coordinator, + service, + ) + + +def _install_real_coordinator_verify_recovery( + service: XgridsK1CompatibilityService, + *, + observed_session_state: str, +) -> None: + """Drive public Verify with real durable reconciliation and no topology I/O.""" + + assert observed_session_state in {"ready", "scanning"} + _seed_supervised_connection(service, connection_mode="bridge", with_control=False) + with service._lock: # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "lease_generation": service._connection_lease_generation, # noqa: SLF001 + "lease_state": "reachable", + } + + async def verify_topology_without_io( + bound_service: XgridsK1CompatibilityService, + request: ConnectionVerifyRequest, + ) -> tuple[str, bool, None]: + operation, created = bound_service._operations.begin( # noqa: SLF001 + facade_module.ACTION_CONNECTION_VERIFY, + operation_id=request.operation_id, + deadline_seconds=10.0, + ) + if created: + bound_service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="topology-confirmed-device-info-pending", + message_code="connection.verify.device_info_pending", + ) + return operation.operation_id, created, None + + service._verify_connection_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + verify_topology_without_io, + service, + ) + _install_real_coordinator_bootstrap( + service, + observed_session_state=observed_session_state, + ) + + +def _retained_physical_recovery_verify_request( + *, + operation_id: str, +) -> ConnectionVerifyRequest: + """Address the exact process-retained K1 seeded by recovery fixtures.""" + + return ConnectionVerifyRequest( + device_id="test-ble-transport", + source="retained-current-process", + compatibility_attestation=ATTESTATION, + operation_id=operation_id, + ) + + +def _install_terminal_camera_error_residual( + service: XgridsK1CompatibilityService, + monkeypatch: pytest.MonkeyPatch, + *, + cleanup_failure: Exception | None = None, + producer_alive: bool = False, +) -> list[str]: + """Reproduce the inert camera/process residue left by archive failure.""" + + source_id = facade_module.DEFAULT_ACQUISITION_CAMERA_SOURCE + camera_state = dict(service.camera_preview.snapshot()) + camera_state.update( + { + "phase": "error", + "generation": 17, + "active_source_id": source_id, + "delivery": None, + "error": { + "code": "segment-too-large", + "message": "synthetic terminal camera archive failure", + }, + } + ) + camera_state["recording"] = { + "active": False, + "source_end_expected": False, + "session": None, + "active_epoch": None, + "committed_media_segment_count": None, + "producer_alive": producer_alive, + "producer_age_ms": None, + "last_segment_age_ms": None, + "preview_consumer_count": 0, + "completed_epochs": 1, + "last_summary": None, + } + camera_gate = threading.RLock() + cleanup_calls: list[str] = [] + + def snapshot() -> dict[str, Any]: + with camera_gate: + copied = dict(camera_state) + copied["recording"] = dict(camera_state["recording"]) + error = camera_state.get("error") + copied["error"] = dict(error) if isinstance(error, dict) else None + return copied + + def stop_current() -> dict[str, Any]: + cleanup_calls.append("stop-current") + if cleanup_failure is not None: + raise cleanup_failure + with camera_gate: + camera_state.update( + { + "phase": "idle", + "generation": None, + "active_source_id": None, + "delivery": None, + "error": None, + } + ) + return snapshot() + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot) + monkeypatch.setattr(service.camera_preview, "stop_current", stop_current) + assert service._ensure_camera_preview_process_lease() is True # noqa: SLF001 + with service._lock: # noqa: SLF001 + service._live_perception_camera_binding = ( # noqa: SLF001 + "camera-failed-session", + source_id, + 17, + ) + service._camera_activation_lineage = ( # noqa: SLF001 + "acq-camera-failed", + "camera-failed-session", + 0, + ) + service._camera_activation_retry_lineage = ( # noqa: SLF001 + "acq-camera-failed", + "camera-failed-session", + 0, + ) + service._camera_activation_retry_not_before_monotonic = 42.0 # noqa: SLF001 + return cleanup_calls + + +def test_explicit_verify_reconciles_persisted_start_to_ready_without_device_io( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="ready", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + operation_id = "op-00000000-0000-4000-8000-000000001201" + + verify_state = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + state = service.state() + + assert coordinator.reconciliation_ids == [f"{operation_id}.physical"] + assert state["physical_command"]["requires_reconciliation"] is False + assert state["physical_command"]["record"]["resolution"] == ("physical-standby-observed") + assert state["connection_lifecycle"]["mode_selection"]["allowed"] is True, state[ + "connection_lifecycle" + ]["mode_selection"] + assert verify_state["last_operation"]["status"] == "succeeded" + assert verify_state["last_operation"]["stage_code"] == ("physical-reconciliation-confirmed") + assert verify_state["last_operation"]["result"]["physical_reconciliation"] == { + "performed": True, + "resolution": "physical-standby-observed", + "observed_session_state": "ready", + "device_write_performed": False, + "automatic_retry": False, + } + assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + +def test_explicit_verify_reconciliation_does_not_require_normal_command_authority( + tmp_path: Path, +) -> None: + """Fresh DeviceInfo/READY resolves the old edge before command authority exists.""" + + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="ready", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + reconciliation_validations: list[str] = [] + + async def bootstrap_with_command_authority_blocked( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + control = bound_service._application_control_session # noqa: SLF001 + + def reject_normal_command_authority() -> None: + raise RuntimeError("normal command authority remains blocked") + + def accept_exact_read_only_reconciliation() -> None: + reconciliation_validations.append("read-only") + + control.validate_connection_binding = reject_normal_command_authority # type: ignore[method-assign] + control.validate_physical_reconciliation_binding = ( # type: ignore[method-assign] + accept_exact_read_only_reconciliation + ) + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_with_command_authority_blocked, + service, + ) + operation_id = "op-00000000-0000-4000-8000-000000001212" + + result = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + assert reconciliation_validations == ["read-only", "read-only"] + assert result["last_operation"]["status"] == "succeeded" + assert result["physical_command"]["record"]["resolution"] == ("physical-standby-observed") + + +def test_explicit_verify_reconciliation_rechecks_binding_before_durable_commit( + tmp_path: Path, +) -> None: + """A route/control change between proof and CAS leaves the ledger unresolved.""" + + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="ready", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + bootstrap = service._bootstrap_prestart_control_ready_owned # noqa: SLF001 + validation_count = 0 + + async def bootstrap_with_binding_change( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + control = bound_service._application_control_session # noqa: SLF001 + + def validate_until_commit() -> None: + nonlocal validation_count + validation_count += 1 + if validation_count == 2: + raise RuntimeError("control generation changed before commit") + + control.validate_physical_reconciliation_binding = ( # type: ignore[method-assign] + validate_until_commit + ) + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_with_binding_change, + service, + ) + operation_id = "op-00000000-0000-4000-8000-000000001213" + + with pytest.raises(facade_module.ConnectionVerificationError) as failure: + asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + assert failure.value.reason_code == "physical-command-reconciliation-failed" + assert validation_count == 2 + assert coordinator.reconciliation_ids == [] + physical = coordinator.snapshot() + assert physical["requires_reconciliation"] is True + assert physical["record"]["resolution"] is None + operation = service.state()["last_operation"] + assert operation["operation_id"] == operation_id + assert operation["status"] == "failed" + + +def test_explicit_verify_reconciles_scanning_as_active_without_unblocking_mode_change( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="scanning", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + + state = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request( + operation_id="op-00000000-0000-4000-8000-000000001202" + ) + ) + ) + + assert state["physical_command"]["requires_reconciliation"] is False + assert state["physical_command"]["record"]["resolution"] == ("physical-active-observed") + assert state["connection_lifecycle"]["mode_selection"]["allowed"] is False + assert ( + "connection-mode-selection-physical-state-unsafe" + in state["connection_lifecycle"]["mode_selection"]["reason_codes"] + ) + assert ( + "physical-device-already-active" + not in state["connection_policy"]["actions"]["scan-ble"]["reason_codes"] + ) + assert ( + "physical-device-already-active" + in state["connection_policy"]["actions"]["provision-fresh-device"]["reason_codes"] + ) + assert state["acquisition"]["acquisition_id"] == "acq-persisted-start" + assert state["acquisition"]["state"] == "failed" + assert state["acquisition"]["cleanup_pending"] is False + assert state["acquisition"]["requested_streams"] == [] + assert state["acquisition"]["result"]["recovery_only"] is True + assert state["acquisition"]["result"]["automatic_replay_allowed"] is False + assert state["application_control_session"]["can_stop"] is True + assert state["last_operation"]["result"]["physical_reconciliation"] == { + "performed": True, + "resolution": "physical-active-observed", + "observed_session_state": "scanning", + "device_write_performed": False, + "automatic_retry": False, + } + + +def test_explicit_verify_scanning_cleans_terminal_camera_residual_for_stop_only_shell( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="scanning", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + cleanup_calls = _install_terminal_camera_error_residual(service, monkeypatch) + operation_id = "op-00000000-0000-4000-8000-000000001214" + + state = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + control = service._application_control_session # noqa: SLF001 + assert isinstance(control, FakeInteractiveControlSession) + assert cleanup_calls == ["stop-current"] + assert coordinator.reconciliation_ids == [f"{operation_id}.physical"] + assert state["physical_command"]["record"]["resolution"] == ( + "physical-active-observed" + ) + assert state["acquisition"]["acquisition_id"] == "acq-persisted-start" + assert state["acquisition"]["state"] == "failed" + assert state["acquisition"]["result"]["recovery_only"] is True + assert state["application_control_session"]["can_stop"] is True + assert service.camera_preview.snapshot()["phase"] == "idle" + assert service._live_perception_camera_binding is None # noqa: SLF001 + assert service._camera_activation_lineage is None # noqa: SLF001 + assert service._camera_activation_retry_lineage is None # noqa: SLF001 + assert service._camera_activation_retry_not_before_monotonic == 0.0 # noqa: SLF001 + assert service._application_control_process_lease_holders == { # noqa: SLF001 + "control" + } + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_explicit_verify_scanning_camera_cleanup_failure_retains_fence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="scanning", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + cleanup_calls = _install_terminal_camera_error_residual( + service, + monkeypatch, + cleanup_failure=RuntimeError("synthetic terminal camera cleanup failed"), + ) + operation_id = "op-00000000-0000-4000-8000-000000001215" + + with pytest.raises(facade_module.ConnectionVerificationError) as failure: + asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + state = service.state() + control = service._application_control_session # noqa: SLF001 + assert isinstance(control, FakeInteractiveControlSession) + assert failure.value.reason_code == "physical-command-reconciliation-failed" + assert cleanup_calls == ["stop-current"] + assert coordinator.reconciliation_ids == [f"{operation_id}.physical"] + assert state["physical_command"]["record"]["resolution"] == ( + "physical-active-observed" + ) + assert state["acquisition"] is None + assert state["last_operation"]["operation_id"] == operation_id + assert state["last_operation"]["status"] == "failed" + assert service.camera_preview.snapshot()["phase"] == "error" + assert service._live_perception_camera_binding is not None # noqa: SLF001 + assert service._camera_activation_lineage is not None # noqa: SLF001 + assert service._camera_activation_retry_lineage is not None # noqa: SLF001 + assert service._application_control_process_lease_holders == { # noqa: SLF001 + "camera" + } + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_explicit_verify_scanning_refuses_to_cleanup_live_camera_producer( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="scanning", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + cleanup_calls = _install_terminal_camera_error_residual( + service, + monkeypatch, + producer_alive=True, + ) + operation_id = "op-00000000-0000-4000-8000-000000001216" + + with pytest.raises(facade_module.ConnectionVerificationError) as failure: + asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + control = service._application_control_session # noqa: SLF001 + assert isinstance(control, FakeInteractiveControlSession) + assert failure.value.reason_code == "physical-command-reconciliation-failed" + assert cleanup_calls == [] + assert service._acquisition is None # noqa: SLF001 + assert service.camera_preview.snapshot()["recording"]["producer_alive"] is True + assert service._application_control_process_lease_holders == { # noqa: SLF001 + "camera" + } + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + + +def test_fresh_service_real_verify_rotation_rehydrates_one_stop_only_target( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + original, _ = service_with_fake_runtime(tmp_path) + _persist_resolved_active_start_for_restart(original) + assert original._acquisition is None # noqa: SLF001 + + # A genuinely fresh facade starts with no process-local acquisition. Its + # only physical authority is the persisted resolved-active START record. + restarted, _ = service_with_fake_runtime(tmp_path) + assert restarted._acquisition is None # noqa: SLF001 + assert restarted.state()["physical_command"]["resolved_active_recovery_required"] is True + _set_scanned_k1(restarted, device_id="test-ble-transport") + + async def read_status( + device_id: str, + *, + timeout_seconds: float, + rediscover: bool, + **_: object, + ) -> dict[str, Any]: + assert (device_id, timeout_seconds, rediscover) == ( + "test-ble-transport", + 20.0, + False, + ) + return _wifi_status_read("10.255.254.77", device_id=device_id) + + monkeypatch.setattr(facade_module, "read_wifi_status_once", read_status) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr(facade_module, "_host_route_class", lambda _target: "direct-or-routed") + monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True) + monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path) + + coordinator = restarted._physical_command_coordinator # noqa: SLF001 + published_physical_edges: list[str] = [] + + async def bootstrap_recovery_control( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + control = bound_service._application_control_session # noqa: SLF001 + verified = control.snapshot().get("verified_control") + assert isinstance(verified, dict) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="f" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-09T12:10:00.000Z", + ) + ) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256=_RECOVERY_PROJECT_HASH, + init_ready=True, + status_message_sha256="1" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-09T12:10:01.000Z", + ) + ) + + original_request_stop = control.request_stop + + def request_stop_with_durable_edge( + *, + confirmation: object, + command_context: object, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert dispatch_admission_deadline_reached is not None + payload = b"recovered-stop" + coordinator.prepare( + command_context, # type: ignore[arg-type] + action="stop", + envelope=OneShotPublishEnvelope( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload=payload, + payload_sha256=hashlib.sha256(payload).hexdigest(), + payload_bytes=len(payload), + qos=2, + retain=False, + ), + ) + published_physical_edges.append("stop") + return original_request_stop( + confirmation=confirmation, + command_context=command_context, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + control.request_stop = request_stop_with_durable_edge # type: ignore[method-assign] + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + restarted._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_recovery_control, + restarted, + ) + verify_operation_id = "op-00000000-0000-4000-8000-000000001230" + verified = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="test-ble-transport", + compatibility_attestation=ATTESTATION, + operation_id=verify_operation_id, + expected_discovery_generation=0, + ) + ) + ) + + acquisition = verified["acquisition"] + assert acquisition["acquisition_id"] == _RECOVERY_ACQUISITION_ID + assert acquisition["device_session_id"] == verified["device_session"]["device_session_id"] + assert acquisition["state"] == "failed" + assert acquisition["cleanup_pending"] is False + assert acquisition["requested_streams"] == [] + assert acquisition["project_name"] is None + assert acquisition["result"] == { + "receiver_stopped": True, + "device_state": "scanning", + "recovery_only": True, + "physical_command_operation_id": _RECOVERY_START_OPERATION_ID, + "physical_reconciliation_id": f"{verify_operation_id}.physical", + "project_id_sha256": _RECOVERY_PROJECT_HASH, + "automatic_replay_allowed": False, + } + assert verified["application_control_session"]["state"] == "scanning" + assert verified["application_control_session"]["can_stop"] is True + assert published_physical_edges == [] + + control = restarted._application_control_session # noqa: SLF001 + control_snapshot = control.snapshot() + with pytest.raises(RuntimeError, match="ожидать сохранения проекта"): + restarted.prepare_acquisition( + _prepare_request( + project_name="must-not-prepare", + compatibility_attestation=ATTESTATION, + expected_control_session_generation=control_snapshot["session_generation"], + expected_control_state_revision=control_snapshot["state_revision"], + ) + ) + with pytest.raises(RuntimeError, match="ещё не готов принять START"): + restarted.start_acquisition( + _start_request( + acquisition_id=_RECOVERY_ACQUISITION_ID, + expected_state_revision=acquisition["state_revision"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=control_snapshot["session_generation"], + expected_control_state_revision=control_snapshot["state_revision"], + ) + ) + assert published_physical_edges == [] + + stop_operation_id = "op-00000000-0000-4000-8000-000000001231" + stop_request = _stop_request( + acquisition_id=_RECOVERY_ACQUISITION_ID, + operation_id=stop_operation_id, + idempotency_key="recovered-stop-once", + mode="graceful", + expected_control_session_generation=control_snapshot["session_generation"], + expected_control_state_revision=control_snapshot["state_revision"], + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + stopped = restarted.stop_acquisition(stop_request) + assert stopped["acquisition"]["result"]["recovery_only"] is True + assert published_physical_edges == ["stop"] + repeated = restarted.stop_acquisition(stop_request) + assert repeated["last_operation"]["operation_id"] == stop_operation_id + assert published_physical_edges == ["stop"] + + payload_hash = hashlib.sha256(b"recovered-stop").hexdigest() + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload_sha256=payload_hash, + qos=2, + retain=False, + packet_id=None, + ) + coordinator.publish_dispatching(dispatch) + returned = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload_sha256=payload_hash, + qos=2, + retain=False, + packet_id=42, + ) + coordinator.publish_result(returned, publish_call_returned=True) + coordinator.qos2_completed(returned) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="modeling:stop", + response_topic="lixel/application/response/modeling", + payload_sha256="2" * 64, + modeling_action="stop", + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + observed_at_utc="2026-08-09T12:10:02.000Z", + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="3" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-09T12:10:03.000Z", + ) + ) + coordinator.resolve("stop") + control.state = "completed" + ready = restarted.state() + + assert ready["physical_command"]["record"]["resolution"] == "stop-standby-observed" + assert ready["physical_command"]["resolved_active_recovery_required"] is False + assert ready["connection_lifecycle"]["mode_selection"]["allowed"] is True + terminal_stop = next( + operation + for operation in ready["operations"] + if operation["operation_id"] == stop_operation_id + ) + assert terminal_stop["status"] == "succeeded" + assert published_physical_edges == ["stop"] + + +def test_active_reconciliation_survives_post_commit_adoption_failure_and_next_ready_verify( + tmp_path: Path, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state="scanning", + reconciliation_ready=True, + ) + _install_synthetic_verify_recovery(service, coordinator) + bootstrap_attempts = 0 + + async def bootstrap_with_first_adoption_failure( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + nonlocal bootstrap_attempts + bootstrap_attempts += 1 + await _synthetic_prestart_control_bootstrap( + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + if bootstrap_attempts == 1: + control = bound_service._application_control_session # noqa: SLF001 + + def fail_after_durable_commit(*_args: object, **_kwargs: object) -> None: + raise RuntimeError("synthetic adoption loss after ledger commit") + + control.adopt_reconciled_scanning = fail_after_durable_commit # type: ignore[method-assign] + + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_with_first_adoption_failure, + service, + ) + first_operation = "op-00000000-0000-4000-8000-000000001210" + + with pytest.raises(facade_module.ConnectionVerificationError): + asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=first_operation) + ) + ) + + after_failure = service.state() + assert coordinator.resolution == "physical-active-observed" + assert after_failure["physical_command"]["resolved_active_recovery_required"] is True + assert after_failure["application_control_session"]["state"] == "idle" + mutation_reasons = after_failure["connection_policy"]["actions"][ + "provision-fresh-device" + ]["reason_codes"] + assert "physical-command-reconciliation-required" in mutation_reasons + assert "physical-device-already-active" in mutation_reasons + assert after_failure["connection_policy"]["actions"]["verify-control-device-info"][ + "allowed" + ] is True + + coordinator.observed_session_state = "ready" + second_operation = "op-00000000-0000-4000-8000-000000001211" + verify_result = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=second_operation) + ) + ) + recovered = service.state() + + assert coordinator.reconciliation_ids == [ + f"{first_operation}.physical", + f"{second_operation}.physical", + ] + assert coordinator.resolution == "physical-standby-observed" + assert verify_result["last_operation"]["status"] == "succeeded" + assert recovered["physical_command"]["resolved_active_recovery_required"] is False + assert recovered["physical_command"]["requires_reconciliation"] is False + assert recovered["connection_lifecycle"]["mode_selection"]["allowed"] is True, recovered[ + "connection_lifecycle" + ]["mode_selection"] + + +@pytest.mark.parametrize( + ("proof_kind", "observed_session_state"), + [ + ("missing-status", None), + ("retained-status", "ready"), + ("stale-device-info", None), + ], +) +def test_explicit_verify_fails_closed_without_fresh_physical_recovery_proof( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + proof_kind: str, + observed_session_state: str | None, +) -> None: + del proof_kind + service, _ = service_with_fake_runtime(tmp_path) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state=observed_session_state, + reconciliation_ready=False, + ) + _install_synthetic_verify_recovery(service, coordinator) + monkeypatch.setattr( + facade_module, + "PHYSICAL_RECONCILIATION_PROOF_TIMEOUT_SECONDS", + 0.0, + ) + operation_id = "op-00000000-0000-4000-8000-000000001203" + + with pytest.raises(facade_module.ConnectionVerificationError) as raised: + asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=operation_id) + ) + ) + + assert raised.value.reason_code == "physical-command-reconciliation-proof-timeout" + assert coordinator.reconciliation_ids == [] + state = service.state() + operation = next(item for item in state["operations"] if item["operation_id"] == operation_id) + assert operation["status"] == "failed" + assert operation["stage_code"] == "physical-reconciliation-failed" + assert operation["error"]["code"] == ("physical-command-reconciliation-proof-timeout") + assert operation["error"]["safe_to_retry"] is True + assert operation["error"]["side_effect_status"] == "none" + assert state["application_control_session"]["state"] == "idle" + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert state["connection_policy"]["actions"]["verify-control-device-info"]["allowed"] is True + + +def test_bootstrap_identity_rejection_retires_only_session_opened_by_attempt( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Rejected DeviceInfo cannot leak a ready socket or process lease.""" + + service, _ = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession(initial_state="idle") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="k1-a", + with_control=False, + ) + physical_edges: list[str] = [] + + def reject_identity(**_: object) -> None: + raise DeviceIdentityPinStoreCorrupt("synthetic DeviceInfo pin rejection") + + monkeypatch.setattr(service, "_pin_or_match_device_identity", reject_identity) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: physical_edges.append("physical-command"), + ) + + with pytest.raises(facade_module.ConnectionVerificationError) as rejected: + asyncio.run( + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned( # noqa: SLF001 + service, + parent_operation_id=None, + connection_mode="bridge", + ) + ) + + assert rejected.value.reason_code == "control-bootstrap-device-identity-unverified" + assert physical_edges == [] + state = service.state() + assert state["application_control_session"]["state"] == "idle" + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert state["active_connection_mode"] is None + assert state["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert state["connection_policy"]["actions"]["verify-control-device-info"]["allowed"] is True + + +@pytest.mark.parametrize("inspection_only", [True, False], ids=["verify", "ordinary"]) +def test_read_only_bootstrap_detaches_physical_observer_before_fresh_control_open( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + inspection_only: bool, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + control = FakeInteractiveControlSession(initial_state="idle") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="k1-a", + with_control=False, + ) + ordering: list[str] = [] + + def detach_before_read_only_open() -> None: + ordering.append("physical-observer-detached") + + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare_read_only_bootstrap", + detach_before_read_only_open, + ) + original_open = control.open + + def open_after_detach( + *, + connection_binding: ApplicationConnectionBinding, + **kwargs: object, + ) -> dict[str, object]: + if inspection_only: + assert ordering == ["physical-observer-detached"] + else: + assert ordering == [] + ordering.append("fresh-control-observer-opened") + return original_open(connection_binding=connection_binding, **kwargs) + + monkeypatch.setattr(control, "open", open_after_detach) + + asyncio.run( + XgridsK1CompatibilityService._bootstrap_prestart_control_ready_owned( # noqa: SLF001 + service, + parent_operation_id=None, + connection_mode="bridge", + inspection_only=inspection_only, + ) + ) + + assert ordering == ( + ["physical-observer-detached", "fresh-control-observer-opened"] + if inspection_only + else ["fresh-control-observer-opened"] + ) + assert control.state == "connection-ready" + + +def test_polling_self_cleans_prepared_acquisition_after_proven_prestart_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A battery/control loss before START recovers without refresh or ABORT.""" + + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + + class TerminalPrestartControl(FakeInteractiveControlSession): + failure: dict[str, object] | None = None + + def snapshot(self) -> dict[str, object]: + snapshot = super().snapshot() + snapshot["failure"] = self.failure + return snapshot + + control = TerminalPrestartControl() + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="TEST001", + host=facade_module.AP_FALLBACK_IPV4, + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + ) + ) + assert prepared["acquisition"]["state"] == "prepared" + service._acquire_application_control_process_lease() # noqa: SLF001 + physical_or_network_edges: list[str] = [] + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: physical_or_network_edges.append("physical-command"), + ) + + async def forbidden_device_write(*_: object, **__: object) -> dict[str, Any]: + physical_or_network_edges.append("network-write") + raise AssertionError("polling recovery must remain read/local-only") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_write) + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "modeling_command_attempted": False, + "safe_to_retry": True, + "network_change_admissible": True, + } + recovered = service.state() + + assert physical_or_network_edges == [] + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert recovered["acquisition"]["state"] == "aborted" + assert recovered["acquisition"]["result"] == { + "receiver_started": False, + "device_command_attempted": False, + "reason_code": "prestart-control-terminal-failure", + } + assert recovered["application_control_session"]["state"] == "idle" + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + assert recovered["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + +@pytest.mark.parametrize("observed_session_state", ["ready", "scanning"]) +def test_verify_classification_settles_original_prepared_stop_as_not_dispatched( + tmp_path: Path, + observed_session_state: str, +) -> None: + service, _ = service_with_fake_runtime(tmp_path) + original_stop_operation_id = "op-00000000-0000-4000-8000-000000001409" + operation, created = service._operations.begin( # noqa: SLF001 + facade_module.ACTION_ACQUISITION_STOP, + operation_id=original_stop_operation_id, + device_id="known-k1", + device_session_id="old-stop-session", + deadline_seconds=60.0, + idempotency_key="old-prepared-stop", + ) + assert created is True + service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="awaiting-external-stop", + message_code="acquisition.stop.device_stopping", + ) + coordinator = _VerifyPhysicalRecoveryCoordinator( + observed_session_state=observed_session_state, + reconciliation_ready=True, + prepared_stop_operation_id=original_stop_operation_id, + ) + _install_synthetic_verify_recovery(service, coordinator) + verify_operation_id = "op-00000000-0000-4000-8000-000000001410" + + verified = asyncio.run( + service.verify_connection( + _retained_physical_recovery_verify_request(operation_id=verify_operation_id) + ) + ) + + original = service._operations.get(original_stop_operation_id) # noqa: SLF001 + assert original.status == "failed" + assert original.stage_code == "physical-stop-classified-not-dispatched" + assert original.error == { + "category": "device", + "code": "physical-stop-not-dispatched", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "none", + "physical_command_sent": False, + "automatic_replay_allowed": False, + } + assert coordinator._resolved_record(f"{verify_operation_id}.physical")["resolution"] == ( # noqa: SLF001 + "not-dispatched" + ) + if observed_session_state == "scanning": + assert verified["acquisition"]["result"]["physical_command_operation_id"] == ( + original_stop_operation_id + ) + assert verified["application_control_session"]["can_stop"] is True + else: + assert verified["application_control_session"]["can_stop"] is False + acquisition = verified.get("acquisition") + assert not ( + isinstance(acquisition, dict) + and isinstance(acquisition.get("result"), dict) + and acquisition["result"].get("recovery_only") is True + ) + + +@pytest.mark.parametrize("observed_session_state", ["ready", "scanning"]) +def test_public_verify_reconciles_fresh_service_resolved_unclassified_stop_without_writes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + observed_session_state: str, +) -> None: + original, _ = service_with_fake_runtime(tmp_path) + stop_operation_id, compatibility_revision = ( + _persist_resolved_unclassified_stop_for_restart(original) + ) + + restarted, runtime = service_with_fake_runtime(tmp_path) + before = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert before is not None + assert before.operation_id == stop_operation_id + assert before.revision == compatibility_revision + assert before.stage == "resolved" + assert before.resolution == "not-dispatched" + assert before.resolved_unclassified_stop_recovery_required is True + assert restarted._acquisition is None # noqa: SLF001 + + forbidden_io: list[str] = [] + + async def forbid_device_io(*_args: object, **_kwargs: object) -> object: + forbidden_io.append("device-io") + raise AssertionError("resolved STOP Verify must stay read-only") + + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbid_device_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbid_device_io) + _install_real_coordinator_verify_recovery( + restarted, + observed_session_state=observed_session_state, + ) + verify_operation_id = "op-00000000-0000-4000-8000-000000001411" + + verified = asyncio.run( + restarted.verify_connection( + _retained_physical_recovery_verify_request( + operation_id=verify_operation_id, + ) + ) + ) + + after = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert after is not None + assert after.operation_id == stop_operation_id + assert after.revision == compatibility_revision + 1 + assert after.stage == "resolved" + assert after.resolution == "not-dispatched" + assert after.resolved_unclassified_stop_recovery_required is False + classification = after.reconciliations[-1] + assert classification.kind == "prepared-stop-classification" + assert classification.original_attempt.stage == "resolved" + assert classification.original_attempt.resolution == "not-dispatched" + assert classification.resolution == ( + "physical-active-observed" + if observed_session_state == "scanning" + else "physical-standby-observed" + ) + assert verified["last_operation"]["operation_id"] == verify_operation_id + assert verified["last_operation"]["status"] == "succeeded" + assert verified["last_operation"]["result"]["write_performed"] is False + assert verified["last_operation"]["result"]["physical_reconciliation"] == { + "performed": True, + "resolution": classification.resolution, + "observed_session_state": observed_session_state, + "device_write_performed": False, + "automatic_retry": False, + } + control = restarted._application_control_session # noqa: SLF001 + assert isinstance(control, FakeInteractiveControlSession) + assert control.start_projects == [] + assert control.stop_calls == 0 + assert runtime.start_calls == [] + assert runtime.stop_calls == 0 + assert forbidden_io == [] + + if observed_session_state == "ready": + assert verified.get("acquisition") is None + assert verified["application_control_session"]["state"] == "connection-ready" + assert verified["application_control_session"]["can_stop"] is False + assert verified["connection_policy"]["actions"]["start-acquisition"][ + "allowed" + ] is True + assert "physical-command-reconciliation-required" not in verified[ + "connection_policy" + ]["actions"]["start-acquisition"]["reason_codes"] + assert verified["connection_lifecycle"]["mode_selection"]["allowed"] is False + with pytest.raises(facade_module.NetworkProvisioningConflict) as mutation: + restarted._require_physical_command_network_mutation_allowed() # noqa: SLF001 + assert mutation.value.reason_code == "physical-command-reconciliation-required" + else: + assert verified["application_control_session"]["state"] == "scanning" + assert verified["application_control_session"]["can_stop"] is True + assert verified["acquisition"]["result"]["recovery_only"] is True + assert verified["acquisition"]["result"]["physical_command_operation_id"] == ( + stop_operation_id + ) + assert verified["connection_policy"]["actions"]["start-acquisition"][ + "allowed" + ] is False + assert verified["connection_policy"]["actions"]["stop-acquisition"][ + "allowed" + ] is True + + +def test_restart_classified_ready_uses_fresh_live_binding_for_one_start( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """C1 proves historical standby; fresh DeviceInfo + live READY on C2 owns START.""" + + original, _ = service_with_fake_runtime(tmp_path) + stop_operation_id, c1 = _persist_classified_ready_stop_for_restart(original) + classified = original._physical_command_ledger.snapshot().record # noqa: SLF001 + assert classified is not None + assert classified.operation_id == stop_operation_id + assert classified.reconciliations[-1].verified_binding.connection == c1 + + restarted, runtime = service_with_fake_runtime(tmp_path) + binding = _seed_supervised_connection( + restarted, + target_ipv4="192.168.68.52", + connection_mode="bridge", + transport_ref="test-ble-transport", + logical_device_id="known-k1", + ) + stable_path = _direct_host_path(binding.target_ipv4) + monkeypatch.setattr( + restarted, + "_sample_host_path", + lambda *_args, **_kwargs: stable_path, + ) + control = _BindingValidatingControlSession(restarted, binding) + control.open(connection_binding=binding) + control.verified_control = _verified_control_for_binding( + binding, + logical_device_id="known-k1", + control_session_id="fresh-live-c2-control", + control_proof_revision=2, + ) + restarted._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + restarted._reconcile_connection_supervisor( # noqa: SLF001 + control.snapshot(), + runtime.snapshot(), + ) + restarted._acquire_application_control_process_lease() # noqa: SLF001 + + coordinator = restarted._physical_command_coordinator # noqa: SLF001 + observed_at_utc = datetime.now(UTC).isoformat(timespec="milliseconds").replace( + "+00:00", + "Z", + ) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:fresh-live-c2:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="6" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc=observed_at_utc, + ) + ) + verified = control.verified_control + assert verified is not None + c2 = PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], # type: ignore[arg-type] + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + assert c2.control_session_id != c1.control_session_id + coordinator.bind_control_session(c2) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=_RECOVERY_VENDOR_HASH, + device_serial_sha256=_RECOVERY_SERIAL_HASH, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="7" * 64, + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + ) + + inherited_reconciliations = classified.reconciliations + ready = restarted.state() + workspace = restarted.enter_application_workspace( + EnterApplicationWorkspaceRequest( + operator_confirmed=True, + expected_session_generation=ready["application_control_session"][ + "session_generation" + ], + expected_state_revision=ready["application_control_session"]["state_revision"], + ) + ) + prepared = restarted.prepare_acquisition( + _prepare_request( + project_name="RESTART_READY_C2", + host=binding.target_ipv4, + compatibility_attestation=ATTESTATION, + expected_control_session_generation=workspace["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=workspace["application_control_session"][ + "state_revision" + ], + ) + ) + before_start = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert before_start == classified + assert before_start.reconciliations == inherited_reconciliations + + prepared_edges: list[str] = [] + original_request_start = control.request_start + + def request_start_with_durable_edge( + *, + project_name: str, + confirmation: object, + command_context: object, + preparation_checkpoint_observer: object | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + payload = b"fresh-start-on-c2" + coordinator.prepare( + command_context, # type: ignore[arg-type] + action="start", + envelope=OneShotPublishEnvelope( + operation_key="modeling:start", + topic="lixel/application/request/modeling", + payload=payload, + payload_sha256=hashlib.sha256(payload).hexdigest(), + payload_bytes=len(payload), + qos=2, + retain=False, + ), + ) + prepared_edges.append("start") + return original_request_start( + project_name=project_name, + confirmation=confirmation, + command_context=command_context, + preparation_checkpoint_observer=preparation_checkpoint_observer, + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + control.request_start = request_start_with_durable_edge # type: ignore[method-assign] + start_operation_id = "op-00000000-0000-4000-8000-000000001412" + started = restarted.start_acquisition( + _start_request( + acquisition_id=prepared["acquisition"]["acquisition_id"], + operation_id=start_operation_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + expected_control_session_generation=prepared["application_control_session"][ + "session_generation" + ], + expected_control_state_revision=prepared["application_control_session"][ + "state_revision" + ], + ) + ) + + successor = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert successor is not None + assert successor.operation_id == start_operation_id + assert successor.parent_operation_id == stop_operation_id + assert successor.action == "start" + assert successor.stage == "prepared" + assert successor.connection.control_session_id == c2.control_session_id + assert successor.connection.host_path_epoch == c2.host_path_epoch + assert successor.connection.producer_generation == c2.producer_generation + assert successor.baseline_status.control_session_id == c2.control_session_id + assert successor.reconciliations == inherited_reconciliations + assert prepared_edges == ["start"] + assert control.start_projects == ["RESTART_READY_C2"] + assert len(runtime.start_calls) == 1 + assert started["acquisition"]["state"] == "starting" + + +def test_recovered_terminal_stop_losing_wifi_stays_acknowledged_unconfirmed_and_never_replays( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service, runtime = service_with_fake_runtime(tmp_path) + + class TerminalStopControl(FakeInteractiveControlSession): + failure: dict[str, object] | None = None + + def snapshot(self) -> dict[str, object]: + snapshot = super().snapshot() + snapshot["failure"] = self.failure + return snapshot + + control = TerminalStopControl(initial_state="workspace-ready") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + connection_mode="bridge", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + prepared = service.prepare_acquisition( + _prepare_request( + project_name="RECOVERED_STOP", + host="192.168.1.20", + compatibility_attestation=ATTESTATION, + ) + ) + acquisition_id = prepared["acquisition"]["acquisition_id"] + service.start_acquisition( + _start_request( + acquisition_id=acquisition_id, + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + ) + start_operation_id = service._acquisition_start_operation_id # noqa: SLF001 + assert isinstance(start_operation_id, str) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: _exact_start_physical_proof( + operation_id=start_operation_id, + acquisition_id=acquisition_id, + binding=binding, + resolved=True, + ), + ) + runtime.mark_ready() + runtime.pcl_frames = 1 + acquiring = service.state() + assert acquiring["acquisition"]["state"] == "acquiring" + + # Model the already-sealed local half after power loss. A later read-only + # Verify has adopted SCANNING, so the terminal acquisition exposes one + # explicit canonical STOP without restarting the receiver or capture. + service._stop_acquisition_sources( # noqa: SLF001 + camera_status="failed", + camera_failure_code="power-loss-before-recovery-stop", + ) + with service._lock: # noqa: SLF001 + assert service._acquisition is not None # noqa: SLF001 + service._acquisition.transition( # noqa: SLF001 + "failed", + message_code="acquisition.connection_lost", + result={"receiver_stopped": True, "device_state": "active-unconfirmed"}, + ) + control.state = "scanning" + stop_operation_id = "op-00000000-0000-4000-8000-000000001220" + stop_request = _stop_request( + acquisition_id=acquisition_id, + operation_id=stop_operation_id, + mode="graceful", + physical_acceptance=PHYSICAL_ACCEPTANCE, + ) + + issued = service.stop_acquisition(stop_request) + + assert issued["acquisition"]["state"] == "failed" + assert control.stop_calls == 1 + stop_operation = next( + operation + for operation in issued["operations"] + if operation["operation_id"] == stop_operation_id + ) + assert stop_operation["status"] == "running" + + control.state = "failed" + control.failure = { + "reason_code": "mqtt_network_loop_failed", + "safe_to_retry": False, + "modeling_command_attempted": True, + "network_change_admissible": True, + "network_change_reconciliation": { + "device_session_state": "scan_stopping", + "device_project_bound": True, + "system_error_code": None, + "stop_complete": True, + "standby_confirmed": False, + "automatic_retry": False, + }, + } + physical_proof = { + "status": "unresolved", + "reason_code": "physical-command-reconciliation-required", + "requires_reconciliation": True, + "resolved_active_recovery_required": False, + "record": { + "operation_id": stop_operation_id, + "action": "stop", + "stage": "observing", + "resolution": None, + "application_response": {"action": "stop", "success": True}, + "last_status": { + "session_state": "scan_stopping", + "mqtt_retained": False, + "project_bound": True, + "system_error_code": None, + }, + "connection": { + "transport_ref": "k1-a", + "connection_mode": "bridge", + }, + "reconciliations": [], + }, + } + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: physical_proof, + ) + + recovered = service.state() + terminal_stop = next( + operation + for operation in recovered["operations"] + if operation["operation_id"] == stop_operation_id + ) + + assert terminal_stop["status"] == "failed" + assert terminal_stop["stage_code"] == "stop-acknowledged-standby-unconfirmed" + assert terminal_stop["error"] == { + "category": "device", + "code": "stop-acknowledged-standby-unconfirmed", + "retryable": False, + "safe_to_retry": False, + "side_effect_status": "unknown", + "automatic_replay_allowed": False, + } + assert recovered["application_control_session"]["state"] == "idle" + assert recovered["source_mode"] == "idle" + assert recovered["acquisition"]["cleanup_pending"] is False + assert control.stop_calls == 1 + + repeated = service.stop_acquisition(stop_request) + repeated_stop = next( + operation + for operation in repeated["operations"] + if operation["operation_id"] == stop_operation_id + ) + assert repeated_stop["stage_code"] == "stop-acknowledged-standby-unconfirmed" + assert control.stop_calls == 1 + + +def test_mode_select_winner_fences_concurrent_prepare_before_stale_record( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """PREPARE and Mode Select share one winner before local publication.""" + + service, runtime = service_with_fake_runtime(tmp_path) + _select_connection_mode(service, "quick-connect") + control = FakeInteractiveControlSession(initial_state="connection-ready") + service._application_control_session = control # type: ignore[assignment] # noqa: SLF001 + binding = _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + ) + control.verified_control = _verified_control_for_binding(binding) + selection_entered = threading.Event() + release_selection = threading.Event() + selection_results: list[dict[str, Any]] = [] + selection_failures: list[BaseException] = [] + original_runtime_snapshot = runtime.snapshot + snapshot_pause_used = False + snapshot_pause_lock = threading.Lock() + + def paused_selection_runtime_snapshot() -> dict[str, object]: + nonlocal snapshot_pause_used + should_pause = False + if threading.current_thread().name == "mode-select-winner": + with snapshot_pause_lock: + if not snapshot_pause_used: + snapshot_pause_used = True + should_pause = True + if should_pause: + selection_entered.set() + assert release_selection.wait(2.0) + return original_runtime_snapshot() + + monkeypatch.setattr(runtime, "snapshot", paused_selection_runtime_snapshot) + + def run_selection() -> None: + try: + selection_results.append( + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="bridge", + expected_revision=1, + ) + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + selection_failures.append(exc) + + selection_thread = threading.Thread( + target=run_selection, + name="mode-select-winner", + daemon=True, + ) + selection_thread.start() + assert selection_entered.wait(2.0) + try: + with pytest.raises(facade_module.ApplicationControlProcessLeaseUnavailable): + service.prepare_acquisition( + _prepare_request( + project_name="STALE01", + host=facade_module.AP_FALLBACK_IPV4, + compatibility_attestation=QUICK_CONNECT_ATTESTATION, + ) + ) + finally: + release_selection.set() + selection_thread.join(timeout=3.0) + + assert selection_thread.is_alive() is False + assert selection_failures == [] + assert selection_results[0]["desired_connection_mode"] == "bridge" + assert selection_results[0]["desired_connection_mode_revision"] == 2 + assert service._acquisition is None # noqa: SLF001 + assert control.state == "connection-ready" + device_edges: list[str] = [] + + async def fake_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert service._acquisition is None # noqa: SLF001 + assert control.state == "idle" + assert on_admitted is not None + on_admitted() + return _ble_scan_result("bridge-candidate") + + async def forbidden_network_write(*_: object, **__: object) -> dict[str, Any]: + device_edges.append("network-write") + raise AssertionError("losing PREPARE must not create a device edge") + + monkeypatch.setattr(facade_module, "scan", fake_scan) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_network_write) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "prepare", + lambda *_args, **_kwargs: device_edges.append("physical-command"), + ) + + scanned = asyncio.run(service.scan_ble(1.0)) + + assert device_edges == [] + assert scanned["acquisition"] is None + assert scanned["desired_connection_mode"] == "bridge" + assert [item["device_id"] for item in scanned["devices"]] == ["bridge-candidate"] + + +@pytest.mark.parametrize( + ("network_stage", "expected_resolution", "expected_terminal_records"), + [ + ("prepared", "not-dispatched", 0), + ("dispatching", "interrupted", 1), + ("observing", "interrupted", 1), + ], + ids=["prepared", "dispatching", "observing"], +) +def test_restart_after_any_open_network_stage_starts_with_a_clean_explicit_flow( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + network_stage: str, + expected_resolution: str, + expected_terminal_records: int, +) -> None: + """CONN-08/11: restart terminalizes audit without restoring live authority.""" + + first, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + first, + target_ipv4="192.168.68.50", + connection_mode="bridge", + transport_ref="k1-a", + logical_device_id="device-a", + ) + _set_scanned_k1(first, device_id="k1-a") + _seed_legacy_network_mutation_without_idempotency( + first, + operation_id=f"restart-{network_stage}-operation", + stage=network_stage, + ) + assert first.state()["selected_device_id"] == "k1-a" + + # A process restart loses every CoreBluetooth object even when the same + # macOS UUID is later advertised again. Construction/recovery is durable + # bookkeeping only and may not discover, read or write the device. + _SYNTHETIC_SCAN_CAPTURES.clear() + implicit_device_edges: list[str] = [] + + async def forbidden_scan(*_: object, **__: object) -> dict[str, Any]: + implicit_device_edges.append("scan") + raise AssertionError("restart recovery must not scan automatically") + + async def forbidden_read(*_: object, **__: object) -> dict[str, Any]: + implicit_device_edges.append("read") + raise AssertionError("restart recovery must not open GATT automatically") + + async def forbidden_write(*_: object, **__: object) -> dict[str, Any]: + implicit_device_edges.append("write") + raise AssertionError("restart recovery must not replay a network write") + + monkeypatch.setattr(facade_module, "scan", forbidden_scan) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_read) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + + restarted, _ = service_with_fake_runtime(tmp_path) + state = restarted.state() + + assert implicit_device_edges == [] + assert state["devices"] == [] + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["current_device_recovery"] is None + assert state["configured_connection_mode"] is None + assert state["active_connection_mode"] is None + assert state["connection_lifecycle"]["active_binding"] is None + assert state["connection_lifecycle"]["connection_ready"] is False + assert state["network_write_reconciliation"] is None + assert state["network_mutation_ledger"]["status"] == "resolved" + assert state["network_mutation_ledger"]["resolution"] == expected_resolution + assert state["network_provisioning_idempotency"]["active_operation_id"] is None + assert state["network_provisioning_idempotency"]["terminal_record_count"] == ( + expected_terminal_records + ) + assert state["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert state["connection_lifecycle"]["automatic_retry"] is False + + # A later advertisement is only a fresh candidate. It does not resurrect + # the old binding, but it does make a new explicit Connect admissible. + _set_scanned_k1(restarted, device_id="k1-a") + fresh = restarted.state() + assert fresh["selected_device_id"] is None + assert fresh["active_connection_mode"] is None + assert fresh["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + assert implicit_device_edges == [] + + +def test_power_loss_before_network_prepare_clears_selection_without_write_or_retry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """CONN-10: a pre-PREPARED disconnect is a clean, retryable no-write end.""" + + service, _ = service_with_fake_runtime(tmp_path) + _seed_supervised_connection( + service, + target_ipv4=facade_module.AP_FALLBACK_IPV4, + connection_mode="quick-connect", + transport_ref="k1-a", + logical_device_id="device-a", + ) + _set_scanned_k1(service, device_id="k1-a") + provisioning_calls = 0 + + async def powered_off_before_prepare( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal provisioning_calls + provisioning_calls += 1 + assert on_write_dispatch is not None + error = RuntimeError("K1 powered off before durable prepare") + error.operation_stage = "gatt-connect" # type: ignore[attr-defined] + error.device_write_attempted = False # type: ignore[attr-defined] + error.device_write_confirmed = False # type: ignore[attr-defined] + raise error + + monkeypatch.setattr(facade_module, "provision_wifi_once", powered_off_before_prepare) + + with pytest.raises(RuntimeError, match="powered off before durable prepare"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=ATTESTATION, + ) + ) + ) + + state = service.state() + operation = next(item for item in state["operations"] if item["action"] == "network.provision") + assert provisioning_calls == 1 + assert state["selected_device_id"] is None + assert state["device_session"] is None + assert state["configured_connection_mode"] is None + assert state["active_connection_mode"] is None + assert state["network_mutation_ledger"]["status"] == "empty" + assert state["network_provisioning_idempotency"]["active_operation_id"] is None + assert operation["status"] == "failed" + assert operation["error"]["side_effect_status"] == "none" + assert operation["error"]["safe_to_retry"] is True + assert operation["context"]["automatic_retry"] is False + assert state["connection_attempt"]["phase"] == "network_not_applied" + assert state["connection_attempt"]["side_effect_status"] == "none" + assert state["connection_attempt"]["automatic_retry"] is False + assert state["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + _set_scanned_k1(service, device_id="k1-a") + assert ( + service.state()["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + ) + assert provisioning_calls == 1 + + +def test_power_loss_after_dispatch_becomes_terminal_unknown_without_replay( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """CONN-12: a dispatched write is unknown audit, never an automatic retry.""" + + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + write_calls = 0 + + async def powered_off_after_dispatch( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal write_calls + write_calls += 1 + _dispatch_test_network_write(on_write_dispatch) + error = RuntimeError("K1 powered off after dispatch") + error.operation_stage = "gatt-write" # type: ignore[attr-defined] + error.device_write_attempted = True # type: ignore[attr-defined] + error.device_write_confirmed = False # type: ignore[attr-defined] + raise error + + monkeypatch.setattr(facade_module, "provision_wifi_once", powered_off_after_dispatch) + + with pytest.raises(RuntimeError, match="powered off after dispatch"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + + # The old advertisement may already be gone after the failed write. The + # durable unresolved ledger still pins an exact read-only target and must + # outrank a generic new scan in current operator guidance. + _set_scanned_devices(service, []) + failed = service.state() + failed_operation = next( + item for item in failed["operations"] if item["action"] == "network.provision" + ) + assert write_calls == 1 + assert failed["selected_device_id"] is None + assert failed["device_session"] is None + assert failed["network_mutation_ledger"]["status"] == "unresolved" + assert failed["network_mutation_ledger"]["stage"] == "dispatching" + assert failed_operation["error"]["side_effect_status"] == "unknown" + assert failed_operation["error"]["safe_to_retry"] is False + assert failed_operation["context"]["automatic_retry"] is False + assert failed["connection_attempt"]["phase"] == "network_outcome_unknown" + assert failed["connection_attempt"]["side_effect_status"] == "unknown" + assert failed["connection_attempt"]["safe_next_action"] == "verify-control-read-only" + assert failed["connection_attempt"]["automatic_retry"] is False + assert ( + failed["connection_policy"]["actions"]["observe-configured-device-network"]["allowed"] + is True + ) + assert failed["connection_policy"]["recommended_action"] == ( + "observe-configured-device-network" + ) + assert failed["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + _set_scanned_k1(service, device_id="k1-a") + assert ( + service.state()["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + ) + assert write_calls == 1 + + _SYNTHETIC_SCAN_CAPTURES.clear() + restarted, _ = service_with_fake_runtime(tmp_path) + terminal = restarted.state() + assert write_calls == 1 + assert terminal["selected_device_id"] is None + assert terminal["device_session"] is None + assert terminal["network_write_reconciliation"] is None + assert terminal["network_mutation_ledger"]["status"] == "resolved" + assert terminal["network_mutation_ledger"]["resolution"] == "interrupted" + assert terminal["network_provisioning_idempotency"]["active_operation_id"] is None + durable_terminal = ( + restarted._require_network_provisioning_idempotency_journal() # noqa: SLF001 + .snapshot() + .records[-1] + ) + assert durable_terminal.terminal is not None + assert durable_terminal.terminal.outcome_code == "network.provision.interrupted" + assert durable_terminal.terminal.side_effect_status == "reconciled" + assert durable_terminal.terminal.safe_to_retry is False + _set_scanned_k1(restarted, device_id="k1-a") + assert ( + restarted.state()["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] + is True + ) + assert write_calls == 1 + + +def test_power_loss_during_observation_preserves_audit_and_releases_ownership( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """CONN-13: bounded observation survives; selection and ownership do not.""" + + service, _ = service_with_fake_runtime(tmp_path) + _set_scanned_k1(service, device_id="k1-a") + baseline = _wifi_status_read(None, device_id="k1-a")["status"] + bounded_baseline = {key: baseline[key] for key in ("mode", "ipv4", "status_code", "reserved")} + write_calls = 0 + + async def power_lost_while_observing( + *_: object, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **__: object, + ) -> dict[str, Any]: + nonlocal write_calls + write_calls += 1 + assert on_write_dispatch is not None + on_write_dispatch(baseline, "with_response") + return { + "started_at_utc": "2026-08-08T10:00:00Z", + "completed_at_utc": "2026-08-08T10:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "connection-lost-during-status-observation", + "write_mode": "with_response", + "baseline_status": baseline, + "observations": [{"status": baseline}], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", power_lost_while_observing) + + with pytest.raises(RuntimeError, match="не сообщило адрес"): + asyncio.run( + service.connect( + _connect_request( + device_id="k1-a", + ssid="lab-router", + password=SecretStr(PRIMARY_TEST_CREDENTIAL), + compatibility_attestation=ATTESTATION, + ) + ) + ) + + failed = service.state() + failed_record = service._network_mutation_ledger.snapshot().record # noqa: SLF001 + failed_operation = next( + item for item in failed["operations"] if item["action"] == "network.provision" + ) + assert write_calls == 1 + assert failed["selected_device_id"] is None + assert failed["device_session"] is None + assert failed["network_mutation_ledger"]["status"] == "unresolved" + assert failed["network_mutation_ledger"]["stage"] == "observing" + assert failed_record is not None + assert failed_record.write_confirmed is True + assert failed_record.last_observation is not None + assert failed_record.last_observation.as_dict() == bounded_baseline + assert failed_operation["error"]["side_effect_status"] == "unknown" + assert failed_operation["error"]["safe_to_retry"] is False + assert failed_operation["context"]["automatic_retry"] is False + assert failed["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + _set_scanned_k1(service, device_id="k1-a") + assert ( + service.state()["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] is True + ) + assert write_calls == 1 + + _SYNTHETIC_SCAN_CAPTURES.clear() + restarted, _ = service_with_fake_runtime(tmp_path) + terminal = restarted.state() + terminal_record = restarted._network_mutation_ledger.snapshot().record # noqa: SLF001 + assert write_calls == 1 + assert terminal["selected_device_id"] is None + assert terminal["device_session"] is None + assert terminal["network_write_reconciliation"] is None + assert terminal["network_mutation_ledger"]["status"] == "resolved" + assert terminal["network_mutation_ledger"]["resolution"] == "interrupted" + assert terminal_record is not None + assert terminal_record.last_observation is not None + assert terminal_record.last_observation.as_dict() == bounded_baseline + assert terminal["network_provisioning_idempotency"]["active_operation_id"] is None + _set_scanned_k1(restarted, device_id="k1-a") + assert ( + restarted.state()["connection_policy"]["actions"]["provision-fresh-device"]["allowed"] + is True + ) + assert write_calls == 1 diff --git a/tests/test_xgrids_active_acquisition_checkpoint_integration.py b/tests/test_xgrids_active_acquisition_checkpoint_integration.py new file mode 100644 index 0000000..e4d84ef --- /dev/null +++ b/tests/test_xgrids_active_acquisition_checkpoint_integration.py @@ -0,0 +1,1917 @@ +from __future__ import annotations + +import asyncio +import hashlib +import json +import sys +import threading +import time +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import k1link.device_plugins.xgrids_k1.camera as camera_module +from k1link.data_plane import ConsumerFrameContext, DecodedPointCloudView +from k1link.device_plugins.xgrids_k1.active_acquisition_recovery_checkpoint import ( + ActiveAcquisitionRecoveryCheckpointStore, + active_acquisition_project_name_sha256, +) +from k1link.device_plugins.xgrids_k1.connection_supervisor import ( + EndpointTarget, + HostPathProbeResult, + VerifiedControlEvidence, +) +from k1link.device_plugins.xgrids_k1.facade import ( + XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + OperatorPresenceRequest, + StopAcquisitionRequest, + XgridsK1CompatibilityService, + _active_stream_reconciliation_id, +) +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + PhysicalCommandIntentContext, + PhysicalCommandRuntimeBinding, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + PhysicalCommandApplicationResponse, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandStatusEvidence, + PhysicalCommandVerifiedBinding, +) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( + MODELING_STATE_BASE, +) +from k1link.device_plugins.xgrids_k1.viewer.runtime import VisualizationRuntime +from k1link.web.device_lifecycle import AcquisitionRecord + +ACQUISITION_ID = "acq-11111111-1111-4111-8111-111111111111" +START_OPERATION_ID = "physical-start-11111111-1111-4111-8111-111111111111" +FIRST_STOP_OPERATION_ID = "physical-stop-22222222-2222-4222-8222-222222222222" +FINAL_STOP_OPERATION_ID = "physical-stop-33333333-3333-4333-8333-333333333333" +EVIDENCE_SESSION_ID = "20260813T120000Z_viewer_live" +START_PAYLOAD = b"checkpoint-start" +START_PAYLOAD_SHA256 = hashlib.sha256(START_PAYLOAD).hexdigest() +VENDOR_SHA256 = "a" * 64 +SERIAL_SHA256 = "b" * 64 +PROJECT_SHA256 = "c" * 64 + + +def _gated_recoverable_source_end_ffmpeg(tmp_path: Path) -> tuple[Path, Path, Path]: + executable = tmp_path / "gated-recoverable-source-end-ffmpeg" + invocation_count = tmp_path / "gated-recoverable-source-end-count" + release_first = tmp_path / "release-first-camera-source" + executable.write_text( + f"#!{sys.executable}\n" + "import pathlib, sys, time\n" + "def box(kind, payload=b''):\n" + " return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n" + f"counter = pathlib.Path({str(invocation_count)!r})\n" + "count = int(counter.read_text()) + 1 if counter.exists() else 1\n" + "counter.write_text(str(count))\n" + "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n" + "sys.stdout.buffer.write(box(b'moof') + box(b'mdat', b'frame'))\n" + "sys.stdout.buffer.flush()\n" + f"release = pathlib.Path({str(release_first)!r})\n" + "if count == 1:\n" + " deadline = time.monotonic() + 5\n" + " while not release.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.005)\n" + "else:\n" + " time.sleep(10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable, invocation_count, release_first + + +def _wait_until(predicate: Any, *, timeout: float = 5.0) -> None: + assert callable(predicate) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition was not reached before timeout") + + +def _service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> XgridsK1CompatibilityService: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + monkeypatch.setenv("MISSIONCORE_EVIDENCE_DIR", str(tmp_path / "evidence")) + monkeypatch.setenv( + "MISSIONCORE_LEGACY_SESSIONS_DIR", + str(tmp_path / "legacy-sessions"), + ) + return XgridsK1CompatibilityService(tmp_path) + + +def _connection( + *, + control_session_id: str, + host_path_epoch: int, + producer_generation: int, +) -> PhysicalCommandConnectionBinding: + return PhysicalCommandConnectionBinding( + intent_id="intent-checkpoint-integration", + transport_ref="transport-checkpoint-integration", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=host_path_epoch, + control_session_id=control_session_id, + producer_generation=producer_generation, + ) + + +def _status( + connection: PhysicalCommandConnectionBinding, + state: str, + *, + source: str = "live-control-session", + observed_at_utc: str, +) -> PhysicalCommandStatusEvidence: + scanning = state == "scanning" + return PhysicalCommandStatusEvidence( + source=source, # type: ignore[arg-type] + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state=state, # type: ignore[arg-type] + session_state_code={"ready": 300, "scanning": 302}[state], + project_bound=scanning, + project_id_sha256=PROJECT_SHA256 if scanning else None, + init_ready=scanning, + status_message_sha256=hashlib.sha256( + f"{state}:{observed_at_utc}".encode() + ).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + +def _runtime_binding( + connection: PhysicalCommandConnectionBinding, +) -> PhysicalCommandRuntimeBinding: + return PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + host_path_epoch=connection.host_path_epoch, + control_session_id=connection.control_session_id, + producer_generation=connection.producer_generation, + ) + + +def _verified_control( + connection: PhysicalCommandConnectionBinding, +) -> dict[str, object]: + return { + "logical_device_id": "logical-k1-checkpoint", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + "intent_id": connection.intent_id, + "transport_ref": connection.transport_ref, + "connection_mode": connection.connection_mode, + "target_ipv4": connection.target_ipv4, + "target_port": connection.target_port, + "host_path_epoch": connection.host_path_epoch, + "control_session_id": connection.control_session_id, + "producer_generation": connection.producer_generation, + } + + +def _prepare_and_activate_checkpoint( + service: XgridsK1CompatibilityService, + connection: PhysicalCommandConnectionBinding, +) -> None: + coordinator = service._physical_command_coordinator # noqa: SLF001 + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="d" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-13T12:00:00.000Z", + ) + ) + coordinator.bind_control_session(_runtime_binding(connection)) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="e" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:00:01.000Z", + ) + ) + envelope = OneShotPublishEnvelope( + operation_key="modeling:start", + topic="lixel/application/request/modeling", + payload=START_PAYLOAD, + payload_sha256=START_PAYLOAD_SHA256, + payload_bytes=len(START_PAYLOAD), + qos=2, + retain=False, + ) + context = PhysicalCommandIntentContext( + operation_id=START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + ) + coordinator.prepare(context, action="start", envelope=envelope) + out_dir = service.evidence_root / EVIDENCE_SESSION_ID + with service._lock: # noqa: SLF001 + service._acquisition = AcquisitionRecord( # noqa: SLF001 + acquisition_id=ACQUISITION_ID, + device_id="logical-k1-checkpoint", + device_session_id="device-session-checkpoint", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_mode="plugin-commanded", + requested_streams=("pointcloud",), + target_host=connection.target_ipv4, + duration_seconds=None, + evidence_policy="required", + state="starting", + ) + service._acquisition_out_dir = out_dir # noqa: SLF001 + service._acquisition_project_name = "CHAINED_STOP_TEST" # noqa: SLF001 + service._acquisition_mount_type = "handheld" # noqa: SLF001 + service._acquisition_gnss_mode = "none" # noqa: SLF001 + service._acquisition_start_operation_id = START_OPERATION_ID # noqa: SLF001 + service._application_control_session.snapshot = lambda: { # type: ignore[method-assign] + "verified_control": _verified_control(connection) + } + service._observe_start_preparation_checkpoint( # noqa: SLF001 + "prepared", + context, + envelope, + ) + publish_evidence = ApplicationMqttPublishEvidence( + operation_key="modeling:start", + topic=envelope.topic, + payload_sha256=START_PAYLOAD_SHA256, + qos=2, + retain=False, + packet_id=41, + ) + coordinator.publish_dispatching(publish_evidence) + coordinator.publish_result(publish_evidence, publish_call_returned=True) + coordinator.qos2_completed(publish_evidence) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="modeling:start", + response_topic="lixel/application/response/modeling", + payload_sha256="f" * 64, + modeling_action="start", + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + observed_at_utc="2026-08-13T12:00:02.000Z", + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256=PROJECT_SHA256, + init_ready=True, + status_message_sha256="1" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:00:03.000Z", + ) + ) + coordinator.resolve("start") + service._observe_application_scanning_checkpoint() # noqa: SLF001 + + +def test_repeated_reset_reopen_fresh_ready_ceases_old_active_checkpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Fresh READY after Apply settles the old START and its checkpoint. + + The live operator path can carry an ACTIVE checkpoint through several + local scenario reset retirement/reopen cycles. Those append-only audit + edges must neither make the proven START ambiguous nor leave its old + checkpoint blocking the project form after fresh READY is observed. + """ + + service = _service(tmp_path, monkeypatch) + original_connection = _connection( + control_session_id="checkpoint-reset-original-control", + host_path_epoch=1, + producer_generation=1, + ) + _prepare_and_activate_checkpoint(service, original_connection) + ledger = service._physical_command_ledger # noqa: SLF001 + current = ledger.snapshot().record + assert current is not None + + active_rebind_connection = _connection( + control_session_id="checkpoint-reset-active-rebind-control", + host_path_epoch=2, + producer_generation=2, + ) + current = ledger.rebind_resolved_active( + current.operation_id, + reconciliation_id="checkpoint-reset-active-rebind", + verified_binding=PhysicalCommandVerifiedBinding( + verification_id="checkpoint-reset-active-rebind.device-info", + identity=current.identity, + connection=active_rebind_connection, + device_info_message_sha256="2" * 64, + verified_at_utc="2026-08-13T12:01:00.000Z", + ), + observation=_status( + active_rebind_connection, + "scanning", + source="explicit-read-only-reconciliation", + observed_at_utc="2026-08-13T12:01:01.000Z", + ), + ) + for cycle in (1, 2): + retired = ledger.retire_unavailable_target( + retirement_id=f"checkpoint-reset-retirement-{cycle}", + expected_operation_id=current.operation_id, + expected_revision=current.revision, + expected_transport_ref=current.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + current = ledger.reopen_retired_reconciliation( + reopening_id=f"checkpoint-reset-reopen-{cycle}", + expected_revision=retired.revision, + expected_retirement_id=f"checkpoint-reset-retirement-{cycle}", + expected_transport_ref=retired.connection.transport_ref, + expected_discovery_generation=20 + cycle, + reason="reset-network-intent-read-only-settlement", + ) + assert current.stage == "observing" + assert current.reconciled_physical_state == "active" + + # The persisted ACTIVE checkpoint belongs to the process that owned the + # original receiver. The real failure was observed after a service + # restart, while the later reset/reopen audit remained durable. + service._snapshot_runtime_id = "snapshot-runtime-checkpoint-reset-successor" # noqa: SLF001 + + ready_connection = _connection( + control_session_id="checkpoint-reset-ready-control", + host_path_epoch=4, + producer_generation=4, + ) + coordinator = service._physical_command_coordinator # noqa: SLF001 + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:reset-ready:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="3" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-13T12:02:00.000Z", + ) + ) + coordinator.bind_control_session(_runtime_binding(ready_connection)) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="4" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:02:01.000Z", + ) + ) + control_snapshot = { + "state": "connection-ready", + "session_generation": 4, + "state_revision": 4, + "verified_control": _verified_control(ready_connection), + } + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "snapshot", + lambda: dict(control_snapshot), + ) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "validate_physical_reconciliation_binding", + lambda: None, + ) + + result = asyncio.run( + service._reconcile_physical_command_after_verify_owned( # noqa: SLF001 + verify_operation_id="checkpoint-reset-ready-verify", + allow_receiver_rehydrate=False, + ) + ) + + assert result["performed"] is True + assert result["resolution"] == "physical-standby-observed" + settled = ledger.snapshot().record + assert settled is not None + assert settled.stage == "resolved" + assert settled.resolution == "start-active-observed" + assert settled.reconciled_physical_state == "standby" + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "ceased" + assert service._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001 + + +def _published_point_cloud(sequence: int = 41) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=64, + live=True, + ), + frame_id="map", + positions_xyz=((0.0, 0.0, 0.0),), + ) + + +def _install_same_process_recovery_runtime( + service: XgridsK1CompatibilityService, + connection: PhysicalCommandConnectionBinding, + *, + events: list[str], + attempt: int = 3, + producer_generation: int = 23, +) -> tuple[VisualizationRuntime, Any]: + """Install real checkpoint/runtime promotion state without sockets or K1 I/O.""" + + def checkpoint_confirmation( + envelope: DecodedPointCloudView, + generation: int, + recovery_attempt: int, + minimum_sequence: int, + ) -> bool: + events.append("checkpoint") + confirmed = service._confirm_active_acquisition_checkpoint_rebind( # noqa: SLF001 + envelope, + generation, + recovery_attempt, + minimum_sequence, + ) + checkpoint_store = service._active_acquisition_checkpoint # noqa: SLF001 + assert checkpoint_store is not None + checkpoint = checkpoint_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_recovered_at_utc is not None + return confirmed + + runtime = VisualizationRuntime( + normalizer=lambda *_args, **_kwargs: None, + published_envelope_observer=service._observe_published_runtime_envelope, # noqa: SLF001 + recovery_promotion_checkpoint=checkpoint_confirmation, + ) + service.runtime = runtime + target = EndpointTarget(connection.target_ipv4, connection.target_port) + supervisor = service._connection_supervisor # noqa: SLF001 + supervisor.set_intent( + intent_id=connection.intent_id, + requested_mode=connection.connection_mode, + expected_device_id="logical-k1-checkpoint", + ) + assert supervisor.observe_device_network_applied( + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target=target, + source="ble-read-only-status", + ) + host_epoch = supervisor.observe_host_path( + HostPathProbeResult( + available=True, + fingerprint="checkpoint-camera-route", + interface="test0", + source_ipv4="192.168.68.10", + route_class="direct", + kernel_route_fingerprint="checkpoint-camera-route", + ) + ) + assert supervisor.observe_endpoint( + target=target, + intent_id=connection.intent_id, + host_path_epoch=host_epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + host_path_epoch=host_epoch, + target=target, + connection_mode=connection.connection_mode, + logical_device_id="logical-k1-checkpoint", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id=connection.control_session_id, + ) + ) + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + assert acquisition is not None + acquisition.transition("acquiring", message_code="test.recovery.active") + service._selected_device_id = connection.transport_ref # noqa: SLF001 + service._connection_mode = connection.connection_mode # noqa: SLF001 + service._k1_ip = connection.target_ipv4 # noqa: SLF001 + service._device_id = acquisition.device_id # noqa: SLF001 + service._device_session_id = acquisition.device_session_id # noqa: SLF001 + service._acquisition_session_lease = object() # type: ignore[assignment] # noqa: SLF001 + service._application_control_session.snapshot = lambda: { # type: ignore[method-assign] + "state": "scanning", + "session_generation": 1, + "state_revision": 1, + "can_stop": True, + "verified_control": _verified_control(connection), + } + with runtime._lock: # noqa: SLF001 + runtime._phase = "reconnecting" # noqa: SLF001 + runtime._source_mode = "live" # noqa: SLF001 + runtime._source_ready = False # noqa: SLF001 + runtime._producer_generation = producer_generation # noqa: SLF001 + runtime._connection_recovery_enabled = True # noqa: SLF001 + runtime._connection_recovery_state = "reconnecting" # noqa: SLF001 + runtime._connection_recovery_attempt = attempt # noqa: SLF001 + lineage = service._admit_active_stream_recovery_lineage( # noqa: SLF001 + runtime=runtime.snapshot() + ) + assert lineage is not None + with service._lock: # noqa: SLF001 + service._active_stream_recovery_attempt = attempt # noqa: SLF001 + checkpoint_store = service._active_acquisition_checkpoint # noqa: SLF001 + assert checkpoint_store is not None + open_gap = checkpoint_store.snapshot().checkpoint + assert open_gap is not None + assert open_gap.last_gap_started_at_utc is not None + assert open_gap.last_gap_recovered_at_utc is None + return runtime, lineage + + +def _reconcile_direct_start_for_recovery( + service: XgridsK1CompatibilityService, + lineage: Any, + connection: PhysicalCommandConnectionBinding, + *, + attempt: int, +) -> None: + coordinator = service._physical_command_coordinator # noqa: SLF001 + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=f"bootstrap:recovery-{attempt}:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="6" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-13T12:10:00.000Z", + ) + ) + coordinator.bind_control_session(_runtime_binding(connection)) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256=PROJECT_SHA256, + init_ready=True, + status_message_sha256="7" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:10:01.000Z", + ) + ) + coordinator.reconcile_resolved_active( + reconciliation_id=_active_stream_reconciliation_id(lineage, attempt=attempt) + ) + service._application_control_session.snapshot = lambda: { # type: ignore[method-assign] + "state": "scanning", + "session_generation": 1, + "state_revision": 1, + "can_stop": True, + "verified_control": _verified_control(connection), + } + + +def _arm_runtime_recovery_confirmation( + runtime: VisualizationRuntime, + *, + producer_generation: int, + attempt: int, + minimum_sequence: int, + events: list[str], +) -> None: + runtime._arm_connection_recovery_point_cloud_candidate( # noqa: SLF001 + attempt, + minimum_sequence, + generation=producer_generation, + ) + + def confirm_capture(recovery_attempt: int) -> bool: + assert recovery_attempt == attempt + events.append("capture") + return True + + runtime._install_connection_recovery_confirmer( # noqa: SLF001 + confirm_capture, + generation=producer_generation, + ) + + +def _append_two_classified_stop_heads( + service: XgridsK1CompatibilityService, + initial: PhysicalCommandConnectionBinding, +) -> PhysicalCommandConnectionBinding: + """Build S1 -> S0 -> START using only durable no-dispatch evidence.""" + + ledger = service._physical_command_ledger # noqa: SLF001 + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + first_rebind = _connection( + control_session_id="control-camera-classified-s0", + host_path_epoch=8, + producer_generation=10, + ) + second_rebind = _connection( + control_session_id="control-camera-classified-s1", + host_path_epoch=9, + producer_generation=11, + ) + for index, (operation_id, parent_id, before, after) in enumerate( + ( + (FIRST_STOP_OPERATION_ID, START_OPERATION_ID, initial, first_rebind), + ( + FINAL_STOP_OPERATION_ID, + FIRST_STOP_OPERATION_ID, + first_rebind, + second_rebind, + ), + ) + ): + ledger.prepare( + operation_id=operation_id, + parent_operation_id=parent_id, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=before, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256=str(index + 2) * 64, + baseline_status=_status( + before, + "scanning", + observed_at_utc=f"2026-08-13T12:20:0{index * 3}.000Z", + ), + operator_confirmation_id=( + "camera-classified-s1-confirmation" if index == 1 else None + ), + operator_confirmed_at_utc=( + "2026-08-13T12:20:03.500Z" if index == 1 else None + ), + ) + prepared = ledger.snapshot().record + assert prepared is not None + ledger.reconcile_prepared_stop_not_dispatched( + operation_id, + expected_revision=prepared.revision, + reconciliation_id=f"camera-classified-stop-{index}", + verified_binding=PhysicalCommandVerifiedBinding( + verification_id=f"camera-classified-stop-{index}.device-info", + identity=identity, + connection=after, + device_info_message_sha256=str(index + 4) * 64, + verified_at_utc=f"2026-08-13T12:20:0{index * 3 + 1}.000Z", + ), + observation=_status( + after, + "scanning", + source="explicit-read-only-reconciliation", + observed_at_utc=f"2026-08-13T12:20:0{index * 3 + 2}.000Z", + ), + ) + return second_rebind + + +@pytest.mark.parametrize( + "scenario", + ["start", "s1-s0", "stop-race"], +) +def test_first_recovered_pcl_restarts_one_established_dead_camera_epoch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + scenario: str, +) -> None: + service = _service(tmp_path, monkeypatch) + initial = _connection( + control_session_id="control-camera-before-loss", + host_path_epoch=7, + producer_generation=9, + ) + _prepare_and_activate_checkpoint(service, initial) + events: list[str] = [] + attempt = 3 + runtime_generation = 23 + runtime, lineage = _install_same_process_recovery_runtime( + service, + initial, + events=events, + attempt=attempt, + producer_generation=runtime_generation, + ) + next_host_path_epoch = 8 + next_control_generation = 10 + if scenario == "s1-s0": + classified_binding = _append_two_classified_stop_heads(service, initial) + current_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert current_record is not None + lineage = replace( + lineage, + physical_operation_id=FINAL_STOP_OPERATION_ID, + physical_revision=current_record.revision, + ) + with service._lock: # noqa: SLF001 + service._active_stream_recovery_lineage = lineage # noqa: SLF001 + service._acquisition_stop_operation_id = FINAL_STOP_OPERATION_ID # noqa: SLF001 + next_host_path_epoch = classified_binding.host_path_epoch + 1 + next_control_generation = classified_binding.producer_generation + 1 + recovered_control = _connection( + control_session_id="control-camera-after-loss", + host_path_epoch=next_host_path_epoch, + producer_generation=next_control_generation, + ) + _reconcile_direct_start_for_recovery( + service, + lineage, + recovered_control, + attempt=attempt, + ) + + camera: dict[str, Any] = { + "phase": "error", + "generation": 7, + "active_source_id": "sensor.camera.right", + "error": { + "code": "camera-source-ended", + "message": "synthetic combined MQTT and camera loss", + }, + "recording": { + "active": True, + "session": EVIDENCE_SESSION_ID, + "active_epoch": 7, + "producer_alive": False, + "producer_age_ms": 12_000, + "last_segment_age_ms": 12_000, + "last_media_segment_age_ms": 12_000, + "committed_media_segment_count": 4, + "media_ready": True, + "current_epoch": { + "generation": 7, + "init_committed": True, + "init_committed_age_ms": 12_000, + "first_media_committed": True, + "first_media_committed_age_ms": 12_000, + "committed_media_segment_count": 4, + "last_media_segment_age_ms": 12_000, + }, + }, + } + restart_entered = threading.Event() + release_restart = threading.Event() + restart_complete = threading.Event() + restart_calls: list[tuple[int, int | None, int]] = [] + + def snapshot_camera() -> dict[str, Any]: + return {**camera, "recording": dict(camera["recording"])} + + def restart_camera( + _source_id: str, + _target: str, + *, + expected_generation: int, + expected_recording_session: str, + expected_active_epoch: int | None, + expected_recording_media_segment_count: int, + pre_detach_fence: Any, + commit_fence: Any, + committed_before_start: Any, + ) -> dict[str, Any]: + restart_calls.append( + ( + expected_generation, + expected_active_epoch, + expected_recording_media_segment_count, + ) + ) + assert expected_recording_session == EVIDENCE_SESSION_ID + restart_entered.set() + assert release_restart.wait(3.0) + if not pre_detach_fence(lambda: True): + restart_complete.set() + raise ValueError("camera recovery reservation stale") + checkpoint_store = service._active_acquisition_checkpoint # noqa: SLF001 + assert checkpoint_store is not None + checkpoint = checkpoint_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_recovered_at_utc is not None + assert runtime.snapshot()["connection_recovery"]["state"] == "recovered" + candidate = { + **camera, + "phase": "connecting", + "generation": expected_generation + 1, + "error": None, + "recording": { + **camera["recording"], + "active_epoch": expected_active_epoch + 1, + "producer_alive": True, + "producer_age_ms": 0, + "last_segment_age_ms": None, + "last_media_segment_age_ms": None, + "media_ready": False, + "current_epoch": { + "generation": expected_active_epoch + 1, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + }, + }, + } + + def commit() -> bool: + events.append("camera") + camera.clear() + camera.update(candidate) + committed_before_start(snapshot_camera()) + return True + + try: + if not commit_fence(commit): + raise ValueError("camera recovery lineage stale") + return snapshot_camera() + finally: + restart_complete.set() + + monkeypatch.setattr(service.camera_preview, "snapshot", snapshot_camera) + monkeypatch.setattr( + service.camera_preview, + "restart_recording_producer", + restart_camera, + ) + frame = _published_point_cloud() + _arm_runtime_recovery_confirmation( + runtime, + producer_generation=runtime_generation, + attempt=attempt, + minimum_sequence=frame.context.sequence, + events=events, + ) + runtime._metrics.published_pcl(1, time.monotonic_ns(), 0.1) # noqa: SLF001 + + assert restart_calls == [] + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + frame, + generation=runtime_generation, + ) + assert events == ["checkpoint", "capture"] + assert runtime.snapshot()["connection_recovery"]["state"] == "recovered" + + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + assert restart_entered.wait(3.0) + # Duplicate recovered PCL is coalesced while the exact G+1 candidate waits. + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + assert restart_calls == [(7, 7, 4)] + + if scenario == "stop-race": + stop_owned_entered = threading.Event() + release_stop = threading.Event() + stop_results: list[dict[str, str]] = [] + + def stop_owned( + _request: StopAcquisitionRequest, + _preadmitted_stop: object | None = None, + ) -> dict[str, str]: + assert service._camera_stop_priority_counts == {ACQUISITION_ID: 1} # noqa: SLF001 + stop_owned_entered.set() + assert release_stop.wait(3.0) + return {"stop": "priority-owned"} + + monkeypatch.setattr(service, "_stop_acquisition_owned", stop_owned) + stop_worker = threading.Thread( + target=lambda: stop_results.append( + service.stop_acquisition( + StopAcquisitionRequest( + acquisition_id=None, + mode="graceful", + idempotency_key="camera-recovery-stop-race", + physical_acceptance=OperatorPresenceRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + ), + expected_control_session_generation=1, + expected_control_state_revision=1, + ) + ) + ), + daemon=True, + ) + stop_worker.start() + assert stop_owned_entered.wait(3.0) + release_restart.set() + assert restart_complete.wait(3.0) + assert events == ["checkpoint", "capture"] + assert camera["generation"] == 7 + assert camera["recording"]["active_epoch"] == 7 + release_stop.set() + stop_worker.join(3.0) + assert not stop_worker.is_alive() + assert stop_results == [{"stop": "priority-owned"}] + return + + release_restart.set() + assert restart_complete.wait(3.0) + + assert events == ["checkpoint", "capture", "camera"] + assert camera["generation"] == 8 + assert camera["recording"]["active_epoch"] == 8 + assert camera["recording"]["media_ready"] is False + physical = service._physical_command_coordinator.snapshot() # noqa: SLF001 + assert physical["record"]["operation_id"] == ( + FINAL_STOP_OPERATION_ID if scenario == "s1-s0" else START_OPERATION_ID + ) + assert physical["record"]["action"] == ( + "stop" if scenario == "s1-s0" else "start" + ) + + +@pytest.mark.parametrize("failed_recovery_spawns", [0, 1, 3]) +def test_real_sealed_eof_waits_for_recovered_pcl_and_owns_bounded_camera_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failed_recovery_spawns: int, +) -> None: + executable, invocation_count, release_first = ( + _gated_recoverable_source_end_ffmpeg(tmp_path) + ) + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(executable)) + service = _service(tmp_path, monkeypatch) + initial = _connection( + control_session_id="control-real-camera-before-loss", + host_path_epoch=7, + producer_generation=9, + ) + _prepare_and_activate_checkpoint(service, initial) + out_dir = service.evidence_root / EVIDENCE_SESSION_ID + out_dir.mkdir(parents=True, exist_ok=True) + camera_lease_created = service._ensure_camera_preview_process_lease() # noqa: SLF001 + assert camera_lease_created is True + try: + selected = service.camera_preview.select( + "sensor.camera.right", + initial.target_ipv4, + ) + first_generation = selected["generation"] + service.camera_preview.start_recording(out_dir) + _wait_until( + lambda: service.camera_preview.snapshot()["recording"]["media_ready"] + is True + ) + assert invocation_count.read_text(encoding="utf-8") == "1" + + events: list[str] = [] + attempt = 3 + runtime_generation = 23 + runtime, lineage = _install_same_process_recovery_runtime( + service, + initial, + events=events, + attempt=attempt, + producer_generation=runtime_generation, + ) + # The established producer ends only after MQTT/runtime authority has + # entered reconnecting. Its backend watchdog must not restart before + # checkpoint/capture/runtime recovery promotion and a published PCL. + release_first.write_text("release", encoding="utf-8") + _wait_until( + lambda: ( + service.camera_preview.snapshot()["phase"] == "error" + and service.camera_preview.snapshot()["recording"]["active_epoch"] + is None + ) + ) + sealed = service.camera_preview.snapshot() + sealed_recording = sealed["recording"] + assert sealed["error"]["code"] == "camera-source-ended" + assert sealed_recording["producer_alive"] is False + assert sealed_recording["last_summary"]["schema_version"] == ( + "missioncore.camera-recording/v1" + ) + assert invocation_count.read_text(encoding="utf-8") == "1" + + injected_failures = 0 + if failed_recovery_spawns: + real_popen = camera_module.subprocess.Popen + + def fail_once_popen(*args: Any, **kwargs: Any) -> Any: + nonlocal injected_failures + if injected_failures < failed_recovery_spawns: + injected_failures += 1 + raise OSError("injected first recovery Popen failure") + return real_popen(*args, **kwargs) + + monkeypatch.setattr(camera_module.subprocess, "Popen", fail_once_popen) + + recovered_control = _connection( + control_session_id="control-real-camera-after-loss", + host_path_epoch=8, + producer_generation=10, + ) + _reconcile_direct_start_for_recovery( + service, + lineage, + recovered_control, + attempt=attempt, + ) + frame = _published_point_cloud() + _arm_runtime_recovery_confirmation( + runtime, + producer_generation=runtime_generation, + attempt=attempt, + minimum_sequence=frame.context.sequence, + events=events, + ) + runtime._metrics.published_pcl(1, time.monotonic_ns(), 0.1) # noqa: SLF001 + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + frame, + generation=runtime_generation, + ) + assert events == ["checkpoint", "capture"] + assert runtime.snapshot()["connection_recovery"]["state"] == "recovered" + checkpoint_store = service._active_acquisition_checkpoint # noqa: SLF001 + assert checkpoint_store is not None + checkpoint = checkpoint_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_recovered_at_utc is not None + + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + expected_generation = first_generation + failed_recovery_spawns + 1 + if failed_recovery_spawns == 3: + _wait_until( + lambda: ( + service._post_recovery_camera_restart_retry is not None # noqa: SLF001 + and service._post_recovery_camera_restart_inflight is None # noqa: SLF001 + ) + ) + failed_generation = service.camera_preview.snapshot()["generation"] + assert failed_generation == first_generation + 3 + assert invocation_count.read_text(encoding="utf-8") == "1" + retry = service._post_recovery_camera_restart_retry # noqa: SLF001 + assert retry is not None + assert retry.failure_batch == 1 + + # Duplicate 10 Hz PCLs during the capped cooldown are consumed by + # the same exact claim and cannot become an initial-camera spawn. + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + time.sleep(0.02) + assert service.camera_preview.snapshot()["generation"] == failed_generation + assert service._post_recovery_camera_restart_inflight is None # noqa: SLF001 + remaining = max(0.0, retry.not_before_monotonic - time.monotonic()) + time.sleep(remaining + 0.02) + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + _wait_until( + lambda: ( + service.camera_preview.snapshot()["generation"] + == expected_generation + and service.camera_preview.snapshot()["recording"]["media_ready"] + is True + ) + ) + recovered = service.camera_preview.snapshot() + assert recovered["recording"]["active_epoch"] == expected_generation + assert recovered["recording"]["current_epoch"]["generation"] == ( + expected_generation + ) + assert invocation_count.read_text(encoding="utf-8") == "2" + assert injected_failures == failed_recovery_spawns + assert events == ["checkpoint", "capture"] + + # Duplicate recovered PCLs retain only the admitted successor claim. + service._observe_published_runtime_envelope(frame, runtime_generation) # noqa: SLF001 + time.sleep(0.05) + assert invocation_count.read_text(encoding="utf-8") == "2" + assert service.camera_preview.snapshot()["generation"] == expected_generation + claim = service._post_recovery_camera_restart_claim # noqa: SLF001 + assert claim is not None + assert claim.camera_generation == expected_generation + assert service._post_recovery_camera_restart_inflight is None # noqa: SLF001 + finally: + service.camera_preview.close() + service._release_camera_preview_process_lease() # noqa: SLF001 + + +def test_real_camera_first_reservation_does_not_hold_stop_priority_gate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + executable, invocation_count, release_first = ( + _gated_recoverable_source_end_ffmpeg(tmp_path) + ) + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(executable)) + service = _service(tmp_path, monkeypatch) + initial = _connection( + control_session_id="control-camera-first-before-loss", + host_path_epoch=7, + producer_generation=9, + ) + _prepare_and_activate_checkpoint(service, initial) + out_dir = service.evidence_root / EVIDENCE_SESSION_ID + out_dir.mkdir(parents=True, exist_ok=True) + assert service._ensure_camera_preview_process_lease() is True # noqa: SLF001 + release_shutdown = threading.Event() + release_stop = threading.Event() + try: + selected = service.camera_preview.select( + "sensor.camera.right", + initial.target_ipv4, + ) + first_generation = selected["generation"] + service.camera_preview.start_recording(out_dir) + _wait_until( + lambda: service.camera_preview.snapshot()["recording"]["media_ready"] + is True + ) + events: list[str] = [] + runtime, lineage = _install_same_process_recovery_runtime( + service, + initial, + events=events, + attempt=3, + producer_generation=23, + ) + release_first.write_text("release", encoding="utf-8") + _wait_until( + lambda: ( + service.camera_preview.snapshot()["phase"] == "error" + and service.camera_preview.snapshot()["recording"]["active_epoch"] + is None + ) + ) + recovered_control = _connection( + control_session_id="control-camera-first-after-loss", + host_path_epoch=8, + producer_generation=10, + ) + _reconcile_direct_start_for_recovery( + service, + lineage, + recovered_control, + attempt=3, + ) + frame = _published_point_cloud() + _arm_runtime_recovery_confirmation( + runtime, + producer_generation=23, + attempt=3, + minimum_sequence=frame.context.sequence, + events=events, + ) + runtime._metrics.published_pcl(1, time.monotonic_ns(), 0.1) # noqa: SLF001 + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + frame, + generation=23, + ) + assert events == ["checkpoint", "capture"] + + shutdown_entered = threading.Event() + real_shutdown = service.camera_preview._shutdown_producer # noqa: SLF001 + real_discard = service.camera_preview._discard_prepared_producer # noqa: SLF001 + real_popen = camera_module.subprocess.Popen + recovery_popen_calls = 0 + candidate_discarded = threading.Event() + + def observe_recovery_popen(*args: Any, **kwargs: Any) -> Any: + nonlocal recovery_popen_calls + recovery_popen_calls += 1 + return real_popen(*args, **kwargs) + + def blocked_shutdown( + producer: Any, + deliveries: Any, + *, + status: Any, + failure_code: str | None, + ) -> None: + if failure_code == "active-stream-connection-recovery": + shutdown_entered.set() + assert release_shutdown.wait(timeout=3) + real_shutdown( + producer, + deliveries, + status=status, + failure_code=failure_code, + ) + + def observe_discard(*args: Any, **kwargs: Any) -> None: + real_discard(*args, **kwargs) + candidate_discarded.set() + + monkeypatch.setattr( + service.camera_preview, + "_shutdown_producer", + blocked_shutdown, + ) + monkeypatch.setattr( + service.camera_preview, + "_discard_prepared_producer", + observe_discard, + ) + monkeypatch.setattr(camera_module.subprocess, "Popen", observe_recovery_popen) + service._observe_published_runtime_envelope(frame, 23) # noqa: SLF001 + assert shutdown_entered.wait(timeout=3) + reserved = service.camera_preview.snapshot() + assert reserved["generation"] == first_generation + 1 + assert reserved["recording"]["active_epoch"] is None + + stop_entered = threading.Event() + stop_results: list[dict[str, str]] = [] + + def stop_owned( + _request: StopAcquisitionRequest, + _preadmitted_stop: object | None = None, + ) -> dict[str, str]: + assert service._camera_stop_priority_counts == {ACQUISITION_ID: 1} # noqa: SLF001 + stop_entered.set() + assert release_stop.wait(timeout=3) + return {"stop": "priority-owned"} + + monkeypatch.setattr(service, "_stop_acquisition_owned", stop_owned) + stop_worker = threading.Thread( + target=lambda: stop_results.append( + service.stop_acquisition( + StopAcquisitionRequest( + acquisition_id=None, + mode="graceful", + idempotency_key="real-camera-first-stop-race", + physical_acceptance=OperatorPresenceRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + ), + expected_control_session_generation=1, + expected_control_state_revision=1, + ) + ) + ), + daemon=True, + ) + stop_worker.start() + # The camera reservation already won, but blocking archive/process + # cleanup runs outside both shared facade gates. Physical STOP reaches + # its owned seam immediately and its priority rejects the late commit. + assert stop_entered.wait(timeout=3) + release_shutdown.set() + assert candidate_discarded.wait(timeout=3) + after = service.camera_preview.snapshot() + assert after["generation"] == first_generation + 1 + assert after["recording"]["active_epoch"] is None + assert after["recording"]["producer_alive"] is False + assert service._post_recovery_camera_restart_claim is None # noqa: SLF001 + assert recovery_popen_calls == 1 + assert invocation_count.read_text(encoding="utf-8") in {"1", "2"} + + release_stop.set() + stop_worker.join(3.0) + assert not stop_worker.is_alive() + assert stop_results == [{"stop": "priority-owned"}] + finally: + release_shutdown.set() + release_stop.set() + service.camera_preview.close() + service._release_camera_preview_process_lease() # noqa: SLF001 + + +def test_recovered_pcl_waits_for_real_eof_archive_seal_before_one_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + executable, invocation_count, release_first = ( + _gated_recoverable_source_end_ffmpeg(tmp_path) + ) + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(executable)) + service = _service(tmp_path, monkeypatch) + initial = _connection( + control_session_id="control-unsealed-eof-before-loss", + host_path_epoch=7, + producer_generation=9, + ) + _prepare_and_activate_checkpoint(service, initial) + out_dir = service.evidence_root / EVIDENCE_SESSION_ID + out_dir.mkdir(parents=True, exist_ok=True) + assert service._ensure_camera_preview_process_lease() is True # noqa: SLF001 + release_finalize = threading.Event() + release_watchdog = threading.Event() + try: + selected = service.camera_preview.select( + "sensor.camera.right", + initial.target_ipv4, + ) + first_generation = selected["generation"] + service.camera_preview.start_recording(out_dir) + _wait_until( + lambda: service.camera_preview.snapshot()["recording"]["media_ready"] + is True + ) + events: list[str] = [] + runtime, lineage = _install_same_process_recovery_runtime( + service, + initial, + events=events, + attempt=3, + producer_generation=23, + ) + + finalize_entered = threading.Event() + watchdog_entered = threading.Event() + real_finalize = service.camera_preview._finalize_archive # noqa: SLF001 + real_notify = service.camera_preview._notify_producer_stall_observer # noqa: SLF001 + + def blocked_finalize( + producer: Any, + status: Any, + failure_code: str | None, + ) -> None: + if failure_code == "camera-source-ended": + finalize_entered.set() + assert release_finalize.wait(timeout=3) + real_finalize(producer, status, failure_code) + + def blocked_watchdog(producer: Any) -> None: + if producer.failure_code == "camera-source-ended": + watchdog_entered.set() + assert release_watchdog.wait(timeout=3) + real_notify(producer) + + monkeypatch.setattr( + service.camera_preview, + "_finalize_archive", + blocked_finalize, + ) + monkeypatch.setattr( + service.camera_preview, + "_notify_producer_stall_observer", + blocked_watchdog, + ) + release_first.write_text("release", encoding="utf-8") + assert finalize_entered.wait(timeout=3) + unsealed = service.camera_preview.snapshot() + unsealed_recording = unsealed["recording"] + assert unsealed["phase"] == "error" + assert unsealed["error"]["code"] == "camera-source-ended" + assert unsealed["generation"] == first_generation + assert unsealed_recording["active_epoch"] is None + assert unsealed_recording["producer_alive"] is False + assert unsealed_recording["completed_epochs"] == 0 + assert unsealed_recording["last_summary"] is None + + recovered_control = _connection( + control_session_id="control-unsealed-eof-after-loss", + host_path_epoch=8, + producer_generation=10, + ) + _reconcile_direct_start_for_recovery( + service, + lineage, + recovered_control, + attempt=3, + ) + frame = _published_point_cloud() + _arm_runtime_recovery_confirmation( + runtime, + producer_generation=23, + attempt=3, + minimum_sequence=frame.context.sequence, + events=events, + ) + runtime._metrics.published_pcl(1, time.monotonic_ns(), 0.1) # noqa: SLF001 + runtime._confirm_connection_recovery_from_published_point_cloud( # noqa: SLF001 + frame, + generation=23, + ) + assert events == ["checkpoint", "capture"] + before_files = { + path.relative_to(out_dir).as_posix(): path.read_bytes() + for path in out_dir.rglob("*") + if path.is_file() + } + + # The first recovered PCL sees producer=None but no canonical summary. + # It is consumed as pending backend recovery, never as initial camera. + service._observe_published_runtime_envelope(frame, 23) # noqa: SLF001 + time.sleep(0.05) + still_unsealed = service.camera_preview.snapshot() + assert still_unsealed["generation"] == first_generation + assert still_unsealed["recording"]["completed_epochs"] == 0 + assert service._post_recovery_camera_restart_claim is None # noqa: SLF001 + assert service._post_recovery_camera_restart_inflight is None # noqa: SLF001 + assert service._camera_activation_lineage is None # noqa: SLF001 + assert invocation_count.read_text(encoding="utf-8") == "1" + assert { + path.relative_to(out_dir).as_posix(): path.read_bytes() + for path in out_dir.rglob("*") + if path.is_file() + } == before_files + + release_finalize.set() + assert watchdog_entered.wait(timeout=3) + sealed = service.camera_preview.snapshot() + assert sealed["recording"]["completed_epochs"] == 1 + assert sealed["recording"]["last_summary"]["schema_version"] == ( + "missioncore.camera-recording/v1" + ) + + service._observe_published_runtime_envelope(frame, 23) # noqa: SLF001 + _wait_until( + lambda: ( + service.camera_preview.snapshot()["generation"] + == first_generation + 1 + and service.camera_preview.snapshot()["recording"]["media_ready"] + is True + ) + ) + assert invocation_count.read_text(encoding="utf-8") == "2" + claim = service._post_recovery_camera_restart_claim # noqa: SLF001 + assert claim is not None + assert claim.camera_generation == first_generation + 1 + + # The delayed production watchdog now observes a stale G1 and cannot + # churn the PCL-admitted G2 successor. + release_watchdog.set() + time.sleep(0.05) + service._observe_published_runtime_envelope(frame, 23) # noqa: SLF001 + time.sleep(0.05) + assert service.camera_preview.snapshot()["generation"] == first_generation + 1 + assert invocation_count.read_text(encoding="utf-8") == "2" + finally: + release_finalize.set() + release_watchdog.set() + service.camera_preview.close() + service._release_camera_preview_process_lease() # noqa: SLF001 + + +def test_chained_stop_facade_cessation_uses_real_ledger_ancestry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _service(tmp_path, monkeypatch) + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + initial = _connection( + control_session_id="control-initial", + host_path_epoch=1, + producer_generation=1, + ) + _prepare_and_activate_checkpoint(service, initial) + ledger = service._physical_command_ledger # noqa: SLF001 + + ledger.prepare( + operation_id=FIRST_STOP_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=initial, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="2" * 64, + baseline_status=_status( + initial, + "scanning", + observed_at_utc="2026-08-13T12:01:00.000Z", + ), + ) + first_rebind = _connection( + control_session_id="control-first-rebind", + host_path_epoch=2, + producer_generation=2, + ) + first_reconciliation = ledger.reconcile_prepared_stop_not_dispatched( + FIRST_STOP_OPERATION_ID, + expected_revision=ledger.snapshot().record.revision, # type: ignore[union-attr] + reconciliation_id="classification-first-stop", + verified_binding=PhysicalCommandVerifiedBinding( + verification_id="verification-first-stop", + identity=identity, + connection=first_rebind, + device_info_message_sha256="3" * 64, + verified_at_utc="2026-08-13T12:01:01.000Z", + ), + observation=_status( + first_rebind, + "scanning", + source="explicit-read-only-reconciliation", + observed_at_utc="2026-08-13T12:01:02.000Z", + ), + ) + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + active = store.snapshot().checkpoint + assert active is not None + store.record_gap( + transition_id="record-gap-before-first-stop-rebind", + expected_revision=active.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=active.current_binding, + gap_started_at_utc="2026-08-13T12:01:00.500Z", + ) + active = store.snapshot().checkpoint + assert active is not None + first_binding = service._checkpoint_transport_binding( # noqa: SLF001 + identity_vendor_sha256=VENDOR_SHA256, + identity_serial_sha256=SERIAL_SHA256, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + connection=first_rebind.as_dict(), + verified_control=_verified_control(first_rebind), + ) + first_reconciliation_proof = first_reconciliation.reconciliations[-1] + store.rebind_active( + transition_id="rebind-after-first-stop-classification", + expected_revision=active.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=service._checkpoint_status_proof( # noqa: SLF001 + status=first_reconciliation_proof.observation, + binding=first_binding, + evidence_session_id=EVIDENCE_SESSION_ID, + ), + physical_proof=service._checkpoint_physical_proof( # noqa: SLF001 + record=first_reconciliation, + binding=first_binding, + checkpoint=active, + reconciliation=first_reconciliation_proof, + ), + ) + ledger.prepare( + operation_id=FINAL_STOP_OPERATION_ID, + parent_operation_id=FIRST_STOP_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=first_rebind, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="4" * 64, + baseline_status=_status( + first_rebind, + "scanning", + observed_at_utc="2026-08-13T12:01:03.000Z", + ), + operator_confirmation_id="confirmation-final-stop", + operator_confirmed_at_utc="2026-08-13T12:01:04.000Z", + ) + ledger.mark_dispatching(FINAL_STOP_OPERATION_ID) + ledger.mark_observing( + FINAL_STOP_OPERATION_ID, + publish_call_returned=True, + packet_id=42, + ) + ledger.mark_qos2_completed(FINAL_STOP_OPERATION_ID, packet_id=42) + ledger.record_application_response( + FINAL_STOP_OPERATION_ID, + PhysicalCommandApplicationResponse( + operation_id=FINAL_STOP_OPERATION_ID, + action="stop", + control_session_id=first_rebind.control_session_id, + host_path_epoch=first_rebind.host_path_epoch, + producer_generation=first_rebind.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="5" * 64, + observed_at_utc="2026-08-13T12:01:05.000Z", + ), + ) + ledger.record_status_observation( + FINAL_STOP_OPERATION_ID, + _status( + first_rebind, + "ready", + observed_at_utc="2026-08-13T12:01:06.000Z", + ), + ) + final_record = ledger.resolve( + FINAL_STOP_OPERATION_ID, + resolution="stop-standby-observed", + ) + assert first_reconciliation.operation_id == FIRST_STOP_OPERATION_ID + ancestry = ledger.prove_classified_stop_ancestry( + head_operation_id=FINAL_STOP_OPERATION_ID, + expected_start_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + ) + assert ancestry is not None + assert ancestry.head_parent_operation_id == FIRST_STOP_OPERATION_ID + + service._application_control_session.snapshot = lambda: { # type: ignore[method-assign] + "verified_control": _verified_control(first_rebind) + } + assert service._cease_active_acquisition_checkpoint_from_physical_head( # noqa: SLF001 + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + evidence_session_id=EVIDENCE_SESSION_ID, + expected_stop_operation_id=FINAL_STOP_OPERATION_ID, + require_reconciliation=False, + ) + checkpoint = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "ceased" + assert checkpoint.cessation_physical_proof is not None + assert checkpoint.cessation_physical_proof.operation_id == final_record.operation_id + assert checkpoint.cessation_physical_proof.ancestor_chain == ancestry + + +def test_restart_ready_classification_ceases_active_checkpoint_after_undispatched_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The live PREPARED/no-publish STOP shape settles locally after restart.""" + + service = _service(tmp_path, monkeypatch) + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + original = _connection( + control_session_id="control-before-undispatched-stop", + host_path_epoch=1, + producer_generation=1, + ) + _prepare_and_activate_checkpoint(service, original) + ledger = service._physical_command_ledger # noqa: SLF001 + ledger.prepare( + operation_id=FIRST_STOP_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=original, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="2" * 64, + baseline_status=_status( + original, + "scanning", + observed_at_utc="2026-08-13T12:01:00.000Z", + ), + ) + token = service._validate_active_acquisition_checkpoint_lineage() # noqa: SLF001 + assert token is not None + assert token.physical_record.operation_id == FIRST_STOP_OPERATION_ID + assert token.physical_record.stage == "prepared" + + restarted = _connection( + control_session_id="control-after-undispatched-stop", + host_path_epoch=2, + producer_generation=2, + ) + reconciliation_id = "restart-ready-undispatched-stop" + ledger.reconcile_prepared_stop_not_dispatched( + FIRST_STOP_OPERATION_ID, + expected_revision=ledger.snapshot().record.revision, # type: ignore[union-attr] + reconciliation_id=reconciliation_id, + verified_binding=PhysicalCommandVerifiedBinding( + verification_id=f"{reconciliation_id}.device-info", + identity=identity, + connection=restarted, + device_info_message_sha256="3" * 64, + verified_at_utc="2026-08-13T12:01:01.000Z", + ), + observation=_status( + restarted, + "ready", + source="explicit-read-only-reconciliation", + observed_at_utc="2026-08-13T12:01:02.000Z", + ), + ) + # Simulate the exact live crash boundary: the physical classification is + # durable but the prior process exited before checkpoint cessation. + restarted_service = _service(tmp_path, monkeypatch) + assert restarted_service._active_acquisition_checkpoint_reason is None # noqa: SLF001 + checkpoint = ActiveAcquisitionRecoveryCheckpointStore(tmp_path).snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "ceased" + assert checkpoint.cessation_status_proof is not None + assert checkpoint.cessation_status_proof.session_state == "ready" + assert checkpoint.cessation_physical_proof is not None + assert checkpoint.cessation_physical_proof.operation_id == FIRST_STOP_OPERATION_ID + assert checkpoint.cessation_physical_proof.resolution == "not-dispatched" + assert checkpoint.cessation_physical_proof.reconciliation_kind == ( + "prepared-stop-classification" + ) + assert checkpoint.cessation_physical_proof.stop_fence == ( + "definitely-not-dispatched" + ) + + +def test_corrupt_checkpoint_is_redacted_and_grants_no_authority( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + store = ActiveAcquisitionRecoveryCheckpointStore(tmp_path) + store.path.write_text( + json.dumps({"secret_ip": "192.168.68.52", "uuid": "private-device-uuid"}), + encoding="utf-8", + ) + store.path.chmod(0o600) + + service = _service(tmp_path, monkeypatch) + projection = service._active_acquisition_checkpoint_public_snapshot() # noqa: SLF001 + + assert projection == { + "status": "corrupt", + "revision": None, + "state": None, + "transport_revision": None, + "gap_open": False, + "restart_resume_authority": False, + "restart_rehydration_phase": "inactive", + "restart_rehydration_reason_code": None, + "reason_code": "active-acquisition-recovery-checkpoint-corrupt", + } + encoded = json.dumps(projection, sort_keys=True) + assert "192.168.68.52" not in encoded + assert "private-device-uuid" not in encoded + + +def test_project_wire_hash_helper_is_exact_utf8() -> None: + assert active_acquisition_project_name_sha256("Проект K1") == hashlib.sha256( + "Проект K1".encode() + ).hexdigest() + + +def test_corrupt_failed_start_settlement_never_reports_checkpoint_safe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _service(tmp_path, monkeypatch) + service._active_acquisition_checkpoint = SimpleNamespace( # type: ignore[assignment] # noqa: SLF001 + snapshot=lambda: SimpleNamespace(status="corrupt", checkpoint=None) + ) + context = PhysicalCommandIntentContext( + operation_id=START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + ) + envelope = OneShotPublishEnvelope( + operation_key="modeling:start", + topic="lixel/application/request/modeling", + payload=START_PAYLOAD, + payload_sha256=START_PAYLOAD_SHA256, + payload_bytes=len(START_PAYLOAD), + qos=2, + retain=False, + ) + + with pytest.raises(Exception, match="durable no-dispatch|unavailable or corrupt"): + service._observe_start_preparation_checkpoint( # noqa: SLF001 + "resolved-not-dispatched", + context, + envelope, + ) + + +def test_camera_commit_requires_current_active_checkpoint_with_closed_gap( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _service(tmp_path, monkeypatch) + connection = _connection( + control_session_id="control-camera-checkpoint", + host_path_epoch=7, + producer_generation=9, + ) + _prepare_and_activate_checkpoint(service, connection) + committed: list[str] = [] + monkeypatch.setattr( + service.runtime, + "snapshot", + lambda: { + "phase": "live", + "source_mode": "live", + "source_ready": True, + "producer_generation": 17, + "connection_recovery": {"state": "inactive"}, + }, + ) + + assert service._commit_camera_restart_if_still_active( # noqa: SLF001 + acquisition_id=ACQUISITION_ID, + evidence_session_id=EVIDENCE_SESSION_ID, + start_operation_id=START_OPERATION_ID, + runtime_producer_generation=17, + commit=lambda: committed.append("closed-gap") or True, + ) + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + store.record_gap( + transition_id="camera-open-gap-fence", + expected_revision=checkpoint.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=checkpoint.current_binding, + gap_started_at_utc="2026-08-13T12:02:00.000Z", + ) + + assert not service._commit_camera_restart_if_still_active( # noqa: SLF001 + acquisition_id=ACQUISITION_ID, + evidence_session_id=EVIDENCE_SESSION_ID, + start_operation_id=START_OPERATION_ID, + runtime_producer_generation=17, + commit=lambda: committed.append("open-gap") or True, + ) + assert committed == ["closed-gap"] + + +def test_scanning_activation_retry_accepts_exact_post_commit_active_checkpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = _service(tmp_path, monkeypatch) + connection = _connection( + control_session_id="control-activation-post-commit", + host_path_epoch=11, + producer_generation=12, + ) + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + durable_activate = store.activate + post_commit_raised = False + + def activate_then_raise(**kwargs: object) -> object: + nonlocal post_commit_raised + committed = durable_activate(**kwargs) # type: ignore[arg-type] + if not post_commit_raised: + post_commit_raised = True + raise OSError("injected post-commit activation return failure") + return committed + + monkeypatch.setattr(store, "activate", activate_then_raise) + with pytest.raises(OSError, match="post-commit activation"): + _prepare_and_activate_checkpoint(service, connection) + committed = store.snapshot().checkpoint + assert committed is not None + assert committed.state == "active" + + # The session's STOP-admission retry must recognize this exact durable + # postcondition rather than permanently treating ACTIVE as a stale PREPARED. + service._observe_application_scanning_checkpoint() # noqa: SLF001 + assert store.snapshot().checkpoint == committed diff --git a/tests/test_xgrids_active_acquisition_recovery_checkpoint.py b/tests/test_xgrids_active_acquisition_recovery_checkpoint.py new file mode 100644 index 0000000..d524b75 --- /dev/null +++ b/tests/test_xgrids_active_acquisition_recovery_checkpoint.py @@ -0,0 +1,3095 @@ +from __future__ import annotations + +import json +import multiprocessing +import os +import stat +import threading +import time +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from k1link.device_plugins.xgrids_k1 import ( + active_acquisition_recovery_checkpoint as checkpoint_module, +) +from k1link.device_plugins.xgrids_k1.active_acquisition_recovery_checkpoint import ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA, + ActiveAcquisitionRecoveryCheckpointCorrupt, + ActiveAcquisitionRecoveryCheckpointIdempotencyConflict, + ActiveAcquisitionRecoveryCheckpointStore, + ActiveAcquisitionRecoveryCheckpointTransitionError, + ActiveAcquisitionRecoveryConnection, + ActiveAcquisitionRecoveryFirstPublishedPclProof, + ActiveAcquisitionRecoveryIdentity, + ActiveAcquisitionRecoveryPhysicalLineageProof, + ActiveAcquisitionRecoveryStartOriginProof, + ActiveAcquisitionRecoveryStatusProof, + ActiveAcquisitionRecoveryTransportBinding, + StaleActiveAcquisitionRecoveryCheckpoint, + active_acquisition_project_name_sha256, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + PhysicalCommandClassifiedStopAncestryProof, +) + +ACQUISITION_ID = "acquisition-001" +START_OPERATION_ID = "operation-start-001" +START_PAYLOAD_SHA256 = "1" * 64 +STOP_PAYLOAD_SHA256 = "2" * 64 +ACTIVE_PROJECT_ID_SHA256 = "3" * 64 +STATUS_SCANNING_SHA256 = "4" * 64 +STATUS_READY_SHA256 = "5" * 64 +PROFILE_ID = "xgrids-k1-openapi-v1" +PROJECT_NAME = "Mission Alpha" +ORIGINAL_EVIDENCE_SESSION_ID = "evidence-session-001" + +IDENTITY = ActiveAcquisitionRecoveryIdentity( + logical_device_id="k1-logical-001", + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, +) +CONNECTION = ActiveAcquisitionRecoveryConnection( + transport_ref="ble-transport-001", + connection_mode="bridge", + target_ipv4="192.168.43.1", + target_port=1883, +) + + +class _Clock: + def __init__(self, *values: datetime) -> None: + self._values = list(values) or [datetime(2026, 8, 13, 12, tzinfo=UTC)] + self._index = 0 + + def __call__(self) -> datetime: + value = self._values[min(self._index, len(self._values) - 1)] + self._index += 1 + return value + + +def _binding( + *, + runtime_instance_id: str = "runtime-A", + host_path_epoch: int = 10, + control_session_id: str = "control-A", + producer_generation: int = 7, +) -> ActiveAcquisitionRecoveryTransportBinding: + return ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=runtime_instance_id, + intent_id="intent-001", + transport_ref=CONNECTION.transport_ref, + connection_mode=CONNECTION.connection_mode, + target_ipv4=CONNECTION.target_ipv4, + target_port=CONNECTION.target_port, + host_path_epoch=host_path_epoch, + control_session_id=control_session_id, + producer_generation=producer_generation, + logical_device_id=IDENTITY.logical_device_id, + compatibility_profile_id=PROFILE_ID, + vendor_device_id_sha256=IDENTITY.vendor_device_id_sha256, + device_serial_sha256=IDENTITY.device_serial_sha256, + ) + + +def _status( + state: str, + *, + binding: ActiveAcquisitionRecoveryTransportBinding, + evidence_session_id: str, + observed_at: str, + project_id_sha256: str | None = None, + status_message_sha256: str | None = None, +) -> ActiveAcquisitionRecoveryStatusProof: + scanning = state == "scanning" + return ActiveAcquisitionRecoveryStatusProof( + source="explicit-read-only-reconciliation", + session_state=state, # type: ignore[arg-type] + session_state_code={"scanning": 302, "ready": 300, "scan_over": 304}[state], + binding=binding, + evidence_session_id=evidence_session_id, + project_id_sha256=( + project_id_sha256 or ACTIVE_PROJECT_ID_SHA256 if scanning else None + ), + init_ready=scanning, + status_message_sha256=( + status_message_sha256 + or (STATUS_SCANNING_SHA256 if scanning else STATUS_READY_SHA256) + ), + mqtt_retained=False, + observed_at_utc=observed_at, + ) + + +def _physical( + resolution: str, + *, + binding: ActiveAcquisitionRecoveryTransportBinding, + proof_id: str, + observed_at: str, + operation_id: str | None = None, + payload_sha256: str | None = None, + ledger_revision: int | None = None, + ancestor_chain: PhysicalCommandClassifiedStopAncestryProof | None = None, +) -> ActiveAcquisitionRecoveryPhysicalLineageProof: + if resolution == "not-dispatched": + action = "start" + head_resolution = "not-dispatched" + reconciliation_kind = None + reconciliation_resolution = None + state = None + digest = None + composite = False + operation = START_OPERATION_ID + parent_operation = None + payload = START_PAYLOAD_SHA256 + stop_fence = "none" + elif resolution in {"start-active-observed", "physical-active-observed"}: + action = "start" + head_resolution = "start-active-observed" + reconciliation_kind = ( + "resolved-active-rebind" + if resolution == "physical-active-observed" + else None + ) + reconciliation_resolution = ( + "physical-active-observed" + if resolution == "physical-active-observed" + else None + ) + state = "scanning" + digest = STATUS_SCANNING_SHA256 + composite = True + operation = START_OPERATION_ID + parent_operation = None + payload = START_PAYLOAD_SHA256 + stop_fence = "none" + elif resolution == "stop-standby-observed": + action = "stop" + head_resolution = "stop-standby-observed" + reconciliation_kind = None + reconciliation_resolution = None + state = "ready" + digest = STATUS_READY_SHA256 + composite = True + operation = "operation-stop-001" + parent_operation = START_OPERATION_ID + payload = STOP_PAYLOAD_SHA256 + stop_fence = "none" + else: + assert resolution == "physical-standby-observed" + action = "start" + head_resolution = "physical-standby-observed" + reconciliation_kind = "ambiguous-outcome" + reconciliation_resolution = "physical-standby-observed" + state = "ready" + digest = STATUS_READY_SHA256 + composite = False + operation = START_OPERATION_ID + parent_operation = None + payload = START_PAYLOAD_SHA256 + stop_fence = "none" + default_ledger_revision = { + "not-dispatched": 20, + "start-active-observed": 20, + "physical-active-observed": 21, + "stop-standby-observed": 22, + "physical-standby-observed": 21, + }[resolution] + return ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=ledger_revision or default_ledger_revision, + proof_id=proof_id, + operation_id=( + ancestor_chain.head_operation_id + if ancestor_chain is not None + else operation_id or operation + ), + original_start_operation_id=START_OPERATION_ID, + parent_operation_id=( + ancestor_chain.head_parent_operation_id + if ancestor_chain is not None + else parent_operation + ), + acquisition_id=ACQUISITION_ID, + action=action, # type: ignore[arg-type] + resolution=head_resolution, # type: ignore[arg-type] + payload_sha256=payload_sha256 or payload, + original_start_payload_sha256=START_PAYLOAD_SHA256, + reconciliation_kind=reconciliation_kind, # type: ignore[arg-type] + reconciliation_resolution=reconciliation_resolution, # type: ignore[arg-type] + status_message_sha256=digest, + observed_session_state=state, # type: ignore[arg-type] + binding=binding, + composite_complete=composite, + edge_terminal=True, + late_start_excluded=True, + stop_fence=stop_fence, # type: ignore[arg-type] + observed_at_utc=observed_at, + ancestor_chain=ancestor_chain, + ) + + +def _undispatched_stop_active( + *, + binding: ActiveAcquisitionRecoveryTransportBinding, + proof_id: str, + observed_at: str, + reconciliation_kind: str, + ledger_revision: int, +) -> ActiveAcquisitionRecoveryPhysicalLineageProof: + return ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=ledger_revision, + proof_id=proof_id, + operation_id="operation-stop-undispatched-001", + original_start_operation_id=START_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + action="stop", + resolution="not-dispatched", + payload_sha256=STOP_PAYLOAD_SHA256, + original_start_payload_sha256=START_PAYLOAD_SHA256, + reconciliation_kind=reconciliation_kind, # type: ignore[arg-type] + reconciliation_resolution="physical-active-observed", + status_message_sha256=STATUS_SCANNING_SHA256, + observed_session_state="scanning", + binding=binding, + composite_complete=False, + edge_terminal=True, + late_start_excluded=True, + stop_fence="definitely-not-dispatched", + observed_at_utc=observed_at, + ) + + +def _store( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + clock: _Clock | None = None, +) -> ActiveAcquisitionRecoveryCheckpointStore: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return ActiveAcquisitionRecoveryCheckpointStore( + tmp_path / "repository", + clock=clock, + ) + + +def _prepare_kwargs( + binding: ActiveAcquisitionRecoveryTransportBinding, + *, + transition_id: str = "transition-prepare-001", +) -> dict[str, Any]: + return { + "transition_id": transition_id, + "predecessor_revision": 0, + "acquisition_id": ACQUISITION_ID, + "original_start_operation_id": START_OPERATION_ID, + "start_payload_sha256": START_PAYLOAD_SHA256, + "identity": IDENTITY, + "connection": CONNECTION, + "compatibility_profile_id": PROFILE_ID, + "project_name": PROJECT_NAME, + "project_name_wire_sha256": active_acquisition_project_name_sha256( + PROJECT_NAME + ), + "original_evidence_session_id": ORIGINAL_EVIDENCE_SESSION_ID, + "duration_seconds": None, + "requested_streams": ( + "spatial.point-cloud.live", + "camera.rgb.live", + ), + "evidence_policy": "required", + "mount_type": "handheld", + "gnss_mode": "none", + "prepared_binding": binding, + } + + +def _prepare( + store: ActiveAcquisitionRecoveryCheckpointStore, + binding: ActiveAcquisitionRecoveryTransportBinding, +) -> Any: + return store.prepare(**_prepare_kwargs(binding)) + + +def _ancestry( + *, + head_operation_id: str = "operation-stop-successor-001", + head_parent_operation_id: str = "operation-stop-undispatched-001", + ledger_revision: int = 22, + depth: int = 1, +) -> PhysicalCommandClassifiedStopAncestryProof: + return PhysicalCommandClassifiedStopAncestryProof( + schema_version=PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + ledger_revision=ledger_revision, + head_operation_id=head_operation_id, + head_parent_operation_id=head_parent_operation_id, + root_start_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + depth=depth, + chain_sha256="9" * 64, + ) + + +def _activate( + store: ActiveAcquisitionRecoveryCheckpointStore, + binding: ActiveAcquisitionRecoveryTransportBinding, +) -> Any: + status = _status( + "scanning", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:00:01.000Z", + ) + return store.activate( + transition_id="transition-activate-001", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=_physical( + "start-active-observed", + binding=binding, + proof_id="physical-start-active-001", + observed_at=status.observed_at_utc, + ), + ) + + +def _reconciled_binding() -> ActiveAcquisitionRecoveryTransportBinding: + return _binding( + runtime_instance_id="runtime-restarted", + host_path_epoch=1, + control_session_id="control-restarted", + producer_generation=1, + ) + + +def _baseline_origin_status( + binding: ActiveAcquisitionRecoveryTransportBinding, +) -> ActiveAcquisitionRecoveryStatusProof: + return ActiveAcquisitionRecoveryStatusProof( + source="live-control-session", + session_state="ready", + session_state_code=300, + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + project_id_sha256=None, + init_ready=False, + status_message_sha256=STATUS_READY_SHA256, + mqtt_retained=False, + observed_at_utc="2026-08-13T11:59:59.000Z", + ) + + +def _reconciled_physical( + *, + origin_kind: str, + binding: ActiveAcquisitionRecoveryTransportBinding, + proof_id: str, + observed_at: str, + ledger_revision: int = 21, + reconciliation_kind: str | None = None, +) -> ActiveAcquisitionRecoveryPhysicalLineageProof: + ambiguous = origin_kind == "ambiguous-reconciled" + projected_reconciliation_kind = reconciliation_kind or ( + "ambiguous-outcome" if ambiguous else "resolved-active-rebind" + ) + return ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=ledger_revision, + proof_id=proof_id, + operation_id=START_OPERATION_ID, + original_start_operation_id=START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + action="start", + resolution=( + "physical-active-observed" if ambiguous else "start-active-observed" + ), + payload_sha256=START_PAYLOAD_SHA256, + original_start_payload_sha256=START_PAYLOAD_SHA256, + reconciliation_kind=projected_reconciliation_kind, # type: ignore[arg-type] + reconciliation_resolution="physical-active-observed", + status_message_sha256=STATUS_SCANNING_SHA256, + observed_session_state="scanning", + binding=binding, + composite_complete=not ambiguous, + edge_terminal=True, + late_start_excluded=True, + stop_fence="none", + observed_at_utc=observed_at, + reconciliation_original_attempt_sha256="8" * 64, + ) + + +def _origin( + *, + origin_kind: str, + prepared_binding: ActiveAcquisitionRecoveryTransportBinding, + physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof, +) -> ActiveAcquisitionRecoveryStartOriginProof: + composite = origin_kind == "composite-resolved" + return ActiveAcquisitionRecoveryStartOriginProof( + schema_version=ACTIVE_ACQUISITION_RECOVERY_START_ORIGIN_PROOF_SCHEMA, + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=physical_proof.ledger_revision, + physical_proof_id=physical_proof.proof_id, + reconciliation_id=f"reconciliation-{origin_kind}", + origin_kind=origin_kind, # type: ignore[arg-type] + original_attempt_revision=20, + original_attempt_stage="resolved" if composite else "observing", + original_attempt_resolution=( + "start-active-observed" if composite else None + ), + original_attempt_sha256="8" * 64, + operation_id=START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + payload_sha256=START_PAYLOAD_SHA256, + baseline_status_proof=_baseline_origin_status(prepared_binding), + original_project_id_sha256=( + ACTIVE_PROJECT_ID_SHA256 if composite else None + ), + automatic_replay_allowed=False, + project_evidence_strength=( + "exact-vendor-project-id" + if composite + else "edge-correlated-vendor-project-id" + ), + reconciled_active_project_id_sha256=ACTIVE_PROJECT_ID_SHA256, + ) + + +def _first_pcl( + binding: ActiveAcquisitionRecoveryTransportBinding, + *, + evidence_session_id: str = "evidence-restarted", +) -> ActiveAcquisitionRecoveryFirstPublishedPclProof: + return ActiveAcquisitionRecoveryFirstPublishedPclProof( + schema_version=ACTIVE_ACQUISITION_RECOVERY_FIRST_PUBLISHED_PCL_PROOF_SCHEMA, + proof_id="first-pcl-restarted", + runtime_instance_id=binding.runtime_instance_id, + evidence_session_id=evidence_session_id, + viewer_producer_generation=42, + frame_sequence=1, + captured_at_epoch_ns=1_786_619_000_000_000_000, + point_count=10, + published_pcl_frames=1, + published_at_utc="2026-08-13T12:00:03.000Z", + ) + + +def _activate_reconciled_kwargs( + prepared_binding: ActiveAcquisitionRecoveryTransportBinding, + *, + origin_kind: str, + reconciliation_kind: str | None = None, +) -> dict[str, Any]: + binding = _reconciled_binding() + status = _status( + "scanning", + binding=binding, + evidence_session_id="evidence-restarted", + observed_at="2026-08-13T12:00:02.000Z", + ) + physical = _reconciled_physical( + origin_kind=origin_kind, + binding=binding, + proof_id=f"physical-{origin_kind}", + observed_at=status.observed_at_utc, + reconciliation_kind=reconciliation_kind, + ) + return { + "transition_id": f"transition-activate-{origin_kind}", + "expected_revision": 1, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "gap_started_at_utc": "2026-08-13T12:00:00.000Z", + "origin_proof": _origin( + origin_kind=origin_kind, + prepared_binding=prepared_binding, + physical_proof=physical, + ), + "status_proof": status, + "physical_proof": physical, + "first_pcl_proof": _first_pcl(binding), + } + + +def _cease_prepared_resolved_start_standby_kwargs( + prepared_binding: ActiveAcquisitionRecoveryTransportBinding, + *, + terminal_state: str, +) -> dict[str, Any]: + binding = _reconciled_binding() + status = _status( + terminal_state, + binding=binding, + evidence_session_id="evidence-restarted", + observed_at="2026-08-13T12:00:03.000Z", + ) + physical = ActiveAcquisitionRecoveryPhysicalLineageProof( + ledger_schema_version=ACTIVE_ACQUISITION_RECOVERY_PHYSICAL_LEDGER_SCHEMA, + ledger_revision=21, + proof_id="physical-composite-resolved-standby", + operation_id=START_OPERATION_ID, + original_start_operation_id=START_OPERATION_ID, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + action="start", + resolution="start-active-observed", + payload_sha256=START_PAYLOAD_SHA256, + original_start_payload_sha256=START_PAYLOAD_SHA256, + reconciliation_kind="resolved-active-cessation", + reconciliation_resolution="physical-standby-observed", + status_message_sha256=status.status_message_sha256, + observed_session_state=status.session_state, + binding=binding, + composite_complete=True, + edge_terminal=True, + late_start_excluded=True, + stop_fence="none", + observed_at_utc=status.observed_at_utc, + ) + origin = _origin( + origin_kind="composite-resolved", + prepared_binding=prepared_binding, + physical_proof=physical, + ) + assert origin.original_project_id_sha256 is not None + return { + "transition_id": ( + f"transition-cease-prepared-resolved-start-{terminal_state}" + ), + "expected_revision": 1, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "gap_started_at_utc": "2026-08-13T12:00:00.000Z", + "origin_proof": origin, + "cessation_status_proof": status, + "cessation_physical_proof": physical, + "reconciliation_id": origin.reconciliation_id, + "reconciliation_original_attempt_sha256": ( + origin.original_attempt_sha256 + ), + "reconciliation_original_project_id_sha256": ( + origin.original_project_id_sha256 + ), + } + + +def _cease_active_reconciled_standby_kwargs( + active: Any, + *, + terminal_state: str, + gap_started_at_utc: str = "2026-08-13T12:05:00.000Z", + transition_id: str | None = None, +) -> dict[str, Any]: + binding = _binding( + runtime_instance_id="runtime-active-restarted", + host_path_epoch=1, + control_session_id="control-active-restarted", + producer_generation=1, + ) + status = _status( + terminal_state, + binding=binding, + evidence_session_id="evidence-active-restarted", + observed_at="2026-08-13T12:05:02.000Z", + ) + prior_physical = active.current_active_physical_proof + assert prior_physical is not None + physical = replace( + prior_physical, + ledger_revision=prior_physical.ledger_revision + 1, + proof_id=f"physical-active-restart-{terminal_state}", + reconciliation_kind="resolved-active-cessation", + reconciliation_resolution="physical-standby-observed", + status_message_sha256=status.status_message_sha256, + observed_session_state=status.session_state, + binding=binding, + observed_at_utc=status.observed_at_utc, + ) + return { + "transition_id": transition_id + or f"transition-cease-active-restart-{terminal_state}", + "expected_revision": active.revision, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "gap_started_at_utc": gap_started_at_utc, + "cessation_status_proof": status, + "cessation_physical_proof": physical, + } + + +def test_exact_secret_free_roundtrip_preserves_requested_contract( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + kwargs = _prepare_kwargs(binding) + kwargs["duration_seconds"] = 3600 + prepared = store.prepare(**kwargs) + + assert prepared.state == "prepared" + assert prepared.transport_revision == 1 + assert prepared.duration_seconds == 3600 + assert prepared.original_evidence_session_id == ORIGINAL_EVIDENCE_SESSION_ID + assert prepared.current_evidence_session_id == ORIGINAL_EVIDENCE_SESSION_ID + assert prepared.restart_resume_authority is False + raw = store.path.read_text(encoding="utf-8") + assert json.loads(raw) == prepared.as_dict() + assert raw.count('"schema_version"') == 1 + assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 + assert stat.S_IMODE(store.path.parent.stat().st_mode) == 0o700 + assert store.path.stat().st_nlink == 1 + for forbidden in ("password", "credential", "private_key", "access_token"): + assert forbidden not in raw.casefold() + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + assert restarted.snapshot().checkpoint == prepared + assert restarted.snapshot().restart_resume_authority is False + + +@pytest.mark.parametrize( + ("origin_kind", "evidence_strength"), + ( + ("composite-resolved", "exact-vendor-project-id"), + ("ambiguous-reconciled", "edge-correlated-vendor-project-id"), + ), +) +def test_activate_reconciled_atomically_promotes_after_first_pcl( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + origin_kind: str, + evidence_strength: str, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind=origin_kind, + ) + + active = store.activate_reconciled(**kwargs) + + assert active.state == "active" + assert active.revision == prepared.revision + 1 + assert active.transport_revision == 2 + assert active.current_binding == kwargs["status_proof"].binding + assert active.current_evidence_session_id == "evidence-restarted" + assert active.last_gap_failed_binding == prepared_binding + assert active.last_gap_started_transport_revision == 1 + assert active.last_gap_recovered_transport_revision == 2 + assert active.activated_at_utc == kwargs["first_pcl_proof"].published_at_utc + assert active.reconciled_start_origin_proof is not None + assert ( + active.reconciled_start_origin_proof.project_evidence_strength + == evidence_strength + ) + assert active.first_published_pcl_proof == kwargs["first_pcl_proof"] + assert active.restart_resume_authority is False + assert store.activate_reconciled(**kwargs) == active + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == active + ) + + +def test_legacy_reconciled_proofs_without_descendant_certificates_stay_readable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, prepared_binding) + active = store.activate_reconciled( + **_activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + ) + ) + document = active.as_dict() + for field_name in ( + "activation_physical_proof", + "current_active_physical_proof", + ): + proof = document[field_name] + assert isinstance(proof, dict) + proof.pop("reconciliation_original_attempt_sha256") + origin = document["reconciled_start_origin_proof"] + assert isinstance(origin, dict) + origin.pop("reconciled_active_project_id_sha256") + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + loaded = restarted.snapshot().checkpoint + + assert loaded is not None + assert loaded.as_dict() == document + assert ( + loaded.current_active_physical_proof.reconciliation_original_attempt_sha256 + is None + ) + assert loaded.reconciled_start_origin_proof is not None + assert ( + loaded.reconciled_start_origin_proof.reconciled_active_project_id_sha256 + is None + ) + + +def test_activate_reconciled_rejects_wrong_runtime_session_pcl_and_conflict( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + ) + + for field, replacement in ( + ( + "status_proof", + replace( + kwargs["status_proof"], + binding=replace( + kwargs["status_proof"].binding, + runtime_instance_id=prepared_binding.runtime_instance_id, + ), + ), + ), + ( + "status_proof", + replace( + kwargs["status_proof"], + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + ), + ), + ( + "first_pcl_proof", + replace( + kwargs["first_pcl_proof"], + evidence_session_id="evidence-wrong", + ), + ), + ): + invalid = dict(kwargs) + invalid[field] = replacement + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.activate_reconciled(**invalid) + assert store.snapshot().checkpoint == prepared + + active = store.activate_reconciled(**kwargs) + conflicting = dict(kwargs) + conflicting["gap_started_at_utc"] = "2026-08-13T11:59:00.000Z" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointIdempotencyConflict): + store.activate_reconciled(**conflicting) + assert store.snapshot().checkpoint == active + + +def test_ambiguous_prepared_accepts_only_exact_resolved_rebind_descendant( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + reconciliation_kind="resolved-active-rebind", + ) + physical = kwargs["physical_proof"] + origin = kwargs["origin_proof"] + status = kwargs["status_proof"] + + forged_resolved = dict(kwargs) + forged_resolved["physical_proof"] = replace( + physical, + resolution="start-active-observed", + composite_complete=True, + ) + + mismatched_attempt = dict(kwargs) + mismatched_attempt["physical_proof"] = replace( + physical, + reconciliation_original_attempt_sha256="9" * 64, + ) + + mismatched_project = dict(kwargs) + mismatched_project["origin_proof"] = replace( + origin, + reconciled_active_project_id_sha256="9" * 64, + ) + + wrong_binding = replace(status.binding, target_port=status.binding.target_port + 1) + mismatched_binding = dict(kwargs) + mismatched_binding["status_proof"] = replace(status, binding=wrong_binding) + mismatched_binding["physical_proof"] = replace( + physical, + binding=wrong_binding, + ) + + missing_descendant_certificate = dict(kwargs) + missing_descendant_certificate["physical_proof"] = replace( + physical, + reconciliation_original_attempt_sha256=None, + ) + + for index, invalid in enumerate( + ( + forged_resolved, + mismatched_attempt, + mismatched_project, + mismatched_binding, + missing_descendant_certificate, + ) + ): + invalid["transition_id"] = f"transition-invalid-ambiguous-rebind-{index}" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.activate_reconciled(**invalid) + assert store.snapshot().checkpoint == prepared + + active = store.activate_reconciled(**kwargs) + + assert active.state == "active" + assert active.active_project_id_sha256 == ACTIVE_PROJECT_ID_SHA256 + assert active.reconciled_start_origin_proof == origin + assert active.reconciled_start_origin_proof.automatic_replay_allowed is False + assert active.restart_resume_authority is False + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == active + ) + + +def test_start_origin_rejects_bad_or_retained_non_ready_baseline() -> None: + prepared_binding = _binding() + kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + ) + origin = kwargs["origin_proof"] + baseline = origin.baseline_status_proof + scanning = _status( + "scanning", + binding=prepared_binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at=baseline.observed_at_utc, + ) + with pytest.raises(ValueError, match="baseline"): + replace(origin, baseline_status_proof=scanning) + with pytest.raises(ValueError, match="non-retained"): + replace(baseline, mqtt_retained=True) + + +@pytest.mark.parametrize("terminal_state", ("ready", "scan_over")) +def test_cease_prepared_resolved_start_standby_is_terminal_without_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + terminal_state: str, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + kwargs = _cease_prepared_resolved_start_standby_kwargs( + prepared_binding, + terminal_state=terminal_state, + ) + + ceased = store.cease_prepared_resolved_start_standby(**kwargs) + + assert ceased.state == "ceased" + assert ceased.revision == prepared.revision + 1 + assert ceased.transport_revision == prepared.transport_revision + 1 + assert ceased.physical_lineage_head_revision == 21 + assert ceased.active_project_id_sha256 == ACTIVE_PROJECT_ID_SHA256 + assert ceased.current_binding == kwargs["cessation_status_proof"].binding + assert ceased.current_evidence_session_id == "evidence-restarted" + assert ceased.activated_at_utc is None + assert ceased.activation_status_proof is None + assert ceased.activation_physical_proof is None + assert ceased.current_active_status_proof is None + assert ceased.current_active_physical_proof is None + assert ceased.last_rebind_status_proof is None + assert ceased.last_rebind_physical_proof is None + assert ceased.first_published_pcl_proof is None + assert ceased.prepared_resolution_proof == kwargs["cessation_physical_proof"] + assert ceased.cessation_status_proof == kwargs["cessation_status_proof"] + assert ceased.cessation_physical_proof == kwargs["cessation_physical_proof"] + assert ceased.last_gap_started_at_utc == kwargs["gap_started_at_utc"] + assert ceased.last_gap_started_transport_revision == 1 + assert ceased.last_gap_failed_binding == prepared_binding + assert ceased.last_gap_recovered_at_utc == ( + kwargs["cessation_status_proof"].observed_at_utc + ) + assert ceased.last_gap_recovered_transport_revision == 2 + assert ceased.prepared_resolved_start_reconciliation_id == ( + kwargs["reconciliation_id"] + ) + assert ceased.prepared_resolved_start_original_attempt_sha256 == ( + kwargs["reconciliation_original_attempt_sha256"] + ) + assert ceased.prepared_resolved_start_original_project_id_sha256 == ( + kwargs["reconciliation_original_project_id_sha256"] + ) + assert ceased.transition_receipts[-1].kind == ( + "cease-prepared-resolved-start-standby" + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + assert restarted.snapshot().checkpoint == ceased + assert restarted.cease_prepared_resolved_start_standby(**kwargs) == ceased + + conflicting = dict(kwargs) + conflicting["gap_started_at_utc"] = "2026-08-13T11:59:59.000Z" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointIdempotencyConflict): + restarted.cease_prepared_resolved_start_standby(**conflicting) + stale = dict(kwargs) + stale["transition_id"] = "transition-cease-prepared-resolved-start-stale" + with pytest.raises(StaleActiveAcquisitionRecoveryCheckpoint): + restarted.cease_prepared_resolved_start_standby(**stale) + assert restarted.snapshot().checkpoint == ceased + + +def test_cease_prepared_resolved_start_standby_rejects_inexact_restart_proofs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + kwargs = _cease_prepared_resolved_start_standby_kwargs( + prepared_binding, + terminal_state="ready", + ) + status = kwargs["cessation_status_proof"] + physical = kwargs["cessation_physical_proof"] + origin = kwargs["origin_proof"] + + invalid_requests: list[dict[str, Any]] = [] + for binding in ( + replace( + status.binding, + runtime_instance_id=prepared_binding.runtime_instance_id, + ), + replace( + status.binding, + control_session_id=prepared_binding.control_session_id, + ), + ): + invalid = dict(kwargs) + invalid["cessation_status_proof"] = replace(status, binding=binding) + invalid["cessation_physical_proof"] = replace(physical, binding=binding) + invalid_requests.append(invalid) + + same_evidence = dict(kwargs) + same_evidence["cessation_status_proof"] = replace( + status, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + ) + invalid_requests.append(same_evidence) + + non_explicit = dict(kwargs) + non_explicit["cessation_status_proof"] = replace( + status, + source="live-control-session", + ) + invalid_requests.append(non_explicit) + + wrong_attempt_sha = dict(kwargs) + wrong_attempt_sha["origin_proof"] = replace( + origin, + original_attempt_sha256="7" * 64, + ) + invalid_requests.append(wrong_attempt_sha) + + wrong_payload = dict(kwargs) + wrong_payload["origin_proof"] = replace( + origin, + payload_sha256="7" * 64, + ) + invalid_requests.append(wrong_payload) + + wrong_project = dict(kwargs) + wrong_project["origin_proof"] = replace( + origin, + original_project_id_sha256="6" * 64, + ) + invalid_requests.append(wrong_project) + + wrong_proof = dict(kwargs) + wrong_proof["origin_proof"] = replace( + origin, + physical_proof_id="physical-wrong", + ) + invalid_requests.append(wrong_proof) + + wrong_reconciliation = dict(kwargs) + wrong_reconciliation["origin_proof"] = replace( + origin, + reconciliation_id="reconciliation-wrong", + ) + invalid_requests.append(wrong_reconciliation) + + wrong_physical_payload = dict(kwargs) + wrong_physical_payload["cessation_physical_proof"] = replace( + physical, + payload_sha256="7" * 64, + original_start_payload_sha256="7" * 64, + ) + invalid_requests.append(wrong_physical_payload) + + for invalid in invalid_requests: + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease_prepared_resolved_start_standby(**invalid) + assert store.snapshot().checkpoint == prepared + + with pytest.raises(ValueError, match="non-retained"): + replace(status, mqtt_retained=True) + + +@pytest.mark.parametrize("terminal_state", ("ready", "scan_over")) +def test_cease_active_reconciled_standby_atomically_opens_and_closes_gap( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + terminal_state: str, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + active = _activate(store, binding) + kwargs = _cease_active_reconciled_standby_kwargs( + active, + terminal_state=terminal_state, + ) + + ceased = store.cease_active_reconciled_standby(**kwargs) + + status = kwargs["cessation_status_proof"] + physical = kwargs["cessation_physical_proof"] + assert ceased.state == "ceased" + assert ceased.revision == active.revision + 1 + assert ceased.transport_revision == active.transport_revision + 1 + assert ceased.current_binding == status.binding + assert ceased.current_evidence_session_id == status.evidence_session_id + assert ceased.last_gap_started_at_utc == kwargs["gap_started_at_utc"] + assert ceased.last_gap_started_transport_revision == active.transport_revision + assert ceased.last_gap_failed_binding == binding + assert ceased.last_gap_recovered_at_utc == status.observed_at_utc + assert ceased.last_gap_recovered_transport_revision == ceased.transport_revision + assert ceased.cessation_status_proof == status + assert ceased.cessation_physical_proof == physical + assert ceased.activation_status_proof == active.activation_status_proof + assert ceased.activation_physical_proof == active.activation_physical_proof + assert ceased.current_active_status_proof == active.current_active_status_proof + assert ( + ceased.current_active_physical_proof + == active.current_active_physical_proof + ) + assert ceased.first_published_pcl_proof == active.first_published_pcl_proof + assert ceased.transition_receipts[-1].kind == ( + "cease-active-reconciled-standby" + ) + assert store.path.stat().st_size <= ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + assert restarted.snapshot().checkpoint == ceased + assert restarted.cease_active_reconciled_standby(**kwargs) == ceased + + conflicting = dict(kwargs) + conflicting["gap_started_at_utc"] = "2026-08-13T12:05:01.000Z" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointIdempotencyConflict): + restarted.cease_active_reconciled_standby(**conflicting) + stale = dict(kwargs) + stale["transition_id"] = f"transition-active-restart-stale-{terminal_state}" + with pytest.raises(StaleActiveAcquisitionRecoveryCheckpoint): + restarted.cease_active_reconciled_standby(**stale) + + +def test_cease_active_reconciled_standby_atomically_closes_exact_open_gap( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + active = _activate(store, binding) + gap_started_at = "2026-08-13T12:05:00.000Z" + open_gap = store.record_gap( + transition_id="transition-active-restart-gap", + expected_revision=active.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=binding, + gap_started_at_utc=gap_started_at, + ) + kwargs = _cease_active_reconciled_standby_kwargs( + open_gap, + terminal_state="ready", + gap_started_at_utc=gap_started_at, + transition_id="transition-cease-active-existing-gap", + ) + + wrong_gap = dict(kwargs) + wrong_gap["gap_started_at_utc"] = "2026-08-13T12:05:01.000Z" + with pytest.raises( + ActiveAcquisitionRecoveryCheckpointTransitionError, + match="exact open gap", + ): + store.cease_active_reconciled_standby(**wrong_gap) + assert store.snapshot().checkpoint == open_gap + + ceased = store.cease_active_reconciled_standby(**kwargs) + + assert ceased.revision == open_gap.revision + 1 + assert ceased.transport_revision == open_gap.transport_revision + 1 + assert ceased.last_gap_started_at_utc == gap_started_at + assert ceased.last_gap_started_transport_revision == ( + open_gap.last_gap_started_transport_revision + ) + assert ceased.last_gap_failed_binding == binding + assert ceased.last_gap_recovered_transport_revision == ceased.transport_revision + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + +def test_cease_active_reconciled_standby_supports_second_process_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, prepared_binding) + active = store.activate_reconciled( + **_activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + ) + ) + kwargs = _cease_active_reconciled_standby_kwargs( + active, + terminal_state="scan_over", + gap_started_at_utc="2026-08-13T12:06:00.000Z", + transition_id="transition-cease-active-second-restart", + ) + status = kwargs["cessation_status_proof"] + physical = kwargs["cessation_physical_proof"] + kwargs["cessation_status_proof"] = replace( + status, + observed_at_utc="2026-08-13T12:06:02.000Z", + ) + kwargs["cessation_physical_proof"] = replace( + physical, + observed_at_utc="2026-08-13T12:06:02.000Z", + ) + + ceased = store.cease_active_reconciled_standby(**kwargs) + + assert ceased.transport_revision == active.transport_revision + 1 + assert ceased.last_gap_started_transport_revision == active.transport_revision + assert ceased.last_gap_failed_binding == active.current_binding + assert ceased.reconciled_start_origin_proof == ( + active.reconciled_start_origin_proof + ) + assert ceased.first_published_pcl_proof == active.first_published_pcl_proof + assert ceased.current_active_status_proof == active.current_active_status_proof + assert ceased.current_active_physical_proof == ( + active.current_active_physical_proof + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + +def test_cease_active_reconciled_standby_rejects_inexact_restart_proofs( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + active = _activate(store, binding) + kwargs = _cease_active_reconciled_standby_kwargs( + active, + terminal_state="ready", + ) + status = kwargs["cessation_status_proof"] + physical = kwargs["cessation_physical_proof"] + + invalid_requests: list[dict[str, Any]] = [] + for next_binding in ( + replace( + status.binding, + runtime_instance_id=binding.runtime_instance_id, + ), + replace( + status.binding, + control_session_id=binding.control_session_id, + ), + ): + invalid = dict(kwargs) + invalid["cessation_status_proof"] = replace(status, binding=next_binding) + invalid["cessation_physical_proof"] = replace( + physical, + binding=next_binding, + ) + invalid_requests.append(invalid) + + same_evidence = dict(kwargs) + same_evidence["cessation_status_proof"] = replace( + status, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + ) + invalid_requests.append(same_evidence) + + non_explicit = dict(kwargs) + non_explicit["cessation_status_proof"] = replace( + status, + source="live-control-session", + ) + invalid_requests.append(non_explicit) + + wrong_status_sha = dict(kwargs) + wrong_status_sha["cessation_status_proof"] = replace( + status, + status_message_sha256="6" * 64, + ) + invalid_requests.append(wrong_status_sha) + + wrong_payload = dict(kwargs) + wrong_payload["cessation_physical_proof"] = replace( + physical, + payload_sha256="7" * 64, + original_start_payload_sha256="7" * 64, + ) + invalid_requests.append(wrong_payload) + + wrong_reconciliation = dict(kwargs) + wrong_reconciliation["cessation_physical_proof"] = _physical( + "stop-standby-observed", + binding=status.binding, + proof_id="physical-ordinary-stop-not-restart-reconciliation", + observed_at=status.observed_at_utc, + ledger_revision=physical.ledger_revision, + ) + invalid_requests.append(wrong_reconciliation) + + before_prior_active = dict(kwargs) + before_prior_active["gap_started_at_utc"] = "2026-08-13T11:59:59.000Z" + invalid_requests.append(before_prior_active) + + for index, invalid in enumerate(invalid_requests): + invalid["transition_id"] = f"transition-active-invalid-{index}" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease_active_reconciled_standby(**invalid) + assert store.snapshot().checkpoint == active + + with pytest.raises(ValueError, match="non-retained"): + replace(status, mqtt_retained=True) + + document = json.loads(store.path.read_text(encoding="utf-8")) + document["active_project_id_sha256"] = "6" * 64 + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .status + == "corrupt" + ) + + +def test_v4_reconciled_checkpoint_loads_as_v5_and_persists_on_mutation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, prepared_binding) + active = store.activate_reconciled( + **_activate_reconciled_kwargs( + prepared_binding, + origin_kind="composite-resolved", + ) + ) + document = active.as_dict() + document["schema_version"] = ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA + document.pop("prepared_resolved_start_reconciliation_id") + document.pop("prepared_resolved_start_original_attempt_sha256") + document.pop("prepared_resolved_start_original_project_id_sha256") + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + loaded = restarted.snapshot().checkpoint + assert loaded == active + assert loaded is not None + assert loaded.schema_version == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + restarted.record_gap( + transition_id="transition-gap-after-v4-migration", + expected_revision=loaded.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=loaded.current_binding, + gap_started_at_utc="2026-08-13T12:10:00.000Z", + ) + assert json.loads(store.path.read_text(encoding="utf-8"))["schema_version"] == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + ) + + +def test_v3_prepared_checkpoint_loads_as_v5_and_persists_on_mutation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, binding) + document = prepared.as_dict() + document["schema_version"] = ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V3_SCHEMA + document.pop("reconciled_start_origin_proof") + document.pop("first_published_pcl_proof") + document.pop("prepared_resolved_start_reconciliation_id") + document.pop("prepared_resolved_start_original_attempt_sha256") + document.pop("prepared_resolved_start_original_project_id_sha256") + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + loaded = restarted.snapshot().checkpoint + assert loaded is not None + assert loaded.schema_version == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + ceased = restarted.cease( + transition_id="transition-cease-v3-prepared", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=None, + physical_proof=_physical( + "not-dispatched", + binding=binding, + proof_id="physical-v3-not-dispatched", + observed_at="2026-08-13T12:00:04.000Z", + ), + ) + assert ceased.schema_version == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + assert json.loads(store.path.read_text(encoding="utf-8"))["schema_version"] == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + ) + + +@pytest.mark.parametrize("chained", (False, True)) +def test_cease_prepared_reconciled_stop_wins_before_first_pcl( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + chained: bool, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, prepared_binding) + active_kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind="ambiguous-reconciled", + ) + active_status = active_kwargs["status_proof"] + active_physical = active_kwargs["physical_proof"] + terminal_status = _status( + "ready", + binding=active_status.binding, + evidence_session_id=active_status.evidence_session_id, + observed_at="2026-08-13T12:00:03.000Z", + ) + ancestry = ( + _ancestry(ledger_revision=22) if chained else None + ) + terminal_physical = _physical( + "stop-standby-observed", + binding=active_status.binding, + proof_id="physical-stop-before-pcl", + observed_at=terminal_status.observed_at_utc, + ledger_revision=22, + ancestor_chain=ancestry, + ) + kwargs = { + "transition_id": "transition-cease-before-pcl", + "expected_revision": 1, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "gap_started_at_utc": "2026-08-13T12:00:00.000Z", + "origin_proof": active_kwargs["origin_proof"], + "active_status_proof": active_status, + "active_physical_proof": active_physical, + "cessation_status_proof": terminal_status, + "cessation_physical_proof": terminal_physical, + } + + ceased = store.cease_prepared_reconciled(**kwargs) + + assert ceased.state == "ceased" + assert ceased.transport_revision == 2 + assert ceased.activated_at_utc is None + assert ceased.first_published_pcl_proof is None + assert ceased.reconciled_start_origin_proof == active_kwargs["origin_proof"] + assert ceased.current_active_status_proof == active_status + assert ceased.cessation_physical_proof == terminal_physical + assert store.cease_prepared_reconciled(**kwargs) == ceased + with pytest.raises(StaleActiveAcquisitionRecoveryCheckpoint): + store.activate_reconciled(**active_kwargs) + + +def test_cease_prepared_reconciled_rejects_stale_terminal_revision_and_wrong_chain( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + prepared = _prepare(store, prepared_binding) + active_kwargs = _activate_reconciled_kwargs( + prepared_binding, + origin_kind="composite-resolved", + ) + active_status = active_kwargs["status_proof"] + terminal_status = _status( + "scan_over", + binding=active_status.binding, + evidence_session_id=active_status.evidence_session_id, + observed_at="2026-08-13T12:00:03.000Z", + ) + valid = _physical( + "stop-standby-observed", + binding=active_status.binding, + proof_id="physical-stop-before-pcl", + observed_at=terminal_status.observed_at_utc, + ledger_revision=22, + ) + base = { + "transition_id": "transition-cease-before-pcl-invalid", + "expected_revision": 1, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "gap_started_at_utc": "2026-08-13T12:00:00.000Z", + "origin_proof": active_kwargs["origin_proof"], + "active_status_proof": active_status, + "active_physical_proof": active_kwargs["physical_proof"], + "cessation_status_proof": terminal_status, + "cessation_physical_proof": valid, + } + for invalid in (replace(valid, ledger_revision=21),): + kwargs = dict(base) + kwargs["cessation_physical_proof"] = invalid + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease_prepared_reconciled(**kwargs) + assert store.snapshot().checkpoint == prepared + with pytest.raises(ValueError, match="typed ancestry certificate"): + replace(valid, original_start_operation_id="operation-start-wrong") + + +@pytest.mark.parametrize( + ("duration_seconds", "expected"), + ((1.25, 1.25), (2, 2.0)), +) +def test_duration_accepts_fractional_and_json_integer_roundtrip( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + duration_seconds: float | int, + expected: float, +) -> None: + store = _store(tmp_path, monkeypatch) + kwargs = _prepare_kwargs(_binding()) + kwargs["duration_seconds"] = duration_seconds + prepared = store.prepare(**kwargs) + assert prepared.duration_seconds == expected + + if isinstance(duration_seconds, int): + # Exercise the persisted parser with a literal JSON integer, rather + # than relying only on the API's canonical float serialization. + document = json.loads(store.path.read_text(encoding="utf-8")) + document["duration_seconds"] = duration_seconds + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + restored = restarted.snapshot().checkpoint + assert restored is not None + assert restored.duration_seconds == expected + assert isinstance(restored.duration_seconds, float) + + +@pytest.mark.parametrize( + "duration_seconds", + ( + True, + False, + float("nan"), + float("inf"), + float("-inf"), + 10**400, + 0, + 0.999, + -1, + ), +) +def test_duration_rejects_bool_nonfinite_and_values_below_one( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + duration_seconds: object, +) -> None: + store = _store(tmp_path, monkeypatch) + kwargs = _prepare_kwargs(_binding()) + kwargs["duration_seconds"] = duration_seconds + with pytest.raises(ValueError, match="duration_seconds"): + store.prepare(**kwargs) + assert store.snapshot().status == "empty" + + +def test_physical_ledger_revision_accepts_int64_max_and_rejects_overflow( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + status = _status( + "scanning", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:00:01.000Z", + ) + proof = _physical( + "start-active-observed", + binding=binding, + proof_id="physical-start-int64-max", + observed_at=status.observed_at_utc, + ledger_revision=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + ) + active = store.activate( + transition_id="transition-activate-int64-max", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=proof, + ) + assert active.physical_lineage_head_revision == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == active + ) + with pytest.raises(ValueError, match="bounded positive revision"): + replace( + proof, + ledger_revision=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + 1, + ) + + +def test_exact_project_name_rejects_implicit_normalization( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + for project_name in (" Mission Alpha", "Mission Alpha ", "Cafe\u0301"): + kwargs = _prepare_kwargs(_binding()) + kwargs["project_name"] = project_name + kwargs["project_name_wire_sha256"] = "9" * 64 + with pytest.raises(ValueError, match="exact canonical"): + store.prepare(**kwargs) + + +def test_lost_response_replay_is_idempotent_for_every_mutation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + second = _binding( + runtime_instance_id="runtime-B", + host_path_epoch=1, + control_session_id="control-B", + producer_generation=1, + ) + store = _store(tmp_path, monkeypatch) + + prepare_kwargs = _prepare_kwargs(first) + prepared = store.prepare(**prepare_kwargs) + assert store.prepare(**prepare_kwargs) == prepared + active = _activate(store, first) + activation_status = active.activation_status_proof + activation_physical = active.activation_physical_proof + assert activation_status is not None + assert activation_physical is not None + assert store.activate( + transition_id="transition-activate-001", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=activation_status, + physical_proof=activation_physical, + ) == active + gap_kwargs = { + "transition_id": "transition-gap-001", + "expected_revision": 2, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "failed_binding": first, + "gap_started_at_utc": "2026-08-13T12:05:00.000Z", + } + gap = store.record_gap(**gap_kwargs) + assert store.record_gap(**gap_kwargs) == gap + rebind_status = _status( + "scanning", + binding=second, + evidence_session_id="evidence-session-002", + observed_at="2026-08-13T12:05:04.000Z", + ) + rebind_physical = _physical( + "physical-active-observed", + binding=second, + proof_id="physical-rebind-001", + observed_at=rebind_status.observed_at_utc, + ) + rebind_kwargs = { + "transition_id": "transition-rebind-001", + "expected_revision": 3, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "status_proof": rebind_status, + "physical_proof": rebind_physical, + } + rebound = store.rebind_active(**rebind_kwargs) + assert store.rebind_active(**rebind_kwargs) == rebound + ready = _status( + "ready", + binding=second, + evidence_session_id="evidence-session-002", + observed_at="2026-08-13T12:10:00.000Z", + ) + cessation_physical = _physical( + "stop-standby-observed", + binding=second, + proof_id="physical-stop-001", + observed_at=ready.observed_at_utc, + ) + cease_kwargs = { + "transition_id": "transition-cease-001", + "expected_revision": 4, + "expected_acquisition_id": ACQUISITION_ID, + "expected_start_operation_id": START_OPERATION_ID, + "status_proof": ready, + "physical_proof": cessation_physical, + } + ceased = store.cease(**cease_kwargs) + assert store.cease(**cease_kwargs) == ceased + assert ceased.revision == 5 + assert len(ceased.transition_receipts) == 5 + + +def test_reused_transition_id_with_different_fingerprint_conflicts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + kwargs = _prepare_kwargs(_binding()) + store.prepare(**kwargs) + kwargs["evidence_policy"] = "disabled" + with pytest.raises(ActiveAcquisitionRecoveryCheckpointIdempotencyConflict): + store.prepare(**kwargs) + + +def test_restart_host_epoch_reset_advances_only_durable_transport_revision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding(host_path_epoch=99) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + store.record_gap( + transition_id="transition-gap-001", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2099-01-01T00:00:00.000Z", + ) + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + reset_binding = _binding( + runtime_instance_id="runtime-after-restart", + host_path_epoch=1, + control_session_id="control-after-restart", + producer_generation=1, + ) + status = _status( + "scanning", + binding=reset_binding, + evidence_session_id="evidence-after-restart", + observed_at="2020-01-01T00:00:00.000Z", + ) + rebound = restarted.rebind_active( + transition_id="transition-rebind-001", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=_physical( + "physical-active-observed", + binding=reset_binding, + proof_id="physical-rebind-001", + observed_at=status.observed_at_utc, + ), + ) + assert rebound.transport_revision == 2 + assert rebound.current_binding.host_path_epoch == 1 + assert rebound.current_evidence_session_id == "evidence-after-restart" + assert rebound.last_gap_started_at_utc > rebound.last_gap_recovered_at_utc # audit only + + +def test_same_process_rebind_preserves_exact_evidence_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + same_runtime = _binding( + runtime_instance_id=first.runtime_instance_id, + host_path_epoch=first.host_path_epoch + 1, + control_session_id="control-A-recovered", + producer_generation=first.producer_generation + 1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + store.record_gap( + transition_id="transition-gap-same-process", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + status = _status( + "scanning", + binding=same_runtime, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:05:04.000Z", + ) + rebound = store.rebind_active( + transition_id="transition-rebind-same-process", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=_physical( + "physical-active-observed", + binding=same_runtime, + proof_id="physical-rebind-same-process", + observed_at=status.observed_at_utc, + ), + ) + + assert rebound.revision == 4 + assert rebound.transport_revision == 2 + assert rebound.current_binding == same_runtime + assert rebound.current_evidence_session_id == ORIGINAL_EVIDENCE_SESSION_ID + assert rebound.last_gap_recovered_transport_revision == 2 + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == rebound + ) + + +def test_cross_process_rebind_requires_new_evidence_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + next_runtime = _binding( + runtime_instance_id="runtime-after-restart", + host_path_epoch=1, + control_session_id="control-after-restart", + producer_generation=1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + open_gap = store.record_gap( + transition_id="transition-gap-cross-process", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + reused_session = _status( + "scanning", + binding=next_runtime, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:05:04.000Z", + ) + with pytest.raises( + ActiveAcquisitionRecoveryCheckpointTransitionError, + match="across a runtime instance requires a successor evidence session", + ): + store.rebind_active( + transition_id="transition-rebind-cross-process-reused-session", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=reused_session, + physical_proof=_physical( + "physical-active-observed", + binding=next_runtime, + proof_id="physical-rebind-cross-process-reused-session", + observed_at=reused_session.observed_at_utc, + ), + ) + assert store.snapshot().checkpoint == open_gap + + successor_session = _status( + "scanning", + binding=next_runtime, + evidence_session_id="evidence-after-restart", + observed_at="2026-08-13T12:05:05.000Z", + ) + rebound = store.rebind_active( + transition_id="transition-rebind-cross-process-new-session", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=successor_session, + physical_proof=_physical( + "physical-active-observed", + binding=next_runtime, + proof_id="physical-rebind-cross-process-new-session", + observed_at=successor_session.observed_at_utc, + ), + ) + assert rebound.revision == 4 + assert rebound.transport_revision == 2 + assert rebound.current_binding == next_runtime + assert rebound.current_evidence_session_id == "evidence-after-restart" + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == rebound + ) + + +@pytest.mark.parametrize( + "reconciliation_kind", + ("prepared-stop-classification", "resolved-active-rebind"), +) +def test_active_rebind_preserves_canonical_undispatched_stop_head( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + reconciliation_kind: str, +) -> None: + first = _binding() + second = _binding( + runtime_instance_id="runtime-B", + host_path_epoch=1, + control_session_id="control-B", + producer_generation=1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + store.record_gap( + transition_id="transition-gap-stop-head", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + status = _status( + "scanning", + binding=second, + evidence_session_id="evidence-session-002", + observed_at="2026-08-13T12:05:04.000Z", + ) + proof = _undispatched_stop_active( + binding=second, + proof_id=f"physical-stop-head-{reconciliation_kind}", + observed_at=status.observed_at_utc, + reconciliation_kind=reconciliation_kind, + ledger_revision=21, + ) + rebound = store.rebind_active( + transition_id=f"transition-rebind-{reconciliation_kind}", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=proof, + ) + + persisted = rebound.current_active_physical_proof + assert persisted is not None + assert persisted.action == "stop" + assert persisted.resolution == "not-dispatched" + assert persisted.operation_id == "operation-stop-undispatched-001" + assert persisted.parent_operation_id == START_OPERATION_ID + assert persisted.original_start_operation_id == START_OPERATION_ID + assert persisted.original_start_payload_sha256 == START_PAYLOAD_SHA256 + assert persisted.stop_fence == "definitely-not-dispatched" + assert persisted.reconciliation_kind == reconciliation_kind + assert persisted.reconciliation_resolution == "physical-active-observed" + assert persisted.observed_session_state == "scanning" + assert json.loads(store.path.read_text(encoding="utf-8"))[ + "current_active_physical_proof" + ]["action"] == "stop" + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == rebound + ) + with pytest.raises(ValueError, match="requires a typed ancestry certificate"): + replace(proof, parent_operation_id="operation-start-wrong-parent") + + +def test_rebind_rejects_prepared_stop_or_unrelated_active_lineage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + second = _binding( + runtime_instance_id="runtime-B", + host_path_epoch=1, + control_session_id="control-B", + producer_generation=1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + store.record_gap( + transition_id="transition-gap-001", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + status = _status( + "scanning", + binding=second, + evidence_session_id="evidence-session-002", + observed_at="2026-08-13T12:05:04.000Z", + ) + prepared_stop = _physical( + "stop-standby-observed", + binding=second, + proof_id="physical-stop-fence-001", + observed_at="2026-08-13T12:05:04.000Z", + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.rebind_active( + transition_id="transition-rebind-invalid", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=prepared_stop, + ) + unrelated = replace( + _physical( + "physical-active-observed", + binding=second, + proof_id="physical-unrelated-001", + observed_at=status.observed_at_utc, + ), + operation_id="other-start-operation", + original_start_operation_id="other-start-operation", + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.rebind_active( + transition_id="transition-rebind-unrelated", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=unrelated, + ) + + +def test_physical_lineage_requires_terminal_late_edge_and_stop_fence_facts() -> None: + binding = _binding() + active = _physical( + "physical-active-observed", + binding=binding, + proof_id="physical-active-fence-facts", + observed_at="2026-08-13T12:05:04.000Z", + ) + with pytest.raises(ValueError, match="START lineage cannot contain a STOP fence"): + replace(active, stop_fence="definitely-not-dispatched") + with pytest.raises(ValueError, match="terminal and late-edge fenced"): + replace(active, edge_terminal=False) # type: ignore[arg-type] + with pytest.raises(ValueError, match="terminal and late-edge fenced"): + replace(active, late_start_excluded=False) # type: ignore[arg-type] + + +def test_activation_rejects_status_only_or_mismatched_physical_lineage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + status = _status( + "scanning", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:00:01.000Z", + ) + wrong = replace( + _physical( + "start-active-observed", + binding=binding, + proof_id="physical-wrong-payload", + observed_at=status.observed_at_utc, + ), + payload_sha256="f" * 64, + original_start_payload_sha256="f" * 64, + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.activate( + transition_id="transition-activate-invalid", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=wrong, + ) + assert store.snapshot().status == "prepared" + + +def test_prepared_crash_cannot_cease_on_status_alone( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + ready = _status( + "ready", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:01:00.000Z", + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease( + transition_id="transition-cease-ambiguous", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=ready, + physical_proof=_physical( + "not-dispatched", + binding=binding, + proof_id="physical-not-dispatched-001", + observed_at="2026-08-13T12:01:00.000Z", + ), + ) + ceased = store.cease( + transition_id="transition-cease-not-dispatched", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=None, + physical_proof=_physical( + "not-dispatched", + binding=binding, + proof_id="physical-not-dispatched-001", + observed_at="2026-08-13T12:01:00.000Z", + ), + ) + assert ceased.state == "ceased" + assert ceased.prepared_resolution_proof is not None + + +def test_active_cease_requires_exact_current_binding_and_physical_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + other = _binding( + runtime_instance_id="runtime-other", + control_session_id="control-other", + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + _activate(store, binding) + ready_other = _status( + "ready", + binding=other, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:10:00.000Z", + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease( + transition_id="transition-cease-wrong-binding", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=ready_other, + physical_proof=_physical( + "stop-standby-observed", + binding=other, + proof_id="physical-stop-wrong-binding", + observed_at=ready_other.observed_at_utc, + ), + ) + scanning = _status( + "scanning", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:10:00.000Z", + ) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointTransitionError): + store.cease( + transition_id="transition-cease-scanning", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=scanning, + physical_proof=_physical( + "physical-active-observed", + binding=binding, + proof_id="physical-active-not-ceased", + observed_at=scanning.observed_at_utc, + ), + ) + + +def test_chained_stop_cessation_roundtrips_and_legacy_v2_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + _activate(store, binding) + ready = _status( + "ready", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:10:00.000Z", + ) + ancestry = _ancestry() + chained_stop = _physical( + "stop-standby-observed", + binding=binding, + proof_id="physical-stop-successor-001", + observed_at=ready.observed_at_utc, + ancestor_chain=ancestry, + ) + + ceased = store.cease( + transition_id="transition-cease-chained-stop", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=ready, + physical_proof=chained_stop, + ) + + assert ceased.state == "ceased" + assert ceased.cessation_physical_proof is not None + assert ceased.cessation_physical_proof.ancestor_chain == ancestry + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + document = json.loads(store.path.read_text(encoding="utf-8")) + assert document["schema_version"] == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + assert document["cessation_physical_proof"]["ancestor_chain"] == ( + ancestry.as_dict() + ) + + for invalid in ( + None, + replace(ancestry, ledger_revision=ancestry.ledger_revision + 1), + replace(ancestry, root_start_operation_id="operation-start-wrong"), + replace(ancestry, acquisition_id="acquisition-wrong"), + replace(ancestry, head_operation_id="operation-stop-wrong"), + ): + with pytest.raises( + ValueError, + match="typed ancestry certificate|exact physical head", + ): + replace(chained_stop, ancestor_chain=invalid) + + # A v2 document has no certificate field. Direct-parent proofs migrate, but + # accepting this chained head would silently relax physical ancestry. + document["schema_version"] = ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA + document.pop("reconciled_start_origin_proof") + document.pop("first_published_pcl_proof") + document.pop("prepared_resolved_start_reconciliation_id") + document.pop("prepared_resolved_start_original_attempt_sha256") + document.pop("prepared_resolved_start_original_project_id_sha256") + for field in ( + "activation_physical_proof", + "current_active_physical_proof", + "last_rebind_physical_proof", + "cessation_physical_proof", + "prepared_resolution_proof", + ): + proof = document[field] + if isinstance(proof, dict): + proof.pop("ancestor_chain") + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .status + == "corrupt" + ) + + +def test_direct_parent_v2_checkpoint_migrates_on_next_mutation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + active = _activate(store, binding) + document = json.loads(store.path.read_text(encoding="utf-8")) + document["schema_version"] = ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_PREVIOUS_SCHEMA + document.pop("reconciled_start_origin_proof") + document.pop("first_published_pcl_proof") + document.pop("prepared_resolved_start_reconciliation_id") + document.pop("prepared_resolved_start_original_attempt_sha256") + document.pop("prepared_resolved_start_original_project_id_sha256") + for field in ( + "activation_physical_proof", + "current_active_physical_proof", + "last_rebind_physical_proof", + "cessation_physical_proof", + "prepared_resolution_proof", + ): + proof = document[field] + if isinstance(proof, dict): + proof.pop("ancestor_chain") + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + restarted = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + snapshot = restarted.snapshot() + assert snapshot.status == "active" + assert snapshot.checkpoint == active + assert snapshot.checkpoint is not None + assert snapshot.checkpoint.schema_version == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + ) + assert snapshot.checkpoint.current_active_physical_proof is not None + assert snapshot.checkpoint.current_active_physical_proof.ancestor_chain is None + + restarted.record_gap( + transition_id="transition-gap-after-v2-migration", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=binding, + gap_started_at_utc="2026-08-13T12:11:00.000Z", + ) + migrated = json.loads(store.path.read_text(encoding="utf-8")) + assert migrated["schema_version"] == ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA + assert migrated["current_active_physical_proof"]["ancestor_chain"] is None + + +@pytest.mark.parametrize("terminal_state", ("ready", "scan_over")) +def test_same_process_open_gap_cease_preserves_evidence_session_and_reloads( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + terminal_state: str, +) -> None: + first = _binding() + same_runtime = _binding( + runtime_instance_id=first.runtime_instance_id, + host_path_epoch=first.host_path_epoch + 1, + control_session_id="control-A-terminal", + producer_generation=first.producer_generation + 1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + store.record_gap( + transition_id=f"transition-gap-same-process-{terminal_state}", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + + wrong_session = _status( + terminal_state, + binding=same_runtime, + evidence_session_id="unexpected-successor-evidence", + observed_at="2026-08-13T12:05:04.000Z", + ) + wrong_physical = replace( + _physical( + "stop-standby-observed", + binding=same_runtime, + proof_id=f"physical-same-process-wrong-{terminal_state}", + observed_at=wrong_session.observed_at_utc, + ), + observed_session_state=terminal_state, + ) + with pytest.raises( + ActiveAcquisitionRecoveryCheckpointTransitionError, + match="same-runtime terminal recovery must preserve", + ): + store.cease( + transition_id=f"transition-cease-same-process-wrong-{terminal_state}", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=wrong_session, + physical_proof=wrong_physical, + ) + + terminal = _status( + terminal_state, + binding=same_runtime, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:05:05.000Z", + ) + terminal_physical = replace( + _physical( + "stop-standby-observed", + binding=same_runtime, + proof_id=f"physical-same-process-{terminal_state}", + observed_at=terminal.observed_at_utc, + ), + observed_session_state=terminal_state, + ) + ceased = store.cease( + transition_id=f"transition-cease-same-process-{terminal_state}", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=terminal, + physical_proof=terminal_physical, + ) + + assert ceased.state == "ceased" + assert ceased.transport_revision == 2 + assert ceased.current_binding == same_runtime + assert ceased.current_evidence_session_id == ORIGINAL_EVIDENCE_SESSION_ID + assert ceased.last_gap_recovered_at_utc == terminal.observed_at_utc + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + +def test_cross_process_open_gap_cease_requires_successor_evidence_session( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + next_runtime = _binding( + runtime_instance_id="runtime-after-restart", + host_path_epoch=1, + control_session_id="control-after-restart", + producer_generation=1, + ) + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + _activate(store, first) + open_gap = store.record_gap( + transition_id="transition-gap-cross-process-terminal", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T12:05:00.000Z", + ) + + reused_session = _status( + "ready", + binding=next_runtime, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:05:04.000Z", + ) + reused_physical = _physical( + "stop-standby-observed", + binding=next_runtime, + proof_id="physical-cross-process-reused-session", + observed_at=reused_session.observed_at_utc, + ) + with pytest.raises( + ActiveAcquisitionRecoveryCheckpointTransitionError, + match="across a runtime instance requires a successor evidence session", + ): + store.cease( + transition_id="transition-cease-cross-process-reused-session", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=reused_session, + physical_proof=reused_physical, + ) + assert store.snapshot().checkpoint == open_gap + + successor_session = _status( + "scan_over", + binding=next_runtime, + evidence_session_id="evidence-after-restart", + observed_at="2026-08-13T12:05:05.000Z", + ) + successor_physical = replace( + _physical( + "stop-standby-observed", + binding=next_runtime, + proof_id="physical-cross-process-successor-session", + observed_at=successor_session.observed_at_utc, + ), + observed_session_state="scan_over", + ) + ceased = store.cease( + transition_id="transition-cease-cross-process-successor-session", + expected_revision=3, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=successor_session, + physical_proof=successor_physical, + ) + assert ceased.state == "ceased" + assert ceased.current_binding == next_runtime + assert ceased.current_evidence_session_id == "evidence-after-restart" + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + document = json.loads(store.path.read_text(encoding="utf-8")) + document["current_evidence_session_id"] = ORIGINAL_EVIDENCE_SESSION_ID + document["cessation_status_proof"][ + "evidence_session_id" + ] = ORIGINAL_EVIDENCE_SESSION_ID + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .status + == "corrupt" + ) + + +def test_clock_rollback_clamps_local_audit_but_never_orders_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + clock = _Clock( + datetime(2026, 8, 13, 12, tzinfo=UTC), + datetime(2020, 1, 1, tzinfo=UTC), + ) + binding = _binding() + store = _store(tmp_path, monkeypatch, clock=clock) + prepared = _prepare(store, binding) + active = _activate(store, binding) + assert active.updated_at_utc == prepared.updated_at_utc + assert active.activated_at_utc == "2026-08-13T12:00:01.000Z" + + +def test_cas_stale_and_exact_identity_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, binding) + status = _status( + "scanning", + binding=binding, + evidence_session_id=ORIGINAL_EVIDENCE_SESSION_ID, + observed_at="2026-08-13T12:00:01.000Z", + ) + with pytest.raises(StaleActiveAcquisitionRecoveryCheckpoint): + store.activate( + transition_id="transition-activate-stale", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=_physical( + "start-active-observed", + binding=binding, + proof_id="physical-start-active-stale", + observed_at=status.observed_at_utc, + ), + ) + + +def test_receipt_window_is_bounded_and_successor_prepare_resets_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, first) + current = _activate(store, first) + current_binding = first + for index in range(20): + current = store.record_gap( + transition_id=f"transition-gap-{index:03d}", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=current_binding, + gap_started_at_utc=f"2026-08-13T13:{index:02d}:00.000Z", + ) + next_binding = _binding( + runtime_instance_id=f"runtime-{index + 100}", + host_path_epoch=1, + control_session_id=f"control-{index + 100}", + producer_generation=1, + ) + status = _status( + "scanning", + binding=next_binding, + evidence_session_id=f"evidence-{index + 100}", + observed_at=f"2026-08-13T13:{index:02d}:01.000Z", + ) + current = store.rebind_active( + transition_id=f"transition-rebind-{index:03d}", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=status, + physical_proof=_physical( + "physical-active-observed", + binding=next_binding, + proof_id=f"physical-rebind-{index:03d}", + observed_at=status.observed_at_utc, + ledger_revision=21 + index, + ), + ) + current_binding = next_binding + assert len(current.transition_receipts) == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS + ) + assert current.transition_receipts[0].committed_revision > 3 + before_evicted_retry = current + with pytest.raises(StaleActiveAcquisitionRecoveryCheckpoint): + store.record_gap( + transition_id="transition-gap-000", + expected_revision=2, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=first, + gap_started_at_utc="2026-08-13T13:00:00.000Z", + ) + assert store.snapshot().checkpoint == before_evicted_retry + terminal_binding = _binding( + runtime_instance_id="runtime-receipt-window-terminal", + host_path_epoch=1, + control_session_id="control-receipt-window-terminal", + producer_generation=1, + ) + ready = _status( + "ready", + binding=terminal_binding, + evidence_session_id="evidence-receipt-window-terminal", + observed_at="2026-08-13T14:00:00.000Z", + ) + current_physical = current.current_active_physical_proof + assert current_physical is not None + ceased = store.cease_active_reconciled_standby( + transition_id="transition-cease-final", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + gap_started_at_utc="2026-08-13T13:59:59.000Z", + cessation_status_proof=ready, + cessation_physical_proof=replace( + current_physical, + ledger_revision=42, + proof_id="physical-stop-final", + reconciliation_kind="resolved-active-cessation", + reconciliation_resolution="physical-standby-observed", + status_message_sha256=ready.status_message_sha256, + observed_session_state="ready", + binding=terminal_binding, + observed_at_utc=ready.observed_at_utc, + ), + ) + assert len(ceased.transition_receipts) == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS + ) + assert ceased.transition_receipts[-1].kind == ( + "cease-active-reconciled-standby" + ) + assert store.path.stat().st_size <= ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES + ) + successor_kwargs = _prepare_kwargs( + terminal_binding, + transition_id="transition-prepare-successor", + ) + successor_kwargs.update( + predecessor_revision=ceased.revision, + acquisition_id="acquisition-002", + original_start_operation_id="operation-start-002", + original_evidence_session_id="evidence-successor", + ) + successor = store.prepare(**successor_kwargs) + assert len(successor.transition_receipts) == 1 + assert successor.transition_receipts[0].kind == "prepare" + + +def test_maximal_v5_restart_proofs_fit_and_maximal_v4_migrates( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + prepared_binding = _binding() + store = _store(tmp_path, monkeypatch) + _prepare(store, prepared_binding) + restart_binding = replace( + _reconciled_binding(), + runtime_instance_id="r" * 160, + control_session_id="c" * 160, + ) + restart_evidence_session_id = "e" * 160 + restart_status = _status( + "scanning", + binding=restart_binding, + evidence_session_id=restart_evidence_session_id, + observed_at="2026-08-13T12:00:02.000Z", + ) + restart_physical = _reconciled_physical( + origin_kind="ambiguous-reconciled", + binding=restart_binding, + proof_id="p" * 160, + observed_at=restart_status.observed_at_utc, + ledger_revision=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION - 16 + ), + ) + restart_origin = replace( + _origin( + origin_kind="ambiguous-reconciled", + prepared_binding=prepared_binding, + physical_proof=restart_physical, + ), + reconciliation_id="q" * 160, + original_attempt_revision=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION - 17 + ), + ) + first_pcl = replace( + _first_pcl( + restart_binding, + evidence_session_id=restart_evidence_session_id, + ), + proof_id="f" * 160, + viewer_producer_generation=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ), + frame_sequence=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + captured_at_epoch_ns=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + point_count=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + published_pcl_frames=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ), + ) + current = store.activate_reconciled( + transition_id="transition-maximal-v5-restart", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + gap_started_at_utc="2026-08-13T12:00:00.000Z", + origin_proof=restart_origin, + status_proof=restart_status, + physical_proof=restart_physical, + first_pcl_proof=first_pcl, + ) + current_binding = restart_binding + current_physical = restart_physical + for index in range(15): + current = store.record_gap( + transition_id=f"transition-maximal-gap-{index:02d}", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=current_binding, + gap_started_at_utc=f"2026-08-13T13:00:{index * 2:02d}.000Z", + ) + next_binding = _binding( + runtime_instance_id=f"r{index:03d}" + "x" * 156, + host_path_epoch=1, + control_session_id=f"c{index:03d}" + "y" * 156, + producer_generation=1, + ) + next_status = _status( + "scanning", + binding=next_binding, + evidence_session_id=f"e{index:03d}" + "z" * 156, + observed_at=f"2026-08-13T13:00:{index * 2 + 1:02d}.000Z", + ) + current_physical = replace( + current_physical, + ledger_revision=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + - 15 + + index + ), + proof_id=f"p{index:03d}" + "w" * 156, + reconciliation_kind="resolved-active-rebind", + binding=next_binding, + observed_at_utc=next_status.observed_at_utc, + ) + current = store.rebind_active( + transition_id=f"transition-maximal-rebind-{index:02d}", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=next_status, + physical_proof=current_physical, + ) + current_binding = next_binding + + assert len(current.transition_receipts) == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS + ) + terminal_status = _status( + "scan_over", + binding=current_binding, + evidence_session_id=current.current_evidence_session_id, + observed_at="2026-08-13T14:00:00.000Z", + ) + maximum_ancestry = PhysicalCommandClassifiedStopAncestryProof( + schema_version=PHYSICAL_COMMAND_CLASSIFIED_STOP_ANCESTRY_SCHEMA, + ledger_revision=ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION, + head_operation_id="h" * 160, + head_parent_operation_id="n" * 160, + root_start_operation_id=START_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + depth=4096, + chain_sha256="d" * 64, + ) + ceased = store.cease( + transition_id="transition-maximal-v5-cease", + expected_revision=current.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=terminal_status, + physical_proof=replace( + _physical( + "stop-standby-observed", + binding=current_binding, + proof_id="t" * 160, + observed_at=terminal_status.observed_at_utc, + ledger_revision=( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_REVISION + ), + ancestor_chain=maximum_ancestry, + ), + observed_session_state="scan_over", + ), + ) + + assert len(ceased.transition_receipts) == ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_RECEIPTS + ) + assert ceased.reconciled_start_origin_proof == restart_origin + assert ceased.first_published_pcl_proof == first_pcl + assert ceased.cessation_physical_proof is not None + assert ceased.cessation_physical_proof.ancestor_chain == maximum_ancestry + assert store.path.stat().st_size <= ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES + ) + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + v4_document = ceased.as_dict() + v4_document["schema_version"] = ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_V4_SCHEMA + ) + v4_document.pop("prepared_resolved_start_reconciliation_id") + v4_document.pop("prepared_resolved_start_original_attempt_sha256") + v4_document.pop("prepared_resolved_start_original_project_id_sha256") + v4_payload = json.dumps( + v4_document, + sort_keys=True, + separators=(",", ":"), + ) + "\n" + assert len(v4_payload.encode("utf-8")) <= ( + ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES + ) + store.path.write_text(v4_payload, encoding="utf-8") + assert ( + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + .snapshot() + .checkpoint + == ceased + ) + + +@pytest.mark.parametrize( + "payload", + [ + b'{"schema_version":', + b'{"schema_version":"x","schema_version":"x"}\n', + json.dumps( + { + "schema_version": ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_SCHEMA, + "password": "must-not-be-stored", + } + ).encode(), + b"{" + b"x" * ACTIVE_ACQUISITION_RECOVERY_CHECKPOINT_MAX_BYTES + b"}", + ], +) +def test_corrupt_truncated_duplicate_unknown_and_oversize_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + payload: bytes, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + path = ( + tmp_path + / "private-data" + / "xgrids-k1" + / "active-acquisition-recovery-checkpoint.json" + ) + path.parent.mkdir(mode=0o700, parents=True) + (tmp_path / "private-data").chmod(0o700) + path.parent.chmod(0o700) + path.write_bytes(payload) + path.chmod(0o600) + store = ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + assert store.snapshot().status == "corrupt" + before = path.read_bytes() + with pytest.raises(ActiveAcquisitionRecoveryCheckpointCorrupt): + store.prepare(**_prepare_kwargs(_binding())) + assert path.read_bytes() == before + + +def test_symlink_nonprivate_file_and_unsafe_lock_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + parent = tmp_path / "private-data" / "xgrids-k1" + parent.mkdir(mode=0o700, parents=True) + (tmp_path / "private-data").chmod(0o700) + parent.chmod(0o700) + path = parent / "active-acquisition-recovery-checkpoint.json" + outside = tmp_path / "outside.json" + outside.write_text("{}", encoding="utf-8") + outside.chmod(0o600) + path.symlink_to(outside) + assert ActiveAcquisitionRecoveryCheckpointStore( + tmp_path / "repository" + ).snapshot().status == "corrupt" + path.unlink() + path.write_text("{}", encoding="utf-8") + path.chmod(0o644) + assert ActiveAcquisitionRecoveryCheckpointStore( + tmp_path / "repository" + ).snapshot().status == "corrupt" + path.unlink() + lock = parent / ".active-acquisition-recovery-checkpoint.lock" + lock.write_text("unsafe", encoding="utf-8") + lock.chmod(0o600) + with pytest.raises(ActiveAcquisitionRecoveryCheckpointCorrupt): + ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository") + + +def test_atomic_publication_fsyncs_file_and_directory_and_replace_failure_is_safe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + real_fsync = checkpoint_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = checkpoint_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(checkpoint_module.os, "fsync", observe_fsync) + _prepare(store, _binding()) + assert "file" in fsync_kinds + assert fsync_kinds[-1] == "directory" + before = store.path.read_bytes() + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected replace failure") + + monkeypatch.setattr(checkpoint_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + store.cease( + transition_id="transition-cease-not-dispatched", + expected_revision=1, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=None, + physical_proof=_physical( + "not-dispatched", + binding=_binding(), + proof_id="physical-not-dispatched-001", + observed_at="2026-08-13T12:01:00.000Z", + ), + ) + assert store.path.read_bytes() == before + assert not list(store.path.parent.glob(".active-acquisition-*.tmp")) + + +def _hold_checkpoint_write( + data_dir: str, + repository_root: str, + release_path: str, + events: Any, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + store = ActiveAcquisitionRecoveryCheckpointStore(Path(repository_root)) + real_write = checkpoint_module._write_private_json_atomic + + def hold_write( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + events.put(("entered", 0)) + deadline = time.monotonic() + 5 + while not Path(release_path).exists(): + if time.monotonic() >= deadline: + raise TimeoutError("parent did not release checkpoint writer") + time.sleep(0.01) + real_write(path, payload, data_dir=data_dir) + + checkpoint_module._write_private_json_atomic = hold_write + prepared = _prepare(store, _binding()) + events.put(("prepared", prepared.revision)) + + +def test_process_lock_serializes_complete_cas_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + context = multiprocessing.get_context("spawn") + events = context.Queue() + release_path = tmp_path / "release-child" + child = context.Process( + target=_hold_checkpoint_write, + args=( + str(tmp_path / "private-data"), + str(tmp_path / "repository"), + str(release_path), + events, + ), + daemon=True, + ) + parent_finished = threading.Event() + parent_outcome: dict[str, object] = {} + + def prepare_from_parent() -> None: + try: + parent_outcome["record"] = store.prepare( + **_prepare_kwargs( + _binding(), + transition_id="transition-prepare-parent", + ) + ) + except Exception as exc: + parent_outcome["error"] = exc + finally: + parent_finished.set() + + child.start() + parent_thread: threading.Thread | None = None + try: + assert events.get(timeout=5) == ("entered", 0) + parent_thread = threading.Thread(target=prepare_from_parent, daemon=True) + parent_thread.start() + assert parent_finished.wait(timeout=0.2) is False + release_path.touch() + assert events.get(timeout=5) == ("prepared", 1) + child.join(timeout=5) + parent_thread.join(timeout=5) + finally: + if child.is_alive(): + child.terminate() + child.join(timeout=5) + assert child.exitcode == 0 + assert parent_thread is not None and not parent_thread.is_alive() + assert isinstance( + parent_outcome.get("error"), + StaleActiveAcquisitionRecoveryCheckpoint, + ) diff --git a/tests/test_xgrids_active_acquisition_restart_rehydration.py b/tests/test_xgrids_active_acquisition_restart_rehydration.py new file mode 100644 index 0000000..666da31 --- /dev/null +++ b/tests/test_xgrids_active_acquisition_restart_rehydration.py @@ -0,0 +1,2386 @@ +from __future__ import annotations + +import asyncio +import copy +import hashlib +import importlib +import json +import os +import subprocess +import sys +import threading +import time +from dataclasses import replace +from pathlib import Path +from types import MethodType +from typing import Any + +import pytest + +from k1link.data_plane import ConsumerFrameContext, DecodedPointCloudView +from k1link.device_plugins.xgrids_k1 import physical_command_ledger as ledger_module +from k1link.device_plugins.xgrids_k1.facade import ( + ConnectionVerifyRequest, + StopAcquisitionRequest, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) + +restart_support: Any = importlib.import_module( + "test_xgrids_active_acquisition_restart_safety" +) +lifecycle_support: Any = importlib.import_module("test_xgrids_acquisition_lifecycle") +capture_support: Any = importlib.import_module("test_mqtt_capture") + + +def _install_ready_capture_runtime(service: Any) -> Any: + runtime = lifecycle_support.FakeVisualizationRuntime() + + def start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + capture_dir = out_dir / "captures" / "mqtt_live" + capture_dir.mkdir(mode=0o700, parents=True) + (capture_dir / "raw.mqtt").write_bytes(b"K1_RESTART_SUCCESSOR") + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + bound_runtime.mark_ready() + + runtime.start_live = MethodType(start_live, runtime) + service.runtime = runtime + return runtime + + +def _published_pcl(sequence: int) -> DecodedPointCloudView: + return DecodedPointCloudView( + context=ConsumerFrameContext( + sequence=sequence, + captured_at_epoch_ns=sequence, + received_monotonic_ns=sequence, + processing_started_monotonic_ns=sequence, + encoded_size_bytes=32, + live=True, + ), + frame_id="map", + positions_xyz=((1.0, 2.0, 3.0),), + ) + + +def _resolve_start_without_checkpoint_activation(service: Any) -> None: + ledger = service._physical_command_ledger # noqa: SLF001 + connection = restart_support._prepared_connection() + ledger.mark_dispatching(restart_support.START_OPERATION_ID) + ledger.mark_observing( + restart_support.START_OPERATION_ID, + publish_call_returned=True, + packet_id=41, + ) + ledger.mark_qos2_completed(restart_support.START_OPERATION_ID, packet_id=41) + ledger.record_application_response( + restart_support.START_OPERATION_ID, + restart_support.PhysicalCommandApplicationResponse( + operation_id=restart_support.START_OPERATION_ID, + action="start", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=restart_support.PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="f" * 64, + observed_at_utc="2026-08-13T12:00:01.000Z", + ), + ) + scanning = restart_support._scanning_status( + connection, + source="live-control-session", + observed_at_utc="2026-08-13T12:00:02.000Z", + ) + ledger.record_status_observation(restart_support.START_OPERATION_ID, scanning) + ledger.resolve( + restart_support.START_OPERATION_ID, + resolution="start-active-observed", + ) + + +def _seed_historical_rebind_before_restart_start(service: Any) -> str: + """Leave one unrelated reconciliation for the next rebind to compact.""" + + ledger = service._physical_command_ledger # noqa: SLF001 + identity = restart_support.PhysicalCommandIdentity( + vendor_device_id_sha256=restart_support.VENDOR_SHA256, + device_serial_sha256=restart_support.SERIAL_SHA256, + ) + start_operation_id = "historical-start-before-restart-checkpoint" + stop_operation_id = "historical-stop-before-restart-checkpoint" + acquisition_id = "historical-acquisition-before-restart-checkpoint" + initial_connection = restart_support.PhysicalCommandConnectionBinding( + intent_id="historical-initial-intent", + transport_ref="transport-restart-safety", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=1, + control_session_id="historical-initial-control", + producer_generation=1, + ) + ledger.prepare( + operation_id=start_operation_id, + parent_operation_id=None, + acquisition_id=acquisition_id, + action="start", + identity=identity, + connection=initial_connection, + compatibility_profile_id=restart_support.XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="1" * 64, + baseline_status=restart_support._ready_status(initial_connection), # noqa: SLF001 + ) + ledger.mark_dispatching(start_operation_id) + ledger.mark_observing( + start_operation_id, + publish_call_returned=True, + packet_id=31, + ) + ledger.mark_qos2_completed(start_operation_id, packet_id=31) + ledger.record_application_response( + start_operation_id, + restart_support.PhysicalCommandApplicationResponse( + operation_id=start_operation_id, + action="start", + control_session_id=initial_connection.control_session_id, + host_path_epoch=initial_connection.host_path_epoch, + producer_generation=initial_connection.producer_generation, + result_code=restart_support.PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="2" * 64, + observed_at_utc="2026-08-13T10:00:01.000Z", + ), + ) + initial_scanning = restart_support._scanning_status( # noqa: SLF001 + initial_connection, + source="live-control-session", + observed_at_utc="2026-08-13T10:00:02.000Z", + ) + ledger.record_status_observation(start_operation_id, initial_scanning) + ledger.resolve(start_operation_id, resolution="start-active-observed") + + rebound_connection = restart_support.PhysicalCommandConnectionBinding( + intent_id="historical-rebound-intent", + transport_ref="transport-restart-safety", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=2, + control_session_id="historical-rebound-control", + producer_generation=2, + ) + rebound_scanning = restart_support._scanning_status( # noqa: SLF001 + rebound_connection, + source="explicit-read-only-reconciliation", + observed_at_utc="2026-08-13T10:01:00.000Z", + ) + ledger.rebind_resolved_active( + start_operation_id, + reconciliation_id="historical-rebind-before-restart-checkpoint", + verified_binding=restart_support.PhysicalCommandVerifiedBinding( + verification_id="historical-rebind-before-restart-checkpoint.device-info", + identity=identity, + connection=rebound_connection, + device_info_message_sha256="3" * 64, + verified_at_utc="2026-08-13T10:00:59.000Z", + ), + observation=rebound_scanning, + ) + live_rebound_scanning = restart_support._scanning_status( # noqa: SLF001 + rebound_connection, + source="live-control-session", + observed_at_utc="2026-08-13T10:01:01.000Z", + ) + ledger.prepare( + operation_id=stop_operation_id, + parent_operation_id=start_operation_id, + acquisition_id=acquisition_id, + action="stop", + identity=identity, + connection=rebound_connection, + compatibility_profile_id=restart_support.XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="4" * 64, + baseline_status=live_rebound_scanning, + operator_confirmation_id="historical-stop-confirmation", + operator_confirmed_at_utc="2026-08-13T10:01:02.000Z", + ) + ledger.mark_dispatching(stop_operation_id) + ledger.mark_observing( + stop_operation_id, + publish_call_returned=True, + packet_id=32, + ) + ledger.mark_qos2_completed(stop_operation_id, packet_id=32) + ledger.record_application_response( + stop_operation_id, + restart_support.PhysicalCommandApplicationResponse( + operation_id=stop_operation_id, + action="stop", + control_session_id=rebound_connection.control_session_id, + host_path_epoch=rebound_connection.host_path_epoch, + producer_generation=rebound_connection.producer_generation, + result_code=restart_support.PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="5" * 64, + observed_at_utc="2026-08-13T10:01:03.000Z", + ), + ) + ledger.record_status_observation( + stop_operation_id, + restart_support._ready_status(rebound_connection), # noqa: SLF001 + ) + ledger.resolve(stop_operation_id, resolution="stop-standby-observed") + return stop_operation_id + + +def _verify_restart(service: Any, *, operation_id: str) -> dict[str, Any]: + return asyncio.run( + service.verify_connection( + ConnectionVerifyRequest( + device_id="transport-restart-safety", + source="retained-current-process", + compatibility_attestation=lifecycle_support.ATTESTATION, + operation_id=operation_id, + ) + ) + ) + + +def _install_forbidden_restart_device_edges( + monkeypatch: pytest.MonkeyPatch, + *, + service: Any, + calls: list[str], +) -> None: + def forbidden_device_command(_control: Any, **_kwargs: object) -> object: + calls.append("physical-command") + pytest.fail("restart setup/cleanup must not publish START or STOP") + + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_start", + forbidden_device_command, + ) + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_stop", + forbidden_device_command, + ) + + def forbidden_camera_activation(_service: Any, **_kwargs: object) -> bool: + calls.append("camera-activation") + pytest.fail("restart setup/cleanup must not activate the camera") + + service._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + forbidden_camera_activation, + service, + ) + + +def _assert_exact_restart_owner_retained(service: Any) -> tuple[Any, Any, Path]: + with service._lock: # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + pending = service._restart_rehydration_pending # noqa: SLF001 + lease = service._acquisition_session_lease # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + assert acquisition is not None + assert acquisition.acquisition_id == restart_support.ACQUISITION_ID + assert acquisition.state == "awaiting_external_start" + assert pending is not None + assert pending.phase == "blocked" + assert pending.reason_code == "restart-receiver-cleanup-pending" + assert lease is not None + assert out_dir == pending.out_dir + assert lease.session_root == out_dir + projected = service.state() + assert projected["acquisition"]["cleanup_pending"] is True + assert projected["connection_recovery"]["state"] == "blocked" + assert projected["connection_recovery"]["reason_code"] == ( + "restart-receiver-cleanup-pending" + ) + assert projected["connection_recovery"]["camera_media_state"] == "inactive" + assert projected["active_acquisition_recovery_checkpoint"][ + "restart_rehydration_phase" + ] == "blocked" + return pending, lease, out_dir + + +def test_active_restart_verify_opens_new_writer_then_first_pcl_rebinds( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + old_dir = first.evidence_root / restart_support.EVIDENCE_SESSION_ID + old_dir.mkdir(mode=0o700, parents=True) + old_raw = old_dir / "raw.mqtt" + old_raw.write_bytes(b"IMMUTABLE_OLD_CAPTURE") + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + device_command_calls: list[str] = [] + + def forbidden_device_command(_control: Any, **_kwargs: object) -> object: + device_command_calls.append("physical-command") + pytest.fail("restart rehydration must not publish START or STOP") + + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_start", + forbidden_device_command, + ) + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_stop", + forbidden_device_command, + ) + camera_calls: list[tuple[str, str, int]] = [] + + def activate_camera( + _service: Any, + *, + expected_acquisition_id: str, + expected_evidence_session_id: str, + expected_runtime_generation: int, + ) -> bool: + camera_calls.append( + ( + expected_acquisition_id, + expected_evidence_session_id, + expected_runtime_generation, + ) + ) + return True + + restarted._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + activate_camera, + restarted, + ) + result = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="transport-restart-safety", + source="retained-current-process", + compatibility_attestation=lifecycle_support.ATTESTATION, + operation_id="op-55555555-5555-4555-8555-555555555555", + ) + ) + ) + + acquisition = result["acquisition"] + assert acquisition["acquisition_id"] == restart_support.ACQUISITION_ID + assert acquisition["project_name"] == restart_support.PROJECT_NAME + assert acquisition["requested_streams"] == ["pointcloud"] + assert acquisition["result"]["receiver_rehydration"] == "awaiting-first-pcl" + assert result["active_acquisition_recovery_checkpoint"][ + "restart_rehydration_phase" + ] == "awaiting-first-pcl" + assert result["phase"] == "reconnecting" + assert result["source_mode"] == "live" + assert result["connection_recovery"]["state"] == "reconnecting" + assert result["connection_recovery"]["reason_code"] == ( + "restart-receiver-awaiting-first-pcl" + ) + assert result["connection_recovery"]["automatic_read_only_rebind"] is True + assert result["connection_recovery"]["start_performed"] is False + assert result["connection_recovery"]["stop_performed"] is False + assert result["connection_recovery"]["camera_recovery"] == "inactive" + assert result["connection_recovery"]["camera_media_state"] == "inactive" + assert result["connection_recovery"]["camera_media_ready"] is False + assert result["connection_recovery"]["camera_epoch"] is None + assert len(runtime.start_calls) == 1 + new_dir = runtime.start_calls[0][1] + assert new_dir != old_dir + assert old_raw.read_bytes() == b"IMMUTABLE_OLD_CAPTURE" + assert (new_dir / "captures" / "mqtt_live" / "raw.mqtt").read_bytes() == ( + b"K1_RESTART_SUCCESSOR" + ) + pending_checkpoint = store.snapshot().checkpoint + assert pending_checkpoint is not None + assert pending_checkpoint.state == "active" + assert pending_checkpoint.last_gap_started_at_utc is not None + assert pending_checkpoint.last_gap_recovered_at_utc is None + assert device_command_calls == [] + assert camera_calls == [] + + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(41), + runtime.producer_generation, + ) + deadline = time.monotonic() + 1.0 + while not camera_calls and time.monotonic() < deadline: + time.sleep(0.01) + + promoted = store.snapshot().checkpoint + assert promoted is not None + assert promoted.state == "active" + assert promoted.current_evidence_session_id == new_dir.name + assert promoted.current_binding.runtime_instance_id == restarted._snapshot_runtime_id # noqa: SLF001 + assert promoted.last_gap_recovered_at_utc is not None + recovered = restarted.state() + assert recovered["phase"] == "live" + assert recovered["acquisition"]["state"] == "acquiring" + assert recovered["connection_recovery"]["state"] == "recovered" + assert recovered["connection_recovery"]["reason_code"] is None + assert camera_calls == [ + ( + restart_support.ACQUISITION_ID, + new_dir.name, + runtime.producer_generation, + ) + ] + + runtime.pcl_frames = 2 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(42), + runtime.producer_generation, + ) + assert store.snapshot().checkpoint == promoted + assert len(camera_calls) == 1 + assert device_command_calls == [] + ledger_record = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert ledger_record is not None + assert ledger_record.operation_id == restart_support.START_OPERATION_ID + assert ledger_record.action == "start" + + +def test_active_restart_verify_rehydrates_when_rebind_rolls_archive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + parent_operation_id = _seed_historical_rebind_before_restart_start(first) + store = restart_support._seed_matching_prepared_start( + first, + parent_operation_id=parent_operation_id, + ) + restart_support._activate_prepared_start(first, store) + predecessor = first._physical_command_ledger.snapshot().record # noqa: SLF001 + assert predecessor is not None + assert [item.reconciliation_id for item in predecessor.reconciliations] == [ + "historical-rebind-before-restart-checkpoint" + ] + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + verify_operation_id = "op-a4c41e2d-5ed7-4711-8e12-1d44953ba007" + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(restarted._physical_command_ledger.path.read_bytes()) + 128, # noqa: SLF001 + ) + result = _verify_restart( + restarted, + operation_id=verify_operation_id, + ) + + current = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert current is not None + assert current.revision == predecessor.revision + 1 + assert current.archive_segment_count == predecessor.archive_segment_count + 1 + assert [item.reconciliation_id for item in current.reconciliations] == [ + f"{verify_operation_id}.physical" + ] + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "active" + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None + assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001 + assert result["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + assert len(runtime.start_calls) == 1 + assert command_calls == [] + + +def test_cold_restart_control_adoption_gets_fresh_budget_after_slow_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A durable SCANNING proof must not consume local worker-adoption time.""" + + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + original_bootstrap = restarted._bootstrap_prestart_control_ready_owned # noqa: SLF001 + clock = [100.0] + adoption_pending = [False] + adoption_snapshots = [0] + + def monotonic() -> float: + return clock[0] + + class FacadeTimeProxy: + def monotonic(self) -> float: + return monotonic() + + def __getattr__(self, name: str) -> Any: + return getattr(time, name) + + async def bootstrap_with_delayed_adoption( + bound_service: Any, + *, + parent_operation_id: str | None, + connection_mode: Any, + inspection_only: bool = False, + ) -> None: + await original_bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + control = bound_service._application_control_session + original_adopt = control.adopt_reconciled_scanning + original_snapshot = control.snapshot + + def delayed_adopt(**kwargs: object) -> dict[str, object]: + result = original_adopt(**kwargs) + adoption_pending[0] = True + control.state = "active-recovery-requested" + # The old shared proof deadline (105.0) is already exhausted here. + clock[0] = 105.01 + return result + + def delayed_snapshot() -> dict[str, object]: + if adoption_pending[0]: + adoption_snapshots[0] += 1 + if adoption_snapshots[0] >= 2: + control.state = "scanning" + clock[0] = 105.5 + return original_snapshot() + + monkeypatch.setattr(control, "adopt_reconciled_scanning", delayed_adopt) + monkeypatch.setattr(control, "snapshot", delayed_snapshot) + + monkeypatch.setattr( + restart_support.facade_module, + "time", + FacadeTimeProxy(), + ) + restarted._bootstrap_prestart_control_ready_owned = MethodType( # noqa: SLF001 + bootstrap_with_delayed_adoption, + restarted, + ) + + result = _verify_restart( + restarted, + operation_id="op-a1a1a1a1-a1a1-41a1-81a1-a1a1a1a1a1a1", + ) + + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "active" + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None + assert result["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + assert adoption_snapshots[0] >= 2 + assert len(runtime.start_calls) == 1 + assert command_calls == [] + + +def test_cold_restart_control_adoption_timeout_is_typed_and_commandless( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + original_bootstrap = restarted._bootstrap_prestart_control_ready_owned # noqa: SLF001 + clock = [300.0] + adoption_pending = [False] + + class FacadeTimeProxy: + def monotonic(self) -> float: + return clock[0] + + def __getattr__(self, name: str) -> Any: + return getattr(time, name) + + async def bootstrap_with_stalled_adoption( + bound_service: Any, + *, + parent_operation_id: str | None, + connection_mode: Any, + inspection_only: bool = False, + ) -> None: + await original_bootstrap( + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + control = bound_service._application_control_session + original_adopt = control.adopt_reconciled_scanning + original_snapshot = control.snapshot + + def stalled_adopt(**kwargs: object) -> dict[str, object]: + result = original_adopt(**kwargs) + adoption_pending[0] = True + control.state = "active-recovery-requested" + clock[0] = 305.01 + return result + + def stalled_snapshot() -> dict[str, object]: + if adoption_pending[0]: + clock[0] = 310.02 + return original_snapshot() + + monkeypatch.setattr(control, "adopt_reconciled_scanning", stalled_adopt) + monkeypatch.setattr(control, "snapshot", stalled_snapshot) + + monkeypatch.setattr( + restart_support.facade_module, + "time", + FacadeTimeProxy(), + ) + restarted._bootstrap_prestart_control_ready_owned = MethodType( # noqa: SLF001 + bootstrap_with_stalled_adoption, + restarted, + ) + + with pytest.raises( + restart_support.facade_module.ConnectionVerificationError, + ) as failure: + _verify_restart( + restarted, + operation_id="op-b1b1b1b1-b1b1-41b1-81b1-b1b1b1b1b1b1", + ) + + assert failure.value.reason_code == ( + "physical-command-reconciliation-control-adoption-timeout" + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "active" + assert checkpoint.last_gap_started_at_utc is None + assert checkpoint.last_gap_recovered_at_utc is None + assert restarted._acquisition is None # noqa: SLF001 + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert runtime.start_calls == [] + assert command_calls == [] + assert restarted.state()["last_operation"]["error"]["code"] == ( + "physical-command-reconciliation-control-adoption-timeout" + ) + + +def test_restart_checkpoint_public_projection_redacts_all_private_lineage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + result = _verify_restart( + restarted, + operation_id="op-12121212-1212-4212-8212-121212121212", + ) + projection = result["active_acquisition_recovery_checkpoint"] + assert set(projection) == { + "status", + "revision", + "state", + "transport_revision", + "gap_open", + "restart_resume_authority", + "restart_rehydration_phase", + "restart_rehydration_reason_code", + "reason_code", + } + assert projection["status"] == "active" + assert projection["state"] == "active" + assert projection["gap_open"] is True + assert projection["restart_resume_authority"] is False + assert projection["restart_rehydration_phase"] == "awaiting-first-pcl" + + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert pending is not None + encoded = json.dumps(projection, sort_keys=True) + private_values = { + checkpoint.acquisition_id, + checkpoint.original_start_operation_id, + checkpoint.original_evidence_session_id, + checkpoint.current_evidence_session_id, + pending.new_evidence_session_id, + pending.status_proof.status_message_sha256, + pending.physical_proof.proof_id, + pending.reconciliation_id, + } + assert all(private_value not in encoded for private_value in private_values) + + +def test_ambiguous_prepared_restart_waits_for_first_pcl_before_activation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + first._physical_command_ledger.mark_dispatching( # noqa: SLF001 + restart_support.START_OPERATION_ID + ) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + camera_calls: list[int] = [] + + def activate_camera(_service: Any, **_kwargs: object) -> bool: + camera_calls.append(1) + return True + + restarted._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + activate_camera, + restarted, + ) + result = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="transport-restart-safety", + source="retained-current-process", + compatibility_attestation=lifecycle_support.ATTESTATION, + operation_id="op-66666666-6666-4666-8666-666666666666", + ) + ) + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None and checkpoint.state == "prepared" + assert result["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + assert camera_calls == [] + + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(51), + runtime.producer_generation, + ) + promoted = store.snapshot().checkpoint + assert promoted is not None and promoted.state == "active" + assert promoted.reconciled_start_origin_proof is not None + assert promoted.reconciled_start_origin_proof.origin_kind == "ambiguous-reconciled" + assert promoted.reconciled_start_origin_proof.project_evidence_strength == ( + "edge-correlated-vendor-project-id" + ) + assert promoted.first_published_pcl_proof is not None + assert promoted.first_published_pcl_proof.frame_sequence == 51 + ledger_record = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert ledger_record is not None + assert ledger_record.operation_id == restart_support.START_OPERATION_ID + assert ledger_record.action == "start" + assert ledger_record.resolution == "physical-active-observed" + + +def test_composite_prepared_restart_uses_exact_origin_and_zero_start_replay( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + _resolve_start_without_checkpoint_activation(first) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + result = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="transport-restart-safety", + source="retained-current-process", + compatibility_attestation=lifecycle_support.ATTESTATION, + operation_id="op-77777777-7777-4777-8777-777777777777", + ) + ) + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None and checkpoint.state == "prepared" + assert result["acquisition"]["acquisition_id"] == restart_support.ACQUISITION_ID + assert result["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(61), + runtime.producer_generation, + ) + promoted = store.snapshot().checkpoint + assert promoted is not None and promoted.state == "active" + assert promoted.reconciled_start_origin_proof is not None + assert promoted.reconciled_start_origin_proof.origin_kind == "composite-resolved" + assert promoted.reconciled_start_origin_proof.project_evidence_strength == ( + "exact-vendor-project-id" + ) + record = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert record is not None + assert record.operation_id == restart_support.START_OPERATION_ID + assert record.action == "start" + assert record.resolution == "start-active-observed" + assert all( + operation["action"] != "acquisition.start" + for operation in restarted._operations.snapshot() # noqa: SLF001 + ) + + +def test_restart_receiver_link_loss_before_first_pcl_rebinds_read_only_then_promotes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + device_command_calls: list[str] = [] + camera_calls: list[tuple[str, str, int]] = [] + + def forbidden_device_command(_control: Any, **_kwargs: object) -> object: + device_command_calls.append("physical-command") + pytest.fail("pre-PCL recovery must not publish START or STOP") + + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_start", + forbidden_device_command, + ) + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_stop", + forbidden_device_command, + ) + + def activate_camera( + _service: Any, + *, + expected_acquisition_id: str, + expected_evidence_session_id: str, + expected_runtime_generation: int, + ) -> bool: + camera_calls.append( + ( + expected_acquisition_id, + expected_evidence_session_id, + expected_runtime_generation, + ) + ) + return True + + restarted._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + activate_camera, + restarted, + ) + _verify_restart( + restarted, + operation_id="op-34343434-3434-4434-8434-343434343434", + ) + original_pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert original_pending is not None + assert original_pending.phase == "awaiting-first-pcl" + original_checkpoint = store.snapshot().checkpoint + assert original_checkpoint is not None + assert original_checkpoint.last_gap_started_at_utc is not None + assert original_checkpoint.last_gap_recovered_at_utc is None + assert camera_calls == [] + assert device_command_calls == [] + assert runtime.request_connection_recovery( + "synthetic-link-loss-before-first-pcl", + expected_generation=runtime.producer_generation, + ) + lineage = restarted._admit_active_stream_recovery_lineage( # noqa: SLF001 + runtime=runtime.snapshot(), + ) + assert lineage is not None + + async def monitor_quiescent() -> bool: + return True + + monkeypatch.setattr( + restarted, + "_await_connection_monitor_quiescence", + monitor_quiescent, + ) + monkeypatch.setattr( + restarted, + "_probe_control_endpoint", + lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001 + reachable=True, + reason_code=None, + ), + ) + monkeypatch.setattr( + restarted, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + restarted, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + restart_bootstrap = restarted._bootstrap_prestart_control_ready_owned # noqa: SLF001 + + async def bootstrap_fresh_control(**kwargs: object) -> None: + kwargs.pop("force_new_binding") + await restart_bootstrap(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr( + restarted, + "_bootstrap_prestart_control_ready_owned", + bootstrap_fresh_control, + ) + + decision = asyncio.run( + restarted._recover_active_stream_connection_owned( # noqa: SLF001 + lineage, + 2, + ) + ) + assert decision == "resume" + with restarted._lock: # noqa: SLF001 + restarted._active_stream_recovery_attempt = 2 # noqa: SLF001 + assert camera_calls == [] + assert device_command_calls == [] + + frame = _published_pcl(63) + assert restarted._confirm_active_acquisition_checkpoint_rebind( # noqa: SLF001 + frame, + runtime.producer_generation, + 2, + frame.context.sequence, + ) + refreshed_pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert refreshed_pending is not None + assert refreshed_pending.reconciliation_id != original_pending.reconciliation_id + assert refreshed_pending.new_evidence_session_id == ( + original_pending.new_evidence_session_id + ) + assert refreshed_pending.gap_started_at_utc == original_pending.gap_started_at_utc + assert store.snapshot().checkpoint == original_checkpoint + assert camera_calls == [] + assert device_command_calls == [] + + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + runtime.recovery_request_pending = False + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + deadline = time.monotonic() + 1.0 + while not camera_calls and time.monotonic() < deadline: + time.sleep(0.01) + + committed = store.snapshot().checkpoint + assert committed is not None + assert committed.state == "active" + assert committed.last_gap_started_at_utc == original_pending.gap_started_at_utc + assert committed.last_gap_recovered_at_utc is not None + assert committed.current_evidence_session_id == ( + original_pending.new_evidence_session_id + ) + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert camera_calls == [ + ( + restart_support.ACQUISITION_ID, + original_pending.new_evidence_session_id, + runtime.producer_generation, + ) + ] + assert device_command_calls == [] + + +def test_ambiguous_prepared_restart_second_rebind_promotes_only_after_fresh_pcl( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + first._physical_command_ledger.mark_dispatching( # noqa: SLF001 + restart_support.START_OPERATION_ID + ) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + camera_calls: list[tuple[str, str, int]] = [] + + def forbidden_device_command(_control: Any, **_kwargs: object) -> object: + command_calls.append("physical-command") + pytest.fail("ambiguous PREPARED recovery must not replay START or STOP") + + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_start", + forbidden_device_command, + ) + monkeypatch.setattr( + lifecycle_support.FakeInteractiveControlSession, + "request_stop", + forbidden_device_command, + ) + + def activate_camera( + _service: Any, + *, + expected_acquisition_id: str, + expected_evidence_session_id: str, + expected_runtime_generation: int, + ) -> bool: + camera_calls.append( + ( + expected_acquisition_id, + expected_evidence_session_id, + expected_runtime_generation, + ) + ) + return True + + restarted._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + activate_camera, + restarted, + ) + _verify_restart( + restarted, + operation_id="op-45454545-4545-4545-8545-454545454545", + ) + initial_pending = restarted._restart_rehydration_pending # noqa: SLF001 + initial_checkpoint = store.snapshot().checkpoint + assert initial_pending is not None + assert initial_pending.phase == "awaiting-first-pcl" + assert initial_pending.origin_proof is not None + assert initial_pending.origin_proof.origin_kind == "ambiguous-reconciled" + assert initial_checkpoint is not None + assert initial_checkpoint.state == "prepared" + assert initial_checkpoint.acquisition_id == restart_support.ACQUISITION_ID + assert initial_checkpoint.current_evidence_session_id == ( + restart_support.EVIDENCE_SESSION_ID + ) + assert command_calls == [] + assert camera_calls == [] + + assert runtime.request_connection_recovery( + "synthetic-ambiguous-prepared-pre-pcl-loss", + expected_generation=runtime.producer_generation, + ) + lineage = restarted._admit_active_stream_recovery_lineage( # noqa: SLF001 + runtime=runtime.snapshot(), + ) + assert lineage is not None + + async def monitor_quiescent() -> bool: + return True + + monkeypatch.setattr( + restarted, + "_await_connection_monitor_quiescence", + monitor_quiescent, + ) + monkeypatch.setattr( + restarted, + "_probe_control_endpoint", + lambda target_ipv4: restart_support.facade_module._CorrelatedEndpointObservation( # noqa: SLF001 + path=lifecycle_support._direct_host_path(target_ipv4), # noqa: SLF001 + reachable=True, + reason_code=None, + ), + ) + monkeypatch.setattr( + restarted, + "_acquire_k1_lifecycle_process_lease", + lambda _holder: None, + ) + monkeypatch.setattr( + restarted, + "_release_k1_lifecycle_process_lease", + lambda _holder: None, + ) + original_bootstrap = restarted._bootstrap_prestart_control_ready_owned # noqa: SLF001 + + async def bootstrap_fresh_control(**kwargs: object) -> None: + kwargs.pop("force_new_binding") + await original_bootstrap(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr( + restarted, + "_bootstrap_prestart_control_ready_owned", + bootstrap_fresh_control, + ) + + assert ( + asyncio.run( + restarted._recover_active_stream_connection_owned( # noqa: SLF001 + lineage, + 2, + ) + ) + == "resume" + ) + with restarted._lock: # noqa: SLF001 + restarted._active_stream_recovery_attempt = 2 # noqa: SLF001 + + frame = _published_pcl(64) + assert restarted._confirm_active_acquisition_checkpoint_rebind( # noqa: SLF001 + frame, + runtime.producer_generation, + 2, + frame.context.sequence, + ) + refreshed_pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert refreshed_pending is not None + assert refreshed_pending.reconciliation_id != initial_pending.reconciliation_id + assert refreshed_pending.new_evidence_session_id == ( + initial_pending.new_evidence_session_id + ) + assert refreshed_pending.gap_started_at_utc == initial_pending.gap_started_at_utc + assert refreshed_pending.origin_proof is not None + assert refreshed_pending.origin_proof.origin_kind == "ambiguous-reconciled" + assert ( + refreshed_pending.origin_proof.original_attempt_sha256 + == initial_pending.origin_proof.original_attempt_sha256 + ) + assert refreshed_pending.physical_proof.reconciliation_kind == ( + "resolved-active-rebind" + ) + assert store.snapshot().checkpoint == initial_checkpoint + assert command_calls == [] + assert camera_calls == [] + + runtime.phase = "live" + runtime.source_ready = True + runtime.recovery_state = "recovered" + runtime.recovery_request_pending = False + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + frame, + runtime.producer_generation, + ) + deadline = time.monotonic() + 1.0 + while not camera_calls and time.monotonic() < deadline: + time.sleep(0.01) + + committed = store.snapshot().checkpoint + assert committed is not None + assert committed.state == "active" + assert committed.acquisition_id == restart_support.ACQUISITION_ID + assert committed.current_evidence_session_id == ( + initial_pending.new_evidence_session_id + ) + assert committed.last_gap_started_at_utc == initial_pending.gap_started_at_utc + assert committed.last_gap_recovered_at_utc is not None + assert committed.reconciled_start_origin_proof is not None + assert committed.reconciled_start_origin_proof.origin_kind == ( + "ambiguous-reconciled" + ) + assert committed.restart_resume_authority is False + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert command_calls == [] + assert camera_calls == [ + ( + restart_support.ACQUISITION_ID, + initial_pending.new_evidence_session_id, + runtime.producer_generation, + ) + ] + + +@pytest.mark.parametrize( + "mismatch", + ("runtime-generation", "pending-intent", "latest-reconciliation"), +) +def test_restart_pending_recovery_rejects_inexact_frozen_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mismatch: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + _verify_restart( + restarted, + operation_id="op-56565656-5656-4656-8656-565656565656", + ) + pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert pending is not None + admitted_lineage = restarted._active_stream_recovery_lineage # noqa: SLF001 + assert admitted_lineage is not None + assert runtime.request_connection_recovery( + "synthetic-negative-pre-pcl-loss", + expected_generation=runtime.producer_generation, + ) + + runtime_snapshot = runtime.snapshot() + physical_snapshot = restarted._physical_command_coordinator.snapshot() # noqa: SLF001 + if mismatch == "runtime-generation": + runtime_snapshot = { + **runtime_snapshot, + "producer_generation": runtime.producer_generation + 1, + } + elif mismatch == "pending-intent": + wrong_binding = replace( + pending.status_proof.binding, + intent_id="intent-restart-pending-mismatch", + ) + with restarted._lock: # noqa: SLF001 + restarted._restart_rehydration_pending = replace( # noqa: SLF001 + pending, + status_proof=replace( + pending.status_proof, + binding=wrong_binding, + ), + ) + else: + physical_snapshot = json.loads(json.dumps(physical_snapshot)) + physical_snapshot["record"]["reconciliations"][-1][ + "reconciliation_id" + ] = "reconciliation-mismatch" + + assert restarted._admit_active_stream_recovery_lineage( # noqa: SLF001 + runtime=runtime_snapshot, + physical=physical_snapshot, + ) is None + assert restarted._active_stream_recovery_lineage == admitted_lineage # noqa: SLF001 + + +def test_first_pcl_finishes_exact_commit_after_store_response_is_lost( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + camera_calls: list[int] = [] + restarted._activate_default_acquisition_camera = MethodType( # noqa: SLF001 + lambda _service, **_kwargs: camera_calls.append(1) is None or True, + restarted, + ) + _verify_restart( + restarted, + operation_id="op-88888888-8888-4888-8888-888888888888", + ) + restarted_store = restarted._active_acquisition_checkpoint # noqa: SLF001 + assert restarted_store is not None + assert restarted_store is not store + assert restarted_store.path == store.path + original_rebind = restarted_store.rebind_active + raised_after_commit = False + + def commit_then_raise(**kwargs: object) -> Any: + nonlocal raised_after_commit + committed = original_rebind(**kwargs) # type: ignore[arg-type] + if not raised_after_commit: + raised_after_commit = True + raise OSError("lost response after checkpoint fsync") + return committed + + monkeypatch.setattr(restarted_store, "rebind_active", commit_then_raise) + runtime.pcl_frames = 1 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(71), + runtime.producer_generation, + ) + committed = store.snapshot().checkpoint + assert committed is not None + assert raised_after_commit is True + assert committed.last_gap_recovered_at_utc is not None + with restarted._lock: # noqa: SLF001 + lost_response_pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert lost_response_pending is not None + assert lost_response_pending.phase == "awaiting-first-pcl" + after_lost_response_state = restarted.state() + assert after_lost_response_state["active_acquisition_recovery_checkpoint"][ + "restart_rehydration_phase" + ] == "awaiting-first-pcl" + after_lost_response = after_lost_response_state["acquisition"] + assert after_lost_response["state"] == "awaiting_external_start" + assert after_lost_response["result"] == { + "read_only_recovery": True, + "device_state": "scanning", + "receiver_rehydration": "awaiting-first-pcl", + "automatic_replay_allowed": False, + } + assert camera_calls == [] + + runtime.pcl_frames = 2 + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(72), + runtime.producer_generation + 1, + ) + stale_generation_state = restarted.state() + assert stale_generation_state["acquisition"]["state"] == ( + "awaiting_external_start" + ) + assert stale_generation_state["active_acquisition_recovery_checkpoint"][ + "restart_rehydration_phase" + ] == "awaiting-first-pcl" + assert camera_calls == [] + + restarted._observe_published_runtime_envelope( # noqa: SLF001 + _published_pcl(73), + runtime.producer_generation, + ) + assert store.snapshot().checkpoint == committed + assert restarted.state()["acquisition"]["state"] == "acquiring" + deadline = time.monotonic() + 1.0 + while not camera_calls and time.monotonic() < deadline: + time.sleep(0.01) + assert camera_calls == [1] + + +def test_backend_process_restart_before_first_pcl_reuses_open_gap_without_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + repository_root = Path(__file__).resolve().parents[1] + subprocess_code = """ +import json +import os +import sys +from pathlib import Path + +import pytest + +repository_root = Path(sys.argv[1]).resolve() +test_root = repository_root / "tests" +sys.path.insert(0, str(test_root)) + +import test_xgrids_active_acquisition_restart_rehydration as rehydration_support +import test_xgrids_active_acquisition_restart_safety as restart_support + +monkeypatch = pytest.MonkeyPatch() +service_root = Path(sys.argv[2]).resolve() +service = restart_support._service(service_root, monkeypatch) +runtime = rehydration_support._install_ready_capture_runtime(service) +restart_support._install_exact_restart_verify(service) +command_edges = [] +camera_edges = [] + +def forbidden_start_or_stop(_control, **kwargs): + command_edges.append(sorted(kwargs)) + raise AssertionError("restart receiver must not publish START or STOP") + +monkeypatch.setattr( + rehydration_support.lifecycle_support.FakeInteractiveControlSession, + "request_start", + forbidden_start_or_stop, +) +monkeypatch.setattr( + rehydration_support.lifecycle_support.FakeInteractiveControlSession, + "request_stop", + forbidden_start_or_stop, +) +service._activate_default_acquisition_camera = lambda **kwargs: ( + camera_edges.append(sorted(kwargs)) + or (_ for _ in ()).throw( + AssertionError("camera must wait for the first published PCL") + ) +) +result = rehydration_support._verify_restart( + service, + operation_id=sys.argv[3], +) +checkpoint = service._active_acquisition_checkpoint.snapshot().checkpoint +assert checkpoint is not None +print( + "RESTART_PROCESS_JSON=" + + json.dumps( + { + "pid": os.getpid(), + "snapshot_runtime_id": service._snapshot_runtime_id, + "receiver_rehydration": result["acquisition"]["result"][ + "receiver_rehydration" + ], + "gap_started_at_utc": checkpoint.last_gap_started_at_utc, + "gap_recovered_at_utc": checkpoint.last_gap_recovered_at_utc, + "out_dir": str(runtime.start_calls[0][1]), + "command_edges": command_edges, + "camera_edges": camera_edges, + }, + sort_keys=True, + ) +) +""" + environment = os.environ.copy() + python_path = [str(repository_root / "src"), str(repository_root / "tests")] + inherited_python_path = environment.get("PYTHONPATH") + if inherited_python_path: + python_path.append(inherited_python_path) + environment["PYTHONPATH"] = os.pathsep.join(python_path) + def run_restart_process(operation_id: str) -> dict[str, Any]: + process = subprocess.run( + [ + sys.executable, + "-c", + subprocess_code, + str(repository_root), + str(tmp_path), + operation_id, + ], + cwd=repository_root, + env=environment, + capture_output=True, + text=True, + timeout=15.0, + check=False, + ) + assert process.returncode == 0, process.stderr + payload_line = next( + line + for line in process.stdout.splitlines() + if line.startswith("RESTART_PROCESS_JSON=") + ) + return json.loads(payload_line.removeprefix("RESTART_PROCESS_JSON=")) + + process_payload = run_restart_process( + "op-99999999-9999-4999-8999-999999999999" + ) + assert process_payload["pid"] != os.getpid() + assert process_payload["receiver_rehydration"] == "awaiting-first-pcl" + assert process_payload["gap_recovered_at_utc"] is None + assert process_payload["command_edges"] == [] + assert process_payload["camera_edges"] == [] + second_dir = Path(process_payload["out_dir"]) + assert second_dir.is_dir() + + open_gap = store.snapshot().checkpoint + assert open_gap is not None + assert open_gap.last_gap_started_at_utc == process_payload["gap_started_at_utc"] + assert open_gap.last_gap_recovered_at_utc is None + + second_process_payload = run_restart_process( + "op-aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" + ) + still_open = store.snapshot().checkpoint + assert still_open is not None + assert still_open.last_gap_started_at_utc == open_gap.last_gap_started_at_utc + assert still_open.last_gap_recovered_at_utc is None + assert second_process_payload["pid"] != os.getpid() + assert second_process_payload["pid"] != process_payload["pid"] + assert ( + second_process_payload["snapshot_runtime_id"] + != process_payload["snapshot_runtime_id"] + ) + assert second_process_payload["gap_started_at_utc"] == ( + process_payload["gap_started_at_utc"] + ) + assert second_process_payload["gap_recovered_at_utc"] is None + assert Path(second_process_payload["out_dir"]) != second_dir + assert second_process_payload["command_edges"] == [] + assert second_process_payload["camera_edges"] == [] + + +def test_setup_cleanup_linearizes_before_late_stop_priority( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = lifecycle_support.FakeVisualizationRuntime() + cleanup_entered = threading.Event() + allow_cleanup = threading.Event() + + def failing_start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + capture_support.capture_mqtt( + host, + out_dir / "captures" / "mqtt_live", + duration_seconds=duration_seconds, + _client_factory=capture_support.FakeClient, + ) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + raise RuntimeError("deterministic setup failure") + + runtime.start_live = MethodType(failing_start_live, runtime) + original_stop = runtime.stop + + def paused_stop() -> None: + cleanup_entered.set() + assert allow_cleanup.wait(2.0) + original_stop() + + runtime.stop = paused_stop + restarted.runtime = runtime + restart_support._install_exact_restart_verify(restarted) + verify_results: list[dict[str, Any]] = [] + verify_errors: list[BaseException] = [] + stop_results: list[dict[str, Any]] = [] + stop_errors: list[BaseException] = [] + + def run_verify() -> None: + try: + verify_results.append( + _verify_restart( + restarted, + operation_id="op-bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + ) + ) + except BaseException as exc: # pragma: no cover - diagnostic capture + verify_errors.append(exc) + + verify_thread = threading.Thread(target=run_verify, daemon=True) + verify_thread.start() + assert cleanup_entered.wait(2.0) + + control = restarted._application_control_session # noqa: SLF001 + coordinator = restarted._physical_command_coordinator # noqa: SLF001 + original_request_stop = control.request_stop + stop_payload = b"restart-cleanup-race-stop" + prepared_stop_operations: list[str] = [] + + def request_stop_with_durable_edge( + *, + confirmation: object, + command_context: Any, + dispatch_admission_deadline_reached: Any = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert command_context is not None + coordinator.prepare( + command_context, + action="stop", + envelope=OneShotPublishEnvelope( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload=stop_payload, + payload_sha256=hashlib.sha256(stop_payload).hexdigest(), + payload_bytes=len(stop_payload), + qos=2, + retain=False, + ), + ) + prepared_stop_operations.append(command_context.operation_id) + result = original_request_stop( + confirmation=confirmation, + command_context=command_context, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + # The production session returns synchronously from the durable + # PREPARED handoff in `stop-requested`; its worker advances to + # `awaiting-standby-confirmation` only after a real publish edge. + control.state = "stop-requested" + return {**result, "state": "stop-requested"} + + control.request_stop = request_stop_with_durable_edge + control_snapshot = control.snapshot() + control_generation = control_snapshot["session_generation"] + control_revision = control_snapshot["state_revision"] + assert isinstance(control_generation, int) + assert isinstance(control_revision, int) + + def run_stop() -> None: + try: + stop_results.append( + restarted.stop_acquisition( + StopAcquisitionRequest( + acquisition_id=restart_support.ACQUISITION_ID, + operation_id="op-cccccccc-cccc-4ccc-8ccc-cccccccccccc", + idempotency_key="restart-cleanup-race-public-stop", + mode="graceful", + expected_control_session_generation=control_generation, + expected_control_state_revision=control_revision, + physical_acceptance=lifecycle_support.PHYSICAL_ACCEPTANCE, + ) + ) + ) + except BaseException as exc: # pragma: no cover - diagnostic capture + stop_errors.append(exc) + + stop_thread = threading.Thread(target=run_stop, daemon=True) + stop_thread.start() + priority_deadline = time.monotonic() + 2.0 + while time.monotonic() < priority_deadline: + with restarted._camera_restart_commit_gate: # noqa: SLF001 + priority_published = bool( + restarted._camera_stop_priority_counts.get( # noqa: SLF001 + restart_support.ACQUISITION_ID, + 0, + ) + ) + if priority_published: + break + time.sleep(0.005) + assert priority_published is True + assert stop_thread.is_alive() + allow_cleanup.set() + verify_thread.join(3.0) + stop_thread.join(3.0) + assert not verify_thread.is_alive() + assert not stop_thread.is_alive() + assert verify_errors == [] + assert len(verify_results) == 1 + assert stop_errors == [] + assert len(stop_results) == 1 + with restarted._lock: # noqa: SLF001 + acquisition = restarted._acquisition # noqa: SLF001 + lease = restarted._acquisition_session_lease # noqa: SLF001 + out_dir = restarted._acquisition_out_dir # noqa: SLF001 + pending = restarted._restart_rehydration_pending # noqa: SLF001 + assert acquisition is not None + assert acquisition.acquisition_id == restart_support.ACQUISITION_ID + assert acquisition.state == "awaiting_external_stop" + assert lease is not None and out_dir is not None + assert pending is not None + assert store.snapshot().checkpoint is not None + # The late STOP owns final teardown. Cleanup may stop the failed runtime, + # but it must not release/replace its evidence owner after priority wins. + assert runtime.source_mode == "idle" + assert out_dir.exists() + capture_root = out_dir / "captures" / "mqtt_live" + assert (capture_root / "mqtt.timeline.origin.json").is_file() + assert (capture_root / "mqtt.timeline.json").is_file() + assert (capture_root / "mqtt.summary.json").is_file() + assert prepared_stop_operations == ["op-cccccccc-cccc-4ccc-8ccc-cccccccccccc"] + physical = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert physical is not None + assert physical.operation_id == "op-cccccccc-cccc-4ccc-8ccc-cccccccccccc" + assert physical.action == "stop" + assert physical.stage == "prepared" + assert control.state == "stop-requested" + projected_after_stop = restarted.state()["acquisition"] + assert projected_after_stop["state"] == "awaiting_external_stop" + assert projected_after_stop["result"] is None + with restarted._camera_restart_commit_gate: # noqa: SLF001 + assert ( + restarted._camera_stop_priority_counts.get( # noqa: SLF001 + restart_support.ACQUISITION_ID, + 0, + ) + == 0 + ) + + +@pytest.mark.parametrize("failure_mode", ("before-commit", "after-commit")) +def test_restart_record_gap_failure_keeps_zero_local_owner_and_retries_exactly( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure_mode: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + restarted_store = restarted._active_acquisition_checkpoint # noqa: SLF001 + assert restarted_store is not None + original_record_gap = restarted_store.record_gap + committed_gap: list[Any] = [] + + def fail_record_gap(**kwargs: object) -> Any: + if failure_mode == "after-commit": + committed_gap.append(original_record_gap(**kwargs)) # type: ignore[arg-type] + raise OSError(f"synthetic record-gap {failure_mode} failure") + + with monkeypatch.context() as fault: + fault.setattr(restarted_store, "record_gap", fail_record_gap) + with pytest.raises(restart_support.facade_module.ConnectionVerificationError): + _verify_restart( + restarted, + operation_id="op-d1d1d1d1-d1d1-41d1-81d1-d1d1d1d1d1d1", + ) + + checkpoint_after_failure = durable_store.snapshot().checkpoint + assert checkpoint_after_failure is not None + assert checkpoint_after_failure.state == "active" + assert checkpoint_after_failure.last_gap_recovered_at_utc is None + assert bool(checkpoint_after_failure.last_gap_started_at_utc) is ( + failure_mode == "after-commit" + ) + assert bool(committed_gap) is (failure_mode == "after-commit") + assert restarted._acquisition is None # noqa: SLF001 + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert runtime.start_calls == [] + assert command_calls == [] + + retried = _verify_restart( + restarted, + operation_id="op-d2d2d2d2-d2d2-42d2-82d2-d2d2d2d2d2d2", + ) + checkpoint_after_retry = durable_store.snapshot().checkpoint + assert checkpoint_after_retry is not None + assert checkpoint_after_retry.last_gap_started_at_utc is not None + assert checkpoint_after_retry.last_gap_recovered_at_utc is None + if committed_gap: + assert checkpoint_after_retry.last_gap_started_at_utc == ( + committed_gap[0].last_gap_started_at_utc + ) + assert retried["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + assert len(runtime.start_calls) == 1 + assert command_calls == [] + + +@pytest.mark.parametrize( + "forgery", + ("operation-id", "project-id", "retained-status"), +) +def test_restart_no_gap_descendant_admission_rejects_forged_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + forgery: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + restarted_store = restarted._active_acquisition_checkpoint # noqa: SLF001 + assert restarted_store is not None + + with monkeypatch.context() as fault: + fault.setattr( + restarted_store, + "record_gap", + lambda **_kwargs: (_ for _ in ()).throw( + OSError("synthetic no-gap publication failure") + ), + ) + with pytest.raises(restart_support.facade_module.ConnectionVerificationError): + _verify_restart( + restarted, + operation_id="op-d3d3d3d3-d3d3-43d3-83d3-d3d3d3d3d3d3", + ) + + checkpoint = durable_store.snapshot().checkpoint + record = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert checkpoint is not None + assert record is not None + assert checkpoint.last_gap_started_at_utc is None + assert restarted._active_checkpoint_allows_restart_start_descendant( # noqa: SLF001 + checkpoint=checkpoint, + record=record, + ) + + forged_record = copy.deepcopy(record) + if forgery == "operation-id": + forged_record = replace( + forged_record, + operation_id="start-forged-descendant-33333333-3333-4333-8333-333333333333", + ) + else: + latest = forged_record.reconciliations[-1] + if forgery == "project-id": + object.__setattr__( + latest.observation, + "project_id_sha256", + "d" * 64, + ) + else: + object.__setattr__(latest.observation, "mqtt_retained", True) + + assert not restarted._active_checkpoint_allows_restart_start_descendant( # noqa: SLF001 + checkpoint=checkpoint, + record=forged_record, + ) + + +def test_restart_lease_acquire_failure_preserves_open_gap_and_retryable_no_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + + def fail_lease_acquire( + _lease_type: type[object], + _sessions_root: Path, + _session_root: Path, + ) -> object: + raise OSError("synthetic active-session lease acquire failure") + + with monkeypatch.context() as fault: + fault.setattr( + restart_support.facade_module.ActiveSessionLease, + "acquire", + classmethod(fail_lease_acquire), + ) + with pytest.raises(restart_support.facade_module.ConnectionVerificationError): + _verify_restart( + restarted, + operation_id="op-e1e1e1e1-e1e1-41e1-81e1-e1e1e1e1e1e1", + ) + + open_gap = durable_store.snapshot().checkpoint + assert open_gap is not None + assert open_gap.state == "active" + assert open_gap.last_gap_started_at_utc is not None + assert open_gap.last_gap_recovered_at_utc is None + assert restarted._acquisition is None # noqa: SLF001 + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert runtime.start_calls == [] + assert command_calls == [] + + retried = _verify_restart( + restarted, + operation_id="op-e2e2e2e2-e2e2-42e2-82e2-e2e2e2e2e2e2", + ) + assert retried["acquisition"]["result"]["receiver_rehydration"] == ( + "awaiting-first-pcl" + ) + assert durable_store.snapshot().checkpoint is not None + assert ( + durable_store.snapshot().checkpoint.last_gap_started_at_utc + == open_gap.last_gap_started_at_utc + ) + assert len(runtime.start_calls) == 1 + assert command_calls == [] + + +def test_restart_suback_timeout_downgrades_without_waiting_or_device_edges( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = lifecycle_support.FakeVisualizationRuntime() + + def start_without_suback( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + + runtime.start_live = MethodType(start_without_suback, runtime) + restarted.runtime = runtime + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + monkeypatch.setattr( + restart_support.facade_module, + "RESTART_RECEIVER_SOURCE_READY_TIMEOUT_SECONDS", + 0.0, + ) + + result = _verify_restart( + restarted, + operation_id="op-e3e3e3e3-e3e3-43e3-83e3-e3e3e3e3e3e3", + ) + + assert result["acquisition"]["state"] == "failed" + recovery_result = result["acquisition"]["result"] + assert { + key: recovery_result[key] + for key in ( + "receiver_stopped", + "device_state", + "recovery_only", + "automatic_replay_allowed", + ) + } == { + "receiver_stopped": True, + "device_state": "scanning", + "recovery_only": True, + "automatic_replay_allowed": False, + } + assert not any(key.startswith("checkpoint_") for key in recovery_result) + private_checkpoint_values = { + durable_store.snapshot().checkpoint.current_evidence_session_id, + runtime.start_calls[0][1].name, + } + encoded_acquisition = json.dumps(result["acquisition"], sort_keys=True) + assert all(value not in encoded_acquisition for value in private_checkpoint_values) + checkpoint_owner = restarted._restart_stop_only_checkpoint_owner # noqa: SLF001 + assert checkpoint_owner is not None + assert checkpoint_owner[0] is restarted._acquisition # noqa: SLF001 + assert checkpoint_owner[1].original_start_operation_id == ( + restart_support.START_OPERATION_ID + ) + assert restarted._trusted_recovery_only_checkpoint_lineage( # noqa: SLF001 + restarted._acquisition # noqa: SLF001 + ) == checkpoint_owner[1] + assert runtime.start_calls + assert runtime.stop_calls == 1 + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert command_calls == [] + checkpoint = durable_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None + + +def test_restart_ingress_begin_failure_cleanly_downgrades_to_stop_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = _install_ready_capture_runtime(restarted) + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + monkeypatch.setattr( + restarted.live_perception_ingress, + "begin_session", + lambda _session_id: (_ for _ in ()).throw( + RuntimeError("synthetic ingress begin failure") + ), + ) + + result = _verify_restart( + restarted, + operation_id="op-f1f1f1f1-f1f1-41f1-81f1-f1f1f1f1f1f1", + ) + acquisition = result["acquisition"] + assert acquisition["state"] == "failed" + assert acquisition["result"]["recovery_only"] is True + assert acquisition["result"]["receiver_stopped"] is True + assert acquisition["result"]["automatic_replay_allowed"] is False + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert runtime.start_calls == [] + assert runtime.stop_calls == 1 + assert command_calls == [] + checkpoint = durable_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "active" + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None + + +@pytest.mark.parametrize("origin", ("active", "ambiguous-prepared")) +def test_restart_partial_start_failure_without_stop_downgrades_to_stop_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + origin: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + if origin == "active": + restart_support._activate_prepared_start(first, durable_store) + else: + first._physical_command_ledger.mark_dispatching( # noqa: SLF001 + restart_support.START_OPERATION_ID + ) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = lifecycle_support.FakeVisualizationRuntime() + + def partial_start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + raise RuntimeError("synthetic partial restart receiver failure") + + runtime.start_live = MethodType(partial_start_live, runtime) + restarted.runtime = runtime + restart_support._install_exact_restart_verify(restarted) + original_request_stop = lifecycle_support.FakeInteractiveControlSession.request_stop + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + + result = _verify_restart( + restarted, + operation_id=( + "op-a1a1a1a1-a1a1-41a1-81a1-a1a1a1a1a1a1" + if origin == "active" + else "op-a2a2a2a2-a2a2-42a2-82a2-a2a2a2a2a2a2" + ), + ) + acquisition = result["acquisition"] + assert acquisition["state"] == "failed" + assert acquisition["result"]["recovery_only"] is True + assert acquisition["result"]["receiver_stopped"] is True + assert acquisition["result"]["automatic_replay_allowed"] is False + assert restarted._acquisition_session_lease is None # noqa: SLF001 + assert restarted._acquisition_out_dir is None # noqa: SLF001 + assert restarted._restart_rehydration_pending is None # noqa: SLF001 + assert runtime.stop_calls == 1 + assert command_calls == [] + checkpoint = durable_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == ("active" if origin == "active" else "prepared") + + control = restarted._application_control_session.snapshot() # noqa: SLF001 + assert control["state"] == "scanning" + assert control["can_stop"] is True + stop_projection = result["connection_policy"]["actions"]["stop-acquisition"] + assert stop_projection["allowed"] is True + + if origin == "ambiguous-prepared": + control = restarted._application_control_session # noqa: SLF001 + coordinator = restarted._physical_command_coordinator # noqa: SLF001 + stop_payload = b"ambiguous-prepared-restart-explicit-stop" + explicit_stop_edges: list[str] = [] + + def request_explicit_stop( + *, + confirmation: object, + command_context: Any, + dispatch_admission_deadline_reached: Any = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + coordinator.prepare( + command_context, + action="stop", + envelope=OneShotPublishEnvelope( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload=stop_payload, + payload_sha256=hashlib.sha256(stop_payload).hexdigest(), + payload_bytes=len(stop_payload), + qos=2, + retain=False, + ), + ) + explicit_stop_edges.append(command_context.operation_id) + return original_request_stop( + control, + confirmation=confirmation, + command_context=command_context, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + control.request_stop = request_explicit_stop + control_snapshot = control.snapshot() + control_generation = control_snapshot["session_generation"] + control_revision = control_snapshot["state_revision"] + assert isinstance(control_generation, int) + assert isinstance(control_revision, int) + stopped = restarted.stop_acquisition( + StopAcquisitionRequest( + acquisition_id=restart_support.ACQUISITION_ID, + operation_id="op-a3a3a3a3-a3a3-43a3-83a3-a3a3a3a3a3a3", + idempotency_key="ambiguous-prepared-restart-explicit-stop", + mode="graceful", + expected_control_session_generation=control_generation, + expected_control_state_revision=control_revision, + physical_acceptance=lifecycle_support.PHYSICAL_ACCEPTANCE, + ) + ) + assert stopped["application_control_session"]["state"] == ( + "awaiting-standby-confirmation" + ) + assert explicit_stop_edges == [ + "op-a3a3a3a3-a3a3-43a3-83a3-a3a3a3a3a3a3" + ] + + +@pytest.mark.parametrize("cleanup_failure", ("raise", "non-idle")) +def test_restart_runtime_cleanup_failure_retains_exact_owner_and_stop_capability( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cleanup_failure: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = lifecycle_support.FakeVisualizationRuntime() + + def partial_start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + raise RuntimeError("synthetic partial restart receiver failure") + + runtime.start_live = MethodType(partial_start_live, runtime) + if cleanup_failure == "raise": + runtime.stop_error = RuntimeError("synthetic runtime cleanup failure") + else: + runtime.stop = lambda: setattr(runtime, "phase", "stopping") + restarted.runtime = runtime + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + + result = _verify_restart( + restarted, + operation_id=( + "op-b1b1b1b1-b1b1-41b1-81b1-b1b1b1b1b1b1" + if cleanup_failure == "raise" + else "op-b2b2b2b2-b2b2-42b2-82b2-b2b2b2b2b2b2" + ), + ) + pending, lease, out_dir = _assert_exact_restart_owner_retained(restarted) + assert result["acquisition"]["acquisition_id"] == restart_support.ACQUISITION_ID + assert result["acquisition"]["state"] == "awaiting_external_start" + assert restarted._restart_rehydration_pending is pending # noqa: SLF001 + assert restarted._acquisition_session_lease is lease # noqa: SLF001 + assert restarted._acquisition_out_dir == out_dir # noqa: SLF001 + assert out_dir.exists() + assert result["active_acquisition_recovery_checkpoint"]["reason_code"] == ( + "restart-receiver-cleanup-pending" + ) + assert command_calls == [] + checkpoint = durable_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None + assert result["connection_policy"]["actions"]["stop-acquisition"][ + "allowed" + ] is True + + +@pytest.mark.parametrize("cleanup_failure", ("ingress-end", "lease-release")) +def test_restart_post_stop_cleanup_failure_retains_exact_owner_and_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cleanup_failure: str, +) -> None: + first = restart_support._service(tmp_path, monkeypatch) + durable_store = restart_support._seed_matching_prepared_start(first) + restart_support._activate_prepared_start(first, durable_store) + + restarted = restart_support._service(tmp_path, monkeypatch) + runtime = lifecycle_support.FakeVisualizationRuntime() + + def partial_start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + raise RuntimeError("synthetic partial restart receiver failure") + + runtime.start_live = MethodType(partial_start_live, runtime) + restarted.runtime = runtime + restart_support._install_exact_restart_verify(restarted) + command_calls: list[str] = [] + _install_forbidden_restart_device_edges( + monkeypatch, + service=restarted, + calls=command_calls, + ) + if cleanup_failure == "ingress-end": + monkeypatch.setattr( + restarted.live_perception_ingress, + "end_session", + lambda _session_id: (_ for _ in ()).throw( + RuntimeError("synthetic ingress end failure") + ), + ) + else: + monkeypatch.setattr( + restart_support.facade_module.ActiveSessionLease, + "release", + lambda _lease: (_ for _ in ()).throw( + OSError("synthetic lease release failure") + ), + ) + + result = _verify_restart( + restarted, + operation_id=( + "op-c1c1c1c1-c1c1-41c1-81c1-c1c1c1c1c1c1" + if cleanup_failure == "ingress-end" + else "op-c2c2c2c2-c2c2-42c2-82c2-c2c2c2c2c2c2" + ), + ) + pending, lease, out_dir = _assert_exact_restart_owner_retained(restarted) + assert restarted._restart_rehydration_pending is pending # noqa: SLF001 + assert restarted._acquisition_session_lease is lease # noqa: SLF001 + assert restarted._acquisition_out_dir == out_dir # noqa: SLF001 + assert result["active_acquisition_recovery_checkpoint"]["reason_code"] == ( + "restart-receiver-cleanup-pending" + ) + assert command_calls == [] + checkpoint = durable_store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.last_gap_started_at_utc is not None + assert checkpoint.last_gap_recovered_at_utc is None diff --git a/tests/test_xgrids_active_acquisition_restart_safety.py b/tests/test_xgrids_active_acquisition_restart_safety.py new file mode 100644 index 0000000..e1901e8 --- /dev/null +++ b/tests/test_xgrids_active_acquisition_restart_safety.py @@ -0,0 +1,1269 @@ +from __future__ import annotations + +import asyncio +import hashlib +import importlib +import json +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from pathlib import Path +from types import MethodType +from typing import Any + +import pytest + +from k1link.device_plugins.xgrids_k1 import facade as facade_module +from k1link.device_plugins.xgrids_k1.active_acquisition_recovery_checkpoint import ( + ActiveAcquisitionRecoveryCheckpointError, + ActiveAcquisitionRecoveryCheckpointStore, + ActiveAcquisitionRecoveryConnection, + ActiveAcquisitionRecoveryIdentity, + ActiveAcquisitionRecoveryTransportBinding, + active_acquisition_project_name_sha256, +) +from k1link.device_plugins.xgrids_k1.facade import ( + XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + ConnectionVerifyRequest, + StopAcquisitionRequest, + XgridsK1CompatibilityService, +) +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + PhysicalCommandIntentContext, + PhysicalCommandRuntimeBinding, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + PhysicalCommandApplicationResponse, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandStatusEvidence, + PhysicalCommandVerifiedBinding, +) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) +from k1link.device_plugins.xgrids_k1.protocol.application_session import ( + OperatorPresenceConfirmation, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( + MODELING_STATE_BASE, +) +from k1link.web.device_lifecycle import AcquisitionRecord + +lifecycle_support: Any = importlib.import_module( + "test_xgrids_acquisition_lifecycle" +) +capture_support: Any = importlib.import_module("test_mqtt_capture") + +ACQUISITION_ID = "acq-restart-safety-11111111-1111-4111-8111-111111111111" +START_OPERATION_ID = "start-restart-safety-11111111-1111-4111-8111-111111111111" +FIRST_STOP_OPERATION_ID = "stop-restart-safety-22222222-2222-4222-8222-222222222222" +FINAL_STOP_OPERATION_ID = "op-33333333-3333-4333-8333-333333333333" +START_PAYLOAD_SHA256 = hashlib.sha256(b"restart-safety-start").hexdigest() +VENDOR_SHA256 = "a" * 64 +SERIAL_SHA256 = "b" * 64 +PROJECT_NAME = "RESTART_SAFETY" +EVIDENCE_SESSION_ID = "20260813T120000Z_restart_safety" + + +def _install_exact_restart_verify( + service: XgridsK1CompatibilityService, +) -> None: + """Drive public Verify with the real ledger/coordinator and no device I/O.""" + + lifecycle_support._seed_supervised_connection( # noqa: SLF001 + service, + target_ipv4="192.168.68.52", + transport_ref="transport-restart-safety", + logical_device_id="logical-restart-safety", + connection_mode="bridge", + with_control=False, + ) + with service._lock: # noqa: SLF001 + service._connection_verification = { # noqa: SLF001 + "lease_generation": service._connection_lease_generation, # noqa: SLF001 + "lease_state": "reachable", + } + + async def verify_topology_without_io( + bound_service: XgridsK1CompatibilityService, + request: ConnectionVerifyRequest, + ) -> tuple[str, bool, None]: + operation, created = bound_service._operations.begin( # noqa: SLF001 + facade_module.ACTION_CONNECTION_VERIFY, + operation_id=request.operation_id, + deadline_seconds=10.0, + ) + if created: + bound_service._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="topology-confirmed-device-info-pending", + message_code="connection.verify.device_info_pending", + ) + return operation.operation_id, created, None + + coordinator = service._physical_command_coordinator # noqa: SLF001 + + async def bootstrap_exact_control( + bound_service: XgridsK1CompatibilityService, + *, + parent_operation_id: str | None, + connection_mode: facade_module.ConnectionMode, + inspection_only: bool = False, + ) -> None: + coordinator.prepare_read_only_bootstrap() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key=f"bootstrap:{parent_operation_id}:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256="7" * 64, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-13T12:20:00.000Z", + ) + ) + await lifecycle_support._synthetic_prestart_control_bootstrap( # noqa: SLF001 + bound_service, + parent_operation_id=parent_operation_id, + connection_mode=connection_mode, + inspection_only=inspection_only, + ) + verified = bound_service._application_control_session.snapshot().get( # noqa: SLF001 + "verified_control" + ) + assert isinstance(verified, dict) + coordinator.bind_control_session( + PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + intent_id=str(verified["intent_id"]), + transport_ref=str(verified["transport_ref"]), + connection_mode=verified["connection_mode"], + target_ipv4=str(verified["target_ipv4"]), + target_port=int(verified["target_port"]), + host_path_epoch=int(verified["host_path_epoch"]), + control_session_id=str(verified["control_session_id"]), + producer_generation=int(verified["producer_generation"]), + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="scanning", + session_state_code=MODELING_STATE_BASE + 302, + project_bound=True, + project_id_sha256="e" * 64, + init_ready=True, + status_message_sha256="8" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:20:01.000Z", + ) + ) + bound_service._acquire_application_control_process_lease() # noqa: SLF001 + + service._verify_connection_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + verify_topology_without_io, + service, + ) + service._bootstrap_prestart_control_ready_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001 + bootstrap_exact_control, + service, + ) + + +def _service( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> XgridsK1CompatibilityService: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + monkeypatch.setenv("MISSIONCORE_EVIDENCE_DIR", str(tmp_path / "evidence")) + monkeypatch.setenv( + "MISSIONCORE_LEGACY_SESSIONS_DIR", + str(tmp_path / "legacy-sessions"), + ) + return XgridsK1CompatibilityService(tmp_path) + + +def _install_ready_restart_receiver( + service: XgridsK1CompatibilityService, +) -> Any: + """Install one deterministic receiver that proves its local SUBACK edge.""" + + runtime = lifecycle_support.FakeVisualizationRuntime() + + def start_live( + bound_runtime: Any, + host: str, + out_dir: Path, + *, + duration_seconds: float | None, + project_name: str, + recover_connection: Any = None, + ) -> None: + out_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + capture_dir = out_dir / "captures" / "mqtt_live" + capture_support.capture_mqtt( + host, + capture_dir, + duration_seconds=duration_seconds, + _client_factory=capture_support.FakeClient, + ) + lifecycle_support.FakeVisualizationRuntime.start_live( + bound_runtime, + host, + out_dir, + duration_seconds=duration_seconds, + project_name=project_name, + recover_connection=recover_connection, + ) + bound_runtime.mark_ready() + + runtime.start_live = MethodType(start_live, runtime) + service.runtime = runtime + return runtime + + +def _activate_retained_fixture_checkpoint( + service: XgridsK1CompatibilityService, + fixture: Any, +) -> ActiveAcquisitionRecoveryCheckpointStore: + """Require or add the retained fixture's exact typed ACTIVE checkpoint.""" + + store = service._active_acquisition_checkpoint # noqa: SLF001 + acquisition = service._acquisition # noqa: SLF001 + out_dir = service._acquisition_out_dir # noqa: SLF001 + record = fixture.ledger.snapshot().record + assert store is not None + assert acquisition is not None + assert out_dir is not None + assert record is not None and record.last_status is not None + snapshot = store.snapshot() + if snapshot.status == "active": + checkpoint = snapshot.checkpoint + assert checkpoint is not None + assert checkpoint.acquisition_id == acquisition.acquisition_id + assert checkpoint.original_start_operation_id == record.operation_id + assert checkpoint.project_name == "STOP_RACE" + assert checkpoint.original_evidence_session_id == out_dir.name + assert checkpoint.current_active_status_proof is not None + assert checkpoint.current_active_physical_proof is not None + service._active_acquisition_checkpoint_trust = "trusted" # noqa: SLF001 + service._active_acquisition_checkpoint_reason = None # noqa: SLF001 + return store + assert snapshot.status == "empty" + binding = ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=service._snapshot_runtime_id, # noqa: SLF001 + intent_id=record.connection.intent_id, + transport_ref=record.connection.transport_ref, + connection_mode=record.connection.connection_mode, + target_ipv4=record.connection.target_ipv4, + target_port=record.connection.target_port, + host_path_epoch=record.connection.host_path_epoch, + control_session_id=record.connection.control_session_id, + producer_generation=record.connection.producer_generation, + logical_device_id=acquisition.device_id, + compatibility_profile_id=record.compatibility_profile_id, + vendor_device_id_sha256=record.identity.vendor_device_id_sha256, + device_serial_sha256=record.identity.device_serial_sha256, + ) + prepared = store.prepare( + transition_id="restart-safety-retained-prepare", + predecessor_revision=0, + acquisition_id=acquisition.acquisition_id, + original_start_operation_id=record.operation_id, + start_payload_sha256=record.payload_sha256, + identity=ActiveAcquisitionRecoveryIdentity( + logical_device_id=acquisition.device_id, + vendor_device_id_sha256=record.identity.vendor_device_id_sha256, + device_serial_sha256=record.identity.device_serial_sha256, + ), + connection=ActiveAcquisitionRecoveryConnection( + transport_ref=record.connection.transport_ref, + connection_mode=record.connection.connection_mode, + target_ipv4=record.connection.target_ipv4, + target_port=record.connection.target_port, + ), + compatibility_profile_id=record.compatibility_profile_id, + project_name="STOP_RACE", + project_name_wire_sha256=active_acquisition_project_name_sha256( + "STOP_RACE" + ), + original_evidence_session_id=out_dir.name, + duration_seconds=acquisition.duration_seconds, + requested_streams=acquisition.requested_streams, + evidence_policy=acquisition.evidence_policy, + mount_type="handheld", + gnss_mode="none", + prepared_binding=binding, + ) + status_proof = service._checkpoint_status_proof( # noqa: SLF001 + status=record.last_status, + binding=binding, + evidence_session_id=out_dir.name, + ) + store.activate( + transition_id="restart-safety-retained-activate", + expected_revision=prepared.revision, + expected_acquisition_id=prepared.acquisition_id, + expected_start_operation_id=prepared.original_start_operation_id, + status_proof=status_proof, + physical_proof=service._checkpoint_physical_proof( # noqa: SLF001 + record=record, + binding=binding, + checkpoint=prepared, + ), + ) + service._active_acquisition_checkpoint_trust = "trusted" # noqa: SLF001 + service._active_acquisition_checkpoint_reason = None # noqa: SLF001 + return store + + +def _prepared_connection() -> PhysicalCommandConnectionBinding: + return PhysicalCommandConnectionBinding( + intent_id="intent-restart-safety", + transport_ref="transport-restart-safety", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=1, + control_session_id="control-restart-safety", + producer_generation=1, + ) + + +def _ready_status( + connection: PhysicalCommandConnectionBinding, +) -> PhysicalCommandStatusEvidence: + return PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="c" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:00:00.000Z", + ) + + +def _scanning_status( + connection: PhysicalCommandConnectionBinding, + *, + source: str, + observed_at_utc: str, +) -> PhysicalCommandStatusEvidence: + return PhysicalCommandStatusEvidence( + source=source, # type: ignore[arg-type] + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state="scanning", + session_state_code=302, + project_bound=True, + project_id_sha256="e" * 64, + init_ready=True, + status_message_sha256=hashlib.sha256(observed_at_utc.encode()).hexdigest(), + mqtt_retained=False, + observed_at_utc=observed_at_utc, + ) + + +def _seed_matching_prepared_start( + service: XgridsK1CompatibilityService, + *, + checkpoint_payload_sha256: str = START_PAYLOAD_SHA256, + parent_operation_id: str | None = None, +) -> ActiveAcquisitionRecoveryCheckpointStore: + connection = _prepared_connection() + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + service._physical_command_ledger.prepare( # noqa: SLF001 + operation_id=START_OPERATION_ID, + parent_operation_id=parent_operation_id, + acquisition_id=ACQUISITION_ID, + action="start", + identity=identity, + connection=connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256=START_PAYLOAD_SHA256, + baseline_status=_ready_status(connection), + ) + store = service._active_acquisition_checkpoint # noqa: SLF001 + assert store is not None + store.prepare( + transition_id="prepare-restart-safety", + predecessor_revision=0, + acquisition_id=ACQUISITION_ID, + original_start_operation_id=START_OPERATION_ID, + start_payload_sha256=checkpoint_payload_sha256, + identity=ActiveAcquisitionRecoveryIdentity( + logical_device_id="logical-restart-safety", + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ), + connection=ActiveAcquisitionRecoveryConnection( + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + ), + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + project_name=PROJECT_NAME, + project_name_wire_sha256=active_acquisition_project_name_sha256( + PROJECT_NAME + ), + original_evidence_session_id=EVIDENCE_SESSION_ID, + duration_seconds=None, + requested_streams=("pointcloud",), + evidence_policy="required", + mount_type="handheld", + gnss_mode="none", + prepared_binding=ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=service._snapshot_runtime_id, # noqa: SLF001 + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + host_path_epoch=connection.host_path_epoch, + control_session_id=connection.control_session_id, + producer_generation=connection.producer_generation, + logical_device_id="logical-restart-safety", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ), + ) + return store + + +def _activate_prepared_start( + service: XgridsK1CompatibilityService, + store: ActiveAcquisitionRecoveryCheckpointStore, +) -> None: + ledger = service._physical_command_ledger # noqa: SLF001 + connection = _prepared_connection() + ledger.mark_dispatching(START_OPERATION_ID) + ledger.mark_observing( + START_OPERATION_ID, + publish_call_returned=True, + packet_id=41, + ) + ledger.mark_qos2_completed(START_OPERATION_ID, packet_id=41) + ledger.record_application_response( + START_OPERATION_ID, + PhysicalCommandApplicationResponse( + operation_id=START_OPERATION_ID, + action="start", + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="f" * 64, + observed_at_utc="2026-08-13T12:00:01.000Z", + ), + ) + scanning = _scanning_status( + connection, + source="live-control-session", + observed_at_utc="2026-08-13T12:00:02.000Z", + ) + ledger.record_status_observation(START_OPERATION_ID, scanning) + record = ledger.resolve( + START_OPERATION_ID, + resolution="start-active-observed", + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + store.activate( + transition_id="activate-restart-safety", + expected_revision=checkpoint.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=service._checkpoint_status_proof( # noqa: SLF001 + status=scanning, + binding=checkpoint.prepared_binding, + evidence_session_id=EVIDENCE_SESSION_ID, + ), + physical_proof=service._checkpoint_physical_proof( # noqa: SLF001 + record=record, + binding=checkpoint.prepared_binding, + checkpoint=checkpoint, + ), + ) + + +def _rebind_connection( + index: int, +) -> PhysicalCommandConnectionBinding: + return PhysicalCommandConnectionBinding( + intent_id="intent-restart-safety", + transport_ref="transport-restart-safety", + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=index + 1, + control_session_id=f"control-restart-safety-{index}", + producer_generation=index + 1, + ) + + +def _checkpoint_binding( + service: XgridsK1CompatibilityService, + connection: PhysicalCommandConnectionBinding, +) -> ActiveAcquisitionRecoveryTransportBinding: + return ActiveAcquisitionRecoveryTransportBinding( + runtime_instance_id=service._snapshot_runtime_id, # noqa: SLF001 + intent_id=connection.intent_id, + transport_ref=connection.transport_ref, + connection_mode=connection.connection_mode, + target_ipv4=connection.target_ipv4, + target_port=connection.target_port, + host_path_epoch=connection.host_path_epoch, + control_session_id=connection.control_session_id, + producer_generation=connection.producer_generation, + logical_device_id="logical-restart-safety", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + + +def _classify_stop_active_and_rebind( + service: XgridsK1CompatibilityService, + store: ActiveAcquisitionRecoveryCheckpointStore, + *, + operation_id: str, + parent_operation_id: str, + current_connection: PhysicalCommandConnectionBinding, + recovered_connection: PhysicalCommandConnectionBinding, + sequence: int, + operator_confirmation: bool, +) -> str: + ledger = service._physical_command_ledger # noqa: SLF001 + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + ledger.prepare( + operation_id=operation_id, + parent_operation_id=parent_operation_id, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=current_connection, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256=hashlib.sha256(operation_id.encode()).hexdigest(), + baseline_status=_scanning_status( + current_connection, + source="live-control-session", + observed_at_utc=f"2026-08-13T12:0{sequence}:00.000Z", + ), + operator_confirmation_id=( + f"confirmation-restart-safety-{sequence}" + if operator_confirmation + else None + ), + operator_confirmed_at_utc=( + f"2026-08-13T12:0{sequence}:01.000Z" + if operator_confirmation + else None + ), + ) + reconciliation_id = f"classification-restart-safety-{sequence}" + observation = _scanning_status( + recovered_connection, + source="explicit-read-only-reconciliation", + observed_at_utc=f"2026-08-13T12:0{sequence}:03.000Z", + ) + prepared = ledger.snapshot().record + assert prepared is not None + record = ledger.reconcile_prepared_stop_not_dispatched( + operation_id, + expected_revision=prepared.revision, + reconciliation_id=reconciliation_id, + verified_binding=PhysicalCommandVerifiedBinding( + verification_id=f"verification-restart-safety-{sequence}", + identity=identity, + connection=recovered_connection, + device_info_message_sha256=hashlib.sha256( + f"device-info-{sequence}".encode() + ).hexdigest(), + verified_at_utc=f"2026-08-13T12:0{sequence}:02.000Z", + ), + observation=observation, + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + store.record_gap( + transition_id=f"gap-restart-safety-{sequence}", + expected_revision=checkpoint.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=checkpoint.current_binding, + gap_started_at_utc=f"2026-08-13T12:0{sequence}:01.500Z", + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + binding = _checkpoint_binding(service, recovered_connection) + reconciliation = record.reconciliations[-1] + store.rebind_active( + transition_id=f"rebind-restart-safety-{sequence}", + expected_revision=checkpoint.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + status_proof=service._checkpoint_status_proof( # noqa: SLF001 + status=observation, + binding=binding, + evidence_session_id=EVIDENCE_SESSION_ID, + ), + physical_proof=service._checkpoint_physical_proof( # noqa: SLF001 + record=record, + binding=binding, + checkpoint=checkpoint, + reconciliation=reconciliation, + ), + ) + return reconciliation_id + + +def test_restart_settles_exact_prepared_start_after_no_dispatch_without_device_io( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start(first) + device_edges: list[str] = [] + + async def forbidden_device_io(*_args: object, **_kwargs: object) -> object: + device_edges.append("device-io") + raise AssertionError("startup checkpoint settlement must not touch K1") + + monkeypatch.setattr(facade_module, "scan", forbidden_device_io) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_io) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_io) + + restarted = _service(tmp_path, monkeypatch) + physical = restarted._physical_command_ledger.snapshot() # noqa: SLF001 + checkpoint = store.snapshot().checkpoint + + assert device_edges == [] + assert physical.status == "resolved" + assert physical.record is not None + assert physical.record.operation_id == START_OPERATION_ID + assert physical.record.resolution == "not-dispatched" + assert checkpoint is not None + assert checkpoint.state == "ceased" + assert checkpoint.prepared_resolution_proof is not None + assert checkpoint.prepared_resolution_proof.operation_id == START_OPERATION_ID + assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001 + + # Lost-response/idempotent startup repetition does not advance the receipt. + revision = checkpoint.revision + _service(tmp_path, monkeypatch) + repeated = store.snapshot().checkpoint + assert repeated is not None + assert repeated.revision == revision + + +def test_restart_mismatch_preserves_checkpoint_bytes_and_revokes_start_admission( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start( + first, + checkpoint_payload_sha256="d" * 64, + ) + checkpoint_bytes = store.path.read_bytes() + + restarted = _service(tmp_path, monkeypatch) + + assert store.path.read_bytes() == checkpoint_bytes + assert restarted._active_acquisition_checkpoint_trust == "unavailable" # noqa: SLF001 + projection = restarted._active_acquisition_checkpoint_public_snapshot() # noqa: SLF001 + assert projection["status"] == "unavailable" + assert projection["restart_resume_authority"] is False + policy = restarted.state()["connection_policy"] + assert policy["actions"]["start-acquisition"]["allowed"] is False + assert "start-acquisition" not in policy["allowed_actions"] + with pytest.raises(ActiveAcquisitionRecoveryCheckpointError): + restarted._require_active_acquisition_checkpoint_store() # noqa: SLF001 + + +def test_restart_recomputes_chained_stop_certificate_and_verify_rechecks_it( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start(first) + _activate_prepared_start(first, store) + initial = _prepared_connection() + first_rebind = _rebind_connection(1) + _classify_stop_active_and_rebind( + first, + store, + operation_id=FIRST_STOP_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + current_connection=initial, + recovered_connection=first_rebind, + sequence=1, + operator_confirmation=False, + ) + second_rebind = _rebind_connection(2) + _classify_stop_active_and_rebind( + first, + store, + operation_id=FINAL_STOP_OPERATION_ID, + parent_operation_id=FIRST_STOP_OPERATION_ID, + current_connection=first_rebind, + recovered_connection=second_rebind, + sequence=2, + operator_confirmation=True, + ) + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.current_active_physical_proof is not None + assert checkpoint.current_active_physical_proof.ancestor_chain is not None + + restarted = _service(tmp_path, monkeypatch) + assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001 + + monkeypatch.setattr( + restarted._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: { + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": False, + "resolved_scan_over_recovery_required": False, + "observed_session_state": None, + }, + ) + result = asyncio.run( + restarted._reconcile_physical_command_after_verify_owned( # noqa: SLF001 + verify_operation_id="verify-restart-safety" + ) + ) + assert result["performed"] is False + assert restarted._active_acquisition_checkpoint_trust == "trusted" # noqa: SLF001 + + document = json.loads(store.path.read_text(encoding="utf-8")) + for field in ( + "current_active_physical_proof", + "last_rebind_physical_proof", + ): + proof = document[field] + assert proof["ancestor_chain"] is not None + proof["ancestor_chain"]["chain_sha256"] = "0" * 64 + store.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + store.path.chmod(0o600) + mismatched_bytes = store.path.read_bytes() + + mismatched = _service(tmp_path, monkeypatch) + + assert store.path.read_bytes() == mismatched_bytes + assert mismatched._active_acquisition_checkpoint_trust == "unavailable" # noqa: SLF001 + assert ( + mismatched._active_acquisition_checkpoint_public_snapshot()["status"] # noqa: SLF001 + == "unavailable" + ) + + +def test_restart_trust_token_rejects_real_checkpoint_cas_change_but_verify_remains_allowed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start(first) + _activate_prepared_start(first, store) + restarted = _service(tmp_path, monkeypatch) + + token = restarted._validate_active_acquisition_checkpoint_lineage() # noqa: SLF001 + assert token is not None + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + store.record_gap( + transition_id="gap-after-trust-token", + expected_revision=checkpoint.revision, + expected_acquisition_id=ACQUISITION_ID, + expected_start_operation_id=START_OPERATION_ID, + failed_binding=checkpoint.current_binding, + gap_started_at_utc="2026-08-13T12:10:00.000Z", + ) + + assert not restarted._active_acquisition_checkpoint_trust_token_is_current( # noqa: SLF001 + token + ) + assert restarted._active_acquisition_checkpoint_trust == "unavailable" # noqa: SLF001 + monkeypatch.setattr( + restarted._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: { + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": False, + "resolved_scan_over_recovery_required": False, + "observed_session_state": None, + }, + ) + verified = asyncio.run( + restarted._reconcile_physical_command_after_verify_owned( # noqa: SLF001 + verify_operation_id="verify-after-checkpoint-cas" + ) + ) + assert verified["performed"] is False + assert restarted.state()["connection_policy"]["actions"]["start-acquisition"][ + "allowed" + ] is False + + +def test_retained_adoption_rechecks_real_checkpoint_cas_inside_lifecycle_gate( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, runtime = lifecycle_support.service_with_fake_runtime(tmp_path) + fixture = lifecycle_support._install_real_prepared_stop_dispatch_fixture( # noqa: SLF001 + service, + runtime, + ) + store = _activate_retained_fixture_checkpoint(service, fixture) + service.stop_acquisition(fixture.request) + owner = lifecycle_support._settle_real_prepared_stop_before_publish( # noqa: SLF001 + fixture + ) + assert owner.checkpoint_trust_token is not None + reconciliation_id = "restart-safety-retained-cas-boundary" + reconciled = lifecycle_support._classify_real_prepared_stop( # noqa: SLF001 + fixture, + owner, + session_state="scanning", + reconciliation_id=reconciliation_id, + ) + verified_binding = reconciled["reconciliations"][-1]["verified_binding"][ + "connection" + ] + fixture.control.state = "scanning" + fixture.control.state_revision += 1 + fixture.control.failure = None + fixture.control.verified_control = { + **dict(verified_binding), + "logical_device_id": "known-k1", + "compatibility_profile_id": XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + } + service._device_session_id = "restart-safety-cas-device-session" # noqa: SLF001 + + original_access = service._acquisition_lifecycle_access # noqa: SLF001 + mutation_count = 0 + + @contextmanager + def mutate_checkpoint_after_gate_acquisition() -> Iterator[None]: + nonlocal mutation_count + with original_access(): + if mutation_count == 0: + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + store.record_gap( + transition_id="restart-safety-retained-cas-gap", + expected_revision=checkpoint.revision, + expected_acquisition_id=checkpoint.acquisition_id, + expected_start_operation_id=( + checkpoint.original_start_operation_id + ), + failed_binding=checkpoint.current_binding, + gap_started_at_utc="2026-08-13T12:30:00.000Z", + ) + mutation_count += 1 + yield + + monkeypatch.setattr( + service, + "_acquisition_lifecycle_access", + mutate_checkpoint_after_gate_acquisition, + ) + adopted = service._adopt_classified_prepared_stop_scanning_in_place( # noqa: SLF001 + owner=owner, + reconciliation_id=reconciliation_id, + reconciled_record=reconciled, + checkpoint_trust_token=owner.checkpoint_trust_token, + ) + + assert adopted is False + assert mutation_count == 1 + assert service._active_acquisition_checkpoint_trust == "unavailable" # noqa: SLF001 + assert service._prepared_stop_recovery_owner is None # noqa: SLF001 + assert service._acquisition_out_dir is None # noqa: SLF001 + assert service._acquisition is not owner.acquisition # noqa: SLF001 + assert service._acquisition is not None # noqa: SLF001 + assert service._acquisition.state == "failed" # noqa: SLF001 + assert service._acquisition.result is not None # noqa: SLF001 + assert service._acquisition.result["recovery_only"] is True # noqa: SLF001 + assert service._acquisition.result["automatic_replay_allowed"] is False # noqa: SLF001 + + +def test_trusted_s2_restart_verify_stop_ready_ceases_checkpoint( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start(first) + _activate_prepared_start(first, store) + initial = _prepared_connection() + first_rebind = _rebind_connection(1) + _classify_stop_active_and_rebind( + first, + store, + operation_id=FIRST_STOP_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + current_connection=initial, + recovered_connection=first_rebind, + sequence=1, + operator_confirmation=False, + ) + second_rebind = _rebind_connection(2) + _classify_stop_active_and_rebind( + first, + store, + operation_id=FINAL_STOP_OPERATION_ID, + parent_operation_id=FIRST_STOP_OPERATION_ID, + current_connection=first_rebind, + recovered_connection=second_rebind, + sequence=2, + operator_confirmation=True, + ) + + restarted = _service(tmp_path, monkeypatch) + _install_ready_restart_receiver(restarted) + _install_exact_restart_verify(restarted) + verify_operation_id = "op-44444444-4444-4444-8444-444444444444" + verified = asyncio.run( + restarted.verify_connection( + ConnectionVerifyRequest( + device_id="transport-restart-safety", + source="retained-current-process", + compatibility_attestation=lifecycle_support.ATTESTATION, + operation_id=verify_operation_id, + ) + ) + ) + acquisition = verified["acquisition"] + assert acquisition["acquisition_id"] == ACQUISITION_ID + assert acquisition["result"]["read_only_recovery"] is True + assert acquisition["result"]["receiver_rehydration"] == "awaiting-first-pcl" + assert not any(key.startswith("checkpoint_") for key in acquisition["result"]) + assert EVIDENCE_SESSION_ID not in json.dumps(acquisition, sort_keys=True) + checkpoint_owner = restarted._restart_stop_only_checkpoint_owner # noqa: SLF001 + assert checkpoint_owner is None + + coordinator = restarted._physical_command_coordinator # noqa: SLF001 + control = restarted._application_control_session # noqa: SLF001 + original_request_stop = control.request_stop + stop_payload = b"trusted-restart-stop" + stop_payload_sha256 = hashlib.sha256(stop_payload).hexdigest() + + def request_stop_with_durable_edge( + *, + confirmation: OperatorPresenceConfirmation, + command_context: PhysicalCommandIntentContext | None, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + expected_session_generation: int | None = None, + expected_state_revision: int | None = None, + ) -> dict[str, object]: + assert command_context is not None + coordinator.prepare( + command_context, + action="stop", + envelope=OneShotPublishEnvelope( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload=stop_payload, + payload_sha256=stop_payload_sha256, + payload_bytes=len(stop_payload), + qos=2, + retain=False, + ), + ) + return original_request_stop( + confirmation=confirmation, + command_context=command_context, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + expected_session_generation=expected_session_generation, + expected_state_revision=expected_state_revision, + ) + + control.request_stop = request_stop_with_durable_edge # type: ignore[assignment,method-assign] + control_snapshot = control.snapshot() + control_generation = control_snapshot["session_generation"] + control_revision = control_snapshot["state_revision"] + assert isinstance(control_generation, int) and not isinstance( + control_generation, + bool, + ) + assert isinstance(control_revision, int) and not isinstance( + control_revision, + bool, + ) + stop_operation_id = "op-55555555-5555-4555-8555-555555555555" + restarted.stop_acquisition( + StopAcquisitionRequest( + acquisition_id=ACQUISITION_ID, + operation_id=stop_operation_id, + idempotency_key="trusted-restart-stop-once", + mode="graceful", + expected_control_session_generation=control_generation, + expected_control_state_revision=control_revision, + physical_acceptance=lifecycle_support.PHYSICAL_ACCEPTANCE, + ) + ) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload_sha256=stop_payload_sha256, + qos=2, + retain=False, + packet_id=None, + ) + coordinator.publish_dispatching(dispatch) + returned = ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic="lixel/application/request/modeling", + payload_sha256=stop_payload_sha256, + qos=2, + retain=False, + packet_id=61, + ) + coordinator.publish_result(returned, publish_call_returned=True) + coordinator.qos2_completed(returned) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="modeling:stop", + response_topic="lixel/application/response/modeling", + payload_sha256="9" * 64, + modeling_action="stop", + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + observed_at_utc="2026-08-13T12:20:02.000Z", + ) + ) + coordinator.device_status( + ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + session_state="ready", + session_state_code=MODELING_STATE_BASE + 300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="0" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:20:03.000Z", + ) + ) + coordinator.resolve("stop") + fake_control: Any = control + fake_control.state = "completed" + terminal = restarted.state() + + checkpoint = store.snapshot().checkpoint + assert checkpoint is not None + assert checkpoint.state == "ceased" + assert checkpoint.original_start_operation_id == START_OPERATION_ID + assert checkpoint.current_evidence_session_id != EVIDENCE_SESSION_ID + assert control.start_projects == [] + assert control.stop_calls == 1 + assert restarted._operations.get(stop_operation_id).status == "succeeded" # noqa: SLF001 + assert restarted._acquisition_stop_operation_id is None # noqa: SLF001 + assert terminal["physical_command"]["record"]["resolution"] == ( + "stop-standby-observed" + ) + + +def test_corrupt_checkpoint_allows_only_exact_recovery_stop_local_teardown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _service(tmp_path, monkeypatch) + store = _seed_matching_prepared_start(first) + _activate_prepared_start(first, store) + initial = _prepared_connection() + recovered = _rebind_connection(1) + reconciliation_id = _classify_stop_active_and_rebind( + first, + store, + operation_id=FIRST_STOP_OPERATION_ID, + parent_operation_id=START_OPERATION_ID, + current_connection=initial, + recovered_connection=recovered, + sequence=1, + operator_confirmation=False, + ) + ledger = first._physical_command_ledger # noqa: SLF001 + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + ) + ledger.prepare( + operation_id=FINAL_STOP_OPERATION_ID, + parent_operation_id=FIRST_STOP_OPERATION_ID, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=identity, + connection=recovered, + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + payload_sha256="1" * 64, + baseline_status=_scanning_status( + recovered, + source="live-control-session", + observed_at_utc="2026-08-13T12:03:00.000Z", + ), + operator_confirmation_id="confirmation-terminal-restart-safety", + operator_confirmed_at_utc="2026-08-13T12:03:01.000Z", + ) + ledger.mark_dispatching(FINAL_STOP_OPERATION_ID) + ledger.mark_observing( + FINAL_STOP_OPERATION_ID, + publish_call_returned=True, + packet_id=51, + ) + ledger.mark_qos2_completed(FINAL_STOP_OPERATION_ID, packet_id=51) + ledger.record_application_response( + FINAL_STOP_OPERATION_ID, + PhysicalCommandApplicationResponse( + operation_id=FINAL_STOP_OPERATION_ID, + action="stop", + control_session_id=recovered.control_session_id, + host_path_epoch=recovered.host_path_epoch, + producer_generation=recovered.producer_generation, + result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + success=True, + payload_sha256="2" * 64, + observed_at_utc="2026-08-13T12:03:02.000Z", + ), + ) + ready = PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=VENDOR_SHA256, + device_serial_sha256=SERIAL_SHA256, + control_session_id=recovered.control_session_id, + host_path_epoch=recovered.host_path_epoch, + producer_generation=recovered.producer_generation, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="3" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-13T12:03:03.000Z", + ) + ledger.record_status_observation(FINAL_STOP_OPERATION_ID, ready) + ledger.resolve( + FINAL_STOP_OPERATION_ID, + resolution="stop-standby-observed", + ) + + store.path.write_text("{\"truncated\":", encoding="utf-8") + store.path.chmod(0o600) + corrupt_bytes = store.path.read_bytes() + restarted = _service(tmp_path, monkeypatch) + assert restarted._active_acquisition_checkpoint_trust == "corrupt" # noqa: SLF001 + recovery = AcquisitionRecord( + acquisition_id=ACQUISITION_ID, + device_id="logical-restart-safety", + device_session_id="device-session-restart-safety", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_mode="plugin-commanded", + requested_streams=(), + target_host=recovered.target_ipv4, + duration_seconds=None, + evidence_policy="disabled", + state="failed", + message_code="acquisition.physical_active_recovered_stop_only", + result={ + "receiver_stopped": True, + "device_state": "scanning", + "recovery_only": True, + "physical_command_operation_id": FIRST_STOP_OPERATION_ID, + "physical_reconciliation_id": reconciliation_id, + "project_id_sha256": "e" * 64, + "automatic_replay_allowed": False, + }, + ) + with restarted._lock: # noqa: SLF001 + restarted._acquisition = recovery # noqa: SLF001 + restarted._acquisition_out_dir = None # noqa: SLF001 + restarted._acquisition_session_lease = None # noqa: SLF001 + restarted._acquisition_start_operation_id = None # noqa: SLF001 + restarted._acquisition_stop_operation_id = FINAL_STOP_OPERATION_ID # noqa: SLF001 + operation, created = restarted._operations.begin( # noqa: SLF001 + facade_module.ACTION_ACQUISITION_STOP, + operation_id=FINAL_STOP_OPERATION_ID, + device_id=recovery.device_id, + device_session_id=recovery.device_session_id, + deadline_seconds=60.0, + ) + assert created + restarted._operations.transition( # noqa: SLF001 + operation.operation_id, + "running", + stage_code="awaiting-external-stop", + message_code="acquisition.stop.device_stopping", + ) + physical = restarted._physical_command_coordinator.snapshot() # noqa: SLF001 + assert restarted._recovery_only_terminal_without_trusted_checkpoint( # noqa: SLF001 + physical_command_proof=physical, + acquisition_id=ACQUISITION_ID, + stop_operation_id=FINAL_STOP_OPERATION_ID, + ) + assert not restarted._recovery_only_terminal_without_trusted_checkpoint( # noqa: SLF001 + physical_command_proof=physical, + acquisition_id=ACQUISITION_ID, + stop_operation_id="wrong-stop-operation", + ) + assert recovery.result is not None + recovery.result["physical_reconciliation_id"] = "wrong-parent-reconciliation" + assert not restarted._recovery_only_terminal_without_trusted_checkpoint( # noqa: SLF001 + physical_command_proof=physical, + acquisition_id=ACQUISITION_ID, + stop_operation_id=FINAL_STOP_OPERATION_ID, + ) + recovery.result["physical_reconciliation_id"] = reconciliation_id + + restarted._reconcile_acquisition( # noqa: SLF001 + restarted.runtime.snapshot(), + restarted.camera_preview.snapshot(), + restarted._application_control_session.snapshot(), # noqa: SLF001 + physical_command_proof=physical, + ) + + assert restarted._operations.get(FINAL_STOP_OPERATION_ID).status == "succeeded" # noqa: SLF001 + assert restarted._acquisition_stop_operation_id is None # noqa: SLF001 + assert store.path.read_bytes() == corrupt_bytes + assert restarted._active_acquisition_checkpoint_trust == "corrupt" # noqa: SLF001 + policy = restarted.state()["connection_policy"] + assert policy["actions"]["start-acquisition"]["allowed"] is False diff --git a/tests/test_xgrids_ap_activation.py b/tests/test_xgrids_ap_activation.py index 3fadd41..e388f45 100644 --- a/tests/test_xgrids_ap_activation.py +++ b/tests/test_xgrids_ap_activation.py @@ -1,9 +1,11 @@ import asyncio from collections.abc import Iterator +from pathlib import Path from types import SimpleNamespace from typing import Any import pytest +from bleak.backends.device import BLEDevice from bleak.exc import BleakDeviceNotFoundError, BleakGATTProtocolError import k1link.device_plugins.xgrids_k1.ble.ap_activation as ap_module @@ -15,20 +17,25 @@ from k1link.device_plugins.xgrids_k1.ble.ap_activation import ( build_ap_activation_frame, is_ap_ready_status, ) +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleRuntimeBusy, + bind_ble_runtime_owner_loop, + ble_runtime_snapshot, + configure_ble_runtime_process_lease, + reset_ble_runtime_arbiter_for_tests, + wait_for_ble_runtime_idle, +) from k1link.device_plugins.xgrids_k1.ble.wifi_provisioning import WifiStatus @pytest.fixture(autouse=True) -def reset_runtime_handle_lease() -> Iterator[None]: - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 +def reset_runtime_handle_lease(tmp_path: Path) -> Iterator[None]: + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + configure_ble_runtime_process_lease(tmp_path) yield - with scanner_module._runtime_handle_lock: # noqa: SLF001 - scanner_module._runtime_handles.clear() # noqa: SLF001 - scanner_module._runtime_handle_observed_at_monotonic = None # noqa: SLF001 - scanner_module._runtime_handle_generation = 0 # noqa: SLF001 + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() def _seed_scan_lease(handles: dict[str, object], *, observed_at: float) -> None: @@ -70,7 +77,7 @@ def test_ap_activation_uses_retained_handle_without_rediscovery( monkeypatch: pytest.MonkeyPatch, ) -> None: device_id = "synthetic-corebluetooth-uuid" - retained_handle = object() + retained_handle = BLEDevice(device_id, "XGR-K1", details=object()) rediscovery_calls: list[tuple[str, float]] = [] client_calls: list[object] = [] @@ -95,19 +102,39 @@ def test_ap_activation_uses_retained_handle_without_rediscovery( ) monkeypatch.setattr(ap_module, "BleakClient", CapturingClient) - with pytest.raises(SelectedHandleObserved) as caught: - asyncio.run( - ap_module.activate_device_ap_once( + async def scenario() -> SelectedHandleObserved: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=retained_handle, + macos_uuid=device_id, + owner_epoch=owner_epoch, + ) + scanner_module.pin_connected_device_handle( + captured, + device_session_id="device-session-a", + ) + with pytest.raises(SelectedHandleObserved) as caught: + await ap_module.activate_device_ap_once( device_id, timeout_seconds=1.0, + captured_device=captured, ) + assert ( + scanner_module.connected_device_capture( + device_id, + device_session_id="device-session-a", + ) + is None ) + return caught.value + + error = asyncio.run(scenario()) assert rediscovery_calls == [] assert client_calls == [retained_handle] - assert caught.value.operation_stage == "connect" # type: ignore[attr-defined] - assert caught.value.device_write_attempted is False # type: ignore[attr-defined] - assert caught.value.device_write_confirmed is False # type: ignore[attr-defined] + assert error.operation_stage == "connect" # type: ignore[attr-defined] + assert error.device_write_attempted is False # type: ignore[attr-defined] + assert error.device_write_confirmed is False # type: ignore[attr-defined] def test_ap_activation_does_not_fallback_when_fresh_scan_omits_device( @@ -150,7 +177,7 @@ def test_ap_activation_write_error_keeps_type_and_adds_safe_gatt_facts( write_characteristic = SimpleNamespace( uuid=ap_module.WRITE_CHARACTERISTIC_UUID, service_uuid=ap_module.SERVICE_UUID, - properties=["write"], + properties=["write-without-response", "write"], max_write_without_response_size=512, ) status_characteristic = SimpleNamespace( @@ -206,5 +233,136 @@ def test_ap_activation_write_error_keeps_type_and_adds_safe_gatt_facts( assert error.operation_stage == "gatt-write" # type: ignore[attr-defined] assert error.device_write_attempted is True # type: ignore[attr-defined] assert error.device_write_confirmed is False # type: ignore[attr-defined] + assert error.resolved_write_mode == "without_response" # type: ignore[attr-defined] + assert error.write_characteristic_properties == ( # type: ignore[attr-defined] + "write", + "write-without-response", + ) + assert error.max_write_without_response_size == 512 # type: ignore[attr-defined] + assert error.frame_length == FRAME_LENGTH # type: ignore[attr-defined] assert error.att_error_code == 0x03 # type: ignore[attr-defined] assert error.att_error_name == "WRITE_NOT_PERMITTED" # type: ignore[attr-defined] + + +def test_ap_activation_keeps_same_client_alive_through_caller_handoff( + monkeypatch: pytest.MonkeyPatch, +) -> None: + device_id = "synthetic-corebluetooth-uuid" + retained_handle = BLEDevice(device_id, "XGR-K1", details=object()) + service = SimpleNamespace(uuid=ap_module.SERVICE_UUID) + write_characteristic = SimpleNamespace( + uuid=ap_module.WRITE_CHARACTERISTIC_UUID, + service_uuid=ap_module.SERVICE_UUID, + properties=["write"], + max_write_without_response_size=512, + ) + status_characteristic = SimpleNamespace( + uuid=ap_module.STATUS_CHARACTERISTIC_UUID, + service_uuid=ap_module.SERVICE_UUID, + properties=["read"], + ) + entered_clients = 0 + exited_clients = 0 + writes: list[tuple[bytes, bool]] = [] + + class FakeServices: + def get_service(self, uuid: str) -> object | None: + return service if uuid == ap_module.SERVICE_UUID else None + + def get_characteristic(self, uuid: str) -> object | None: + if uuid == ap_module.WRITE_CHARACTERISTIC_UUID: + return write_characteristic + if uuid == ap_module.STATUS_CHARACTERISTIC_UUID: + return status_characteristic + return None + + class ReadyClient: + def __init__(self, device: object, **_kwargs: object) -> None: + assert device is retained_handle + self.services = FakeServices() + self.name = "XGR-K1" + self.is_connected = False + self._write_completed = False + + async def __aenter__(self) -> Any: + nonlocal entered_clients + entered_clients += 1 + self.is_connected = True + return self + + async def __aexit__(self, *_args: object) -> None: + nonlocal exited_clients + exited_clients += 1 + self.is_connected = False + + async def read_gatt_char(self, _characteristic: object) -> bytes: + if not self._write_completed: + return bytes(52) + ready = bytearray(52) + mode = b"WIFI_AP" + ready[0] = len(mode) + ready[1 : 1 + len(mode)] = mode + ready[33] = 4 + ready[34:38] = bytes((192, 168, 56, 1)) + ready[50] = 1 + ready[51] = 1 + return bytes(ready) + + async def write_gatt_char( + self, + _characteristic: object, + value: bytes, + *, + response: bool, + ) -> None: + writes.append((bytes(value), response)) + self._write_completed = True + + monkeypatch.setattr(scanner_module, "monotonic", lambda: 100.0) + _seed_scan_lease({device_id: retained_handle}, observed_at=100.0) + monkeypatch.setattr(ap_module, "BleakClient", ReadyClient) + + async def scenario() -> None: + owner_epoch = bind_ble_runtime_owner_loop() + captured = scanner_module.CapturedDiscoveredDevice( + device=retained_handle, + macos_uuid=device_id, + owner_epoch=owner_epoch, + ) + scanner_module.pin_connected_device_handle( + captured, + device_session_id="device-session-a", + ) + async with ap_module.device_ap_activation_session( + device_id, + timeout_seconds=0.1, + poll_interval_seconds=0.01, + captured_device=captured, + ) as result: + assert result["ready_observed"] is True + assert entered_clients == 1 + assert exited_clients == 0 + assert ble_runtime_snapshot()["active_operation_kind"] == "ap-enable" + with pytest.raises(BleRuntimeBusy) as busy: + await scanner_module.scan(0.01) + assert busy.value.active_operation_kind == "ap-enable" + # This represents the host CoreWLAN association window: the setup + # deadline is over, but the exact same BLE client must remain alive. + await asyncio.sleep(0.02) + assert exited_clients == 0 + assert scanner_module.connected_device_recovery_snapshot( + device_id, + device_session_id="device-session-a", + )["gatt_validated_recently"] is True + + assert exited_clients == 1 + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + assert len(writes) == 1 + payload, response = writes[0] + assert len(payload) == FRAME_LENGTH + assert payload[:COMMAND_OFFSET] == bytes(COMMAND_OFFSET) + assert payload[COMMAND_OFFSET] == ENABLE_AP_COMMAND + assert response is True diff --git a/tests/test_xgrids_application_acceptance.py b/tests/test_xgrids_application_acceptance.py index 419938d..77bac18 100644 --- a/tests/test_xgrids_application_acceptance.py +++ b/tests/test_xgrids_application_acceptance.py @@ -1,7 +1,7 @@ from __future__ import annotations import hashlib -from collections.abc import Collection, Sequence +from collections.abc import Callable, Collection, Sequence import pytest from pydantic import JsonValue, TypeAdapter @@ -20,6 +20,9 @@ from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( ShadowApplicationBootstrapOrchestrator, build_canonical_post_start_observation, ) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttTransportError, +) from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( OneShotPublishEnvelope, ) @@ -120,7 +123,19 @@ class SyntheticAcceptanceTransport: envelopes: Sequence[OneShotPublishEnvelope], *, required_response_operation_keys: Collection[str], + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, ) -> dict[str, bytes]: + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + if dispatch_admission_commit is not None: + dispatch_admission_commit() self.batches.append(tuple(envelope.operation_key for envelope in envelopes)) responses: dict[str, bytes] = {} modeling_operation = next( @@ -357,6 +372,114 @@ def test_canonical_session_owns_start_active_scan_stop_and_save_boundary() -> No TypeAdapter(JsonValue).validate_python(executor.snapshot()) +def test_stop_deadline_expiring_during_dispatch_validation_consumes_no_permit_or_publish() -> None: + transport = SyntheticAcceptanceTransport() + executor = PhysicalAcceptanceDialogueExecutor(transport) + authority = ApplicationControlAuthority(openapi_key=APPLICATION_KEY) + orchestrator = ShadowApplicationBootstrapOrchestrator( + authority, + epoch_seconds=1_752_680_000, + timezone_name="Europe/Moscow", + ) + binding = executor.run_connection_stage(orchestrator) + executor.run_workspace_entry_stage( + orchestrator, + executor.wait_for_operator_checkpoint("workspace-entered", lambda: True), + ) + executor.run_project_prompt_stage( + orchestrator, + executor.wait_for_operator_checkpoint("project-prompt-opened", lambda: True), + ) + start_command = ShadowModelingCommand.from_command( + encode_modeling_start( + CommandHeaderIdentity( + device_id=binding.vendor_device_id, + openapi_key=APPLICATION_KEY, + ), + project_name="SAFE_PROJECT", + record_mode=RecordMode.RECORD_AND_CALCULATE, + scan_mode=ScanMode.LCC, + mount_type=MountType.HANDHELD, + ) + ) + executor.execute_canonical_start( + start_command, + build_canonical_post_start_observation(authority, binding), + authority=authority, + binding=binding, + permit=PhysicalAcceptancePermit(_checklist(ModelingAction.START)), + checkpoint=executor.wait_for_operator_checkpoint( + "start-confirmed", + lambda: True, + ), + ) + executor.maintain_active_until_stop_requested(lambda: True) + stop_command = ShadowModelingCommand.from_command( + encode_modeling_stop( + CommandHeaderIdentity( + device_id=binding.vendor_device_id, + openapi_key=APPLICATION_KEY, + ) + ) + ) + stop_permit = PhysicalAcceptancePermit(_checklist(ModelingAction.STOP)) + expired = False + + def slow_dispatch_validation() -> None: + nonlocal expired + expired = True + + with pytest.raises(ApplicationMqttTransportError) as raised: + executor.execute_canonical_stop( + stop_command, + stop_permit, + dispatch_guard=slow_dispatch_validation, + dispatch_admission_deadline_reached=lambda: expired, + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert stop_permit.snapshot()["consumed"] is False + assert transport.stop_emitted is False + assert executor.snapshot()["stop_attempted"] is False + assert executor.snapshot()["dialogue_stage"] == "stop-requested" + + +def test_read_only_inspection_publishes_only_ordinal_one_device_info() -> None: + transport = SyntheticAcceptanceTransport() + executor = PhysicalAcceptanceDialogueExecutor(transport) + orchestrator = ShadowApplicationBootstrapOrchestrator( + ApplicationControlAuthority(openapi_key=APPLICATION_KEY), + epoch_seconds=1_752_680_000, + timezone_name="Europe/Moscow", + ) + + binding = executor.run_read_only_inspection_stage(orchestrator) + + assert binding.vendor_device_id == VENDOR_DEVICE_ID + assert transport.batches == [("bootstrap:1:DeviceInfoRequest",)] + outbound = {operation for batch in transport.batches for operation in batch} + assert outbound == {"bootstrap:1:DeviceInfoRequest"} + assert not any("DeviceConfig" in operation for operation in outbound) + assert not any("ModelingStatus" in operation for operation in outbound) + assert not any(operation.startswith("modeling:") for operation in outbound) + assert executor.snapshot()["dialogue_stage"] == "inspection-ready" + + completed = executor.complete_connection_stage( + orchestrator, + expected_binding=binding, + ) + + assert completed == binding + assert [len(batch) for batch in transport.batches] == [1, 5] + assert transport.batches[1] == ( + "bootstrap:2:ModelingStatusRequest", + "bootstrap:3:GetRtkAdvanceRequest", + "bootstrap:4:DeviceConfigRequest", + "bootstrap:5:DeviceInfoRequest", + "bootstrap:6:GetRtkAdvanceRequest", + ) + + def test_start_initialization_wait_has_fail_closed_watchdog() -> None: class NeverInitializedTransport(SyntheticAcceptanceTransport): def scan_initialization_complete(self, _binding: object) -> bool: diff --git a/tests/test_xgrids_application_authority.py b/tests/test_xgrids_application_authority.py index 072ed5d..db77db7 100644 --- a/tests/test_xgrids_application_authority.py +++ b/tests/test_xgrids_application_authority.py @@ -11,11 +11,28 @@ from k1link.device_plugins.xgrids_k1.protocol.application_authority import ( ApplicationAuthorityLoadError, MacOSKeychainApplicationAuthorityLoader, MacOSKeychainApplicationAuthorityProvisioner, + _keychain_authority_reason_code, ) PRIVATE_AUTHORITY = b"11111111-2222-3333-4444-555555555555\n" +@pytest.mark.parametrize( + ("status", "expected_reason"), + [ + (-25308, "keychain-authorization-required"), + (-25293, "keychain-authorization-denied"), + (-128, "keychain-authorization-cancelled"), + (-25300, "application_authority_unavailable"), + ], +) +def test_keychain_osstatus_is_reduced_to_redacted_failure_class( + status: int, + expected_reason: str, +) -> None: + assert _keychain_authority_reason_code(status) == expected_reason + + @patch("platform.system", return_value="Darwin") def test_runtime_authority_loads_through_security_framework_without_subprocess( _system: object, diff --git a/tests/test_xgrids_application_control_process_lease.py b/tests/test_xgrids_application_control_process_lease.py new file mode 100644 index 0000000..e471de6 --- /dev/null +++ b/tests/test_xgrids_application_control_process_lease.py @@ -0,0 +1,821 @@ +from __future__ import annotations + +import asyncio +import fcntl +import multiprocessing +import os +import subprocess +import sys +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + APPLICATION_CONTROL_LOCK_FILENAME, + ApplicationControlProcessLease, + ApplicationControlProcessLeaseError, + ApplicationControlProcessLeaseReleaseAmbiguous, + ApplicationControlProcessLeaseUnavailable, +) +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationHardTimeout, + BleOperationProgress, + BleRuntimeProcessLeaseBorrowInvalid, + ble_runtime_snapshot, + borrow_ble_runtime_process_lease, + configure_ble_runtime_process_lease, + reset_ble_runtime_arbiter_for_tests, + run_ble_operation, + wait_for_ble_runtime_idle, +) +from k1link.device_plugins.xgrids_k1.connection_supervisor import EndpointTarget +from k1link.device_plugins.xgrids_k1.facade import ( + OpenApplicationControlSessionRequest, + XgridsK1CompatibilityService, + _recover_physical_command_after_process_restart, +) +from k1link.web.device_lifecycle import OperationJournal + + +@pytest.fixture(autouse=True) +def reset_process_ble_runtime() -> None: + reset_ble_runtime_arbiter_for_tests() + yield + reset_ble_runtime_arbiter_for_tests() + + +def _configure_private_data( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, Path]: + data_dir = tmp_path / "private-data" + repository_root = tmp_path / "repository" + repository_root.mkdir() + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(data_dir)) + return repository_root, data_dir + + +def _hold_process_lease( + repository_root: str, + data_dir: str, + ready: multiprocessing.synchronize.Event, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + lease = ApplicationControlProcessLease.acquire(Path(repository_root)) + ready.set() + try: + while True: + time.sleep(0.05) + finally: + lease.release() + + +def test_lease_is_nonblocking_private_and_reusable_after_release( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure_private_data(tmp_path, monkeypatch) + + first = ApplicationControlProcessLease.acquire(repository_root) + lock_path = data_dir / "xgrids-k1" / APPLICATION_CONTROL_LOCK_FILENAME + assert first.path == lock_path + assert lock_path.is_file() + assert lock_path.stat().st_mode & 0o777 == 0o600 + assert lock_path.parent.stat().st_mode & 0o777 == 0o700 + + with pytest.raises(ApplicationControlProcessLeaseUnavailable) as busy: + ApplicationControlProcessLease.acquire(repository_root) + assert busy.value.reason_code == "application-control-process-lease-unavailable" + + first.release() + first.release() + with ApplicationControlProcessLease.acquire(repository_root) as second: + assert second.path == lock_path + assert lock_path.exists() + + +def test_release_success_is_idempotent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + lease = ApplicationControlProcessLease.acquire(repository_root) + unlock_calls = 0 + close_calls = 0 + original_flock = fcntl.flock + original_close = os.close + + def count_flock(descriptor: int, operation: int) -> None: + nonlocal unlock_calls + if descriptor == lease._descriptor and operation == fcntl.LOCK_UN: # noqa: SLF001 + unlock_calls += 1 + original_flock(descriptor, operation) + + def count_close(descriptor: int) -> None: + nonlocal close_calls + if descriptor == lease._descriptor: # noqa: SLF001 + close_calls += 1 + original_close(descriptor) + + monkeypatch.setattr(fcntl, "flock", count_flock) + monkeypatch.setattr(os, "close", count_close) + + first = lease.release() + second = lease.release() + + assert first.disposition == "released" + assert first.unlock_error_code is None + assert first.close_error_code is None + assert second.disposition == "already-released" + assert lease.release_state == "released" + assert (unlock_calls, close_calls) == (1, 1) + + +def test_release_unlock_success_close_error_is_terminal_released( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + lease = ApplicationControlProcessLease.acquire(repository_root) + descriptor = lease._descriptor # noqa: SLF001 + original_close = os.close + close_calls = 0 + + def fail_close(candidate: int) -> None: + nonlocal close_calls + if candidate == descriptor: + close_calls += 1 + raise OSError(5, "synthetic close failure") + original_close(candidate) + + monkeypatch.setattr(os, "close", fail_close) + outcome = lease.release() + + assert outcome.disposition == "released" + assert outcome.unlock_error_code is None + assert outcome.close_error_code == "OSError:5" + assert lease.release_state == "released" + assert lease.release().disposition == "already-released" + assert close_calls == 1 + with pytest.raises(ApplicationControlProcessLeaseError): + lease.duplicate_descriptor_for_child() + configure_ble_runtime_process_lease(repository_root) + with ( + pytest.raises(BleRuntimeProcessLeaseBorrowInvalid), + borrow_ble_runtime_process_lease(lease), + ): + pass + with ApplicationControlProcessLease.acquire(repository_root): + pass + original_close(descriptor) + + +def test_release_unlock_error_close_success_is_terminal_released( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + lease = ApplicationControlProcessLease.acquire(repository_root) + descriptor = lease._descriptor # noqa: SLF001 + original_flock = fcntl.flock + + def fail_unlock(candidate: int, operation: int) -> None: + if candidate == descriptor and operation == fcntl.LOCK_UN: + raise OSError(5, "synthetic unlock failure") + original_flock(candidate, operation) + + monkeypatch.setattr(fcntl, "flock", fail_unlock) + outcome = lease.release() + + assert outcome.disposition == "released" + assert outcome.unlock_error_code == "OSError:5" + assert outcome.close_error_code is None + assert lease.release_state == "released" + with ApplicationControlProcessLease.acquire(repository_root): + pass + + +def test_release_unlock_and_close_error_quarantines_without_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + lease = ApplicationControlProcessLease.acquire(repository_root) + descriptor = lease._descriptor # noqa: SLF001 + lock_identity = lease.path.stat().st_dev, lease.path.stat().st_ino + original_flock = fcntl.flock + original_close = os.close + unlock_calls = 0 + close_calls = 0 + + def fail_unlock(candidate: int, operation: int) -> None: + nonlocal unlock_calls + if candidate == descriptor and operation == fcntl.LOCK_UN: + unlock_calls += 1 + raise OSError(5, "synthetic unlock failure") + original_flock(candidate, operation) + + def fail_close(candidate: int) -> None: + nonlocal close_calls + if candidate == descriptor: + close_calls += 1 + raise OSError(5, "synthetic close failure") + original_close(candidate) + + monkeypatch.setattr(fcntl, "flock", fail_unlock) + monkeypatch.setattr(os, "close", fail_close) + try: + with pytest.raises(ApplicationControlProcessLeaseReleaseAmbiguous): + lease.release() + with pytest.raises(ApplicationControlProcessLeaseReleaseAmbiguous): + lease.release() + assert lease.release_state == "ambiguous" + assert (unlock_calls, close_calls) == (1, 1) + with pytest.raises(ApplicationControlProcessLeaseError): + lease.duplicate_descriptor_for_child() + configure_ble_runtime_process_lease(repository_root) + with ( + pytest.raises(BleRuntimeProcessLeaseBorrowInvalid), + borrow_ble_runtime_process_lease(lease), + ): + pass + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(repository_root) + assert (lease.path.stat().st_dev, lease.path.stat().st_ino) == lock_identity + finally: + monkeypatch.setattr(fcntl, "flock", original_flock) + monkeypatch.setattr(os, "close", original_close) + original_flock(descriptor, fcntl.LOCK_UN) + original_close(descriptor) + + +def test_ble_admission_rejects_borrow_token_quarantined_after_creation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + lease = ApplicationControlProcessLease.acquire(repository_root) + descriptor = lease._descriptor # noqa: SLF001 + original_flock = fcntl.flock + original_close = os.close + operation_calls = 0 + + def fail_unlock(candidate: int, operation: int) -> None: + if candidate == descriptor and operation == fcntl.LOCK_UN: + raise OSError(5, "synthetic unlock failure") + original_flock(candidate, operation) + + def fail_close(candidate: int) -> None: + if candidate == descriptor: + raise OSError(5, "synthetic close failure") + original_close(candidate) + + async def operation(_progress: BleOperationProgress) -> None: + nonlocal operation_calls + operation_calls += 1 + + monkeypatch.setattr(fcntl, "flock", fail_unlock) + monkeypatch.setattr(os, "close", fail_close) + try: + with borrow_ble_runtime_process_lease(lease): + with pytest.raises(ApplicationControlProcessLeaseReleaseAmbiguous): + lease.release() + with pytest.raises(BleRuntimeProcessLeaseBorrowInvalid): + asyncio.run( + run_ble_operation( + "scan", + hard_timeout_seconds=1.0, + operation=operation, + ) + ) + assert operation_calls == 0 + assert ble_runtime_snapshot()["active_operation_kind"] is None + finally: + monkeypatch.setattr(fcntl, "flock", original_flock) + monkeypatch.setattr(os, "close", original_close) + original_flock(descriptor, fcntl.LOCK_UN) + original_close(descriptor) + + +def test_process_crash_releases_os_ownership_without_deleting_lock_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure_private_data(tmp_path, monkeypatch) + context = multiprocessing.get_context("spawn") + ready = context.Event() + process = context.Process( + target=_hold_process_lease, + args=(str(repository_root), str(data_dir), ready), + ) + process.start() + try: + assert ready.wait(timeout=10.0) + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(repository_root) + finally: + process.terminate() + process.join(timeout=10.0) + if process.is_alive(): + process.kill() + process.join(timeout=10.0) + + lock_path = data_dir / "xgrids-k1" / APPLICATION_CONTROL_LOCK_FILENAME + assert lock_path.exists() + with ApplicationControlProcessLease.acquire(repository_root): + pass + + +@pytest.mark.skipif(os.name != "posix", reason="camera flock inheritance is POSIX-only") +def test_inherited_camera_descriptor_survives_parent_crash_until_child_exits( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + ready_path = tmp_path / "camera-child-ready" + release_path = tmp_path / "camera-child-release" + owner = ApplicationControlProcessLease.acquire(repository_root) + inherited_descriptor = owner.duplicate_descriptor_for_child() + child: subprocess.Popen[bytes] | None = None + parent_descriptor_closed = False + try: + child = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import pathlib,sys,time;" + "ready=pathlib.Path(sys.argv[1]);" + "release=pathlib.Path(sys.argv[2]);" + "ready.write_text('ready');" + "\nwhile not release.exists(): time.sleep(0.01)" + ), + str(ready_path), + str(release_path), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + pass_fds=(inherited_descriptor,), + start_new_session=True, + ) + os.close(inherited_descriptor) + inherited_descriptor = -1 + deadline = time.monotonic() + 5.0 + while not ready_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert ready_path.exists() + + # A process crash closes descriptors without explicitly unlocking the + # shared flock description. Model that exact boundary; the child now + # owns the only remaining duplicate inherited through exec. + os.close(owner._descriptor) # noqa: SLF001 + owner._released = True # noqa: SLF001 + parent_descriptor_closed = True + + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(repository_root) + + release_path.touch() + assert child.wait(timeout=5.0) == 0 + with ApplicationControlProcessLease.acquire(repository_root): + pass + finally: + if inherited_descriptor >= 0: + os.close(inherited_descriptor) + if not parent_descriptor_closed: + owner.release() + release_path.touch(exist_ok=True) + if child is not None and child.poll() is None: + child.terminate() + child.wait(timeout=5.0) + + +def test_unsafe_existing_lock_file_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure_private_data(tmp_path, monkeypatch) + data_dir.mkdir(mode=0o700) + lock_dir = data_dir / "xgrids-k1" + lock_dir.mkdir(mode=0o700) + lock_path = lock_dir / APPLICATION_CONTROL_LOCK_FILENAME + lock_path.write_text("unsafe", encoding="utf-8") + lock_path.chmod(0o644) + + with pytest.raises(ApplicationControlProcessLeaseError, match="private regular file"): + ApplicationControlProcessLease.acquire(repository_root) + assert lock_path.stat().st_mode & 0o777 == 0o644 + + +def test_unsafe_existing_lock_directory_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure_private_data(tmp_path, monkeypatch) + data_dir.mkdir(mode=0o700) + lock_dir = data_dir / "xgrids-k1" + lock_dir.mkdir(mode=0o755) + # Earlier security-oriented tests may legitimately tighten the process + # umask. The fixture must still create the exact unsafe mode it claims to + # exercise instead of silently becoming 0700 under that inherited umask. + lock_dir.chmod(0o755) + + with pytest.raises(ApplicationControlProcessLeaseError, match="directory is not private"): + ApplicationControlProcessLease.acquire(repository_root) + + +def test_symlink_lock_file_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure_private_data(tmp_path, monkeypatch) + data_dir.mkdir(mode=0o700) + lock_dir = data_dir / "xgrids-k1" + lock_dir.mkdir(mode=0o700) + target = tmp_path / "unrelated-private-file" + target.touch(mode=0o600) + (lock_dir / APPLICATION_CONTROL_LOCK_FILENAME).symlink_to(target) + + with pytest.raises(ApplicationControlProcessLeaseError, match="opened safely"): + ApplicationControlProcessLease.acquire(repository_root) + + +def _bare_service(repository_root: Path) -> XgridsK1CompatibilityService: + service = object.__new__(XgridsK1CompatibilityService) + service.repository_root = repository_root + service._lock = threading.Lock() # noqa: SLF001 + service._acquisition_lifecycle_gate = threading.RLock() # noqa: SLF001 + service._acquisition_lifecycle_admission = threading.Condition() # noqa: SLF001 + service._acquisition_lifecycle_reader_depth = threading.local() # noqa: SLF001 + service._acquisition_lifecycle_writer_token = None # noqa: SLF001 + service._acquisition_lifecycle_writer_thread_id = None # noqa: SLF001 + service._k1_lifecycle_transition_gate = threading.Lock() # noqa: SLF001 + service._k1_command_dispatch_gate = threading.Lock() # noqa: SLF001 + service._k1_process_lease_gate = threading.Lock() # noqa: SLF001 + service._connection_monitor_contact_gate = threading.Lock() # noqa: SLF001 + service._application_control_process_lease = None # noqa: SLF001 + service._application_control_process_lease_holders = set() # noqa: SLF001 + service._application_control_process_lease_quarantine = None # noqa: SLF001 + return service + + +def test_facade_holds_lease_until_worker_and_socket_are_retired( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + owner = _bare_service(repository_root) + peer = _bare_service(repository_root) + + owner._acquire_application_control_process_lease() # noqa: SLF001 + owner._reconcile_application_control_process_lease( # noqa: SLF001 + {"state": "failed", "can_open": False} + ) + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + peer._acquire_application_control_process_lease() # noqa: SLF001 + + owner._reconcile_application_control_process_lease( # noqa: SLF001 + {"state": "failed", "can_open": True} + ) + peer._acquire_application_control_process_lease() # noqa: SLF001 + peer._release_application_control_process_lease() # noqa: SLF001 + + +def test_network_and_control_share_one_process_lease_until_both_retire( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + owner = _bare_service(repository_root) + peer = _bare_service(repository_root) + + owner._acquire_application_control_process_lease() # noqa: SLF001 + owner._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + owner._release_application_control_process_lease() # noqa: SLF001 + + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + peer._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + + owner._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + peer._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + peer._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + + +def test_facade_rejects_new_holder_when_active_pointer_is_not_owned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + owner = _bare_service(repository_root) + lease = ApplicationControlProcessLease.acquire(repository_root) + lease.release() + owner._application_control_process_lease = lease # noqa: SLF001 + owner._application_control_process_lease_holders = {"control"} # noqa: SLF001 + + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + owner._acquire_k1_lifecycle_process_lease("camera") # noqa: SLF001 + + assert owner._application_control_process_lease is lease # noqa: SLF001 + assert owner._application_control_process_lease_holders == {"control"} # noqa: SLF001 + + +def test_facade_quarantines_ambiguous_final_release_and_never_retries_descriptor( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + owner = _bare_service(repository_root) + owner._acquire_k1_lifecycle_process_lease("camera") # noqa: SLF001 + lease = owner._application_control_process_lease # noqa: SLF001 + assert lease is not None + descriptor = lease._descriptor # noqa: SLF001 + original_flock = fcntl.flock + original_close = os.close + unlock_calls = 0 + close_calls = 0 + + def fail_unlock(candidate: int, operation: int) -> None: + nonlocal unlock_calls + if candidate == descriptor and operation == fcntl.LOCK_UN: + unlock_calls += 1 + raise OSError(5, "synthetic unlock failure") + original_flock(candidate, operation) + + def fail_close(candidate: int) -> None: + nonlocal close_calls + if candidate == descriptor: + close_calls += 1 + raise OSError(5, "synthetic close failure") + original_close(candidate) + + monkeypatch.setattr(fcntl, "flock", fail_unlock) + monkeypatch.setattr(os, "close", fail_close) + try: + with pytest.raises(ApplicationControlProcessLeaseReleaseAmbiguous): + owner._release_k1_lifecycle_process_lease("camera") # noqa: SLF001 + assert owner._application_control_process_lease is None # noqa: SLF001 + assert owner._application_control_process_lease_holders == set() # noqa: SLF001 + quarantine = owner._application_control_process_lease_quarantine # noqa: SLF001 + assert quarantine == ( + lease, + "application-control-process-lease-release-ambiguous", + ) + + owner._release_k1_lifecycle_process_lease("camera") # noqa: SLF001 + assert (unlock_calls, close_calls) == (1, 1) + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + owner._acquire_k1_lifecycle_process_lease("camera") # noqa: SLF001 + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + owner._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + assert (unlock_calls, close_calls) == (1, 1) + finally: + monkeypatch.setattr(fcntl, "flock", original_flock) + monkeypatch.setattr(os, "close", original_close) + original_flock(descriptor, fcntl.LOCK_UN) + original_close(descriptor) + + +def test_network_process_lease_blocks_other_process_control_owner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + network_owner = _bare_service(repository_root) + control_peer = _bare_service(repository_root) + + network_owner._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + control_peer._acquire_application_control_process_lease() # noqa: SLF001 + network_owner._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + + +@pytest.mark.parametrize( + ("action", "expected_after_release"), + [("start", True), ("stop", False)], +) +def test_physical_prepared_recovery_waits_for_exclusive_lifecycle_lease_and_preserves_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + action: str, + expected_after_release: bool, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + live_owner = ApplicationControlProcessLease.acquire(repository_root) + resolutions: list[tuple[str, str]] = [] + + class PreparedLedger: + @staticmethod + def snapshot() -> SimpleNamespace: + return SimpleNamespace( + status="unresolved", + record=SimpleNamespace( + operation_id="physical-prepared-by-live-owner", + stage="prepared", + action=action, + ), + ) + + @staticmethod + def resolve(operation_id: str, *, resolution: str) -> None: + resolutions.append((operation_id, resolution)) + + ledger = PreparedLedger() + _recover_physical_command_after_process_restart( + repository_root, + physical_ledger=ledger, # type: ignore[arg-type] + ) + assert resolutions == [] + + live_owner.release() + _recover_physical_command_after_process_restart( + repository_root, + physical_ledger=ledger, # type: ignore[arg-type] + ) + assert resolutions == ( + [("physical-prepared-by-live-owner", "not-dispatched")] + if expected_after_release + else [] + ) + + +def test_detached_ble_cleanup_retains_cross_process_lifecycle_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + owner = _bare_service(repository_root) + peer = _bare_service(repository_root) + + async def scenario() -> None: + cleanup_entered = asyncio.Event() + cleanup_release = asyncio.Event() + + async def stubborn_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cleanup_entered.set() + await cleanup_release.wait() + raise + + owner._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + lease = owner._application_control_process_lease # noqa: SLF001 + assert lease is not None + with ( + borrow_ble_runtime_process_lease(lease), + pytest.raises(BleOperationHardTimeout), + ): + await run_ble_operation( + "scan", + hard_timeout_seconds=0.01, + operation=stubborn_cleanup, + ) + await cleanup_entered.wait() + owner._release_network_process_lease_after_ble_cleanup() # noqa: SLF001 + + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + peer._acquire_application_control_process_lease() # noqa: SLF001 + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + peer._acquire_application_control_process_lease() # noqa: SLF001 + peer._release_application_control_process_lease() # noqa: SLF001 + + asyncio.run(scenario()) + + +def test_failed_ble_cleanup_keeps_cross_process_lease_until_process_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + owner = _bare_service(repository_root) + peer = _bare_service(repository_root) + + async def scenario() -> None: + cleanup_entered = asyncio.Event() + cleanup_failure = asyncio.Event() + + async def failed_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError as exc: + cleanup_entered.set() + await cleanup_failure.wait() + raise RuntimeError("synthetic native cleanup failure") from exc + + owner._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + lease = owner._application_control_process_lease # noqa: SLF001 + assert lease is not None + with ( + borrow_ble_runtime_process_lease(lease), + pytest.raises(BleOperationHardTimeout), + ): + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.01, + operation=failed_cleanup, + ) + await cleanup_entered.wait() + owner._release_network_process_lease_after_ble_cleanup() # noqa: SLF001 + cleanup_failure.set() + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + + assert ble_runtime_snapshot()["poisoned"] is True + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + peer._acquire_application_control_process_lease() # noqa: SLF001 + + try: + asyncio.run(scenario()) + finally: + # Production releases this flock only through process exit. The test + # simulates that boundary explicitly so it cannot contaminate peers. + reset_ble_runtime_arbiter_for_tests() + owner._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + + +def test_explicit_open_failure_releases_newly_acquired_facade_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _ = _configure_private_data(tmp_path, monkeypatch) + configure_ble_runtime_process_lease(repository_root) + service = _bare_service(repository_root) + service._acquisition_lifecycle_gate = threading.RLock() # noqa: SLF001 + service._provisioning_active = False # noqa: SLF001 + service._device_id = "known-k1" # noqa: SLF001 + service._device_session_id = "known-session" # noqa: SLF001 + service._selected_device_id = "test-ble-transport" # noqa: SLF001 + service._compatibility_attestation = { # noqa: SLF001 + "firmware_version": "3.0.2", + "topology": "direct-lan", + "verification": "live-device-info", + } + service._acquisition = None # noqa: SLF001 + service._operations = OperationJournal() # noqa: SLF001 + service._application_control = SimpleNamespace(disarm=lambda: None) # noqa: SLF001 + service._physical_command_coordinator = SimpleNamespace( # type: ignore[assignment] # noqa: SLF001 + snapshot=lambda: {"status": "empty", "record": None} + ) + + target = EndpointTarget("192.168.1.20", 1883) + service._connection_supervisor = SimpleNamespace( # type: ignore[assignment] # noqa: SLF001 + snapshot=lambda: SimpleNamespace( + intent=SimpleNamespace(intent_id="intent-1", requested_mode="bridge"), + device_network=SimpleNamespace( + state="applied", + intent_id="intent-1", + transport_ref="test-ble-transport", + connection_mode="bridge", + target=target, + ), + endpoint=SimpleNamespace( + target=target, + tcp_state="reachable", + host_path_epoch=1, + ), + host_path=SimpleNamespace(epoch=1), + ) + ) + service._reuse_or_recover_control_target = lambda: ( # type: ignore[method-assign] # noqa: SLF001 + target.ipv4, + {"device_write_performed": False}, + ) + service._validate_application_connection_path = lambda _binding: None # type: ignore[method-assign] # noqa: SLF001 + + class FailingSession: + def snapshot(self) -> dict[str, object]: + return {"state": "idle", "can_open": True} + + def open(self, **_: object) -> None: + raise RuntimeError("synthetic pre-worker open failure") + + service._application_control_session = FailingSession() # type: ignore[assignment] # noqa: SLF001 + + with pytest.raises(RuntimeError, match="synthetic pre-worker open failure"): + service.open_application_control_session( + OpenApplicationControlSessionRequest( + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + timezone_name="Europe/Moscow", + ) + ) + + peer = _bare_service(repository_root) + peer._acquire_application_control_process_lease() # noqa: SLF001 + peer._release_application_control_process_lease() # noqa: SLF001 diff --git a/tests/test_xgrids_application_mqtt.py b/tests/test_xgrids_application_mqtt.py index 71385cd..e091fc3 100644 --- a/tests/test_xgrids_application_mqtt.py +++ b/tests/test_xgrids_application_mqtt.py @@ -1,7 +1,10 @@ from __future__ import annotations import hashlib +import threading from collections import deque +from collections.abc import Callable +from datetime import UTC, datetime from types import SimpleNamespace from typing import Any, cast @@ -16,11 +19,17 @@ from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( LiveDeviceControlBinding, ) from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + CONTROL_HEARTBEAT_TOPIC, CONTROL_SUBSCRIPTION_GROUPS, MODELING_RESPONSE_TOPIC, SYSTEM_ERROR_TOPIC, ApplicationCommandOutcomeUnknown, ApplicationControlDeviceFault, + ApplicationControlProofStale, + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, + ApplicationMqttTransportError, ReviewedApplicationMqttTransport, ) from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( @@ -28,6 +37,7 @@ from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( ) from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( MODELING_STATE_BASE, + OPENAPI_SUCCESS, SYSTEM_ERROR_STATE_BASE, ModelingAction, SessionState, @@ -127,8 +137,15 @@ class FakeControlClient: elif kind == "publish": self.on_publish(self, None, value, FakeReasonCode(), None) elif kind == "message": - topic, payload = value - self.on_message(self, None, SimpleNamespace(topic=topic, payload=payload)) + message_parts = cast(tuple[object, ...], value) + topic = cast(str, message_parts[0]) + payload = cast(bytes, message_parts[1]) + retained = bool(message_parts[2]) if len(message_parts) == 3 else False + self.on_message( + self, + None, + SimpleNamespace(topic=topic, payload=payload, retain=retained), + ) return mqtt.MQTT_ERR_SUCCESS def unsubscribe(self, topics: list[str]) -> tuple[mqtt.MQTTErrorCode, int]: @@ -140,6 +157,59 @@ class FakeControlClient: return mqtt.MQTT_ERR_SUCCESS +class RecordingEvidenceObserver: + def __init__(self) -> None: + self.transport: ReviewedApplicationMqttTransport | None = None + self.events: list[tuple[str, object]] = [] + + def _prove_transport_lock_is_not_held(self) -> None: + if self.transport is not None: + self.transport.snapshot() + + def publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: + self._prove_transport_lock_is_not_held() + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + if dispatch_admission_commit is not None: + dispatch_admission_commit() + self.events.append(("dispatching", evidence)) + + def publish_result( + self, + evidence: ApplicationMqttPublishEvidence, + *, + publish_call_returned: bool, + ) -> None: + self._prove_transport_lock_is_not_held() + self.events.append( + ("publish-returned" if publish_call_returned else "publish-failed", evidence) + ) + + def qos2_completed(self, evidence: ApplicationMqttPublishEvidence) -> None: + self._prove_transport_lock_is_not_held() + self.events.append(("qos2", evidence)) + + def application_response(self, evidence: ApplicationMqttResponseEvidence) -> None: + self._prove_transport_lock_is_not_held() + self.events.append(("response", evidence)) + + def device_status(self, evidence: ApplicationMqttDeviceStatusEvidence) -> None: + self._prove_transport_lock_is_not_held() + self.events.append(("status", evidence)) + + def _application_message( session_id: str, *, @@ -335,6 +405,35 @@ def test_acceptance_transport_connects_once_and_completes_one_qos2_exchange() -> assert snapshot["operation_keys_consumed"] == 1 +@pytest.mark.parametrize( + ("connect_error", "expected_reason"), + [ + (ConnectionRefusedError("private broker address"), "mqtt_connect_rejected"), + (TimeoutError("private broker address"), "mqtt_connection_timeout"), + ], +) +def test_connect_socket_failure_preserves_only_reviewed_mqtt_class( + connect_error: OSError, + expected_reason: str, +) -> None: + class FailingConnectClient(FakeControlClient): + def connect(self, host: str, port: int, keepalive: int) -> mqtt.MQTTErrorCode: + del host, port, keepalive + raise connect_error + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, FailingConnectClient()), + ) + + with pytest.raises(ApplicationMqttTransportError) as raised: + transport.open() + + assert raised.value.reason_code == expected_reason + assert "private broker address" not in str(raised.value) + assert transport.snapshot().automatic_retry is False + + def test_consumed_operation_key_can_never_be_published_again() -> None: fake = FakeControlClient() transport = ReviewedApplicationMqttTransport( @@ -381,9 +480,7 @@ def test_same_topic_inflight_responses_are_routed_by_exact_session_without_retry required_response_operation_keys={unbound.operation_key, bound.operation_key}, ) - assert responses[unbound.operation_key] == _application_message( - ":GetRtkAdvanceRequest" - ) + assert responses[unbound.operation_key] == _application_message(":GetRtkAdvanceRequest") assert responses[bound.operation_key] == _application_message( bound_session, device_id=VENDOR_DEVICE_ID, @@ -413,11 +510,7 @@ def test_duplicate_start_response_blocks_stop_before_stop_is_published() -> None retain: bool, ) -> SimpleNamespace: info = super().publish(topic, payload, qos, retain) - action = ( - ModelingAction.START - if len(self.publish_calls) == 1 - else ModelingAction.STOP - ) + action = ModelingAction.START if len(self.publish_calls) == 1 else ModelingAction.STOP self.events.append(("publish", info.mid)) self.events.append( ( @@ -482,9 +575,7 @@ def test_unanswered_optional_status_does_not_steal_same_identity_required_refres # Ordinals 12 and 14 intentionally reuse the same protocol # identity. The capture has only one response after ordinal # 14; unanswered optional ordinal 12 must not swallow it. - self.events.append( - ("message", (MODELING_STATUS_RESPONSE_TOPIC, payload)) - ) + self.events.append(("message", (MODELING_STATUS_RESPONSE_TOPIC, payload))) return info fake = ReusedStatusClient() @@ -531,9 +622,7 @@ def test_already_arrived_optional_status_is_consumed_before_required_refresh() - info = super().publish(topic, payload, qos, retain) self.events.append(("publish", info.mid)) if len(self.publish_calls) == 2: - self.events.append( - ("message", (MODELING_STATUS_RESPONSE_TOPIC, payload)) - ) + self.events.append(("message", (MODELING_STATUS_RESPONSE_TOPIC, payload))) return info fake = ReusedStatusClient() @@ -551,9 +640,7 @@ def test_already_arrived_optional_status_is_consumed_before_required_refresh() - ) # This callback is already available before ordinal 14 is admitted. The # retained socket's zero-wait service turn must consume it as ordinal 12. - fake.events.append( - ("message", (MODELING_STATUS_RESPONSE_TOPIC, immediate.payload)) - ) + fake.events.append(("message", (MODELING_STATUS_RESPONSE_TOPIC, immediate.payload))) responses = transport.exchange_batch_once( [refresh], required_response_operation_keys={refresh.operation_key}, @@ -594,9 +681,7 @@ def test_unbound_optional_status_remains_distinct_from_bound_required_refresh() ), ) ) - self.events.append( - ("message", (MODELING_STATUS_RESPONSE_TOPIC, payload)) - ) + self.events.append(("message", (MODELING_STATUS_RESPONSE_TOPIC, payload))) return info fake = DistinctStatusClient() @@ -701,6 +786,23 @@ def test_control_owner_uses_live_state_gates_and_surfaces_system_error() -> None ) transport.open() + fake.events.append( + ( + "message", + ( + DEVICE_STATUS_TOPIC, + _device_status( + SessionState.READY, + project_bound=False, + init_ready=True, + ), + ), + ) + ) + transport.maintain_open_for(1.0) + assert transport.pre_start_ready(_binding()) is False + assert transport.standby_complete(_binding()) is False + fake.events.append( ( "message", @@ -745,7 +847,7 @@ def test_control_owner_uses_live_state_gates_and_surfaces_system_error() -> None transport.maintain_open_for(1.0) snapshot = transport.snapshot().as_dict() - assert snapshot["device_status_reports"] == 2 + assert snapshot["device_status_reports"] == 3 assert snapshot["system_error_reports"] == 1 assert snapshot["latest_system_error_code"] == 0x32040133 assert snapshot["latest_system_error_state"] == "algorithm_error" @@ -806,8 +908,860 @@ def test_network_loop_failure_keeps_exact_paho_result_and_phase() -> None: snapshot = transport.snapshot().as_dict() assert snapshot["state"] == "poisoned" assert snapshot["last_loop_result_code"] == int(mqtt.MQTT_ERR_CONN_LOST) - assert snapshot["last_loop_result_name"] == mqtt.error_string( - int(mqtt.MQTT_ERR_CONN_LOST) - ) + assert snapshot["last_loop_result_name"] == mqtt.error_string(int(mqtt.MQTT_ERR_CONN_LOST)) assert snapshot["last_loop_phase"] == "post-publish-drain" assert len(fake.publish_calls) == 1 + + +def test_control_proof_expires_without_remote_packets_and_heartbeat_refreshes_it() -> None: + monotonic = FakeClock() + suspend_aware = [1_000.0] + fake = FakeControlClient(clock=monotonic) + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + monotonic=monotonic, + suspend_aware_clock=lambda: suspend_aware[0], + control_proof_ttl_seconds=2.0, + ) + transport.open() + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + + initial = transport.snapshot().as_dict() + assert initial["control_proof_revision"] == 1 + assert initial["control_proof_source"] == "correlated-application-response" + assert initial["control_proof_fresh"] is True + transport.validate_control_proof(_binding()) + + monotonic.now += 2.01 + with pytest.raises(ApplicationControlProofStale) as stale: + transport.validate_control_proof(_binding()) + assert stale.value.reason_code == "control_proof_stale" + + fake.events.append(("message", (CONTROL_HEARTBEAT_TOPIC, b"opaque-vendor-heartbeat"))) + transport.maintain_open_for(1.0) + + refreshed = transport.snapshot().as_dict() + assert refreshed["control_proof_revision"] == 2 + assert refreshed["control_proof_source"] == "mqtt-heartbeat" + assert refreshed["control_proof_fresh"] is True + transport.validate_control_proof(_binding()) + + +def test_control_proof_uses_suspend_aware_elapsed_time() -> None: + monotonic = FakeClock() + suspend_aware = [5_000.0] + fake = FakeControlClient(clock=monotonic) + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + monotonic=monotonic, + suspend_aware_clock=lambda: suspend_aware[0], + control_proof_ttl_seconds=2.0, + ) + transport.open() + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + monotonic_before_sleep = monotonic.now + + suspend_aware[0] += 60.0 + + assert monotonic.now == monotonic_before_sleep + assert transport.snapshot().control_proof_fresh is False + with pytest.raises(ApplicationControlProofStale): + transport.validate_control_proof(_binding()) + + +def test_device_status_refresh_is_promoted_only_after_exact_identity_validation() -> None: + monotonic = FakeClock() + fake = FakeControlClient(clock=monotonic) + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + monotonic=monotonic, + ) + transport.open() + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + fake.events.append( + ( + "message", + ( + DEVICE_STATUS_TOPIC, + _device_status( + SessionState.READY, + project_bound=False, + init_ready=False, + ), + ), + ) + ) + transport.maintain_open_for(1.0) + + unbound = transport.snapshot().as_dict() + assert unbound["device_status_reports"] == 1 + assert unbound["control_proof_revision"] == 1 + assert unbound["control_proof_source"] == "correlated-application-response" + + transport.validate_control_proof(_binding()) + + promoted = transport.snapshot().as_dict() + assert promoted["control_proof_revision"] == 2 + assert promoted["control_proof_source"] == "bound-device-status" + + +def test_evidence_observer_sees_exact_publish_order_with_immediate_qos_callback() -> None: + class ImmediateQosClient(FakeControlClient): + def __init__(self) -> None: + super().__init__(emit_exchange=False, response_topic=MODELING_RESPONSE_TOPIC) + + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + self.publish_calls.append((topic, payload, qos, retain)) + mid = self.next_mid + self.next_mid += 1 + # Paho is allowed to complete a publish from inside publish(). The + # observer must still see publish-result before QoS2 completion. + self.on_publish(self, None, mid, FakeReasonCode(), None) + response = ( + _application_message( + f"{VENDOR_DEVICE_ID}:ModelingRequest", + device_id=VENDOR_DEVICE_ID, + ) + + _uint(2, ModelingAction.START) + + _bytes(15, _uint(1, OPENAPI_SUCCESS)) + ) + self.events.append(("message", (MODELING_RESPONSE_TOPIC, response))) + return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=mid) + + observed_at = datetime(2026, 8, 7, 9, 30, tzinfo=UTC) + fake = ImmediateQosClient() + observer = RecordingEvidenceObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + utc_now=lambda: observed_at, + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + envelope = _modeling_envelope(ModelingAction.START) + + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + ) + + assert [kind for kind, _evidence in observer.events] == [ + "dispatching", + "publish-returned", + "qos2", + "response", + ] + dispatch = cast(ApplicationMqttPublishEvidence, observer.events[0][1]) + returned = cast(ApplicationMqttPublishEvidence, observer.events[1][1]) + qos2 = cast(ApplicationMqttPublishEvidence, observer.events[2][1]) + response = cast(ApplicationMqttResponseEvidence, observer.events[3][1]) + assert dispatch.operation_key == "modeling:start" + assert dispatch.payload_sha256 == envelope.payload_sha256 + assert dispatch.packet_id is None + assert returned.packet_id == 20 + assert qos2 == returned + assert response.operation_key == "modeling:start" + assert response.modeling_action == "start" + assert response.result_code == OPENAPI_SUCCESS + assert response.success is True + assert response.observed_at_utc == "2026-08-07T09:30:00.000Z" + + +def test_publish_exception_emits_observing_failure_without_inventing_packet_or_qos() -> None: + class FailingPublishClient(FakeControlClient): + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + self.publish_calls.append((topic, payload, qos, retain)) + raise OSError("socket disappeared during publish") + + fake = FailingPublishClient() + observer = RecordingEvidenceObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + + with pytest.raises(ApplicationCommandOutcomeUnknown, match="publish call failed"): + transport.exchange_batch_once( + [_modeling_envelope(ModelingAction.START)], + required_response_operation_keys={"modeling:start"}, + ) + + assert [kind for kind, _evidence in observer.events] == [ + "dispatching", + "publish-failed", + ] + failure = cast(ApplicationMqttPublishEvidence, observer.events[1][1]) + assert failure.operation_key == "modeling:start" + assert failure.packet_id is None + assert transport.snapshot().qos2_completions == 0 + + +def test_device_status_evidence_preserves_wire_hash_retain_identity_and_project() -> None: + observed_at = datetime(2026, 8, 7, 10, 15, tzinfo=UTC) + fake = FakeControlClient(emit_exchange=False) + observer = RecordingEvidenceObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + utc_now=lambda: observed_at, + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + payload = _device_status( + SessionState.SCANNING, + project_bound=True, + init_ready=True, + ) + + fake.on_message( + fake, + None, + SimpleNamespace(topic=DEVICE_STATUS_TOPIC, payload=payload, retain=False), + ) + + assert [kind for kind, _evidence in observer.events] == ["status"] + evidence = cast(ApplicationMqttDeviceStatusEvidence, observer.events[0][1]) + assert evidence.vendor_device_id_sha256 == hashlib.sha256(VENDOR_DEVICE_ID.encode()).hexdigest() + assert evidence.device_serial_sha256 == hashlib.sha256(DEVICE_SERIAL.encode()).hexdigest() + assert evidence.session_state == "scanning" + assert evidence.session_state_code == MODELING_STATE_BASE + SessionState.SCANNING + assert evidence.project_bound is True + assert evidence.project_id_sha256 == hashlib.sha256(b"project-present").hexdigest() + assert evidence.init_ready is True + assert evidence.status_message_sha256 == hashlib.sha256(payload).hexdigest() + assert evidence.mqtt_retained is False + assert evidence.observed_at_utc == "2026-08-07T10:15:00.000Z" + + +def test_evidence_observer_can_only_be_installed_before_transport_open() -> None: + fake = FakeControlClient() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.open() + + with pytest.raises(ApplicationMqttTransportError) as raised: + transport.install_evidence_observer(RecordingEvidenceObserver()) + + assert raised.value.reason_code == "evidence_observer_install_too_late" + + +def test_qos_observer_failure_poisoning_is_processed_without_transport_lock_deadlock() -> None: + class RaisingQosObserver(RecordingEvidenceObserver): + def qos2_completed(self, evidence: ApplicationMqttPublishEvidence) -> None: + super().qos2_completed(evidence) + raise RuntimeError("ledger persistence unavailable") + + fake = FakeControlClient() + observer = RaisingQosObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + + with pytest.raises(ApplicationCommandOutcomeUnknown, match="evidence observer failed"): + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + + assert transport.snapshot().state == "poisoned" + assert [kind for kind, _evidence in observer.events][:3] == [ + "dispatching", + "publish-returned", + "qos2", + ] + + +def test_late_old_pubcomp_cannot_be_reused_as_qos_evidence_for_a_future_mid() -> None: + fake = FakeControlClient() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.open() + first = _envelope() + transport.exchange_batch_once( + [first], + required_response_operation_keys={first.operation_key}, + ) + assert len(fake.publish_calls) == 1 + + # The first exchange already consumed MID 20. A delayed duplicate arrives + # while no publish call owns it, immediately before Paho would reuse 20. + fake.events.append(("publish", 20)) + fake.next_mid = 20 + second = _envelope( + "bootstrap:2:DeviceInfoRequest", + session_id=":DeviceInfoRequest2", + ) + + with pytest.raises(ApplicationCommandOutcomeUnknown, match="unowned or duplicate PUBCOMP"): + transport.exchange_batch_once( + [second], + required_response_operation_keys={second.operation_key}, + ) + + assert len(fake.publish_calls) == 1 + assert transport.snapshot().qos2_completions == 1 + + +def test_retained_device_info_response_cannot_satisfy_live_correlation() -> None: + class RetainedResponseClient(FakeControlClient): + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + info = super().publish(topic, payload, qos, retain) + # Replace the ordinary response queued by the fake with retained + # broker history carrying the same deterministic session identity. + self.events.pop() + self.events.append(("message", (self.response_topic, payload, True))) + return info + + fake = RetainedResponseClient() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.open() + + with pytest.raises(ApplicationCommandOutcomeUnknown) as raised: + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + + assert raised.value.reason_code == "retained_application_response" + snapshot = transport.snapshot() + assert snapshot.correlated_responses == 0 + assert snapshot.control_proof_revision == 0 + assert snapshot.retained_application_responses_rejected == 1 + + +def test_retained_modeling_response_cannot_acknowledge_a_physical_command() -> None: + class RetainedModelingResponseClient(FakeControlClient): + def __init__(self) -> None: + super().__init__(emit_exchange=False, response_topic=MODELING_RESPONSE_TOPIC) + + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + info = super().publish(topic, payload, qos, retain) + self.events.append(("publish", info.mid)) + self.events.append( + ( + "message", + ( + MODELING_RESPONSE_TOPIC, + _modeling_response(ModelingAction.START), + True, + ), + ) + ) + return info + + fake = RetainedModelingResponseClient() + observer = RecordingEvidenceObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + + with pytest.raises(ApplicationCommandOutcomeUnknown) as raised: + transport.exchange_batch_once( + [_modeling_envelope(ModelingAction.START)], + required_response_operation_keys={"modeling:start"}, + ) + + assert raised.value.reason_code == "retained_application_response" + assert "response" not in [kind for kind, _evidence in observer.events] + assert transport.snapshot().retained_application_responses_rejected == 1 + + +@pytest.mark.parametrize("state", [SessionState.READY, SessionState.SCANNING]) +def test_retained_device_status_cannot_change_authoritative_live_state( + state: SessionState, +) -> None: + fake = FakeControlClient() + observer = RecordingEvidenceObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + observer.events.clear() + payload = _device_status( + state, + project_bound=state is SessionState.SCANNING, + init_ready=state is SessionState.SCANNING, + ) + + fake.on_message( + fake, + None, + SimpleNamespace(topic=DEVICE_STATUS_TOPIC, payload=payload, retain=True), + ) + + snapshot = transport.snapshot() + assert snapshot.latest_device_session_state is None + assert snapshot.latest_device_project_bound is None + assert snapshot.latest_device_init_ready is None + assert snapshot.retained_control_reports_ignored == 1 + assert observer.events == [] + assert transport.pre_start_ready(_binding()) is False + assert transport.scan_initialization_complete(_binding()) is False + assert transport.standby_complete(_binding()) is False + + +def test_retained_heartbeat_cannot_refresh_bound_control_proof() -> None: + monotonic = FakeClock() + fake = FakeControlClient(clock=monotonic) + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + monotonic=monotonic, + ) + transport.open() + transport.exchange_batch_once( + [_envelope()], + required_response_operation_keys={"bootstrap:1:DeviceInfoRequest"}, + ) + before = transport.snapshot() + + fake.on_message( + fake, + None, + SimpleNamespace( + topic=CONTROL_HEARTBEAT_TOPIC, + payload=b"retained-old-heartbeat", + retain=True, + ), + ) + + after = transport.snapshot() + assert after.control_proof_revision == before.control_proof_revision + assert after.control_proof_source == before.control_proof_source + assert after.retained_control_reports_ignored == 1 + + +def test_dispatch_guard_runs_before_every_packet_and_blocks_partial_batch_continuation() -> None: + fake = FakeControlClient() + guard_calls = 0 + + def guard() -> None: + nonlocal guard_calls + guard_calls += 1 + if guard_calls == 2: + raise RuntimeError("connection epoch changed") + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.install_dispatch_guard(guard) + transport.open() + first = _envelope() + second = _envelope( + "bootstrap:2:DeviceInfoRequest", + session_id=":DeviceInfoRequest2", + ) + + with pytest.raises(ApplicationCommandOutcomeUnknown, match="dispatch guard failed"): + transport.exchange_batch_once( + [first, second], + required_response_operation_keys={first.operation_key, second.operation_key}, + ) + + assert guard_calls == 2 + assert len(fake.publish_calls) == 1 + assert transport.snapshot().state == "poisoned" + + +def test_dispatch_guard_release_covers_exact_publish_window() -> None: + events: list[str] = [] + + class WindowAwareClient(FakeControlClient): + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + assert events == ["acquired"] + events.append("publish") + return super().publish(topic, payload, qos, retain) + + fake = WindowAwareClient() + + def guard() -> Callable[[], None]: + events.append("acquired") + return lambda: events.append("released") + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.install_dispatch_guard(guard) + transport.open() + envelope = _envelope() + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + ) + + assert events == ["acquired", "publish", "released"] + + +def test_dispatch_deadline_is_checked_under_guard_before_durable_mark_and_publish() -> None: + events: list[str] = [] + fake = FakeControlClient() + evidence = RecordingEvidenceObserver() + deadline_checks = 0 + admission_commits = 0 + + def deadline_reached() -> bool: + nonlocal deadline_checks + deadline_checks += 1 + # Entry and post-drain checks are open. Expire only while the exact + # dispatch lease is held, immediately before durable DISPATCHING. + return deadline_checks == 3 + + def commit_admission() -> None: + nonlocal admission_commits + admission_commits += 1 + + def guard() -> Callable[[], None]: + events.append("acquired") + return lambda: events.append("released") + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + evidence.transport = transport + transport.install_evidence_observer(evidence) + transport.install_dispatch_guard(guard) + transport.open() + envelope = _modeling_envelope(ModelingAction.STOP) + + with pytest.raises(ApplicationMqttTransportError) as raised: + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + dispatch_admission_deadline_reached=deadline_reached, + dispatch_admission_commit=commit_admission, + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert deadline_checks == 3 + assert admission_commits == 0 + assert events == ["acquired", "released"] + assert fake.publish_calls == [] + assert evidence.events == [] + assert transport.snapshot().publish_attempts == 0 + + +def test_failure_after_atomic_dispatch_commit_is_unknown_and_never_publishes() -> None: + fake = FakeControlClient() + admission_commits = 0 + + class MarkThenFailObserver(RecordingEvidenceObserver): + def publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: + super().publish_dispatching( + evidence, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + dispatch_admission_commit=dispatch_admission_commit, + ) + raise OSError("injected failure after durable dispatch admission") + + def commit_admission() -> None: + nonlocal admission_commits + admission_commits += 1 + + observer = MarkThenFailObserver() + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + observer.transport = transport + transport.install_evidence_observer(observer) + transport.open() + envelope = _modeling_envelope(ModelingAction.STOP) + + with pytest.raises(ApplicationCommandOutcomeUnknown): + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + dispatch_admission_deadline_reached=lambda: False, + dispatch_admission_commit=commit_admission, + ) + + assert admission_commits == 1 + assert [kind for kind, _evidence in observer.events] == ["dispatching"] + assert fake.publish_calls == [] + assert transport.snapshot().state == "poisoned" + + +def test_dispatch_deadline_expiring_during_preflight_never_reaches_dispatch_guard() -> None: + expired = False + guard_calls = 0 + fake = FakeControlClient() + evidence = RecordingEvidenceObserver() + + class ExpiringPreflightTransport(ReviewedApplicationMqttTransport): + def _drain_responses( # type: ignore[override] + self, + pending: object, + responses: object, + ) -> None: + nonlocal expired + super()._drain_responses(pending, responses) # type: ignore[arg-type] + expired = True + + def guard() -> None: + nonlocal guard_calls + guard_calls += 1 + + transport = ExpiringPreflightTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + evidence.transport = transport + transport.install_evidence_observer(evidence) + transport.install_dispatch_guard(guard) + transport.open() + envelope = _modeling_envelope(ModelingAction.STOP) + + with pytest.raises(ApplicationMqttTransportError) as raised: + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + dispatch_admission_deadline_reached=lambda: expired, + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert guard_calls == 0 + assert fake.publish_calls == [] + assert evidence.events == [] + assert transport.snapshot().publish_attempts == 0 + + +def test_dispatch_deadline_expiring_after_mark_preserves_published_outcome() -> None: + class StopResponseClient(FakeControlClient): + def __init__(self) -> None: + super().__init__(emit_exchange=False, response_topic=MODELING_RESPONSE_TOPIC) + + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + info = super().publish(topic, payload, qos, retain) + self.events.append(("publish", info.mid)) + self.events.append( + ("message", (MODELING_RESPONSE_TOPIC, _modeling_response(ModelingAction.STOP))) + ) + return info + + fake = StopResponseClient() + expired = False + + class ExpiringDispatchEvidence(RecordingEvidenceObserver): + def publish_dispatching( + self, + evidence: ApplicationMqttPublishEvidence, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: + nonlocal expired + super().publish_dispatching( + evidence, + dispatch_admission_deadline_reached=( + dispatch_admission_deadline_reached + ), + dispatch_admission_commit=dispatch_admission_commit, + ) + # DISPATCHING is already durable at this callback boundary. A + # later cutoff must not relabel the result as no-dispatch. + expired = True + + evidence = ExpiringDispatchEvidence() + deadline_checks = 0 + + def deadline_reached() -> bool: + nonlocal deadline_checks + deadline_checks += 1 + return expired + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + evidence.transport = transport + transport.install_evidence_observer(evidence) + transport.open() + envelope = _modeling_envelope(ModelingAction.STOP) + + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + dispatch_admission_deadline_reached=deadline_reached, + ) + + assert deadline_checks == 3 + assert expired is True + assert len(fake.publish_calls) == 1 + assert [kind for kind, _item in evidence.events[:2]] == [ + "dispatching", + "publish-returned", + ] + + +def test_dispatch_guard_release_runs_when_publish_outcome_is_unknown() -> None: + events: list[str] = [] + + class FailingPublishClient(FakeControlClient): + def publish( + self, + topic: str, + payload: bytes, + qos: int, + retain: bool, + ) -> SimpleNamespace: + self.publish_calls.append((topic, payload, qos, retain)) + events.append("publish-failed") + raise OSError("socket disappeared during publish") + + fake = FailingPublishClient() + + def guard() -> Callable[[], None]: + events.append("acquired") + return lambda: events.append("released") + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + transport.install_dispatch_guard(guard) + transport.open() + + with pytest.raises(ApplicationCommandOutcomeUnknown, match="publish call failed"): + transport.exchange_batch_once( + [_modeling_envelope(ModelingAction.START)], + required_response_operation_keys={"modeling:start"}, + ) + + assert events == ["acquired", "publish-failed", "released"] + + +def test_close_after_dispatch_precheck_prevents_queued_publish_after_gate_release() -> None: + fake = FakeControlClient() + guard_entered = threading.Event() + release_guard = threading.Event() + worker_errors: list[BaseException] = [] + evidence = RecordingEvidenceObserver() + + def guard() -> Callable[[], None]: + guard_entered.set() + assert release_guard.wait(3.0) + return lambda: None + + transport = ReviewedApplicationMqttTransport( + "192.168.1.20", + client_factory=lambda: cast(mqtt.Client, fake), + ) + evidence.transport = transport + transport.install_evidence_observer(evidence) + transport.install_dispatch_guard(guard) + transport.open() + envelope = _modeling_envelope(ModelingAction.STOP) + + def exchange() -> None: + try: + transport.exchange_batch_once( + [envelope], + required_response_operation_keys={envelope.operation_key}, + ) + except BaseException as exc: + worker_errors.append(exc) + + worker = threading.Thread(target=exchange, daemon=True) + worker.start() + assert guard_entered.wait(3.0) + + # The worker passed the batch-level ready check but has not acquired the + # facade publish lease. Closing in this seam must be a no-future-publish + # proof once the lease becomes available. + transport.close() + release_guard.set() + worker.join(3.0) + + assert worker.is_alive() is False + assert len(worker_errors) == 1 + assert isinstance(worker_errors[0], ApplicationMqttTransportError) + assert worker_errors[0].reason_code == "transport_closed_before_publish" # type: ignore[attr-defined] + assert fake.publish_calls == [] + assert transport.snapshot().publish_attempts == 0 + assert evidence.events == [] diff --git a/tests/test_xgrids_application_session.py b/tests/test_xgrids_application_session.py index b5a167c..8834a69 100644 --- a/tests/test_xgrids_application_session.py +++ b/tests/test_xgrids_application_session.py @@ -1,24 +1,115 @@ from __future__ import annotations +import hashlib import logging import threading import time +from collections.abc import Callable, Collection, Sequence from dataclasses import dataclass +from pathlib import Path from typing import Any import pytest +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + LedgerPhysicalCommandCoordinator, + PhysicalCommandIntentContext, + PhysicalCommandRuntimeBinding, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandLedger, + PhysicalCommandStatusEvidence, +) from k1link.device_plugins.xgrids_k1.protocol import application_session as session_module +from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( + ApplicationAcceptanceError, +) from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import ( + COMPATIBILITY_PROFILE_ID, ApplicationControlAuthority, LiveDeviceControlBinding, ) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationControlProofStale, + ApplicationMqttTransportError, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) from k1link.device_plugins.xgrids_k1.protocol.application_session import ( + ApplicationConnectionBinding, + ApplicationConnectionBindingLost, + ApplicationControlProofExpired, + ApplicationControlStateConflict, + ApplicationStartCheckpointSettlementError, InteractiveApplicationControlSession, OperatorPresenceConfirmation, ) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import OPENAPI_SUCCESS APPLICATION_KEY = "00000000-0000-0000-0000-000000000000" +RETIRED_VENDOR_DEVICE_ID = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" +RETIRED_DEVICE_SERIAL = "K1SERIAL01" + + +def _test_varint(value: int) -> bytes: + encoded = bytearray() + while value > 0x7F: + encoded.append((value & 0x7F) | 0x80) + value >>= 7 + encoded.append(value) + return bytes(encoded) + + +def _test_uint(number: int, value: int) -> bytes: + return _test_varint(number << 3) + _test_varint(value) + + +def _test_bytes(number: int, value: bytes) -> bytes: + return _test_varint((number << 3) | 2) + _test_varint(len(value)) + value + + +def _test_text(number: int, value: str) -> bytes: + return _test_bytes(number, value.encode()) + + +def _test_response_header(session_id: str) -> bytes: + return b"".join( + ( + _test_text(4, RETIRED_VENDOR_DEVICE_ID), + _test_text(5, session_id), + _test_text(6, APPLICATION_KEY), + ) + ) + + +def _test_device_info_response(session_id: str) -> bytes: + base_info = b"".join( + ( + _test_text(2, "V3.0.2_20250624.122658"), + _test_text(3, "V3.0.2"), + _test_text(6, "LixelKity K1"), + _test_text(7, RETIRED_DEVICE_SERIAL), + _test_text(8, "A4"), + ) + ) + device_info = _test_bytes(2, base_info) + _test_bytes(7, _test_uint(1, 1)) + return b"".join( + ( + _test_bytes(1, _test_response_header(session_id)), + _test_bytes(2, device_info), + _test_bytes(15, _test_uint(1, OPENAPI_SUCCESS)), + ) + ) + + +def _test_generic_response(session_id: str) -> bytes: + return _test_bytes(1, _test_response_header(session_id)) + _test_bytes( + 15, + _test_uint(1, OPENAPI_SUCCESS), + ) class FakeAuthorityLoader: @@ -34,11 +125,13 @@ class FakeAuthorityLoader: class FakeTransportSnapshot: state: str ready: bool + proof_fresh: bool + publish_attempts: int = 0 def as_dict(self) -> dict[str, object]: return { "state": self.state, - "publish_attempts": 0, + "publish_attempts": self.publish_attempts, "qos2_completions": 0, "correlated_responses": 0, "ignored_known_responses": 0, @@ -47,6 +140,10 @@ class FakeTransportSnapshot: "latest_device_project_bound": not self.ready, "automatic_retry": False, "automatic_reconnect": False, + "control_proof_revision": 1, + "control_proof_source": "correlated-application-response", + "control_proof_fresh": self.proof_fresh, + "control_proof_age_seconds": 0.0, } @@ -55,6 +152,18 @@ class FakeTransport: self.host = host self.state = "new" self.ready = True + self.proof_fresh = True + self.publish_attempts = 0 + self.evidence_observer: object | None = None + self.dispatch_guard: Callable[[], None] | None = None + + def install_evidence_observer(self, observer: object) -> None: + assert self.state == "new" + self.evidence_observer = observer + + def install_dispatch_guard(self, guard: Callable[[], None]) -> None: + assert self.state == "new" + self.dispatch_guard = guard def open(self) -> FakeTransportSnapshot: self.state = "ready" @@ -64,7 +173,93 @@ class FakeTransport: self.state = "closed" def snapshot(self) -> FakeTransportSnapshot: - return FakeTransportSnapshot(self.state, self.ready) + return FakeTransportSnapshot( + self.state, + self.ready, + self.proof_fresh, + self.publish_attempts, + ) + + def validate_control_proof(self, _binding: LiveDeviceControlBinding) -> None: + if not self.proof_fresh: + raise ApplicationControlProofStale("test control proof expired") + + +class FakePhysicalCommandCoordinator: + def __init__(self, *, fail_prepare: bool = False) -> None: + self.fail_prepare = fail_prepare + self.bindings: list[PhysicalCommandRuntimeBinding] = [] + self.prepares: list[tuple[PhysicalCommandIntentContext, str, object]] = [] + self.resolutions: list[tuple[str, str | None]] = [] + self.phase_probe: Callable[[], str] | None = None + self.snapshot_override: dict[str, object] | None = None + + def bind_control_session(self, binding: PhysicalCommandRuntimeBinding) -> None: + self.bindings.append(binding) + + def prepare( + self, + context: PhysicalCommandIntentContext, + *, + action: str, + envelope: object, + ) -> None: + self.prepares.append((context, action, envelope)) + if self.fail_prepare: + raise RuntimeError("durable prepare failed") + + def resolve(self, action: str) -> None: + phase = self.phase_probe() if self.phase_probe is not None else None + self.resolutions.append((action, phase)) + + def resolve_prepared_not_dispatched(self, action: str) -> None: + phase = self.phase_probe() if self.phase_probe is not None else None + self.resolutions.append((f"{action}-not-dispatched", phase)) + + def snapshot(self) -> dict[str, object]: + if self.snapshot_override is not None: + return self.snapshot_override + return { + "status": "test", + "requires_reconciliation": False, + "automatic_replay_allowed": False, + } + + def publish_dispatching( + self, + _evidence: object, + *, + dispatch_admission_deadline_reached: Callable[[], bool] | None = None, + dispatch_admission_commit: Callable[[], None] | None = None, + ) -> None: + if ( + dispatch_admission_deadline_reached is not None + and dispatch_admission_deadline_reached() + ): + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + if dispatch_admission_commit is not None: + dispatch_admission_commit() + return + + def publish_result( + self, + _evidence: object, + *, + publish_call_returned: bool, + ) -> None: + del publish_call_returned + + def qos2_completed(self, _evidence: object) -> None: + return + + def application_response(self, _evidence: object) -> None: + return + + def device_status(self, _evidence: object) -> None: + return class FakeExecutor: @@ -86,24 +281,62 @@ class FakeExecutor: self.records.append("connection:1-6") return self.binding + def run_read_only_inspection_stage( + self, + _orchestrator: object, + ) -> LiveDeviceControlBinding: + self.records.append("inspection:1") + return self.binding + + def complete_connection_stage( + self, + _orchestrator: object, + *, + expected_binding: LiveDeviceControlBinding, + ) -> LiveDeviceControlBinding: + assert expected_binding == self.binding + self.records.append("connection:2-6") + return self.binding + def wait_for_operator_checkpoint( self, event: str, observed: Any, - ) -> str: + *, + reconciled_active_observed: Any | None = None, + ) -> str | None: self.records.append(f"wait:{event}") deadline = time.monotonic() + 2.0 while not observed(): + if ( + reconciled_active_observed is not None + and reconciled_active_observed() + ): + return None if time.monotonic() >= deadline: raise TimeoutError(f"test did not release {event}") threading.Event().wait(0.005) return event + def adopt_reconciled_scanning( + self, + *, + authority: ApplicationControlAuthority, + binding: LiveDeviceControlBinding, + ) -> None: + del authority + assert binding == self.binding + self.records.append("adopt:scanning") + def run_workspace_entry_stage( self, _orchestrator: object, _checkpoint: object, + *, + dispatch_guard: Callable[[], None] | None = None, ) -> LiveDeviceControlBinding: + if dispatch_guard is not None: + dispatch_guard() self.records.append("workspace:7") return self.binding @@ -111,7 +344,11 @@ class FakeExecutor: self, _orchestrator: object, _checkpoint: object, + *, + dispatch_guard: Callable[[], None] | None = None, ) -> LiveDeviceControlBinding: + if dispatch_guard is not None: + dispatch_guard() self.records.append("project:8-10") return self.binding @@ -166,6 +403,33 @@ def _wait_phase( raise AssertionError(f"session did not reach {expected}: {session.snapshot()}") +def _connection_binding() -> ApplicationConnectionBinding: + return ApplicationConnectionBinding( + intent_id="test-intent-1", + transport_ref="test-transport-1", + host_path_epoch=1, + target_ipv4="192.168.1.20", + target_port=1883, + connection_mode="bridge", + ) + + +def _physical_context(action: str) -> PhysicalCommandIntentContext: + return PhysicalCommandIntentContext( + operation_id=f"operation-{action}-1", + parent_operation_id=("operation-start-1" if action == "stop" else None), + acquisition_id="acquisition-1", + ) + + +def _successful_start_checkpoint_observer( + _phase: str, + _context: PhysicalCommandIntentContext, + _envelope: object, +) -> None: + return None + + def test_canonical_stages_require_operator_events_but_device_standby_does_not( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -189,10 +453,16 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not( host="192.168.1.20", timezone_name="Europe/Moscow", confirmation=_confirmation(), + connection_binding=_connection_binding(), ) - _wait_phase(session, "connection-ready") + connected = _wait_phase(session, "connection-ready") + assert connected["verified_control"]["transport_ref"] == "test-transport-1" # type: ignore[index] threading.Event().wait(0.02) - assert FakeExecutor.records == ["connection:1-6", "wait:workspace-entered"] + assert FakeExecutor.records == [ + "inspection:1", + "connection:2-6", + "wait:workspace-entered", + ] session.enter_workspace() _wait_phase(session, "workspace-ready") @@ -216,7 +486,8 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not( assert completed["scanning_observer_errors"] == 0 assert completed["scripted_transitions"] is False assert FakeExecutor.records == [ - "connection:1-6", + "inspection:1", + "connection:2-6", "wait:workspace-entered", "workspace:7", "wait:project-prompt-opened", @@ -229,6 +500,1179 @@ def test_canonical_stages_require_operator_events_but_device_standby_does_not( ] +def test_read_only_device_info_open_does_not_require_physical_acceptance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Connect may prove DeviceInfo without inventing START/STOP confirmations.""" + + FakeExecutor.records = [] + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + FakeExecutor, + ) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + + opened = session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + assert opened["state"] in {"connecting", "connection-ready"} + ready = _wait_phase(session, "connection-ready") + + assert ready["verified_control"] is not None + assert coordinator.prepares == [] + assert "start:11-14" not in FakeExecutor.records + assert "stop" not in FakeExecutor.records + session.close_prestart( + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + + +def test_inspection_session_rejects_workspace_until_verify_releases_boundary( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + ) + + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + inspection_only=True, + ) + ready = _wait_phase(session, "connection-ready") + + with pytest.raises(ApplicationAcceptanceError, match="read-only inspection"): + session.enter_workspace( + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + assert FakeExecutor.records == ["inspection:1", "wait:workspace-entered"] + + released = session.release_inspection_for_operator_dialogue( + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + session.enter_workspace( + expected_session_generation=released["session_generation"], # type: ignore[arg-type] + expected_state_revision=released["state_revision"], # type: ignore[arg-type] + ) + _wait_phase(session, "workspace-ready") + assert FakeExecutor.records[:4] == [ + "inspection:1", + "wait:workspace-entered", + "connection:2-6", + "workspace:7", + ] + + +@pytest.mark.parametrize( + ("reconciliation_kind", "record_resolution"), + [ + ("ambiguous-outcome", "physical-active-observed"), + ("resolved-active-rebind", "start-active-observed"), + ("prepared-stop-classification", "not-dispatched"), + ], +) +def test_reconciled_scanning_adoption_emits_no_start_and_one_explicit_stop( + monkeypatch: pytest.MonkeyPatch, + reconciliation_kind: str, + record_resolution: str, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + inspection_only=True, + ) + ready = _wait_phase(session, "connection-ready") + verified = ready["verified_control"] + assert isinstance(verified, dict) + reconciliation_id = "verify-operation-1.physical" + exact_connection = { + field: verified[field] + for field in ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + } + coordinator.snapshot_override = { + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": True, + "record": { + "resolution": record_resolution, + "reconciliations": [{ + "reconciliation_id": reconciliation_id, + "kind": reconciliation_kind, + "resolution": "physical-active-observed", + "verified_binding": {"connection": exact_connection}, + "observation": { + "source": "explicit-read-only-reconciliation", + "control_session_id": verified["control_session_id"], + "host_path_epoch": verified["host_path_epoch"], + "producer_generation": verified["producer_generation"], + "session_state": "scanning", + "project_bound": True, + "init_ready": True, + "mqtt_retained": False, + }, + }], + }, + } + + requested = session.adopt_reconciled_scanning( + reconciliation_id=reconciliation_id, + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + scanning = _wait_phase(session, "scanning") + assert requested["state"] in {"active-recovery-requested", "scanning"} + assert "start:11-14" not in FakeExecutor.records + assert FakeExecutor.records.count("adopt:scanning") == 1 + + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + expected_session_generation=scanning["session_generation"], # type: ignore[arg-type] + expected_state_revision=scanning["state_revision"], # type: ignore[arg-type] + ) + _wait_phase(session, "completed") + assert FakeExecutor.records.count("stop") == 1 + + +def test_reconciled_scanning_adoption_requires_latest_exact_reconciliation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An inherited prepared classification cannot override a newer rebind.""" + + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + inspection_only=True, + ) + ready = _wait_phase(session, "connection-ready") + verified = ready["verified_control"] + assert isinstance(verified, dict) + exact_connection = { + field: verified[field] + for field in ( + "intent_id", + "transport_ref", + "connection_mode", + "target_ipv4", + "target_port", + "host_path_epoch", + "control_session_id", + "producer_generation", + ) + } + + def reconciliation(reconciliation_id: str, kind: str) -> dict[str, object]: + return { + "reconciliation_id": reconciliation_id, + "kind": kind, + "resolution": "physical-active-observed", + "verified_binding": {"connection": exact_connection}, + "observation": { + "source": "explicit-read-only-reconciliation", + "control_session_id": verified["control_session_id"], + "host_path_epoch": verified["host_path_epoch"], + "producer_generation": verified["producer_generation"], + "session_state": "scanning", + "project_bound": True, + "init_ready": True, + "mqtt_retained": False, + }, + } + + coordinator.snapshot_override = { + "status": "resolved", + "requires_reconciliation": False, + "resolved_active_recovery_required": True, + "record": { + "resolution": "start-active-observed", + "reconciliations": [ + reconciliation("older-prepared-stop", "prepared-stop-classification"), + reconciliation("latest-rebind", "resolved-active-rebind"), + ], + }, + } + + with pytest.raises(ApplicationAcceptanceError, match="does not match"): + session.adopt_reconciled_scanning( + reconciliation_id="older-prepared-stop", + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + + admitted = session.adopt_reconciled_scanning( + reconciliation_id="latest-rebind", + expected_session_generation=ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=ready["state_revision"], # type: ignore[arg-type] + ) + assert admitted["state"] in {"active-recovery-requested", "scanning"} + scanning = _wait_phase(session, "scanning") + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + expected_session_generation=scanning["session_generation"], # type: ignore[arg-type] + expected_state_revision=scanning["state_revision"], # type: ignore[arg-type] + ) + _wait_phase(session, "completed") + assert [action for _context, action, _envelope in coordinator.prepares] == ["stop"] + + with pytest.raises(ApplicationControlStateConflict): + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + expected_session_generation=scanning["session_generation"], # type: ignore[arg-type] + expected_state_revision=scanning["state_revision"], # type: ignore[arg-type] + ) + assert FakeExecutor.records.count("stop") == 1 + + +def test_control_checkpoint_revision_blocks_stale_browser_continuation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + FakeExecutor, + ) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + ) + + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + connection_ready = _wait_phase(session, "connection-ready") + generation = connection_ready["session_generation"] + revision = connection_ready["state_revision"] + assert generation == 1 + assert isinstance(revision, int) + + session.enter_workspace( + expected_session_generation=generation, + expected_state_revision=revision, + ) + with pytest.raises(ApplicationControlStateConflict, match="session changed"): + session.enter_workspace( + expected_session_generation=generation, + expected_state_revision=revision, + ) + + workspace_ready = _wait_phase(session, "workspace-ready") + with pytest.raises(ApplicationControlStateConflict, match="session changed"): + session.open_project_prompt( + expected_session_generation=generation, + expected_state_revision=revision, + ) + assert session.snapshot()["state"] == "workspace-ready" + + session.open_project_prompt( + expected_session_generation=workspace_ready["session_generation"], # type: ignore[arg-type] + expected_state_revision=workspace_ready["state_revision"], # type: ignore[arg-type] + ) + _wait_phase(session, "project-ready") + + +def test_close_wins_over_workspace_continuation_already_in_preflight() -> None: + validation_entered = threading.Event() + release_validation = threading.Event() + + def validate_binding(_binding: ApplicationConnectionBinding) -> bool: + validation_entered.set() + assert release_validation.wait(timeout=1.0) + return True + + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + connection_path_validator=lambda _binding: True, + connection_binding_validator=validate_binding, + ) + # Isolate the operator-CAS race without opening a background MQTT worker. + # The public facade supplies these exact fields from a real snapshot. + with session._lock: # noqa: SLF001 + session._connection_binding = _connection_binding() # noqa: SLF001 + session._run_generation = 1 # noqa: SLF001 + session._phase = "connection-ready" # noqa: SLF001 + session._state_revision = 2 # noqa: SLF001 + + failures: list[BaseException] = [] + + def enter_workspace() -> None: + try: + session.enter_workspace( + expected_session_generation=1, + expected_state_revision=2, + ) + except BaseException as exc: # pragma: no branch - asserted below + failures.append(exc) + + thread = threading.Thread(target=enter_workspace) + thread.start() + assert validation_entered.wait(timeout=1.0) + + closed = session.close_prestart( + expected_session_generation=1, + expected_state_revision=2, + ) + release_validation.set() + thread.join(timeout=1.0) + + assert not thread.is_alive() + assert len(failures) == 1 + assert isinstance(failures[0], ApplicationControlStateConflict) + assert closed["state_revision"] == 3 + assert session.snapshot()["state"] == "connection-ready" + assert session._workspace_requested.is_set() is False # noqa: SLF001 + + +def test_physical_reconciliation_uses_read_only_path_not_command_authority() -> None: + """The proof that unlocks an ambiguous edge cannot require that edge unlocked.""" + + path_validations: list[ApplicationConnectionBinding] = [] + command_validations: list[ApplicationConnectionBinding] = [] + + def validate_path(binding: ApplicationConnectionBinding) -> bool: + path_validations.append(binding) + return True + + def reject_command_authority(binding: ApplicationConnectionBinding) -> bool: + command_validations.append(binding) + return False + + transport = FakeTransport("192.168.1.20") + transport.open() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + connection_path_validator=validate_path, + connection_binding_validator=reject_command_authority, + ) + binding = _connection_binding() + with session._lock: # noqa: SLF001 + session._connection_binding = binding # noqa: SLF001 + session._live_control_binding = FakeExecutor(transport).binding # noqa: SLF001 + session._transport = transport # type: ignore[assignment] # noqa: SLF001 + session._verified_control = {"logical_device_id": "device-id"} # noqa: SLF001 + session._phase = "connection-ready" # noqa: SLF001 + + session.validate_physical_reconciliation_binding() + + assert path_validations == [binding] + assert command_validations == [] + with pytest.raises(ApplicationConnectionBindingLost): + session.validate_connection_binding() + assert command_validations == [binding] + + +def test_physical_reconciliation_rejects_stale_mqtt_control_proof() -> None: + """Read-only reconciliation never outlives its retained MQTT proof.""" + + transport = FakeTransport("192.168.1.20") + transport.open() + transport.proof_fresh = False + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + connection_path_validator=lambda _binding: True, + ) + with session._lock: # noqa: SLF001 + session._connection_binding = _connection_binding() # noqa: SLF001 + session._live_control_binding = FakeExecutor(transport).binding # noqa: SLF001 + session._transport = transport # type: ignore[assignment] # noqa: SLF001 + session._verified_control = {"logical_device_id": "device-id"} # noqa: SLF001 + session._phase = "connection-ready" # noqa: SLF001 + + with pytest.raises(ApplicationControlProofExpired): + session.validate_physical_reconciliation_binding() + + +def test_physical_reconciliation_rejects_host_path_drift() -> None: + """A fresh MQTT proof cannot authorize a changed host-path epoch.""" + + transport = FakeTransport("192.168.1.20") + transport.open() + path_validations: list[ApplicationConnectionBinding] = [] + + def reject_path(binding: ApplicationConnectionBinding) -> bool: + path_validations.append(binding) + return False + + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + connection_path_validator=reject_path, + ) + binding = _connection_binding() + with session._lock: # noqa: SLF001 + session._connection_binding = binding # noqa: SLF001 + session._live_control_binding = FakeExecutor(transport).binding # noqa: SLF001 + session._transport = transport # type: ignore[assignment] # noqa: SLF001 + session._verified_control = {"logical_device_id": "device-id"} # noqa: SLF001 + session._phase = "connection-ready" # noqa: SLF001 + + with pytest.raises(ApplicationConnectionBindingLost): + session.validate_physical_reconciliation_binding() + assert path_validations == [binding] + + +def test_durable_prepare_failure_does_not_release_start_checkpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator(fail_prepare=True) + transport = FakeTransport("192.168.1.20") + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + + with pytest.raises(RuntimeError, match="durable prepare failed"): + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=_successful_start_checkpoint_observer, # type: ignore[arg-type] + ) + + snapshot = session.snapshot() + assert snapshot["state"] == "project-ready" + assert snapshot["pending_operator_action"] == "start" + assert session._start_requested.is_set() is False # noqa: SLF001 + assert "start:11-14" not in FakeExecutor.records + assert len(coordinator.prepares) == 1 + + +def test_start_checkpoint_is_committed_before_worker_event_and_publish( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + observed: list[str] = [] + + def checkpoint( + phase: str, + _context: PhysicalCommandIntentContext, + _envelope: object, + ) -> None: + assert phase == "prepared" + assert len(coordinator.prepares) == 1 + assert session._start_requested.is_set() is False # noqa: SLF001 + assert "start:11-14" not in FakeExecutor.records + observed.append(phase) + + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=checkpoint, # type: ignore[arg-type] + ) + + assert observed == ["prepared"] + assert session._start_requested.is_set() is True # noqa: SLF001 + _wait_phase(session, "scanning") + + +def test_scanning_checkpoint_activation_linearizes_before_stop_prepare( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + activation_entered = threading.Event() + release_activation = threading.Event() + + def scanning_checkpoint() -> None: + activation_entered.set() + assert release_activation.wait(2.0) + + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + scanning_observer=scanning_checkpoint, + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=_successful_start_checkpoint_observer, # type: ignore[arg-type] + ) + assert activation_entered.wait(2.0) + assert session.snapshot()["state"] == "initializing" + + stop_errors: list[BaseException] = [] + + def request_stop() -> None: + try: + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + ) + except BaseException as exc: # pragma: no cover - asserted below + stop_errors.append(exc) + + stop_thread = threading.Thread(target=request_stop, daemon=True) + stop_thread.start() + threading.Event().wait(0.05) + assert stop_thread.is_alive() + assert [action for _context, action, _envelope in coordinator.prepares] == [ + "start" + ] + + release_activation.set() + stop_thread.join(2.0) + assert stop_thread.is_alive() is False + assert stop_errors == [] + assert [action for _context, action, _envelope in coordinator.prepares] == [ + "start", + "stop", + ] + _wait_phase(session, "completed") + + +def test_failed_scanning_checkpoint_cannot_be_bypassed_by_stop_prepare( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + observer_calls = 0 + + def failed_checkpoint() -> None: + nonlocal observer_calls + observer_calls += 1 + raise RuntimeError("checkpoint activation unavailable") + + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + scanning_observer=failed_checkpoint, + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=_successful_start_checkpoint_observer, # type: ignore[arg-type] + ) + _wait_phase(session, "scanning") + + with pytest.raises(ApplicationAcceptanceError, match="durably activated"): + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + ) + + assert observer_calls == 2 + assert [action for _context, action, _envelope in coordinator.prepares] == [ + "start" + ] + assert session.snapshot()["state"] == "scanning" + + +def test_start_checkpoint_failure_settles_no_dispatch_before_cease_callback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + coordinator.phase_probe = lambda: str(session.snapshot()["state"]) + phases: list[str] = [] + + def checkpoint( + phase: str, + _context: PhysicalCommandIntentContext, + _envelope: object, + ) -> None: + phases.append(phase) + if phase == "prepared": + raise RuntimeError("checkpoint fsync failed") + assert coordinator.resolutions == [ + ("start-not-dispatched", "project-ready") + ] + + with pytest.raises(RuntimeError, match="checkpoint fsync failed"): + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=checkpoint, # type: ignore[arg-type] + ) + + assert phases == ["prepared", "resolved-not-dispatched"] + assert session.snapshot()["state"] == "project-ready" + assert session._start_requested.is_set() is False # noqa: SLF001 + assert "start:11-14" not in FakeExecutor.records + + +def test_start_checkpoint_cease_failure_is_nonretryable_settlement_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + coordinator.phase_probe = lambda: str(session.snapshot()["state"]) + + def checkpoint( + phase: str, + _context: PhysicalCommandIntentContext, + _envelope: object, + ) -> None: + raise RuntimeError( + "checkpoint prepare failed" if phase == "prepared" else "cease fsync failed" + ) + + with pytest.raises(ApplicationStartCheckpointSettlementError): + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=checkpoint, # type: ignore[arg-type] + ) + + assert coordinator.resolutions == [ + ("start-not-dispatched", "project-ready") + ] + assert session.snapshot()["state"] == "project-ready" + assert session._start_requested.is_set() is False # noqa: SLF001 + assert "start:11-14" not in FakeExecutor.records + + +def test_session_generation_and_resolution_order_are_bound_to_durable_coordinator( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = FakePhysicalCommandCoordinator() + transport = FakeTransport("192.168.1.20") + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + coordinator.phase_probe = lambda: str(session.snapshot()["state"]) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + assert transport.evidence_observer is coordinator + assert len(coordinator.bindings) == 1 + runtime_binding = coordinator.bindings[0] + assert runtime_binding.producer_generation == session._run_generation == 1 # noqa: SLF001 + assert runtime_binding.control_session_id.startswith("application-control-1-") + + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=_successful_start_checkpoint_observer, # type: ignore[arg-type] + ) + _wait_phase(session, "scanning") + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + ) + _wait_phase(session, "completed") + + assert [action for _context, action, _envelope in coordinator.prepares] == [ + "start", + "stop", + ] + assert coordinator.resolutions == [ + ("start", "initializing"), + ("stop", "awaiting-standby-confirmation"), + ] + + +def test_retired_identity_under_new_transport_is_rejected_after_device_info_only( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """A new BLE UUID cannot move a retired K1 past read-only DeviceInfo.""" + + old_transport_ref = "A161D9D5-C352-1069-D430-5FB0BC13F7F9" + new_transport_ref = "B272E0E6-D463-2170-E541-6FC1CD24A8A0" + identity = PhysicalCommandIdentity( + vendor_device_id_sha256=hashlib.sha256( + RETIRED_VENDOR_DEVICE_ID.encode() + ).hexdigest(), + device_serial_sha256=hashlib.sha256( + RETIRED_DEVICE_SERIAL.encode() + ).hexdigest(), + ) + old_connection = PhysicalCommandConnectionBinding( + intent_id="retired-intent-1", + transport_ref=old_transport_ref, + connection_mode="bridge", + target_ipv4="192.168.68.51", + target_port=1883, + host_path_epoch=7, + control_session_id="retired-control-session-1", + producer_generation=11, + ) + baseline = PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=old_connection.control_session_id, + host_path_epoch=old_connection.host_path_epoch, + producer_generation=old_connection.producer_generation, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="d" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-10T18:00:00.000Z", + ) + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ledger = PhysicalCommandLedger(tmp_path / "repository") + operation_id = "retired-start-before-bootstrap-ordering-test" + ledger.prepare( + operation_id=operation_id, + parent_operation_id=None, + acquisition_id="retired-acquisition-before-bootstrap-ordering-test", + action="start", + identity=identity, + connection=old_connection, + compatibility_profile_id=COMPATIBILITY_PROFILE_ID, + payload_sha256="e" * 64, + baseline_status=baseline, + ) + ledger.mark_dispatching(operation_id) + unresolved = ledger.snapshot().record + assert unresolved is not None + ledger.retire_unavailable_target( + retirement_id="retired-bootstrap-ordering-test", + expected_operation_id=operation_id, + expected_revision=unresolved.revision, + expected_transport_ref=old_transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + durable_bytes = ledger.path.read_bytes() + + @dataclass + class OrderingTransportSnapshot: + state: str + publish_attempts: int + correlated_responses: int + + def as_dict(self) -> dict[str, object]: + return { + "state": self.state, + "publish_attempts": self.publish_attempts, + "qos2_completions": self.correlated_responses, + "correlated_responses": self.correlated_responses, + "ignored_known_responses": 0, + "late_known_responses": 0, + "automatic_retry": False, + "automatic_reconnect": False, + } + + class OrderingTransport(FakeTransport): + def __init__(self, host: str) -> None: + super().__init__(host) + self.batches: list[tuple[str, ...]] = [] + self.publish_attempts = 0 + self.correlated_responses = 0 + + def exchange_batch_once( + self, + envelopes: Sequence[OneShotPublishEnvelope], + *, + required_response_operation_keys: Collection[str], + ) -> dict[str, bytes]: + operations = tuple(envelope.operation_key for envelope in envelopes) + self.batches.append(operations) + self.publish_attempts += len(envelopes) + responses: dict[str, bytes] = {} + for operation_key in required_response_operation_keys: + _, ordinal_text, message_type = operation_key.split(":", 2) + ordinal = int(ordinal_text) + session_id = ( + f":{message_type}" + if ordinal <= 3 + else f"{RETIRED_VENDOR_DEVICE_ID}:{message_type}" + ) + if message_type == "DeviceConfigRequest": + session_id += ":Publish_Proto_DeviceConfig_SetTime" + responses[operation_key] = ( + _test_device_info_response(session_id) + if message_type == "DeviceInfoRequest" + else _test_generic_response(session_id) + ) + self.correlated_responses += 1 + return responses + + def snapshot(self) -> OrderingTransportSnapshot: + return OrderingTransportSnapshot( + self.state, + self.publish_attempts, + self.correlated_responses, + ) + + def maintain_open_for( + self, + _duration_seconds: float, + *, + allowed_response_topics: Collection[str] = (), + ) -> None: + del allowed_response_topics + + def scan_initialization_complete(self, _binding: object) -> bool: + return False + + def pre_start_ready(self, _binding: object) -> bool: + return True + + def standby_complete(self, _binding: object) -> bool: + return False + + def validate_bound_status(self, _binding: object) -> None: + return None + + transport = OrderingTransport("192.168.68.51") + coordinator = LedgerPhysicalCommandCoordinator(ledger) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + physical_command_coordinator=coordinator, + ) + session.open( + host="192.168.68.51", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=ApplicationConnectionBinding( + intent_id="fresh-intent-for-retired-identity", + transport_ref=new_transport_ref, + host_path_epoch=8, + target_ipv4="192.168.68.51", + target_port=1883, + connection_mode="bridge", + ), + ) + failed = _wait_phase(session, "failed") + worker = session._thread # noqa: SLF001 + assert worker is not None + worker.join(timeout=2.0) + assert not worker.is_alive() + + assert transport.batches == [("bootstrap:1:DeviceInfoRequest",)] + emitted = tuple(operation for batch in transport.batches for operation in batch) + assert not any("DeviceConfigRequest" in operation for operation in emitted) + assert not any(operation.startswith("dialogue:") for operation in emitted) + assert not any(operation.startswith("modeling:") for operation in emitted) + assert failed["verified_control"] is None + assert failed["can_enter_workspace"] is False + assert failed["can_prepare_project"] is False + assert failed["can_start"] is False + assert failed["failure"]["reason_code"] == ( # type: ignore[index] + "physical-command-reconciliation-required" + ) + assert ledger.path.read_bytes() == durable_bytes + + +def test_start_preflight_fails_closed_when_control_proof_is_stale( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + FakeExecutor, + ) + transport = FakeTransport("192.168.1.20") + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + epoch_seconds=lambda: 1_752_680_000, + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + + transport.proof_fresh = False + + session.request_start(project_name="TEST001", confirmation=_confirmation()) + failed = _wait_phase(session, "failed") + failure = failed["failure"] + assert isinstance(failure, dict) + assert failure["reason_code"] == ApplicationControlProofExpired.reason_code + assert failure["modeling_command_attempted"] is False + assert "start:11-14" not in FakeExecutor.records + + +@pytest.mark.parametrize( + ("guarded_stage", "forbidden_record", "outcome_unknown"), + [ + ("workspace", "workspace:7", False), + ("project", "project:8-10", False), + ("start", "start:11-14", False), + ("stop", "stop", True), + ], +) +def test_stale_connection_binding_blocks_each_pending_canonical_command( + monkeypatch: pytest.MonkeyPatch, + guarded_stage: str, + forbidden_record: str, + outcome_unknown: bool, +) -> None: + FakeExecutor.records = [] + binding_current = True + allow_one_stale_preflight = False + + def validate_binding(_binding: ApplicationConnectionBinding) -> bool: + nonlocal allow_one_stale_preflight + if allow_one_stale_preflight: + allow_one_stale_preflight = False + return True + return binding_current + + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + FakeExecutor, + ) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + connection_binding_validator=validate_binding, + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + + if guarded_stage != "workspace": + session.enter_workspace() + _wait_phase(session, "workspace-ready") + if guarded_stage not in {"workspace", "project"}: + session.open_project_prompt() + _wait_phase(session, "project-ready") + if guarded_stage == "stop": + session.request_start(project_name="TEST001", confirmation=_confirmation()) + _wait_phase(session, "scanning") + + binding_current = False + if guarded_stage == "workspace": + allow_one_stale_preflight = True + session.enter_workspace() + elif guarded_stage == "project": + allow_one_stale_preflight = True + session.open_project_prompt() + elif guarded_stage == "start": + session.request_start(project_name="TEST001", confirmation=_confirmation()) + else: + session.request_stop(confirmation=_confirmation()) + + failed = _wait_phase(session, "failed") + worker = session._thread # noqa: SLF001 + assert worker is not None + worker.join(timeout=2.0) + assert not worker.is_alive() + failed = session.snapshot() + + assert forbidden_record not in FakeExecutor.records + assert failed["outcome_unknown"] is outcome_unknown + failure = failed["failure"] + assert isinstance(failure, dict) + assert failure["reason_code"] == "application-connection-binding-lost" + assert failure["modeling_command_attempted"] is False + assert failure["safe_to_retry"] is (not outcome_unknown) + + +def test_binding_is_revalidated_after_correlated_response_before_phase_promotion( + monkeypatch: pytest.MonkeyPatch, +) -> None: + FakeExecutor.records = [] + binding_current = True + + class RouteChangingExecutor(FakeExecutor): + def run_workspace_entry_stage( + self, + orchestrator: object, + checkpoint: object, + *, + dispatch_guard: Callable[[], None] | None = None, + ) -> LiveDeviceControlBinding: + nonlocal binding_current + result = super().run_workspace_entry_stage( + orchestrator, + checkpoint, + dispatch_guard=dispatch_guard, + ) + binding_current = False + return result + + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + RouteChangingExecutor, + ) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + connection_binding_validator=lambda _binding: binding_current, + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + + session.enter_workspace() + failed = _wait_phase(session, "failed") + + assert "workspace:7" in FakeExecutor.records + assert failed["state"] != "workspace-ready" + assert failed["failure"]["reason_code"] == ( # type: ignore[index] + "application-connection-binding-lost" + ) + + def test_prestart_failure_is_reported_and_requires_a_new_operator_click( monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, @@ -236,7 +1680,7 @@ def test_prestart_failure_is_reported_and_requires_a_new_operator_click( caplog.set_level(logging.ERROR) class BootstrapFailureExecutor(FakeExecutor): - def run_connection_stage( + def run_read_only_inspection_stage( self, _orchestrator: object, ) -> LiveDeviceControlBinding: @@ -284,12 +1728,14 @@ def test_prestart_failure_is_reported_and_requires_a_new_operator_click( "dialogue_stage": "connection", "transport_state": "ready", "publish_attempts": 0, + "stop_publish_attempts": None, "qos2_completions": 0, "correlated_responses": 0, "ignored_known_responses": 0, - "late_known_responses": 0, - "modeling_command_attempted": False, - "diagnostic_snapshot_unavailable": [], + "late_known_responses": 0, + "modeling_command_attempted": False, + "stop_command_attempted": False, + "diagnostic_snapshot_unavailable": [], "diagnostic_evidence_unavailable": [], "correlation_failure": None, "compatibility_failure": None, @@ -325,7 +1771,7 @@ def test_correlated_read_only_profile_mismatch_allows_only_a_fresh_explicit_atte return CorrelatedReadSnapshot(self.state) class ProfileMismatchExecutor(FakeExecutor): - def run_connection_stage( + def run_read_only_inspection_stage( self, _orchestrator: object, ) -> LiveDeviceControlBinding: @@ -429,13 +1875,13 @@ def test_new_explicit_session_waits_for_old_worker_transport_retirement( type(self).instances += 1 self.instance = type(self).instances - def run_connection_stage( + def run_read_only_inspection_stage( self, orchestrator: object, ) -> LiveDeviceControlBinding: if self.instance == 1: raise RuntimeError("first generation failed before publish") - return super().run_connection_stage(orchestrator) + return super().run_read_only_inspection_stage(orchestrator) def transport_factory(host: str) -> TaggedTransport: transport = TaggedTransport(host, f"generation-{len(transports) + 1}") @@ -573,8 +2019,29 @@ def test_start_outcome_unknown_blocks_reopen_even_when_transport_is_closed( session.retire_for_network_change() +@pytest.mark.parametrize( + ("disconnect_error", "expected_reason_code"), + [ + ( + session_module.ApplicationCommandOutcomeUnknown( + "control MQTT network loop returned an error", + reason_code="mqtt_network_loop_failed", + ), + "mqtt_network_loop_failed", + ), + ( + session_module.ApplicationConnectionBindingLost( + "Mac Wi-Fi route changed before READY confirmation" + ), + "application-connection-binding-lost", + ), + ], + ids=["mqtt-network-loop-lost", "host-route-binding-lost"], +) def test_acknowledged_stop_disconnect_allows_only_explicit_network_change( monkeypatch: pytest.MonkeyPatch, + disconnect_error: Exception, + expected_reason_code: str, ) -> None: @dataclass class StopDisconnectTransportSnapshot: @@ -622,16 +2089,11 @@ def test_acknowledged_stop_disconnect_allows_only_explicit_network_change( def maintain_post_stop_until_standby(self) -> None: self.records.append("wait:device-standby") self.transport.state = "poisoned" - raise session_module.ApplicationCommandOutcomeUnknown( - "control MQTT network loop returned an error", - reason_code="mqtt_network_loop_failed", - ) + raise disconnect_error def snapshot(self) -> dict[str, object]: return { - "dialogue_stage": ( - "stop-acknowledged" if self.stop_complete else "test-stage" - ), + "dialogue_stage": ("stop-acknowledged" if self.stop_complete else "test-stage"), "start_attempted": True, "start_complete": True, "stop_attempted": self.stop_attempted, @@ -675,6 +2137,7 @@ def test_acknowledged_stop_disconnect_allows_only_explicit_network_change( assert failed["outcome_unknown"] is True failure = failed["failure"] assert isinstance(failure, dict) + assert failure["reason_code"] == expected_reason_code assert failure["safe_to_retry"] is False assert failure["network_change_admissible"] is True assert failure["network_change_reconciliation"] == { @@ -693,6 +2156,171 @@ def test_acknowledged_stop_disconnect_allows_only_explicit_network_change( assert retired["can_open"] is True +def test_stop_dispatch_deadline_is_definite_zero_publish_not_unknown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + deadline_checks = 0 + expired = False + + def deadline_reached() -> bool: + nonlocal deadline_checks + deadline_checks += 1 + return expired + + class DeadlineExecutor(FakeExecutor): + def execute_canonical_start(self, *_args: object, **_kwargs: object) -> object: + result = super().execute_canonical_start(*_args, **_kwargs) + self.transport.publish_attempts = 6 + return result + + def execute_canonical_stop(self, *_args: object, **kwargs: object) -> object: + nonlocal expired + predicate = kwargs.get("dispatch_admission_deadline_reached") + assert callable(predicate) + expired = True + assert predicate() is True + self.transport.state = "failed" + raise ApplicationMqttTransportError( + "control command dispatch deadline expired before publish admission", + reason_code="physical-command-dispatch-deadline-expired", + ) + + def snapshot(self) -> dict[str, object]: + return { + "dialogue_stage": "stop-requested", + "start_attempted": True, + "stop_attempted": False, + "stop_complete": False, + "automatic_retry": False, + } + + FakeExecutor.records = [] + monkeypatch.setattr( + session_module, + "PhysicalAcceptanceDialogueExecutor", + DeadlineExecutor, + ) + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + confirmation=_confirmation(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + session.request_start(project_name="TEST001", confirmation=_confirmation()) + _wait_phase(session, "scanning") + + session.request_stop( + confirmation=_confirmation(), + dispatch_admission_deadline_reached=deadline_reached, + ) + failed = _wait_phase(session, "failed") + + assert deadline_checks >= 4 + assert failed["outcome_unknown"] is False + failure = failed["failure"] + assert isinstance(failure, dict) + assert failure["reason_code"] == "physical-command-dispatch-deadline-expired" + assert failure["publish_attempts"] == 6 + assert failure["stop_publish_attempts"] == 0 + assert failure["modeling_command_attempted"] is True + assert failure["stop_command_attempted"] is False + assert failure["diagnostic_evidence_unavailable"] == [] + + +def test_stop_deadline_expiring_during_durable_prepare_settles_exact_no_dispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + expired = False + + class ExpiringStopPrepareCoordinator(FakePhysicalCommandCoordinator): + def prepare( + self, + context: PhysicalCommandIntentContext, + *, + action: str, + envelope: object, + ) -> None: + nonlocal expired + super().prepare(context, action=action, envelope=envelope) + if action == "stop": + expired = True + + FakeExecutor.records = [] + monkeypatch.setattr(session_module, "PhysicalAcceptanceDialogueExecutor", FakeExecutor) + coordinator = ExpiringStopPrepareCoordinator() + transport = FakeTransport("192.168.1.20") + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + transport_factory=lambda _host: transport, # type: ignore[arg-type] + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + session.open( + host="192.168.1.20", + timezone_name="Europe/Moscow", + connection_binding=_connection_binding(), + ) + _wait_phase(session, "connection-ready") + session.enter_workspace() + _wait_phase(session, "workspace-ready") + session.open_project_prompt() + _wait_phase(session, "project-ready") + session.request_start( + project_name="TEST001", + confirmation=_confirmation(), + command_context=_physical_context("start"), + preparation_checkpoint_observer=_successful_start_checkpoint_observer, + ) + scanning = _wait_phase(session, "scanning") + publish_attempts_before = transport.publish_attempts + + with pytest.raises(ApplicationMqttTransportError) as raised: + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + dispatch_admission_deadline_reached=lambda: expired, + expected_session_generation=scanning["session_generation"], # type: ignore[arg-type] + expected_state_revision=scanning["state_revision"], # type: ignore[arg-type] + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert session.snapshot()["state"] == "scanning" + assert session._stop_requested.is_set() is False # noqa: SLF001 + assert transport.publish_attempts - publish_attempts_before == 0 + assert [action for _context, action, _envelope in coordinator.prepares] == [ + "start", + "stop", + ] + assert coordinator.resolutions[-1][0] == "stop-not-dispatched" + assert FakeExecutor.records.count("stop") == 0 + + +def test_preadmission_deadline_rejects_stop_before_validation_or_prepare() -> None: + coordinator = FakePhysicalCommandCoordinator() + session = InteractiveApplicationControlSession( + FakeAuthorityLoader(), + physical_command_coordinator=coordinator, # type: ignore[arg-type] + ) + + with pytest.raises(ApplicationMqttTransportError) as raised: + session.request_stop( + confirmation=_confirmation(), + command_context=_physical_context("stop"), + dispatch_admission_deadline_reached=lambda: True, + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert coordinator.prepares == [] + assert session.snapshot()["state"] == "idle" + + def test_prestart_loop_failure_allows_only_fresh_explicit_retry_after_ready_status( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -711,6 +2339,7 @@ def test_prestart_loop_failure_allows_only_fresh_explicit_retry_after_ready_stat "device_status_reports": 1, "latest_device_session_state": "ready", "latest_device_project_bound": False, + "latest_device_init_ready": False, "latest_system_error_code": None, "last_loop_result_code": 7, "last_loop_result_name": "The connection was lost.", @@ -724,10 +2353,13 @@ def test_prestart_loop_failure_allows_only_fresh_explicit_retry_after_ready_stat return ReconciledTransportSnapshot(self.state) class PrestartLoopFailureExecutor(FakeExecutor): - def run_connection_stage( + def complete_connection_stage( self, _orchestrator: object, + *, + expected_binding: LiveDeviceControlBinding, ) -> LiveDeviceControlBinding: + assert expected_binding == self.binding raise session_module.ApplicationCommandOutcomeUnknown( "control MQTT network loop returned an error", reason_code="mqtt_network_loop_failed", @@ -790,10 +2422,13 @@ def test_unavailable_transport_snapshot_after_publish_blocks_reopen( return super().snapshot() class PublishedThenFailedExecutor(FakeExecutor): - def run_connection_stage( + def complete_connection_stage( self, _orchestrator: object, + *, + expected_binding: LiveDeviceControlBinding, ) -> LiveDeviceControlBinding: + assert expected_binding == self.binding executor_entered.set() assert release_failure.wait(timeout=2.0) assert isinstance(self.transport, SnapshotFailureTransport) @@ -852,7 +2487,7 @@ def test_unavailable_dialogue_snapshot_is_outcome_unknown( monkeypatch: pytest.MonkeyPatch, ) -> None: class DialogueSnapshotFailureExecutor(FakeExecutor): - def run_connection_stage( + def run_read_only_inspection_stage( self, _orchestrator: object, ) -> LiveDeviceControlBinding: diff --git a/tests/test_xgrids_ble_runtime_arbiter.py b/tests/test_xgrids_ble_runtime_arbiter.py new file mode 100644 index 0000000..8660a9f --- /dev/null +++ b/tests/test_xgrids_ble_runtime_arbiter.py @@ -0,0 +1,823 @@ +from __future__ import annotations + +import asyncio +import multiprocessing +import os +import threading +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +import pytest + +import k1link.device_plugins.xgrids_k1.ble.scanner as scanner_module +import k1link.device_plugins.xgrids_k1.ble.wifi_provisioning as wifi_module +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + ApplicationControlProcessLease, + ApplicationControlProcessLeaseUnavailable, +) +from k1link.device_plugins.xgrids_k1.ble.runtime_arbiter import ( + BleOperationHardTimeout, + BleOperationProgress, + BleRuntimeBusy, + BleRuntimeOwnerLoopConflict, + BleRuntimePoisoned, + BleRuntimeProcessLeaseNotConfigured, + bind_ble_runtime_owner_loop, + ble_runtime_snapshot, + borrow_ble_runtime_process_lease, + configure_ble_runtime_process_lease, + defer_until_ble_runtime_idle, + reset_ble_runtime_arbiter_for_tests, + run_ble_operation, + run_ble_operation_session, + wait_for_ble_runtime_idle, +) + + +def _hold_external_process_lease( + repository_root: str, + data_dir: str, + ready: Any, + release: Any, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + lease = ApplicationControlProcessLease.acquire(Path(repository_root)) + ready.set() + try: + release.wait(timeout=10.0) + finally: + lease.release() + + +@pytest.fixture(autouse=True) +def configured_process_ble_runtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[Path]: + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + repository_root = tmp_path / "repository" + repository_root.mkdir() + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + configure_ble_runtime_process_lease(repository_root) + yield repository_root + scanner_module.reset_runtime_handles_for_tests() + reset_ble_runtime_arbiter_for_tests() + + +def test_direct_low_level_ble_call_is_blocked_by_external_process_owner( + configured_process_ble_runtime: Path, +) -> None: + context = multiprocessing.get_context("spawn") + ready = context.Event() + release = context.Event() + process = context.Process( + target=_hold_external_process_lease, + args=( + str(configured_process_ble_runtime), + os.environ["MISSIONCORE_DATA_DIR"], + ready, + release, + ), + ) + process.start() + operation_called = False + + async def operation(_progress: BleOperationProgress) -> None: + nonlocal operation_called + operation_called = True + + try: + assert ready.wait(timeout=10.0) + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + asyncio.run( + run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=operation, + ) + ) + assert operation_called is False + finally: + release.set() + process.join(timeout=10.0) + if process.is_alive(): + process.terminate() + process.join(timeout=10.0) + + assert process.exitcode == 0 + + +def test_borrowed_external_process_lease_avoids_double_acquire( + configured_process_ble_runtime: Path, +) -> None: + external_lease = ApplicationControlProcessLease.acquire( + configured_process_ble_runtime + ) + try: + with borrow_ble_runtime_process_lease(external_lease): + result = asyncio.run( + run_ble_operation( + "status-read", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_value("borrowed"), + ) + ) + assert result == "borrowed" + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(configured_process_ble_runtime) + finally: + external_lease.release() + + with ApplicationControlProcessLease.acquire(configured_process_ble_runtime): + pass + + +def test_session_owns_os_lease_until_native_context_exit( + configured_process_ble_runtime: Path, +) -> None: + async def scenario() -> None: + @asynccontextmanager + async def ready_session( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + yield "ready" + + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.1, + operation=ready_session, + ) as value: + assert value == "ready" + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(configured_process_ble_runtime) + + assert await wait_for_ble_runtime_idle() + with ApplicationControlProcessLease.acquire(configured_process_ble_runtime): + pass + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("contender", ["status-read", "wifi-provision"]) +def test_scan_lease_rejects_wifi_entrypoints_process_wide( + monkeypatch: pytest.MonkeyPatch, + contender: str, +) -> None: + async def scenario() -> None: + scan_entered = asyncio.Event() + release_scan = asyncio.Event() + resolution_attempted = False + + async def blocked_discover(**_kwargs: object) -> dict[str, object]: + scan_entered.set() + await release_scan.wait() + return {} + + async def forbidden_resolution(*_args: object, **_kwargs: object) -> None: + nonlocal resolution_attempted + resolution_attempted = True + raise AssertionError("busy contender must not touch CoreBluetooth") + + monkeypatch.setattr(scanner_module.BleakScanner, "discover", blocked_discover) + monkeypatch.setattr( + wifi_module.BleakScanner, + "find_device_by_address", + forbidden_resolution, + ) + scan_task = asyncio.create_task(scanner_module.scan(1.0)) + await scan_entered.wait() + try: + if contender == "status-read": + with pytest.raises(BleRuntimeBusy) as raised: + await wifi_module.read_wifi_status_once( + "synthetic-device", + timeout_seconds=0.1, + ) + else: + with pytest.raises(BleRuntimeBusy) as raised: + await wifi_module.provision_wifi_once( + "synthetic-device", + "LabNet", + "synthetic-password", + timeout_seconds=0.1, + ) + assert raised.value.reason_code == "ble-runtime-busy" + assert raised.value.active_operation_kind == "scan" + assert resolution_attempted is False + finally: + release_scan.set() + await scan_task + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_hard_deadline_returns_before_cleanup_and_quarantines_reentry() -> None: + async def scenario() -> None: + cancellation_observed = asyncio.Event() + cleanup_release = asyncio.Event() + + async def stubborn_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancellation_observed.set() + await cleanup_release.wait() + raise + + loop = asyncio.get_running_loop() + started = loop.time() + with pytest.raises(BleOperationHardTimeout) as timed_out: + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.01, + operation=stubborn_cleanup, + ) + elapsed = loop.time() - started + assert elapsed < 0.25 + assert timed_out.value.reason_code == "ble-status-read-timeout" + await cancellation_observed.wait() + assert ble_runtime_snapshot()["cleanup_pending"] is True + + with pytest.raises(BleRuntimeBusy) as busy: + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert busy.value.reason_code == "ble-runtime-cleanup-pending" + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + result = await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_value("recovered"), + ) + assert result == "recovered" + + asyncio.run(scenario()) + + +def test_idle_callback_runs_synchronously_once_when_runtime_is_idle() -> None: + calls: list[str] = [] + + disposition = defer_until_ble_runtime_idle(lambda: calls.append("released")) + + assert disposition == "released" + assert calls == ["released"] + + +def test_idle_callback_waits_for_detached_cleanup_and_runs_once( + configured_process_ble_runtime: Path, +) -> None: + async def scenario() -> None: + cancellation_observed = asyncio.Event() + cleanup_release = asyncio.Event() + calls: list[str] = [] + + async def stubborn_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + cancellation_observed.set() + await cleanup_release.wait() + raise + + with pytest.raises(BleOperationHardTimeout): + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.01, + operation=stubborn_cleanup, + ) + await cancellation_observed.wait() + + disposition = defer_until_ble_runtime_idle(lambda: calls.append("released")) + assert disposition == "deferred" + assert calls == [] + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(configured_process_ble_runtime) + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + assert calls == ["released"] + await asyncio.sleep(0) + assert calls == ["released"] + with ApplicationControlProcessLease.acquire(configured_process_ble_runtime): + pass + + asyncio.run(scenario()) + + +def test_idle_callback_is_retained_forever_when_cleanup_poisoned( + configured_process_ble_runtime: Path, +) -> None: + async def scenario() -> None: + cancellation_observed = asyncio.Event() + fail_cleanup = asyncio.Event() + calls: list[str] = [] + + async def failing_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError as exc: + cancellation_observed.set() + await fail_cleanup.wait() + raise RuntimeError("synthetic native cleanup failure") from exc + + with pytest.raises(BleOperationHardTimeout): + await run_ble_operation( + "wifi-provision", + hard_timeout_seconds=0.01, + operation=failing_cleanup, + ) + await cancellation_observed.wait() + assert defer_until_ble_runtime_idle(lambda: calls.append("before-poison")) == "deferred" + + fail_cleanup.set() + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + + assert ble_runtime_snapshot()["poisoned"] is True + assert calls == [] + assert ( + defer_until_ble_runtime_idle(lambda: calls.append("after-poison")) + == "poisoned" + ) + assert await wait_for_ble_runtime_idle(timeout_seconds=0.01) is False + assert calls == [] + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(configured_process_ble_runtime) + + asyncio.run(scenario()) + + +def test_test_reset_releases_poisoned_owned_lock_and_clears_configuration( + configured_process_ble_runtime: Path, +) -> None: + async def poison_runtime() -> None: + async def failing_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError as exc: + raise RuntimeError("synthetic cleanup failure") from exc + + with pytest.raises(BleOperationHardTimeout): + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.01, + operation=failing_cleanup, + ) + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + assert ble_runtime_snapshot()["poisoned"] is True + + asyncio.run(poison_runtime()) + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(configured_process_ble_runtime) + + reset_ble_runtime_arbiter_for_tests() + with ApplicationControlProcessLease.acquire(configured_process_ble_runtime): + pass + with pytest.raises(BleRuntimeProcessLeaseNotConfigured): + asyncio.run( + run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + ) + + +def test_idle_callback_registration_racing_release_executes_exactly_once() -> None: + async def race_once() -> None: + operation_entered = asyncio.Event() + operation_release = asyncio.Event() + registration_gate = threading.Barrier(2) + registration_finished = threading.Event() + registration_errors: list[BaseException] = [] + dispositions: list[str] = [] + calls = 0 + calls_lock = threading.Lock() + + async def operation(_progress: BleOperationProgress) -> None: + operation_entered.set() + await operation_release.wait() + + operation_task = asyncio.create_task( + run_ble_operation( + "scan", + hard_timeout_seconds=1.0, + operation=operation, + ) + ) + await operation_entered.wait() + + def callback() -> None: + nonlocal calls + with calls_lock: + calls += 1 + + def register() -> None: + try: + registration_gate.wait() + dispositions.append(defer_until_ble_runtime_idle(callback)) + except BaseException as exc: + registration_errors.append(exc) + finally: + registration_finished.set() + + registration_thread = threading.Thread(target=register, daemon=True) + registration_thread.start() + registration_gate.wait() + operation_release.set() + await operation_task + while not registration_finished.is_set(): + await asyncio.sleep(0) + registration_thread.join(timeout=0.1) + assert registration_thread.is_alive() is False + assert registration_errors == [] + assert dispositions in (["deferred"], ["released"]) + assert await wait_for_ble_runtime_idle() + assert calls == 1 + + async def scenario() -> None: + for _ in range(32): + await race_once() + + asyncio.run(scenario()) + + +def test_failed_deferred_idle_callback_poisons_and_retains_ownership() -> None: + async def scenario() -> None: + operation_entered = asyncio.Event() + operation_release = asyncio.Event() + callback_calls = 0 + later_calls: list[str] = [] + + async def operation(_progress: BleOperationProgress) -> None: + operation_entered.set() + await operation_release.wait() + + def failed_release() -> None: + nonlocal callback_calls + callback_calls += 1 + raise RuntimeError("synthetic external lease release failure") + + operation_task = asyncio.create_task( + run_ble_operation( + "status-read", + hard_timeout_seconds=1.0, + operation=operation, + ) + ) + await operation_entered.wait() + assert defer_until_ble_runtime_idle(failed_release) == "deferred" + operation_release.set() + await operation_task + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + + snapshot = ble_runtime_snapshot() + assert callback_calls == 1 + assert snapshot["poisoned"] is True + assert snapshot["cleanup_pending"] is True + assert snapshot["active_operation_kind"] == "status-read" + assert ( + defer_until_ble_runtime_idle(lambda: later_calls.append("unsafe-release")) + == "poisoned" + ) + assert later_calls == [] + with pytest.raises(BleRuntimePoisoned): + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + + asyncio.run(scenario()) + + +def test_hard_deadline_cleanup_failure_poison_blocks_next_ble_lease() -> None: + async def scenario() -> None: + async def failing_cancel_cleanup(_progress: BleOperationProgress) -> None: + try: + await asyncio.Event().wait() + except asyncio.CancelledError as exc: + raise RuntimeError("synthetic disconnect failure") from exc + + with pytest.raises(BleOperationHardTimeout): + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.01, + operation=failing_cancel_cleanup, + ) + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + snapshot = ble_runtime_snapshot() + assert snapshot["cleanup_pending"] is True + assert snapshot["poisoned"] is True + with pytest.raises(BleRuntimePoisoned): + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + + asyncio.run(scenario()) + + +def test_session_setup_deadline_returns_before_noncooperative_cleanup() -> None: + async def scenario() -> None: + cancellation_observed = asyncio.Event() + cleanup_release = asyncio.Event() + + @asynccontextmanager + async def blocked_session( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + try: + await asyncio.Event().wait() + yield "unreachable" + except asyncio.CancelledError: + cancellation_observed.set() + await cleanup_release.wait() + raise + + loop = asyncio.get_running_loop() + started = loop.time() + with pytest.raises(BleOperationHardTimeout) as timed_out: + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.01, + operation=blocked_session, + ): + raise AssertionError("blocked setup must never yield") + assert loop.time() - started < 0.25 + assert timed_out.value.reason_code == "ble-ap-enable-timeout" + await cancellation_observed.wait() + assert ble_runtime_snapshot()["cleanup_pending"] is True + + with pytest.raises(BleRuntimeBusy) as busy: + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert busy.value.reason_code == "ble-runtime-cleanup-pending" + assert busy.value.active_operation_kind == "ap-enable" + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_session_setup_cleanup_failure_requires_runtime_restart() -> None: + async def scenario() -> None: + @asynccontextmanager + async def failing_setup_cleanup( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + try: + await asyncio.Event().wait() + yield "unreachable" + except asyncio.CancelledError as exc: + raise RuntimeError("synthetic setup teardown failure") from exc + + with pytest.raises(BleOperationHardTimeout): + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.01, + operation=failing_setup_cleanup, + ): + raise AssertionError("blocked setup must never yield") + + for _ in range(100): + if ble_runtime_snapshot()["poisoned"]: + break + await asyncio.sleep(0.001) + snapshot = ble_runtime_snapshot() + assert snapshot["cleanup_pending"] is True + assert snapshot["poisoned"] is True + with pytest.raises(BleRuntimePoisoned) as poisoned: + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert poisoned.value.reason_code == "ble-runtime-restart-required" + + asyncio.run(scenario()) + + +def test_session_setup_deadline_stops_after_ready_while_lease_stays_held() -> None: + async def scenario() -> None: + cleanup_completed = asyncio.Event() + + @asynccontextmanager + async def ready_session( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + try: + yield "ready" + finally: + cleanup_completed.set() + + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.01, + operation=ready_session, + ) as value: + assert value == "ready" + await asyncio.sleep(0.05) + assert ble_runtime_snapshot()["active_operation_kind"] == "ap-enable" + with pytest.raises(BleRuntimeBusy) as busy: + await run_ble_operation( + "status-read", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert busy.value.reason_code == "ble-runtime-busy" + + assert cleanup_completed.is_set() + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_session_cleanup_deadline_detaches_without_releasing_process_lease() -> None: + async def scenario() -> None: + cleanup_entered = asyncio.Event() + cleanup_release = asyncio.Event() + + @asynccontextmanager + async def stubborn_cleanup_session( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + try: + yield "ready" + finally: + cleanup_entered.set() + await cleanup_release.wait() + + loop = asyncio.get_running_loop() + started = loop.time() + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.1, + hard_cleanup_timeout_seconds=0.01, + operation=stubborn_cleanup_session, + ) as value: + assert value == "ready" + assert loop.time() - started < 0.25 + await cleanup_entered.wait() + assert ble_runtime_snapshot()["cleanup_pending"] is True + + with pytest.raises(BleRuntimeBusy) as busy: + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert busy.value.reason_code == "ble-runtime-cleanup-pending" + + cleanup_release.set() + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_session_cleanup_failure_requires_runtime_restart() -> None: + async def scenario() -> None: + @asynccontextmanager + async def failing_cleanup_session( + _progress: BleOperationProgress, + ) -> AsyncIterator[str]: + try: + yield "ready" + finally: + raise RuntimeError("synthetic native disconnect failure") + + with pytest.raises(RuntimeError, match="disconnect failure"): + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=0.1, + operation=failing_cleanup_session, + ) as value: + assert value == "ready" + + await asyncio.sleep(0) + snapshot = ble_runtime_snapshot() + assert snapshot["cleanup_pending"] is True + assert snapshot["poisoned"] is True + # Dispatchers may still bind non-BLE safety/state actions on the same + # owner loop; only a new BLE lease is prohibited until process restart. + assert bind_ble_runtime_owner_loop() == snapshot["owner_epoch"] + with pytest.raises(BleRuntimePoisoned) as poisoned: + await run_ble_operation( + "scan", + hard_timeout_seconds=0.1, + operation=lambda _progress: _completed_none(), + ) + assert poisoned.value.reason_code == "ble-runtime-restart-required" + + asyncio.run(scenario()) + + +def test_caller_cancellation_preserves_ble_write_progress() -> None: + async def scenario() -> None: + entered = asyncio.Event() + + async def blocked_write(progress: BleOperationProgress) -> None: + progress.operation_stage = "gatt-write" + progress.device_write_attempted = True + progress.device_write_confirmed = True + entered.set() + await asyncio.Event().wait() + + task = asyncio.create_task( + run_ble_operation( + "wifi-provision", + hard_timeout_seconds=5.0, + operation=blocked_write, + ) + ) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError) as raised: + await task + assert raised.value.operation_stage == "gatt-write" # type: ignore[attr-defined] + assert raised.value.device_write_attempted is True # type: ignore[attr-defined] + assert raised.value.device_write_confirmed is True # type: ignore[attr-defined] + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_session_body_cancellation_preserves_ble_write_progress() -> None: + async def scenario() -> None: + body_entered = asyncio.Event() + + @asynccontextmanager + async def ready_after_write( + progress: BleOperationProgress, + ) -> AsyncIterator[str]: + progress.operation_stage = "status-poll" + progress.device_write_attempted = True + progress.device_write_confirmed = True + yield "ready" + + async def caller() -> None: + async with run_ble_operation_session( + "ap-enable", + hard_setup_timeout_seconds=1.0, + operation=ready_after_write, + ): + body_entered.set() + await asyncio.Event().wait() + + task = asyncio.create_task(caller()) + await body_entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError) as raised: + await task + assert raised.value.operation_stage == "status-poll" # type: ignore[attr-defined] + assert raised.value.device_write_attempted is True # type: ignore[attr-defined] + assert raised.value.device_write_confirmed is True # type: ignore[attr-defined] + assert await wait_for_ble_runtime_idle() + + asyncio.run(scenario()) + + +def test_live_owner_loop_cannot_be_replaced() -> None: + first_loop = asyncio.new_event_loop() + second_loop = asyncio.new_event_loop() + try: + first_epoch = bind_ble_runtime_owner_loop(first_loop) + with pytest.raises(BleRuntimeOwnerLoopConflict) as raised: + bind_ble_runtime_owner_loop(second_loop) + assert raised.value.reason_code == "ble-runtime-owner-loop-conflict" + assert ble_runtime_snapshot()["owner_epoch"] == first_epoch + finally: + first_loop.close() + second_loop.close() + + +async def _completed_none() -> None: + return None + + +async def _completed_value(value: str) -> str: + return value diff --git a/tests/test_xgrids_camera_gateway.py b/tests/test_xgrids_camera_gateway.py index e3086a8..137c7fd 100644 --- a/tests/test_xgrids_camera_gateway.py +++ b/tests/test_xgrids_camera_gateway.py @@ -1,8 +1,10 @@ from __future__ import annotations +import asyncio import hashlib import json import sys +import threading import time from io import BytesIO from pathlib import Path @@ -12,10 +14,25 @@ import pytest import k1link.device_plugins.xgrids_k1.camera as camera_module from k1link.device_plugins.xgrids_k1.camera import ( CAMERA_MEDIA_TYPE, + CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS, + CAMERA_SOURCE_IO_TIMEOUT_MICROSECONDS, + MAX_CAMERA_PREVIEW_CONSUMERS, + MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS, + MAX_CAMERA_PREVIEW_QUEUED_BYTES, + MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS, + MAX_FMP4_SEGMENT_BYTES, CommittedCameraSegment, XgridsK1CameraGateway, _build_ffmpeg_argv, + _CameraPreviewSegmentQueue, _read_mp4_box, + build_xgrids_k1_camera_router, + classify_camera_recording_health, +) +from k1link.device_plugins.xgrids_k1.connection_supervisor import ( + EndpointTarget, + HostPathProbeResult, + VerifiedControlEvidence, ) from k1link.device_plugins.xgrids_k1.facade import ( XGRIDS_K1_COMPATIBILITY_PROFILE_ID, @@ -65,6 +82,66 @@ def _burst_ffmpeg(tmp_path: Path) -> Path: return executable +def _preview_buffer_ffmpeg( + tmp_path: Path, + *, + fragments: int, + payload_bytes: int, + interval_seconds: float = 0.002, +) -> Path: + executable = tmp_path / f"preview-buffer-{fragments}-{payload_bytes}" + executable.write_text( + f"#!{sys.executable}\n" + "import sys, time\n" + "def box(kind, payload=b''):\n" + " return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n" + "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n" + "sys.stdout.buffer.flush()\n" + f"payload = bytes({payload_bytes})\n" + f"for index in range({fragments}):\n" + " sys.stdout.buffer.write(box(b'moof') + box(b'mdat', payload))\n" + " sys.stdout.buffer.flush()\n" + f" time.sleep({interval_seconds!r})\n" + "time.sleep(10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable + + +def _single_fragment_ffmpeg(tmp_path: Path, *, complete_fragment_bytes: int) -> Path: + assert complete_fragment_bytes >= 16 + executable = tmp_path / f"single-fragment-{complete_fragment_bytes}" + executable.write_text( + f"#!{sys.executable}\n" + "import sys, time\n" + "def box(kind, payload=b''):\n" + " return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n" + "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n" + "sys.stdout.buffer.flush()\n" + f"payload_bytes = {complete_fragment_bytes} - 16\n" + "payload = b'PRIVATE-FRAME-BYTES' + bytes(payload_bytes - 19)\n" + "sys.stdout.buffer.write(box(b'moof') + box(b'mdat', payload))\n" + "sys.stdout.buffer.flush()\n" + "time.sleep(10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable + + +def _silent_ffmpeg(tmp_path: Path) -> Path: + executable = tmp_path / "silent-ffmpeg" + executable.write_text( + f"#!{sys.executable}\n" + "import time\n" + "time.sleep(10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable + + def _clean_source_end_ffmpeg(tmp_path: Path) -> Path: executable = tmp_path / "clean-source-end-ffmpeg" executable.write_text( @@ -82,6 +159,57 @@ def _clean_source_end_ffmpeg(tmp_path: Path) -> Path: return executable +def _recoverable_source_end_ffmpeg(tmp_path: Path) -> tuple[Path, Path]: + executable = tmp_path / "recoverable-source-end-ffmpeg" + invocation_count = tmp_path / "recoverable-source-end-count" + executable.write_text( + f"#!{sys.executable}\n" + "import pathlib, sys, time\n" + "def box(kind, payload=b''):\n" + " return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n" + f"counter = pathlib.Path({str(invocation_count)!r})\n" + "count = int(counter.read_text()) + 1 if counter.exists() else 1\n" + "counter.write_text(str(count))\n" + "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n" + "sys.stdout.buffer.write(box(b'moof') + box(b'mdat', b'frame'))\n" + "sys.stdout.buffer.flush()\n" + "time.sleep(0.15 if count == 1 else 10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable, invocation_count + + +def _gated_media_progress_ffmpeg( + tmp_path: Path, + *, + release_second_media: Path, +) -> Path: + executable = tmp_path / "gated-media-progress-ffmpeg" + first_media_ready = tmp_path / "gated-media-progress-first-ready" + executable.write_text( + f"#!{sys.executable}\n" + "import pathlib, sys, time\n" + "def box(kind, payload=b''):\n" + " return (8 + len(payload)).to_bytes(4, 'big') + kind + payload\n" + "sys.stdout.buffer.write(box(b'ftyp', b'isom') + box(b'moov'))\n" + "sys.stdout.buffer.write(box(b'moof') + box(b'mdat', b'first'))\n" + "sys.stdout.buffer.flush()\n" + f"pathlib.Path({str(first_media_ready)!r}).write_text('ready')\n" + f"release = pathlib.Path({str(release_second_media)!r})\n" + "deadline = time.monotonic() + 5\n" + "while not release.exists() and time.monotonic() < deadline:\n" + " time.sleep(0.005)\n" + "if release.exists():\n" + " sys.stdout.buffer.write(box(b'moof') + box(b'mdat', b'second'))\n" + " sys.stdout.buffer.flush()\n" + "time.sleep(10)\n", + encoding="utf-8", + ) + executable.chmod(0o700) + return executable + + def _buffered_tail_ffmpeg(tmp_path: Path, sentinel: Path, *, fragments: int) -> Path: executable = tmp_path / "buffered-tail-ffmpeg" executable.write_text( @@ -180,12 +308,7 @@ def test_camera_derived_observer_runs_only_after_durable_archive_commit( observed: list[CommittedCameraSegment] = [] def observe(segment: CommittedCameraSegment) -> None: - archive = ( - session - / "media" - / segment.source_id - / f"epoch-{segment.generation}" - ) + archive = session / "media" / segment.source_id / f"epoch-{segment.generation}" committed = ( archive / "init.mp4" if segment.kind == "init" @@ -209,6 +332,49 @@ def test_camera_derived_observer_runs_only_after_durable_archive_commit( gateway.close() +def test_live_perception_rejects_late_camera_generation_after_replacement( + tmp_path: Path, +) -> None: + service = XgridsK1CompatibilityService(tmp_path) + try: + service.live_perception_ingress.begin_session("acquisition-new") + service._live_perception_camera_binding = ( # noqa: SLF001 + "acquisition-new", + "sensor.camera.right", + 12, + ) + stale = CommittedCameraSegment( + source_id="sensor.camera.right", + generation=11, + kind="media", + sequence=4, + host_epoch_ns=10, + host_monotonic_ns=20, + payload=b"stale", + ) + current = CommittedCameraSegment( + source_id="sensor.camera.right", + generation=12, + kind="media", + sequence=1, + host_epoch_ns=30, + host_monotonic_ns=40, + payload=b"current", + ) + + service._observe_committed_camera_segment(stale) # noqa: SLF001 + after_stale = service.live_perception_ingress.snapshot() + assert after_stale["queues"]["camera-frame"]["published"] == 0 + assert after_stale["queues"]["camera-frame"]["depth"] == 0 + + service._observe_committed_camera_segment(current) # noqa: SLF001 + after_current = service.live_perception_ingress.snapshot() + assert after_current["queues"]["camera-frame"]["published"] == 1 + assert after_current["queues"]["camera-frame"]["depth"] == 1 + finally: + service.close() + + def test_camera_ffmpeg_command_is_allowlisted_copy_remux() -> None: argv = _build_ffmpeg_argv( Path("/trusted/ffmpeg"), @@ -220,6 +386,7 @@ def test_camera_ffmpeg_command_is_allowlisted_copy_remux() -> None: assert argv[argv.index("-i") + 1] == ("rtsp://10.0.0.24:8554/live/chn_left_main") assert argv[argv.index("-c:v") + 1] == "copy" assert argv[argv.index("-allowed_media_types") + 1] == "video" + assert argv[argv.index("-timeout") + 1] == str(CAMERA_SOURCE_IO_TIMEOUT_MICROSECONDS) assert argv[argv.index("-flush_packets") + 1] == "1" assert "-c:v" in argv assert ";" not in " ".join(argv) @@ -246,6 +413,198 @@ def test_iso_bmff_reader_preserves_complete_boxes() -> None: _read_mp4_box(BytesIO(oversized)) +def test_current_epoch_is_not_media_ready_from_popen_or_epoch_allocation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_silent_ffmpeg(tmp_path))) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "sessions" / "camera-pending-media" + session.mkdir(parents=True) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + first = gateway.start_recording(session) + assert first["phase"] == "connecting" + assert first["recording"]["active_epoch"] == selected["generation"] + assert first["recording"]["media_ready"] is False + assert first["recording"]["current_epoch"] == { + "generation": selected["generation"], + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + "first_media_committed_age_ms": None, + "committed_media_segment_count": 0, + "last_media_segment_age_ms": None, + } + + restarted = gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=selected["generation"], + expected_recording_session=session.name, + expected_active_epoch=selected["generation"], + expected_recording_media_segment_count=0, + pre_detach_fence=lambda reserve: reserve(), + ) + replacement = selected["generation"] + 1 + assert restarted["generation"] == replacement + assert restarted["recording"]["active_epoch"] == replacement + assert restarted["recording"]["media_ready"] is False + assert restarted["recording"]["current_epoch"]["generation"] == replacement + assert restarted["recording"]["current_epoch"]["init_committed"] is False + assert restarted["recording"]["current_epoch"]["first_media_committed"] is False + finally: + gateway.close() + + +def test_camera_health_classifier_splits_no_init_and_no_media_stalls() -> None: + base = { + "phase": "connecting", + "recording": { + "active": True, + "producer_age_ms": 10_000, + "current_epoch": { + "generation": 4, + "init_committed": False, + "init_committed_age_ms": None, + "first_media_committed": False, + }, + }, + "error": None, + } + assert classify_camera_recording_health(base) == "camera-connecting-no-init-stalled" + current = base["recording"]["current_epoch"] + current.update({"init_committed": True, "init_committed_age_ms": 10_000}) + assert classify_camera_recording_health(base) == "camera-connecting-no-media-stalled" + current["first_media_committed"] = True + assert classify_camera_recording_health(base) is None + + +def test_backend_camera_watchdog_reports_stall_without_browser_polling( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_silent_ffmpeg(tmp_path))) + observed: list[tuple[str, int]] = [] + wake = threading.Event() + + def observe(reason: str, generation: int) -> None: + observed.append((reason, generation)) + wake.set() + + gateway = XgridsK1CameraGateway( + tmp_path, + XGRIDS_K1_PLUGIN_ID, + producer_stall_observer=observe, + producer_stall_milliseconds=30, + producer_watchdog_interval_seconds=0.01, + ) + session = tmp_path / "sessions" / "camera-watchdog" + session.mkdir(parents=True) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + assert wake.wait(timeout=1.0) is True + assert observed == [ + ("camera-connecting-no-init-stalled", selected["generation"]) + ] + time.sleep(0.05) + assert len(observed) == 1 + finally: + gateway.close() + + +def test_fragment_above_one_mib_is_durably_committed_and_media_ready( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + complete_fragment_bytes = 2 * 1024 * 1024 + monkeypatch.setenv( + "MISSIONCORE_FFMPEG_BINARY", + str( + _single_fragment_ffmpeg( + tmp_path, + complete_fragment_bytes=complete_fragment_bytes, + ) + ), + ) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "sessions" / "camera-large-valid-fragment" + session.mkdir(parents=True) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + _wait_until(lambda: gateway.snapshot()["recording"]["media_ready"] is True) + snapshot = gateway.snapshot() + current = snapshot["recording"]["current_epoch"] + assert current["generation"] == selected["generation"] + assert current["init_committed"] is True + assert current["first_media_committed"] is True + assert current["committed_media_segment_count"] == 1 + assert isinstance(current["init_committed_age_ms"], int) + assert isinstance(current["first_media_committed_age_ms"], int) + segment = ( + session + / "media" + / "sensor.camera.right" + / f"epoch-{selected['generation']}" + / "segments" + / "1.m4s" + ) + assert segment.stat().st_size == complete_fragment_bytes + finally: + gateway.close() + + +def test_fragment_above_eight_mib_fails_closed_with_size_only_diagnostic( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + observed_bytes = MAX_FMP4_SEGMENT_BYTES + 1 + monkeypatch.setenv( + "MISSIONCORE_FFMPEG_BINARY", + str( + _single_fragment_ffmpeg( + tmp_path, + complete_fragment_bytes=observed_bytes, + ) + ), + ) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "sessions" / "camera-oversized-fragment" + session.mkdir(parents=True) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + _wait_until( + lambda: ( + gateway.snapshot()["phase"] == "error" + and gateway.snapshot()["recording"]["completed_epochs"] == 1 + ) + ) + snapshot = gateway.snapshot() + assert snapshot["error"] == { + "code": "segment-too-large", + "message": "Camera adapter отклонил слишком большой video segment.", + "observed_size_bytes": observed_bytes, + "maximum_size_bytes": MAX_FMP4_SEGMENT_BYTES, + } + assert snapshot["recording"]["committed_media_segment_count"] == 0 + serialized = json.dumps(snapshot) + assert "PRIVATE-FRAME-BYTES" not in serialized + epoch = ( + session + / "media" + / "sensor.camera.right" + / f"epoch-{selected['generation']}" + ) + assert list((epoch / "segments").glob("*.m4s")) == [] + summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8")) + assert summary["failure_code"] == "segment-too-large" + finally: + gateway.close() + + def test_gateway_emits_init_and_complete_media_segments_without_transcoding( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -333,9 +692,7 @@ def test_acquisition_records_without_browser_and_source_switch_seals_epochs( assert summary["status"] == "complete" assert summary["segment_count"] == 1 assert summary["entry_count"] == 1 - assert summary["valid_bytes"] == len(init) + sum( - entry["length"] for entry in entries - ) + assert summary["valid_bytes"] == len(init) + sum(entry["length"] for entry in entries) assert summaries[0]["failure_code"] == "source-switch" assert summaries[1]["failure_code"] is None @@ -346,6 +703,466 @@ def test_acquisition_records_without_browser_and_source_switch_seals_epochs( gateway.close() +def test_acquisition_allows_independent_browser_consumers_without_lease_churn( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """React/window replacement cannot starve the currently visible camera.""" + + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "sessions" / "camera-browser-consumers" + session.mkdir(parents=True) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + first = gateway.open_delivery(selected["generation"]) + second = gateway.open_delivery(selected["generation"]) + + assert first.segments.get(timeout=3)[0] == "init" + assert second.segments.get(timeout=3)[0] == "init" + assert gateway.snapshot()["recording"]["preview_consumer_count"] == 2 + + gateway.release_delivery(first, client_closed=True) + assert gateway.snapshot()["recording"]["preview_consumer_count"] == 1 + assert second.process.poll() is None + assert second.failure_code is None + + gateway.release_delivery(second, client_closed=True) + state = gateway.snapshot() + assert state["recording"]["preview_consumer_count"] == 0 + assert state["recording"]["active"] is True + assert state["recording"]["producer_alive"] is True + finally: + gateway.close() + + +def test_release_delivery_wakes_blocked_reader_without_stopping_recording( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "released-browser-reader" + session.mkdir(parents=True) + released: list[tuple[str, bytes] | None] = [] + reader_started = threading.Event() + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + lease = gateway.open_delivery(selected["generation"]) + assert lease.segments.get(timeout=3)[0] == "init" + assert lease.segments.get(timeout=3)[0] == "media" + + def wait_for_next_segment() -> None: + reader_started.set() + released.append(lease.segments.get()) + + reader = threading.Thread(target=wait_for_next_segment) + reader.start() + assert reader_started.wait(timeout=1) + time.sleep(0.02) + assert released == [] + + gateway.release_delivery(lease, client_closed=True) + reader.join(timeout=1) + assert reader.is_alive() is False + assert released == [None] + recording = gateway.snapshot()["recording"] + assert recording["active"] is True + assert recording["producer_alive"] is True + assert recording["preview_consumer_count"] == 0 + finally: + gateway.close() + + +def test_preview_queue_enforces_exact_fragment_byte_and_age_bounds() -> None: + now = [10.0] + fragment_bounded = _CameraPreviewSegmentQueue(clock=lambda: now[0]) + for index in range(MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS): + assert fragment_bounded.offer(("media", bytes([index % 256]))) is True + assert fragment_bounded.offer(("media", b"overflow")) is False + + byte_bounded = _CameraPreviewSegmentQueue(clock=lambda: now[0]) + one_mebibyte = bytes(1024 * 1024) + for _ in range(MAX_CAMERA_PREVIEW_QUEUED_BYTES // len(one_mebibyte)): + assert byte_bounded.offer(("media", one_mebibyte)) is True + assert byte_bounded.queued_bytes == MAX_CAMERA_PREVIEW_QUEUED_BYTES + assert byte_bounded.offer(("media", b"x")) is False + + valid_maximum = _CameraPreviewSegmentQueue() + assert valid_maximum.offer(("init", bytes(512 * 1024))) is True + assert valid_maximum.offer(("media", bytes(MAX_FMP4_SEGMENT_BYTES))) is True + assert valid_maximum.queued_bytes == MAX_FMP4_SEGMENT_BYTES + 512 * 1024 + + age_bounded = _CameraPreviewSegmentQueue(clock=lambda: now[0]) + assert age_bounded.offer(("init", b"init")) is True + now[0] += MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS + assert age_bounded.offer(("media", b"at-boundary")) is True + now[0] += 0.001 + assert age_bounded.offer(("media", b"too-old")) is False + + +def test_camera_router_closes_lagging_preview_with_explicit_private_code( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "router-send-timeout" + session.mkdir() + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + _wait_until(lambda: gateway.snapshot()["recording"]["active_epoch"] is not None) + + class SlowWebSocket: + def __init__(self) -> None: + self.accepted = False + self.close_code: int | None = None + self.close_reason: str | None = None + + async def accept(self) -> None: + self.accepted = True + + async def send_bytes(self, _payload: bytes) -> None: + await asyncio.sleep(CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS * 4) + + async def close(self, *, code: int = 1000, reason: str = "") -> None: + self.close_code = code + self.close_reason = reason + + websocket = SlowWebSocket() + route = build_xgrids_k1_camera_router(gateway, XGRIDS_K1_PLUGIN_ID).routes[0] + monkeypatch.setattr(camera_module, "CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS", 0.01) + try: + asyncio.run(route.endpoint(websocket, selected["generation"])) + assert websocket.accepted is True + assert websocket.close_code == 4008 + assert websocket.close_reason == "preview-consumer-lagged" + recording = gateway.snapshot()["recording"] + assert recording["preview_consumer_count"] == 0 + assert recording["producer_alive"] is True + finally: + gateway.close() + + +def test_preview_consumer_cardinality_is_bounded_and_newest_keeps_streaming( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_burst_ffmpeg(tmp_path))) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "sessions" / "bounded-browser-consumers" + session.mkdir(parents=True) + leases = [] + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + _wait_until(lambda: gateway.snapshot()["recording"]["active_epoch"] is not None) + for _ in range(MAX_CAMERA_PREVIEW_CONSUMERS + 3): + leases.append(gateway.open_delivery(selected["generation"])) + + state = gateway.snapshot() + assert state["recording"]["preview_consumer_count"] == MAX_CAMERA_PREVIEW_CONSUMERS + assert all(lease.failure_code == "consumer-superseded" for lease in leases[:3]) + newest = leases[-1] + assert newest.segments.get(timeout=3)[0] == "init" + assert newest.segments.get(timeout=3)[0] == "media" + assert newest.failure_code is None + assert newest.process.poll() is None + recording = gateway.snapshot()["recording"] + assert recording["producer_alive"] is True + assert recording["committed_media_segment_count"] >= 1 + finally: + for lease in leases: + gateway.release_delivery(lease, client_closed=True) + gateway.close() + + +def test_acquisition_recovery_cas_restarts_stalled_ffmpeg_and_preserves_old_epoch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "sessions" / "camera-recovery" + session.mkdir(parents=True) + try: + gateway.start_recording(session) + selected = gateway.select("sensor.camera.right", "192.168.1.20") + first_generation = selected["generation"] + first_epoch = session / "media" / "sensor.camera.right" / f"epoch-{first_generation}" + _wait_until(lambda: gateway.snapshot()["phase"] == "streaming") + _wait_until(lambda: (first_epoch / "init.mp4").is_file()) + before = gateway.snapshot() + assert before["recording"]["producer_alive"] is True + assert before["recording"]["active_epoch"] == first_generation + + restarted = gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=before["generation"], + expected_recording_session=session.name, + expected_active_epoch=before["recording"]["active_epoch"], + expected_recording_media_segment_count=before["recording"][ + "committed_media_segment_count" + ], + pre_detach_fence=lambda reserve: reserve(), + ) + + assert restarted["generation"] == first_generation + 1 + assert restarted["recording"]["active_epoch"] == first_generation + 1 + assert restarted["recording"]["completed_epochs"] == 1 + first_summary = json.loads((first_epoch / "summary.json").read_text(encoding="utf-8")) + assert first_summary["status"] == "interrupted" + assert first_summary["failure_code"] == "active-stream-connection-recovery" + second_epoch = session / "media" / "sensor.camera.right" / f"epoch-{first_generation + 1}" + _wait_until(lambda: (second_epoch / "init.mp4").is_file()) + + with pytest.raises(ValueError, match="устарело|lineage"): + gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=before["generation"], + expected_recording_session=session.name, + expected_active_epoch=before["recording"]["active_epoch"], + expected_recording_media_segment_count=before["recording"][ + "committed_media_segment_count" + ], + pre_detach_fence=lambda reserve: reserve(), + ) + assert gateway.snapshot()["generation"] == first_generation + 1 + finally: + gateway.close() + + +def test_stop_first_pre_detach_denial_leaves_g7_and_archive_bytes_unchanged( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "sessions" / "camera-stop-first-reservation" + session.mkdir(parents=True) + try: + # Build a nontrivial lineage so the test proves the exact caller CAS, + # not an incidental initial-generation special case. + for source_id in ( + "sensor.camera.right", + "sensor.camera.left", + "sensor.camera.right", + "sensor.camera.left", + "sensor.camera.right", + "sensor.camera.left", + "sensor.camera.right", + ): + selected = gateway.select(source_id, "192.168.1.20") + assert selected["generation"] == 7 + gateway.start_recording(session) + _wait_until(lambda: gateway.snapshot()["recording"]["media_ready"] is True) + before = gateway.snapshot() + before_recording = before["recording"] + before_files = { + path.relative_to(session).as_posix(): path.read_bytes() + for path in session.rglob("*") + if path.is_file() + } + + with pytest.raises(ValueError, match="устарела|lineage"): + gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=7, + expected_recording_session=session.name, + expected_active_epoch=7, + expected_recording_media_segment_count=before_recording[ + "committed_media_segment_count" + ], + pre_detach_fence=lambda _reserve: False, + ) + + after = gateway.snapshot() + after_recording = after["recording"] + assert after["generation"] == before["generation"] == 7 + assert after["revision"] == before["revision"] + assert after["phase"] == before["phase"] == "streaming" + assert after["error"] == before["error"] is None + assert after_recording["active_epoch"] == before_recording["active_epoch"] == 7 + assert after_recording["producer_alive"] is True + assert after_recording["completed_epochs"] == before_recording["completed_epochs"] + assert after_recording["committed_media_segment_count"] == before_recording[ + "committed_media_segment_count" + ] + assert { + path.relative_to(session).as_posix(): path.read_bytes() + for path in session.rglob("*") + if path.is_file() + } == before_files + finally: + gateway.close() + + +def test_fresh_durable_media_supersedes_stale_watchdog_restart_cas( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + release_second_media = tmp_path / "release-second-camera-media" + monkeypatch.setenv( + "MISSIONCORE_FFMPEG_BINARY", + str( + _gated_media_progress_ffmpeg( + tmp_path, + release_second_media=release_second_media, + ) + ), + ) + session = tmp_path / "sessions" / "camera-progress-cas" + session.mkdir(parents=True) + watchdog_snapshotted = threading.Event() + allow_stale_cas = threading.Event() + cas_done = threading.Event() + cas_rejections: list[str] = [] + observer_errors: list[Exception] = [] + gateway: XgridsK1CameraGateway | None = None + + def backend_watchdog(reason: str, generation: int) -> None: + if reason != "camera-stream-stalled": + return + try: + assert gateway is not None + stale = gateway.snapshot() + stale_recording = stale["recording"] + assert stale_recording["committed_media_segment_count"] == 1 + watchdog_snapshotted.set() + assert allow_stale_cas.wait(timeout=3) + try: + gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=generation, + expected_recording_session=session.name, + expected_active_epoch=stale_recording["active_epoch"], + expected_recording_media_segment_count=stale_recording[ + "committed_media_segment_count" + ], + pre_detach_fence=lambda reserve: reserve(), + ) + except ValueError as exc: + cas_rejections.append(str(exc)) + else: + raise AssertionError("fresh durable media must supersede stale camera CAS") + except Exception as exc: + observer_errors.append(exc) + finally: + cas_done.set() + + gateway = XgridsK1CameraGateway( + tmp_path, + XGRIDS_K1_PLUGIN_ID, + producer_stall_observer=backend_watchdog, + producer_stall_milliseconds=40, + producer_watchdog_interval_seconds=0.01, + ) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + _wait_until( + lambda: gateway is not None + and gateway.snapshot()["recording"]["committed_media_segment_count"] == 1 + ) + assert watchdog_snapshotted.wait(timeout=3) + + # This archive append lands after the watchdog's health snapshot but + # before its lifecycle-CAS entry. Generation and active epoch remain + # unchanged; only the durable progress token can fence the stale restart. + release_second_media.write_text("release", encoding="utf-8") + _wait_until( + lambda: gateway is not None + and gateway.snapshot()["recording"]["committed_media_segment_count"] == 2 + ) + allow_stale_cas.set() + assert cas_done.wait(timeout=3) + + current = gateway.snapshot() + assert observer_errors == [] + assert len(cas_rejections) == 1 + assert "lineage" in cas_rejections[0] + assert current["generation"] == selected["generation"] + assert current["phase"] == "streaming" + assert current["error"] is None + assert current["recording"]["producer_alive"] is True + assert current["recording"]["active_epoch"] == selected["generation"] + assert current["recording"]["committed_media_segment_count"] == 2 + assert current["recording"]["completed_epochs"] == 0 + epoch = ( + session + / "media" + / "sensor.camera.right" + / f"epoch-{selected['generation']}" + ) + assert len(list((epoch / "segments").glob("*.m4s"))) == 2 + finally: + allow_stale_cas.set() + gateway.close() + + +def test_partial_same_session_activation_can_retry_exact_local_producer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + gateway = _gateway(tmp_path, monkeypatch) + session = tmp_path / "sessions" / "camera-partial-activation" + session.mkdir(parents=True) + real_popen = camera_module.subprocess.Popen + spawn_attempts = 0 + + def fail_first_popen(*args: object, **kwargs: object) -> object: + nonlocal spawn_attempts + spawn_attempts += 1 + if spawn_attempts == 1: + raise OSError("synthetic first Popen failure") + return real_popen(*args, **kwargs) + + monkeypatch.setattr(camera_module.subprocess, "Popen", fail_first_popen) + try: + gateway.select("sensor.camera.right", "192.168.1.20") + with pytest.raises(RuntimeError, match="camera adapter"): + gateway.start_recording(session) + + partial = gateway.snapshot() + assert partial["phase"] == "error" + assert partial["active_source_id"] == "sensor.camera.right" + assert partial["recording"]["active"] is True + assert partial["recording"]["session"] == session.name + assert partial["recording"]["active_epoch"] is None + failed_epoch = session / "media" / "sensor.camera.right" / f"epoch-{partial['generation']}" + failed_summary = json.loads((failed_epoch / "summary.json").read_text(encoding="utf-8")) + assert failed_summary["status"] == "failed" + assert failed_summary["failure_code"] == "ffmpeg-start-failed" + + retried = gateway.retry_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=partial["generation"], + expected_recording_session=session.name, + pre_retry_fence=lambda reserve: reserve(), + ) + + assert spawn_attempts == 2 + assert retried["phase"] in {"connecting", "streaming"} + assert retried["generation"] == partial["generation"] + 1 + assert retried["recording"]["active_epoch"] == partial["generation"] + 1 + assert retried["recording"]["producer_alive"] is True + assert json.loads((failed_epoch / "summary.json").read_text(encoding="utf-8")) == ( + failed_summary + ) + with pytest.raises(ValueError, match="lineage"): + gateway.retry_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=partial["generation"], + expected_recording_session=session.name, + pre_retry_fence=lambda reserve: reserve(), + ) + finally: + gateway.close() + + def test_expected_camera_source_end_during_device_stop_seals_complete_epoch( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -373,12 +1190,7 @@ def test_expected_camera_source_end_during_device_stop_seals_complete_epoch( assert state["recording"]["source_end_expected"] is False gateway.stop_recording(status="complete") - epoch = ( - session - / "media" - / "sensor.camera.left" - / f"epoch-{selected['generation']}" - ) + epoch = session / "media" / "sensor.camera.left" / f"epoch-{selected['generation']}" summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8")) assert summary["status"] == "complete" assert summary["failure_code"] is None @@ -402,13 +1214,13 @@ def test_unexpected_camera_source_end_remains_a_recording_failure( selected = gateway.select("sensor.camera.left", "192.168.1.20") gateway.start_recording(session) - _wait_until(lambda: gateway.snapshot()["phase"] == "error") - epoch = ( - session - / "media" - / "sensor.camera.left" - / f"epoch-{selected['generation']}" + _wait_until( + lambda: gateway.snapshot()["recording"]["completed_epochs"] == 1, ) + state = gateway.snapshot() + assert state["phase"] == "error" + assert state["error"]["code"] == "camera-source-ended" + epoch = session / "media" / "sensor.camera.left" / f"epoch-{selected['generation']}" summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8")) assert summary["status"] == "interrupted" assert summary["failure_code"] == "camera-source-ended" @@ -416,6 +1228,224 @@ def test_unexpected_camera_source_end_remains_a_recording_failure( gateway.close() +def test_browser_reconnect_cannot_respawn_sealed_epoch_before_backend_cas( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + executable, invocation_count = _recoverable_source_end_ffmpeg(tmp_path) + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(executable)) + session = tmp_path / "sessions" / "camera-browser-backend-recovery-race" + session.mkdir(parents=True) + watchdog_entered = threading.Event() + allow_backend_cas = threading.Event() + backend_cas_done = threading.Event() + observer_errors: list[Exception] = [] + recovered_snapshots: list[dict[str, object]] = [] + gateway: XgridsK1CameraGateway | None = None + + def backend_watchdog(reason: str, generation: int) -> None: + try: + assert reason == "camera-source-ended" + watchdog_entered.set() + assert allow_backend_cas.wait(timeout=3) + assert gateway is not None + recovered_snapshots.append( + gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=generation, + expected_recording_session=session.name, + expected_active_epoch=None, + expected_recording_media_segment_count=1, + pre_detach_fence=lambda reserve: reserve(), + ) + ) + except Exception as exc: + observer_errors.append(exc) + finally: + backend_cas_done.set() + + gateway = XgridsK1CameraGateway( + tmp_path, + XGRIDS_K1_PLUGIN_ID, + producer_stall_observer=backend_watchdog, + ) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + first_generation = selected["generation"] + gateway.start_recording(session) + assert watchdog_entered.wait(timeout=3) + + sealed = gateway.snapshot() + first_epoch = ( + session + / "media" + / "sensor.camera.right" + / f"epoch-{first_generation}" + ) + first_summary = json.loads( + (first_epoch / "summary.json").read_text(encoding="utf-8") + ) + assert invocation_count.read_text(encoding="utf-8") == "1" + assert sealed["generation"] == first_generation + assert sealed["phase"] == "error" + assert sealed["error"]["code"] == "camera-source-ended" + assert sealed["recording"]["producer_alive"] is False + assert sealed["recording"]["active_epoch"] is None + assert sealed["recording"]["completed_epochs"] == 1 + + # A browser reconnect reaches the sealed generation before the backend + # observer is allowed to take its CAS. It must neither spawn FFmpeg nor + # reopen the finalized epoch or misclassify the collision as storage. + with pytest.raises(RuntimeError, match="backend recovery"): + gateway.open_delivery(first_generation) + + after_browser = gateway.snapshot() + assert invocation_count.read_text(encoding="utf-8") == "1" + assert after_browser["generation"] == first_generation + assert after_browser["recording"]["producer_alive"] is False + assert after_browser["recording"]["active_epoch"] is None + assert after_browser["recording"]["completed_epochs"] == 1 + assert after_browser["error"]["code"] == "camera-source-ended" + assert list((first_epoch.parent).glob("epoch-*")) == [first_epoch] + assert json.loads( + (first_epoch / "summary.json").read_text(encoding="utf-8") + ) == first_summary + + allow_backend_cas.set() + assert backend_cas_done.wait(timeout=3) + assert observer_errors == [] + _wait_until( + lambda: ( + gateway is not None + and gateway.snapshot()["generation"] == first_generation + 1 + and gateway.snapshot()["recording"]["media_ready"] is True + ) + ) + + recovered = gateway.snapshot() + assert len(recovered_snapshots) == 1 + assert invocation_count.read_text(encoding="utf-8") == "2" + assert recovered["generation"] == first_generation + 1 + assert recovered["phase"] == "streaming" + assert recovered["error"] is None + assert recovered["recording"]["active_epoch"] == first_generation + 1 + assert recovered["recording"]["media_ready"] is True + assert recovered["recording"]["current_epoch"]["generation"] == ( + first_generation + 1 + ) + assert json.loads( + (first_epoch / "summary.json").read_text(encoding="utf-8") + ) == first_summary + finally: + allow_backend_cas.set() + gateway.close() + + +def test_source_end_none_epoch_cas_waits_for_canonical_archive_seal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + executable, _invocation_count = _recoverable_source_end_ffmpeg(tmp_path) + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(executable)) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "sessions" / "camera-source-end-seal-fence" + session.mkdir(parents=True) + finalize_entered = threading.Event() + release_finalize = threading.Event() + real_finalize = gateway._finalize_archive # noqa: SLF001 + + def blocked_finalize(*args: object, **kwargs: object) -> None: + finalize_entered.set() + assert release_finalize.wait(timeout=3) + real_finalize(*args, **kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(gateway, "_finalize_archive", blocked_finalize) + try: + selected = gateway.select("sensor.camera.right", "192.168.1.20") + gateway.start_recording(session) + assert finalize_entered.wait(timeout=3) + unsealed = gateway.snapshot() + assert unsealed["phase"] == "error" + assert unsealed["error"]["code"] == "camera-source-ended" + assert unsealed["recording"]["active_epoch"] is None + assert unsealed["recording"]["completed_epochs"] == 0 + + with pytest.raises(ValueError, match="устарела|lineage"): + gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=selected["generation"], + expected_recording_session=session.name, + expected_active_epoch=None, + expected_recording_media_segment_count=1, + pre_detach_fence=lambda reserve: reserve(), + ) + denied = gateway.snapshot() + assert denied["generation"] == selected["generation"] + assert denied["recording"]["completed_epochs"] == 0 + + release_finalize.set() + _wait_until(lambda: gateway.snapshot()["recording"]["completed_epochs"] == 1) + sealed = gateway.snapshot() + summary = sealed["recording"]["last_summary"] + assert summary["schema_version"] == "missioncore.camera-recording/v1" + assert summary["failure_code"] == "camera-source-ended" + restarted = gateway.restart_recording_producer( + "sensor.camera.right", + "192.168.1.20", + expected_generation=selected["generation"], + expected_recording_session=session.name, + expected_active_epoch=None, + expected_recording_media_segment_count=1, + pre_detach_fence=lambda reserve: reserve(), + ) + assert restarted["generation"] == selected["generation"] + 1 + finally: + release_finalize.set() + gateway.close() + + +def test_acquisition_candidate_binds_epoch_before_any_reader_thread_starts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_silent_ffmpeg(tmp_path))) + session = tmp_path / "sessions" / "camera-bind-before-reader" + session.mkdir(parents=True) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + bound_generation: list[int] = [] + reader_start_generation: list[int] = [] + + def bind(committed: dict[str, object]) -> None: + recording = committed["recording"] + assert isinstance(recording, dict) + generation = recording["active_epoch"] + assert isinstance(generation, int) + bound_generation.append(generation) + + def start_threads(producer: object) -> None: + generation = producer.generation # type: ignore[attr-defined] + assert bound_generation == [generation] + reader_start_generation.append(generation) + + monkeypatch.setattr(gateway, "_start_producer_threads", start_threads) + try: + activated = gateway.activate_recording_producer( + "sensor.camera.right", + "192.168.1.20", + session, + commit_fence=lambda commit: commit(), + committed_before_start=bind, + ) + + assert bound_generation == [activated["recording"]["active_epoch"]] + assert reader_start_generation == bound_generation + assert activated["recording"]["media_ready"] is False + finally: + gateway.close() + + def test_camera_storage_open_failure_is_loud_and_never_starts_ffmpeg( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -487,8 +1517,7 @@ def test_camera_storage_append_failure_fails_producer_and_epoch_loudly( _wait_until( lambda: ( summary_path.is_file() - and json.loads(summary_path.read_text(encoding="utf-8"))["status"] - == "failed" + and json.loads(summary_path.read_text(encoding="utf-8"))["status"] == "failed" ) ) summary = json.loads(summary_path.read_text(encoding="utf-8")) @@ -499,11 +1528,67 @@ def test_camera_storage_append_failure_fails_producer_and_epoch_loudly( gateway.close() -def test_slow_browser_is_dropped_without_stopping_archive_producer( +def test_routine_browser_stall_is_buffered_without_dropping_live_delivery( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setenv("MISSIONCORE_FFMPEG_BINARY", str(_burst_ffmpeg(tmp_path))) + fragment_count = 40 + monkeypatch.setenv( + "MISSIONCORE_FFMPEG_BINARY", + str( + _preview_buffer_ffmpeg( + tmp_path, + fragments=fragment_count, + payload_bytes=80 * 1024, + ) + ), + ) + gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) + session = tmp_path / "routine-preview-stall" + session.mkdir() + try: + selected = gateway.select("sensor.camera.left", "192.168.1.20") + gateway.start_recording(session) + lease = gateway.open_delivery(selected["generation"]) + + # A busy Rerun/browser main thread can defer this WebSocket task for + # several seconds. That is ordinary scheduling jitter, not a dead + # consumer, and must not terminate the visible camera transport. + _wait_until( + lambda: ( + gateway.snapshot()["recording"]["committed_media_segment_count"] == fragment_count + ), + ) + assert lease.failure_code is None + assert gateway.snapshot()["recording"]["preview_consumer_count"] == 1 + assert lease.segments.queued_segments == fragment_count + 1 + assert lease.segments.queued_bytes < MAX_CAMERA_PREVIEW_QUEUED_BYTES + + assert lease.segments.get(timeout=1)[0] == "init" + for _ in range(fragment_count): + assert lease.segments.get(timeout=1)[0] == "media" + assert lease.segments.queued_segments == 0 + assert lease.segments.queued_bytes == 0 + finally: + gateway.close() + + +def test_permanently_slow_browser_is_dropped_without_stopping_archive_producer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fragment_count = MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS + 20 + monkeypatch.setenv( + "MISSIONCORE_FFMPEG_BINARY", + str( + _preview_buffer_ffmpeg( + tmp_path, + fragments=fragment_count, + payload_bytes=160 * 1024, + interval_seconds=0.02, + ) + ), + ) gateway = XgridsK1CameraGateway(tmp_path, XGRIDS_K1_PLUGIN_ID) session = tmp_path / "session" session.mkdir() @@ -511,6 +1596,19 @@ def test_slow_browser_is_dropped_without_stopping_archive_producer( selected = gateway.select("sensor.camera.left", "192.168.1.20") gateway.start_recording(session) lease = gateway.open_delivery(selected["generation"]) + fast = gateway.open_delivery(selected["generation"]) + producer_pid = fast.process.pid + active_epoch = gateway.snapshot()["recording"]["active_epoch"] + fast_segments: list[str] = [] + + def drain_fast_reader() -> None: + for _ in range(fragment_count + 1): + segment = fast.segments.get(timeout=3) + assert segment is not None + fast_segments.append(segment[0]) + + fast_reader = threading.Thread(target=drain_fast_reader) + fast_reader.start() # Deliberately never drain the bounded queue. _wait_until(lambda: lease.failure_code == "consumer-too-slow") @@ -518,23 +1616,31 @@ def test_slow_browser_is_dropped_without_stopping_archive_producer( assert lease.process.poll() is None assert gateway.snapshot()["recording"]["active"] is True + replacement = gateway.open_delivery(selected["generation"]) + assert replacement.segments.get(timeout=1)[0] == "init" + assert replacement.segments.get(timeout=3)[0] == "media" + assert replacement.process.pid == producer_pid + assert gateway.snapshot()["generation"] == selected["generation"] + assert gateway.snapshot()["recording"]["active_epoch"] == active_epoch + gateway.release_delivery(lease, client_closed=False) - epoch = ( - session - / "media" - / "sensor.camera.left" - / f"epoch-{selected['generation']}" - ) + epoch = session / "media" / "sensor.camera.left" / f"epoch-{selected['generation']}" _wait_until( - lambda: len(list((epoch / "segments").glob("*.m4s"))) == 10, + lambda: len(list((epoch / "segments").glob("*.m4s"))) == fragment_count, ) + fast_reader.join(timeout=3) + assert fast_reader.is_alive() is False + assert fast_segments == ["init", *("media" for _ in range(fragment_count))] + assert fast.failure_code is None + gateway.release_delivery(fast, client_closed=True) + gateway.release_delivery(replacement, client_closed=True) gateway.stop_recording(status="complete") summary_path = epoch / "summary.json" summary = json.loads(summary_path.read_text(encoding="utf-8")) assert summary["status"] == "complete" - assert summary["segment_count"] == 10 - assert summary["media_segment_count"] == 10 + assert summary["segment_count"] == fragment_count + assert summary["media_segment_count"] == fragment_count finally: gateway.close() @@ -559,12 +1665,7 @@ def test_clean_stop_drains_ffmpeg_stdout_before_sealing_archive( gateway.stop_recording(status="complete") - epoch = ( - session - / "media" - / "sensor.camera.left" - / f"epoch-{selected['generation']}" - ) + epoch = session / "media" / "sensor.camera.left" / f"epoch-{selected['generation']}" summary = json.loads((epoch / "summary.json").read_text(encoding="utf-8")) assert summary["status"] == "complete" assert summary["segment_count"] == fragment_count @@ -621,7 +1722,9 @@ def test_service_publishes_two_dynamic_camera_rows_and_stale_stop_is_safe( service = XgridsK1CompatibilityService(tmp_path) try: with service._lock: + service._selected_device_id = "ble-transport-test" service._k1_ip = "192.168.1.20" + service._connection_mode = "bridge" service._device_id = "device-k1-test" service._device_session_id = "device-session-test" service._device_session_opened_at = "2026-07-16T20:00:00Z" @@ -632,6 +1735,55 @@ def test_service_publishes_two_dynamic_camera_rows_and_stale_stop_is_safe( "basis": "selected-profile-live-device-info-required", "observed_at": "2026-07-16T20:00:00Z", } + supervisor = service._connection_supervisor # noqa: SLF001 + target = EndpointTarget("192.168.1.20") + supervisor.set_intent( + intent_id="camera-test-intent", + requested_mode="bridge", + ) + assert supervisor.observe_device_network_applied( + intent_id="camera-test-intent", + transport_ref="ble-transport-test", + connection_mode="bridge", + target=target, + source="ble-read-only-status", + ) + host_epoch = supervisor.observe_host_path( + HostPathProbeResult( + available=True, + fingerprint="camera-test-route", + interface="en0", + source_ipv4="192.168.1.10", + route_class="direct", + ) + ) + assert supervisor.observe_endpoint( + target=target, + intent_id="camera-test-intent", + host_path_epoch=host_epoch, + reachable=True, + ) + assert supervisor.observe_control_evidence( + VerifiedControlEvidence( + intent_id="camera-test-intent", + transport_ref="ble-transport-test", + host_path_epoch=host_epoch, + target=target, + connection_mode="bridge", + logical_device_id="device-k1-test", + compatibility_profile_id=XGRIDS_K1_COMPATIBILITY_PROFILE_ID, + control_session_id="camera-test-control-session", + ) + ) + # This bounded gateway test supplies an explicit supervisor proof + # instead of opening the real MQTT application dialogue. Prevent the + # ordinary state reducer from correctly expiring that synthetic proof + # when it observes the deliberately idle test control session. + monkeypatch.setattr( + service, + "_reconcile_connection_supervisor", + lambda *_args, **_kwargs: None, + ) state = service.select_camera_preview( CameraPreviewSelectRequest( diff --git a/tests/test_xgrids_connection_scenario_reset.py b/tests/test_xgrids_connection_scenario_reset.py new file mode 100644 index 0000000..0fd387d --- /dev/null +++ b/tests/test_xgrids_connection_scenario_reset.py @@ -0,0 +1,2886 @@ +from __future__ import annotations + +import asyncio +import hashlib +import os +import threading +import time +from collections.abc import Callable, Mapping +from concurrent.futures import Future +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from pydantic import SecretStr + +from k1link.device_plugins.xgrids_k1 import facade as facade_module +from k1link.device_plugins.xgrids_k1 import ( + network_mutation_ledger as network_ledger_module, +) +from k1link.device_plugins.xgrids_k1 import physical_command_ledger as ledger_module +from k1link.device_plugins.xgrids_k1.facade import ( + BleScanRequest, + CompatibilityAttestationRequest, + ConnectRequest, + DesiredConnectionModeRequest, + NetworkProvisioningConflict, + XgridsK1CompatibilityService, +) +from k1link.device_plugins.xgrids_k1.network_mutation_ledger import ( + NetworkStatusEvidence, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_ARCHIVE_DIRECTORY, + PhysicalCommandArchiveSegment, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandRecord, + PhysicalCommandStatusEvidence, + PhysicalCommandVerifiedBinding, +) +from k1link.web.device_lifecycle import AcquisitionRecord + +RESET_ATTESTATION = CompatibilityAttestationRequest( + firmware_version="3.0.2", + topology="direct-lan", + verification="live-device-info", +) +RESET_TEST_CREDENTIAL = "r" * 24 + + +def _service(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> XgridsK1CompatibilityService: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "data")) + monkeypatch.setenv("MISSIONCORE_EVIDENCE_DIR", str(tmp_path / "evidence")) + monkeypatch.setenv( + "MISSIONCORE_LEGACY_SESSIONS_DIR", + str(tmp_path / "legacy"), + ) + return XgridsK1CompatibilityService(tmp_path) + + +def _reset( + *, + mode: str, + revision: int, + reset_id: str, +) -> DesiredConnectionModeRequest: + return DesiredConnectionModeRequest( + connection_mode=mode, # type: ignore[arg-type] + expected_revision=revision, + reset_scenario=True, + reset_id=reset_id, + ) + + +def _seed_resettable_physical_record( + service: XgridsK1CompatibilityService, + *, + closed_cycles: int = 0, + prepared_only: bool = False, +) -> PhysicalCommandRecord: + ledger = service._physical_command_ledger # noqa: SLF001 + connection = PhysicalCommandConnectionBinding( + intent_id="reset-preflight-intent-0001", + transport_ref="RESET-PREFLIGHT-K1-UUID", + connection_mode="bridge", + target_ipv4="192.168.68.51", + target_port=1883, + host_path_epoch=7, + control_session_id="reset-preflight-control-session-0001", + producer_generation=11, + ) + operation_id = "reset-preflight-physical-start-0001" + ledger.prepare( + operation_id=operation_id, + parent_operation_id=None, + acquisition_id="reset-preflight-acquisition-0001", + action="start", + identity=PhysicalCommandIdentity( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + ), + connection=connection, + compatibility_profile_id="xgrids-k1-reset-preflight-v1", + payload_sha256="c" * 64, + baseline_status=PhysicalCommandStatusEvidence( + source="live-control-session", + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, + control_session_id=connection.control_session_id, + host_path_epoch=connection.host_path_epoch, + producer_generation=connection.producer_generation, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256="d" * 64, + mqtt_retained=False, + observed_at_utc="2026-08-12T09:00:00.000Z", + ), + ) + if prepared_only: + record = ledger.snapshot().record + assert record is not None + return record + ledger.mark_dispatching(operation_id) + record = ledger.mark_observing(operation_id, publish_call_returned=False) + for cycle in range(closed_cycles): + retired = ledger.retire_unavailable_target( + retirement_id=f"reset-preflight-retirement-{cycle + 1}", + expected_operation_id=operation_id, + expected_revision=record.revision, + expected_transport_ref=connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + record = ledger.reopen_retired_reconciliation( + reopening_id=f"reset-preflight-reopening-{cycle + 1}", + expected_revision=retired.revision, + expected_retirement_id=f"reset-preflight-retirement-{cycle + 1}", + expected_transport_ref=connection.transport_ref, + expected_discovery_generation=100 + cycle, + reason="device-returned-for-explicit-reconciliation", + ) + return record + + +def _install_exact_reset_scan( + service: XgridsK1CompatibilityService, + monkeypatch: pytest.MonkeyPatch, + *, + device_ids: tuple[str, ...] = ("RESET-PREFLIGHT-K1-UUID",), +) -> list[facade_module.CapturedDiscoveredDevice]: + """Install a public Scan seam with one exact object per generation.""" + + current_captures: list[facade_module.CapturedDiscoveredDevice] = [] + + async def exact_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + current_captures[:] = [ + facade_module.CapturedDiscoveredDevice( + device=SimpleNamespace(address=device_id), # type: ignore[arg-type] + macos_uuid=device_id, + owner_epoch=19, + scan_generation=service._ble_discovery_generation, # noqa: SLF001 + ) + for device_id in device_ids + ] + return { + "devices": [ + { + "macos_uuid": device_id, + "name": "XGR-K1", + "local_name": "XGR-K1", + "rssi": -41, + "k1_name_candidate": True, + } + for device_id in device_ids + ] + } + + def capture(device_id: str) -> facade_module.CapturedDiscoveredDevice | None: + return next( + (item for item in current_captures if item.macos_uuid == device_id), + None, + ) + + monkeypatch.setattr(facade_module, "scan", exact_scan) + monkeypatch.setattr(facade_module, "_capture_network_intent_device", capture) + monkeypatch.setattr( + facade_module, + "pin_connected_device_handle", + lambda *_args, **_kwargs: None, + ) + return current_captures + + +def _reset_connect_request( + *, + device_id: str, + discovery_generation: int, + mode_revision: int = 1, + operation_id: str = "op-00000000-0000-4000-8000-000000000501", +) -> ConnectRequest: + return ConnectRequest( + device_id=device_id, + ssid="reset-lab-network", + password=SecretStr(RESET_TEST_CREDENTIAL), + connection_mode="bridge", + compatibility_attestation=RESET_ATTESTATION, + operation_id=operation_id, + idempotency_key=f"reset-retired-reprovision-{operation_id}", + expected_mode_revision=mode_revision, + expected_discovery_generation=discovery_generation, + ) + + +def _network_operation_documents( + service: XgridsK1CompatibilityService, +) -> list[dict[str, Any]]: + return [ + item + for item in service._operations.snapshot(limit=128) # noqa: SLF001 + if item.get("action") == facade_module.ACTION_NETWORK_PROVISION + ] + + +def _forbid_reset_teardown( + service: XgridsK1CompatibilityService, + monkeypatch: pytest.MonkeyPatch, +) -> list[str]: + calls: list[str] = [] + + def forbidden(*_args: object, **_kwargs: object) -> None: + calls.append("called") + raise AssertionError("scenario reset crossed preflight into teardown") + + for name in ( + "_cleanup_terminal_acquisition_sources_for_scenario_reset", + "_force_finish_active_acquisition_locally", + "_cleanup_residual_sources_for_connection_scenario_reset", + "_retire_network_scenario_for_operator_reset", + ): + monkeypatch.setattr(service, name, forbidden) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "retire_connection_scenario", + forbidden, + ) + return calls + + +def test_cold_scenario_reset_is_local_only_and_exact_retry_converges( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-cold-00000001", + ) + + first = service.select_connection_mode(request) + retried = service.select_connection_mode(request) + + assert first["desired_connection_mode"] == "quick-connect" + assert first["desired_connection_mode_revision"] == 1 + assert retried["desired_connection_mode_revision"] == 1 + assert retried["connection_scenario_reset"] == first["connection_scenario_reset"] + assert first["connection_scenario_reset"] == { + "reset_id": "op-reset-cold-00000001", + "request_revision": 0, + "revision": 1, + "desired_mode": "quick-connect", + "active": True, + "settled_by_discovery_generation": None, + "local_session_closed": True, + "previous_device_may_continue_scanning": False, + "physical_disposition": "empty", + "network_disposition": None, + "device_command_performed": False, + "network_write_performed": False, + "automatic_scan": False, + "operation_sequence": 0, + } + assert first["devices"] == [] + assert first["selected_device_id"] is None + assert first["connection_attempt"] is None + assert first["connection_scenario_reset_pending"] is None + assert service.state()["connection_scenario_reset_pending"] is None + assert first["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + +def test_reset_attempt_cutoff_uses_operation_identity_not_local_stage_sequence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + old_operation, created = service._operations.begin( # noqa: SLF001 + facade_module.ACTION_NETWORK_PROVISION, + operation_id="op-00000000-0000-4000-8000-000000000101", + context={"connection_mode": "bridge"}, + ) + assert created is True + for stage in range(4): + service._operations.transition( # noqa: SLF001 + old_operation.operation_id, + "running", + stage_code=f"old-stage-{stage}", + message_code="network.provision.running", + ) + service._operations.transition( # noqa: SLF001 + old_operation.operation_id, + "failed", + stage_code="old-failed", + message_code="network.provision.failed", + error={ + "code": "old-network-failure", + "side_effect_status": "none", + "retryable": True, + "safe_to_retry": True, + }, + ) + assert old_operation.sequence == 6 + + reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-operation-identity-cutoff-01", + ) + ) + assert reset["connection_scenario_reset"]["operation_sequence"] == 6 + assert reset["connection_attempt"] is None + + new_operation, created = service._operations.begin( # noqa: SLF001 + facade_module.ACTION_NETWORK_PROVISION, + operation_id="op-00000000-0000-4000-8000-000000000102", + context={"connection_mode": "bridge"}, + ) + assert created is True + service._operations.transition( # noqa: SLF001 + new_operation.operation_id, + "running", + stage_code="new-running", + message_code="network.provision.running", + ) + service._operations.transition( # noqa: SLF001 + new_operation.operation_id, + "failed", + stage_code="new-failed", + message_code="network.provision.failed", + error={ + "code": "new-network-failure", + "side_effect_status": "unknown", + "retryable": False, + "safe_to_retry": False, + }, + ) + assert new_operation.sequence == 3 + + projected = service.state()["connection_attempt"] + assert projected is not None + assert projected["attempt_id"] == new_operation.operation_id + assert projected["phase"] == "network_outcome_unknown" + assert projected["status"] == "failed" + + +def test_scenario_reset_preserves_physical_audit_and_unrelated_plugin_state( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + unrelated_state = tmp_path / "data" / "unrelated-plugin" / "state.json" + unrelated_state.parent.mkdir(parents=True) + unrelated_bytes = b'{"connection":"must-survive-k1-reset","revision":17}\n' + unrelated_state.write_bytes(unrelated_bytes) + unrelated_inode = unrelated_state.stat().st_ino + + before = _seed_resettable_physical_record(service) + reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-preserve-durable-audit-01", + ) + ) + + after = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert after is not None + assert after.revision == before.revision + 1 + assert after.stage == "resolved" + assert after.resolution == "operator-retired-outcome-unknown" + assert after.operation_id == before.operation_id + assert after.parent_operation_id == before.parent_operation_id + assert after.acquisition_id == before.acquisition_id + assert after.action == before.action + assert after.identity == before.identity + assert after.connection == before.connection + assert after.compatibility_profile_id == before.compatibility_profile_id + assert after.payload_sha256 == before.payload_sha256 + assert after.baseline_status == before.baseline_status + assert after.created_at_utc == before.created_at_utc + assert len(after.operator_retirements) == len(before.operator_retirements) + 1 + retirement = after.operator_retirements[-1] + assert retirement.reason == "connection-scenario-reset-by-operator" + assert retirement.retired_transport_ref == before.connection.transport_ref + assert retirement.original_attempt.operation_id == before.operation_id + assert retirement.original_attempt.acquisition_id == before.acquisition_id + assert retirement.original_attempt.payload_sha256 == before.payload_sha256 + + assert reset["connection_scenario_reset"]["physical_disposition"] == ( + "operator-retired-outcome-unknown" + ) + assert reset["connection_scenario_reset"]["previous_device_may_continue_scanning"] is True + assert reset["connection_scenario_reset"]["device_command_performed"] is False + assert reset["connection_scenario_reset"]["network_write_performed"] is False + assert reset["connection_scenario_reset"]["automatic_scan"] is False + assert reset["phase"] == "idle" + assert reset["devices"] == [] + assert reset["selected_device_id"] is None + assert reset["device_ref"] is None + assert reset["device_session"] is None + assert reset["connection_attempt"] is None + assert reset["connection_reconfiguration"]["status"] == "idle" + assert reset["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + assert unrelated_state.read_bytes() == unrelated_bytes + assert unrelated_state.stat().st_ino == unrelated_inode + + +def test_existing_reset_retires_failed_reconnect_reopen_for_manual_new_flow( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + first_reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-before-failed-reconnect-01", + ) + ) + first_retired = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert first_retired is not None + assert first_retired.resolution == "operator-retired-outcome-unknown" + first_retirement = first_retired.operator_retirements[-1] + + reopened = service._physical_command_coordinator.reopen_retired_reconciliation( # noqa: SLF001 + reopening_id="op-reopen-that-failed-read-only-verify-01", + expected_revision=first_retired.revision, + expected_retirement_id=first_retirement.retirement_id, + expected_transport_ref=first_retirement.retired_transport_ref, + expected_discovery_generation=17, + reason="device-returned-for-explicit-reconciliation", + ) + assert reopened["stage"] == "observing" + assert reopened["resolution"] is None + + # A failed read-only Verify appends no physical classification. The + # existing explicit reset must still be able to retire that reopened local + # reconciliation and return to a clean, manually scanned scenario. + manual_new = service.select_connection_mode( + _reset( + mode="bridge", + revision=first_reset["desired_connection_mode_revision"], + reset_id="op-reset-after-failed-reconnect-01", + ) + ) + + final_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert final_record is not None + assert final_record.stage == "resolved" + assert final_record.resolution == "operator-retired-outcome-unknown" + assert len(final_record.operator_retirements) == 2 + assert final_record.operator_retirements[0] == first_retirement + assert final_record.operator_retirements[-1].reason == ( + "connection-scenario-reset-by-operator" + ) + assert manual_new["desired_connection_mode_revision"] == 2 + assert manual_new["connection_scenario_reset"]["physical_disposition"] == ( + "operator-retired-outcome-unknown" + ) + assert manual_new["connection_scenario_reset"][ + "previous_device_may_continue_scanning" + ] is True + assert manual_new["devices"] == [] + assert manual_new["selected_device_id"] is None + assert manual_new["connection_attempt"] is None + assert manual_new["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + + +@pytest.mark.parametrize( + ("prepared_only", "bound"), + [ + (False, "segment-count"), + (False, "segment-bytes"), + (False, "archive-bytes"), + (False, "noncompactable-live-frontier"), + (True, "prepared-noncompactable-live-frontier"), + ], +) +def test_physical_persistence_preflight_blocks_before_any_reset_teardown( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + prepared_only: bool, + bound: str, +) -> None: + service = _service(monkeypatch, tmp_path) + record = _seed_resettable_physical_record( + service, + closed_cycles=0, + prepared_only=prepared_only, + ) + ledger = service._physical_command_ledger # noqa: SLF001 + main_bytes = ledger.path.read_bytes() + main_inode = ledger.path.stat().st_ino + monkeypatch.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(main_bytes) + 1, + ) + if bound == "segment-count": + monkeypatch.setattr(ledger_module, "PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS", 0) + elif bound == "segment-bytes": + monkeypatch.setattr( + ledger_module, + "PHYSICAL_COMMAND_ARCHIVE_SEGMENT_MAX_BYTES", + len(main_bytes) - 1, + ) + elif bound == "archive-bytes": + monkeypatch.setattr(ledger_module, "PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES", 1) + teardown_calls = _forbid_reset_teardown(service, monkeypatch) + state_before = service.state() + + with pytest.raises(NetworkProvisioningConflict) as conflict: + service.select_connection_mode( + _reset( + mode="quick-connect", + revision=0, + reset_id=f"op-reset-preflight-{bound}", + ) + ) + + assert conflict.value.reason_code == "connection-scenario-reset-physical-audit-blocked" + assert teardown_calls == [] + assert ledger.path.read_bytes() == main_bytes + assert ledger.path.stat().st_ino == main_inode + assert ledger.snapshot().record == record + state_after = service.state() + assert state_after["desired_connection_mode_revision"] == 0 + assert state_after["connection_scenario_reset"] is None + assert state_after["connection_scenario_reset_pending"] is None + assert state_after["application_control_session"] == state_before["application_control_session"] + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + assert not archive_dir.exists() + + +@pytest.mark.parametrize("filesystem_blocker", ["symlink", "collision", "hardlink"]) +def test_archive_filesystem_preflight_fails_before_any_reset_teardown( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + filesystem_blocker: str, +) -> None: + service = _service(monkeypatch, tmp_path) + record = _seed_resettable_physical_record(service, closed_cycles=4) + ledger = service._physical_command_ledger # noqa: SLF001 + main_bytes = ledger.path.read_bytes() + main_inode = ledger.path.stat().st_ino + monkeypatch.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(main_bytes) + 1, + ) + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + outside = tmp_path / "outside-archive" + outside.mkdir(mode=0o700) + segment = PhysicalCommandArchiveSegment( + sequence=1, + predecessor_sha256=None, + record_sha256=ledger_module._sha256_document(record.as_dict()), + record=record, + ) + payload = ledger_module._canonical_json_bytes(segment.as_dict()) + digest = hashlib.sha256(payload).hexdigest() + if filesystem_blocker == "symlink": + archive_dir.symlink_to(outside, target_is_directory=True) + else: + archive_dir.mkdir(mode=0o700) + destination = archive_dir / f"{digest}.json" + destination.write_bytes( + b"different-content-address-bytes\n" if filesystem_blocker == "collision" else payload + ) + destination.chmod(0o600) + if filesystem_blocker == "hardlink": + os.link(destination, outside / "unrelated-hard-link.json") + teardown_calls = _forbid_reset_teardown(service, monkeypatch) + + with pytest.raises(NetworkProvisioningConflict) as conflict: + service.select_connection_mode( + _reset( + mode="quick-connect", + revision=0, + reset_id=f"op-reset-preflight-fs-{filesystem_blocker}", + ) + ) + + assert conflict.value.reason_code == "connection-scenario-reset-physical-audit-blocked" + assert teardown_calls == [] + assert ledger.path.read_bytes() == main_bytes + assert ledger.path.stat().st_ino == main_inode + assert ledger.snapshot().record == record + assert service.state()["desired_connection_mode_revision"] == 0 + assert service.state()["connection_scenario_reset_pending"] is None + if filesystem_blocker == "symlink": + assert tuple(outside.iterdir()) == () + elif filesystem_blocker == "collision": + assert (archive_dir / f"{digest}.json").read_bytes() == ( + b"different-content-address-bytes\n" + ) + else: + assert (archive_dir / f"{digest}.json").stat().st_nlink == 2 + + +def test_successful_explicit_scan_settles_only_the_marker_presentation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + request = _reset( + mode="bridge", + revision=0, + reset_id="op-reset-settled-by-scan-01", + ) + committed = service.select_connection_mode(request) + assert committed["connection_scenario_reset"]["active"] is True + assert committed["connection_scenario_reset"]["settled_by_discovery_generation"] is None + + async def empty_successful_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return {"devices": []} + + monkeypatch.setattr(facade_module, "scan", empty_successful_scan) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + + marker = scanned["connection_scenario_reset"] + assert marker["reset_id"] == request.reset_id + assert marker["active"] is False + assert marker["settled_by_discovery_generation"] == scanned["ble_discovery_generation"] + assert scanned["devices"] == [] + + replayed = service.select_connection_mode(request) + + assert replayed["desired_connection_mode_revision"] == 1 + assert replayed["connection_scenario_reset"] == marker + assert replayed["connection_scenario_reset"]["active"] is False + + +def test_failed_and_cancelled_scans_leave_reset_marker_active( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-scan-not-settled-01", + ) + ) + + async def failing_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + raise RuntimeError("injected scan failure") + + monkeypatch.setattr(facade_module, "scan", failing_scan) + with pytest.raises(RuntimeError, match="injected scan failure"): + asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + failed_marker = service.state()["connection_scenario_reset"] + assert failed_marker["active"] is True + assert failed_marker["settled_by_discovery_generation"] is None + + async def cancelled_scenario() -> None: + admitted = asyncio.Event() + + async def blocked_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + admitted.set() + await asyncio.Event().wait() + raise AssertionError("unreachable") + + monkeypatch.setattr(facade_module, "scan", blocked_scan) + scan_task = asyncio.create_task(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + await asyncio.wait_for(admitted.wait(), timeout=1.0) + scan_task.cancel() + with pytest.raises(asyncio.CancelledError): + await scan_task + + asyncio.run(cancelled_scenario()) + cancelled_marker = service.state()["connection_scenario_reset"] + assert cancelled_marker["active"] is True + assert cancelled_marker["settled_by_discovery_generation"] is None + + +def test_old_scan_cannot_settle_marker_when_new_reset_is_queued( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-before-racing-scan-01", + ) + ) + with service._lock: # noqa: SLF001 + old_marker = service._connection_scenario_reset # noqa: SLF001 + assert old_marker is not None + + async def scenario() -> dict[str, Any]: + scan_admitted = asyncio.Event() + finish_scan = asyncio.Event() + + async def delayed_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + scan_admitted.set() + await finish_scan.wait() + return {"devices": []} + + monkeypatch.setattr(facade_module, "scan", delayed_scan) + scan_task = asyncio.create_task(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + await asyncio.wait_for(scan_admitted.wait(), timeout=1.0) + newer_request = _reset( + mode="quick-connect", + revision=1, + reset_id="op-reset-queued-after-old-scan-01", + ) + reset_task = asyncio.create_task( + asyncio.to_thread(service.select_connection_mode, newer_request) + ) + deadline = asyncio.get_running_loop().time() + 1.0 + while asyncio.get_running_loop().time() < deadline: + with service._lock: # noqa: SLF001 + pending = service._connection_scenario_reset_pending # noqa: SLF001 + if pending and pending.get("reset_id") == newer_request.reset_id: + break + await asyncio.sleep(0.01) + else: # pragma: no cover - timeout assertion aid + raise AssertionError("newer reset did not enter the pending boundary") + + finish_scan.set() + await asyncio.wait_for(scan_task, timeout=2.0) + return await asyncio.wait_for(reset_task, timeout=3.0) + + reset_state = asyncio.run(scenario()) + + # The older successful callback saw a different pending reset and did not + # mutate its captured marker. The newer reset then committed a fresh + # active presentation boundary. + assert old_marker["active"] is True + assert old_marker["settled_by_discovery_generation"] is None + assert reset_state["desired_connection_mode_revision"] == 2 + assert reset_state["connection_scenario_reset"]["reset_id"] == ( + "op-reset-queued-after-old-scan-01" + ) + assert reset_state["connection_scenario_reset"]["active"] is True + assert reset_state["connection_scenario_reset"]["settled_by_discovery_generation"] is None + + +def test_scan_entered_before_reset_cannot_settle_marker_after_delayed_admission( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-captured-at-scan-entry", + ) + ) + + async def scenario() -> tuple[dict[str, Any], dict[str, Any]]: + scan_entered = asyncio.Event() + allow_scan_admission = asyncio.Event() + + async def delayed_prepare() -> None: + scan_entered.set() + await allow_scan_admission.wait() + + async def successful_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + if on_admitted is not None: + on_admitted() + return {"devices": []} + + monkeypatch.setattr(service, "_prepare_pending_mode_scan", delayed_prepare) + monkeypatch.setattr(facade_module, "scan", successful_scan) + scan_task = asyncio.create_task(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + await asyncio.wait_for(scan_entered.wait(), timeout=1.0) + + reset_state = await asyncio.to_thread( + service.select_connection_mode, + _reset( + mode="quick-connect", + revision=1, + reset_id="op-reset-after-scan-entry-01", + ), + ) + allow_scan_admission.set() + scanned_state = await asyncio.wait_for(scan_task, timeout=2.0) + return reset_state, scanned_state + + reset_state, scanned_state = asyncio.run(scenario()) + + assert reset_state["connection_scenario_reset"]["active"] is True + assert reset_state["desired_connection_mode_revision"] == 2 + marker = scanned_state["connection_scenario_reset"] + assert marker["reset_id"] == "op-reset-after-scan-entry-01" + assert marker["active"] is True + assert marker["settled_by_discovery_generation"] is None + + +def test_emergency_shell_reset_clears_ephemeral_state_to_bridge_without_device_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service.select_connection_mode( + DesiredConnectionModeRequest( + connection_mode="quick-connect", + expected_revision=0, + ) + ) + + runtime_active = True + runtime_stop_calls = 0 + base_runtime = service.runtime.snapshot() + + def runtime_snapshot() -> dict[str, object]: + snapshot = dict(base_runtime) + snapshot.update( + { + "phase": "replay" if runtime_active else "idle", + "source_mode": "replay" if runtime_active else "idle", + "rerun_grpc_url": "rerun+http://127.0.0.1:9876/proxy" if runtime_active else None, + } + ) + return snapshot + + def stop_runtime() -> None: + nonlocal runtime_active, runtime_stop_calls + runtime_stop_calls += 1 + runtime_active = False + + camera_active = True + camera_stop_calls = 0 + base_camera = service.camera_preview.snapshot() + + def camera_snapshot() -> dict[str, object]: + snapshot = dict(base_camera) + snapshot.update( + { + "phase": "streaming" if camera_active else "idle", + "active_source_id": "sensor.camera.right" if camera_active else None, + "error": {"code": "stale-preview", "message": "old"} if camera_active else None, + } + ) + snapshot["recording"] = { + **dict(base_camera["recording"]), + "active": False, + } + return snapshot + + def stop_camera() -> dict[str, object]: + nonlocal camera_active, camera_stop_calls + camera_stop_calls += 1 + camera_active = False + return camera_snapshot() + + monkeypatch.setattr(service.runtime, "snapshot", runtime_snapshot) + monkeypatch.setattr(service.runtime, "stop", stop_runtime) + monkeypatch.setattr(service.camera_preview, "snapshot", camera_snapshot) + monkeypatch.setattr(service.camera_preview, "stop_current", stop_camera) + + shadow_disarm = service._application_control.disarm # noqa: SLF001 + shadow_disarm_calls = 0 + + def disarm_shadow() -> object: + nonlocal shadow_disarm_calls + shadow_disarm_calls += 1 + return shadow_disarm() + + monkeypatch.setattr(service._application_control, "disarm", disarm_shadow) # noqa: SLF001 + + forbidden_action_calls: list[str] = [] + + def forbidden_io(*_args: object, **_kwargs: object) -> None: + forbidden_action_calls.append("called") + raise AssertionError("scenario reset crossed a forbidden device/host I/O boundary") + + for forbidden_name in ( + "scan", + "read_wifi_status_once", + "provision_wifi_once", + "associate_with_wifi_profile_once", + "associate_with_ephemeral_wifi_credentials_once", + ): + monkeypatch.setattr(facade_module, forbidden_name, forbidden_io) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "request_start", + forbidden_io, + ) + monkeypatch.setattr( + service._application_control_session, # noqa: SLF001 + "request_stop", + forbidden_io, + ) + for forbidden_service_action in ( + "scan_ble", + "connect", + "verify_connection", + "probe_configured_endpoint", + "start_acquisition", + "stop_acquisition", + ): + monkeypatch.setattr(service, forbidden_service_action, forbidden_io) + for forbidden_publish_edge in ( + "publish_dispatching", + "publish_result", + "qos2_completed", + "application_response", + ): + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + forbidden_publish_edge, + forbidden_io, + ) + + observed = time.monotonic() + stale_recovery_future: Future[object] = Future() + with service._lock: # noqa: SLF001 + service._operation_phase = "error" # noqa: SLF001 + service._operation_message = "old connection error" # noqa: SLF001 + service._devices = [ # noqa: SLF001 + { + "device_id": "stale-k1", + "name": "XGR-K1", + "rssi": -42, + "address": None, + "connectable": True, + "likely_k1": True, + } + ] + service._ble_device_last_seen_monotonic = { # noqa: SLF001 + "stale-k1": observed + } + service._ble_device_last_seen_suspend_aware = { # noqa: SLF001 + "stale-k1": time.time() + } + service._selected_device_id = "stale-k1" # noqa: SLF001 + service._k1_ip = "192.168.68.51" # noqa: SLF001 + service._connection_mode = "quick-connect" # noqa: SLF001 + service._device_id = "logical-stale-k1" # noqa: SLF001 + service._device_session_id = "session-stale-k1" # noqa: SLF001 + service._device_session_opened_at = "2026-08-12T10:00:00Z" # noqa: SLF001 + service._connection_reconfiguration_intent_id = "old-draft" # noqa: SLF001 + service._connection_reconfiguration_intent = "select-device" # noqa: SLF001 + service._connection_reconfiguration_required_connection_mode = ( # noqa: SLF001 + "bridge" + ) + service._connection_verification = { # noqa: SLF001 + "status": "unreachable", + "lease_state": "disconnected", + "lease_generation": 99, + "endpoint_validation": "failed", + "network_reachability": "unreachable", + "reason_code": "old-error", + "observed_at": "2026-08-12T10:00:00Z", + } + service._configured_endpoint_probe = { # noqa: SLF001 + **service._configured_endpoint_probe, # noqa: SLF001 + "status": "failed", + "endpoint": "192.168.68.51:1883", + "reason_code": "old-probe-error", + } + service._active_stream_recovery_state = "blocked" # noqa: SLF001 + service._active_stream_recovery_future = stale_recovery_future # noqa: SLF001 + service._active_stream_recovery_attempt = 4 # noqa: SLF001 + service._active_stream_recovery_reason_code = "old-recovery-error" # noqa: SLF001 + service._active_stream_recovery_started_at_utc = ( # noqa: SLF001 + "2026-08-12T10:00:00Z" + ) + service._active_stream_recovery_started_monotonic = observed # noqa: SLF001 + + reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=1, + reset_id="op-shell-emergency-reset-01", + ) + ) + + assert runtime_stop_calls == 1 + assert camera_stop_calls == 1 + assert shadow_disarm_calls == 1 + assert stale_recovery_future.cancelled() is True + assert reset["desired_connection_mode"] == "bridge" + assert reset["desired_connection_mode_revision"] == 2 + assert reset["phase"] == "idle" + assert reset["source_mode"] == "idle" + assert reset["rerun_grpc_url"] is None + assert reset["camera_preview"]["phase"] == "idle" + assert reset["camera_preview"]["active_source_id"] is None + assert reset["camera_preview"]["error"] is None + assert reset["devices"] == [] + assert reset["selected_device_id"] is None + assert reset["device_ref"] is None + assert reset["device_session"] is None + assert reset["connection_mode"] is None + assert reset["connection_reconfiguration"]["status"] == "idle" + assert reset["application_control_execution"]["state"] == "disarmed" + assert reset["application_control_session"]["state"] == "idle" + assert reset["connection_verification"]["status"] == "not-probed" + assert reset["configured_endpoint_probe"]["status"] == "not-probed" + assert reset["configured_endpoint_probe"]["endpoint"] is None + assert reset["connection_recovery"]["state"] == "inactive" + assert reset["connection_recovery"]["attempt"] == 0 + assert reset["connection_recovery"]["reason_code"] is None + assert reset["connection_attempt"] is None + assert reset["connection_scenario_reset"]["active"] is True + assert reset["connection_scenario_reset"]["settled_by_discovery_generation"] is None + assert reset["connection_policy"]["actions"]["scan-ble"]["allowed"] is True + assert reset["connection_scenario_reset"]["device_command_performed"] is False + assert reset["connection_scenario_reset"]["network_write_performed"] is False + assert reset["connection_scenario_reset"]["automatic_scan"] is False + assert forbidden_action_calls == [] + + +def test_residual_source_cleanup_failure_keeps_reset_retryable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + base_runtime = service.runtime.snapshot() + runtime_active = True + stop_attempts = 0 + + def runtime_snapshot() -> dict[str, object]: + snapshot = dict(base_runtime) + snapshot.update( + { + "phase": "replay" if runtime_active else "idle", + "source_mode": "replay" if runtime_active else "idle", + } + ) + return snapshot + + def stop_runtime() -> None: + nonlocal runtime_active, stop_attempts + stop_attempts += 1 + if stop_attempts == 1: + raise RuntimeError("injected standalone replay cleanup failure") + runtime_active = False + + monkeypatch.setattr(service.runtime, "snapshot", runtime_snapshot) + monkeypatch.setattr(service.runtime, "stop", stop_runtime) + request = _reset( + mode="bridge", + revision=0, + reset_id="op-reset-residual-cleanup-retry", + ) + + with pytest.raises(RuntimeError, match="standalone replay cleanup failure"): + service.select_connection_mode(request) + + with service._lock: # noqa: SLF001 + assert service._desired_connection_mode_revision == 0 # noqa: SLF001 + assert service._connection_scenario_reset is None # noqa: SLF001 + assert service._connection_scenario_reset_pending is None # noqa: SLF001 + assert service._active_stream_recovery_state == "blocked" # noqa: SLF001 + + recovered = service.select_connection_mode(request) + + assert stop_attempts == 2 + assert recovered["desired_connection_mode_revision"] == 1 + assert recovered["source_mode"] == "idle" + assert recovered["connection_recovery"]["state"] == "inactive" + assert recovered["connection_scenario_reset"]["reset_id"] == request.reset_id + + +def test_reset_id_cannot_be_rebound_to_another_fingerprint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + reset_id = "op-reset-fingerprint-0001" + service.select_connection_mode(_reset(mode="bridge", revision=0, reset_id=reset_id)) + + with pytest.raises(NetworkProvisioningConflict) as conflict: + service.select_connection_mode(_reset(mode="quick-connect", revision=0, reset_id=reset_id)) + + assert conflict.value.reason_code == "connection-scenario-reset-id-conflict" + assert service.state()["desired_connection_mode_revision"] == 1 + + +def test_reset_retry_converges_after_durable_retirement_failpoint( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + coordinator = service._physical_command_coordinator # noqa: SLF001 + retire = coordinator.retire_connection_scenario + attempts = 0 + + def fail_once(**kwargs: object) -> dict[str, object]: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("injected post-network/pre-physical failpoint") + return retire(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(coordinator, "retire_connection_scenario", fail_once) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-partial-retry-001", + ) + + with pytest.raises(RuntimeError, match="injected"): + service.select_connection_mode(request) + with service._lock: # noqa: SLF001 + assert service._desired_connection_mode_revision == 0 # noqa: SLF001 + assert service._connection_scenario_reset is None # noqa: SLF001 + assert service._connection_scenario_reset_pending is None # noqa: SLF001 + + recovered = service.select_connection_mode(request) + + assert attempts == 2 + assert recovered["desired_connection_mode"] == "quick-connect" + assert recovered["desired_connection_mode_revision"] == 1 + + +def test_committed_replay_marker_waits_for_fallible_local_tail( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + retire_binding = service._retire_ephemeral_device_binding_for_new_intent # noqa: SLF001 + attempts = 0 + + def fail_once() -> None: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("injected local-tail failpoint") + retire_binding() + + monkeypatch.setattr( + service, + "_retire_ephemeral_device_binding_for_new_intent", + fail_once, + ) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-local-tail-retry-01", + ) + + with pytest.raises(RuntimeError, match="local-tail"): + service.select_connection_mode(request) + with service._lock: # noqa: SLF001 + assert service._desired_connection_mode_revision == 0 # noqa: SLF001 + assert service._connection_scenario_reset is None # noqa: SLF001 + + recovered = service.select_connection_mode(request) + + assert attempts == 2 + assert recovered["desired_connection_mode_revision"] == 1 + assert recovered["connection_scenario_reset"]["reset_id"] == request.reset_id + + +def test_new_waiting_reset_supersedes_old_waiter_without_old_teardown( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service._acquisition_lifecycle_gate.acquire() # noqa: SLF001 + outcomes: dict[str, object] = {} + + def invoke(name: str, request: DesiredConnectionModeRequest) -> None: + try: + outcomes[name] = service.select_connection_mode(request) + except BaseException as exc: # pragma: no cover - assertion aid + outcomes[name] = exc + + first = threading.Thread( + target=invoke, + args=( + "first", + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-waiter-first-01", + ), + ), + daemon=True, + ) + second = threading.Thread( + target=invoke, + args=( + "second", + _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-waiter-second-1", + ), + ), + daemon=True, + ) + first.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + pending = service._connection_scenario_reset_pending # noqa: SLF001 + if pending and pending.get("reset_id") == "op-reset-waiter-first-01": + break + time.sleep(0.01) + second.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + pending = service._connection_scenario_reset_pending # noqa: SLF001 + if pending and pending.get("reset_id") == "op-reset-waiter-second-1": + break + time.sleep(0.01) + service._acquisition_lifecycle_gate.release() # noqa: SLF001 + first.join(timeout=3.0) + second.join(timeout=3.0) + + assert isinstance(outcomes["first"], NetworkProvisioningConflict) + assert outcomes["first"].reason_code == "connection-scenario-reset-superseded" # type: ignore[union-attr] + assert isinstance(outcomes["second"], dict) + assert outcomes["second"]["desired_connection_mode"] == "quick-connect" # type: ignore[index] + assert service.state()["desired_connection_mode_revision"] == 1 + + +def test_service_close_wakes_a_reset_waiter_and_clears_pending( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service._acquisition_lifecycle_gate.acquire() # noqa: SLF001 + outcome: list[BaseException] = [] + + def invoke_reset() -> None: + try: + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-close-waiter-001", + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + outcome.append(exc) + + reset_thread = threading.Thread(target=invoke_reset, daemon=True) + reset_thread.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + if service._connection_scenario_reset_pending is not None: # noqa: SLF001 + break + time.sleep(0.01) + close_thread = threading.Thread(target=service.close, daemon=True) + close_thread.start() + reset_thread.join(timeout=2.0) + with service._lock: # noqa: SLF001 + pending = service._connection_scenario_reset_pending # noqa: SLF001 + service._acquisition_lifecycle_gate.release() # noqa: SLF001 + close_thread.join(timeout=3.0) + + assert len(outcome) == 1 + assert isinstance(outcome[0], NetworkProvisioningConflict) + assert outcome[0].reason_code == "connection-scenario-reset-service-closing" + assert pending is None + assert close_thread.is_alive() is False + + +def test_concurrent_exact_retry_replays_one_committed_reset( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service._acquisition_lifecycle_gate.acquire() # noqa: SLF001 + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-concurrent-retry-01", + ) + outcomes: list[dict[str, object] | BaseException] = [] + + def invoke_reset() -> None: + try: + outcomes.append(service.select_connection_mode(request)) + except BaseException as exc: # pragma: no cover - assertion aid + outcomes.append(exc) + + first = threading.Thread(target=invoke_reset, daemon=True) + second = threading.Thread(target=invoke_reset, daemon=True) + first.start() + second.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._lock: # noqa: SLF001 + pending = service._connection_scenario_reset_pending # noqa: SLF001 + if pending and pending.get("reset_id") == request.reset_id: + break + time.sleep(0.01) + service._acquisition_lifecycle_gate.release() # noqa: SLF001 + first.join(timeout=3.0) + second.join(timeout=3.0) + + assert len(outcomes) == 2 + assert all(isinstance(outcome, dict) for outcome in outcomes) + states = [outcome for outcome in outcomes if isinstance(outcome, dict)] + assert {state["desired_connection_mode_revision"] for state in states} == {1} + assert { + state["connection_scenario_reset"]["reset_id"] # type: ignore[index] + for state in states + } == {request.reset_id} + assert service.state()["desired_connection_mode_revision"] == 1 + + +def test_reset_wait_timeout_clears_pending_and_is_retryable( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr( + facade_module, + "CONNECTION_SCENARIO_RESET_DRAIN_TIMEOUT_SECONDS", + 0.05, + ) + service = _service(monkeypatch, tmp_path) + gate_held = threading.Event() + release_gate = threading.Event() + + def hold_gate() -> None: + service._acquisition_lifecycle_gate.acquire() # noqa: SLF001 + gate_held.set() + release_gate.wait(timeout=2.0) + service._acquisition_lifecycle_gate.release() # noqa: SLF001 + + holder = threading.Thread(target=hold_gate, daemon=True) + holder.start() + assert gate_held.wait(timeout=1.0) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-timeout-retry-0001", + ) + + with pytest.raises(NetworkProvisioningConflict) as conflict: + service.select_connection_mode(request) + + assert conflict.value.reason_code == "connection-scenario-reset-lifecycle-timeout" + with service._lock: # noqa: SLF001 + assert service._connection_scenario_reset_pending is None # noqa: SLF001 + release_gate.set() + holder.join(timeout=1.0) + + recovered = service.select_connection_mode(request) + + assert recovered["desired_connection_mode_revision"] == 1 + assert recovered["connection_scenario_reset"]["reset_id"] == request.reset_id + + +def test_reset_wait_releases_old_owner_gates_and_keeps_state_responsive( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + owner_ready = threading.Event() + owner_may_finish = threading.Event() + owner_reacquired_transition = threading.Event() + + def old_network_owner() -> None: + service._acquire_k1_lifecycle_process_lease("network") # noqa: SLF001 + owner_ready.set() + owner_may_finish.wait(timeout=2.0) + acquired = service._k1_lifecycle_transition_gate.acquire( # noqa: SLF001 + timeout=1.0 + ) + if acquired: + owner_reacquired_transition.set() + service._k1_lifecycle_transition_gate.release() # noqa: SLF001 + service._release_k1_lifecycle_process_lease("network") # noqa: SLF001 + + owner = threading.Thread(target=old_network_owner, daemon=True) + owner.start() + assert owner_ready.wait(timeout=1.0) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-old-network-owner-01", + ) + reset_outcome: list[dict[str, object] | BaseException] = [] + + def invoke_reset() -> None: + try: + reset_outcome.append(service.select_connection_mode(request)) + except BaseException as exc: # pragma: no cover - assertion aid + reset_outcome.append(exc) + + reset_thread = threading.Thread(target=invoke_reset, daemon=True) + reset_thread.start() + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline: + pending = service.state().get("connection_scenario_reset_pending") + if isinstance(pending, dict) and pending.get("reset_id") == request.reset_id: + break + time.sleep(0.01) + + state_outcome: list[dict[str, object]] = [] + state_thread = threading.Thread( + target=lambda: state_outcome.append(service.state()), + daemon=True, + ) + state_thread.start() + state_thread.join(timeout=1.0) + assert state_thread.is_alive() is False + assert state_outcome[0]["connection_scenario_reset_pending"] is not None + + owner_may_finish.set() + owner.join(timeout=2.0) + reset_thread.join(timeout=3.0) + + assert owner_reacquired_transition.is_set() + assert owner.is_alive() is False + assert reset_thread.is_alive() is False + assert len(reset_outcome) == 1 + assert isinstance(reset_outcome[0], dict) + assert reset_outcome[0]["desired_connection_mode_revision"] == 1 # type: ignore[index] + + +def test_reset_writer_priority_blocks_late_state_pollers_until_commit( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + service._acquisition_lifecycle_gate.acquire() # noqa: SLF001 + order: list[str] = [] + order_lock = threading.Lock() + reset_outcome: list[dict[str, object] | BaseException] = [] + original_retire_binding = ( + service._retire_ephemeral_device_binding_for_new_intent # noqa: SLF001 + ) + + def mark_reset_commit() -> None: + with order_lock: + order.append("reset-owned") + original_retire_binding() + + monkeypatch.setattr( + service, + "_retire_ephemeral_device_binding_for_new_intent", + mark_reset_commit, + ) + + def invoke_reset() -> None: + try: + reset_outcome.append( + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-writer-priority-01", + ) + ) + ) + except BaseException as exc: # pragma: no cover - assertion aid + reset_outcome.append(exc) + + reset_thread = threading.Thread(target=invoke_reset, daemon=True) + reset_thread.start() + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + with service._acquisition_lifecycle_admission: # noqa: SLF001 + if service._acquisition_lifecycle_writer_token is not None: # noqa: SLF001 + break + time.sleep(0.005) + with service._acquisition_lifecycle_admission: # noqa: SLF001 + assert service._acquisition_lifecycle_writer_token is not None # noqa: SLF001 + + reader_threads: list[threading.Thread] = [] + + def poll_state(index: int) -> None: + service.state() + with order_lock: + order.append(f"reader-{index}") + + for index in range(8): + reader = threading.Thread(target=poll_state, args=(index,), daemon=True) + reader_threads.append(reader) + reader.start() + time.sleep(0.05) + assert order == [] + + released_at = time.monotonic() + service._acquisition_lifecycle_gate.release() # noqa: SLF001 + reset_thread.join(timeout=2.0) + for reader in reader_threads: + reader.join(timeout=2.0) + + assert reset_thread.is_alive() is False + assert all(reader.is_alive() is False for reader in reader_threads) + assert time.monotonic() - released_at < 2.0 + assert len(reset_outcome) == 1 + assert isinstance(reset_outcome[0], dict) + assert order[0] == "reset-owned" + assert sorted(order[1:]) == [f"reader-{index}" for index in range(8)] + with service._acquisition_lifecycle_admission: # noqa: SLF001 + assert service._acquisition_lifecycle_writer_token is None # noqa: SLF001 + assert service._acquisition_lifecycle_writer_thread_id is None # noqa: SLF001 + with service._lock: # noqa: SLF001 + assert service._application_control_process_lease_holders == set() # noqa: SLF001 + + +def test_reset_writer_owner_can_reenter_state_snapshot( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + outcome: list[dict[str, object] | BaseException] = [] + + def invoke_owned_state() -> None: + token = object() + try: + assert service._try_register_acquisition_lifecycle_writer(token) # noqa: SLF001 + outcome.append(service.state()) + except BaseException as exc: # pragma: no cover - assertion aid + outcome.append(exc) + finally: + service._unregister_acquisition_lifecycle_writer(token) # noqa: SLF001 + + owner = threading.Thread(target=invoke_owned_state, daemon=True) + owner.start() + owner.join(timeout=2.0) + + assert owner.is_alive() is False + assert len(outcome) == 1 + assert isinstance(outcome[0], dict) + assert outcome[0]["snapshot_runtime_id"] == service._snapshot_runtime_id # type: ignore[index] # noqa: SLF001 + + +def test_active_local_scanning_evidence_keeps_physical_outcome_truthful( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + acquisition = AcquisitionRecord( + acquisition_id="acq-reset-active-local-evidence", + device_id="device-reset-active-local-evidence", + device_session_id="device-session-reset-active-local-evidence", + compatibility_profile_id=("xgrids.lixelkity-k1.fw-3.0.2.local-network.v2"), + control_mode="plugin-commanded", + requested_streams=("points",), + target_host="192.168.68.51", + duration_seconds=None, + evidence_policy="required", + ) + acquisition.transition( + "acquiring", + message_code="acquisition.active", + ) + with service._lock: # noqa: SLF001 + service._acquisition = acquisition # noqa: SLF001 + + reset = service.select_connection_mode( + _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-active-local-evidence", + ) + ) + + marker = reset["connection_scenario_reset"] + assert marker["local_session_closed"] is True + assert marker["previous_device_may_continue_scanning"] is True + assert marker["physical_disposition"] == "empty" + assert marker["device_command_performed"] is False + assert marker["network_write_performed"] is False + assert reset["acquisition"]["state"] == "interrupted" + + +def test_terminal_cleanup_pending_is_locally_sealed_by_mode_reset( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + acquisition = AcquisitionRecord( + acquisition_id="acq-reset-terminal-cleanup", + device_id="device-reset-terminal-cleanup", + device_session_id="device-session-reset-terminal-cleanup", + compatibility_profile_id=("xgrids.lixelkity-k1.fw-3.0.2.local-network.v2"), + control_mode="plugin-commanded", + requested_streams=("points", "camera"), + target_host="192.168.68.51", + duration_seconds=None, + evidence_policy="required", + ) + original_result = {"failure_code": "receiver-disconnected", "device_state": "unknown"} + acquisition.transition( + "failed", + message_code="acquisition.receiver_failed", + result=original_result, + ) + + class FakeLease: + releases = 0 + + def release(self) -> None: + self.releases += 1 + + lease = FakeLease() + with service._lock: # noqa: SLF001 + service._acquisition = acquisition # noqa: SLF001 + service._acquisition_session_lease = lease # type: ignore[assignment] # noqa: SLF001 + service._application_control_process_lease_holders.add("camera") # noqa: SLF001 + + base_runtime = service.runtime.snapshot() + runtime_live = True + runtime_stop_calls = 0 + + def runtime_snapshot() -> dict[str, object]: + snapshot = dict(base_runtime) + snapshot.update( + { + "phase": "live" if runtime_live else "idle", + "source_mode": "live" if runtime_live else "idle", + } + ) + return snapshot + + def stop_runtime() -> None: + nonlocal runtime_live, runtime_stop_calls + runtime_stop_calls += 1 + runtime_live = False + + monkeypatch.setattr(service.runtime, "snapshot", runtime_snapshot) + monkeypatch.setattr(service.runtime, "stop", stop_runtime) + original_stop_recording = service.camera_preview.stop_recording + original_stop_current = service.camera_preview.stop_current + camera_recording_stop_calls = 0 + camera_preview_stop_calls = 0 + + def stop_recording(**kwargs: object) -> object: + nonlocal camera_recording_stop_calls + camera_recording_stop_calls += 1 + return original_stop_recording(**kwargs) # type: ignore[arg-type] + + def stop_current() -> dict[str, object]: + nonlocal camera_preview_stop_calls + camera_preview_stop_calls += 1 + return original_stop_current() + + monkeypatch.setattr(service.camera_preview, "stop_recording", stop_recording) + monkeypatch.setattr(service.camera_preview, "stop_current", stop_current) + + reset = service.select_connection_mode( + _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-terminal-cleanup-01", + ) + ) + + assert runtime_stop_calls == 1 + assert camera_recording_stop_calls == 1 + assert camera_preview_stop_calls == 1 + assert lease.releases == 1 + with service._lock: # noqa: SLF001 + assert service._acquisition_session_lease is None # noqa: SLF001 + assert "camera" not in service._application_control_process_lease_holders # noqa: SLF001 + assert reset["acquisition"]["state"] == "failed" + assert reset["acquisition"]["result"] == original_result + assert reset["acquisition"]["cleanup_pending"] is False + assert reset["connection_scenario_reset"]["device_command_performed"] is False + assert reset["connection_scenario_reset"]["network_write_performed"] is False + + +def test_terminal_cleanup_failure_stays_visible_and_exact_reset_can_retry( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + acquisition = AcquisitionRecord( + acquisition_id="acq-reset-terminal-cleanup-retry", + device_id="device-reset-terminal-cleanup-retry", + device_session_id="device-session-reset-terminal-cleanup-retry", + compatibility_profile_id=("xgrids.lixelkity-k1.fw-3.0.2.local-network.v2"), + control_mode="plugin-commanded", + requested_streams=("points",), + target_host="192.168.68.51", + duration_seconds=None, + evidence_policy="required", + ) + original_result = {"failure_code": "local-seal-pending", "device_state": "unknown"} + acquisition.transition( + "failed", + message_code="acquisition.cleanup_pending", + result=original_result, + ) + + class FakeLease: + releases = 0 + + def release(self) -> None: + self.releases += 1 + + lease = FakeLease() + with service._lock: # noqa: SLF001 + service._acquisition = acquisition # noqa: SLF001 + service._acquisition_session_lease = lease # type: ignore[assignment] # noqa: SLF001 + original_cleanup = service._stop_acquisition_sources # noqa: SLF001 + cleanup_attempts = 0 + + def fail_once(**kwargs: object) -> None: + nonlocal cleanup_attempts + cleanup_attempts += 1 + if cleanup_attempts == 1: + raise RuntimeError("injected terminal local cleanup failure") + original_cleanup(**kwargs) # type: ignore[arg-type] + + monkeypatch.setattr(service, "_stop_acquisition_sources", fail_once) + request = _reset( + mode="quick-connect", + revision=0, + reset_id="op-reset-terminal-cleanup-retry", + ) + + with pytest.raises(RuntimeError, match="terminal local cleanup failure"): + service.select_connection_mode(request) + + failed = service.state() + assert failed["desired_connection_mode_revision"] == 0 + assert failed["connection_scenario_reset"] is None + assert failed["connection_scenario_reset_pending"] is None + assert failed["acquisition"]["state"] == "failed" + assert failed["acquisition"]["result"] == original_result + assert failed["acquisition"]["cleanup_pending"] is True + assert lease.releases == 0 + + recovered = service.select_connection_mode(request) + + assert cleanup_attempts == 2 + assert lease.releases == 1 + assert recovered["desired_connection_mode_revision"] == 1 + assert recovered["acquisition"]["state"] == "failed" + assert recovered["acquisition"]["result"] == original_result + assert recovered["acquisition"]["cleanup_pending"] is False + assert recovered["connection_scenario_reset"]["device_command_performed"] is False + assert recovered["connection_scenario_reset"]["network_write_performed"] is False + + +@pytest.mark.parametrize("observed_state", ["ready", "scanning"]) +def test_reset_fresh_same_retired_uuid_apply_uses_exact_capture_and_one_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + observed_state: str, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-retired-reprovision-positive-01", + ) + ) + captures = _install_exact_reset_scan(service, monkeypatch) + + first_scan = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + first_settled_generation = first_scan["connection_scenario_reset"][ + "settled_by_discovery_generation" + ] + second_scan = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + current_generation = second_scan["ble_discovery_generation"] + + assert first_settled_generation == first_scan["ble_discovery_generation"] + assert current_generation > first_settled_generation + assert second_scan["connection_scenario_reset"][ + "settled_by_discovery_generation" + ] == first_settled_generation + assert second_scan["connection_policy"]["facts"]["retired_transport_refs"] == [ + "RESET-PREFLIGHT-K1-UUID" + ] + assert second_scan["connection_policy"]["facts"][ + "eligible_fresh_transport_refs" + ] == ["RESET-PREFLIGHT-K1-UUID"] + assert second_scan["connection_policy"]["actions"]["provision-fresh-device"][ + "allowed" + ] is True + + physical_before = service._physical_command_coordinator.snapshot()["record"] # noqa: SLF001 + events: list[str] = [] + writes = 0 + network_ledger = service._network_mutation_ledger # noqa: SLF001 + idempotency_journal = ( + service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + ) + real_network_prepare = network_ledger.prepare + real_physical_reopen = ( + service._physical_command_coordinator.reopen_retired_reconciliation # noqa: SLF001 + ) + real_mark_unresolved = idempotency_journal.mark_unresolved + real_mark_dispatching = network_ledger.mark_dispatching + real_operation_transition = service._operations.transition # noqa: SLF001 + + def traced_network_prepare(*args: object, **kwargs: object) -> object: + result = real_network_prepare(*args, **kwargs) # type: ignore[arg-type] + events.append("network-prepared-durable") + return result + + def traced_physical_reopen(*args: object, **kwargs: object) -> object: + result = real_physical_reopen(*args, **kwargs) # type: ignore[arg-type] + events.append("physical-reopen-durable") + return result + + def traced_mark_unresolved(*args: object, **kwargs: object) -> object: + result = real_mark_unresolved(*args, **kwargs) # type: ignore[arg-type] + events.append("idempotency-unresolved-durable") + return result + + def traced_mark_dispatching(*args: object, **kwargs: object) -> object: + result = real_mark_dispatching(*args, **kwargs) # type: ignore[arg-type] + events.append("network-dispatching-durable") + return result + + def traced_operation_transition(*args: object, **kwargs: object) -> object: + result = real_operation_transition(*args, **kwargs) # type: ignore[arg-type] + if kwargs.get("stage_code") == "ble-write-dispatched": + events.append("operation-ble-write-dispatched") + return result + + monkeypatch.setattr(network_ledger, "prepare", traced_network_prepare) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reopen_retired_reconciliation", + traced_physical_reopen, + ) + monkeypatch.setattr( + idempotency_journal, + "mark_unresolved", + traced_mark_unresolved, + ) + monkeypatch.setattr( + network_ledger, + "mark_dispatching", + traced_mark_dispatching, + ) + monkeypatch.setattr( + service._operations, # noqa: SLF001 + "transition", + traced_operation_transition, + ) + + async def provision_once( + device_id: str, + ssid: str, + _password: str, + *, + captured_device: facade_module.CapturedDiscoveredDevice | None = None, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + nonlocal writes + assert device_id == "RESET-PREFLIGHT-K1-UUID" + assert ssid == "reset-lab-network" + assert captured_device is captures[0] + assert on_write_dispatch is not None + baseline = { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + } + events.append("live-gatt-baseline-7f02") + on_write_dispatch(baseline, "with_response") + events.append("write-7f01") + writes += 1 + result = { + "started_at_utc": "2026-08-14T08:00:00Z", + "completed_at_utc": "2026-08-14T08:00:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": baseline, + "observations": [ + { + "observed_at_utc": "2026-08-14T08:00:01Z", + "status": { + "mode": "WIFI_CLIENT", + "ipv4": "192.168.68.51", + "status_code": 1, + "reserved": 0, + "network_name": "reset-lab-network", + }, + } + ], + } + events.append("helper-return") + return result + + monkeypatch.setattr(facade_module, "provision_wifi_once", provision_once) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + service, + "_schedule_control_bootstrap_continuation", + lambda **_kwargs: None, + ) + request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=current_generation, + ).model_copy( + update={ + # The browser always carries the current numeric revision, even + # when there is no active reconfiguration intent. + "expected_reconfiguration_revision": second_scan[ + "connection_reconfiguration" + ]["revision"], + } + ) + + applied = asyncio.run(service.connect(request)) + physical_after_apply = service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + reopened_record = service._physical_command_ledger.snapshot().record # noqa: SLF001 + assert reopened_record is not None + proof_connection = PhysicalCommandConnectionBinding( + intent_id=request.operation_id or "reset-network-intent", + transport_ref=request.device_id, + connection_mode="bridge", + target_ipv4="192.168.68.51", + target_port=1883, + host_path_epoch=8, + control_session_id=f"reset-post-apply-{observed_state}-control", + producer_generation=12, + ) + service._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001 + reopened_record.operation_id, + reconciliation_id=f"reset-post-apply-{observed_state}-physical", + resolution=( + "physical-active-observed" + if observed_state == "scanning" + else "physical-standby-observed" + ), + verified_binding=PhysicalCommandVerifiedBinding( + verification_id=f"reset-post-apply-{observed_state}-device-info", + identity=reopened_record.identity, + connection=proof_connection, + device_info_message_sha256="e" * 64, + verified_at_utc=reopened_record.updated_at_utc, + ), + observation=PhysicalCommandStatusEvidence( + source="explicit-read-only-reconciliation", + vendor_device_id_sha256=( + reopened_record.identity.vendor_device_id_sha256 + ), + device_serial_sha256=reopened_record.identity.device_serial_sha256, + control_session_id=proof_connection.control_session_id, + host_path_epoch=proof_connection.host_path_epoch, + producer_generation=proof_connection.producer_generation, + session_state=observed_state, + session_state_code=302 if observed_state == "scanning" else 300, + project_bound=observed_state == "scanning", + project_id_sha256="f" * 64 if observed_state == "scanning" else None, + init_ready=observed_state == "scanning", + status_message_sha256="d" * 64, + mqtt_retained=False, + observed_at_utc=reopened_record.updated_at_utc, + ), + ) + physical_after_settlement = service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + replay_request = request.model_copy(update={"operation_id": None}) + replayed = asyncio.run(service.connect(replay_request)) + physical_after_replay = service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + + assert reset["connection_scenario_reset"]["physical_disposition"] == ( + "operator-retired-outcome-unknown" + ) + assert events == [ + "live-gatt-baseline-7f02", + "network-prepared-durable", + "physical-reopen-durable", + "idempotency-unresolved-durable", + "network-dispatching-durable", + "write-7f01", + "helper-return", + "operation-ble-write-dispatched", + ] + assert writes == 1 + network_operations = _network_operation_documents(service) + assert len(network_operations) == 1 + assert network_operations[0]["status"] == "succeeded" + assert applied["selected_device_id"] == "RESET-PREFLIGHT-K1-UUID" + assert replayed["selected_device_id"] == "RESET-PREFLIGHT-K1-UUID" + assert isinstance(physical_before, dict) + assert isinstance(physical_after_apply, dict) + assert isinstance(physical_after_settlement, dict) + assert physical_after_replay == physical_after_settlement + assert physical_after_apply["operation_id"] == physical_before["operation_id"] + assert physical_after_apply["created_at_utc"] == physical_before["created_at_utc"] + assert physical_after_apply["revision"] == physical_before["revision"] + 1 + assert physical_after_apply["stage"] == "observing" + assert physical_after_apply["resolution"] is None + reopens = physical_after_apply["operator_reconciliation_reopens"] + assert isinstance(reopens, list) + assert len(reopens) == 1 + assert reopens[0]["reason"] == "reset-network-intent-read-only-settlement" + assert reopens[0]["retired_record_revision"] == physical_before["revision"] + assert reopens[0]["reopened_transport_ref"] == "RESET-PREFLIGHT-K1-UUID" + assert reopens[0]["discovery_generation"] == current_generation + assert physical_after_settlement["revision"] == physical_after_apply["revision"] + 1 + assert physical_after_settlement["stage"] == "resolved" + assert physical_after_settlement["reconciliations"][-1]["resolution"] == ( + "physical-active-observed" + if observed_state == "scanning" + else "physical-standby-observed" + ) + assert replayed["connection_policy"]["facts"]["retired_transport_refs"] == [] + + changed_key = request.model_copy( + update={"idempotency_key": "reset-retired-reprovision-new-key-0001"} + ) + with pytest.raises(NetworkProvisioningConflict) as changed_key_error: + asyncio.run(service.connect(changed_key)) + assert changed_key_error.value.reason_code == "physical-command-target-retired" + + changed_request = request.model_copy(update={"ssid": "other-reset-network"}) + with pytest.raises(NetworkProvisioningConflict) as changed_request_error: + asyncio.run(service.connect(changed_request)) + assert changed_request_error.value.reason_code == "physical-command-target-retired" + + trusted_journal = service._network_provisioning_idempotency_journal # noqa: SLF001 + service._network_provisioning_idempotency_journal = None # noqa: SLF001 + with pytest.raises(NetworkProvisioningConflict) as missing_journal_error: + asyncio.run(service.connect(replay_request)) + assert missing_journal_error.value.reason_code == "physical-command-target-retired" + service._network_provisioning_idempotency_journal = SimpleNamespace( # type: ignore[assignment] # noqa: SLF001 + snapshot=lambda: SimpleNamespace(status="corrupt", records=()), + ) + with pytest.raises(NetworkProvisioningConflict) as corrupt_journal_error: + asyncio.run(service.connect(replay_request)) + assert corrupt_journal_error.value.reason_code == "physical-command-target-retired" + service._network_provisioning_idempotency_journal = trusted_journal # noqa: SLF001 + assert writes == 1 + assert len(_network_operation_documents(service)) == 1 + + +def test_retired_uuid_without_current_reset_fails_before_network_journal_or_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + record = _seed_resettable_physical_record(service) + service._physical_command_coordinator.retire_connection_scenario( # noqa: SLF001 + retirement_id="retired-without-current-reset", + expected_revision=record.revision, + ) + _install_exact_reset_scan(service, monkeypatch) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + capture_calls: list[str] = [] + + def forbidden_capture(device_id: str) -> None: + capture_calls.append(device_id) + raise AssertionError("retired request reached exact-handle capture") + + async def forbidden_write(*_args: object, **_kwargs: object) -> None: + raise AssertionError("retired request reached GATT provisioning") + + monkeypatch.setattr(facade_module, "_capture_network_intent_device", forbidden_capture) + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_write) + operations_before = service._operations.snapshot(limit=128) # noqa: SLF001 + + with pytest.raises(NetworkProvisioningConflict) as error: + asyncio.run( + service.connect( + _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + mode_revision=0, + operation_id="op-00000000-0000-4000-8000-000000000502", + ) + ) + ) + + assert error.value.reason_code == "physical-command-target-retired" + assert service._operations.snapshot(limit=128) == operations_before # noqa: SLF001 + assert _network_operation_documents(service) == [] + assert service._network_mutation_ledger.snapshot().status == "empty" # noqa: SLF001 + assert capture_calls == [] + + +@pytest.mark.parametrize("dispatch_commits", [False, True]) +def test_ordinary_candidate_burns_retired_exception_only_after_dispatch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + dispatch_commits: bool, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-ordinary-first-burns-retired-authority-01", + ) + ) + captures = _install_exact_reset_scan( + service, + monkeypatch, + device_ids=("RESET-PREFLIGHT-K1-UUID", "ORDINARY-K1-UUID"), + ) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + writes: list[str] = [] + + async def provision_once( + device_id: str, + _ssid: str, + _password: str, + *, + captured_device: facade_module.CapturedDiscoveredDevice | None = None, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + assert device_id == "ORDINARY-K1-UUID" + assert captured_device is next( + item for item in captures if item.macos_uuid == device_id + ) + assert on_write_dispatch is not None + baseline = { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + } + on_write_dispatch(baseline, "with_response") + writes.append(device_id) + return { + "started_at_utc": "2026-08-14T08:10:00Z", + "completed_at_utc": "2026-08-14T08:10:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": baseline, + "observations": [ + { + "observed_at_utc": "2026-08-14T08:10:01Z", + "status": { + "mode": "WIFI_CLIENT", + "ipv4": "192.168.68.52", + "status_code": 1, + "reserved": 0, + "network_name": "reset-lab-network", + }, + } + ], + } + + monkeypatch.setattr(facade_module, "provision_wifi_once", provision_once) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + service, + "_schedule_control_bootstrap_continuation", + lambda **_kwargs: None, + ) + if not dispatch_commits: + monkeypatch.setattr( + service._network_mutation_ledger, # noqa: SLF001 + "mark_dispatching", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("synthetic ordinary-B dispatch precommit fault") + ), + ) + ordinary_request = _reset_connect_request( + device_id="ORDINARY-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000511", + ) + if dispatch_commits: + asyncio.run(service.connect(ordinary_request)) + else: + with pytest.raises(RuntimeError, match="ordinary-B dispatch precommit"): + asyncio.run(service.connect(ordinary_request)) + + with service._lock: # noqa: SLF001 + assert ( # noqa: SLF001 + service._connection_scenario_reset_retired_transport_authority + is not None + ) + assert writes == [] + failed_operation = _network_operation_documents(service)[0] + assert not any( + event.get("stage_code") == "ble-write-dispatched" + for event in failed_operation["events"] + ) + retired_request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000512", + ) + service._require_network_provision_transport_allowed(retired_request) # noqa: SLF001 + return + + with service._lock: # noqa: SLF001 + assert ( # noqa: SLF001 + service._connection_scenario_reset_retired_transport_authority is None + ) + assert writes == ["ORDINARY-K1-UUID"] + + repeat_scan = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + retired_request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=repeat_scan["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000512", + ) + with pytest.raises(NetworkProvisioningConflict) as denied: + asyncio.run(service.connect(retired_request)) + assert denied.value.reason_code == "physical-command-target-retired" + assert writes == ["ORDINARY-K1-UUID"] + assert len(_network_operation_documents(service)) == 1 + + +def test_reopen_commit_then_authority_cas_failure_is_no_network_dispatch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-reopen-commit-authority-cas-failure-01", + ) + ) + _install_exact_reset_scan(service, monkeypatch) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + gatt_writes: list[str] = [] + + async def no_write_after_callback_failure( + _device_id: str, + _ssid: str, + _password: str, + *, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + on_write_dispatch( + { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + }, + "with_response", + ) + gatt_writes.append("write-7f01") + raise AssertionError("callback failure must prevent the helper write") + + original_reopen = ( + service._physical_command_coordinator.reopen_retired_reconciliation # noqa: SLF001 + ) + + def reopen_then_supersede_authority(**kwargs: object) -> dict[str, object]: + reopened = original_reopen(**kwargs) # type: ignore[arg-type] + with service._lock: # noqa: SLF001 + service._connection_scenario_reset_retired_transport_authority = None # noqa: SLF001 + return reopened + + monkeypatch.setattr(facade_module, "provision_wifi_once", no_write_after_callback_failure) + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "reopen_retired_reconciliation", + reopen_then_supersede_authority, + ) + request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000521", + ) + + with pytest.raises(NetworkProvisioningConflict) as failed: + asyncio.run(service.connect(request)) + assert failed.value.reason_code == "connection-scenario-reset-superseded" + assert gatt_writes == [] + + network = service._network_mutation_ledger.snapshot() # noqa: SLF001 + assert network.status == "resolved" + assert network.record is not None + assert network.record.stage == "resolved" + assert network.record.resolution == "not-dispatched" + physical = service._physical_command_coordinator.snapshot() # noqa: SLF001 + assert physical["requires_reconciliation"] is True + record = physical["record"] + assert isinstance(record, dict) + assert record["stage"] == "observing" + assert record["resolution"] is None + assert len(record["operator_reconciliation_reopens"]) == 1 + journal = service._network_provisioning_idempotency_journal # noqa: SLF001 + assert journal is not None + exact_rows = [ + item + for item in journal.snapshot().records + if item.operation_id == request.operation_id + ] + assert len(exact_rows) == 1 + assert exact_rows[0].stage == "terminal" + assert exact_rows[0].terminal is not None + assert exact_rows[0].terminal.side_effect_status == "none" + + replay = asyncio.run(service.connect(request)) + assert replay["connection_attempt"]["status"] == "failed" + assert gatt_writes == [] + assert len(record["operator_reconciliation_reopens"]) == 1 + + +@pytest.mark.parametrize( + "dispatch_fault", + ["before-commit", "after-return", "replace-before-directory-fsync"], +) +def test_network_dispatch_marker_failure_never_causes_second_write( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + dispatch_fault: str, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id=( + "op-reset-dispatch-marker-fault-" + f"{dispatch_fault.replace('-', '_')}-01" + ), + ) + ) + _install_exact_reset_scan(service, monkeypatch) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + writes: list[str] = [] + + async def provision_once( + _device_id: str, + _ssid: str, + _password: str, + *, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + baseline = { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + } + on_write_dispatch(baseline, "with_response") + writes.append("write-7f01") + return { + "started_at_utc": "2026-08-14T08:20:00Z", + "completed_at_utc": "2026-08-14T08:20:01Z", + "profile_id": "xgrids-k1-fw3-wifi-v1", + "outcome": "lan_address_observed", + "write_mode": "with_response", + "baseline_status": baseline, + "observations": [ + { + "observed_at_utc": "2026-08-14T08:20:01Z", + "status": { + "mode": "WIFI_CLIENT", + "ipv4": "192.168.68.53", + "status_code": 1, + "reserved": 0, + "network_name": "reset-lab-network", + }, + } + ], + } + + original_mark_dispatching = service._network_mutation_ledger.mark_dispatching # noqa: SLF001 + original_fsync_directory = network_ledger_module._fsync_directory # noqa: SLF001 + fail_mark_directory_fsync = False + + def fsync_with_mark_fault(path: Path) -> None: + if fail_mark_directory_fsync: + raise OSError("synthetic replace-before-directory-fsync fault") + original_fsync_directory(path) + + monkeypatch.setattr( + network_ledger_module, + "_fsync_directory", + fsync_with_mark_fault, + ) + + def fail_mark_dispatching( + operation_id: str, + *, + expected_revision: int, + ) -> object: + nonlocal fail_mark_directory_fsync + if dispatch_fault == "before-commit": + raise RuntimeError("synthetic mark-dispatching precommit fault") + if dispatch_fault == "replace-before-directory-fsync": + fail_mark_directory_fsync = True + try: + return original_mark_dispatching( + operation_id, + expected_revision=expected_revision, + ) + finally: + fail_mark_directory_fsync = False + if dispatch_fault == "after-return": + original_mark_dispatching( + operation_id, + expected_revision=expected_revision, + ) + raise RuntimeError("synthetic mark-dispatching return-path fault") + + monkeypatch.setattr(facade_module, "provision_wifi_once", provision_once) + monkeypatch.setattr(facade_module, "_target_is_local_ipv4", lambda _target: False) + monkeypatch.setattr( + service, + "_schedule_control_bootstrap_continuation", + lambda **_kwargs: None, + ) + monkeypatch.setattr( + service._network_mutation_ledger, # noqa: SLF001 + "mark_dispatching", + fail_mark_dispatching, + ) + request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + operation_id=( + "op-00000000-0000-4000-8000-000000000532" + if dispatch_fault == "after-return" + else "op-00000000-0000-4000-8000-000000000533" + if dispatch_fault == "replace-before-directory-fsync" + else "op-00000000-0000-4000-8000-000000000531" + ), + ) + + dispatch_confirmed = dispatch_fault != "before-commit" + if dispatch_confirmed: + result = asyncio.run(service.connect(request)) + assert result["selected_device_id"] == "RESET-PREFLIGHT-K1-UUID" + assert writes == ["write-7f01"] + assert service._network_mutation_ledger.snapshot().record.resolution == ( # noqa: SLF001 + "target-observed" + ) + else: + with pytest.raises(RuntimeError, match="mark-dispatching"): + asyncio.run(service.connect(request)) + assert writes == [] + network = service._network_mutation_ledger.snapshot() # noqa: SLF001 + assert network.status == "resolved" + assert network.record is not None + assert network.record.resolution == "not-dispatched" + + physical_after_first = service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + replay = asyncio.run(service.connect(request)) + assert replay["connection_attempt"]["status"] == ( + "succeeded" if dispatch_confirmed else "failed" + ) + assert writes == (["write-7f01"] if dispatch_confirmed else []) + assert service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] == physical_after_first + + +def test_idempotency_unresolved_commit_return_fault_terminalizes_no_dispatch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-idempotency-unresolved-return-fault-01", + ) + ) + _install_exact_reset_scan(service, monkeypatch) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + writes: list[str] = [] + + async def callback_must_abort_before_write( + _device_id: str, + _ssid: str, + _password: str, + *, + on_write_dispatch: Callable[[dict[str, Any], str], None] | None = None, + **_kwargs: object, + ) -> dict[str, Any]: + assert on_write_dispatch is not None + on_write_dispatch( + { + "mode": "WIFI_AP", + "ipv4": "192.168.56.1", + "status_code": 1, + "reserved": 0, + }, + "with_response", + ) + writes.append("write-7f01") + raise AssertionError("unresolved journal fault must abort callback") + + journal = service._network_provisioning_idempotency_journal # noqa: SLF001 + assert journal is not None + original_mark_unresolved = journal.mark_unresolved + + def commit_unresolved_then_raise( + operation_id: str, + *, + expected_revision: int, + ) -> object: + original_mark_unresolved( + operation_id, + expected_revision=expected_revision, + ) + raise RuntimeError("synthetic unresolved journal return-path fault") + + monkeypatch.setattr(facade_module, "provision_wifi_once", callback_must_abort_before_write) + monkeypatch.setattr(journal, "mark_unresolved", commit_unresolved_then_raise) + request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000541", + ) + + with pytest.raises(RuntimeError, match="unresolved journal"): + asyncio.run(service.connect(request)) + assert writes == [] + network = service._network_mutation_ledger.snapshot() # noqa: SLF001 + assert network.status == "resolved" + assert network.record is not None + assert network.record.resolution == "not-dispatched" + row = next( + item + for item in journal.snapshot().records + if item.operation_id == request.operation_id + ) + assert row.stage == "terminal" + assert row.terminal is not None + assert row.terminal.side_effect_status == "none" + physical_after_fault = service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] + + replay = asyncio.run(service.connect(request.model_copy(update={"operation_id": None}))) + assert replay["connection_attempt"]["status"] == "failed" + assert writes == [] + assert service._physical_command_coordinator.snapshot()[ # noqa: SLF001 + "record" + ] == physical_after_fault + + +def test_reset_retired_uuid_old_scan_generation_fails_before_journal_or_io( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-retired-stale-generation-01", + ) + ) + _install_exact_reset_scan(service, monkeypatch) + first = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + second = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + assert second["ble_discovery_generation"] > first["ble_discovery_generation"] + operations_before = service._operations.snapshot(limit=128) # noqa: SLF001 + monkeypatch.setattr( + facade_module, + "_capture_network_intent_device", + lambda _device_id: (_ for _ in ()).throw( + AssertionError("stale generation reached capture") + ), + ) + + with pytest.raises(NetworkProvisioningConflict) as error: + asyncio.run( + service.connect( + _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=first["ble_discovery_generation"], + operation_id="op-00000000-0000-4000-8000-000000000503", + ) + ) + ) + + assert error.value.reason_code == "physical-command-target-retired" + assert service._operations.snapshot(limit=128) == operations_before # noqa: SLF001 + assert _network_operation_documents(service) == [] + + +@pytest.mark.parametrize( + "requested_transport_ref", + ["RESET-PREFLIGHT-K1-UUID", "OTHER-RETIRED-K1-UUID"], +) +def test_reset_retirement_identity_drift_or_other_retired_ref_fails_before_journal( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + requested_transport_ref: str, +) -> None: + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id="op-reset-retirement-audit-drift-01", + ) + ) + _install_exact_reset_scan( + service, + monkeypatch, + device_ids=(requested_transport_ref,), + ) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + physical = service._physical_command_coordinator.snapshot() # noqa: SLF001 + record = physical["record"] + assert isinstance(record, Mapping) + drifted_record = dict(record) + drifted_record["revision"] = int(drifted_record["revision"]) + 1 + retirements = list(drifted_record["operator_retirements"]) + future_retirement = dict(retirements[-1]) + future_retirement["retirement_id"] = ( + "future-retirement-after-old-reset-" + requested_transport_ref.lower() + ) + future_retirement["retired_transport_ref"] = requested_transport_ref + future_retirement["retired_at_utc"] = "2026-08-14T08:30:00.000Z" + retirements.append(future_retirement) + drifted_record["operator_retirements"] = retirements + drifted_physical = dict(physical) + drifted_physical["record"] = drifted_record + monkeypatch.setattr( + service._physical_command_coordinator, # noqa: SLF001 + "snapshot", + lambda: drifted_physical, + ) + operations_before = service._operations.snapshot(limit=128) # noqa: SLF001 + + with pytest.raises(NetworkProvisioningConflict) as error: + asyncio.run( + service.connect( + _reset_connect_request( + device_id=requested_transport_ref, + discovery_generation=scanned["ble_discovery_generation"], + operation_id=( + "op-00000000-0000-4000-8000-000000000504" + if requested_transport_ref.startswith("RESET") + else "op-00000000-0000-4000-8000-000000000505" + ), + ) + ) + ) + + assert error.value.reason_code == "physical-command-target-retired" + assert service._operations.snapshot(limit=128) == operations_before # noqa: SLF001 + assert _network_operation_documents(service) == [] + assert service._network_mutation_ledger.snapshot().status == "empty" # noqa: SLF001 + + +@pytest.mark.parametrize("journal_stage", ["prepared", "unresolved"]) +def test_restart_after_hidden_reopen_terminal_replays_only_exact_no_write_intent( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + journal_stage: str, +) -> None: + """Crash after the local reopen stays no-write and exact-request replayable.""" + + service = _service(monkeypatch, tmp_path) + _seed_resettable_physical_record(service) + reset = service.select_connection_mode( + _reset( + mode="bridge", + revision=0, + reset_id=f"op-reset-hidden-reopen-restart-{journal_stage}", + ) + ) + _install_exact_reset_scan(service, monkeypatch) + scanned = asyncio.run(service.scan_ble(BleScanRequest(duration_seconds=1.0))) + operation_id = ( + "op-00000000-0000-4000-8000-000000000571" + if journal_stage == "prepared" + else "op-00000000-0000-4000-8000-000000000572" + ) + request = _reset_connect_request( + device_id="RESET-PREFLIGHT-K1-UUID", + discovery_generation=scanned["ble_discovery_generation"], + mode_revision=reset["desired_connection_mode_revision"], + operation_id=operation_id, + ) + request_binding = service._network_provision_request_binding_sha256(request) # noqa: SLF001 + journal = service._require_network_provisioning_idempotency_journal() # noqa: SLF001 + admitted = journal.begin( + idempotency_key=request.idempotency_key, + action=facade_module.ACTION_NETWORK_PROVISION, + operation_id=operation_id, + request_binding_sha256=request_binding, + ) + prepared_network = service._network_mutation_ledger.prepare( # noqa: SLF001 + operation_id=operation_id, + transport_ref=request.device_id, + intended_mode="bridge", + write_mode="with_response", + baseline_status=NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=0, + ), + ) + authority = service._connection_scenario_reset_retired_transport_authority # noqa: SLF001 + assert authority is not None + reopening_id = ( + "reset-network-intent-" + + hashlib.sha256( + (operation_id + ":" + request_binding).encode("utf-8") + ).hexdigest()[:48] + ) + reopened = service._physical_command_coordinator.reopen_retired_reconciliation( # noqa: SLF001 + reopening_id=reopening_id, + expected_revision=authority.physical_record_revision, + expected_retirement_id=authority.retirement_id, + expected_transport_ref=authority.retired_transport_ref, + expected_discovery_generation=request.expected_discovery_generation, + reason="reset-network-intent-read-only-settlement", + ) + if journal_stage == "unresolved": + journal.mark_unresolved( + operation_id, + expected_revision=admitted.record.revision, + ) + + assert prepared_network.stage == "prepared" + assert reopened["stage"] == "observing" + assert len(reopened["operator_reconciliation_reopens"]) == 1 + + io_events: list[str] = [] + + async def forbidden_device_io(*_args: object, **_kwargs: object) -> object: + io_events.append("device-io") + raise AssertionError("restart/exact replay must not touch BLE or GATT") + + monkeypatch.setattr(facade_module, "provision_wifi_once", forbidden_device_io) + monkeypatch.setattr(facade_module, "read_wifi_status_once", forbidden_device_io) + restarted = _service(monkeypatch, tmp_path) + startup = restarted.state() + restarted_physical = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + restarted_journal = ( + restarted._require_network_provisioning_idempotency_journal().snapshot() # noqa: SLF001 + ) + + assert io_events == [] + assert startup["network_mutation_ledger"]["status"] == "resolved" + assert startup["network_mutation_ledger"]["resolution"] == "not-dispatched" + assert restarted_journal.active_record is None + terminal = restarted_journal.records[-1] + assert terminal.operation_id == operation_id + assert terminal.stage == "terminal" + assert terminal.terminal is not None + assert terminal.terminal.side_effect_status == "none" + assert terminal.terminal.safe_to_retry is True + assert restarted_physical is not None + assert restarted_physical.stage == "observing" + assert restarted_physical.resolution is None + assert len(restarted_physical.operator_reconciliation_reopens) == 1 + assert restarted_physical.operator_reconciliation_reopens[0].reopening_id == ( + reopening_id + ) + + replayed = asyncio.run( + restarted.connect(request.model_copy(update={"operation_id": None})) + ) + replay_operation = next( + item + for item in replayed["operations"] + if item["action"] == facade_module.ACTION_NETWORK_PROVISION + ) + assert replay_operation["operation_id"] == operation_id + assert replay_operation["status"] == "failed" + assert replay_operation["stage_code"] == "durable-terminal-replay" + assert replay_operation["error"]["side_effect_status"] == "none" + assert io_events == [] + + async def explicit_scan( + _duration_seconds: float, + *, + on_admitted: Callable[[], None] | None = None, + ) -> dict[str, Any]: + assert on_admitted is not None + on_admitted() + io_events.append("explicit-scan") + return { + "devices": [ + { + "macos_uuid": request.device_id, + "name": "XGR-K1", + "local_name": "XGR-K1", + "rssi": -40, + "k1_name_candidate": True, + } + ] + } + + monkeypatch.setattr(facade_module, "scan", explicit_scan) + fresh = asyncio.run(restarted.scan_ble(BleScanRequest(duration_seconds=1.0))) + changed_body = request.model_copy( + update={ + "ssid": "different-reset-network", + "expected_discovery_generation": fresh["ble_discovery_generation"], + } + ) + changed_key = request.model_copy( + update={ + "idempotency_key": f"different-key-{journal_stage}", + "expected_discovery_generation": fresh["ble_discovery_generation"], + } + ) + for changed in (changed_body, changed_key): + with pytest.raises(NetworkProvisioningConflict) as blocked: + asyncio.run(restarted.connect(changed)) + assert blocked.value.reason_code == "physical-command-reconciliation-required" + + final_physical = restarted._physical_command_ledger.snapshot().record # noqa: SLF001 + assert final_physical == restarted_physical + assert io_events == ["explicit-scan"] diff --git a/tests/test_xgrids_device_identity_pin_store.py b/tests/test_xgrids_device_identity_pin_store.py new file mode 100644 index 0000000..38a8b2f --- /dev/null +++ b/tests/test_xgrids_device_identity_pin_store.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import json +import os +import stat +import threading +from collections.abc import Mapping +from pathlib import Path + +import pytest + +from k1link.device_plugins.xgrids_k1 import device_identity_pin_store as store_module +from k1link.device_plugins.xgrids_k1.device_identity_pin_store import ( + DEVICE_IDENTITY_PIN_FILENAME, + DEVICE_IDENTITY_PIN_LOCK_FILENAME, + DEVICE_IDENTITY_PIN_MAX_BYTES, + DEVICE_IDENTITY_PIN_SCHEMA, + DeviceIdentityPin, + DeviceIdentityPinMismatch, + DeviceIdentityPinStore, + DeviceIdentityPinStoreCorrupt, +) + +TRANSPORT_A = "A161D9D5-C352-1069-D430-5FB0BC13F7F9" +TRANSPORT_B = "B262E0E6-D463-2170-E541-6FC1CD24A8EA" +VENDOR_DEVICE_A = "K1-DEVICE-0001" +VENDOR_DEVICE_B = "K1-DEVICE-0002" +PROFILE_ID = "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2" + + +def _configure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, Path]: + repository_root = tmp_path / "repository" + repository_root.mkdir() + data_dir = tmp_path / "private-data" + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(data_dir)) + return repository_root, data_dir + + +def _store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> DeviceIdentityPinStore: + repository_root, _data_dir = _configure(tmp_path, monkeypatch) + return DeviceIdentityPinStore(repository_root) + + +def _pin( + store: DeviceIdentityPinStore, + *, + transport_ref: str = TRANSPORT_A, + vendor_device_id: str = VENDOR_DEVICE_A, + compatibility_profile_id: str = PROFILE_ID, +) -> None: + store.pin_or_match( + transport_ref=transport_ref, + vendor_device_id=vendor_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + + +def test_first_contact_is_private_secret_free_and_survives_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure(tmp_path, monkeypatch) + store = DeviceIdentityPinStore(repository_root) + + decision = store.pin_or_match( + transport_ref=TRANSPORT_A, + vendor_device_id=VENDOR_DEVICE_A, + compatibility_profile_id=PROFILE_ID, + ) + + assert decision.created is True + assert decision.revision == 1 + assert decision.pin == DeviceIdentityPin( + transport_ref=TRANSPORT_A, + vendor_device_id=VENDOR_DEVICE_A, + compatibility_profile_id=PROFILE_ID, + ) + assert store.path == data_dir / "xgrids-k1" / DEVICE_IDENTITY_PIN_FILENAME + assert stat.S_IMODE(data_dir.stat().st_mode) == 0o700 + assert stat.S_IMODE(store.path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 + assert store.path.stat().st_nlink == 1 + lock_path = store.path.parent / DEVICE_IDENTITY_PIN_LOCK_FILENAME + assert stat.S_IMODE(lock_path.stat().st_mode) == 0o600 + assert lock_path.stat().st_nlink == 1 + + document = json.loads(store.path.read_text(encoding="utf-8")) + assert document == { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "revision": 1, + "pins": [decision.pin.as_dict()], + } + assert set(document) == {"schema_version", "revision", "pins"} + assert set(document["pins"][0]) == { + "transport_ref", + "vendor_device_id", + "compatibility_profile_id", + } + serialized = store.path.read_text(encoding="utf-8").casefold() + assert "ssid" not in serialized + assert "password" not in serialized + assert "credential" not in serialized + assert "secret" not in serialized + assert "ipv4" not in serialized + + restarted = DeviceIdentityPinStore(repository_root) + snapshot = restarted.snapshot() + assert snapshot.status == "available" + assert snapshot.revision == 1 + assert snapshot.pins == (decision.pin,) + assert snapshot.for_transport(TRANSPORT_A) == decision.pin + assert snapshot.as_public_dict() == { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "status": "available", + "revision": 1, + "pin_count": 1, + "reason_code": None, + } + assert VENDOR_DEVICE_A not in str(snapshot.as_public_dict()) + + +def test_same_exact_identity_is_read_only_and_does_not_bump_revision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + _pin(store) + before = store.path.read_bytes() + before_identity = (store.path.stat().st_dev, store.path.stat().st_ino) + + decision = store.pin_or_match( + transport_ref=TRANSPORT_A, + vendor_device_id=VENDOR_DEVICE_A, + compatibility_profile_id=PROFILE_ID, + ) + + assert decision.created is False + assert decision.revision == 1 + assert store.path.read_bytes() == before + assert (store.path.stat().st_dev, store.path.stat().st_ino) == before_identity + + +@pytest.mark.parametrize( + ("vendor_device_id", "compatibility_profile_id"), + [ + (VENDOR_DEVICE_B, PROFILE_ID), + (VENDOR_DEVICE_A, "xgrids.lixelkity-k1.incompatible.v9"), + (VENDOR_DEVICE_B, "xgrids.lixelkity-k1.incompatible.v9"), + ], +) +def test_mismatch_is_typed_and_never_overwrites_first_contact( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + vendor_device_id: str, + compatibility_profile_id: str, +) -> None: + store = _store(tmp_path, monkeypatch) + _pin(store) + before = store.path.read_bytes() + + with pytest.raises(DeviceIdentityPinMismatch) as raised: + store.pin_or_match( + transport_ref=TRANSPORT_A, + vendor_device_id=vendor_device_id, + compatibility_profile_id=compatibility_profile_id, + ) + + assert raised.value.reason_code == "device-identity-pin-mismatch" + assert raised.value.transport_ref == TRANSPORT_A + assert raised.value.expected_vendor_device_id == VENDOR_DEVICE_A + assert raised.value.observed_vendor_device_id == vendor_device_id + assert store.path.read_bytes() == before + assert store.snapshot().revision == 1 + + +def test_multiple_transport_mappings_are_canonical_and_restart_safe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + _pin( + store, + transport_ref=TRANSPORT_B, + vendor_device_id=VENDOR_DEVICE_B, + ) + _pin(store) + + snapshot = DeviceIdentityPinStore(tmp_path / "repository").snapshot() + assert snapshot.revision == 2 + assert [pin.transport_ref for pin in snapshot.pins] == [TRANSPORT_A, TRANSPORT_B] + document = json.loads(store.path.read_text(encoding="utf-8")) + assert [pin["transport_ref"] for pin in document["pins"]] == [ + TRANSPORT_A, + TRANSPORT_B, + ] + + +@pytest.mark.parametrize( + "payload", + [ + ( + b'{"schema_version":"missioncore.xgrids-k1-device-identity-pins/v1",' + b'"schema_version":"missioncore.xgrids-k1-device-identity-pins/v1"}\n' + ), + json.dumps( + { + "schema_version": "missioncore.xgrids-k1-device-identity-pins/v999", + "revision": 1, + "pins": [ + { + "transport_ref": TRANSPORT_A, + "vendor_device_id": VENDOR_DEVICE_A, + "compatibility_profile_id": PROFILE_ID, + } + ], + } + ).encode(), + json.dumps( + { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "revision": 1, + "pins": [ + { + "transport_ref": TRANSPORT_A, + "vendor_device_id": VENDOR_DEVICE_A, + "compatibility_profile_id": PROFILE_ID, + } + ], + "ssid": "must-not-be-stored", + } + ).encode(), + json.dumps( + { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "revision": 2, + "pins": [ + { + "transport_ref": TRANSPORT_A, + "vendor_device_id": VENDOR_DEVICE_A, + "compatibility_profile_id": PROFILE_ID, + } + ], + } + ).encode(), + json.dumps( + { + "schema_version": DEVICE_IDENTITY_PIN_SCHEMA, + "revision": 2, + "pins": [ + { + "transport_ref": TRANSPORT_A, + "vendor_device_id": VENDOR_DEVICE_A, + "compatibility_profile_id": PROFILE_ID, + }, + { + "transport_ref": TRANSPORT_A, + "vendor_device_id": VENDOR_DEVICE_B, + "compatibility_profile_id": PROFILE_ID, + }, + ], + } + ).encode(), + b"{" + b"x" * DEVICE_IDENTITY_PIN_MAX_BYTES + b"}", + ], +) +def test_corruption_fails_closed_without_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + payload: bytes, +) -> None: + repository_root, data_dir = _configure(tmp_path, monkeypatch) + parent = data_dir / "xgrids-k1" + parent.mkdir(mode=0o700, parents=True) + data_dir.chmod(0o700) + parent.chmod(0o700) + path = parent / DEVICE_IDENTITY_PIN_FILENAME + path.write_bytes(payload) + path.chmod(0o600) + + store = DeviceIdentityPinStore(repository_root) + assert store.snapshot().status == "corrupt" + before = path.read_bytes() + with pytest.raises(DeviceIdentityPinStoreCorrupt): + _pin(store) + assert path.read_bytes() == before + + +def test_symlink_nonregular_hardlink_and_nonprivate_file_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure(tmp_path, monkeypatch) + parent = data_dir / "xgrids-k1" + parent.mkdir(mode=0o700, parents=True) + data_dir.chmod(0o700) + parent.chmod(0o700) + path = parent / DEVICE_IDENTITY_PIN_FILENAME + target = tmp_path / "outside.json" + target.write_text("{}", encoding="utf-8") + target.chmod(0o600) + path.symlink_to(target) + + assert DeviceIdentityPinStore(repository_root).snapshot().status == "corrupt" + path.unlink() + + path.mkdir(mode=0o700) + assert DeviceIdentityPinStore(repository_root).snapshot().status == "corrupt" + path.rmdir() + + path.write_text("{}", encoding="utf-8") + path.chmod(0o600) + hardlink = tmp_path / "second-link.json" + os.link(path, hardlink) + assert DeviceIdentityPinStore(repository_root).snapshot().status == "corrupt" + hardlink.unlink() + path.unlink() + + path.write_text("{}", encoding="utf-8") + path.chmod(0o644) + assert DeviceIdentityPinStore(repository_root).snapshot().status == "corrupt" + + +def test_nonprivate_directory_and_unsafe_lock_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, data_dir = _configure(tmp_path, monkeypatch) + data_dir.mkdir(mode=0o755) + data_dir.chmod(0o755) + with pytest.raises(DeviceIdentityPinStoreCorrupt, match="permissions"): + DeviceIdentityPinStore(repository_root) + + data_dir.chmod(0o700) + parent = data_dir / "xgrids-k1" + parent.mkdir(mode=0o700) + lock_path = parent / DEVICE_IDENTITY_PIN_LOCK_FILENAME + lock_path.write_bytes(b"not-empty") + lock_path.chmod(0o600) + with pytest.raises(DeviceIdentityPinStoreCorrupt, match="stable private"): + DeviceIdentityPinStore(repository_root) + + lock_path.unlink() + target = tmp_path / "unrelated-private-file" + target.touch(mode=0o600) + lock_path.symlink_to(target) + with pytest.raises(DeviceIdentityPinStoreCorrupt, match="opened safely"): + DeviceIdentityPinStore(repository_root) + + +def test_invalid_inputs_are_rejected_without_creating_a_pin_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + + with pytest.raises(ValueError): + _pin(store, transport_ref="../../other-device") + with pytest.raises(ValueError): + _pin(store, vendor_device_id="K1 DEVICE WITH SPACES") + with pytest.raises(ValueError): + _pin(store, compatibility_profile_id="profile\npassword") + + assert store.snapshot().status == "empty" + assert store.path.exists() is False + + +def test_atomic_publication_fsyncs_file_and_parent_and_cleans_temp_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + real_fsync = store_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = store_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(store_module.os, "fsync", observe_fsync) + _pin(store) + + assert "file" in fsync_kinds + assert fsync_kinds[-1] == "directory" + assert not list(store.path.parent.glob(f".{DEVICE_IDENTITY_PIN_FILENAME}.*.tmp")) + + +def test_failed_atomic_replace_preserves_previous_mapping( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + _pin(store) + before = store.path.read_bytes() + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected replace failure") + + monkeypatch.setattr(store_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + _pin( + store, + transport_ref=TRANSPORT_B, + vendor_device_id=VENDOR_DEVICE_B, + ) + assert store.path.read_bytes() == before + assert not list(store.path.parent.glob(f".{DEVICE_IDENTITY_PIN_FILENAME}.*.tmp")) + + +def test_two_instances_serialize_first_contact_and_mismatch_without_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root, _data_dir = _configure(tmp_path, monkeypatch) + first = DeviceIdentityPinStore(repository_root) + second = DeviceIdentityPinStore(repository_root) + real_write = store_module._write_private_json_atomic + first_write_entered = threading.Event() + release_first_write = threading.Event() + second_started = threading.Event() + second_finished = threading.Event() + write_count = 0 + count_lock = threading.Lock() + results: dict[str, object] = {} + + def blocked_first_write( + path: Path, + payload: Mapping[str, object], + *, + data_dir: Path, + ) -> None: + nonlocal write_count + with count_lock: + write_count += 1 + should_block = write_count == 1 + if should_block: + first_write_entered.set() + assert release_first_write.wait(timeout=5) + real_write(path, payload, data_dir=data_dir) + + def run_first() -> None: + results["first"] = first.pin_or_match( + transport_ref=TRANSPORT_A, + vendor_device_id=VENDOR_DEVICE_A, + compatibility_profile_id=PROFILE_ID, + ) + + def run_second() -> None: + second_started.set() + try: + second.pin_or_match( + transport_ref=TRANSPORT_A, + vendor_device_id=VENDOR_DEVICE_B, + compatibility_profile_id=PROFILE_ID, + ) + except DeviceIdentityPinMismatch as exc: + results["second"] = exc + finally: + second_finished.set() + + monkeypatch.setattr(store_module, "_write_private_json_atomic", blocked_first_write) + first_thread = threading.Thread(target=run_first, daemon=True) + second_thread = threading.Thread(target=run_second, daemon=True) + first_thread.start() + assert first_write_entered.wait(timeout=5) + second_thread.start() + assert second_started.wait(timeout=5) + try: + assert second_finished.wait(timeout=0.2) is False + finally: + release_first_write.set() + first_thread.join(timeout=5) + second_thread.join(timeout=5) + + assert first_thread.is_alive() is False + assert second_thread.is_alive() is False + assert results["first"].created is True # type: ignore[union-attr] + assert isinstance(results["second"], DeviceIdentityPinMismatch) + snapshot = DeviceIdentityPinStore(repository_root).snapshot() + assert snapshot.revision == 1 + assert snapshot.for_transport(TRANSPORT_A) is not None + assert snapshot.for_transport(TRANSPORT_A).vendor_device_id == VENDOR_DEVICE_A # type: ignore[union-attr] diff --git a/tests/test_xgrids_host_diagnostics.py b/tests/test_xgrids_host_diagnostics.py new file mode 100644 index 0000000..0497194 --- /dev/null +++ b/tests/test_xgrids_host_diagnostics.py @@ -0,0 +1,367 @@ +from __future__ import annotations + +import json +import socket + +import pytest +from bleak.exc import ( + BleakBluetoothNotAvailableError, + BleakBluetoothNotAvailableReason, +) + +from k1link.device_plugins.xgrids_k1.host_diagnostics import ( + host_diagnostic_for_exception, + host_diagnostic_for_reason, + host_diagnostics_for_reasons, +) +from k1link.device_plugins.xgrids_k1.network_mutation_ledger import ( + NetworkMutationLedgerSnapshot, +) + + +@pytest.mark.parametrize( + ("reason_code", "expected_code", "expected_domain", "expected_action"), + [ + ( + "ble-permission-denied", + "host.bluetooth.permission-denied", + "corebluetooth", + "grant-bluetooth-permission", + ), + ( + "ble-adapter-powered-off", + "host.bluetooth.adapter-powered-off", + "corebluetooth", + "power-on-bluetooth", + ), + ( + "ble-adapter-unavailable", + "host.bluetooth.adapter-unavailable", + "corebluetooth", + "restore-bluetooth-adapter", + ), + ( + "ble-provisioning-timeout", + "host.bluetooth.operation-timeout", + "corebluetooth", + "explicit-retry", + ), + ( + "corewlan-authorization-denied", + "host.wifi.permission-denied", + "corewlan", + "grant-wifi-permission", + ), + ( + "wifi-interface-inactive", + "host.wifi.adapter-powered-off", + "corewlan", + "power-on-wifi", + ), + ( + "wifi-interface-unavailable", + "host.wifi.interface-unavailable", + "corewlan", + "restore-wifi-interface", + ), + ( + "network-not-found", + "host.wifi.ssid-unavailable", + "corewlan", + "join-expected-network", + ), + ( + "keychain-authorization-required", + "host.keychain.interaction-required", + "keychain", + "unlock-or-authorize-keychain", + ), + ( + "keychain-authorization-denied", + "host.keychain.permission-denied", + "keychain", + "review-keychain-access", + ), + ( + "keychain-access-failed", + "host.keychain.unavailable", + "keychain", + "unlock-or-authorize-keychain", + ), + ( + "host-route-unavailable", + "host.route.unavailable", + "route", + "inspect-host-route", + ), + ( + "tcp-connection-refused", + "host.tcp.connection-refused", + "tcp", + "verify-broker-endpoint", + ), + ( + "tcp-connection-timeout", + "host.tcp.connection-timeout", + "tcp", + "verify-broker-endpoint", + ), + ( + "mqtt_connection_timeout", + "host.mqtt.connection-timeout", + "mqtt", + "verify-broker-endpoint", + ), + ( + "mqtt_network_loop_failed", + "host.mqtt.transport-unavailable", + "mqtt", + "verify-broker-endpoint", + ), + ( + "network-mutation-ledger-corrupt", + "host.filesystem.ledger-unavailable", + "filesystem", + "inspect-local-storage", + ), + ], +) +def test_reviewed_host_reason_codes_map_to_typed_redacted_diagnostics( + reason_code: str, + expected_code: str, + expected_domain: str, + expected_action: str, +) -> None: + diagnostic = host_diagnostic_for_reason(reason_code) + + assert diagnostic is not None + document = diagnostic.as_dict() + assert document["schema_version"] == "missioncore.host-failure-diagnostic/v1" + assert document["code"] == expected_code + assert document["domain"] == expected_domain + assert document["operator_action"] == expected_action + assert document["automatic_retry"] is False + assert document["redacted"] is True + + +@pytest.mark.parametrize( + ("private_message", "expected_code"), + [ + ( + "CoreBluetooth not authorized; private device UUID 1111", + "host.bluetooth.permission-denied", + ), + ( + "CBManagerStatePoweredOff for private adapter record", + "host.bluetooth.adapter-powered-off", + ), + ( + "No Bluetooth adapter; private host path /Users/operator", + "host.bluetooth.adapter-unavailable", + ), + ], +) +def test_corebluetooth_string_classification_never_reflects_private_details( + private_message: str, + expected_code: str, +) -> None: + diagnostic = host_diagnostic_for_exception( + RuntimeError(private_message), + boundary="corebluetooth", + ) + + assert diagnostic is not None + document = diagnostic.as_dict() + assert document["code"] == expected_code + assert private_message not in json.dumps(document) + assert "1111" not in json.dumps(document) + assert "/Users/operator" not in json.dumps(document) + + +@pytest.mark.parametrize( + ("reason", "expected_code"), + [ + ( + BleakBluetoothNotAvailableReason.DENIED_BY_USER, + "host.bluetooth.permission-denied", + ), + ( + BleakBluetoothNotAvailableReason.POWERED_OFF, + "host.bluetooth.adapter-powered-off", + ), + ( + BleakBluetoothNotAvailableReason.NO_BLUETOOTH, + "host.bluetooth.adapter-unavailable", + ), + ], +) +def test_structured_bleak_adapter_failure_maps_without_message_reflection( + reason: BleakBluetoothNotAvailableReason, + expected_code: str, +) -> None: + private_message = "private CoreBluetooth registry record" + diagnostic = host_diagnostic_for_exception( + BleakBluetoothNotAvailableError(private_message, reason), + boundary="corebluetooth", + ) + + assert diagnostic is not None + assert diagnostic.code == expected_code + assert private_message not in json.dumps(diagnostic.as_dict()) + + +def test_builtin_host_errors_require_an_explicit_boundary() -> None: + assert host_diagnostic_for_exception(PermissionError("private")) is None + assert host_diagnostic_for_exception(TimeoutError("private")) is None + assert host_diagnostic_for_exception(ConnectionRefusedError("private")) is None + + filesystem = host_diagnostic_for_exception( + PermissionError("private ledger path"), + boundary="filesystem", + ) + tcp = host_diagnostic_for_exception( + TimeoutError("private broker address"), + boundary="tcp", + ) + mqtt = host_diagnostic_for_exception( + ConnectionRefusedError("private broker address"), + boundary="mqtt", + ) + + assert filesystem is not None + assert filesystem.code == "host.filesystem.permission-denied" + assert tcp is not None + assert tcp.code == "host.tcp.connection-timeout" + assert mqtt is not None + assert mqtt.code == "host.mqtt.connection-refused" + + +def test_unknown_reason_is_not_reflected_and_duplicate_diagnostics_are_collapsed() -> None: + private_reason = "private-ssid-or-ledger-path" + assert host_diagnostic_for_reason(private_reason) is None + + diagnostics = host_diagnostics_for_reasons( + "host-route-unavailable", + "host-path-unavailable", + private_reason, + ) + + assert [diagnostic.code for diagnostic in diagnostics] == ["host.route.unavailable"] + assert private_reason not in json.dumps([item.as_dict() for item in diagnostics]) + + +def test_operation_error_projects_nested_diagnostic_without_raw_exception_text() -> None: + from k1link.device_plugins.xgrids_k1.facade import _operation_error + + private_message = "CoreBluetooth not authorized for UUID private-device-17" + error = _operation_error( + RuntimeError(private_message), + category="transport", + side_effect_status="none", + host_boundary="corebluetooth", + ) + + assert error["code"] == "RuntimeError" + assert error["host_diagnostic"] == { + "schema_version": "missioncore.host-failure-diagnostic/v1", + "code": "host.bluetooth.permission-denied", + "domain": "corebluetooth", + "impact": "discovery", + "operator_action": "grant-bluetooth-permission", + "automatic_retry": False, + "redacted": True, + } + assert private_message not in json.dumps(error) + assert "private-device-17" not in json.dumps(error) + + +def test_control_session_failure_projects_keychain_diagnostic_without_message_reflection() -> None: + from k1link.device_plugins.xgrids_k1.facade import ( + _application_control_session_public_snapshot, + ) + + private_message = "private Keychain item label and account" + document = _application_control_session_public_snapshot( + { + "state": "failed", + "failure": { + "reason_code": "keychain-authorization-required", + "message": private_message, + }, + } + ) + + failure = document["failure"] + assert isinstance(failure, dict) + assert failure["host_diagnostic"] == { + "schema_version": "missioncore.host-failure-diagnostic/v1", + "code": "host.keychain.interaction-required", + "domain": "keychain", + "impact": "control", + "operator_action": "unlock-or-authorize-keychain", + "automatic_retry": False, + "redacted": True, + } + assert private_message not in json.dumps(failure["host_diagnostic"]) + + +def test_corrupt_ledger_projection_exposes_only_typed_storage_diagnostic() -> None: + from k1link.device_plugins.xgrids_k1.facade import ( + _network_mutation_ledger_public_snapshot, + ) + + document = _network_mutation_ledger_public_snapshot( + NetworkMutationLedgerSnapshot( + status="corrupt", + record=None, + reason_code="network-mutation-ledger-corrupt", + ) + ) + + assert document["mutation_allowed"] is False + assert document["diagnostic"] == { + "schema_version": "missioncore.host-failure-diagnostic/v1", + "code": "host.filesystem.ledger-unavailable", + "domain": "filesystem", + "impact": "durable-safety", + "operator_action": "inspect-local-storage", + "automatic_retry": False, + "redacted": True, + } + + +@pytest.mark.parametrize( + ("socket_error", "expected_reason", "expected_code"), + [ + ( + ConnectionRefusedError("private broker address refused"), + "tcp-connection-refused", + "host.tcp.connection-refused", + ), + ( + TimeoutError("private broker address timed out"), + "tcp-connection-timeout", + "host.tcp.connection-timeout", + ), + ], +) +def test_broker_tcp_probe_preserves_only_refused_or_timeout_class( + monkeypatch: pytest.MonkeyPatch, + socket_error: OSError, + expected_reason: str, + expected_code: str, +) -> None: + from k1link.device_plugins.xgrids_k1.facade import _probe_control_endpoint_socket + + def fail_connect(*_args: object, **_kwargs: object) -> socket.socket: + raise socket_error + + monkeypatch.setattr(socket, "create_connection", fail_connect) + result = _probe_control_endpoint_socket("192.168.68.52") + diagnostic = host_diagnostic_for_reason(result.reason_code) + + assert result.reachable is False + assert result.reason_code == expected_reason + assert diagnostic is not None + assert diagnostic.code == expected_code + assert "private broker" not in json.dumps(diagnostic.as_dict()) diff --git a/tests/test_xgrids_macos_wifi.py b/tests/test_xgrids_macos_wifi.py index 50853ec..511506e 100644 --- a/tests/test_xgrids_macos_wifi.py +++ b/tests/test_xgrids_macos_wifi.py @@ -1,13 +1,19 @@ from __future__ import annotations -import hashlib import json import os import subprocess +import sys +import threading +import time from pathlib import Path import pytest +from k1link.device_plugins.xgrids_k1.application_control_process_lease import ( + ApplicationControlProcessLease, + ApplicationControlProcessLeaseUnavailable, +) from k1link.host_network import wifi TEST_PASSWORD = "fixture-only-network-secret" @@ -15,17 +21,394 @@ TEST_PROFILE_ID = "fixture.quick-connect.v1" TEST_CREDENTIAL_SOURCE_ID = "fixture.firmware-provider.v1" -def _helper(tmp_path: Path, *, seed_compiled_cache: bool = True) -> Path: +def _helper(tmp_path: Path) -> Path: helper = tmp_path / "associate_wifi.swift" helper.write_text("// offline fixture\n", encoding="utf-8") - if seed_compiled_cache: - executable = wifi._compiled_macos_helper_path(helper) - executable.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - executable.write_bytes(b"offline compiled fixture\n") - executable.chmod(0o700) return helper +def _private_process_lease( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[Path, ApplicationControlProcessLease]: + repository_root = tmp_path / "repository" + repository_root.mkdir() + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return repository_root, ApplicationControlProcessLease.acquire(repository_root) + + +def test_association_identity_is_opaque_process_scoped_evidence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + captured: dict[str, object] = {} + live_inputs: list[bytearray] = [] + + def fake_runner(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: + assert isinstance(kwargs["input"], bytearray) + live_inputs.append(kwargs["input"]) + captured.update({"argv": argv, **kwargs, "input": bytes(kwargs["input"])}) + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_identity":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + b'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","association_evidence":"ssid+bssid"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=bytes(range(32)), + ) + + result = probe.observe("en0") + + assert result == { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "associated", + "evidence_quality": "ssid+bssid", + "continuity_proven": True, + "continuity_token": "a" * 64, + "reason_code": None, + } + request = json.loads(bytes(captured["input"]).decode("utf-8")) + assert request == { + "action": "inspect-association", + "profile_id": "host-association-inspection.v1", + "interface_name": "en0", + "continuity_key_hex": bytes(range(32)).hex(), + } + assert "continuity_key_hex" not in " ".join(captured["argv"]) + assert "ssid" not in request + assert "bssid" not in request + assert live_inputs and not any(live_inputs[0]) + + +def test_association_identity_changes_an_otherwise_identical_route_fingerprint() -> None: + base_route = "c" * 64 + first: wifi.HostWifiAssociationIdentityResult = { + "schema_version": 1, + "adapter": "CoreWLAN", + "wifi_interface": True, + "association_state": "associated", + "evidence_quality": "bssid-only", + "continuity_proven": True, + "continuity_token": "a" * 64, + "reason_code": None, + } + second: wifi.HostWifiAssociationIdentityResult = { + **first, + "continuity_token": "b" * 64, + } + + assert wifi.bind_route_fingerprint_to_wifi_association( + base_route, first + ) != wifi.bind_route_fingerprint_to_wifi_association(base_route, second) + + +def test_association_identity_reuses_only_a_subsecond_read_only_observation( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + clocks = {"monotonic": 100.0, "wall": 1_000.0} + calls = 0 + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + nonlocal calls + calls += 1 + identity = "a" * 64 if calls == 1 else "b" * 64 + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + + f'"association_identity":"{identity}",'.encode() + + b'"association_evidence":"ssid+bssid"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"c" * 32, + monotonic_clock=lambda: clocks["monotonic"], + wall_clock=lambda: clocks["wall"], + ) + + first = probe.observe("en0") + first["continuity_token"] = "f" * 64 + clocks["monotonic"] += 0.74 + clocks["wall"] += 0.74 + cached = probe.observe("en0") + + assert calls == 1 + assert cached["continuity_token"] == "a" * 64 + + clocks["monotonic"] += 0.02 + clocks["wall"] += 0.02 + refreshed = probe.observe("en0") + + assert calls == 2 + assert refreshed["continuity_token"] == "b" * 64 + + +def test_association_identity_timeout_includes_waiting_for_an_inflight_observer( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + runner_entered = threading.Event() + release_runner = threading.Event() + first_result: list[wifi.HostWifiAssociationIdentityResult] = [] + runner_calls = 0 + + def blocked_runner( + argv: list[str], + **_: object, + ) -> subprocess.CompletedProcess[bytes]: + nonlocal runner_calls + runner_calls += 1 + runner_entered.set() + assert release_runner.wait(timeout=2.0) + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_identity":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + b'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","association_evidence":"bssid-only"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=blocked_runner, + continuity_key=b"t" * 32, + max_cache_age_seconds=0, + ) + owner = threading.Thread( + target=lambda: first_result.append( + probe.observe("en0", timeout_seconds=1.0) + ) + ) + owner.start() + assert runner_entered.wait(timeout=1.0) + + started = time.monotonic() + timed_out = probe.observe("en0", timeout_seconds=0.05) + elapsed = time.monotonic() - started + + assert elapsed < 0.25 + assert timed_out["continuity_proven"] is False + assert timed_out["reason_code"] == "host-wifi-operation-timeout" + assert runner_calls == 1 + + release_runner.set() + owner.join(timeout=2.0) + assert owner.is_alive() is False + assert first_result[0]["continuity_proven"] is True + + +def test_association_identity_interface_transition_is_an_immediate_cache_barrier( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + calls = 0 + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + nonlocal calls + calls += 1 + identity = format(calls, "x") * 64 + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + + f'"association_identity":"{identity}",'.encode() + + b'"association_evidence":"ssid+bssid"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"d" * 32, + monotonic_clock=lambda: 100.0, + wall_clock=lambda: 1_000.0, + ) + + first = probe.observe("en0") + second = probe.observe("en1") + third = probe.observe("en0") + + assert calls == 3 + assert first["continuity_token"] == "1" * 64 + assert second["continuity_token"] == "2" * 64 + assert third["continuity_token"] == "3" * 64 + + +def test_association_identity_wall_clock_sleep_expires_cache_fail_closed( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + clocks = {"monotonic": 100.0, "wall": 1_000.0} + calls = 0 + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + nonlocal calls + calls += 1 + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_identity":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + b'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","association_evidence":"bssid-only"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"e" * 32, + monotonic_clock=lambda: clocks["monotonic"], + wall_clock=lambda: clocks["wall"], + ) + + probe.observe("en0") + clocks["wall"] += 10.0 + probe.observe("en0") + + assert calls == 2 + + +@pytest.mark.parametrize("max_cache_age_seconds", [-0.01, 0.751, float("nan")]) +def test_association_identity_cache_age_cannot_exceed_safety_bound( + tmp_path: Path, + max_cache_age_seconds: float, +) -> None: + with pytest.raises(ValueError, match="between 0 and 0.75 seconds"): + wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + max_cache_age_seconds=max_cache_age_seconds, + ) + + +def test_unavailable_association_identity_keeps_one_process_scoped_route_token( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_evidence":"unavailable",' + b'"reason_code":"association-identity-unavailable"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"x" * 32, + ) + + first = probe.observe("en0") + second = probe.observe("en0") + + assert first["continuity_proven"] is False + assert first["association_state"] == "unavailable" + assert first["reason_code"] == "association-identity-unavailable" + assert first["continuity_token"] == second["continuity_token"] + assert wifi.bind_route_fingerprint_to_wifi_association( + "route", first + ) == wifi.bind_route_fingerprint_to_wifi_association("route", second) + + +def test_unavailable_association_identity_changes_with_interface_scope( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":true,' + b'"association_evidence":"unavailable",' + b'"reason_code":"association-identity-unavailable"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"z" * 32, + ) + + first = probe.observe("en0") + second = probe.observe("en1") + + assert first["continuity_proven"] is False + assert second["continuity_proven"] is False + assert first["continuity_token"] != second["continuity_token"] + + +def test_non_wifi_interface_has_stable_secret_free_continuity_evidence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + + def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","wifi_interface":false,' + b'"association_identity":"dddddddddddddddddddddddddddddddd' + b'dddddddddddddddddddddddddddddddd","association_evidence":"not-wifi"}' + ), + stderr=b"", + ) + + probe = wifi.HostWifiAssociationIdentityProbe( + _helper(tmp_path), + runner=fake_runner, + continuity_key=b"y" * 32, + ) + + first = probe.observe("en7") + second = probe.observe("en7") + + assert first == second + assert first["wifi_interface"] is False + assert first["association_state"] == "not-wifi" + assert first["continuity_proven"] is True + + def test_association_exposes_only_profile_id_and_expected_ssid_to_platform_helper( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -44,7 +427,7 @@ def test_association_exposes_only_profile_id_and_expected_ssid_to_platform_helpe stdout=( b'{"ok":true,"adapter":"CoreWLAN","already_associated":false,' b'"profile_enrolled":true,"scan_attempt_count":3,' - b'"scan_elapsed_ms":1840,"credential_source":"system-wifi-keychain"}' + b'"scan_elapsed_ms":1840,"credential_source":"exact-firmware-profile"}' ), stderr=b"", ) @@ -65,23 +448,81 @@ def test_association_exposes_only_profile_id_and_expected_ssid_to_platform_helpe "profile_enrolled": True, "scan_attempt_count": 3, "scan_elapsed_ms": 1840, - "credential_source": "system-wifi-keychain", + "credential_source": "exact-firmware-profile", } assert len(calls) == 1 call = calls[0] - assert call["argv"] == [str(wifi._compiled_macos_helper_path(helper))] + assert call["argv"] == [ + "/usr/bin/xcrun", + "swift", + str(helper.resolve()), + ] + assert "swiftc" not in call["argv"] request = json.loads(bytes(call["input"]).decode("utf-8")) assert request == { "action": "associate", "profile_id": TEST_PROFILE_ID, "ssid": "XGR-OFFLINE", - "scan_timeout_seconds": 15.0, + "scan_timeout_seconds": 30.0, } assert call["check"] is False assert call["timeout"] == 180.0 assert live_inputs and not any(live_inputs[0]) +def test_ephemeral_bridge_association_uses_stdin_and_does_not_enroll_profile( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setattr(wifi.sys, "platform", "darwin") + captured: dict[str, object] = {} + live_inputs: list[bytearray] = [] + + def fake_runner(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: + assert isinstance(kwargs["input"], bytearray) + live_inputs.append(kwargs["input"]) + captured.update({"argv": argv, **kwargs, "input": bytes(kwargs["input"])}) + return subprocess.CompletedProcess( + argv, + 0, + stdout=( + b'{"ok":true,"adapter":"CoreWLAN","already_associated":false,' + b'"profile_enrolled":false,"scan_attempt_count":2,' + b'"scan_elapsed_ms":910,"credential_source":"operation-memory"}' + ), + stderr=b"", + ) + + result = wifi.associate_with_ephemeral_wifi_credentials_once( + _helper(tmp_path), + "DCEXPRESS", + TEST_PASSWORD, + runner=fake_runner, + ) + + assert result == { + "schema_version": 1, + "adapter": "CoreWLAN", + "outcome": "associated", + "already_associated": False, + "profile_enrolled": False, + "scan_attempt_count": 2, + "scan_elapsed_ms": 910, + "credential_source": "operation-memory", + } + request = json.loads(bytes(captured["input"]).decode("utf-8")) + assert request == { + "action": "associate-ephemeral", + "profile_id": "bridge-operation-memory.v1", + "ssid": "DCEXPRESS", + "password": TEST_PASSWORD, + "scan_timeout_seconds": 30.0, + } + assert TEST_PASSWORD not in " ".join(captured["argv"]) + assert "env" not in captured + assert live_inputs and not any(live_inputs[0]) + + def test_profile_store_passes_secret_only_through_stdin( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -111,6 +552,7 @@ def test_profile_store_passes_secret_only_through_stdin( assert result["outcome"] == "stored" assert TEST_PASSWORD not in " ".join(captured["argv"]) + assert "env" not in captured request = json.loads(bytes(captured["input"]).decode("utf-8")) assert request == { "action": "store-profile", @@ -149,6 +591,7 @@ def test_firmware_material_store_passes_secret_only_through_stdin( assert result["outcome"] == "stored" assert TEST_PASSWORD not in " ".join(captured["argv"]) + assert "env" not in captured request = json.loads(bytes(captured["input"]).decode("utf-8")) assert request == { "action": "store-credential-material", @@ -220,7 +663,11 @@ def test_profile_preflight_checks_only_the_expected_keychain_item( stdout=( b'{"ok":true,"adapter":"macOS Keychain","profile_available":' + encoded_available - + b"}" + + ( + b',"credential_source":"exact-firmware-profile"}' + if available + else b"}" + ) ), stderr=b"", ) @@ -236,6 +683,9 @@ def test_profile_preflight_checks_only_the_expected_keychain_item( "schema_version": 1, "adapter": "macOS Keychain", "available": available, + "credential_source": ( + "exact-firmware-profile" if available else None + ), } request = json.loads(bytes(captured["input"]).decode("utf-8")) assert request == { @@ -302,6 +752,110 @@ def test_association_reports_operator_timeout_separately_from_missing_helper( assert raised.value.helper_elapsed_ms is None +@pytest.mark.skipif(os.name != "posix", reason="helper flock inheritance is POSIX-only") +def test_fenced_host_helper_child_keeps_lifecycle_lock_after_parent_crash( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repository_root, owner = _private_process_lease(tmp_path, monkeypatch) + ready_path = tmp_path / "host-helper-ready" + release_path = tmp_path / "host-helper-release" + outcome: list[subprocess.CompletedProcess[bytes] | BaseException] = [] + parent_descriptor_closed = False + + def run_helper() -> None: + try: + outcome.append( + wifi._run_macos_helper_with_process_fence( # noqa: SLF001 + [ + sys.executable, + "-c", + ( + "import pathlib,sys,time\n" + "ready=pathlib.Path(sys.argv[1])\n" + "release=pathlib.Path(sys.argv[2])\n" + "ready.write_text('ready')\n" + "while not release.exists(): time.sleep(0.01)\n" + ), + str(ready_path), + str(release_path), + ], + request_bytes=bytearray(b"{}"), + timeout_seconds=5.0, + process_fence_descriptor_factory=( + owner.duplicate_descriptor_for_child + ), + ) + ) + except BaseException as exc: # pragma: no cover - asserted below + outcome.append(exc) + + worker = threading.Thread(target=run_helper, daemon=True) + worker.start() + try: + deadline = time.monotonic() + 5.0 + while not ready_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert ready_path.exists() + + os.close(owner._descriptor) # noqa: SLF001 + owner._released = True # noqa: SLF001 + parent_descriptor_closed = True + with pytest.raises(ApplicationControlProcessLeaseUnavailable): + ApplicationControlProcessLease.acquire(repository_root) + + release_path.touch() + worker.join(timeout=5.0) + assert not worker.is_alive() + assert len(outcome) == 1 + assert isinstance(outcome[0], subprocess.CompletedProcess) + assert outcome[0].returncode == 0 + with ApplicationControlProcessLease.acquire(repository_root): + pass + finally: + release_path.touch(exist_ok=True) + worker.join(timeout=5.0) + if not parent_descriptor_closed: + owner.release() + + +@pytest.mark.skipif(os.name != "posix", reason="helper process groups are POSIX-only") +def test_fenced_host_helper_timeout_kills_descendants_before_lock_release( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + repository_root, owner = _private_process_lease(tmp_path, monkeypatch) + descendant_code = "import time; time.sleep(30)" + group_leader_code = ( + "import os,subprocess,sys,time\n" + "fds=[]\n" + "for fd in range(3,256):\n" + " try: os.fstat(fd)\n" + " except OSError: continue\n" + " fds.append(fd)\n" + "subprocess.Popen([sys.executable,'-c',sys.argv[1]],pass_fds=tuple(fds))\n" + "time.sleep(30)\n" + ) + try: + with pytest.raises(subprocess.TimeoutExpired): + wifi._run_macos_helper_with_process_fence( # noqa: SLF001 + [sys.executable, "-c", group_leader_code, descendant_code], + request_bytes=bytearray(b"{}"), + timeout_seconds=0.2, + process_fence_descriptor_factory=( + owner.duplicate_descriptor_for_child + ), + ) + finally: + owner.release() + + # A surviving descendant inherited the lifecycle descriptor deliberately. + # Successful reacquisition therefore proves killpg + reap completed before + # the mutating helper returned its timeout. + with ApplicationControlProcessLease.acquire(repository_root): + pass + + def test_association_reports_an_unavailable_helper_separately_from_timeout( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -327,269 +881,151 @@ def test_association_reports_an_unavailable_helper_separately_from_timeout( assert raised.value.reason_code == "host-wifi-helper-unavailable" -def test_cold_helper_build_uses_source_hash_and_separate_timeout( +def test_missing_helper_is_rejected_before_starting_a_process( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, ) -> None: monkeypatch.setattr(wifi.sys, "platform", "darwin") - helper = _helper(tmp_path, seed_compiled_cache=False) - executable = wifi._compiled_macos_helper_path(helper) - calls: list[dict[str, object]] = [] + missing_helper = tmp_path / "missing-associate-wifi.swift" + runner_called = False - def fake_runner(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: - calls.append({"argv": argv, **kwargs}) - if argv[:2] == ["/usr/bin/xcrun", "swiftc"]: - assert argv[2] == str(helper.resolve()) - assert argv[3] == "-o" - assert kwargs["stdin"] == subprocess.DEVNULL - assert "input" not in kwargs - assert 0 < float(kwargs["timeout"]) <= wifi.DEFAULT_HELPER_BUILD_TIMEOUT_SECONDS - staging = Path(argv[4]) - staging.write_bytes(b"compiled fixture\n") - staging.chmod(0o700) - return subprocess.CompletedProcess(argv, 0, stdout=b"", stderr=b"") - - assert argv == [str(executable)] - assert kwargs["timeout"] == 30.0 - return subprocess.CompletedProcess( - argv, - 0, - stdout=( - b'{"ok":true,"adapter":"macOS Keychain",' - b'"profile_available":true}' - ), - stderr=b"", - ) - - result = wifi.check_wifi_profile( - helper, - TEST_PROFILE_ID, - "XGR-OFFLINE", - runner=fake_runner, - ) - - assert result["available"] is True - assert len(calls) == 2 - assert calls[0]["argv"][:2] == ["/usr/bin/xcrun", "swiftc"] - assert calls[1]["argv"] == [str(executable)] - assert executable.is_file() - assert executable.stat().st_mode & 0o111 - assert executable.name.endswith(hashlib.sha256(helper.read_bytes()).hexdigest()) - assert not list(executable.parent.glob(f".{executable.name}.*.tmp")) - - -def test_plugin_helper_cache_is_stable_under_repository_runtime(tmp_path: Path) -> None: - helper = tmp_path / "repo" / "plugins" / "xgrids-k1" / "macos" / "associate_wifi.swift" - helper.parent.mkdir(parents=True) - helper.write_text("// offline fixture\n", encoding="utf-8") - - assert wifi._helper_cache_directory(helper) == ( - tmp_path / "repo" / ".runtime" / "mission-core" / "helpers" - ) - - -def test_helper_lock_and_compile_share_one_build_deadline( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setattr(wifi.sys, "platform", "darwin") - helper = _helper(tmp_path, seed_compiled_cache=False) - monotonic_values = iter((100.0, 100.0, 104.0)) - monkeypatch.setattr(wifi.time, "monotonic", lambda: next(monotonic_values)) - monkeypatch.setattr(wifi.time, "monotonic_ns", lambda: 0) - lock_timeouts: list[float] = [] - compile_timeouts: list[float] = [] - - class FakeLock: - def __enter__(self) -> None: - return None - - def __exit__(self, *_args: object) -> None: - return None - - def fake_lock(_path: Path, *, timeout_seconds: float) -> FakeLock: - lock_timeouts.append(timeout_seconds) - return FakeLock() - - def fake_runner(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[bytes]: - if argv[:2] == ["/usr/bin/xcrun", "swiftc"]: - compile_timeouts.append(float(kwargs["timeout"])) - staging = Path(argv[4]) - staging.write_bytes(b"compiled fixture\n") - staging.chmod(0o700) - return subprocess.CompletedProcess(argv, 0, stdout=b"", stderr=b"") - return subprocess.CompletedProcess( - argv, - 0, - stdout=b'{"ok":true,"adapter":"macOS Keychain","profile_available":true}', - stderr=b"", - ) - - monkeypatch.setattr(wifi, "_exclusive_helper_build_lock", fake_lock) - result = wifi._run_macos_helper( - helper, - {"action": "check-profile"}, - timeout_seconds=3.0, - build_timeout_seconds=10.0, - runner=fake_runner, - ) - - assert result["ok"] is True - assert lock_timeouts == [10.0] - assert compile_timeouts == [6.0] - - -def test_compiled_helper_is_reused_without_recompiling( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setattr(wifi.sys, "platform", "darwin") - helper = _helper(tmp_path, seed_compiled_cache=False) - executable = wifi._compiled_macos_helper_path(helper) - compile_count = 0 - runtime_count = 0 - - def fake_runner(argv: list[str], **_: object) -> subprocess.CompletedProcess[bytes]: - nonlocal compile_count, runtime_count - if argv[:2] == ["/usr/bin/xcrun", "swiftc"]: - compile_count += 1 - staging = Path(argv[4]) - staging.write_bytes(b"compiled fixture\n") - staging.chmod(0o700) - return subprocess.CompletedProcess(argv, 0, stdout=b"", stderr=b"") - runtime_count += 1 - assert argv == [str(executable)] - return subprocess.CompletedProcess( - argv, - 0, - stdout=( - b'{"ok":true,"adapter":"macOS Keychain",' - b'"profile_available":true}' - ), - stderr=b"", - ) - - for _ in range(2): - wifi.check_wifi_profile( - helper, - TEST_PROFILE_ID, - "XGR-OFFLINE", - runner=fake_runner, - ) - - assert compile_count == 1 - assert runtime_count == 2 - - -def test_helper_build_timeout_is_separate_from_operation_timeout( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - monkeypatch.setattr(wifi.sys, "platform", "darwin") - helper = _helper(tmp_path, seed_compiled_cache=False) - seen_timeout: float | None = None - monotonic_values = iter((1_000_000_000, 1_123_000_000)) - monkeypatch.setattr(wifi.time, "monotonic_ns", lambda: next(monotonic_values)) - - def timed_out_compiler( - argv: list[str], **kwargs: object + def unexpected_runner( + _: list[str], **__: object ) -> subprocess.CompletedProcess[bytes]: - nonlocal seen_timeout - assert argv[:2] == ["/usr/bin/xcrun", "swiftc"] - seen_timeout = float(kwargs["timeout"]) - raise subprocess.TimeoutExpired(argv, timeout=seen_timeout) + nonlocal runner_called + runner_called = True + raise AssertionError("missing helper must fail before subprocess startup") with pytest.raises( wifi.HostWifiProfileError, - match="host-wifi-helper-build-timeout", + match="host-wifi-helper-missing", ) as raised: - wifi._run_macos_helper( - helper, - {"action": "check-profile"}, - timeout_seconds=3.0, - build_timeout_seconds=7.5, - runner=timed_out_compiler, - ) - - assert raised.value.reason_code == "host-wifi-helper-build-timeout" - assert raised.value.helper_stage == "compile" - assert raised.value.helper_elapsed_ms == 123 - assert seen_timeout is not None - assert 0 < seen_timeout <= 7.5 - assert not wifi._compiled_macos_helper_path(helper).exists() - - -def test_helper_build_lock_contention_is_bounded( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - fcntl = pytest.importorskip("fcntl") - lock_path = tmp_path / "helper.lock" - descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600) - fcntl.flock(descriptor, fcntl.LOCK_EX) - monotonic_values = iter((100.0, 100.02)) - monkeypatch.setattr(wifi.time, "monotonic", lambda: next(monotonic_values)) - - try: - with ( - pytest.raises(wifi.HostWifiProfileError) as raised, - wifi._exclusive_helper_build_lock( - lock_path, - timeout_seconds=0.01, - ), - ): - raise AssertionError("contended lock must not be acquired") - finally: - fcntl.flock(descriptor, fcntl.LOCK_UN) - os.close(descriptor) - - assert raised.value.reason_code == "host-wifi-helper-build-timeout" - assert raised.value.helper_stage == "compile-lock" - assert raised.value.helper_elapsed_ms == 19 - - -@pytest.mark.parametrize( - ("compiler_failure", "expected_reason_code"), - [ - ("exit", "host-wifi-helper-build-failed"), - ("unavailable", "host-wifi-helper-compiler-unavailable"), - ], -) -def test_helper_build_errors_have_sanitized_build_taxonomy( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - compiler_failure: str, - expected_reason_code: str, -) -> None: - monkeypatch.setattr(wifi.sys, "platform", "darwin") - helper = _helper(tmp_path, seed_compiled_cache=False) - - def failing_compiler( - argv: list[str], **_: object - ) -> subprocess.CompletedProcess[bytes]: - assert argv[:2] == ["/usr/bin/xcrun", "swiftc"] - if compiler_failure == "unavailable": - raise OSError(f"private diagnostic {TEST_PASSWORD}") - return subprocess.CompletedProcess( - argv, - 1, - stdout=b"", - stderr=f"private diagnostic {TEST_PASSWORD}".encode(), - ) - - with pytest.raises(wifi.HostWifiProfileError) as raised: wifi.check_wifi_profile( - helper, + missing_helper, TEST_PROFILE_ID, "XGR-OFFLINE", - runner=failing_compiler, + runner=unexpected_runner, ) - assert raised.value.reason_code == expected_reason_code - assert raised.value.helper_stage == "compile" - assert isinstance(raised.value.helper_elapsed_ms, int) - assert raised.value.helper_elapsed_ms >= 0 - assert TEST_PASSWORD not in str(raised.value) + assert raised.value.reason_code == "host-wifi-helper-missing" + assert runner_called is False + + +def test_swift_helper_source_forbids_interactive_runtime_password_fallbacks() -> None: + source_path = ( + Path(__file__).resolve().parents[1] + / "plugins" + / "xgrids-k1" + / "macos" + / "associate_wifi.swift" + ) + source = source_path.read_text(encoding="utf-8") + + for forbidden in ( + "import AppKit", + "CWKeychainFindWiFiPassword", + "loadSystemWiFiProfile", + "promptForDevicePassword", + "NSApplication.shared", + "NSAlert", + '"native-secure-prompt"', + '"system-wifi-keychain"', + ): + assert forbidden not in source + + metadata_check = source[ + source.index("private func keychainItemExists") : source.index( + "private func keychainReasonCode" + ) + ] + assert "kSecReturnAttributes as String] = true" in metadata_check + assert "kSecReturnData" not in metadata_check + assert "nonInteractiveAuthenticationContext()" in metadata_check + assert "context.interactionNotAllowed = true" in source + assert "interactionAllowed: false" in source + assert "coreWLANReasonCode(error)" in source + assert 'return "corewlan-authorization-denied"' in source + assert 'return "host-wifi-operation-timeout"' in source + ensure_profile = source[ + source.index('if request.action == "ensure-profile"') : source.index( + 'if request.action == "check-profile"' + ) + ] + assert "profile.ssid == ssid" in ensure_profile + assert 'profile.credentialSource == "exact-firmware-profile"' in ensure_profile + assert "interactionAllowed: false" in ensure_profile + check_profile = source[ + source.index('if request.action == "check-profile"') : source.index( + 'guard request.action == "associate"' + ) + ] + assert "profile.ssid == expectedSSID" in check_profile + assert 'profile.credentialSource == "exact-firmware-profile"' in check_profile + assert "interactionAllowed: false" in check_profile + + association_inspection = source[ + source.index('if request.action == "inspect-association"') : source.index( + 'if request.action == "store-profile"' + ) + ] + assert "interface.ssid()" in association_inspection + assert "interface.bssid()" in association_inspection + assert "associationIdentity(" in association_inspection + assert "ssid: currentSSID" not in association_inspection + assert "bssid: currentBSSID" in association_inspection + # Raw network identifiers remain local variables inside the helper. The + # response contract exports only an opaque keyed identity and quality. + response_contract = source[ + source.index("private struct HostWifiResponse") : source.index( + "private func emit" + ) + ] + assert "let ssid" not in response_contract + assert "let bssid" not in response_contract + assert "associationIdentity" in response_contract + assert "associationEvidence" in response_contract + ephemeral_start = source.index('if request.action == "associate-ephemeral"') + ephemeral_association = source[ + ephemeral_start : source.index(" } else {", ephemeral_start) + ] + assert 'credentialSource = "operation-memory"' in ephemeral_association + assert "storeProfile(" not in ephemeral_association + assert "loadProfile(" not in ephemeral_association + + +def test_swift_association_identity_uses_bssid_not_ssid_visibility() -> None: + source_path = ( + Path(__file__).resolve().parents[1] + / "plugins" + / "xgrids-k1" + / "macos" + / "associate_wifi.swift" + ) + source = source_path.read_text(encoding="utf-8") + identity_function = source[ + source.index("private func associationIdentity(") : source.index( + "private let input" + ) + ] + association_inspection = source[ + source.index('if request.action == "inspect-association"') : source.index( + 'if request.action == "store-profile"' + ) + ] + + # With the same interface+BSSID, switching between ssid+bssid and + # bssid-only evidence feeds identical material to the HMAC. SSID remains a + # local evidence-quality signal and never changes the continuity token. + assert "mission-core/host-wifi-association/v2" in identity_function + assert " ssid: String" not in identity_function + assert "appendLengthPrefixed(ssid" not in identity_function + assert "appendLengthPrefixed(interfaceName" in identity_function + assert "appendLengthPrefixed(bssid.lowercased()" in identity_function + assert "interface.ssid()" in association_inspection + assert '"bssid-only"' in association_inspection + assert '"ssid+bssid"' in association_inspection + assert "ssid: currentSSID" not in association_inspection + assert "bssid: currentBSSID" in association_inspection def test_association_rejects_an_uninstalled_platform( diff --git a/tests/test_xgrids_network_mutation_ledger.py b/tests/test_xgrids_network_mutation_ledger.py new file mode 100644 index 0000000..f630da6 --- /dev/null +++ b/tests/test_xgrids_network_mutation_ledger.py @@ -0,0 +1,781 @@ +from __future__ import annotations + +import json +import stat +import threading +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from k1link.device_plugins.xgrids_k1 import network_mutation_ledger as ledger_module +from k1link.device_plugins.xgrids_k1.network_mutation_ledger import ( + NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA, + NETWORK_MUTATION_LEDGER_SCHEMA, + NetworkMutationBlocked, + NetworkMutationLedger, + NetworkMutationLedgerCorrupt, + NetworkMutationTransitionError, + NetworkStatusEvidence, + PreviousConnectionEvidence, +) + +OPERATION_ID = "op-11111111-1111-4111-8111-111111111111" +SECOND_OPERATION_ID = "op-22222222-2222-4222-8222-222222222222" +TRANSPORT_REF = "A161D9D5-C352-1069-D430-5FB0BC13F7F9" + + +def _clock() -> datetime: + return datetime(2026, 8, 6, 12, 30, tzinfo=UTC) + + +def _baseline() -> NetworkStatusEvidence: + return NetworkStatusEvidence( + mode="WIFI_AP", + ipv4="192.168.56.1", + status_code=1, + reserved=1, + ) + + +def _target() -> NetworkStatusEvidence: + return NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.50", + status_code=1, + reserved=0, + ) + + +def _later_target() -> NetworkStatusEvidence: + return NetworkStatusEvidence( + mode="WIFI_CLIENT", + ipv4="192.168.68.51", + status_code=1, + reserved=0, + ) + + +def _ledger(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> NetworkMutationLedger: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return NetworkMutationLedger(tmp_path / "repository", clock=_clock) + + +def _prepare(ledger: NetworkMutationLedger, *, operation_id: str = OPERATION_ID) -> None: + ledger.prepare( + operation_id=operation_id, + transport_ref=TRANSPORT_REF, + intended_mode="bridge", + write_mode="with_response", + baseline_status=_baseline(), + previous_connection=PreviousConnectionEvidence( + transport_ref=TRANSPORT_REF, + mode="quick-connect", + ipv4="192.168.56.1", + device_session_id="device-session-11111111-1111-4111-8111-111111111111", + ), + ) + + +def test_ledger_uses_private_data_path_and_secret_free_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + + expected_data_dir = tmp_path / "private-data" + assert ledger.path == expected_data_dir / "xgrids-k1" / "network-mutation.json" + assert stat.S_IMODE(expected_data_dir.stat().st_mode) == 0o700 + assert stat.S_IMODE(ledger.path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(ledger.path.stat().st_mode) == 0o600 + + document = json.loads(ledger.path.read_text(encoding="utf-8")) + assert document["schema_version"] == NETWORK_MUTATION_LEDGER_SCHEMA + assert set(document) == { + "schema_version", + "revision", + "operation_id", + "transport_ref", + "intended_mode", + "stage", + "write_mode", + "baseline_status", + "previous_connection", + "write_confirmed", + "last_observation", + "resolution", + "created_at_utc", + "updated_at_utc", + } + assert set(document["previous_connection"]) == { + "transport_ref", + "mode", + "ipv4", + "device_session_id", + } + serialized = ledger.path.read_text(encoding="utf-8").casefold() + assert "ssid" not in serialized + assert "password" not in serialized + assert "credential" not in serialized + assert "payload" not in serialized + + +def test_unresolved_record_survives_restart_and_resolved_record_allows_next_write( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + + restarted = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + snapshot = restarted.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.stage == "prepared" + assert snapshot.record.revision == 1 + assert snapshot.mutation_allowed is False + with pytest.raises(NetworkMutationBlocked): + restarted.require_mutation_allowed() + + dispatching = restarted.mark_dispatching(OPERATION_ID, expected_revision=1) + assert dispatching.stage == "dispatching" + assert dispatching.revision == 2 + observing = restarted.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=True, + observation=_target(), + ) + assert observing.stage == "observing" + assert observing.revision == 3 + resolved = restarted.resolve( + OPERATION_ID, + expected_revision=observing.revision, + resolution="target-observed", + ) + assert resolved.stage == "resolved" + assert resolved.resolution == "target-observed" + assert resolved.revision == 4 + + after_resolution_restart = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + final_snapshot = after_resolution_restart.snapshot() + assert final_snapshot.status == "resolved" + assert final_snapshot.mutation_allowed is True + after_resolution_restart.require_mutation_allowed() + next_record = after_resolution_restart.prepare( + operation_id=SECOND_OPERATION_ID, + transport_ref=TRANSPORT_REF, + intended_mode="quick-connect", + write_mode="with_response", + baseline_status=_target(), + ) + assert next_record.revision == 5 + assert next_record.stage == "prepared" + + +@pytest.mark.parametrize("resolution", ["interrupted", "superseded"]) +def test_dispatched_session_can_be_terminalized_as_audit_without_status_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + resolution: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + dispatching = ledger.mark_dispatching(OPERATION_ID, expected_revision=1) + + terminal = ledger.resolve( + OPERATION_ID, + expected_revision=dispatching.revision, + resolution=resolution, # type: ignore[arg-type] + ) + + assert terminal.stage == "resolved" + assert terminal.resolution == resolution + assert terminal.last_observation is None + assert ledger.snapshot().mutation_allowed is True + + +def test_wall_clock_rollback_never_corrupts_serialized_network_stages( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ticks = iter(datetime(2026, 8, 6, hour, tzinfo=UTC) for hour in (12, 11, 10, 9)) + ledger = NetworkMutationLedger( + tmp_path / "repository", + clock=lambda: next(ticks), + ) + + def assert_restart(*, stage: str, revision: int) -> None: + snapshot = NetworkMutationLedger(tmp_path / "repository", clock=_clock).snapshot() + assert snapshot.record is not None + assert snapshot.record.stage == stage + assert snapshot.record.revision == revision + assert snapshot.record.created_at_utc == "2026-08-06T12:00:00.000Z" + assert snapshot.record.updated_at_utc == "2026-08-06T12:00:00.000Z" + + _prepare(ledger) + assert_restart(stage="prepared", revision=1) + dispatching = ledger.mark_dispatching(OPERATION_ID, expected_revision=1) + assert_restart(stage="dispatching", revision=2) + ledger.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=True, + observation=_target(), + ) + assert_restart(stage="observing", revision=3) + ledger.resolve( + OPERATION_ID, + expected_revision=3, + resolution="target-observed", + ) + assert_restart(stage="resolved", revision=4) + + +def test_dispatch_persist_failure_reaches_caller_before_ble_side_effect( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + before = ledger.path.read_bytes() + ble_write_called = False + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected pre-dispatch persistence failure") + + def ble_caller() -> None: + nonlocal ble_write_called + ledger.mark_dispatching(OPERATION_ID, expected_revision=1) + ble_write_called = True + + monkeypatch.setattr(ledger_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="pre-dispatch persistence failure"): + ble_caller() + + assert ble_write_called is False + assert ledger.path.read_bytes() == before + snapshot = NetworkMutationLedger(tmp_path / "repository", clock=_clock).snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.stage == "prepared" + + +def test_uncertain_dispatch_confirmation_refsyncs_exact_current_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + prepared = ledger.prepare( + operation_id=OPERATION_ID, + transport_ref=TRANSPORT_REF, + intended_mode="bridge", + write_mode="with_response", + baseline_status=_baseline(), + ) + dispatching = ledger.mark_dispatching( + OPERATION_ID, + expected_revision=prepared.revision, + ) + fsynced_directories: list[Path] = [] + real_fsync_directory = ledger_module._fsync_directory + + def observe_fsync(path: Path) -> None: + fsynced_directories.append(path) + real_fsync_directory(path) + + monkeypatch.setattr(ledger_module, "_fsync_directory", observe_fsync) + + confirmed = ledger.confirm_dispatching_after_uncertain_return( + OPERATION_ID, + expected_prepared=prepared, + ) + + assert confirmed == dispatching + assert confirmed.revision == prepared.revision + 1 + assert confirmed.stage == "dispatching" + assert fsynced_directories == [ledger.path.parent] + restarted = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == dispatching + + +@pytest.mark.parametrize("drift", ["predecessor", "current"]) +def test_uncertain_dispatch_confirmation_rejects_predecessor_or_current_drift( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + drift: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + prepared = ledger.prepare( + operation_id=OPERATION_ID, + transport_ref=TRANSPORT_REF, + intended_mode="bridge", + write_mode="with_response", + baseline_status=_baseline(), + ) + dispatching = ledger.mark_dispatching( + OPERATION_ID, + expected_revision=prepared.revision, + ) + expected = prepared + if drift == "predecessor": + expected = replace(prepared, transport_ref="different-k1") + else: + ledger.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=False, + ) + before = ledger.path.read_bytes() + + with pytest.raises( + NetworkMutationTransitionError, + match="does not match its predecessor", + ): + ledger.confirm_dispatching_after_uncertain_return( + OPERATION_ID, + expected_prepared=expected, + ) + + assert ledger.path.read_bytes() == before + + +def test_secret_free_legacy_v1_record_migrates_to_v2_without_changing_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + before = json.loads(ledger.path.read_text(encoding="utf-8")) + before["schema_version"] = NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA + ledger.path.write_text( + json.dumps(before, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + + restarted = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + + snapshot = restarted.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.schema_version == NETWORK_MUTATION_LEDGER_SCHEMA + assert snapshot.record.operation_id == OPERATION_ID + assert snapshot.record.previous_connection is not None + assert snapshot.record.previous_connection.transport_ref == TRANSPORT_REF + migrated = json.loads(restarted.path.read_text(encoding="utf-8")) + assert migrated["schema_version"] == NETWORK_MUTATION_LEDGER_SCHEMA + assert migrated["revision"] == before["revision"] + + +def test_ambiguous_legacy_v1_previous_connection_fails_closed_and_is_preserved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + document = json.loads(ledger.path.read_text(encoding="utf-8")) + document["schema_version"] = NETWORK_MUTATION_LEDGER_LEGACY_SCHEMA + assert isinstance(document["previous_connection"], dict) + document["previous_connection"].pop("transport_ref") + payload = (json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n").encode() + ledger.path.write_bytes(payload) + ledger.path.chmod(0o600) + + restarted = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + + assert restarted.snapshot().status == "corrupt" + with pytest.raises(NetworkMutationLedgerCorrupt): + restarted.require_mutation_allowed() + assert restarted.path.read_bytes() == payload + + +def test_resolution_contract_does_not_invent_no_side_effect_after_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + + with pytest.raises(NetworkMutationTransitionError, match="target-observed"): + ledger.resolve( + OPERATION_ID, + expected_revision=1, + resolution="target-observed", + ) + + ledger.mark_dispatching(OPERATION_ID, expected_revision=1) + with pytest.raises(NetworkMutationTransitionError, match="not-dispatched"): + ledger.resolve( + OPERATION_ID, + expected_revision=2, + resolution="not-dispatched", + ) + with pytest.raises(NetworkMutationTransitionError, match="status evidence"): + ledger.resolve( + OPERATION_ID, + expected_revision=2, + resolution="target-observed", + ) + + assert ledger.snapshot().status == "unresolved" + + +def test_prepared_record_can_be_explicitly_resolved_as_not_dispatched( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + + resolved = ledger.resolve( + OPERATION_ID, + expected_revision=1, + resolution="not-dispatched", + ) + assert resolved.stage == "resolved" + assert resolved.resolution == "not-dispatched" + assert resolved.write_confirmed is None + assert resolved.last_observation is None + ledger.require_mutation_allowed() + + +@pytest.mark.parametrize( + "corrupt_payload", + [ + b"{corrupt\n", + json.dumps( + { + "schema_version": NETWORK_MUTATION_LEDGER_SCHEMA, + "password": "must-never-enter-ledger", + } + ).encode("utf-8"), + ], +) +def test_corrupt_or_noncanonical_record_fails_closed_without_being_overwritten( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + corrupt_payload: bytes, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + path = tmp_path / "private-data" / "xgrids-k1" / "network-mutation.json" + path.parent.mkdir(mode=0o700, parents=True) + path.parent.chmod(0o700) + path.write_bytes(corrupt_payload) + path.chmod(0o600) + before = path.read_bytes() + + ledger = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + snapshot = ledger.snapshot() + assert snapshot.status == "corrupt" + assert snapshot.reason_code == "network-mutation-ledger-corrupt" + assert snapshot.mutation_allowed is False + with pytest.raises(NetworkMutationLedgerCorrupt): + ledger.require_mutation_allowed() + with pytest.raises(NetworkMutationLedgerCorrupt): + _prepare(ledger) + assert path.read_bytes() == before + + +def test_atomic_publication_fsyncs_file_and_parent_and_cleans_temporary_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + real_fsync = ledger_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = ledger_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(ledger_module.os, "fsync", observe_fsync) + _prepare(ledger) + + assert "file" in fsync_kinds + assert "directory" in fsync_kinds + assert fsync_kinds.index("file") < len(fsync_kinds) - 1 + assert fsync_kinds[-1] == "directory" + assert not list(ledger.path.parent.glob(".*.tmp")) + + +def test_failed_atomic_replace_preserves_previous_resolved_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare(ledger) + ledger.resolve( + OPERATION_ID, + expected_revision=1, + resolution="not-dispatched", + ) + before = ledger.path.read_bytes() + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected replace failure") + + monkeypatch.setattr(ledger_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + ledger.prepare( + operation_id=SECOND_OPERATION_ID, + transport_ref=TRANSPORT_REF, + intended_mode="quick-connect", + write_mode="with_response", + baseline_status=_target(), + ) + + assert ledger.path.read_bytes() == before + assert not list(ledger.path.parent.glob(".*.tmp")) + + +def test_stale_same_operation_revision_cannot_mark_dispatching( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = _ledger(tmp_path, monkeypatch) + _prepare(current) + stale = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + + dispatching = current.mark_dispatching(OPERATION_ID, expected_revision=1) + + with pytest.raises(NetworkMutationTransitionError, match="stale record revision"): + stale.mark_dispatching(OPERATION_ID, expected_revision=1) + + persisted = stale.snapshot().record + assert persisted == dispatching + assert persisted is not None + assert persisted.stage == "dispatching" + assert persisted.revision == 2 + + +def test_stale_same_operation_revision_cannot_mark_observing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = _ledger(tmp_path, monkeypatch) + _prepare(current) + dispatching = current.mark_dispatching(OPERATION_ID, expected_revision=1) + stale = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + + observing = current.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=True, + observation=_target(), + ) + + with pytest.raises(NetworkMutationTransitionError, match="stale record revision"): + stale.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=False, + observation=_later_target(), + ) + + persisted = stale.snapshot().record + assert persisted == observing + assert persisted is not None + assert persisted.stage == "observing" + assert persisted.revision == 3 + assert persisted.write_confirmed is True + assert persisted.last_observation == _target() + + +def test_stale_same_operation_revision_cannot_resolve( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = _ledger(tmp_path, monkeypatch) + _prepare(current) + dispatching = current.mark_dispatching(OPERATION_ID, expected_revision=1) + first_observation = current.mark_observing( + OPERATION_ID, + expected_revision=dispatching.revision, + write_confirmed=False, + observation=_target(), + ) + stale = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + latest = current.mark_observing( + OPERATION_ID, + expected_revision=first_observation.revision, + write_confirmed=True, + observation=_later_target(), + ) + + with pytest.raises(NetworkMutationTransitionError, match="stale record revision"): + stale.resolve( + OPERATION_ID, + expected_revision=first_observation.revision, + resolution="target-observed", + ) + + persisted = stale.snapshot().record + assert persisted == latest + assert persisted is not None + assert persisted.stage == "observing" + assert persisted.revision == 4 + assert persisted.resolution is None + assert persisted.last_observation == _later_target() + + +def test_two_instances_cannot_prepare_from_the_same_revision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + second = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + real_write = ledger_module._write_private_json_atomic + first_write_entered = threading.Event() + release_first_write = threading.Event() + second_started = threading.Event() + second_finished = threading.Event() + write_calls: list[str] = [] + call_lock = threading.Lock() + outcomes: dict[str, BaseException | None] = {} + + def blocked_first_write( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + with call_lock: + write_calls.append(str(payload["operation_id"])) + is_first_write = len(write_calls) == 1 + if is_first_write: + first_write_entered.set() + assert release_first_write.wait(timeout=5) + real_write(path, payload, data_dir=data_dir) + + def prepare_in_thread( + name: str, + ledger: NetworkMutationLedger, + operation_id: str, + *, + started: threading.Event | None = None, + finished: threading.Event | None = None, + ) -> None: + if started is not None: + started.set() + try: + _prepare(ledger, operation_id=operation_id) + except BaseException as exc: # captured for assertion in the test thread + outcomes[name] = exc + else: + outcomes[name] = None + finally: + if finished is not None: + finished.set() + + monkeypatch.setattr(ledger_module, "_write_private_json_atomic", blocked_first_write) + first_thread = threading.Thread( + target=prepare_in_thread, + args=("first", first, OPERATION_ID), + daemon=True, + ) + second_thread = threading.Thread( + target=prepare_in_thread, + args=("second", second, SECOND_OPERATION_ID), + kwargs={"started": second_started, "finished": second_finished}, + daemon=True, + ) + first_thread.start() + assert first_write_entered.wait(timeout=5) + second_thread.start() + assert second_started.wait(timeout=5) + try: + assert second_finished.wait(timeout=0.2) is False + assert write_calls == [OPERATION_ID] + finally: + release_first_write.set() + first_thread.join(timeout=5) + second_thread.join(timeout=5) + + assert first_thread.is_alive() is False + assert second_thread.is_alive() is False + assert outcomes["first"] is None + assert isinstance(outcomes["second"], NetworkMutationBlocked) + assert write_calls == [OPERATION_ID] + snapshot = second.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.operation_id == OPERATION_ID + assert snapshot.record.revision == 1 + + +def test_second_instance_snapshot_waits_for_in_flight_prepare_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + _prepare(first) + first.resolve( + OPERATION_ID, + expected_revision=1, + resolution="not-dispatched", + ) + second = NetworkMutationLedger(tmp_path / "repository", clock=_clock) + real_write = ledger_module._write_private_json_atomic + prepare_write_entered = threading.Event() + release_prepare_write = threading.Event() + snapshot_started = threading.Event() + snapshot_finished = threading.Event() + outcomes: dict[str, object] = {} + + def blocked_prepare_write( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + prepare_write_entered.set() + assert release_prepare_write.wait(timeout=5) + real_write(path, payload, data_dir=data_dir) + + def prepare_next() -> None: + try: + _prepare(first, operation_id=SECOND_OPERATION_ID) + except BaseException as exc: # captured for assertion in the test thread + outcomes["prepare"] = exc + else: + outcomes["prepare"] = None + + def read_snapshot() -> None: + snapshot_started.set() + try: + outcomes["snapshot"] = second.snapshot() + except BaseException as exc: # captured for assertion in the test thread + outcomes["snapshot"] = exc + finally: + snapshot_finished.set() + + monkeypatch.setattr(ledger_module, "_write_private_json_atomic", blocked_prepare_write) + prepare_thread = threading.Thread(target=prepare_next, daemon=True) + snapshot_thread = threading.Thread(target=read_snapshot, daemon=True) + prepare_thread.start() + assert prepare_write_entered.wait(timeout=5) + snapshot_thread.start() + assert snapshot_started.wait(timeout=5) + try: + assert snapshot_finished.wait(timeout=0.2) is False + finally: + release_prepare_write.set() + prepare_thread.join(timeout=5) + snapshot_thread.join(timeout=5) + + assert prepare_thread.is_alive() is False + assert snapshot_thread.is_alive() is False + assert outcomes["prepare"] is None + snapshot = outcomes["snapshot"] + assert isinstance(snapshot, ledger_module.NetworkMutationLedgerSnapshot) + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.operation_id == SECOND_OPERATION_ID + assert snapshot.record.revision == 3 + with pytest.raises(NetworkMutationBlocked): + second.require_mutation_allowed() diff --git a/tests/test_xgrids_network_provisioning_idempotency_journal.py b/tests/test_xgrids_network_provisioning_idempotency_journal.py new file mode 100644 index 0000000..c49039e --- /dev/null +++ b/tests/test_xgrids_network_provisioning_idempotency_journal.py @@ -0,0 +1,815 @@ +from __future__ import annotations + +import hashlib +import json +import multiprocessing +import os +import stat +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest + +from k1link.device_plugins.xgrids_k1 import ( + network_provisioning_idempotency_journal as journal_module, +) +from k1link.device_plugins.xgrids_k1.network_provisioning_idempotency_journal import ( + NetworkProvisioningIdempotencyBlocked, + NetworkProvisioningIdempotencyConflict, + NetworkProvisioningIdempotencyCorrupt, + NetworkProvisioningIdempotencyJournal, + NetworkProvisioningIdempotencyTransitionError, + NetworkProvisioningLegacyAdoptionProvenance, + NetworkProvisioningTerminalMetadata, + derive_request_binding_sha256, +) + +ACTION = "network.provision" +KEY = "network-provision:31d482b6-77ff-4cd4-a4b5-2bc565c21cb0" +OPERATION_ID = "op-network-001" + + +def _clock() -> datetime: + return datetime(2026, 8, 7, 0, 0, tzinfo=UTC) + + +def _binding( + key: str = KEY, + *, + device_id: str = "device-k1-owned", + password: str = "never-persist-this-password", +) -> str: + canonical = json.dumps( + { + "connection_mode": "bridge", + "device_id": device_id, + "password": password, + "ssid": "never-persist-this-ssid", + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return derive_request_binding_sha256( + key, + action=ACTION, + canonical_request=canonical, + ) + + +def _success( + *, + side_effect_status: str = "applied", +) -> NetworkProvisioningTerminalMetadata: + return NetworkProvisioningTerminalMetadata( + outcome="succeeded", + outcome_code="network.provision.completed", + error_code=None, + side_effect_status=side_effect_status, # type: ignore[arg-type] + retryable=False, + safe_to_retry=False, + ) + + +def _failure_no_side_effect() -> NetworkProvisioningTerminalMetadata: + return NetworkProvisioningTerminalMetadata( + outcome="failed", + outcome_code="network.provision.failed", + error_code="host.precondition.failed", + side_effect_status="none", + retryable=True, + safe_to_retry=True, + ) + + +def _legacy_provenance( + *, + operation_id: str = "legacy-network-operation", + transport_ref: str = "legacy-k1-transport", +) -> NetworkProvisioningLegacyAdoptionProvenance: + return NetworkProvisioningLegacyAdoptionProvenance( + source_schema_version="missioncore.xgrids-k1-network-mutation/v2", + operation_id=operation_id, + transport_ref=transport_ref, + intended_mode="bridge", + write_mode="with_response", + baseline_status_sha256=hashlib.sha256(b"legacy-baseline-evidence").hexdigest(), + previous_connection_sha256=None, + created_at_utc="2026-08-06T17:00:00.000Z", + ) + + +def _journal( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + max_terminal_records: int = 64, + clock: Any = _clock, +) -> NetworkProvisioningIdempotencyJournal: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return NetworkProvisioningIdempotencyJournal( + tmp_path / "repository", + max_terminal_records=max_terminal_records, + clock=clock, + ) + + +def _complete_success( + journal: NetworkProvisioningIdempotencyJournal, + *, + key: str, + operation_id: str, + device_id: str, +) -> None: + admission = journal.begin( + idempotency_key=key, + action=ACTION, + operation_id=operation_id, + request_binding_sha256=_binding(key, device_id=device_id), + ) + unresolved = journal.mark_unresolved( + operation_id, + expected_revision=admission.record.revision, + ) + journal.complete( + operation_id, + expected_revision=unresolved.revision, + terminal=_success(), + ) + + +def _cross_process_admit( + repository_root: str, + data_dir: str, + key: str, + operation_id: str, + start: Any, + results: Any, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + journal = NetworkProvisioningIdempotencyJournal(Path(repository_root), clock=_clock) + start.wait(timeout=10) + try: + admission = journal.begin( + idempotency_key=key, + action=ACTION, + operation_id=operation_id, + request_binding_sha256=_binding(key, device_id=operation_id), + ) + except NetworkProvisioningIdempotencyBlocked: + results.put(("blocked", operation_id)) + else: + results.put((admission.disposition, operation_id)) + + +def _cross_process_replay( + repository_root: str, + data_dir: str, + results: Any, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + journal = NetworkProvisioningIdempotencyJournal(Path(repository_root), clock=_clock) + admission = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id="op-new-process-generated-id", + request_binding_sha256=_binding(), + ) + terminal = admission.record.terminal + results.put( + ( + admission.disposition, + admission.record.operation_id, + terminal.outcome if terminal is not None else None, + ) + ) + + +def _cross_process_adopt_legacy( + repository_root: str, + data_dir: str, + operation_id: str, + start: Any, + results: Any, +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + journal = NetworkProvisioningIdempotencyJournal(Path(repository_root), clock=_clock) + start.wait(timeout=10) + try: + adopted = journal.adopt_legacy_unresolved( + action=ACTION, + provenance=_legacy_provenance(operation_id=operation_id), + ) + except NetworkProvisioningIdempotencyBlocked: + results.put(("blocked", operation_id)) + else: + results.put((adopted.stage, operation_id)) + + +def test_journal_persists_only_hashes_and_private_files( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + + admission = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + + document = journal.path.read_text(encoding="utf-8") + assert admission.disposition == "admitted" + assert KEY not in document + assert "never-persist-this-ssid" not in document + assert "never-persist-this-password" not in document + assert stat.S_IMODE(journal.path.stat().st_mode) == 0o600 + assert journal.path.stat().st_nlink == 1 + assert stat.S_IMODE(journal.path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(journal._process_lock_path.stat().st_mode) == 0o600 # noqa: SLF001 + assert journal._process_lock_path.stat().st_nlink == 1 # noqa: SLF001 + + +def test_legacy_adoption_atomically_publishes_direct_unresolved_identity( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + provenance = _legacy_provenance() + + adopted = journal.adopt_legacy_unresolved( + action=ACTION, + provenance=provenance, + ) + persisted = journal.path.read_text(encoding="utf-8") + + assert adopted.operation_id == provenance.operation_id + assert adopted.stage == "unresolved" + assert adopted.terminal is None + assert adopted.created_revision == 1 + assert adopted.revision == 1 + assert "prepared" not in persisted + assert provenance.transport_ref not in persisted + assert provenance.created_at_utc not in persisted + assert adopted.idempotency_key_sha256 in persisted + assert adopted.request_binding_sha256 in persisted + + # A crash after atomic replace but before returning is retried exactly, + # without creating a competing record or advancing durable revision. + restarted = NetworkProvisioningIdempotencyJournal( + tmp_path / "repository", + clock=_clock, + ) + replayed = restarted.adopt_legacy_unresolved( + action=ACTION, + provenance=provenance, + ) + assert replayed == adopted + assert restarted.snapshot().revision == 1 + assert len(restarted.snapshot().records) == 1 + + +def test_legacy_adoption_refuses_competing_active_and_operation_rebinding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + active = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + + with pytest.raises(NetworkProvisioningIdempotencyBlocked): + journal.adopt_legacy_unresolved( + action=ACTION, + provenance=_legacy_provenance(), + ) + assert journal.snapshot().active_record == active.record + + journal.complete( + OPERATION_ID, + expected_revision=active.record.revision, + terminal=_failure_no_side_effect(), + ) + first = journal.adopt_legacy_unresolved( + action=ACTION, + provenance=_legacy_provenance(), + ) + assert first.stage == "unresolved" + with pytest.raises(NetworkProvisioningIdempotencyConflict): + journal.adopt_legacy_unresolved( + action=ACTION, + provenance=_legacy_provenance(transport_ref="different-legacy-transport"), + ) + snapshot = journal.snapshot() + assert snapshot.active_record == first + assert len(snapshot.records) == 2 + + +def test_failed_atomic_legacy_adoption_leaves_no_prepared_or_partial_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + + def fail_replace(_source: object, _target: object) -> None: + raise OSError("simulated legacy adoption publication failure") + + with monkeypatch.context() as publication_failure: + publication_failure.setattr(journal_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="legacy adoption publication failure"): + journal.adopt_legacy_unresolved( + action=ACTION, + provenance=_legacy_provenance(), + ) + + assert not journal.path.exists() + assert not list(journal.path.parent.glob(".*.tmp")) + restarted = NetworkProvisioningIdempotencyJournal( + tmp_path / "repository", + clock=_clock, + ) + assert restarted.snapshot().status == "empty" + assert restarted.snapshot().active_record is None + + +def test_same_terminal_request_replays_across_restart_without_rebinding_operation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + _complete_success( + journal, + key=KEY, + operation_id=OPERATION_ID, + device_id="device-k1-owned", + ) + + restarted = NetworkProvisioningIdempotencyJournal( + tmp_path / "repository", + clock=_clock, + ) + replay = restarted.begin( + idempotency_key=KEY, + action=ACTION, + operation_id="op-generated-after-restart", + request_binding_sha256=_binding(), + ) + + assert replay.disposition == "terminal-replay" + assert replay.created is False + assert replay.record.operation_id == OPERATION_ID + assert replay.record.terminal == _success() + + +def test_same_key_cannot_be_rebound_to_another_request_or_action( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + _complete_success( + journal, + key=KEY, + operation_id=OPERATION_ID, + device_id="device-k1-owned", + ) + + with pytest.raises(NetworkProvisioningIdempotencyConflict, match="another request"): + journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id="op-different-binding", + request_binding_sha256=_binding(device_id="different-device"), + ) + with pytest.raises(NetworkProvisioningIdempotencyConflict, match="another request"): + journal.begin( + idempotency_key=KEY, + action="network.reconcile", + operation_id="op-different-action", + request_binding_sha256=_binding(), + ) + + +@pytest.mark.parametrize("cross_ambiguity_boundary", [False, True]) +def test_nonterminal_request_blocks_its_replay_and_every_new_side_effect( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + cross_ambiguity_boundary: bool, +) -> None: + journal = _journal(tmp_path, monkeypatch) + admission = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + if cross_ambiguity_boundary: + journal.mark_unresolved( + OPERATION_ID, + expected_revision=admission.record.revision, + ) + + with pytest.raises(NetworkProvisioningIdempotencyBlocked) as replay_error: + journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id="op-replay", + request_binding_sha256=_binding(), + ) + with pytest.raises(NetworkProvisioningIdempotencyBlocked) as new_error: + journal.begin( + idempotency_key="different-key", + action=ACTION, + operation_id="op-new", + request_binding_sha256=_binding("different-key", device_id="different-device"), + ) + + assert replay_error.value.record is not None + assert new_error.value.record is not None + assert replay_error.value.record.operation_id == OPERATION_ID + assert new_error.value.record.operation_id == OPERATION_ID + + +def test_revision_cas_and_stage_contract_guard_the_side_effect_boundary( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + admission = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + + with pytest.raises(NetworkProvisioningIdempotencyTransitionError, match="stale"): + journal.mark_unresolved( + OPERATION_ID, + expected_revision=admission.record.revision + 1, + ) + with pytest.raises(NetworkProvisioningIdempotencyTransitionError, match="no-side-effect"): + journal.complete( + OPERATION_ID, + expected_revision=admission.record.revision, + terminal=_success(), + ) + + failed = journal.complete( + OPERATION_ID, + expected_revision=admission.record.revision, + terminal=_failure_no_side_effect(), + ) + assert failed.stage == "terminal" + assert ( + journal.complete( + OPERATION_ID, + expected_revision=admission.record.revision, + terminal=_failure_no_side_effect(), + ) + == failed + ) + + +def test_terminal_history_is_bounded_without_evicting_active_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch, max_terminal_records=2) + for index in range(3): + _complete_success( + journal, + key=f"key-{index}", + operation_id=f"op-{index}", + device_id=f"device-{index}", + ) + + snapshot = journal.snapshot() + assert snapshot.status == "ready" + assert [record.operation_id for record in snapshot.records] == ["op-1", "op-2"] + + active = journal.begin( + idempotency_key="key-active", + action=ACTION, + operation_id="op-active", + request_binding_sha256=_binding("key-active", device_id="device-active"), + ) + assert active.created is True + blocked = journal.snapshot() + assert blocked.status == "blocked" + assert blocked.active_record is not None + assert blocked.active_record.operation_id == "op-active" + assert len([record for record in blocked.records if record.stage == "terminal"]) == 2 + + +def test_corruption_fails_closed_and_is_not_overwritten( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + path = tmp_path / "private-data" / "xgrids-k1" / "network-provisioning-idempotency.json" + path.parent.mkdir(mode=0o700, parents=True) + (tmp_path / "private-data").chmod(0o700) + path.parent.chmod(0o700) + path.write_bytes(b'{"schema_version":"wrong","password":"must-stay-unread"}\n') + path.chmod(0o600) + before = path.read_bytes() + + journal = NetworkProvisioningIdempotencyJournal(tmp_path / "repository", clock=_clock) + + assert journal.snapshot().status == "corrupt" + with pytest.raises(NetworkProvisioningIdempotencyCorrupt): + journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + assert path.read_bytes() == before + + +def test_journal_rejects_symlink_and_hardlink_storage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + target = journal.path.parent / "journal-target.json" + journal.path.replace(target) + journal.path.symlink_to(target) + + assert journal.snapshot().status == "corrupt" + journal.path.unlink() + target.replace(journal.path) + hardlink = journal.path.parent / "journal-hardlink.json" + os.link(journal.path, hardlink) + + assert journal.snapshot().status == "corrupt" + assert journal.path.stat().st_nlink == 2 + + +def test_cross_process_lock_rejects_an_extra_hardlink( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + hardlink = journal.path.parent / "lock-hardlink" + os.link(journal._process_lock_path, hardlink) # noqa: SLF001 + + with pytest.raises(NetworkProvisioningIdempotencyCorrupt, match="lock is unsafe"): + journal.snapshot() + + +def test_atomic_publication_fsyncs_file_then_parent_and_cleans_tempfiles( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + real_fsync = journal_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = journal_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(journal_module.os, "fsync", observe_fsync) + journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + + assert "file" in fsync_kinds + assert fsync_kinds[-1] == "directory" + assert fsync_kinds.index("file") < len(fsync_kinds) - 1 + assert not list(journal.path.parent.glob(".*.tmp")) + + +def test_failed_atomic_replace_preserves_previous_terminal_history( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + _complete_success( + journal, + key=KEY, + operation_id=OPERATION_ID, + device_id="device-k1-owned", + ) + before = journal.path.read_bytes() + + def fail_replace(_source: object, _target: object) -> None: + raise OSError("simulated atomic publication failure") + + with monkeypatch.context() as publication_failure: + publication_failure.setattr(journal_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="atomic publication failure"): + journal.begin( + idempotency_key="key-after-terminal", + action=ACTION, + operation_id="op-after-terminal", + request_binding_sha256=_binding( + "key-after-terminal", + device_id="device-after-terminal", + ), + ) + + assert journal.path.read_bytes() == before + assert not list(journal.path.parent.glob(".*.tmp")) + restarted = NetworkProvisioningIdempotencyJournal(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().status == "ready" + + +def test_revision_not_wall_clock_orders_transitions_and_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + moments = iter( + [ + datetime(2026, 8, 7, 12, 0, tzinfo=UTC), + datetime(2026, 8, 7, 11, 0, tzinfo=UTC), + datetime(2026, 8, 7, 10, 0, tzinfo=UTC), + datetime(2026, 8, 7, 9, 0, tzinfo=UTC), + ] + ) + journal = _journal(tmp_path, monkeypatch, clock=lambda: next(moments)) + admission = journal.begin( + idempotency_key=KEY, + action=ACTION, + operation_id=OPERATION_ID, + request_binding_sha256=_binding(), + ) + unresolved = journal.mark_unresolved( + OPERATION_ID, + expected_revision=admission.record.revision, + ) + terminal = journal.complete( + OPERATION_ID, + expected_revision=unresolved.revision, + terminal=_success(), + ) + later_revision = journal.begin( + idempotency_key="key-after-clock-rollback", + action=ACTION, + operation_id="op-after-clock-rollback", + request_binding_sha256=_binding( + "key-after-clock-rollback", + device_id="device-after-clock-rollback", + ), + ).record + + assert admission.record.revision < unresolved.revision < terminal.revision + assert terminal.updated_at_utc == admission.record.created_at_utc + assert later_revision.revision > terminal.revision + assert later_revision.created_at_utc < terminal.updated_at_utc + restarted = NetworkProvisioningIdempotencyJournal(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().status == "blocked" + + +def test_discontinuous_record_revision_lineage_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + journal = _journal(tmp_path, monkeypatch) + _complete_success( + journal, + key=KEY, + operation_id=OPERATION_ID, + device_id="device-k1-owned", + ) + document = json.loads(journal.path.read_text(encoding="utf-8")) + record = document["records"][0] + assert record["revision"] == 3 + assert record["previous_revision"] == 2 + record["previous_revision"] = 1 + journal.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + + restarted = NetworkProvisioningIdempotencyJournal(tmp_path / "repository", clock=_clock) + + assert restarted.snapshot().status == "corrupt" + with pytest.raises(NetworkProvisioningIdempotencyCorrupt): + restarted.begin( + idempotency_key="blocked-after-lineage-corruption", + action=ACTION, + operation_id="blocked-after-lineage-corruption", + request_binding_sha256=_binding( + "blocked-after-lineage-corruption", + device_id="blocked-after-lineage-corruption", + ), + ) + + +def test_flock_allows_only_one_cross_process_admission( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + data_dir = tmp_path / "private-data" + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(data_dir)) + repository_root = tmp_path / "repository" + context = multiprocessing.get_context("fork") + start = context.Event() + results = context.Queue() + processes = [ + context.Process( + target=_cross_process_admit, + args=( + str(repository_root), + str(data_dir), + f"cross-process-key-{index}", + f"cross-process-op-{index}", + start, + results, + ), + ) + for index in range(2) + ] + for process in processes: + process.start() + start.set() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + + outcomes = sorted(results.get(timeout=2) for _ in processes) + assert [outcome[0] for outcome in outcomes] == ["admitted", "blocked"] + snapshot = NetworkProvisioningIdempotencyJournal(repository_root, clock=_clock).snapshot() + assert snapshot.status == "blocked" + assert len(snapshot.records) == 1 + + +def test_flock_allows_only_one_competing_cross_process_legacy_adoption( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + data_dir = tmp_path / "private-data" + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(data_dir)) + repository_root = tmp_path / "repository" + context = multiprocessing.get_context("fork") + start = context.Event() + results = context.Queue() + processes = [ + context.Process( + target=_cross_process_adopt_legacy, + args=( + str(repository_root), + str(data_dir), + f"legacy-cross-process-op-{index}", + start, + results, + ), + ) + for index in range(2) + ] + for process in processes: + process.start() + start.set() + for process in processes: + process.join(timeout=15) + assert process.exitcode == 0 + + outcomes = sorted(results.get(timeout=2) for _ in processes) + assert [outcome[0] for outcome in outcomes] == ["blocked", "unresolved"] + snapshot = NetworkProvisioningIdempotencyJournal(repository_root, clock=_clock).snapshot() + assert snapshot.status == "blocked" + assert len(snapshot.records) == 1 + assert snapshot.active_record is not None + assert snapshot.active_record.stage == "unresolved" + + +def test_terminal_replay_is_identical_in_another_process( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + data_dir = tmp_path / "private-data" + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(data_dir)) + repository_root = tmp_path / "repository" + journal = NetworkProvisioningIdempotencyJournal(repository_root, clock=_clock) + _complete_success( + journal, + key=KEY, + operation_id=OPERATION_ID, + device_id="device-k1-owned", + ) + context = multiprocessing.get_context("fork") + results = context.Queue() + process = context.Process( + target=_cross_process_replay, + args=(str(repository_root), str(data_dir), results), + ) + process.start() + process.join(timeout=15) + + assert process.exitcode == 0 + assert results.get(timeout=2) == ("terminal-replay", OPERATION_ID, "succeeded") diff --git a/tests/test_xgrids_physical_command_coordinator.py b/tests/test_xgrids_physical_command_coordinator.py new file mode 100644 index 0000000..c7fe5b2 --- /dev/null +++ b/tests/test_xgrids_physical_command_coordinator.py @@ -0,0 +1,1393 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + LedgerPhysicalCommandCoordinator, + PhysicalCommandIntentContext, + PhysicalCommandPrepareSettlementError, + PhysicalCommandRuntimeBinding, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PhysicalCommandLedger, +) +from k1link.device_plugins.xgrids_k1.protocol.application_acceptance import ( + PhysicalAcceptanceChecklist, + PhysicalAcceptancePermit, +) +from k1link.device_plugins.xgrids_k1.protocol.application_mqtt import ( + ApplicationMqttDeviceStatusEvidence, + ApplicationMqttPublishEvidence, + ApplicationMqttResponseEvidence, + ApplicationMqttTransportError, +) +from k1link.device_plugins.xgrids_k1.protocol.application_publish import ( + OneShotPublishEnvelope, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import ( + MODELING_STATE_BASE, + OPENAPI_SUCCESS, + ModelingAction, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_safety import MODELING_REQUEST_TOPIC + +VENDOR_HASH = hashlib.sha256(b"vendor-id").hexdigest() +SERIAL_HASH = hashlib.sha256(b"serial-id").hexdigest() +PROJECT_HASH = hashlib.sha256(b"project-id").hexdigest() +PAYLOAD = b"exact-modeling-command" +PAYLOAD_HASH = hashlib.sha256(PAYLOAD).hexdigest() + + +@dataclass(frozen=True) +class FakeLedgerSnapshot: + status: str = "empty" + reason_code: str | None = None + record: object | None = None + requires_reconciliation: bool = False + + +@dataclass(frozen=True) +class FakePhysicalRecord: + operation_id: str + revision: int = 1 + stage: str = "observing" + action: str = "stop" + resolution: str | None = None + reconciled_physical_state: str | None = None + reopened_physical_state_requires_reconciliation: bool = False + resolved_unclassified_stop_recovery_required: bool = False + + def as_dict(self) -> dict[str, object]: + return {"operation_id": self.operation_id} + + +@dataclass(frozen=True) +class FakeReconciledRecord: + operation_id: str + + def as_dict(self) -> dict[str, object]: + return {"operation_id": self.operation_id, "stage": "resolved"} + + +@dataclass(frozen=True) +class FakeScenarioConnection: + transport_ref: str = "bridge:192.168.1.20" + + +@dataclass(frozen=True) +class FakeScenarioRecord: + operation_id: str = "operation-ambiguous-reset" + revision: int = 7 + stage: str = "observing" + resolution: str | None = None + reconciled_physical_state: str | None = None + reopened_physical_state_requires_reconciliation: bool = False + classified_stop_standby_successor_pending: bool = False + connection: FakeScenarioConnection = FakeScenarioConnection() + operator_retirements: tuple[object, ...] = () + operator_reconciliation_reopens: tuple[object, ...] = () + + def as_dict(self) -> dict[str, object]: + return { + "operation_id": self.operation_id, + "revision": self.revision, + "stage": self.stage, + "resolution": self.resolution, + } + + +class FakePhysicalLedger: + def __init__(self) -> None: + self.events: list[tuple[str, object]] = [] + self.snapshot_result = FakeLedgerSnapshot() + self.runtime_guard_calls: list[tuple[object, object]] = [] + self.runtime_guard_error: BaseException | None = None + + def snapshot(self) -> FakeLedgerSnapshot: + return self.snapshot_result + + def require_runtime_not_retired( + self, + *, + identity: object, + connection: object, + ) -> None: + self.runtime_guard_calls.append((identity, connection)) + if self.runtime_guard_error is not None: + raise self.runtime_guard_error + + def prepare(self, **kwargs: Any) -> None: + self.events.append(("prepare", kwargs)) + + def mark_dispatching(self, operation_id: str) -> None: + self.events.append(("dispatching", operation_id)) + + def mark_observing(self, operation_id: str, **kwargs: Any) -> None: + self.events.append(("observing", (operation_id, kwargs))) + + def mark_qos2_completed(self, operation_id: str, **kwargs: Any) -> None: + self.events.append(("qos2", (operation_id, kwargs))) + + def record_application_response(self, operation_id: str, response: object) -> None: + self.events.append(("response", (operation_id, response))) + + def record_status_observation(self, operation_id: str, status: object) -> None: + self.events.append(("status", (operation_id, status))) + + def resolve(self, operation_id: str, **kwargs: Any) -> None: + self.events.append(("resolve", (operation_id, kwargs))) + + def preflight_resolve_not_dispatched(self, operation_id: str) -> None: + self.events.append(("preflight-resolve", operation_id)) + + def reconcile_ambiguous(self, operation_id: str, **kwargs: Any) -> FakeReconciledRecord: + self.events.append(("reconcile", (operation_id, kwargs))) + return FakeReconciledRecord(operation_id) + + def reconcile_prepared_stop_not_dispatched( + self, + operation_id: str, + **kwargs: Any, + ) -> FakeReconciledRecord: + self.events.append(("reconcile-prepared-stop", (operation_id, kwargs))) + return FakeReconciledRecord(operation_id) + + def retire_unavailable_target(self, **kwargs: Any) -> FakeReconciledRecord: + self.events.append(("retire", kwargs)) + return FakeReconciledRecord(str(kwargs["expected_operation_id"])) + + def preflight_retire_unavailable_target(self, **kwargs: Any) -> FakeReconciledRecord: + self.events.append(("preflight-retire", kwargs)) + return FakeReconciledRecord(str(kwargs["expected_operation_id"])) + + def reopen_retired_reconciliation(self, **kwargs: Any) -> FakeReconciledRecord: + self.events.append(("reopen", kwargs)) + return FakeReconciledRecord("operation-reopened-for-reconciliation") + + +def _runtime( + *, + host_path_epoch: int = 7, + control_session_id: str = "application-control-3-100", + producer_generation: int = 3, +) -> PhysicalCommandRuntimeBinding: + return PhysicalCommandRuntimeBinding( + vendor_device_id_sha256=VENDOR_HASH, + device_serial_sha256=SERIAL_HASH, + compatibility_profile_id="xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", + intent_id="connection-intent-1", + transport_ref="bridge:192.168.1.20", + connection_mode="bridge", + target_ipv4="192.168.1.20", + target_port=1883, + host_path_epoch=host_path_epoch, + control_session_id=control_session_id, + producer_generation=producer_generation, + ) + + +def _status( + state: str, + code: int, + *, + project_bound: bool, + init_ready: bool, + retained: bool = False, + observed_at: str, +) -> ApplicationMqttDeviceStatusEvidence: + payload_hash = hashlib.sha256(f"{state}:{observed_at}".encode()).hexdigest() + return ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=VENDOR_HASH, + device_serial_sha256=SERIAL_HASH, + session_state=state, + session_state_code=code, + project_bound=project_bound, + project_id_sha256=PROJECT_HASH if project_bound else None, + init_ready=init_ready, + status_message_sha256=payload_hash, + mqtt_retained=retained, + observed_at_utc=observed_at, + ) + + +def _envelope(action: str) -> OneShotPublishEnvelope: + return OneShotPublishEnvelope( + operation_key=f"modeling:{action}", + topic=MODELING_REQUEST_TOPIC, + payload=PAYLOAD, + payload_sha256=PAYLOAD_HASH, + payload_bytes=len(PAYLOAD), + qos=2, + retain=False, + ) + + +def _stop_permit(clock: Any) -> PhysicalAcceptancePermit: + return PhysicalAcceptancePermit( + PhysicalAcceptanceChecklist( + action=ModelingAction.STOP, + operator_present=True, + owner_controlled_device=True, + lixelgo_closed=True, + battery_storage_confirmed=True, + expected_physical_state_confirmed=True, + ), + ttl_seconds=15.0, + monotonic=clock, + ) + + +def _prepare_stop_coordinator() -> tuple[ + FakePhysicalLedger, + LedgerPhysicalCommandCoordinator, + ApplicationMqttPublishEvidence, +]: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-stop-atomic-admission", + parent_operation_id="operation-start-parent", + acquisition_id="acquisition-atomic-admission", + ), + action="stop", + envelope=_envelope("stop"), + ) + return ( + ledger, + coordinator, + ApplicationMqttPublishEvidence( + operation_key="modeling:stop", + topic=MODELING_REQUEST_TOPIC, + payload_sha256=PAYLOAD_HASH, + qos=2, + retain=False, + packet_id=None, + ), + ) + + +def test_atomic_dispatch_admission_checks_deadline_before_permit_and_mark() -> None: + ledger, coordinator, dispatch = _prepare_stop_coordinator() + now = [100.0] + permit = _stop_permit(lambda: now[0]) + reservation = permit.reserve(ModelingAction.STOP) + + with pytest.raises(ApplicationMqttTransportError) as raised: + coordinator.publish_dispatching( + dispatch, + dispatch_admission_deadline_reached=lambda: True, + dispatch_admission_commit=reservation.commit, + ) + + assert raised.value.reason_code == "physical-command-dispatch-deadline-expired" + assert permit.snapshot()["consumed"] is False + assert [kind for kind, _payload in ledger.events] == ["prepare"] + reservation.release() + + +def test_atomic_dispatch_admission_rejects_expired_permit_before_mark() -> None: + ledger, coordinator, dispatch = _prepare_stop_coordinator() + now = [100.0] + permit = _stop_permit(lambda: now[0]) + reservation = permit.reserve(ModelingAction.STOP) + now[0] += 15.0 + + with pytest.raises(ApplicationMqttTransportError) as raised: + coordinator.publish_dispatching( + dispatch, + dispatch_admission_deadline_reached=lambda: False, + dispatch_admission_commit=reservation.commit, + ) + + assert raised.value.reason_code == ( + "physical-acceptance-permit-expired-before-dispatch" + ) + assert permit.snapshot()["consumed"] is False + assert [kind for kind, _payload in ledger.events] == ["prepare"] + + +def test_mark_failure_after_atomic_permit_commit_stays_fail_closed() -> None: + ledger, coordinator, dispatch = _prepare_stop_coordinator() + now = [100.0] + permit = _stop_permit(lambda: now[0]) + reservation = permit.reserve(ModelingAction.STOP) + + def fail_mark(operation_id: str) -> None: + ledger.events.append(("dispatching-started", operation_id)) + raise OSError("injected durable mark failure") + + ledger.mark_dispatching = fail_mark # type: ignore[method-assign] + + with pytest.raises(OSError, match="durable mark failure"): + coordinator.publish_dispatching( + dispatch, + dispatch_admission_deadline_reached=lambda: False, + dispatch_admission_commit=reservation.commit, + ) + + assert permit.snapshot()["consumed"] is True + assert [kind for kind, _payload in ledger.events] == [ + "prepare", + "dispatching-started", + ] + + +def test_coordinator_maps_exact_transport_evidence_to_one_durable_start_edge() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + baseline = _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + coordinator.device_status(baseline) + context = PhysicalCommandIntentContext( + operation_id="operation-start-1", + parent_operation_id=None, + acquisition_id="acquisition-1", + ) + envelope = _envelope("start") + + coordinator.prepare(context, action="start", envelope=envelope) + dispatch = ApplicationMqttPublishEvidence( + operation_key="modeling:start", + topic=MODELING_REQUEST_TOPIC, + payload_sha256=PAYLOAD_HASH, + qos=2, + retain=False, + packet_id=None, + ) + coordinator.publish_dispatching(dispatch) + returned = ApplicationMqttPublishEvidence( + operation_key="modeling:start", + topic=MODELING_REQUEST_TOPIC, + payload_sha256=PAYLOAD_HASH, + qos=2, + retain=False, + packet_id=42, + ) + coordinator.publish_result(returned, publish_call_returned=True) + coordinator.qos2_completed(returned) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="modeling:start", + response_topic="lixel/application/response/modeling", + payload_sha256=hashlib.sha256(b"response").hexdigest(), + modeling_action="start", + result_code=OPENAPI_SUCCESS, + success=True, + observed_at_utc="2026-08-07T10:00:01.000Z", + ) + ) + target = _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:02.000Z", + ) + coordinator.device_status(target) + coordinator.resolve("start") + + assert [kind for kind, _payload in ledger.events] == [ + "prepare", + "dispatching", + "observing", + "qos2", + "response", + "status", + "resolve", + ] + prepared = ledger.events[0][1] + assert isinstance(prepared, dict) + assert prepared["operation_id"] == "operation-start-1" + assert prepared["payload_sha256"] == PAYLOAD_HASH + assert prepared["connection"].producer_generation == 3 + assert prepared["baseline_status"].session_state == "ready" + assert prepared["baseline_status"].session_state_code == 300 + response = ledger.events[4][1][1] + assert response.control_session_id == "application-control-3-100" + assert response.producer_generation == 3 + resolution = ledger.events[-1][1][1] + assert resolution["resolution"] == "start-active-observed" + + +def test_coordinator_durably_classifies_scan_over_then_requires_fresh_ready( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ledger = PhysicalCommandLedger(tmp_path / "repository") + coordinator = LedgerPhysicalCommandCoordinator(ledger) + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-start-scan-over", + parent_operation_id=None, + acquisition_id="acquisition-scan-over", + ), + action="start", + envelope=_envelope("start"), + ) + dispatching = ApplicationMqttPublishEvidence( + operation_key="modeling:start", + topic=MODELING_REQUEST_TOPIC, + payload_sha256=PAYLOAD_HASH, + qos=2, + retain=False, + packet_id=None, + ) + published = ApplicationMqttPublishEvidence( + operation_key="modeling:start", + topic=MODELING_REQUEST_TOPIC, + payload_sha256=PAYLOAD_HASH, + qos=2, + retain=False, + packet_id=42, + ) + coordinator.publish_dispatching(dispatching) + coordinator.publish_result(published, publish_call_returned=True) + coordinator.qos2_completed(published) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="modeling:start", + response_topic="lixel/application/response/modeling", + payload_sha256=hashlib.sha256(b"start-response").hexdigest(), + modeling_action="start", + result_code=OPENAPI_SUCCESS, + success=True, + observed_at_utc="2026-08-07T10:00:01.000Z", + ) + ) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:02.000Z", + ) + ) + coordinator.resolve("start") + + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:2:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=hashlib.sha256(b"scan-over-device-info").hexdigest(), + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:01:00.000Z", + ) + ) + coordinator.bind_control_session( + _runtime( + host_path_epoch=8, + control_session_id="application-control-scan-over", + producer_generation=4, + ) + ) + coordinator.device_status( + _status( + "scan_over", + MODELING_STATE_BASE + 304, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:01:01.000Z", + ) + ) + before = coordinator.snapshot() + assert before["resolved_active_recovery_required"] is True + scan_over = coordinator.reconcile_resolved_active(reconciliation_id="reconcile-scan-over") + after_scan_over = coordinator.snapshot() + + assert scan_over["operation_id"] == "operation-start-scan-over" + assert after_scan_over["status"] == "resolved" + assert after_scan_over["resolved_active_recovery_required"] is False + assert after_scan_over["resolved_scan_over_recovery_required"] is True + assert after_scan_over["requires_reconciliation"] is True + assert after_scan_over["reason_code"] == ("physical-command-scan-over-awaiting-ready") + record = after_scan_over["record"] + assert isinstance(record, dict) + assert record["reconciliations"][-1]["observation"]["session_state"] == ( # type: ignore[index] + "scan_over" + ) + + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:3:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=hashlib.sha256(b"ready-device-info").hexdigest(), + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:02:00.000Z", + ) + ) + coordinator.bind_control_session( + _runtime( + host_path_epoch=9, + control_session_id="application-control-ready-after-scan-over", + producer_generation=5, + ) + ) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:02:01.000Z", + ) + ) + coordinator.reconcile_resolved_active(reconciliation_id="reconcile-ready-after-scan-over") + after_ready = coordinator.snapshot() + assert after_ready["resolved_scan_over_recovery_required"] is False + assert after_ready["requires_reconciliation"] is False + + +def test_coordinator_prepare_requires_a_live_status_baseline() -> None: + coordinator = LedgerPhysicalCommandCoordinator(FakePhysicalLedger()) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + + with pytest.raises(RuntimeError, match="baseline is unavailable"): + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-start-1", + parent_operation_id=None, + acquisition_id="acquisition-1", + ), + action="start", + envelope=_envelope("start"), + ) + + +def test_prepare_exception_after_durable_replace_is_sealed_not_dispatched( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ledger = PhysicalCommandLedger(tmp_path / "repository") + coordinator = LedgerPhysicalCommandCoordinator(ledger) + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + durable_prepare = ledger.prepare + + def replace_then_raise(**kwargs: Any) -> None: + durable_prepare(**kwargs) + raise OSError("injected post-replace failure") + + monkeypatch.setattr(ledger, "prepare", replace_then_raise) + context = PhysicalCommandIntentContext( + operation_id="operation-start-post-replace-failure", + parent_operation_id=None, + acquisition_id="acquisition-post-replace-failure", + ) + + with pytest.raises(OSError, match="post-replace failure"): + coordinator.prepare(context, action="start", envelope=_envelope("start")) + + snapshot = ledger.snapshot() + assert snapshot.status == "resolved" + assert snapshot.record is not None + assert snapshot.record.operation_id == context.operation_id + assert snapshot.record.resolution == "not-dispatched" + assert coordinator.snapshot()["active_operation_id"] is None + + +def test_prepare_exception_with_unreadable_settlement_fails_closed() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + + def failed_prepare(**_kwargs: Any) -> None: + ledger.snapshot_result = None # type: ignore[assignment] + raise OSError("injected prepare failure") + + ledger.prepare = failed_prepare # type: ignore[method-assign] + + with pytest.raises(PhysicalCommandPrepareSettlementError): + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-start-unreadable-settlement", + parent_operation_id=None, + acquisition_id="acquisition-unreadable-settlement", + ), + action="start", + envelope=_envelope("start"), + ) + + +def test_prepare_conflict_never_settles_another_binding_same_operation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ledger = PhysicalCommandLedger(tmp_path / "repository") + original = LedgerPhysicalCommandCoordinator(ledger) + original.bind_control_session(_runtime()) + original.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + context = PhysicalCommandIntentContext( + operation_id="operation-start-binding-conflict", + parent_operation_id=None, + acquisition_id="acquisition-binding-conflict", + ) + original.prepare(context, action="start", envelope=_envelope("start")) + + conflicting = LedgerPhysicalCommandCoordinator(ledger) + conflicting.bind_control_session( + _runtime( + host_path_epoch=8, + control_session_id="application-control-conflict", + producer_generation=4, + ) + ) + conflicting.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:01:00.000Z", + ) + ) + + with pytest.raises(PhysicalCommandPrepareSettlementError): + conflicting.prepare(context, action="start", envelope=_envelope("start")) + + snapshot = ledger.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.operation_id == context.operation_id + assert snapshot.record.stage == "prepared" + assert snapshot.record.resolution is None + assert snapshot.record.connection.control_session_id == "application-control-3-100" + + +def test_coordinator_rejects_retired_deviceinfo_identity_before_runtime_binding() -> None: + ledger = FakePhysicalLedger() + ledger.runtime_guard_error = RuntimeError( + "retired physical target cannot bind a control session" + ) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + with pytest.raises(RuntimeError, match="retired physical target"): + coordinator.bind_control_session( + _runtime( + control_session_id="retired-deviceinfo-session", + producer_generation=41, + ) + ) + + snapshot = coordinator.snapshot() + assert len(ledger.runtime_guard_calls) == 1 + assert snapshot["runtime_bound"] is False + assert snapshot["reconciliation_ready"] is False + assert ledger.events == [] + + +def test_scenario_reset_retires_ambiguous_edge_without_device_io() -> None: + ledger = FakePhysicalLedger() + record = FakeScenarioRecord() + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=record, + requires_reconciliation=True, + ) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + preview = coordinator.preview_connection_scenario_retirement( + retirement_id="scenario-reset-test-ambiguous", + ) + result = coordinator.retire_connection_scenario( + retirement_id="scenario-reset-test-ambiguous", + expected_revision=record.revision, + ) + + assert preview == { + "disposition": "operator-retired-outcome-unknown", + "record_revision": 7, + "operation_id": "operation-ambiguous-reset", + "transport_ref": "bridge:192.168.1.20", + } + assert result["disposition"] == "operator-retired-outcome-unknown" + assert ledger.events == [ + ( + "preflight-retire", + { + "retirement_id": "scenario-reset-test-ambiguous", + "expected_operation_id": "operation-ambiguous-reset", + "expected_revision": 7, + "expected_transport_ref": "bridge:192.168.1.20", + "reason": "connection-scenario-reset-by-operator", + }, + ), + ( + "retire", + { + "retirement_id": "scenario-reset-test-ambiguous", + "expected_operation_id": "operation-ambiguous-reset", + "expected_revision": 7, + "expected_transport_ref": "bridge:192.168.1.20", + "reason": "connection-scenario-reset-by-operator", + }, + ), + ] + snapshot = coordinator.snapshot() + assert snapshot["runtime_bound"] is False + assert snapshot["active_operation_id"] is None + + +def test_scenario_reset_retires_pending_classified_stop_reopen_without_state_claim() -> None: + ledger = FakePhysicalLedger() + record = FakeScenarioRecord( + stage="resolved", + resolution="not-dispatched", + reconciled_physical_state=None, + reopened_physical_state_requires_reconciliation=True, + ) + ledger.snapshot_result = FakeLedgerSnapshot(status="resolved", record=record) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + preview = coordinator.preview_connection_scenario_retirement( + retirement_id="scenario-reset-pending-classified-reopen", + ) + result = coordinator.retire_connection_scenario( + retirement_id="scenario-reset-pending-classified-reopen", + expected_revision=record.revision, + ) + + assert preview["disposition"] == "operator-retired-outcome-unknown" + assert result["disposition"] == "operator-retired-outcome-unknown" + expected_request = { + "retirement_id": "scenario-reset-pending-classified-reopen", + "expected_operation_id": record.operation_id, + "expected_revision": record.revision, + "expected_transport_ref": record.connection.transport_ref, + "reason": "connection-scenario-reset-by-operator", + } + assert ledger.events == [ + ("preflight-retire", expected_request), + ("retire", expected_request), + ] + + +def test_scenario_reset_retires_classified_stop_standby_successor_binding() -> None: + ledger = FakePhysicalLedger() + record = FakeScenarioRecord( + stage="resolved", + resolution="not-dispatched", + reconciled_physical_state="standby", + classified_stop_standby_successor_pending=True, + ) + ledger.snapshot_result = FakeLedgerSnapshot(status="resolved", record=record) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + preview = coordinator.preview_connection_scenario_retirement( + retirement_id="scenario-reset-classified-standby", + ) + result = coordinator.retire_connection_scenario( + retirement_id="scenario-reset-classified-standby", + expected_revision=record.revision, + ) + + assert preview["disposition"] == "operator-retired-outcome-unknown" + assert result["disposition"] == "operator-retired-outcome-unknown" + expected_request = { + "retirement_id": "scenario-reset-classified-standby", + "expected_operation_id": record.operation_id, + "expected_revision": record.revision, + "expected_transport_ref": record.connection.transport_ref, + "reason": "connection-scenario-reset-by-operator", + } + assert ledger.events == [ + ("preflight-retire", expected_request), + ("retire", expected_request), + ] + + +def test_scenario_reset_preflights_exact_prepared_no_dispatch_resolution() -> None: + ledger = FakePhysicalLedger() + record = FakeScenarioRecord(stage="prepared") + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=record, + requires_reconciliation=True, + ) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + preview = coordinator.preview_connection_scenario_retirement( + retirement_id="scenario-reset-prepared-preflight", + ) + + assert preview["disposition"] == "not-dispatched" + assert ledger.events == [("preflight-resolve", record.operation_id)] + + +def test_coordinator_refuses_operator_retirement_while_command_owner_is_active() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-stop-active-owner", + parent_operation_id="operation-start-active-owner", + acquisition_id="acquisition-active-owner", + ), + action="stop", + envelope=_envelope("stop"), + ) + events_before = list(ledger.events) + + with pytest.raises(RuntimeError, match="owner is still active"): + coordinator.retire_unavailable_target( + retirement_id="retirement-active-owner", + expected_operation_id="operation-stop-active-owner", + expected_revision=2, + expected_transport_ref=_runtime().transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert ledger.events == events_before + assert coordinator.snapshot()["active_operation_id"] == ("operation-stop-active-owner") + + +def test_coordinator_reopen_is_local_only_and_revokes_all_runtime_proof() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + + result = coordinator.reopen_retired_reconciliation( + reopening_id="reopening-coordinator", + expected_revision=18, + expected_retirement_id="retirement-coordinator", + expected_transport_ref=_runtime().transport_ref, + expected_discovery_generation=7, + reason="device-returned-for-explicit-reconciliation", + ) + + assert result == { + "operation_id": "operation-reopened-for-reconciliation", + "stage": "resolved", + } + assert ledger.events == [ + ( + "reopen", + { + "reopening_id": "reopening-coordinator", + "expected_revision": 18, + "expected_retirement_id": "retirement-coordinator", + "expected_transport_ref": _runtime().transport_ref, + "expected_discovery_generation": 7, + "reason": "device-returned-for-explicit-reconciliation", + }, + ) + ] + snapshot = coordinator.snapshot() + assert snapshot["runtime_bound"] is False + assert snapshot["reconciliation_ready"] is False + + +def test_coordinator_rejects_status_from_an_identity_outside_the_bound_session() -> None: + coordinator = LedgerPhysicalCommandCoordinator(FakePhysicalLedger()) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + foreign = ApplicationMqttDeviceStatusEvidence( + vendor_device_id_sha256=hashlib.sha256(b"other-device").hexdigest(), + device_serial_sha256=SERIAL_HASH, + session_state="ready", + session_state_code=300, + project_bound=False, + project_id_sha256=None, + init_ready=False, + status_message_sha256=hashlib.sha256(b"foreign-status").hexdigest(), + mqtt_retained=False, + observed_at_utc="2026-08-07T10:00:00.000Z", + ) + + with pytest.raises(RuntimeError, match="identity does not match"): + coordinator.device_status(foreign) + + +def test_coordinator_reconciles_ambiguous_stop_from_fresh_read_only_proofs() -> None: + ledger = FakePhysicalLedger() + ledger.snapshot_result = FakeLedgerSnapshot( + status="blocked", + record=FakePhysicalRecord("operation-stop-ambiguous"), + requires_reconciliation=True, + ) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + device_info_hash = hashlib.sha256(b"fresh-device-info").hexdigest() + + # DeviceInfo is correlated before ApplicationControlSession can bind the + # exact route/session generation. The coordinator must consume that proof + # once when the runtime binding arrives. + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=device_info_hash, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:00:01.000Z", + ) + ) + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:00:02.000Z", + ) + ) + + snapshot = coordinator.snapshot() + assert snapshot["reconciliation_ready"] is True + assert snapshot["observed_session_state"] == "ready" + result = coordinator.reconcile_unresolved(reconciliation_id="reconciliation-read-only-1") + + assert result == { + "operation_id": "operation-stop-ambiguous", + "stage": "resolved", + } + assert [kind for kind, _payload in ledger.events] == ["reconcile"] + operation_id, kwargs = ledger.events[0][1] + assert operation_id == "operation-stop-ambiguous" + assert kwargs["resolution"] == "physical-standby-observed" + assert kwargs["verified_binding"].device_info_message_sha256 == device_info_hash + assert ( + kwargs["verified_binding"].connection.control_session_id + == kwargs["observation"].control_session_id + == _runtime().control_session_id + ) + assert kwargs["observation"].source == "explicit-read-only-reconciliation" + assert kwargs["observation"].session_state == "ready" + + +def test_new_control_binding_does_not_append_status_to_an_old_unresolved_edge() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + coordinator.prepare( + PhysicalCommandIntentContext( + operation_id="operation-stop-old-session", + parent_operation_id="operation-start-old-session", + acquisition_id="acquisition-old-session", + ), + action="stop", + envelope=_envelope("stop"), + ) + ledger.snapshot_result = FakeLedgerSnapshot( + status="blocked", + record=FakePhysicalRecord("operation-stop-old-session"), + requires_reconciliation=True, + ) + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:1:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=hashlib.sha256(b"new-session-device-info").hexdigest(), + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:01:00.000Z", + ) + ) + ledger.events.clear() + + coordinator.bind_control_session( + _runtime( + host_path_epoch=8, + control_session_id="application-control-4-200", + producer_generation=4, + ) + ) + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:01:01.000Z", + ) + ) + + assert ledger.events == [] + snapshot = coordinator.snapshot() + assert snapshot["reconciliation_ready"] is True + + +@pytest.mark.parametrize("state", ["ready", "scanning"]) +def test_prepared_stop_detaches_before_read_only_bootstrap_without_poisoning_status( + state: str, +) -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + old_runtime = _runtime() + coordinator.bind_control_session(old_runtime) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + context = PhysicalCommandIntentContext( + operation_id="operation-stop-prepared-bootstrap", + parent_operation_id="operation-start-prepared-bootstrap", + acquisition_id="acquisition-prepared-bootstrap", + ) + coordinator.prepare(context, action="stop", envelope=_envelope("stop")) + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=FakePhysicalRecord( + operation_id=context.operation_id, + revision=7, + stage="prepared", + action="stop", + ), + requires_reconciliation=True, + ) + ledger.events.clear() + + coordinator.prepare_read_only_bootstrap() + # A status can arrive after the observer is installed but before DeviceInfo + # creates the new exact binding. It is ignored, not appended to PREPARED. + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:01:00.000Z", + ) + ) + assert ledger.events == [] + assert coordinator.snapshot()["runtime_bound"] is False + + device_info_hash = hashlib.sha256(b"prepared-stop-fresh-device-info").hexdigest() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:2:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=device_info_hash, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:01:01.000Z", + ) + ) + fresh_runtime = _runtime( + host_path_epoch=8, + control_session_id="application-control-prepared-recovery", + producer_generation=4, + ) + coordinator.bind_control_session(fresh_runtime) + coordinator.device_status( + _status( + state, + MODELING_STATE_BASE + (300 if state == "ready" else 302), + project_bound=state == "scanning", + init_ready=state == "scanning", + observed_at="2026-08-07T10:01:02.000Z", + ) + ) + coordinator.reconcile_unresolved( + reconciliation_id="prepared-stop-read-only-reconciliation" + ) + assert [kind for kind, _payload in ledger.events] == [ + "reconcile-prepared-stop" + ] + operation_id, kwargs = ledger.events[0][1] + assert operation_id == context.operation_id + assert kwargs["expected_revision"] == 7 + assert kwargs["observation"].source == "explicit-read-only-reconciliation" + assert kwargs["observation"].session_state == state + + +def test_exact_active_prepared_stop_can_be_resolved_not_dispatched_only_once() -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + context = PhysicalCommandIntentContext( + operation_id="operation-stop-deadline-expired", + parent_operation_id="operation-start-deadline-expired", + acquisition_id="acquisition-deadline-expired", + ) + coordinator.prepare(context, action="stop", envelope=_envelope("stop")) + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=FakePhysicalRecord( + operation_id=context.operation_id, + revision=7, + stage="prepared", + action="stop", + ), + requires_reconciliation=True, + ) + ledger.events.clear() + + coordinator.resolve_prepared_not_dispatched("stop") + + assert ledger.events == [ + ( + "resolve", + (context.operation_id, {"resolution": "not-dispatched"}), + ) + ] + assert coordinator.snapshot()["active_operation_id"] is None + with pytest.raises(RuntimeError, match="no matching durable preparation"): + coordinator.resolve_prepared_not_dispatched("stop") + + +@pytest.mark.parametrize("stage", ["dispatching", "observing"]) +def test_not_dispatched_resolution_refuses_post_dispatch_stage(stage: str) -> None: + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + context = PhysicalCommandIntentContext( + operation_id=f"operation-stop-{stage}", + parent_operation_id="operation-start-post-dispatch", + acquisition_id="acquisition-post-dispatch", + ) + coordinator.prepare(context, action="stop", envelope=_envelope("stop")) + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=FakePhysicalRecord( + operation_id=context.operation_id, + revision=8, + stage=stage, + action="stop", + ), + requires_reconciliation=True, + ) + ledger.events.clear() + + with pytest.raises(RuntimeError, match="exact active PREPARED"): + coordinator.resolve_prepared_not_dispatched("stop") + + assert ledger.events == [] + assert coordinator.snapshot()["active_operation_id"] == context.operation_id + +@pytest.mark.parametrize("state", ["ready", "scanning"]) +def test_resolved_unclassified_stop_projects_recovery_and_routes_combined_classification( + state: str, +) -> None: + ledger = FakePhysicalLedger() + record = FakePhysicalRecord( + operation_id="operation-stop-resolved-unclassified", + revision=14, + stage="resolved", + action="stop", + resolution="not-dispatched", + resolved_unclassified_stop_recovery_required=True, + ) + ledger.snapshot_result = FakeLedgerSnapshot( + status="resolved", + record=record, + requires_reconciliation=False, + ) + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + + before = coordinator.snapshot() + assert before["requires_reconciliation"] is True + assert before["resolved_unclassified_stop_recovery_required"] is True + assert before["reconciliation_ready"] is False + assert before["reason_code"] == ( + "physical-command-stop-awaiting-state-classification" + ) + assert before["recovery_requirement"] == ( + "explicit-read-only-deviceinfo-and-non-retained-devicestatus" + ) + + device_info_hash = hashlib.sha256(b"resolved-stop-device-info").hexdigest() + coordinator.application_response( + ApplicationMqttResponseEvidence( + operation_key="bootstrap:resolved-stop:DeviceInfoRequest", + response_topic="lixel/application/response/device_info", + payload_sha256=device_info_hash, + modeling_action=None, + result_code=None, + success=None, + observed_at_utc="2026-08-07T10:02:01.000Z", + ) + ) + coordinator.bind_control_session( + _runtime( + host_path_epoch=9, + control_session_id="application-control-resolved-stop-recovery", + producer_generation=5, + ) + ) + coordinator.device_status( + _status( + state, + MODELING_STATE_BASE + (300 if state == "ready" else 302), + project_bound=state == "scanning", + init_ready=state == "scanning", + observed_at="2026-08-07T10:02:02.000Z", + ) + ) + assert coordinator.snapshot()["reconciliation_ready"] is True + + coordinator.reconcile_unresolved( + reconciliation_id="resolved-stop-read-only-reconciliation" + ) + assert [kind for kind, _payload in ledger.events] == [ + "reconcile-prepared-stop" + ] + operation_id, kwargs = ledger.events[0][1] + assert operation_id == record.operation_id + assert kwargs["expected_revision"] == record.revision + assert kwargs["observation"].source == "explicit-read-only-reconciliation" + assert kwargs["observation"].session_state == state + + +def test_read_only_bootstrap_detaches_exact_ambiguous_edge_without_resolving_it() -> None: + for stage in ("dispatching", "observing"): + ledger = FakePhysicalLedger() + coordinator = LedgerPhysicalCommandCoordinator(ledger) # type: ignore[arg-type] + coordinator.bind_control_session(_runtime()) + coordinator.device_status( + _status( + "scanning", + MODELING_STATE_BASE + 302, + project_bound=True, + init_ready=True, + observed_at="2026-08-07T10:00:00.000Z", + ) + ) + context = PhysicalCommandIntentContext( + operation_id=f"operation-stop-{stage}-bootstrap", + parent_operation_id="operation-start-bootstrap", + acquisition_id="acquisition-bootstrap", + ) + coordinator.prepare(context, action="stop", envelope=_envelope("stop")) + ledger.snapshot_result = FakeLedgerSnapshot( + status="unresolved", + record=FakePhysicalRecord( + operation_id=context.operation_id, + revision=8, + stage=stage, + action="stop", + ), + requires_reconciliation=True, + ) + + ledger.events.clear() + coordinator.prepare_read_only_bootstrap() + coordinator.device_status( + _status( + "ready", + MODELING_STATE_BASE + 300, + project_bound=False, + init_ready=False, + observed_at="2026-08-07T10:01:00.000Z", + ) + ) + assert ledger.events == [] + assert coordinator.snapshot()["active_operation_id"] is None + assert coordinator.snapshot()["runtime_bound"] is False diff --git a/tests/test_xgrids_physical_command_ledger.py b/tests/test_xgrids_physical_command_ledger.py new file mode 100644 index 0000000..4cd6c2b --- /dev/null +++ b/tests/test_xgrids_physical_command_ledger.py @@ -0,0 +1,5978 @@ +from __future__ import annotations + +import copy +import hashlib +import json +import os +import stat +import subprocess +import sys +import threading +import time +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest + +from k1link.device_plugins.xgrids_k1 import physical_command_ledger as ledger_module +from k1link.device_plugins.xgrids_k1.physical_command_coordinator import ( + LedgerPhysicalCommandCoordinator, +) +from k1link.device_plugins.xgrids_k1.physical_command_ledger import ( + PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + PHYSICAL_COMMAND_ARCHIVE_DIRECTORY, + PHYSICAL_COMMAND_LEDGER_LEGACY_SCHEMA, + PHYSICAL_COMMAND_LEDGER_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V2_SCHEMA, + PHYSICAL_COMMAND_LEDGER_V3_SCHEMA, + PhysicalCommandApplicationResponse, + PhysicalCommandBlocked, + PhysicalCommandConnectionBinding, + PhysicalCommandIdentity, + PhysicalCommandLedger, + PhysicalCommandLedgerCorrupt, + PhysicalCommandStatusEvidence, + PhysicalCommandTransitionError, + PhysicalCommandVerifiedBinding, + active_operator_retirements, +) +from k1link.device_plugins.xgrids_k1.protocol.modeling_control import OPENAPI_SUCCESS + +START_OPERATION = "physical-start-11111111-1111-4111-8111-111111111111" +STOP_OPERATION = "physical-stop-22222222-2222-4222-8222-222222222222" +NEXT_START_OPERATION = "physical-start-33333333-3333-4333-8333-333333333333" +STOP_RETRY_OPERATION = "physical-stop-44444444-4444-4444-8444-444444444444" +ACQUISITION_ID = "acquisition-11111111-1111-4111-8111-111111111111" +TRANSPORT_REF = "A161D9D5-C352-1069-D430-5FB0BC13F7F9" +CONTROL_SESSION_ID = "control-11111111-1111-4111-8111-111111111111" +IDENTITY = PhysicalCommandIdentity( + vendor_device_id_sha256="a" * 64, + device_serial_sha256="b" * 64, +) + + +def _clock() -> datetime: + return datetime(2026, 8, 7, 9, 30, tzinfo=UTC) + + +def _connection( + *, + control_session_id: str = CONTROL_SESSION_ID, + host_path_epoch: int = 7, + producer_generation: int = 11, + transport_ref: str = TRANSPORT_REF, +) -> PhysicalCommandConnectionBinding: + return PhysicalCommandConnectionBinding( + intent_id="intent-11111111-1111-4111-8111-111111111111", + transport_ref=transport_ref, + connection_mode="bridge", + target_ipv4="192.168.68.52", + target_port=1883, + host_path_epoch=host_path_epoch, + control_session_id=control_session_id, + producer_generation=producer_generation, + ) + + +def _status( + state: str, + *, + connection: PhysicalCommandConnectionBinding | None = None, + source: str = "live-control-session", + retained: bool = False, + observed_at: str = "2026-08-07T09:29:00.000Z", + project_id_sha256: str = "c" * 64, + identity: PhysicalCommandIdentity = IDENTITY, +) -> PhysicalCommandStatusEvidence: + binding = connection or _connection() + state_code = { + "ready": 300, + "scan_starting": 301, + "scanning": 302, + "scan_stopping": 303, + "scan_over": 304, + }[state] + project_bound = state in {"scan_starting", "scanning", "scan_stopping"} + init_ready = state == "scanning" + return PhysicalCommandStatusEvidence( + source=source, # type: ignore[arg-type] + vendor_device_id_sha256=identity.vendor_device_id_sha256, + device_serial_sha256=identity.device_serial_sha256, + control_session_id=binding.control_session_id, + host_path_epoch=binding.host_path_epoch, + producer_generation=binding.producer_generation, + session_state=state, # type: ignore[arg-type] + session_state_code=state_code, + project_bound=project_bound, + project_id_sha256=project_id_sha256 if project_bound else None, + init_ready=init_ready, + status_message_sha256="d" * 64, + mqtt_retained=retained, + observed_at_utc=observed_at, + ) + + +def _response( + operation_id: str, + action: str, + *, + connection: PhysicalCommandConnectionBinding | None = None, + result_code: int = PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + observed_at: str = "2026-08-07T09:29:30.000Z", +) -> PhysicalCommandApplicationResponse: + binding = connection or _connection() + return PhysicalCommandApplicationResponse( + operation_id=operation_id, + action=action, # type: ignore[arg-type] + control_session_id=binding.control_session_id, + host_path_epoch=binding.host_path_epoch, + producer_generation=binding.producer_generation, + result_code=result_code, + success=result_code == PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE, + payload_sha256="e" * 64, + observed_at_utc=observed_at, + ) + + +def _ledger(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> PhysicalCommandLedger: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + + +def _prepare_start( + ledger: PhysicalCommandLedger, + *, + operation_id: str = START_OPERATION, + parent_operation_id: str | None = None, + connection: PhysicalCommandConnectionBinding | None = None, + acquisition_id: str = ACQUISITION_ID, + baseline_observed_at: str = "2026-08-07T09:29:00.000Z", +) -> None: + binding = connection or _connection() + ledger.prepare( + operation_id=operation_id, + parent_operation_id=parent_operation_id, + acquisition_id=acquisition_id, + action="start", + identity=IDENTITY, + connection=binding, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="f" * 64, + baseline_status=_status( + "ready", + connection=binding, + observed_at=baseline_observed_at, + ), + ) + + +def _complete_start(ledger: PhysicalCommandLedger) -> None: + ledger.mark_dispatching(START_OPERATION) + ledger.mark_observing( + START_OPERATION, + publish_call_returned=True, + packet_id=41, + ) + ledger.record_status_observation( + START_OPERATION, + _status("scanning", observed_at="2026-08-07T09:29:40.000Z"), + ) + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start"), + ) + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + ledger.resolve(START_OPERATION, resolution="start-active-observed") + + +def _prepare_stop( + ledger: PhysicalCommandLedger, + *, + operation_id: str = STOP_OPERATION, + parent_operation_id: str = START_OPERATION, + connection: PhysicalCommandConnectionBinding | None = None, + operator_confirmation_id: str | None = None, + operator_confirmed_at_utc: str | None = None, + baseline_observed_at: str = "2026-08-07T09:29:00.000Z", +) -> None: + binding = connection or _connection() + ledger.prepare( + operation_id=operation_id, + parent_operation_id=parent_operation_id, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=IDENTITY, + connection=binding, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="1" * 64, + baseline_status=_status( + "scanning", + connection=binding, + observed_at=baseline_observed_at, + ), + operator_confirmation_id=operator_confirmation_id, + operator_confirmed_at_utc=operator_confirmed_at_utc, + ) + + +def _reconciliation_connection() -> PhysicalCommandConnectionBinding: + return _connection( + control_session_id="reconcile-55555555-5555-4555-8555-555555555555", + host_path_epoch=8, + producer_generation=12, + ) + + +def test_successor_stop_confirmation_is_reserved_for_reconciled_active_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + + same_session = ledger.snapshot().record + assert same_session is not None + assert same_session.successor_stop_requires_operator_confirmation is False + + reconciled = ledger.rebind_resolved_active( + same_session.operation_id, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + verified_binding=_verified_binding(connection=_reconciliation_connection()), + observation=_status( + "scanning", + connection=_reconciliation_connection(), + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + assert reconciled.successor_stop_requires_operator_confirmation is True + + +def _verified_binding( + *, + identity: PhysicalCommandIdentity = IDENTITY, + connection: PhysicalCommandConnectionBinding | None = None, + verification_id: str = "verification-55555555-5555-4555-8555-555555555555", + verified_at: str = "2026-08-07T09:30:00.000Z", +) -> PhysicalCommandVerifiedBinding: + return PhysicalCommandVerifiedBinding( + verification_id=verification_id, + identity=identity, + connection=connection or _reconciliation_connection(), + device_info_message_sha256="6" * 64, + verified_at_utc=verified_at, + ) + + +def _complete_stop(ledger: PhysicalCommandLedger) -> None: + ledger.mark_dispatching(STOP_OPERATION) + ledger.mark_observing(STOP_OPERATION, publish_call_returned=True, packet_id=42) + ledger.mark_qos2_completed(STOP_OPERATION, packet_id=42) + ledger.record_application_response( + STOP_OPERATION, + _response(STOP_OPERATION, "stop"), + ) + ledger.record_status_observation( + STOP_OPERATION, + _status("ready", observed_at="2026-08-07T09:29:50.000Z"), + ) + ledger.resolve(STOP_OPERATION, resolution="stop-standby-observed") + + +def _prepared_stop_classification( + ledger: PhysicalCommandLedger, + state: str, + *, + reconciliation_id: str = "prepared-stop-classification-1", + connection: PhysicalCommandConnectionBinding | None = None, +) -> tuple[PhysicalCommandConnectionBinding, object]: + proof_connection = connection or _reconciliation_connection() + before = ledger.snapshot().record + assert before is not None + classified = ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=before.revision, + reconciliation_id=reconciliation_id, + verified_binding=_verified_binding( + connection=proof_connection, + verification_id=f"{reconciliation_id}.device-info", + ), + observation=_status( + state, + connection=proof_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + return proof_connection, classified + + +def _classify_named_stop_active( + ledger: PhysicalCommandLedger, + *, + operation_id: str, + connection: PhysicalCommandConnectionBinding, + reconciliation_id: str, + observed_at: str, +) -> ledger_module.PhysicalCommandRecord: + before = ledger.snapshot().record + assert before is not None + return ledger.reconcile_prepared_stop_not_dispatched( + operation_id, + expected_revision=before.revision, + reconciliation_id=reconciliation_id, + verified_binding=_verified_binding( + connection=connection, + verification_id=f"{reconciliation_id}.device-info", + verified_at=observed_at, + ), + observation=_status( + "scanning", + connection=connection, + source="explicit-read-only-reconciliation", + observed_at=observed_at, + ), + ) + + +def _install_classified_stop_chain( + ledger: PhysicalCommandLedger, + *, + include_second_classified_stop: bool, +) -> tuple[PhysicalCommandConnectionBinding, PhysicalCommandConnectionBinding | None]: + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + first_recovery_connection = _connection( + control_session_id="ancestry-control-s0", + host_path_epoch=20, + producer_generation=20, + ) + _classify_named_stop_active( + ledger, + operation_id=STOP_OPERATION, + connection=first_recovery_connection, + reconciliation_id="ancestry-classification-s0", + observed_at="2026-08-07T09:31:00.000Z", + ) + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=first_recovery_connection, + operator_confirmation_id="ancestry-confirmation-s1", + operator_confirmed_at_utc="2026-08-07T09:32:00.000Z", + baseline_observed_at="2026-08-07T09:31:30.000Z", + ) + if not include_second_classified_stop: + return first_recovery_connection, None + second_recovery_connection = _connection( + control_session_id="ancestry-control-s1", + host_path_epoch=21, + producer_generation=21, + ) + _classify_named_stop_active( + ledger, + operation_id=STOP_RETRY_OPERATION, + connection=second_recovery_connection, + reconciliation_id="ancestry-classification-s1", + observed_at="2026-08-07T09:33:00.000Z", + ) + return first_recovery_connection, second_recovery_connection + + +def _retire_ambiguous_stop( + ledger: PhysicalCommandLedger, + *, + retirement_id: str = "retirement-reopen-cycle-1", + observing: bool = True, +): + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + if observing: + ledger.mark_observing( + STOP_OPERATION, + publish_call_returned=True, + packet_id=42, + ) + before = ledger.snapshot().record + assert before is not None + retired = ledger.retire_unavailable_target( + retirement_id=retirement_id, + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + return before, retired + + +@pytest.mark.parametrize( + ("state", "physical_state", "requires_confirmation"), + [ + ("ready", "standby", False), + ("scanning", "active", True), + ], +) +def test_prepared_stop_classification_keeps_no_dispatch_separate_from_physical_state( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state: str, + physical_state: str, + requires_confirmation: bool, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + prepared = ledger.snapshot().record + assert prepared is not None + proof_connection, classified = _prepared_stop_classification(ledger, state) + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + + assert classified.stage == "resolved" + assert classified.resolution == "not-dispatched" + assert classified.original_command_outcome == "not-dispatched" + assert classified.reconciled_physical_state == physical_state + assert ( + classified.successor_stop_requires_operator_confirmation + is requires_confirmation + ) + audit = classified.reconciliations[-1] + assert audit.kind == "prepared-stop-classification" + assert audit.original_attempt.stage == "prepared" + assert audit.original_attempt.action == "stop" + assert audit.original_attempt.revision == prepared.revision + assert audit.resolution == ( + "physical-active-observed" + if state == "scanning" + else "physical-standby-observed" + ) + assert audit.verified_binding.connection == proof_connection + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == classified + restarted_projection = LedgerPhysicalCommandCoordinator(restarted).snapshot() + assert restarted_projection["reconciled_physical_state"] == physical_state + assert restarted_projection["physical_active"] is requires_confirmation + assert ( + restarted_projection["resolved_active_recovery_required"] + is requires_confirmation + ) + + exact_retry = restarted.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-classification-1", + verified_binding=audit.verified_binding, + observation=audit.observation, + ) + assert exact_retry == classified + + +def test_classified_stop_ancestry_proves_direct_and_s1_after_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + direct = _ledger(tmp_path / "direct", monkeypatch) + _prepare_start(direct) + _complete_start(direct) + _prepare_stop(direct) + assert ( + direct.prove_classified_stop_ancestry( + head_operation_id=STOP_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + is None + ) + + chained_root = tmp_path / "chained" + chained = _ledger(chained_root, monkeypatch) + _install_classified_stop_chain(chained, include_second_classified_stop=False) + proof = chained.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + assert proof is not None + assert proof.head_operation_id == STOP_RETRY_OPERATION + assert proof.head_parent_operation_id == STOP_OPERATION + assert proof.root_start_operation_id == START_OPERATION + assert proof.acquisition_id == ACQUISITION_ID + assert proof.depth == 1 + assert len(proof.chain_sha256) == 64 + + restarted = PhysicalCommandLedger(chained_root / "repository", clock=_clock) + assert ( + restarted.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + == proof + ) + with pytest.raises(PhysicalCommandTransitionError, match="head/acquisition"): + restarted.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id="wrong-acquisition", + ) + with pytest.raises(PhysicalCommandTransitionError, match="successor confirmation"): + restarted.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id="wrong-start-operation", + acquisition_id=ACQUISITION_ID, + ) + + +def test_classified_stop_s2_ancestry_survives_forced_rollover_and_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, second_connection = _install_classified_stop_chain( + ledger, + include_second_classified_stop=True, + ) + assert second_connection is not None + before_bytes = ledger.path.read_bytes() + second_retry_operation = "physical-stop-55555555-5555-4555-8555-555555555555" + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(before_bytes) + 128, + ) + _prepare_stop( + ledger, + operation_id=second_retry_operation, + parent_operation_id=STOP_RETRY_OPERATION, + connection=second_connection, + operator_confirmation_id="ancestry-confirmation-s2", + operator_confirmed_at_utc="2026-08-07T09:34:00.000Z", + baseline_observed_at="2026-08-07T09:33:30.000Z", + ) + current = ledger.snapshot().record + assert current is not None + assert current.archive_segment_count == 1 + assert "ancestry-classification-s0" not in { + item.reconciliation_id for item in current.reconciliations + } + proof = ledger.prove_classified_stop_ancestry( + head_operation_id=second_retry_operation, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + assert proof is not None + assert proof.depth == 2 + assert proof.head_parent_operation_id == STOP_RETRY_OPERATION + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert ( + restarted.prove_classified_stop_ancestry( + head_operation_id=second_retry_operation, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + == proof + ) + + +def _install_resolved_start_rebind_rollover( + ledger: PhysicalCommandLedger, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[ + ledger_module.PhysicalCommandRecord, + ledger_module.PhysicalCommandRecord, + Path, +]: + """Create the field shape: old audits compact while one START rebind appends.""" + + _prepare_start(ledger) + _complete_start(ledger) + first_rebind_connection = _reconciliation_connection() + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="archive-proof-historical-rebind", + verified_binding=_verified_binding( + connection=first_rebind_connection, + verification_id="archive-proof-historical-rebind.device-info", + ), + observation=_status( + "scanning", + connection=first_rebind_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + _prepare_stop( + ledger, + connection=first_rebind_connection, + operator_confirmation_id="archive-proof-stop-confirmation", + operator_confirmed_at_utc="2026-08-07T09:31:30.000Z", + baseline_observed_at="2026-08-07T09:31:00.000Z", + ) + ledger.mark_dispatching(STOP_OPERATION) + ledger.mark_observing(STOP_OPERATION, publish_call_returned=True, packet_id=42) + ledger.mark_qos2_completed(STOP_OPERATION, packet_id=42) + ledger.record_application_response( + STOP_OPERATION, + _response(STOP_OPERATION, "stop", connection=first_rebind_connection), + ) + ledger.record_status_observation( + STOP_OPERATION, + _status( + "ready", + connection=first_rebind_connection, + observed_at="2026-08-07T09:32:00.000Z", + ), + ) + ledger.resolve(STOP_OPERATION, resolution="stop-standby-observed") + + current_connection = _connection( + control_session_id="archive-proof-current-control", + host_path_epoch=9, + producer_generation=13, + ) + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="archive-proof-current-acquisition", + connection=current_connection, + baseline_observed_at="2026-08-07T09:32:00.000Z", + ) + ledger.mark_dispatching(NEXT_START_OPERATION) + ledger.mark_observing( + NEXT_START_OPERATION, + publish_call_returned=True, + packet_id=43, + ) + ledger.mark_qos2_completed(NEXT_START_OPERATION, packet_id=43) + ledger.record_application_response( + NEXT_START_OPERATION, + _response(NEXT_START_OPERATION, "start", connection=current_connection), + ) + ledger.record_status_observation( + NEXT_START_OPERATION, + _status( + "scanning", + connection=current_connection, + observed_at="2026-08-07T09:33:00.000Z", + ), + ) + predecessor = ledger.resolve( + NEXT_START_OPERATION, + resolution="start-active-observed", + ) + assert { + item.reconciliation_id for item in predecessor.reconciliations + } == {"archive-proof-historical-rebind"} + + rebound_connection = _connection( + control_session_id="archive-proof-rebound-control", + host_path_epoch=10, + producer_generation=14, + ) + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(ledger.path.read_bytes()) + 128, + ) + current = ledger.rebind_resolved_active( + NEXT_START_OPERATION, + reconciliation_id="archive-proof-current-rebind", + verified_binding=_verified_binding( + connection=rebound_connection, + verification_id="archive-proof-current-rebind.device-info", + ), + observation=_status( + "scanning", + connection=rebound_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:34:00.000Z", + ), + ) + assert current.revision == predecessor.revision + 1 + assert current.archive_segment_count == predecessor.archive_segment_count + 1 + assert [item.reconciliation_id for item in current.reconciliations] == [ + "archive-proof-current-rebind" + ] + assert current.archive_head_sha256 is not None + archive_path = ( + ledger.path.parent + / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + / f"{current.archive_head_sha256}.json" + ) + return predecessor, current, archive_path + + +def test_immediate_archived_reconciliation_successor_proves_exact_compacted_transition( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + predecessor, current, _archive_path = _install_resolved_start_rebind_rollover( + ledger, + monkeypatch, + ) + + assert ledger.prove_immediate_archived_reconciliation_successor( + expected_current=current, + predecessor=predecessor, + ) + assert not ledger.prove_immediate_archived_reconciliation_successor( + expected_current=current, + predecessor=replace( + predecessor, + updated_at_utc="2026-08-07T09:32:59.999Z", + ), + ) + assert not ledger.prove_immediate_archived_reconciliation_successor( + expected_current=replace(current, updated_at_utc="2026-08-07T09:34:00.001Z"), + predecessor=predecessor, + ) + current_reconciliation = current.reconciliations[-1] + with pytest.raises( + ValueError, + match="resolved-active cessation must observe standby state", + ): + replace( + current_reconciliation, + kind="resolved-active-cessation", + ) + + +@pytest.mark.parametrize("archive_fault", ["missing", "corrupt"]) +def test_immediate_archived_reconciliation_successor_fails_closed_on_archive_fault( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + archive_fault: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + predecessor, current, archive_path = _install_resolved_start_rebind_rollover( + ledger, + monkeypatch, + ) + if archive_fault == "missing": + archive_path.unlink() + else: + archive_path.write_bytes(archive_path.read_bytes() + b"\n") + + with pytest.raises(PhysicalCommandLedgerCorrupt): + ledger.prove_immediate_archived_reconciliation_successor( + expected_current=current, + predecessor=predecessor, + ) + + +def test_classified_stop_ancestry_rejects_missing_confirmation_cycle_and_conflict( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + missing = _ledger(tmp_path / "missing", monkeypatch) + _install_classified_stop_chain(missing, include_second_classified_stop=False) + missing_document = json.loads(missing.path.read_text(encoding="utf-8")) + missing_document["operator_confirmations"] = [] + missing.path.write_bytes(ledger_module._canonical_json_bytes(missing_document)) + missing.path.chmod(0o600) + missing_restarted = PhysicalCommandLedger( + tmp_path / "missing" / "repository", + clock=_clock, + ) + with pytest.raises(PhysicalCommandTransitionError, match="successor confirmation"): + missing_restarted.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + + cycle = _ledger(tmp_path / "cycle", monkeypatch) + _install_classified_stop_chain(cycle, include_second_classified_stop=False) + cycle_document = json.loads(cycle.path.read_text(encoding="utf-8")) + cycle_document["parent_operation_id"] = STOP_RETRY_OPERATION + cycle.path.write_bytes(ledger_module._canonical_json_bytes(cycle_document)) + cycle.path.chmod(0o600) + cycle_restarted = PhysicalCommandLedger( + tmp_path / "cycle" / "repository", + clock=_clock, + ) + with pytest.raises(PhysicalCommandLedgerCorrupt, match="operation cycle"): + cycle_restarted.prove_classified_stop_ancestry( + head_operation_id=STOP_RETRY_OPERATION, + expected_start_operation_id=START_OPERATION, + acquisition_id=ACQUISITION_ID, + ) + + conflict = _ledger(tmp_path / "conflict", monkeypatch) + _install_classified_stop_chain(conflict, include_second_classified_stop=False) + conflict_record = conflict.snapshot().record + assert conflict_record is not None + confirmation = conflict_record.operator_confirmations[-1] + history = ledger_module._extend_archive_history( + ledger_module._PhysicalCommandArchiveHistory(), + archived_record=conflict_record, + head_sha256="a" * 64, + segment_bytes=1, + ) + with pytest.raises(PhysicalCommandLedgerCorrupt, match="conflicting operator"): + ledger_module._extend_archive_history( + history, + archived_record=replace( + conflict_record, + operator_confirmations=( + replace(confirmation, confirmation_id="different-confirmation"), + ), + ), + head_sha256="b" * 64, + segment_bytes=1, + ) + + +def test_prepared_stop_ready_survives_restart_and_allows_fresh_live_start_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "ready") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + classification = classified.reconciliations[-1] + fresh_connection = _connection( + control_session_id="fresh-ready-66666666-6666-4666-8666-666666666666", + host_path_epoch=9, + producer_generation=13, + ) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == classified + assert classification.verified_binding.connection != fresh_connection + + with pytest.raises(PhysicalCommandTransitionError, match="permits only fresh START"): + restarted.prepare( + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="7" * 64, + baseline_status=_status("scanning", connection=fresh_connection), + ) + prepared = restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-fresh-after-manual-ready", + action="start", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=_status("ready", connection=fresh_connection), + ) + assert prepared.revision == classified.revision + 1 + assert prepared.parent_operation_id == STOP_OPERATION + assert prepared.connection == fresh_connection + assert prepared.baseline_status == _status("ready", connection=fresh_connection) + assert prepared.reconciliations[-1] == classification + + reopened = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + reopened_record = reopened.snapshot().record + assert reopened_record == prepared + assert reopened_record is not None + assert reopened_record.reconciliations[-1] == classification + + +def test_prepared_stop_ready_fresh_start_keeps_identity_profile_and_live_baseline_gates( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _prepared_stop_classification(ledger, "ready") + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + fresh_connection = _connection( + control_session_id="fresh-ready-77777777-7777-4777-8777-777777777777", + host_path_epoch=9, + producer_generation=13, + ) + valid_status = _status("ready", connection=fresh_connection) + before = restarted.path.read_bytes() + + with pytest.raises(PhysicalCommandTransitionError, match="chain the prior durable operation"): + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + acquisition_id="acquisition-wrong-parent", + action="start", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=valid_status, + ) + wrong_identity = PhysicalCommandIdentity( + vendor_device_id_sha256="8" * 64, + device_serial_sha256="9" * 64, + ) + with pytest.raises(PhysicalCommandTransitionError, match="device identity changed"): + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-wrong-identity", + action="start", + identity=wrong_identity, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=_status( + "ready", + connection=fresh_connection, + identity=wrong_identity, + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="compatibility profile changed"): + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-wrong-profile", + action="start", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.7.0", + payload_sha256="8" * 64, + baseline_status=valid_status, + ) + with pytest.raises(PhysicalCommandTransitionError, match="live-path generation"): + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-wrong-binding", + action="start", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=replace(valid_status, control_session_id="wrong-live-binding"), + ) + for invalid_status in ( + replace(valid_status, source="explicit-read-only-reconciliation"), + replace(valid_status, mqtt_retained=True), + _status("scanning", connection=fresh_connection), + ): + with pytest.raises(PhysicalCommandTransitionError): + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-invalid-live-ready", + action="start", + identity=IDENTITY, + connection=fresh_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=invalid_status, + ) + assert restarted.path.read_bytes() == before + + +def test_prepared_stop_scanning_allows_exactly_one_confirmed_stop_on_fresh_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + proof_connection, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + different_connection = _connection( + control_session_id="wrong-scanning-88888888-8888-4888-8888-888888888888", + host_path_epoch=9, + producer_generation=13, + ) + + with pytest.raises(PhysicalCommandTransitionError, match="freshly verified binding"): + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=different_connection, + operator_confirmation_id="wrong-scanning-confirmation-1", + operator_confirmed_at_utc="2026-08-07T09:32:00.000Z", + ) + + with pytest.raises(PhysicalCommandTransitionError, match="requires explicit"): + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=proof_connection, + ) + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=proof_connection, + operator_confirmation_id="prepared-stop-confirmation-1", + operator_confirmed_at_utc="2026-08-07T09:32:00.000Z", + ) + successor = ledger.snapshot().record + assert successor is not None + assert successor.operator_confirmations[-1].reconciliation_id == ( + "prepared-stop-classification-1" + ) + + +def test_prepared_stop_classification_rejects_stale_retained_mismatch_and_nonlatest_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + prepared = ledger.snapshot().record + assert prepared is not None + proof_connection = _reconciliation_connection() + binding = _verified_binding(connection=proof_connection) + ready = _status( + "ready", + connection=proof_connection, + source="explicit-read-only-reconciliation", + ) + before = ledger.path.read_bytes() + with pytest.raises(PhysicalCommandTransitionError, match="stale"): + ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision - 1, + reconciliation_id="prepared-stop-stale", + verified_binding=binding, + observation=ready, + ) + with pytest.raises(PhysicalCommandTransitionError, match="retained"): + ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-retained", + verified_binding=binding, + observation=replace(ready, mqtt_retained=True), + ) + wrong_project = _status( + "scanning", + connection=proof_connection, + source="explicit-read-only-reconciliation", + project_id_sha256="9" * 64, + ) + with pytest.raises(PhysicalCommandTransitionError, match="same-project"): + ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-wrong-project", + verified_binding=binding, + observation=wrong_project, + ) + assert ledger.path.read_bytes() == before + + _, classified = _prepared_stop_classification(ledger, "ready") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-classification-1", + verified_binding=classified.reconciliations[-1].verified_binding, + observation=replace( + classified.reconciliations[-1].observation, + status_message_sha256="0" * 64, + ), + ) + + +def test_two_processes_share_prepared_stop_classification_exact_cas( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + _prepare_start(first) + _complete_start(first) + _prepare_stop(first) + prepared = first.snapshot().record + assert prepared is not None + stale_second = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + proof_connection = _reconciliation_connection() + request = { + "expected_revision": prepared.revision, + "reconciliation_id": "prepared-stop-process-cas", + "verified_binding": _verified_binding( + connection=proof_connection, + verification_id="prepared-stop-process-cas.device-info", + ), + "observation": _status( + "ready", + connection=proof_connection, + source="explicit-read-only-reconciliation", + ), + } + + committed = first.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **request, # type: ignore[arg-type] + ) + observed = stale_second.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **request, # type: ignore[arg-type] + ) + assert observed == committed + conflicting = dict(request) + conflicting["reconciliation_id"] = "prepared-stop-process-conflict" + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + stale_second.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **conflicting, # type: ignore[arg-type] + ) + + +@pytest.mark.parametrize( + ("state", "physical_state"), + [("ready", "standby"), ("scanning", "active")], +) +def test_resolved_unclassified_stop_requires_fresh_classification_before_successor( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + state: str, + physical_state: str, +) -> None: + first = _ledger(tmp_path, monkeypatch) + _prepare_start(first) + _complete_start(first) + _prepare_stop(first) + locally_resolved = first.resolve(STOP_OPERATION, resolution="not-dispatched") + + assert locally_resolved.resolved_unclassified_stop_recovery_required is True + assert locally_resolved.reconciled_physical_state is None + projection = LedgerPhysicalCommandCoordinator(first).snapshot() + assert projection["status"] == "resolved" + assert projection["requires_reconciliation"] is True + assert projection["resolved_unclassified_stop_recovery_required"] is True + assert projection["recovery_requirement"] == ( + "explicit-read-only-deviceinfo-and-non-retained-devicestatus" + ) + with pytest.raises(PhysicalCommandBlocked, match="physical classification"): + if state == "ready": + _prepare_start( + first, + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=_reconciliation_connection(), + acquisition_id="acquisition-before-stop-classification", + ) + else: + _prepare_stop( + first, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=_reconciliation_connection(), + operator_confirmation_id="confirmation-before-stop-classification", + operator_confirmed_at_utc="2026-08-07T09:31:00.000Z", + ) + + stale_second = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + proof_connection = _reconciliation_connection() + request = { + "expected_revision": locally_resolved.revision, + "reconciliation_id": "resolved-stop-read-only-classification", + "verified_binding": _verified_binding( + connection=proof_connection, + verification_id="resolved-stop-read-only-classification.device-info", + ), + "observation": _status( + state, + connection=proof_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + } + before = first.path.read_bytes() + with pytest.raises(PhysicalCommandTransitionError, match="stale"): + first.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **{**request, "expected_revision": locally_resolved.revision - 1}, + ) + assert first.path.read_bytes() == before + + classified = first.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **request, # type: ignore[arg-type] + ) + exact_cas_retry = stale_second.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **request, # type: ignore[arg-type] + ) + assert exact_cas_retry == classified + assert classified.resolved_unclassified_stop_recovery_required is False + assert classified.reconciled_physical_state == physical_state + audit = classified.reconciliations[-1] + assert audit.kind == "prepared-stop-classification" + assert audit.original_attempt.stage == "resolved" + assert audit.original_attempt.resolution == "not-dispatched" + assert audit.original_attempt.revision == locally_resolved.revision + + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + stale_second.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **{ + **request, + "reconciliation_id": "resolved-stop-conflicting-classification", + }, + ) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == classified + assert restarted.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + **request, # type: ignore[arg-type] + ) == classified + if state == "ready": + _prepare_start( + restarted, + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=proof_connection, + acquisition_id="acquisition-after-resolved-stop-ready", + ) + else: + _prepare_stop( + restarted, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=proof_connection, + operator_confirmation_id="resolved-stop-active-confirmation", + operator_confirmed_at_utc="2026-08-07T09:32:00.000Z", + ) + + +def test_resolved_unclassified_stop_active_classification_supports_fresh_followups( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + locally_resolved = ledger.resolve(STOP_OPERATION, resolution="not-dispatched") + first_connection = _reconciliation_connection() + classified = ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=locally_resolved.revision, + reconciliation_id="resolved-stop-active-classification", + verified_binding=_verified_binding( + connection=first_connection, + verification_id="resolved-stop-active-classification.device-info", + ), + observation=_status( + "scanning", + connection=first_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + second_connection = _connection( + control_session_id="resolved-stop-active-followup-control", + host_path_epoch=9, + producer_generation=13, + ) + rebound = ledger.rebind_resolved_active( + STOP_OPERATION, + reconciliation_id="resolved-stop-active-followup", + verified_binding=_verified_binding( + connection=second_connection, + verification_id="resolved-stop-active-followup.device-info", + ), + observation=_status( + "scanning", + connection=second_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:32:00.000Z", + ), + ) + assert rebound.original_command_outcome == "not-dispatched" + assert rebound.reconciled_physical_state == "active" + assert rebound.reconciliations[0].original_attempt.stage == "resolved" + assert all( + item.original_attempt == classified.reconciliations[0].original_attempt + for item in rebound.reconciliations + if item.original_attempt.operation_id == STOP_OPERATION + ) + + third_connection = _connection( + control_session_id="resolved-stop-standby-followup-control", + host_path_epoch=10, + producer_generation=14, + ) + standby = ledger.observe_resolved_active_standby( + STOP_OPERATION, + reconciliation_id="resolved-stop-standby-followup", + verified_binding=_verified_binding( + connection=third_connection, + verification_id="resolved-stop-standby-followup.device-info", + ), + observation=_status( + "ready", + connection=third_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:33:00.000Z", + ), + ) + assert standby.original_command_outcome == "not-dispatched" + assert standby.reconciled_physical_state == "standby" + assert PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot().record == ( + standby + ) + + +@pytest.mark.parametrize("locally_resolved_before_classification", [False, True]) +def test_near_capacity_undispatched_stop_classification_rolls_archive_once_and_restarts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + locally_resolved_before_classification: bool, +) -> None: + source = _ledger(tmp_path / "source", monkeypatch) + _prepare_start(source) + source.mark_dispatching(START_OPERATION) + source_connection = _reconciliation_connection() + source_resolved = source.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="prepared-stop-archive-source", + resolution="physical-standby-observed", + verified_binding=_verified_binding( + connection=source_connection, + verification_id="prepared-stop-archive-source.device-info", + ), + observation=_status( + "ready", + connection=source_connection, + source="explicit-read-only-reconciliation", + ), + ) + source_audit = source_resolved.reconciliations[-1].as_dict() + + target = _ledger(tmp_path / "target", monkeypatch) + _prepare_start(target) + _complete_start(target) + _prepare_stop(target) + if locally_resolved_before_classification: + target.resolve(STOP_OPERATION, resolution="not-dispatched") + document = json.loads(target.path.read_text(encoding="utf-8")) + historical: list[dict[str, object]] = [] + while True: + index = len(historical) + 1 + audit = copy.deepcopy(source_audit) + audit["reconciliation_id"] = f"prepared-stop-archive-history-{index:03d}" + original = audit["original_attempt"] + assert isinstance(original, dict) + original["operation_id"] = f"prepared-stop-archive-operation-{index:03d}" + audit["original_attempt_sha256"] = ledger_module._sha256_document(original) + verified = audit["verified_binding"] + assert isinstance(verified, dict) + verified["verification_id"] = ( + f"prepared-stop-archive-verification-{index:03d}" + ) + verified_connection = verified["connection"] + assert isinstance(verified_connection, dict) + verified_connection["control_session_id"] = ( + f"prepared-stop-archive-control-{index:03d}" + ) + observation = audit["observation"] + assert isinstance(observation, dict) + observation["control_session_id"] = verified_connection["control_session_id"] + historical.append(audit) + document["reconciliations"] = historical + serialized = ledger_module._canonical_json_bytes(document) + if len(serialized) >= 62_500: + break + assert len(historical) < 40 + assert len(serialized) < ledger_module.PHYSICAL_COMMAND_LEDGER_MAX_BYTES + target.path.write_bytes(serialized) + target.path.chmod(0o600) + loaded = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + prepared = loaded.snapshot().record + assert prepared is not None + assert prepared.stage == ( + "resolved" if locally_resolved_before_classification else "prepared" + ) + assert prepared.resolved_unclassified_stop_recovery_required is ( + locally_resolved_before_classification + ) + + proof_connection = _reconciliation_connection() + classified = loaded.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-archive-classification", + verified_binding=_verified_binding( + connection=proof_connection, + verification_id="prepared-stop-archive-classification.device-info", + ), + observation=_status( + "ready", + connection=proof_connection, + source="explicit-read-only-reconciliation", + ), + ) + assert classified.archive_segment_count == 1 + assert classified.archive_head_sha256 is not None + assert [item.reconciliation_id for item in classified.reconciliations] == [ + "prepared-stop-archive-classification" + ] + archive_path = ( + loaded.path.parent + / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + / f"{classified.archive_head_sha256}.json" + ) + archive_bytes = archive_path.read_bytes() + restarted = PhysicalCommandLedger( + tmp_path / "target" / "repository", + clock=_clock, + ) + assert restarted.snapshot().record == classified + retried = restarted.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=prepared.revision, + reconciliation_id="prepared-stop-archive-classification", + verified_binding=classified.reconciliations[-1].verified_binding, + observation=classified.reconciliations[-1].observation, + ) + assert retried == classified + assert archive_path.read_bytes() == archive_bytes + assert len(tuple(archive_path.parent.glob("*.json"))) == 1 + + next_acquisition = "acquisition-after-prepared-stop-archive" + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id=next_acquisition, + action="start", + identity=IDENTITY, + connection=proof_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="7" * 64, + baseline_status=_status("ready", connection=proof_connection), + ) + restarted.mark_dispatching(NEXT_START_OPERATION) + restarted.mark_observing( + NEXT_START_OPERATION, + publish_call_returned=True, + packet_id=51, + ) + restarted.record_application_response( + NEXT_START_OPERATION, + _response(NEXT_START_OPERATION, "start", connection=proof_connection), + ) + restarted.record_status_observation( + NEXT_START_OPERATION, + _status("scanning", connection=proof_connection), + ) + restarted.mark_qos2_completed(NEXT_START_OPERATION, packet_id=51) + restarted.resolve(NEXT_START_OPERATION, resolution="start-active-observed") + restarted.prepare( + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=NEXT_START_OPERATION, + acquisition_id=next_acquisition, + action="stop", + identity=IDENTITY, + connection=proof_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="8" * 64, + baseline_status=_status("scanning", connection=proof_connection), + ) + second_prepared = restarted.snapshot().record + assert second_prepared is not None + second_proof = _connection( + control_session_id="prepared-stop-archive-second-control", + host_path_epoch=9, + producer_generation=13, + ) + with pytest.raises(PhysicalCommandTransitionError, match="already been used"): + restarted.reconcile_prepared_stop_not_dispatched( + STOP_RETRY_OPERATION, + expected_revision=second_prepared.revision, + reconciliation_id="prepared-stop-archive-history-001", + verified_binding=_verified_binding( + connection=second_proof, + verification_id="prepared-stop-archive-second.device-info", + ), + observation=_status( + "ready", + connection=second_proof, + source="explicit-read-only-reconciliation", + ), + ) + + second_classified = restarted.reconcile_prepared_stop_not_dispatched( + STOP_RETRY_OPERATION, + expected_revision=second_prepared.revision, + reconciliation_id="prepared-stop-archive-second-classification", + verified_binding=_verified_binding( + connection=second_proof, + verification_id="prepared-stop-archive-second-classification.device-info", + ), + observation=_status( + "scanning", + connection=second_proof, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:37:00.000Z", + ), + ) + second_retired = restarted.retire_unavailable_target( + retirement_id="prepared-stop-archive-second-retirement", + expected_operation_id=second_classified.operation_id, + expected_revision=second_classified.revision, + expected_transport_ref=second_classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + second_reopened = restarted.reopen_retired_reconciliation( + reopening_id="prepared-stop-archive-second-reopen", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-archive-second-retirement", + expected_transport_ref=second_retired.connection.transport_ref, + expected_discovery_generation=103, + reason="device-returned-for-explicit-reconciliation", + ) + assert second_reopened.original_command_outcome == "not-dispatched" + assert second_reopened.reconciled_physical_state is None + assert second_reopened.reopened_physical_state_requires_reconciliation is True + assert second_reopened.archive_head_sha256 == classified.archive_head_sha256 + assert second_reopened.archive_segment_count == 1 + assert archive_path.read_bytes() == archive_bytes + assert len(tuple(archive_path.parent.glob("*.json"))) == 1 + + restarted_pending = PhysicalCommandLedger( + tmp_path / "target" / "repository", + clock=_clock, + ) + assert restarted_pending.snapshot().record == second_reopened + exact_reopen_retry = restarted_pending.reopen_retired_reconciliation( + reopening_id="prepared-stop-archive-second-reopen", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-archive-second-retirement", + expected_transport_ref=second_retired.connection.transport_ref, + expected_discovery_generation=103, + reason="device-returned-for-explicit-reconciliation", + ) + assert exact_reopen_retry == second_reopened + assert archive_path.read_bytes() == archive_bytes + + final_proof = _connection( + control_session_id="prepared-stop-archive-final-control", + host_path_epoch=10, + producer_generation=14, + ) + final_ready = restarted_pending.observe_resolved_active_standby( + STOP_RETRY_OPERATION, + reconciliation_id="prepared-stop-archive-final-ready", + verified_binding=_verified_binding( + connection=final_proof, + verification_id="prepared-stop-archive-final-ready.device-info", + ), + observation=_status( + "ready", + connection=final_proof, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:38:00.000Z", + ), + ) + assert final_ready.original_command_outcome == "not-dispatched" + assert final_ready.reconciled_physical_state == "standby" + assert final_ready.reopened_physical_state_requires_reconciliation is False + assert final_ready.archive_head_sha256 == classified.archive_head_sha256 + assert archive_path.read_bytes() == archive_bytes + + +def test_active_prepared_stop_classification_can_retire_and_reopen_without_losing_outcome( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + + retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-active-retirement", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.original_command_outcome == "not-dispatched" + assert retired.reconciled_physical_state is None + assert retired.operator_retirements[-1].original_attempt.resolution == ( + "not-dispatched" + ) + canonical_retired_bytes = ledger.path.read_bytes() + forged = json.loads(canonical_retired_bytes) + forged["reconciliations"] = [] + ledger.path.write_bytes(ledger_module._canonical_json_bytes(forged)) + ledger.path.chmod(0o600) + assert PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot().status == ( + "corrupt" + ) + ledger.path.write_bytes(canonical_retired_bytes) + ledger.path.chmod(0o600) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == retired + + reopened = restarted.reopen_retired_reconciliation( + reopening_id="prepared-stop-active-reopen", + expected_revision=retired.revision, + expected_retirement_id="prepared-stop-active-retirement", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=91, + reason="device-returned-for-explicit-reconciliation", + ) + assert reopened.stage == "resolved" + assert reopened.resolution == "not-dispatched" + assert reopened.original_command_outcome == "not-dispatched" + assert reopened.reconciled_physical_state is None + assert reopened.reopened_physical_state_requires_reconciliation is True + + with pytest.raises(PhysicalCommandBlocked, match="requires fresh read-only"): + _prepare_stop( + restarted, + operation_id="prepared-stop-before-fresh-proof", + parent_operation_id=STOP_OPERATION, + connection=reopened.reconciliations[-1].verified_binding.connection, + operator_confirmation_id="prepared-stop-before-fresh-proof-confirmation", + operator_confirmed_at_utc="2026-08-07T09:32:30.000Z", + ) + with pytest.raises(PhysicalCommandBlocked, match="requires fresh read-only"): + restarted.prepare( + operation_id="prepared-start-before-fresh-proof", + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-before-fresh-proof", + action="start", + identity=IDENTITY, + connection=reopened.reconciliations[-1].verified_binding.connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="2" * 64, + baseline_status=_status( + "ready", + connection=reopened.reconciliations[-1].verified_binding.connection, + ), + ) + restarted_pending = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + pending = restarted_pending.snapshot().record + assert pending == reopened + assert pending is not None + assert pending.reconciled_physical_state is None + assert pending.reopened_physical_state_requires_reconciliation is True + pending_projection = LedgerPhysicalCommandCoordinator(restarted_pending).snapshot() + assert pending_projection["physical_active"] is False + assert pending_projection["reconciled_physical_state"] is None + assert pending_projection["reopened_physical_state_recovery_required"] is True + assert pending_projection["resolved_active_recovery_required"] is True + + followup_connection = _connection( + control_session_id="prepared-stop-followup-control", + host_path_epoch=9, + producer_generation=13, + ) + ceased = restarted_pending.observe_resolved_active_standby( + STOP_OPERATION, + reconciliation_id="prepared-stop-active-followup-ready", + verified_binding=_verified_binding( + connection=followup_connection, + verification_id="prepared-stop-active-followup-ready.device-info", + ), + observation=_status( + "ready", + connection=followup_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:33:00.000Z", + ), + ) + assert ceased.resolution == "not-dispatched" + assert ceased.original_command_outcome == "not-dispatched" + assert ceased.reconciled_physical_state == "standby" + assert ceased.reopened_physical_state_requires_reconciliation is False + assert PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot().record == ceased + + +def test_pending_prepared_stop_reopen_can_retire_reopen_again_before_fresh_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + first_retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-pending-cycle-retirement-1", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + first_reopened = ledger.reopen_retired_reconciliation( + reopening_id="prepared-stop-pending-cycle-reopen-1", + expected_revision=first_retired.revision, + expected_retirement_id="prepared-stop-pending-cycle-retirement-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=101, + reason="device-returned-for-explicit-reconciliation", + ) + assert first_reopened.reconciled_physical_state is None + assert first_reopened.reopened_physical_state_requires_reconciliation is True + + first_process = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + stale_process = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + retirement_request = { + "retirement_id": "prepared-stop-pending-cycle-retirement-2", + "expected_operation_id": STOP_OPERATION, + "expected_revision": first_reopened.revision, + "expected_transport_ref": TRANSPORT_REF, + "reason": "connection-scenario-reset-by-operator", + } + main_before_preflight = first_process.path.read_bytes() + prospective = first_process.preflight_retire_unavailable_target( + **retirement_request, # type: ignore[arg-type] + ) + assert prospective.original_command_outcome == "not-dispatched" + assert prospective.reconciled_physical_state is None + assert first_process.path.read_bytes() == main_before_preflight + + second_retired = first_process.retire_unavailable_target( + **retirement_request, # type: ignore[arg-type] + ) + exact_lost_response_retry = stale_process.retire_unavailable_target( + **retirement_request, # type: ignore[arg-type] + ) + assert second_retired == prospective == exact_lost_response_retry + assert second_retired.original_command_outcome == "not-dispatched" + assert second_retired.reconciled_physical_state is None + assert [item.retirement_id for item in active_operator_retirements(second_retired)] == [ + "prepared-stop-pending-cycle-retirement-2" + ] + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + first_process.retire_unavailable_target( + **{ + **retirement_request, + "reason": "device-permanently-unavailable-or-replaced", + }, # type: ignore[arg-type] + ) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == second_retired + # Force this second reopen across the rollover boundary. The compact live + # frontier retains the newest pair while the immutable predecessor carries + # the older completed pair; neither side may revive its stale SCANNING. + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(restarted.path.read_bytes()) + 64, + ) + second_reopened = restarted.reopen_retired_reconciliation( + reopening_id="prepared-stop-pending-cycle-reopen-2", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-pending-cycle-retirement-2", + expected_transport_ref=TRANSPORT_REF.lower(), + expected_discovery_generation=102, + reason="device-returned-for-explicit-reconciliation", + ) + assert second_reopened.archive_segment_count == 1 + assert second_reopened.archive_head_sha256 is not None + archive_path = ( + restarted.path.parent + / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + / f"{second_reopened.archive_head_sha256}.json" + ) + archive_bytes = archive_path.read_bytes() + canonical_reopened_bytes = restarted.path.read_bytes() + forged_gap = json.loads(canonical_reopened_bytes) + forged_gap["revision"] += 2 + restarted.path.write_bytes(ledger_module._canonical_json_bytes(forged_gap)) + restarted.path.chmod(0o600) + assert PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot().status == ( + "corrupt" + ) + restarted.path.write_bytes(canonical_reopened_bytes) + restarted.path.chmod(0o600) + exact_reopen_retry = PhysicalCommandLedger( + tmp_path / "repository", + clock=_clock, + ).reopen_retired_reconciliation( + reopening_id="prepared-stop-pending-cycle-reopen-2", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-pending-cycle-retirement-2", + expected_transport_ref=TRANSPORT_REF.lower(), + expected_discovery_generation=102, + reason="device-returned-for-explicit-reconciliation", + ) + assert exact_reopen_retry == second_reopened + assert archive_path.read_bytes() == archive_bytes + assert len(tuple(archive_path.parent.glob("*.json"))) == 1 + assert second_reopened.original_command_outcome == "not-dispatched" + assert second_reopened.reconciled_physical_state is None + assert second_reopened.reopened_physical_state_requires_reconciliation is True + assert active_operator_retirements(second_reopened) == () + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + restarted.reopen_retired_reconciliation( + reopening_id="prepared-stop-pending-cycle-reopen-2", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-pending-cycle-retirement-2", + expected_transport_ref=TRANSPORT_REF.lower(), + expected_discovery_generation=103, + reason="device-returned-for-explicit-reconciliation", + ) + with pytest.raises(PhysicalCommandBlocked, match="requires fresh read-only"): + _prepare_stop( + restarted, + operation_id="prepared-stop-after-second-reopen-before-proof", + parent_operation_id=STOP_OPERATION, + connection=second_reopened.reconciliations[-1].verified_binding.connection, + operator_confirmation_id="prepared-stop-after-second-reopen-confirmation", + operator_confirmed_at_utc="2026-08-07T09:35:30.000Z", + ) + with pytest.raises(PhysicalCommandBlocked, match="requires fresh read-only"): + restarted.prepare( + operation_id="prepared-start-after-second-reopen-before-proof", + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-before-second-reopen-proof", + action="start", + identity=IDENTITY, + connection=second_reopened.reconciliations[-1].verified_binding.connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="4" * 64, + baseline_status=_status( + "ready", + connection=second_reopened.reconciliations[-1].verified_binding.connection, + ), + ) + + followup_connection = _connection( + control_session_id="prepared-stop-pending-cycle-ready-control", + host_path_epoch=11, + producer_generation=15, + ) + ready = restarted.observe_resolved_active_standby( + STOP_OPERATION, + reconciliation_id="prepared-stop-pending-cycle-ready", + verified_binding=_verified_binding( + connection=followup_connection, + verification_id="prepared-stop-pending-cycle-ready.device-info", + ), + observation=_status( + "ready", + connection=followup_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:36:00.000Z", + ), + ) + assert ready.original_command_outcome == "not-dispatched" + assert ready.reconciled_physical_state == "standby" + assert ready.reopened_physical_state_requires_reconciliation is False + restarted.prepare( + operation_id="prepared-start-after-pending-retirement-cycle", + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-after-pending-retirement-cycle", + action="start", + identity=IDENTITY, + connection=followup_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="3" * 64, + baseline_status=_status("ready", connection=followup_connection), + ) + + +def test_reopened_active_prepared_stop_can_rebind_then_confirm_one_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-rebind-retirement", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="prepared-stop-rebind-reopen", + expected_revision=retired.revision, + expected_retirement_id="prepared-stop-rebind-retirement", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=92, + reason="device-returned-for-explicit-reconciliation", + ) + exact_reopen_retry = PhysicalCommandLedger( + tmp_path / "repository", + clock=_clock, + ).reopen_retired_reconciliation( + reopening_id="prepared-stop-rebind-reopen", + expected_revision=retired.revision, + expected_retirement_id="prepared-stop-rebind-retirement", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=92, + reason="device-returned-for-explicit-reconciliation", + ) + assert exact_reopen_retry == reopened + assert reopened.reconciled_physical_state is None + assert reopened.reopened_physical_state_requires_reconciliation is True + followup_connection = _connection( + control_session_id="prepared-stop-rebind-control", + host_path_epoch=10, + producer_generation=14, + ) + rebound = ledger.rebind_resolved_active( + STOP_OPERATION, + reconciliation_id="prepared-stop-rebind-scanning", + verified_binding=_verified_binding( + connection=followup_connection, + verification_id="prepared-stop-rebind-scanning.device-info", + ), + observation=_status( + "scanning", + connection=followup_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:34:00.000Z", + ), + ) + assert rebound.revision == reopened.revision + 1 + assert rebound.resolution == "not-dispatched" + assert rebound.original_command_outcome == "not-dispatched" + assert rebound.reconciled_physical_state == "active" + assert rebound.reopened_physical_state_requires_reconciliation is False + assert rebound.successor_stop_requires_operator_confirmation is True + assert PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot().record == rebound + + _prepare_stop( + ledger, + operation_id="prepared-stop-after-rebind-stop", + parent_operation_id=STOP_OPERATION, + connection=followup_connection, + operator_confirmation_id="prepared-stop-after-rebind-confirmation", + operator_confirmed_at_utc="2026-08-07T09:35:00.000Z", + ) + successor = ledger.snapshot().record + assert successor is not None + assert successor.operator_confirmations[-1].reconciliation_id == ( + "prepared-stop-rebind-scanning" + ) + + +def test_reopened_prepared_stop_fresh_scanning_can_retire_before_operator_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + first_retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-rebound-loss-retirement-1", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + ledger.reopen_retired_reconciliation( + reopening_id="prepared-stop-rebound-loss-reopen-1", + expected_revision=first_retired.revision, + expected_retirement_id="prepared-stop-rebound-loss-retirement-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=111, + reason="device-returned-for-explicit-reconciliation", + ) + rebind_connection = _connection( + control_session_id="prepared-stop-rebound-loss-control", + host_path_epoch=12, + producer_generation=16, + ) + rebound = ledger.rebind_resolved_active( + STOP_OPERATION, + reconciliation_id="prepared-stop-rebound-loss-scanning", + verified_binding=_verified_binding( + connection=rebind_connection, + verification_id="prepared-stop-rebound-loss-scanning.device-info", + ), + observation=_status( + "scanning", + connection=rebind_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:39:00.000Z", + ), + ) + assert rebound.reconciled_physical_state == "active" + + second_retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-rebound-loss-retirement-2", + expected_operation_id=rebound.operation_id, + expected_revision=rebound.revision, + expected_transport_ref=rebound.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert second_retired.original_command_outcome == "not-dispatched" + assert second_retired.reconciled_physical_state is None + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == second_retired + second_reopened = restarted.reopen_retired_reconciliation( + reopening_id="prepared-stop-rebound-loss-reopen-2", + expected_revision=second_retired.revision, + expected_retirement_id="prepared-stop-rebound-loss-retirement-2", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=112, + reason="device-returned-for-explicit-reconciliation", + ) + assert second_reopened.reconciled_physical_state is None + assert second_reopened.reopened_physical_state_requires_reconciliation is True + + +def test_ready_prepared_stop_classification_is_retireable_only_by_scenario_reset( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "ready") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + ledger.retire_unavailable_target( + retirement_id="prepared-stop-ready-retirement-denied", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + retired = ledger.retire_unavailable_target( + retirement_id="prepared-stop-ready-scenario-reset", + expected_operation_id=classified.operation_id, + expected_revision=classified.revision, + expected_transport_ref=classified.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.original_command_outcome == "not-dispatched" + assert retired.reconciled_physical_state is None + assert retired.operator_retirements[-1].original_attempt.revision == ( + classified.revision + ) + + +def test_scenario_reset_retires_effective_active_prepared_stop_classification( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + _, classified = _prepared_stop_classification(ledger, "scanning") + assert isinstance(classified, ledger_module.PhysicalCommandRecord) + coordinator = LedgerPhysicalCommandCoordinator(ledger) + + preview = coordinator.preview_connection_scenario_retirement( + retirement_id="prepared-stop-scenario-reset", + ) + assert preview["disposition"] == "operator-retired-outcome-unknown" + result = coordinator.retire_connection_scenario( + retirement_id="prepared-stop-scenario-reset", + expected_revision=classified.revision, + ) + retired = ledger.snapshot().record + assert result["disposition"] == "operator-retired-outcome-unknown" + assert retired is not None + assert retired.original_command_outcome == "not-dispatched" + assert retired.reconciled_physical_state is None + + +def test_private_secret_free_canonical_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + + assert ledger.path == tmp_path / "private-data" / "xgrids-k1" / "physical-command.json" + assert stat.S_IMODE((tmp_path / "private-data").stat().st_mode) == 0o700 + assert stat.S_IMODE(ledger.path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(ledger.path.stat().st_mode) == 0o600 + + document = json.loads(ledger.path.read_text(encoding="utf-8")) + assert document["schema_version"] == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert set(document) == { + "schema_version", + "revision", + "operation_id", + "parent_operation_id", + "acquisition_id", + "action", + "stage", + "identity", + "connection", + "compatibility_profile_id", + "payload_sha256", + "baseline_status", + "publish_call_returned", + "packet_id", + "qos2_completed", + "application_response", + "last_status", + "reconciliations", + "operator_confirmations", + "operator_retirements", + "operator_reconciliation_reopens", + "resolution", + "created_at_utc", + "updated_at_utc", + "archive_head_sha256", + "archive_segment_count", + "archive_total_bytes", + } + serialized = ledger.path.read_text(encoding="utf-8") + for secret in ( + "real-vendor-device-id", + "real-device-serial", + "real-project-name", + "openapi-key", + "mqtt-password", + ): + assert secret not in serialized + assert document["identity"] == { + "vendor_device_id_sha256": "a" * 64, + "device_serial_sha256": "b" * 64, + } + + +def test_v4_document_loads_read_only_and_first_transition_upgrades_to_v5( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + v5_document = json.loads(ledger.path.read_text(encoding="utf-8")) + v5_document["schema_version"] = "missioncore.xgrids-k1-physical-command/v4" + ledger.path.write_bytes(ledger_module._canonical_json_bytes(v5_document)) + ledger.path.chmod(0o600) + old_bytes = ledger.path.read_bytes() + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + loaded = restarted.snapshot().record + assert loaded is not None + assert loaded.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert restarted.path.read_bytes() == old_bytes + + upgraded = restarted.resolve(START_OPERATION, resolution="not-dispatched") + persisted = json.loads(restarted.path.read_text(encoding="utf-8")) + assert upgraded.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert persisted["schema_version"] == PHYSICAL_COMMAND_LEDGER_SCHEMA + + +def _write_v4_archive_fixture( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + main_schema: str, +) -> tuple[Path, Path, bytes]: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + source_record = json.loads(ledger.path.read_text(encoding="utf-8")) + source_record["schema_version"] = ledger_module.PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA + segment_document = { + "schema_version": ledger_module.PHYSICAL_COMMAND_ARCHIVE_SCHEMA, + "sequence": 1, + "predecessor_sha256": None, + "record_sha256": ledger_module._sha256_document(source_record), + "record": source_record, + } + archive_bytes = ledger_module._canonical_json_bytes(segment_document) + archive_head = hashlib.sha256(archive_bytes).hexdigest() + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + archive_dir.mkdir(mode=0o700) + archive_dir.chmod(0o700) + archive_path = archive_dir / f"{archive_head}.json" + archive_path.write_bytes(archive_bytes) + archive_path.chmod(0o600) + + ledger.resolve(START_OPERATION, resolution="not-dispatched") + main_document = json.loads(ledger.path.read_text(encoding="utf-8")) + main_document["schema_version"] = main_schema + main_document["archive_head_sha256"] = archive_head + main_document["archive_segment_count"] = 1 + main_document["archive_total_bytes"] = len(archive_bytes) + ledger.path.write_bytes(ledger_module._canonical_json_bytes(main_document)) + ledger.path.chmod(0o600) + return ledger.path, archive_path, archive_bytes + + +@pytest.mark.parametrize( + "main_schema", + [ + ledger_module.PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + PHYSICAL_COMMAND_LEDGER_SCHEMA, + ], +) +def test_v4_archive_loads_before_migration_and_next_transition_preserves_immutable_bytes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + main_schema: str, +) -> None: + ledger_path, archive_path, archive_bytes = _write_v4_archive_fixture( + tmp_path, + monkeypatch, + main_schema=main_schema, + ) + main_before = ledger_path.read_bytes() + archive_inode = archive_path.stat().st_ino + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + snapshot = restarted.snapshot() + assert snapshot.status == "resolved" + assert snapshot.record is not None + assert snapshot.record.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert snapshot.record.operation_id == START_OPERATION + assert snapshot.record.resolution == "not-dispatched" + assert ledger_path.read_bytes() == main_before + assert archive_path.read_bytes() == archive_bytes + + _prepare_start( + restarted, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + acquisition_id="acquisition-after-v4-archive", + ) + persisted = json.loads(ledger_path.read_text(encoding="utf-8")) + assert persisted["schema_version"] == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert persisted["archive_head_sha256"] == archive_path.stem + assert persisted["archive_segment_count"] == 1 + assert persisted["archive_total_bytes"] == len(archive_bytes) + assert archive_path.read_bytes() == archive_bytes + assert archive_path.stat().st_ino == archive_inode + assert hashlib.sha256(archive_path.read_bytes()).hexdigest() == archive_path.stem + + +def test_v4_archive_remains_immutable_when_v5_rollover_extends_mixed_schema_chain( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = _ledger(tmp_path / "source", monkeypatch) + _prepare_start(source) + source.mark_dispatching(START_OPERATION) + source_connection = _reconciliation_connection() + source_resolved = source.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="mixed-schema-source", + resolution="physical-standby-observed", + verified_binding=_verified_binding( + connection=source_connection, + verification_id="mixed-schema-source.device-info", + ), + observation=_status( + "ready", + connection=source_connection, + source="explicit-read-only-reconciliation", + ), + ) + source_audit = source_resolved.reconciliations[-1].as_dict() + + ledger_path, v4_archive_path, v4_archive_bytes = _write_v4_archive_fixture( + tmp_path / "target", + monkeypatch, + main_schema=ledger_module.PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA, + ) + main_document = json.loads(ledger_path.read_text(encoding="utf-8")) + historical: list[dict[str, object]] = [] + while len(ledger_module._canonical_json_bytes(main_document)) < 30_000: + index = len(historical) + 1 + audit = copy.deepcopy(source_audit) + audit["reconciliation_id"] = f"mixed-schema-history-{index:03d}" + original = audit["original_attempt"] + assert isinstance(original, dict) + original["operation_id"] = f"mixed-schema-operation-{index:03d}" + audit["original_attempt_sha256"] = ledger_module._sha256_document(original) + verified = audit["verified_binding"] + assert isinstance(verified, dict) + verified["verification_id"] = f"mixed-schema-verification-{index:03d}" + verified_connection = verified["connection"] + assert isinstance(verified_connection, dict) + verified_connection["control_session_id"] = f"mixed-schema-control-{index:03d}" + observation = audit["observation"] + assert isinstance(observation, dict) + observation["control_session_id"] = verified_connection["control_session_id"] + historical.append(audit) + main_document["reconciliations"] = historical + ledger_path.write_bytes(ledger_module._canonical_json_bytes(main_document)) + ledger_path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + before = restarted.snapshot() + assert before.status == "resolved" + assert before.record is not None + assert before.record.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + v4_archive_inode = v4_archive_path.stat().st_ino + main_size = len(ledger_path.read_bytes()) + with monkeypatch.context() as bounded: + bounded.setattr(ledger_module, "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", main_size) + _prepare_start( + restarted, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + acquisition_id="acquisition-after-mixed-schema-rollover", + ) + + extended = restarted.snapshot().record + assert extended is not None + assert extended.archive_segment_count == 2 + assert extended.archive_head_sha256 is not None + v5_archive_path = ( + v4_archive_path.parent / f"{extended.archive_head_sha256}.json" + ) + v5_archive_bytes = v5_archive_path.read_bytes() + v5_archive_inode = v5_archive_path.stat().st_ino + assert json.loads(v4_archive_bytes)["record"]["schema_version"] == ( + ledger_module.PHYSICAL_COMMAND_LEDGER_PREVIOUS_SCHEMA + ) + assert json.loads(v5_archive_bytes)["record"]["schema_version"] == ( + PHYSICAL_COMMAND_LEDGER_SCHEMA + ) + assert v4_archive_path.read_bytes() == v4_archive_bytes + assert v4_archive_path.stat().st_ino == v4_archive_inode + + reloaded = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert reloaded.snapshot().record == extended + assert v4_archive_path.read_bytes() == v4_archive_bytes + assert v4_archive_path.stat().st_ino == v4_archive_inode + assert v5_archive_path.read_bytes() == v5_archive_bytes + assert v5_archive_path.stat().st_ino == v5_archive_inode + + +@pytest.mark.parametrize("tamper", ["record-hash", "noncanonical-encoding"]) +def test_v4_archive_tamper_or_noncanonical_encoding_still_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + tamper: str, +) -> None: + ledger_path, archive_path, _archive_bytes = _write_v4_archive_fixture( + tmp_path, + monkeypatch, + main_schema=PHYSICAL_COMMAND_LEDGER_SCHEMA, + ) + segment_document = json.loads(archive_path.read_text(encoding="utf-8")) + if tamper == "record-hash": + segment_document["record"]["payload_sha256"] = "0" * 64 + forged_bytes = ledger_module._canonical_json_bytes(segment_document) + else: + forged_bytes = ( + json.dumps(segment_document, ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ).encode("utf-8") + forged_head = hashlib.sha256(forged_bytes).hexdigest() + forged_path = archive_path.parent / f"{forged_head}.json" + forged_path.write_bytes(forged_bytes) + forged_path.chmod(0o600) + main_document = json.loads(ledger_path.read_text(encoding="utf-8")) + main_document["archive_head_sha256"] = forged_head + main_document["archive_total_bytes"] = len(forged_bytes) + ledger_path.write_bytes(ledger_module._canonical_json_bytes(main_document)) + ledger_path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + snapshot = restarted.snapshot() + assert snapshot.status == "corrupt" + assert snapshot.record is None + with pytest.raises(PhysicalCommandLedgerCorrupt): + restarted.require_no_unresolved_attempt() + + +def test_unresolved_attempt_survives_restart_and_never_allows_automatic_replay( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + snapshot = restarted.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.requires_reconciliation + assert snapshot.automatic_replay_allowed is False + assert snapshot.record is not None + assert snapshot.record.stage == "dispatching" + assert snapshot.record.automatic_replay_allowed is False + with pytest.raises(PhysicalCommandBlocked): + restarted.require_no_unresolved_attempt() + with pytest.raises(PhysicalCommandBlocked): + _prepare_start(restarted, operation_id=NEXT_START_OPERATION) + + +def test_operator_retirement_is_exact_idempotent_persistent_and_keeps_outcome_unknown( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + ledger.mark_observing(STOP_OPERATION, publish_call_returned=True, packet_id=42) + ledger.mark_qos2_completed(STOP_OPERATION, packet_id=42) + ledger.record_application_response( + STOP_OPERATION, + _response(STOP_OPERATION, "stop"), + ) + before = ledger.snapshot().record + assert before is not None + + retired = ledger.retire_unavailable_target( + retirement_id="retirement-55555555-5555-4555-8555-555555555555", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert retired.stage == "resolved" + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.original_command_outcome == "unknown" + assert retired.reconciled_physical_state is None + assert retired.automatic_replay_allowed is False + assert len(retired.operator_retirements) == 1 + retirement = retired.operator_retirements[0] + assert retirement.original_attempt.operation_id == STOP_OPERATION + assert retirement.original_attempt.revision == before.revision + assert retirement.original_attempt.stage == "observing" + assert retirement.retired_transport_ref == TRANSPORT_REF + + same = ledger.retire_unavailable_target( + retirement_id=retirement.retirement_id, + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + assert same == retired + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + ledger.retire_unavailable_target( + retirement_id="retirement-conflicting-retry", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + persisted = restarted.snapshot().record + assert persisted == retired + assert restarted.snapshot().status == "resolved" + + +def test_proven_active_target_can_be_retired_without_inventing_standby( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + active = ledger.snapshot().record + assert active is not None + assert active.resolution == "start-active-observed" + + retired = ledger.retire_unavailable_target( + retirement_id="retirement-proven-active-power-loss", + expected_operation_id=active.operation_id, + expected_revision=active.revision, + expected_transport_ref=active.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.original_command_outcome == "unknown" + assert retired.reconciled_physical_state is None + assert retired.operator_retirements[-1].original_attempt.resolution == ("start-active-observed") + assert ( + PhysicalCommandLedger( + tmp_path / "repository", + clock=_clock, + ) + .snapshot() + .record + == retired + ) + + +def test_read_only_active_resolution_from_dispatching_can_be_retired( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + recovery_connection = _reconciliation_connection() + recovered_active = ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-dispatching-active-retirement", + resolution="physical-active-observed", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-dispatching-active-retirement", + ), + observation=_status( + "scanning", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + + retired = ledger.retire_unavailable_target( + retirement_id="retirement-dispatching-active-power-loss", + expected_operation_id=recovered_active.operation_id, + expected_revision=recovered_active.revision, + expected_transport_ref=recovered_active.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.operator_retirements[-1].original_attempt.stage == "resolved" + assert retired.operator_retirements[-1].original_attempt.resolution == ( + "physical-active-observed" + ) + assert ( + PhysicalCommandLedger( + tmp_path / "repository", + clock=_clock, + ) + .snapshot() + .record + == retired + ) + + +def test_proven_active_retire_reopen_reconcile_and_retire_again_is_restart_safe( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + active = ledger.snapshot().record + assert active is not None + first_retired = ledger.retire_unavailable_target( + retirement_id="retirement-proven-active-cycle-1", + expected_operation_id=active.operation_id, + expected_revision=active.revision, + expected_transport_ref=active.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-proven-active-cycle-1", + expected_revision=first_retired.revision, + expected_retirement_id="retirement-proven-active-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=21, + reason="device-returned-for-explicit-reconciliation", + ) + assert reopened.stage == "observing" + assert reopened.resolution is None + + recovery_connection = _reconciliation_connection() + recovered_active = ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-proven-active-cycle-1", + resolution="physical-active-observed", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-proven-active-cycle-1", + ), + observation=_status( + "scanning", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + second_retired = ledger.retire_unavailable_target( + retirement_id="retirement-proven-active-cycle-2", + expected_operation_id=recovered_active.operation_id, + expected_revision=recovered_active.revision, + expected_transport_ref=recovered_active.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert second_retired.resolution == "operator-retired-outcome-unknown" + assert second_retired.original_command_outcome == "unknown" + assert second_retired.operator_retirements[-1].original_attempt.resolution == ( + "physical-active-observed" + ) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == second_retired + + forged = json.loads(ledger.path.read_text(encoding="utf-8")) + forged["reconciliations"] = [] + ledger.path.write_text( + json.dumps(forged, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + corrupted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot() + assert corrupted.status == "corrupt" + assert corrupted.record is None + + +def test_repeated_resolved_start_retire_reopen_settles_fresh_ready_as_cessation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A reset reopen must not launder a proven START into ambiguity. + + This is the durable shape produced by an old confirmed acquisition followed + by more than one scenario reset before a new network Apply observes READY. + The original START and its newest active rebind remain proven; the fresh + READY therefore appends a resolved-active cessation instead of a second + ambiguous-outcome classification for the same operation. + """ + + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + first_rebind_connection = _connection( + control_session_id="resolved-start-reset-rebind-control", + host_path_epoch=7, + producer_generation=11, + ) + rebound = ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="resolved-start-reset-rebind", + verified_binding=_verified_binding( + connection=first_rebind_connection, + verification_id="resolved-start-reset-rebind.device-info", + ), + observation=_status( + "scanning", + connection=first_rebind_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + + current = rebound + for cycle in (1, 2): + retired = ledger.retire_unavailable_target( + retirement_id=f"resolved-start-reset-retirement-{cycle}", + expected_operation_id=current.operation_id, + expected_revision=current.revision, + expected_transport_ref=current.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + current = ledger.reopen_retired_reconciliation( + reopening_id=f"resolved-start-reset-reopen-{cycle}", + expected_revision=retired.revision, + expected_retirement_id=f"resolved-start-reset-retirement-{cycle}", + expected_transport_ref=retired.connection.transport_ref, + expected_discovery_generation=40 + cycle, + reason="reset-network-intent-read-only-settlement", + ) + + assert current.stage == "observing" + assert current.resolution is None + assert current.reconciled_physical_state == "active" + + ready_connection = _connection( + control_session_id="resolved-start-reset-ready-control", + host_path_epoch=10, + producer_generation=14, + ) + settled = ledger.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="resolved-start-reset-ready", + verified_binding=_verified_binding( + connection=ready_connection, + verification_id="resolved-start-reset-ready.device-info", + ), + observation=_status( + "ready", + connection=ready_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:34:00.000Z", + ), + ) + + assert settled.stage == "resolved" + assert settled.resolution == "start-active-observed" + assert settled.reconciled_physical_state == "standby" + assert settled.reconciliations[-1].kind == "resolved-active-cessation" + assert settled.reconciliations[-1].original_attempt.operation_id == START_OPERATION + + +def test_legacy_v1_unresolved_attempt_retires_atomically_into_strict_v2( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + + legacy_document = json.loads(ledger.path.read_text(encoding="utf-8")) + legacy_document["schema_version"] = PHYSICAL_COMMAND_LEDGER_LEGACY_SCHEMA + legacy_document.pop("operator_retirements") + legacy_document.pop("operator_reconciliation_reopens") + legacy_document.pop("archive_head_sha256") + legacy_document.pop("archive_segment_count") + legacy_document.pop("archive_total_bytes") + ledger.path.write_text( + json.dumps(legacy_document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + legacy_bytes = ledger.path.read_bytes() + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + before = restarted.snapshot().record + assert before is not None + assert before.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert before.operator_retirements == () + assert restarted.path.read_bytes() == legacy_bytes + + retired = restarted.retire_unavailable_target( + retirement_id="retirement-v1-migration", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + persisted = json.loads(restarted.path.read_text(encoding="utf-8")) + assert persisted["schema_version"] == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert persisted["operator_retirements"][0]["retirement_id"] == ("retirement-v1-migration") + assert retired.resolution == "operator-retired-outcome-unknown" + assert retired.operator_retirements[0].original_attempt.stage == "dispatching" + + +def test_operator_retirement_size_preflight_preserves_unresolved_file_and_hash( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + before_record = ledger.snapshot().record + assert before_record is not None + before_bytes = ledger.path.read_bytes() + before_hash = hashlib.sha256(before_bytes).hexdigest() + before_inode = ledger.path.stat().st_ino + # The existing unresolved v2 document still fits, while adding the full + # immutable attempt audit cannot. The transition must fail before replace. + monkeypatch.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(before_bytes) + 64, + ) + + with pytest.raises(PhysicalCommandBlocked, match="requires archival"): + ledger.retire_unavailable_target( + retirement_id="retirement-too-large", + expected_operation_id=before_record.operation_id, + expected_revision=before_record.revision, + expected_transport_ref=before_record.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + assert ledger.path.read_bytes() == before_bytes + assert hashlib.sha256(ledger.path.read_bytes()).hexdigest() == before_hash + assert ledger.path.stat().st_ino == before_inode + unchanged = ledger.snapshot().record + assert unchanged == before_record + assert unchanged is not None and unchanged.unresolved + + +def test_two_instances_share_exact_retirement_idempotency_after_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + _prepare_start(first) + _complete_start(first) + _prepare_stop(first) + first.mark_dispatching(STOP_OPERATION) + before = first.snapshot().record + assert before is not None + stale_second = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert stale_second.snapshot().record == before + + request = { + "retirement_id": "retirement-shared-idempotency", + "expected_operation_id": before.operation_id, + "expected_revision": before.revision, + "expected_transport_ref": before.connection.transport_ref, + "reason": "device-permanently-unavailable-or-replaced", + } + committed = first.retire_unavailable_target(**request) # type: ignore[arg-type] + observed = stale_second.retire_unavailable_target(**request) # type: ignore[arg-type] + + assert observed == committed + assert observed.revision == before.revision + 1 + assert len(observed.operator_retirements) == 1 + + +def test_v2_retirement_reopens_to_v4_persists_and_preserves_original_audit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(ledger) + document = json.loads(ledger.path.read_text(encoding="utf-8")) + original_retirement = document["operator_retirements"][0] + document["schema_version"] = PHYSICAL_COMMAND_LEDGER_V2_SCHEMA + document.pop("operator_reconciliation_reopens") + document.pop("archive_head_sha256") + document.pop("archive_segment_count") + document.pop("archive_total_bytes") + ledger.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + + migrated = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + v2_record = migrated.snapshot().record + assert v2_record is not None + assert v2_record.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert v2_record.operator_reconciliation_reopens == () + reopened = migrated.reopen_retired_reconciliation( + reopening_id="reopening-v2-migration", + expected_revision=retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF.lower(), + expected_discovery_generation=7, + reason="device-returned-for-explicit-reconciliation", + ) + + assert reopened.stage == "observing" + assert reopened.resolution is None + assert reopened.original_command_outcome == "unknown" + assert active_operator_retirements(reopened) == () + assert reopened.operator_retirements[0].as_dict() == original_retirement + persisted = json.loads(migrated.path.read_text(encoding="utf-8")) + assert persisted["schema_version"] == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert persisted["operator_retirements"][0] == original_retirement + assert ( + persisted["operator_reconciliation_reopens"][0]["reopened_transport_ref"] + == TRANSPORT_REF.lower() + ) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == reopened + + +def test_reopen_is_process_locked_lost_response_idempotent_and_exact_cas( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(first) + stale_second = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + request = { + "reopening_id": "reopening-lost-response", + "expected_revision": retired.revision, + "expected_retirement_id": "retirement-reopen-cycle-1", + "expected_transport_ref": TRANSPORT_REF, + "expected_discovery_generation": 9, + "reason": "device-returned-for-explicit-reconciliation", + } + + committed = first.reopen_retired_reconciliation(**request) # type: ignore[arg-type] + observed = stale_second.reopen_retired_reconciliation(**request) # type: ignore[arg-type] + assert observed == committed + assert observed.revision == retired.revision + 1 + assert len(observed.operator_reconciliation_reopens) == 1 + + conflicting = dict(request) + conflicting["expected_discovery_generation"] = 10 + with pytest.raises(PhysicalCommandTransitionError, match="conflicts"): + first.reopen_retired_reconciliation(**conflicting) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("expected_revision", 1, "stale record checkpoint"), + ("expected_retirement_id", "retirement-wrong", "not active"), + ( + "expected_transport_ref", + "B272E0E6-D463-2170-E541-6FC1CD24A8A0", + "does not match retirement", + ), + ], +) +def test_reopen_rejects_wrong_checkpoint_without_mutating_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: object, + message: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(ledger) + before = ledger.path.read_bytes() + request: dict[str, object] = { + "reopening_id": "reopening-wrong-cas", + "expected_revision": retired.revision, + "expected_retirement_id": "retirement-reopen-cycle-1", + "expected_transport_ref": TRANSPORT_REF, + "expected_discovery_generation": 11, + "reason": "device-returned-for-explicit-reconciliation", + } + request[field] = value + + with pytest.raises(PhysicalCommandTransitionError, match=message): + ledger.reopen_retired_reconciliation(**request) # type: ignore[arg-type] + assert ledger.path.read_bytes() == before + + +@pytest.mark.parametrize( + ("observing", "state", "resolution"), + [ + (False, "ready", "physical-standby-observed"), + (False, "scanning", "physical-active-observed"), + (True, "ready", "physical-standby-observed"), + (True, "scanning", "physical-active-observed"), + ], +) +def test_dispatching_or_observing_retirement_reopens_for_exact_read_only_proof( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + observing: bool, + state: str, + resolution: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(ledger, observing=observing) + reopened = ledger.reopen_retired_reconciliation( + reopening_id=f"reopening-{observing}-{state}", + expected_revision=retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=12, + reason="device-returned-for-explicit-reconciliation", + ) + assert reopened.stage == ("observing" if observing else "dispatching") + proof_connection = _reconciliation_connection() + resolved = ledger.reconcile_ambiguous( + STOP_OPERATION, + reconciliation_id=f"reconciliation-{observing}-{state}", + resolution=resolution, # type: ignore[arg-type] + verified_binding=_verified_binding(connection=proof_connection), + observation=_status( + state, + connection=proof_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + assert resolved.resolution == resolution + assert resolved.original_command_outcome == "unknown" + + +def test_retire_reopen_failed_verify_retire_again_keeps_one_active_cycle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, first_retired = _retire_ambiguous_stop(ledger) + first_reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-cycle-1", + expected_revision=first_retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=13, + reason="device-returned-for-explicit-reconciliation", + ) + second_retired = ledger.retire_unavailable_target( + retirement_id="retirement-reopen-cycle-2", + expected_operation_id=STOP_OPERATION, + expected_revision=first_reopened.revision, + expected_transport_ref=TRANSPORT_REF, + reason="device-permanently-unavailable-or-replaced", + ) + assert [item.retirement_id for item in active_operator_retirements(second_retired)] == [ + "retirement-reopen-cycle-2" + ] + assert len(second_retired.operator_retirements) == 2 + assert len(second_retired.operator_reconciliation_reopens) == 1 + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == second_retired + + second_reopened = restarted.reopen_retired_reconciliation( + reopening_id="reopening-cycle-2", + expected_revision=second_retired.revision, + expected_retirement_id="retirement-reopen-cycle-2", + expected_transport_ref=TRANSPORT_REF.lower(), + expected_discovery_generation=14, + reason="device-returned-for-explicit-reconciliation", + ) + assert active_operator_retirements(second_reopened) == () + + +def test_four_retirement_reopen_cycles_with_later_reconciliation_reload_cleanly( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A later proof belongs only to the cycle whose reopen preceded it. + + A durable field record can contain several failed Verify cycles for one + physical operation, followed by a successful read-only reconciliation. + Parsing each earlier retirement pair must not count that future proof when + checking the pair's contiguous revision window. + """ + + ledger = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(ledger) + + for cycle in range(1, 5): + reopened = ledger.reopen_retired_reconciliation( + reopening_id=f"reopening-multi-cycle-{cycle}", + expected_revision=retired.revision, + expected_retirement_id=f"retirement-reopen-cycle-{cycle}", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=20 + cycle, + reason="device-returned-for-explicit-reconciliation", + ) + if cycle < 4: + retired = ledger.retire_unavailable_target( + retirement_id=f"retirement-reopen-cycle-{cycle + 1}", + expected_operation_id=STOP_OPERATION, + expected_revision=reopened.revision, + expected_transport_ref=TRANSPORT_REF, + reason="device-permanently-unavailable-or-replaced", + ) + + recovery_connection = _reconciliation_connection() + resolved = ledger.reconcile_ambiguous( + STOP_OPERATION, + reconciliation_id="reconciliation-after-four-retirement-cycles", + resolution="physical-standby-observed", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-after-four-retirement-cycles", + ), + observation=_status( + "ready", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + + retirement_revisions = [ + item.original_attempt.revision for item in resolved.operator_retirements + ] + reopened_record_revisions = [ + item.retired_record_revision for item in resolved.operator_reconciliation_reopens + ] + assert len(retirement_revisions) == 4 + assert len(reopened_record_revisions) == 4 + assert all( + current_retirement_revision == prior_retired_record_revision + 1 + for prior_retired_record_revision, current_retirement_revision in zip( + reopened_record_revisions, + retirement_revisions[1:], + strict=False, + ) + ) + later_reconciliation_revision = resolved.reconciliations[-1].original_attempt.revision + assert later_reconciliation_revision == reopened.revision + assert later_reconciliation_revision > retirement_revisions[-1] + + before_reload = ledger.path.read_bytes() + before_inode = ledger.path.stat().st_ino + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + restarted_again = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + + assert restarted.snapshot().status == "resolved" + assert restarted.snapshot().record == resolved + assert restarted_again.snapshot().record == resolved + assert ledger.path.read_bytes() == before_reload + assert ledger.path.stat().st_ino == before_inode + + +def _write_near_capacity_v3_four_cycle_ledger( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[PhysicalCommandLedger, dict[str, object]]: + """Build a secret-free analogue of the 62,718-byte field ledger.""" + + source = _ledger(tmp_path / "source", monkeypatch) + _, source_retired = _retire_ambiguous_stop(source) + source.reopen_retired_reconciliation( + reopening_id="reopening-source-proof", + expected_revision=source_retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=70, + reason="device-returned-for-explicit-reconciliation", + ) + source_connection = _reconciliation_connection() + source_resolved = source.reconcile_ambiguous( + STOP_OPERATION, + reconciliation_id="reconciliation-source-proof", + resolution="physical-standby-observed", + verified_binding=_verified_binding(connection=source_connection), + observation=_status( + "ready", + connection=source_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + source_audit = source_resolved.reconciliations[-1].as_dict() + + target = _ledger(tmp_path / "target", monkeypatch) + _, retired = _retire_ambiguous_stop(target) + for cycle in range(1, 5): + reopened = target.reopen_retired_reconciliation( + reopening_id=f"reopening-capacity-cycle-{cycle}", + expected_revision=retired.revision, + expected_retirement_id=f"retirement-reopen-cycle-{cycle}", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=80 + cycle, + reason="device-returned-for-explicit-reconciliation", + ) + if cycle < 4: + retired = target.retire_unavailable_target( + retirement_id=f"retirement-reopen-cycle-{cycle + 1}", + expected_operation_id=STOP_OPERATION, + expected_revision=reopened.revision, + expected_transport_ref=TRANSPORT_REF, + reason="device-permanently-unavailable-or-replaced", + ) + + document = json.loads(target.path.read_text(encoding="utf-8")) + historical: list[dict[str, object]] = [] + while True: + index = len(historical) + 1 + audit = copy.deepcopy(source_audit) + audit["reconciliation_id"] = f"historical-reconciliation-{index:03d}" + original = audit["original_attempt"] + assert isinstance(original, dict) + original["operation_id"] = f"historical-stop-operation-{index:03d}" + response = original.get("application_response") + if isinstance(response, dict): + response["operation_id"] = original["operation_id"] + audit["original_attempt_sha256"] = ledger_module._sha256_document(original) + verified = audit["verified_binding"] + assert isinstance(verified, dict) + verified["verification_id"] = f"historical-verification-{index:03d}" + verified_connection = verified["connection"] + assert isinstance(verified_connection, dict) + verified_connection["control_session_id"] = f"historical-control-{index:03d}" + observation = audit["observation"] + assert isinstance(observation, dict) + observation["control_session_id"] = verified_connection["control_session_id"] + historical.append(audit) + document["reconciliations"] = historical + serialized = ledger_module._canonical_json_bytes(document) + if len(serialized) >= 62_500: + break + assert len(historical) < 30 + assert len(serialized) < ledger_module.PHYSICAL_COMMAND_LEDGER_MAX_BYTES + + padded = historical[0] + padded["reconciliation_id"] = "historical-reconciliation-padding-" + ("r" * 120) + padded_original = padded["original_attempt"] + assert isinstance(padded_original, dict) + padded_original["operation_id"] = "historical-stop-padding-" + ("o" * 120) + padded_response = padded_original.get("application_response") + if isinstance(padded_response, dict): + padded_response["operation_id"] = padded_original["operation_id"] + padded["original_attempt_sha256"] = ledger_module._sha256_document(padded_original) + padded_verified = padded["verified_binding"] + assert isinstance(padded_verified, dict) + padded_verified["verification_id"] = "historical-verification-padding-" + ("v" * 120) + padded_connection = padded_verified["connection"] + assert isinstance(padded_connection, dict) + padded_connection["control_session_id"] = "historical-control-padding-" + ("c" * 120) + padded_observation = padded["observation"] + assert isinstance(padded_observation, dict) + padded_observation["control_session_id"] = padded_connection["control_session_id"] + + document["schema_version"] = PHYSICAL_COMMAND_LEDGER_V3_SCHEMA + document.pop("archive_head_sha256") + document.pop("archive_segment_count") + document.pop("archive_total_bytes") + serialized = ledger_module._canonical_json_bytes(document) + assert 63_000 <= len(serialized) < ledger_module.PHYSICAL_COMMAND_LEDGER_MAX_BYTES + target.path.write_bytes(serialized) + target.path.chmod(0o600) + loaded = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + snapshot = loaded.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.schema_version == PHYSICAL_COMMAND_LEDGER_SCHEMA + assert len(snapshot.record.operator_retirements) == 4 + assert len(snapshot.record.operator_reconciliation_reopens) == 4 + return loaded, document + + +def test_near_capacity_v3_four_cycles_accept_fifth_reset_and_reload_archive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _v3_document = _write_near_capacity_v3_four_cycle_ledger( + tmp_path, + monkeypatch, + ) + before = ledger.snapshot().record + assert before is not None + coordinator = LedgerPhysicalCommandCoordinator(ledger) + + committed = coordinator.retire_connection_scenario( + retirement_id="scenario-reset-fifth-retirement", + expected_revision=before.revision, + ) + record = ledger.snapshot().record + assert committed["disposition"] == "operator-retired-outcome-unknown" + assert record is not None + assert record.revision == before.revision + 1 + assert record.archive_segment_count == 1 + assert record.archive_head_sha256 is not None + assert len(ledger.path.read_bytes()) < ledger_module.PHYSICAL_COMMAND_LEDGER_MAX_BYTES + assert [item.retirement_id for item in active_operator_retirements(record)] == [ + "scenario-reset-fifth-retirement" + ] + # The live frontier retains the just-closed cycle and the new active fence; + # older complete history remains content-addressed in the archive segment. + assert record.operator_retirements[-1].retirement_id == ("scenario-reset-fifth-retirement") + archive_path = ( + ledger.path.parent + / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + / f"{record.archive_head_sha256}.json" + ) + assert archive_path.is_file() + assert stat.S_IMODE(archive_path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(archive_path.stat().st_mode) == 0o600 + + main_before = ledger.path.read_bytes() + main_inode = ledger.path.stat().st_ino + archive_before = archive_path.read_bytes() + archive_inode = archive_path.stat().st_ino + first_reload = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + second_reload = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert first_reload.snapshot().record == record + assert second_reload.snapshot().record == record + assert ledger.path.read_bytes() == main_before + assert ledger.path.stat().st_ino == main_inode + assert archive_path.read_bytes() == archive_before + assert archive_path.stat().st_ino == archive_inode + + # Lost-response retry observes the exact committed checkpoint and does not + # append or republish an archive segment. + retried = coordinator.retire_connection_scenario( + retirement_id="scenario-reset-fifth-retirement", + expected_revision=before.revision, + ) + assert retried == committed + assert ledger.snapshot().record == record + assert tuple(archive_path.parent.iterdir()) == (archive_path,) + + +def test_retirement_preflight_prepares_exact_archive_without_committing_main( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + main_bytes = ledger.path.read_bytes() + main_inode = ledger.path.stat().st_ino + request = { + "retirement_id": "scenario-reset-preflight-exact-rollover", + "expected_operation_id": before.operation_id, + "expected_revision": before.revision, + "expected_transport_ref": before.connection.transport_ref, + "reason": "connection-scenario-reset-by-operator", + } + + prospective = ledger.preflight_retire_unavailable_target(**request) # type: ignore[arg-type] + + assert prospective.revision == before.revision + 1 + assert prospective.archive_segment_count == 1 + assert prospective.archive_head_sha256 is not None + assert ledger.path.read_bytes() == main_bytes + assert ledger.path.stat().st_ino == main_inode + assert ledger.snapshot().record == before + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + archive_paths = tuple(path for path in archive_dir.iterdir() if path.suffix == ".json") + assert len(archive_paths) == 1 + archive_inode = archive_paths[0].stat().st_ino + + repeated = ledger.preflight_retire_unavailable_target(**request) # type: ignore[arg-type] + + assert repeated == prospective + assert ledger.path.read_bytes() == main_bytes + assert ledger.path.stat().st_ino == main_inode + assert archive_paths[0].stat().st_ino == archive_inode + + committed = ledger.retire_unavailable_target(**request) # type: ignore[arg-type] + + assert committed == prospective + assert tuple(path for path in archive_dir.iterdir() if path.suffix == ".json") == archive_paths + assert archive_paths[0].stat().st_ino == archive_inode + assert ( + PhysicalCommandLedger( + tmp_path / "target" / "repository", + clock=_clock, + ) + .snapshot() + .record + == committed + ) + + +def test_successor_prepare_preserves_archive_and_archived_ids_after_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + fifth = ledger.retire_unavailable_target( + retirement_id="scenario-reset-fifth-before-successor", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-fifth-before-successor", + expected_revision=fifth.revision, + expected_retirement_id="scenario-reset-fifth-before-successor", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=95, + reason="device-returned-for-explicit-reconciliation", + ) + recovery_connection = _reconciliation_connection() + reconciled = ledger.reconcile_ambiguous( + reopened.operation_id, + reconciliation_id="reconciliation-fifth-before-successor", + resolution="physical-standby-observed", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-fifth-before-successor", + ), + observation=_status( + "ready", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + successor = ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=reconciled.operation_id, + acquisition_id="acquisition-successor-after-archive", + action="start", + identity=IDENTITY, + connection=recovery_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="9" * 64, + baseline_status=_status( + "ready", + connection=recovery_connection, + observed_at="2026-08-07T09:32:00.000Z", + ), + ) + assert successor.archive_head_sha256 == fifth.archive_head_sha256 + assert successor.archive_segment_count == fifth.archive_segment_count == 1 + assert successor.archive_total_bytes == fifth.archive_total_bytes + + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + observed = restarted.snapshot().record + assert observed == successor + assert observed.archive_head_sha256 == fifth.archive_head_sha256 + assert observed.archive_segment_count == 1 + assert observed.archive_total_bytes == fifth.archive_total_bytes + + restarted.mark_dispatching(successor.operation_id) + dispatching = restarted.snapshot().record + assert dispatching is not None + with pytest.raises(PhysicalCommandTransitionError, match="retirement id has already"): + restarted.retire_unavailable_target( + retirement_id="retirement-reopen-cycle-1", + expected_operation_id=dispatching.operation_id, + expected_revision=dispatching.revision, + expected_transport_ref=dispatching.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + fresh_retirement = restarted.retire_unavailable_target( + retirement_id="retirement-successor-after-archive", + expected_operation_id=dispatching.operation_id, + expected_revision=dispatching.revision, + expected_transport_ref=dispatching.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + with pytest.raises(PhysicalCommandTransitionError, match="reopen id has already"): + restarted.reopen_retired_reconciliation( + reopening_id="reopening-capacity-cycle-1", + expected_revision=fresh_retirement.revision, + expected_retirement_id="retirement-successor-after-archive", + expected_transport_ref=recovery_connection.transport_ref, + expected_discovery_generation=96, + reason="device-returned-for-explicit-reconciliation", + ) + + replacement_identity = PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ) + replacement_connection = _connection( + control_session_id="replacement-control-after-archive", + host_path_epoch=20, + producer_generation=20, + transport_ref="replacement-transport-after-archive", + ) + with pytest.raises(PhysicalCommandTransitionError, match="archived audit"): + restarted.prepare( + operation_id="historical-stop-operation-002", + parent_operation_id=fresh_retirement.operation_id, + acquisition_id="replacement-acquisition-after-archive", + action="start", + identity=replacement_identity, + connection=replacement_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="a" * 64, + baseline_status=_status( + "ready", + connection=replacement_connection, + identity=replacement_identity, + observed_at="2026-08-07T09:33:00.000Z", + ), + ) + + +def _create_two_segment_archive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> tuple[PhysicalCommandLedger, str]: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + fifth = ledger.retire_unavailable_target( + retirement_id="scenario-reset-fifth-for-two-segments", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + first_head = fifth.archive_head_sha256 + assert first_head is not None + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-fifth-for-two-segments", + expected_revision=fifth.revision, + expected_retirement_id="scenario-reset-fifth-for-two-segments", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=101, + reason="device-returned-for-explicit-reconciliation", + ) + current_bytes = len(ledger.path.read_bytes()) + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + current_bytes + 128, + ) + sixth = ledger.retire_unavailable_target( + retirement_id="scenario-reset-sixth-for-two-segments", + expected_operation_id=reopened.operation_id, + expected_revision=reopened.revision, + expected_transport_ref=reopened.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert sixth.archive_segment_count == 2 + assert sixth.archive_head_sha256 is not None + assert sixth.archive_head_sha256 != first_head + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert restarted.snapshot().record == sixth + return restarted, first_head + + +def test_repeated_rollover_preserves_chain_and_segment_bound_blocks_atomically( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _first_head = _create_two_segment_archive(tmp_path, monkeypatch) + sixth = ledger.snapshot().record + assert sixth is not None + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-sixth-before-segment-bound", + expected_revision=sixth.revision, + expected_retirement_id="scenario-reset-sixth-for-two-segments", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=102, + reason="device-returned-for-explicit-reconciliation", + ) + main_before = ledger.path.read_bytes() + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + archive_before = {path.name: path.read_bytes() for path in archive_dir.iterdir()} + with monkeypatch.context() as bounded: + bounded.setattr(ledger_module, "PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS", 2) + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(main_before) + 128, + ) + with pytest.raises(PhysicalCommandBlocked, match="segment limit"): + ledger.retire_unavailable_target( + retirement_id="scenario-reset-seventh-blocked-by-segment-bound", + expected_operation_id=reopened.operation_id, + expected_revision=reopened.revision, + expected_transport_ref=reopened.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert ledger.path.read_bytes() == main_before + assert {path.name: path.read_bytes() for path in archive_dir.iterdir()} == archive_before + + +def test_archive_byte_bound_blocks_next_rollover_without_partial_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _first_head = _create_two_segment_archive(tmp_path, monkeypatch) + sixth = ledger.snapshot().record + assert sixth is not None + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-sixth-before-byte-bound", + expected_revision=sixth.revision, + expected_retirement_id="scenario-reset-sixth-for-two-segments", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=103, + reason="device-returned-for-explicit-reconciliation", + ) + main_before = ledger.path.read_bytes() + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + archive_before = {path.name: path.read_bytes() for path in archive_dir.iterdir()} + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES", + reopened.archive_total_bytes + 1, + ) + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(main_before) + 128, + ) + with pytest.raises(PhysicalCommandBlocked, match="byte limit"): + ledger.retire_unavailable_target( + retirement_id="scenario-reset-seventh-blocked-by-byte-bound", + expected_operation_id=reopened.operation_id, + expected_revision=reopened.revision, + expected_transport_ref=reopened.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert ledger.path.read_bytes() == main_before + assert {path.name: path.read_bytes() for path in archive_dir.iterdir()} == archive_before + + +def test_rollover_never_compacts_current_operation_reconciliation_or_active_fence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + ledger.mark_observing(STOP_OPERATION, publish_call_returned=True, packet_id=42) + recovery_connection = _reconciliation_connection() + active = ledger.reconcile_ambiguous( + STOP_OPERATION, + reconciliation_id="reconciliation-current-proof", + resolution="physical-active-observed", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-current-proof", + ), + observation=_status( + "scanning", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + first_retired = ledger.retire_unavailable_target( + retirement_id="retirement-current-proof-cycle-1", + expected_operation_id=active.operation_id, + expected_revision=active.revision, + expected_transport_ref=active.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-current-proof-cycle-1", + expected_revision=first_retired.revision, + expected_retirement_id="retirement-current-proof-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=111, + reason="device-returned-for-explicit-reconciliation", + ) + document = json.loads(ledger.path.read_text(encoding="utf-8")) + current_audit = document["reconciliations"][0] + for index in range(1, 3): + historical = copy.deepcopy(current_audit) + historical["reconciliation_id"] = f"historical-current-proof-{index}" + original = historical["original_attempt"] + original["operation_id"] = f"historical-current-proof-operation-{index}" + historical["original_attempt_sha256"] = ledger_module._sha256_document(original) + verified = historical["verified_binding"] + verified["verification_id"] = f"historical-current-proof-verification-{index}" + verified["connection"]["control_session_id"] = f"historical-current-proof-control-{index}" + historical["observation"]["control_session_id"] = verified["connection"][ + "control_session_id" + ] + document["reconciliations"].append(historical) + ledger.path.write_bytes(ledger_module._canonical_json_bytes(document)) + ledger.path.chmod(0o600) + ledger = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + reopened = ledger.snapshot().record + assert reopened is not None + with monkeypatch.context() as bounded: + bounded.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(ledger.path.read_bytes()) + 128, + ) + second_retired = ledger.retire_unavailable_target( + retirement_id="retirement-current-proof-cycle-2", + expected_operation_id=reopened.operation_id, + expected_revision=reopened.revision, + expected_transport_ref=reopened.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert second_retired.archive_segment_count == 1 + assert [item.reconciliation_id for item in second_retired.reconciliations] == [ + "reconciliation-current-proof" + ] + assert [item.retirement_id for item in active_operator_retirements(second_retired)] == [ + "retirement-current-proof-cycle-2" + ] + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == second_retired + + +@pytest.mark.parametrize("tamper", ["predecessor", "sequence"]) +def test_archive_predecessor_or_sequence_forgery_fails_closed_with_valid_address( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + tamper: str, +) -> None: + ledger, _first_head = _create_two_segment_archive(tmp_path, monkeypatch) + record = ledger.snapshot().record + assert record is not None and record.archive_head_sha256 is not None + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + head_path = archive_dir / f"{record.archive_head_sha256}.json" + segment = json.loads(head_path.read_text(encoding="utf-8")) + if tamper == "predecessor": + segment["predecessor_sha256"] = None + else: + segment["sequence"] = 1 + forged_bytes = ledger_module._canonical_json_bytes(segment) + forged_head = hashlib.sha256(forged_bytes).hexdigest() + forged_path = archive_dir / f"{forged_head}.json" + forged_path.write_bytes(forged_bytes) + forged_path.chmod(0o600) + main = json.loads(ledger.path.read_text(encoding="utf-8")) + main["archive_head_sha256"] = forged_head + ledger.path.write_bytes(ledger_module._canonical_json_bytes(main)) + ledger.path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert restarted.snapshot().status == "corrupt" + + +def test_content_addressed_archive_publication_is_idempotent_and_fails_on_collision( + tmp_path: Path, +) -> None: + data_dir = tmp_path / "private-data" + archive_dir = data_dir / "xgrids-k1" / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + payload = b'{"canonical":"archive"}\n' + digest = hashlib.sha256(payload).hexdigest() + ledger_module._write_private_content_addressed_archive( + archive_dir, + expected_sha256=digest, + payload=payload, + data_dir=data_dir, + ) + destination = archive_dir / f"{digest}.json" + inode = destination.stat().st_ino + ledger_module._write_private_content_addressed_archive( + archive_dir, + expected_sha256=digest, + payload=payload, + data_dir=data_dir, + ) + assert destination.stat().st_ino == inode + + destination.write_bytes(b"different-existing-bytes\n") + destination.chmod(0o600) + with pytest.raises(PhysicalCommandLedgerCorrupt, match="different bytes"): + ledger_module._write_private_content_addressed_archive( + archive_dir, + expected_sha256=digest, + payload=payload, + data_dir=data_dir, + ) + + +def test_process_death_after_archive_link_fsync_recovers_exact_alias_and_reloads( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + main_bytes = ledger.path.read_bytes() + main_inode = ledger.path.stat().st_ino + retirement_id = "scenario-reset-after-hardlink-publication-crash" + script = """ +import os +from pathlib import Path +from k1link.device_plugins.xgrids_k1 import physical_command_ledger as ledger_module +from k1link.device_plugins.xgrids_k1.physical_command_ledger import PhysicalCommandLedger + +ledger = PhysicalCommandLedger(Path(os.environ["PHYSICAL_LEDGER_TEST_REPOSITORY"])) +record = ledger.snapshot().record +assert record is not None +ledger_module._PHYSICAL_COMMAND_ARCHIVE_AFTER_LINK_FSYNC_FAILPOINT = lambda: os._exit(93) +ledger.retire_unavailable_target( + retirement_id=os.environ["PHYSICAL_LEDGER_TEST_RETIREMENT_ID"], + expected_operation_id=record.operation_id, + expected_revision=record.revision, + expected_transport_ref=record.connection.transport_ref, + reason="connection-scenario-reset-by-operator", +) +""" + environment = dict(os.environ) + environment["PHYSICAL_LEDGER_TEST_REPOSITORY"] = str(tmp_path / "target" / "repository") + environment["PHYSICAL_LEDGER_TEST_RETIREMENT_ID"] = retirement_id + + crashed = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).parent.parent, + env=environment, + check=False, + capture_output=True, + text=True, + ) + + assert crashed.returncode == 93, crashed.stderr + assert ledger.path.read_bytes() == main_bytes + assert ledger.path.stat().st_ino == main_inode + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + destinations = tuple(path for path in archive_dir.iterdir() if path.suffix == ".json") + temporaries = tuple(path for path in archive_dir.iterdir() if path.suffix == ".tmp") + assert len(destinations) == 1 + assert len(temporaries) == 1 + destination_inode = destinations[0].stat().st_ino + assert temporaries[0].stat().st_ino == destination_inode + assert destinations[0].stat().st_nlink == 2 + + committed = ledger.retire_unavailable_target( + retirement_id=retirement_id, + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + + assert destinations[0].stat().st_ino == destination_inode + assert destinations[0].stat().st_nlink == 1 + assert tuple(path for path in archive_dir.iterdir() if path.suffix == ".tmp") == () + assert tuple(path for path in archive_dir.iterdir() if path.suffix == ".json") == destinations + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert restarted.snapshot().record == committed + + +def test_archive_retry_never_repairs_or_removes_an_unrelated_hard_link( + tmp_path: Path, +) -> None: + data_dir = tmp_path / "private-data" + archive_dir = data_dir / "xgrids-k1" / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + payload = b'{"canonical":"archive-hardlink-deny"}\n' + digest = hashlib.sha256(payload).hexdigest() + ledger_module._write_private_content_addressed_archive( + archive_dir, + expected_sha256=digest, + payload=payload, + data_dir=data_dir, + ) + destination = archive_dir / f"{digest}.json" + unrelated = tmp_path / "unrelated-hard-link.json" + os.link(destination, unrelated) + + with pytest.raises(PhysicalCommandLedgerCorrupt, match="unsafe"): + ledger_module._write_private_content_addressed_archive( + archive_dir, + expected_sha256=digest, + payload=payload, + data_dir=data_dir, + ) + + assert destination.read_bytes() == payload + assert unrelated.read_bytes() == payload + assert destination.stat().st_nlink == 2 + assert tuple(path for path in archive_dir.iterdir() if path.suffix == ".tmp") == () + + +def test_crash_after_archive_before_main_publish_retries_without_duplicate_chain( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + original_writer = ledger_module._write_private_json_atomic + failed_once = False + + def fail_first_main_publish( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + nonlocal failed_once + if path == ledger.path and not failed_once: + failed_once = True + raise OSError("injected crash after archive publication") + original_writer(path, payload, data_dir=data_dir) + + monkeypatch.setattr( + ledger_module, + "_write_private_json_atomic", + fail_first_main_publish, + ) + with pytest.raises(OSError, match="injected crash"): + ledger.retire_unavailable_target( + retirement_id="scenario-reset-after-injected-crash", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + orphaned = tuple(archive_dir.iterdir()) + assert len(orphaned) == 1 + assert ledger.snapshot().record == before + + committed = ledger.retire_unavailable_target( + retirement_id="scenario-reset-after-injected-crash", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + assert committed.archive_segment_count == 1 + assert tuple(archive_dir.iterdir()) == orphaned + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert restarted.snapshot().record == committed + + +@pytest.mark.parametrize( + "tamper", + [ + "missing", + "bytes", + "symlink", + "head", + "replay-count", + "segment-bound", + "byte-bound", + ], +) +def test_archive_missing_tampered_head_sequence_or_symlink_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + tamper: str, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + LedgerPhysicalCommandCoordinator(ledger).retire_connection_scenario( + retirement_id="scenario-reset-for-archive-tamper", + expected_revision=before.revision, + ) + record = ledger.snapshot().record + assert record is not None and record.archive_head_sha256 is not None + archive_path = ( + ledger.path.parent + / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + / f"{record.archive_head_sha256}.json" + ) + if tamper == "missing": + archive_path.unlink() + elif tamper == "bytes": + payload = bytearray(archive_path.read_bytes()) + payload[-2] ^= 1 + archive_path.write_bytes(payload) + archive_path.chmod(0o600) + elif tamper == "symlink": + safe_copy = tmp_path / "archive-copy.json" + safe_copy.write_bytes(archive_path.read_bytes()) + safe_copy.chmod(0o600) + archive_path.unlink() + archive_path.symlink_to(safe_copy) + else: + main = json.loads(ledger.path.read_text(encoding="utf-8")) + if tamper == "head": + main["archive_head_sha256"] = "f" * 64 + elif tamper == "segment-bound": + main["archive_segment_count"] = ledger_module.PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS + 1 + elif tamper == "byte-bound": + main["archive_total_bytes"] = ledger_module.PHYSICAL_COMMAND_ARCHIVE_MAX_BYTES + 1 + else: + main["archive_segment_count"] = 2 + ledger.path.write_bytes(ledger_module._canonical_json_bytes(main)) + ledger.path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "target" / "repository", clock=_clock) + assert restarted.snapshot().status == "corrupt" + assert restarted.snapshot().record is None + with pytest.raises(PhysicalCommandLedgerCorrupt): + restarted.require_no_unresolved_attempt() + + +def test_archive_directory_symlink_fails_before_rollover_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + archive_dir = ledger.path.parent / PHYSICAL_COMMAND_ARCHIVE_DIRECTORY + outside = tmp_path / "outside-archive" + outside.mkdir(mode=0o700) + archive_dir.symlink_to(outside, target_is_directory=True) + before = ledger.snapshot().record + assert before is not None + with pytest.raises((PhysicalCommandLedgerCorrupt, ValueError)): + LedgerPhysicalCommandCoordinator(ledger).retire_connection_scenario( + retirement_id="scenario-reset-blocked-by-archive-symlink", + expected_revision=before.revision, + ) + assert list(outside.iterdir()) == [] + + +def test_archived_retirement_and_reopening_ids_cannot_be_reused( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger, _ = _write_near_capacity_v3_four_cycle_ledger(tmp_path, monkeypatch) + before = ledger.snapshot().record + assert before is not None + fifth = ledger.retire_unavailable_target( + retirement_id="scenario-reset-fifth-for-id-reuse", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="connection-scenario-reset-by-operator", + ) + with pytest.raises(PhysicalCommandTransitionError, match="reopen id has already"): + ledger.reopen_retired_reconciliation( + reopening_id="reopening-capacity-cycle-1", + expected_revision=fifth.revision, + expected_retirement_id="scenario-reset-fifth-for-id-reuse", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=100, + reason="device-returned-for-explicit-reconciliation", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-fifth-for-id-reuse", + expected_revision=fifth.revision, + expected_retirement_id="scenario-reset-fifth-for-id-reuse", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=100, + reason="device-returned-for-explicit-reconciliation", + ) + with pytest.raises(PhysicalCommandTransitionError, match="retirement id has already"): + ledger.retire_unavailable_target( + retirement_id="retirement-reopen-cycle-1", + expected_operation_id=before.operation_id, + expected_revision=reopened.revision, + expected_transport_ref=TRANSPORT_REF, + reason="connection-scenario-reset-by-operator", + ) + + +def test_reopen_size_preflight_is_atomic( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, retired = _retire_ambiguous_stop(ledger) + before_bytes = ledger.path.read_bytes() + before_inode = ledger.path.stat().st_ino + monkeypatch.setattr( + ledger_module, + "PHYSICAL_COMMAND_LEDGER_MAX_BYTES", + len(before_bytes) + 32, + ) + + with pytest.raises(PhysicalCommandBlocked, match="requires archival"): + ledger.reopen_retired_reconciliation( + reopening_id="reopening-too-large", + expected_revision=retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=15, + reason="device-returned-for-explicit-reconciliation", + ) + assert ledger.path.read_bytes() == before_bytes + assert ledger.path.stat().st_ino == before_inode + + +def test_forged_double_active_retirement_cycle_fails_closed_after_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _, first_retired = _retire_ambiguous_stop(ledger) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-for-forgery", + expected_revision=first_retired.revision, + expected_retirement_id="retirement-reopen-cycle-1", + expected_transport_ref=TRANSPORT_REF, + expected_discovery_generation=16, + reason="device-returned-for-explicit-reconciliation", + ) + ledger.retire_unavailable_target( + retirement_id="retirement-forged-second-active", + expected_operation_id=STOP_OPERATION, + expected_revision=reopened.revision, + expected_transport_ref=TRANSPORT_REF, + reason="device-permanently-unavailable-or-replaced", + ) + forged = json.loads(ledger.path.read_text(encoding="utf-8")) + forged["operator_reconciliation_reopens"] = [] + ledger.path.write_text( + json.dumps(forged, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + + snapshot = PhysicalCommandLedger(tmp_path / "repository", clock=_clock).snapshot() + assert snapshot.status == "corrupt" + assert snapshot.record is None + + +@pytest.mark.parametrize( + ("candidate_identity", "candidate_transport"), + [ + ( + PhysicalCommandIdentity( + vendor_device_id_sha256=IDENTITY.vendor_device_id_sha256, + device_serial_sha256="8" * 64, + ), + "B272E0E6-D463-2170-E541-6FC1CD24A8A0", + ), + ( + PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256=IDENTITY.device_serial_sha256, + ), + "B272E0E6-D463-2170-E541-6FC1CD24A8A0", + ), + ( + PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ), + TRANSPORT_REF.lower(), + ), + ], + ids=("vendor-hash-overlap", "serial-hash-overlap", "transport-overlap"), +) +def test_retired_target_rejects_each_independent_identity_or_transport_overlap( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + candidate_identity: PhysicalCommandIdentity, + candidate_transport: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + before = ledger.snapshot().record + assert before is not None + retired = ledger.retire_unavailable_target( + retirement_id="retirement-overlap-fence", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + candidate_connection = _connection( + transport_ref=candidate_transport, + control_session_id="candidate-control-session", + host_path_epoch=20, + producer_generation=21, + ) + + with pytest.raises(PhysicalCommandBlocked, match="bind a control session"): + ledger.require_runtime_not_retired( + identity=candidate_identity, + connection=candidate_connection, + ) + with pytest.raises(PhysicalCommandBlocked, match="retired physical target"): + ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=retired.operation_id, + acquisition_id="candidate-acquisition", + action="start", + identity=candidate_identity, + connection=candidate_connection, + compatibility_profile_id="xgrids-k1-v0.7.0", + payload_sha256="9" * 64, + baseline_status=_status( + "ready", + connection=candidate_connection, + identity=candidate_identity, + ), + ) + + +def test_replacement_rejects_retired_parent_operation_and_acquisition_id_reuse( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + _prepare_stop(ledger) + ledger.mark_dispatching(STOP_OPERATION) + before = ledger.snapshot().record + assert before is not None + retired = ledger.retire_unavailable_target( + retirement_id="retirement-lineage-fence", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + replacement_identity = PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ) + replacement_connection = _connection( + transport_ref="B272E0E6-D463-2170-E541-6FC1CD24A8A0", + control_session_id="replacement-control-session", + host_path_epoch=20, + producer_generation=21, + ) + common = { + "parent_operation_id": retired.operation_id, + "action": "start", + "identity": replacement_identity, + "connection": replacement_connection, + "compatibility_profile_id": "xgrids-k1-v0.7.0", + "payload_sha256": "9" * 64, + "baseline_status": _status( + "ready", + connection=replacement_connection, + identity=replacement_identity, + ), + } + + with pytest.raises(PhysicalCommandTransitionError, match="already consumed"): + ledger.prepare( + operation_id=START_OPERATION, + acquisition_id="fresh-replacement-acquisition", + **common, # type: ignore[arg-type] + ) + with pytest.raises(PhysicalCommandBlocked, match="acquisition id cannot be reused"): + ledger.prepare( + operation_id=NEXT_START_OPERATION, + acquisition_id=ACQUISITION_ID, + **common, # type: ignore[arg-type] + ) + + +def test_retired_target_stays_fenced_while_different_device_starts_new_chain( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + recovered_connection = _reconciliation_connection() + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-before-retirement", + verified_binding=_verified_binding(connection=recovered_connection), + observation=_status( + "scanning", + connection=recovered_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + _prepare_stop( + ledger, + connection=recovered_connection, + operator_confirmation_id="confirmation-before-retirement", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + ledger.mark_dispatching(STOP_OPERATION) + before = ledger.snapshot().record + assert before is not None + retired = ledger.retire_unavailable_target( + retirement_id="retirement-after-reconciled-stop", + expected_operation_id=before.operation_id, + expected_revision=before.revision, + expected_transport_ref=before.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + + replacement_identity = PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ) + replacement_connection = _connection( + transport_ref="B272E0E6-D463-2170-E541-6FC1CD24A8A0", + control_session_id="replacement-control-session", + host_path_epoch=10, + producer_generation=14, + ) + ledger.require_runtime_not_retired( + identity=replacement_identity, + connection=replacement_connection, + ) + with pytest.raises(PhysicalCommandBlocked, match="retired physical target"): + ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=retired.operation_id, + acquisition_id="replacement-acquisition", + action="start", + identity=IDENTITY, + connection=recovered_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="9" * 64, + baseline_status=_status("ready", connection=recovered_connection), + ) + + replacement = ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=retired.operation_id, + acquisition_id="replacement-acquisition", + action="start", + identity=replacement_identity, + connection=replacement_connection, + compatibility_profile_id="xgrids-k1-v0.7.0", + payload_sha256="9" * 64, + baseline_status=_status( + "ready", + connection=replacement_connection, + identity=replacement_identity, + ), + ) + assert replacement.stage == "prepared" + assert replacement.identity == replacement_identity + assert replacement.operator_retirements == retired.operator_retirements + assert replacement.reconciliations == retired.reconciliations + assert replacement.operator_confirmations == retired.operator_confirmations + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().record == replacement + + +def test_wall_clock_rollback_keeps_every_physical_stage_restart_valid( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + ticks = iter(datetime(2026, 8, 7, hour, tzinfo=UTC) for hour in (12, 11, 10, 9, 8, 7, 6, 5)) + repository = tmp_path / "repository" + ledger = PhysicalCommandLedger(repository, clock=lambda: next(ticks)) + + def assert_restart(*, stage: str, revision: int, action: str = "start") -> None: + snapshot = PhysicalCommandLedger(repository, clock=_clock).snapshot() + assert snapshot.status == ("resolved" if stage == "resolved" else "unresolved") + assert snapshot.record is not None + assert snapshot.record.stage == stage + assert snapshot.record.action == action + assert snapshot.record.revision == revision + assert snapshot.record.created_at_utc == "2026-08-07T12:00:00.000Z" + assert snapshot.record.updated_at_utc == "2026-08-07T12:00:00.000Z" + + _prepare_start( + ledger, + baseline_observed_at="2026-08-07T11:30:00.000Z", + ) + assert_restart(stage="prepared", revision=1) + ledger.mark_dispatching(START_OPERATION) + assert_restart(stage="dispatching", revision=2) + ledger.mark_observing(START_OPERATION, publish_call_returned=True, packet_id=41) + assert_restart(stage="observing", revision=3) + ledger.record_application_response( + START_OPERATION, + _response( + START_OPERATION, + "start", + observed_at="2026-08-07T05:30:00.000Z", + ), + ) + assert_restart(stage="observing", revision=4) + ledger.record_status_observation( + START_OPERATION, + _status("scanning", observed_at="2026-08-07T04:30:00.000Z"), + ) + assert_restart(stage="observing", revision=5) + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + assert_restart(stage="observing", revision=6) + ledger.resolve(START_OPERATION, resolution="start-active-observed") + assert_restart(stage="resolved", revision=7) + + _prepare_stop( + ledger, + baseline_observed_at="2026-08-07T03:30:00.000Z", + ) + assert_restart(stage="prepared", revision=8, action="stop") + + +def test_status_only_never_resolves_a_dispatched_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + ledger.mark_observing(START_OPERATION, publish_call_returned=True, packet_id=41) + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + ledger.record_status_observation( + START_OPERATION, + _status("scanning", observed_at="2026-08-07T09:29:40.000Z"), + ) + + with pytest.raises(PhysicalCommandTransitionError, match="successful application response"): + ledger.resolve(START_OPERATION, resolution="start-active-observed") + assert ledger.snapshot().status == "unresolved" + + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start"), + ) + resolved = ledger.resolve(START_OPERATION, resolution="start-active-observed") + assert resolved.stage == "resolved" + assert resolved.resolution == "start-active-observed" + + +def test_resolution_requires_exact_nonretained_live_generation_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + + wrong_generation = _connection(producer_generation=12) + with pytest.raises(PhysicalCommandTransitionError, match="live-path generation"): + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start", connection=wrong_generation), + ) + with pytest.raises(PhysicalCommandTransitionError, match="correlate"): + ledger.record_application_response( + START_OPERATION, + _response(STOP_OPERATION, "start"), + ) + + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start"), + ) + retained = _status( + "scanning", + retained=True, + observed_at="2026-08-07T09:29:40.000Z", + ) + with pytest.raises(PhysicalCommandTransitionError, match="non-retained"): + ledger.resolve( + START_OPERATION, + resolution="start-active-observed", + observation=retained, + ) + reconciliation = _status( + "scanning", + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:29:40.000Z", + ) + ledger.record_status_observation(START_OPERATION, reconciliation) + with pytest.raises(PhysicalCommandTransitionError, match="live-session"): + ledger.resolve(START_OPERATION, resolution="start-active-observed") + + +def test_prepared_is_the_only_stage_that_can_prove_not_dispatched( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + resolved = ledger.resolve(START_OPERATION, resolution="not-dispatched") + assert resolved.stage == "resolved" + assert resolved.publish_call_returned is None + assert resolved.application_response is None + ledger.require_no_unresolved_attempt() + + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + ) + ledger.mark_dispatching(NEXT_START_OPERATION) + with pytest.raises(PhysicalCommandTransitionError, match="prepared stage"): + ledger.resolve(NEXT_START_OPERATION, resolution="not-dispatched") + + +def test_edge_chain_enforces_start_then_same_session_stop_then_new_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + + with pytest.raises(PhysicalCommandTransitionError, match="only by same-session STOP"): + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + ) + with pytest.raises(PhysicalCommandTransitionError, match="live control-session"): + _prepare_stop(ledger, connection=_connection(control_session_id="other-control-session")) + + _prepare_stop(ledger) + _complete_stop(ledger) + with pytest.raises(PhysicalCommandTransitionError, match="fresh START"): + ledger.prepare( + operation_id="physical-stop-44444444-4444-4444-8444-444444444444", + parent_operation_id=STOP_OPERATION, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=IDENTITY, + connection=_connection(), + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="2" * 64, + baseline_status=_status("scanning"), + ) + + next_connection = _connection( + control_session_id="control-33333333-3333-4333-8333-333333333333", + host_path_epoch=8, + producer_generation=12, + ) + next_start = ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=STOP_OPERATION, + acquisition_id="acquisition-33333333-3333-4333-8333-333333333333", + action="start", + identity=IDENTITY, + connection=next_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="3" * 64, + baseline_status=_status( + "ready", + connection=next_connection, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + assert next_start.stage == "prepared" + assert next_start.revision > 1 + + +def test_successful_start_can_rebind_read_only_for_one_confirmed_stop( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + fresh_connection = _reconciliation_connection() + + rebound = ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-55555555-5555-4555-8555-555555555555", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + assert rebound.resolution == "start-active-observed" + assert rebound.original_command_outcome == "succeeded" + assert rebound.reconciled_physical_state == "active" + assert rebound.automatic_replay_allowed is False + active_rebind = rebound.reconciliations[-1] + assert active_rebind.kind == "resolved-active-rebind" + assert active_rebind.resolution == "physical-active-observed" + assert active_rebind.original_attempt.stage == "resolved" + assert active_rebind.original_attempt.resolution == "start-active-observed" + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + persisted = restarted.snapshot().record + assert persisted is not None + assert persisted.original_command_outcome == "succeeded" + with pytest.raises(PhysicalCommandTransitionError, match="latest verified"): + _prepare_stop(restarted) + with pytest.raises(PhysicalCommandTransitionError, match="operator-confirmed STOP"): + _prepare_stop( + restarted, + connection=fresh_connection, + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + + _prepare_stop( + restarted, + connection=fresh_connection, + operator_confirmation_id="confirmation-after-active-rebind", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + prepared_stop = restarted.snapshot().record + assert prepared_stop is not None + assert prepared_stop.action == "stop" + assert prepared_stop.connection == fresh_connection + assert prepared_stop.operator_confirmations[-1].reconciliation_id == ( + "active-rebind-55555555-5555-4555-8555-555555555555" + ) + + +def test_active_rebind_exact_latest_retry_is_process_safe_and_conflicts_fail( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + stale_process = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + fresh_connection = _reconciliation_connection() + verified_binding = _verified_binding(connection=fresh_connection) + observation = _status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ) + request = { + "reconciliation_id": "active-rebind-exact-process-retry", + "verified_binding": verified_binding, + "observation": observation, + } + + committed = ledger.rebind_resolved_active(START_OPERATION, **request) # type: ignore[arg-type] + retried = stale_process.rebind_resolved_active( # type: ignore[arg-type] + START_OPERATION, + **request, + ) + + assert retried == committed + assert retried.revision == committed.revision + assert len(retried.reconciliations) == 1 + with pytest.raises(PhysicalCommandTransitionError, match="already been used"): + stale_process.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-exact-process-retry", + verified_binding=verified_binding, + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + + +def test_active_rebind_exact_retry_rejects_operator_reopened_noncanonical_stage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + fresh_connection = _reconciliation_connection() + verified_binding = _verified_binding(connection=fresh_connection) + observation = _status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ) + reconciliation_id = "active-rebind-before-operator-reopen" + rebound = ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id=reconciliation_id, + verified_binding=verified_binding, + observation=observation, + ) + retired = ledger.retire_unavailable_target( + retirement_id="retirement-after-active-rebind", + expected_operation_id=rebound.operation_id, + expected_revision=rebound.revision, + expected_transport_ref=rebound.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + reopened = ledger.reopen_retired_reconciliation( + reopening_id="reopening-after-active-rebind", + expected_revision=retired.revision, + expected_retirement_id="retirement-after-active-rebind", + expected_transport_ref=rebound.connection.transport_ref, + expected_discovery_generation=31, + reason="device-returned-for-explicit-reconciliation", + ) + assert reopened.stage == "observing" + assert reopened.resolution is None + assert reopened.reconciliations[-1].reconciliation_id == reconciliation_id + + with pytest.raises(PhysicalCommandTransitionError, match="resolved active"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id=reconciliation_id, + verified_binding=verified_binding, + observation=observation, + ) + + +def test_active_standby_exact_latest_retry_is_process_safe_and_conflicts_fail( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + stale_process = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + fresh_connection = _reconciliation_connection() + verified_binding = _verified_binding(connection=fresh_connection) + observation = _status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ) + request = { + "reconciliation_id": "active-standby-exact-process-retry", + "verified_binding": verified_binding, + "observation": observation, + } + + committed = ledger.observe_resolved_active_standby( # type: ignore[arg-type] + START_OPERATION, + **request, + ) + retried = stale_process.observe_resolved_active_standby( # type: ignore[arg-type] + START_OPERATION, + **request, + ) + + assert retried == committed + assert retried.revision == committed.revision + assert len(retried.reconciliations) == 1 + with pytest.raises(PhysicalCommandTransitionError, match="already been used"): + stale_process.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-standby-exact-process-retry", + verified_binding=verified_binding, + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + + +def test_standby_cessation_cannot_enter_operator_retire_reopen_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + fresh_connection = _reconciliation_connection() + cessation = ledger.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-standby-before-retirement-attempt", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + assert cessation.reconciled_physical_state == "standby" + + with pytest.raises(PhysicalCommandTransitionError, match="conflicts with the current record"): + ledger.retire_unavailable_target( + retirement_id="retirement-after-standby-cessation", + expected_operation_id=cessation.operation_id, + expected_revision=cessation.revision, + expected_transport_ref=cessation.connection.transport_ref, + reason="device-permanently-unavailable-or-replaced", + ) + persisted = ledger.snapshot().record + assert persisted == cessation + + +def test_active_rebind_requires_fresh_exact_read_only_scanning_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + fresh_connection = _reconciliation_connection() + + with pytest.raises(PhysicalCommandTransitionError, match="new verified"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-same-session", + verified_binding=_verified_binding(connection=_connection()), + observation=_status( + "scanning", + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="read-only"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-live-status", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="retained"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-retained-status", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + retained=True, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="initialized SCANNING"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-standby-status", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="project does not match"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-wrong-project", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + project_id_sha256="9" * 64, + ), + ) + snapshot = ledger.snapshot() + assert snapshot.status == "resolved" + assert snapshot.record is not None + assert snapshot.record.reconciliations == () + + +def test_latest_active_rebind_invalidates_an_older_recovery_binding( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + first_connection = _reconciliation_connection() + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-first", + verified_binding=_verified_binding(connection=first_connection), + observation=_status( + "scanning", + connection=first_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + second_connection = _connection( + control_session_id="active-rebind-second-session", + host_path_epoch=9, + producer_generation=13, + ) + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="active-rebind-second", + verified_binding=_verified_binding( + connection=second_connection, + verification_id="active-rebind-second-verification", + ), + observation=_status( + "scanning", + connection=second_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + with pytest.raises(PhysicalCommandTransitionError, match="latest verified"): + _prepare_stop( + ledger, + connection=first_connection, + operator_confirmation_id="confirmation-for-stale-rebind", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + _prepare_stop( + ledger, + connection=second_connection, + operator_confirmation_id="confirmation-for-latest-rebind", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + prepared = ledger.snapshot().record + assert prepared is not None + assert prepared.operator_confirmations[-1].reconciliation_id == "active-rebind-second" + + +def test_successful_start_power_loss_ready_preserves_success_and_allows_fresh_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + recovery_connection = _reconciliation_connection() + + cessation = ledger.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-cessation-after-power-loss", + verified_binding=_verified_binding(connection=recovery_connection), + observation=_status( + "ready", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + assert cessation.resolution == "start-active-observed" + assert cessation.original_command_outcome == "succeeded" + assert cessation.reconciled_physical_state == "standby" + assert cessation.reconciliations[-1].kind == "resolved-active-cessation" + assert cessation.automatic_replay_allowed is False + with pytest.raises(PhysicalCommandTransitionError, match="only a fresh START"): + _prepare_stop(ledger, connection=recovery_connection) + with pytest.raises(PhysicalCommandTransitionError, match="live-session"): + ledger.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + acquisition_id="acquisition-after-power-loss", + action="start", + identity=IDENTITY, + connection=recovery_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="3" * 64, + baseline_status=_status( + "ready", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + normal_connection = _connection( + control_session_id="normal-session-after-power-loss", + host_path_epoch=9, + producer_generation=13, + ) + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + connection=normal_connection, + acquisition_id="acquisition-after-power-loss", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + prepared = restarted.snapshot().record + assert prepared is not None + assert prepared.action == "start" + assert prepared.stage == "prepared" + assert prepared.reconciliations[-1].kind == "resolved-active-cessation" + + +def test_scan_over_cessation_is_durable_but_requires_fresh_ready_before_next_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + scan_over_connection = _reconciliation_connection() + + scan_over = ledger.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-cessation-scan-over", + verified_binding=_verified_binding(connection=scan_over_connection), + observation=_status( + "scan_over", + connection=scan_over_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + assert scan_over.original_command_outcome == "succeeded" + assert scan_over.reconciled_physical_state == "standby" + assert scan_over.reconciliations[-1].observation.session_state == "scan_over" + assert scan_over.automatic_replay_allowed is False + with pytest.raises(PhysicalCommandBlocked, match="later fresh READY"): + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + connection=_connection( + control_session_id="premature-start-after-scan-over", + host_path_epoch=9, + producer_generation=13, + ), + acquisition_id="acquisition-after-scan-over", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + persisted = restarted.snapshot().record + assert persisted is not None + assert persisted.reconciliations[-1].observation.session_state == "scan_over" + ready_connection = _connection( + control_session_id="ready-after-scan-over", + host_path_epoch=10, + producer_generation=14, + ) + ready = restarted.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-cessation-ready-after-scan-over", + verified_binding=_verified_binding( + connection=ready_connection, + verification_id="verification-ready-after-scan-over", + ), + observation=_status( + "ready", + connection=ready_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + assert ready.reconciliations[-1].observation.session_state == "ready" + + next_connection = _connection( + control_session_id="start-after-ready", + host_path_epoch=11, + producer_generation=15, + ) + _prepare_start( + restarted, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + connection=next_connection, + acquisition_id="acquisition-after-scan-over", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + assert restarted.snapshot().record is not None + assert restarted.snapshot().record.action == "start" + + +def test_standby_cessation_cannot_be_reinterpreted_as_original_active_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + _complete_start(ledger) + standby_connection = _reconciliation_connection() + ledger.observe_resolved_active_standby( + START_OPERATION, + reconciliation_id="active-cessation-terminal", + verified_binding=_verified_binding(connection=standby_connection), + observation=_status( + "ready", + connection=standby_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + later_connection = _connection( + control_session_id="later-active-session", + host_path_epoch=9, + producer_generation=13, + ) + with pytest.raises(PhysicalCommandTransitionError, match="cessation cannot be rebound"): + ledger.rebind_resolved_active( + START_OPERATION, + reconciliation_id="invalid-active-after-cessation", + verified_binding=_verified_binding( + connection=later_connection, + verification_id="later-active-verification", + ), + observation=_status( + "scanning", + connection=later_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + +def test_initial_stop_and_stale_or_wrong_baselines_fail_before_persistence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + with pytest.raises(PhysicalCommandTransitionError, match="initial STOP"): + ledger.prepare( + operation_id=STOP_OPERATION, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + action="stop", + identity=IDENTITY, + connection=_connection(), + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="1" * 64, + baseline_status=_status("scanning"), + ) + with pytest.raises(PhysicalCommandTransitionError, match="non-retained"): + ledger.prepare( + operation_id=START_OPERATION, + parent_operation_id=None, + acquisition_id=ACQUISITION_ID, + action="start", + identity=IDENTITY, + connection=_connection(), + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="f" * 64, + baseline_status=_status("ready", retained=True), + ) + assert ledger.snapshot().status == "empty" + + +def test_publish_and_evidence_fields_are_monotonic_and_idempotent( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + observing = ledger.mark_observing( + START_OPERATION, + publish_call_returned=True, + packet_id=41, + ) + assert ( + ledger.mark_observing( + START_OPERATION, + publish_call_returned=True, + packet_id=41, + ).revision + == observing.revision + ) + with pytest.raises(PhysicalCommandTransitionError, match="cannot change"): + ledger.mark_observing( + START_OPERATION, + publish_call_returned=False, + ) + with pytest.raises(PhysicalCommandTransitionError, match="packet_id"): + ledger.mark_qos2_completed(START_OPERATION, packet_id=99) + + response = _response(START_OPERATION, "start") + recorded = ledger.record_application_response(START_OPERATION, response) + assert ( + ledger.record_application_response(START_OPERATION, response).revision == recorded.revision + ) + with pytest.raises(PhysicalCommandTransitionError, match="cannot change"): + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start", observed_at="2026-08-07T09:29:31.000Z"), + ) + + +def test_publish_failure_crosses_the_no_replay_boundary_and_cannot_be_reversed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + failed = ledger.mark_observing( + START_OPERATION, + publish_call_returned=False, + ) + assert failed.stage == "observing" + assert failed.publish_call_returned is False + + with pytest.raises(PhysicalCommandTransitionError, match="cannot change"): + ledger.mark_observing( + START_OPERATION, + publish_call_returned=True, + packet_id=41, + ) + with pytest.raises(PhysicalCommandTransitionError, match="cannot reverse"): + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + with pytest.raises(PhysicalCommandTransitionError, match="prepared stage"): + ledger.resolve(START_OPERATION, resolution="not-dispatched") + assert ledger.snapshot().requires_reconciliation + + +def test_application_rejection_and_wrong_action_resolution_remain_unresolved( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start", result_code=7), + ) + ledger.record_status_observation( + START_OPERATION, + _status("scanning", observed_at="2026-08-07T09:29:40.000Z"), + ) + + with pytest.raises(PhysicalCommandTransitionError, match="command action"): + ledger.resolve(START_OPERATION, resolution="stop-standby-observed") + with pytest.raises(PhysicalCommandTransitionError, match="successful application response"): + ledger.resolve(START_OPERATION, resolution="start-active-observed") + assert ledger.snapshot().status == "unresolved" + + +def test_post_dispatch_transitions_are_rejected_before_dispatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + + with pytest.raises(PhysicalCommandTransitionError, match="after dispatch"): + ledger.mark_observing(START_OPERATION, publish_call_returned=False) + with pytest.raises(PhysicalCommandTransitionError, match="dispatched physical command"): + ledger.mark_qos2_completed(START_OPERATION, packet_id=41) + with pytest.raises(PhysicalCommandTransitionError, match="dispatched physical command"): + ledger.record_application_response( + START_OPERATION, + _response(START_OPERATION, "start"), + ) + with pytest.raises(PhysicalCommandTransitionError, match="dispatched physical command"): + ledger.record_status_observation(START_OPERATION, _status("scanning")) + assert ledger.snapshot().record is not None + assert ledger.snapshot().record.stage == "prepared" + + +def test_application_response_uses_reviewed_vendor_success_code() -> None: + assert PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE == 302252033 + assert PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE == OPENAPI_SUCCESS + accepted = _response(START_OPERATION, "start") + assert accepted.success + assert accepted.result_code == 302252033 + + with pytest.raises(ValueError, match="agree with result_code"): + PhysicalCommandApplicationResponse( + operation_id=START_OPERATION, + action="start", + control_session_id=CONTROL_SESSION_ID, + host_path_epoch=7, + producer_generation=11, + result_code=0, + success=True, + payload_sha256="e" * 64, + observed_at_utc="2026-08-07T09:29:30.000Z", + ) + + +def test_restart_reconciliation_active_preserves_original_and_new_binding_audit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + ledger.mark_observing(START_OPERATION, publish_call_returned=False) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + fresh_connection = _reconciliation_connection() + verified = _verified_binding(connection=fresh_connection) + observed_active = _status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ) + reconciled = restarted.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-active-observed", + verified_binding=verified, + observation=observed_active, + ) + + assert reconciled.stage == "resolved" + assert reconciled.resolution == "physical-active-observed" + assert reconciled.original_command_outcome == "unknown" + assert reconciled.reconciled_physical_state == "active" + assert reconciled.application_response is None + assert reconciled.qos2_completed is False + assert len(reconciled.reconciliations) == 1 + audit = reconciled.reconciliations[0] + assert audit.original_attempt.operation_id == START_OPERATION + assert audit.original_attempt.stage == "observing" + assert audit.original_attempt.publish_call_returned is False + assert audit.original_attempt.connection == _connection() + assert audit.verified_binding.connection == fresh_connection + assert audit.observation == observed_active + assert audit.resolution == "physical-active-observed" + assert audit.original_attempt_sha256 + assert reconciled.operator_confirmations == () + assert ( + restarted.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-active-observed", + verified_binding=verified, + observation=observed_active, + ).revision + == reconciled.revision + ) + + after_second_restart = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + persisted = after_second_restart.snapshot().record + assert persisted is not None + assert persisted.reconciliations == reconciled.reconciliations + with pytest.raises( + PhysicalCommandTransitionError, + match="requires reconcile_ambiguous", + ): + after_second_restart.resolve( + START_OPERATION, + resolution="physical-active-observed", # type: ignore[arg-type] + ) + + +def test_active_reconciliation_consumes_one_operator_confirmation_per_stop_attempt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + fresh_connection = _reconciliation_connection() + reconciliation_id = "reconciliation-55555555-5555-4555-8555-555555555555" + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id=reconciliation_id, + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + + with pytest.raises(PhysicalCommandTransitionError, match="may authorize only"): + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + connection=fresh_connection, + ) + with pytest.raises(PhysicalCommandTransitionError, match="freshly verified binding"): + _prepare_stop( + ledger, + operator_confirmation_id="confirmation-1", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + with pytest.raises(PhysicalCommandTransitionError, match="operator-confirmed STOP"): + _prepare_stop( + ledger, + connection=fresh_connection, + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + + _prepare_stop( + ledger, + connection=fresh_connection, + operator_confirmation_id="confirmation-1", + operator_confirmed_at_utc="2026-08-07T08:30:00.000Z", + baseline_observed_at="2026-08-07T08:29:00.000Z", + ) + prepared = ledger.snapshot().record + assert prepared is not None + assert prepared.action == "stop" + assert prepared.connection == fresh_connection + assert len(prepared.reconciliations) == 1 + assert len(prepared.operator_confirmations) == 1 + confirmation = prepared.operator_confirmations[0] + assert confirmation.reconciliation_id == reconciliation_id + assert confirmation.stop_operation_id == STOP_OPERATION + + ledger.resolve(STOP_OPERATION, resolution="not-dispatched") + with pytest.raises(PhysicalCommandBlocked, match="physical classification"): + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=fresh_connection, + operator_confirmation_id="confirmation-before-classification", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + recovery_connection = _connection( + control_session_id="reconcile-stop-retry-control-session", + host_path_epoch=9, + producer_generation=13, + ) + resolved_stop = ledger.snapshot().record + assert resolved_stop is not None + ledger.reconcile_prepared_stop_not_dispatched( + STOP_OPERATION, + expected_revision=resolved_stop.revision, + reconciliation_id="reconciliation-undispatched-stop-retry", + verified_binding=_verified_binding( + connection=recovery_connection, + verification_id="verification-undispatched-stop-retry", + ), + observation=_status( + "scanning", + connection=recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:31:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="operator-confirmed STOP"): + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=recovery_connection, + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + with pytest.raises(PhysicalCommandTransitionError, match="already been consumed"): + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=recovery_connection, + operator_confirmation_id="confirmation-1", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + _prepare_stop( + ledger, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=recovery_connection, + operator_confirmation_id="confirmation-2", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + retried = ledger.snapshot().record + assert retried is not None + assert [item.confirmation_id for item in retried.operator_confirmations] == [ + "confirmation-1", + "confirmation-2", + ] + assert [item.stop_operation_id for item in retried.operator_confirmations] == [ + STOP_OPERATION, + STOP_RETRY_OPERATION, + ] + + +def test_active_reconciliation_requires_same_acquisition_identity_and_profile( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + fresh_connection = _reconciliation_connection() + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + common = { + "operation_id": STOP_OPERATION, + "parent_operation_id": START_OPERATION, + "action": "stop", + "connection": fresh_connection, + "payload_sha256": "1" * 64, + "baseline_status": _status("scanning", connection=fresh_connection), + "operator_confirmation_id": "confirmation-1", + "operator_confirmed_at_utc": "2026-08-07T09:30:00.000Z", + } + with pytest.raises(PhysicalCommandTransitionError, match="original acquisition"): + ledger.prepare( + **common, # type: ignore[arg-type] + acquisition_id="different-acquisition", + identity=IDENTITY, + compatibility_profile_id="xgrids-k1-v0.6.0", + ) + with pytest.raises(PhysicalCommandTransitionError, match="identity does not match"): + ledger.prepare( + **common, # type: ignore[arg-type] + acquisition_id=ACQUISITION_ID, + identity=PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ), + compatibility_profile_id="xgrids-k1-v0.6.0", + ) + with pytest.raises(PhysicalCommandTransitionError, match="profile changed"): + ledger.prepare( + **common, # type: ignore[arg-type] + acquisition_id=ACQUISITION_ID, + identity=IDENTITY, + compatibility_profile_id="xgrids-k1-v0.7.0", + ) + assert ledger.snapshot().record is not None + assert ledger.snapshot().record.resolution == "physical-active-observed" + + +def test_crashed_reconciled_stop_requires_new_reconciliation_and_new_confirmation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + first_recovery_connection = _reconciliation_connection() + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=first_recovery_connection), + observation=_status( + "scanning", + connection=first_recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + _prepare_stop( + ledger, + connection=first_recovery_connection, + operator_confirmation_id="confirmation-1", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + ledger.mark_dispatching(STOP_OPERATION) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + second_recovery_connection = _connection( + control_session_id="reconcile-77777777-7777-4777-8777-777777777777", + host_path_epoch=9, + producer_generation=13, + ) + restarted.reconcile_ambiguous( + STOP_OPERATION, + reconciliation_id="reconciliation-77777777-7777-4777-8777-777777777777", + resolution="physical-active-observed", + verified_binding=_verified_binding( + connection=second_recovery_connection, + verification_id="verification-77777777-7777-4777-8777-777777777777", + ), + observation=_status( + "scanning", + connection=second_recovery_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + twice_reconciled = restarted.snapshot().record + assert twice_reconciled is not None + assert [item.original_attempt.operation_id for item in twice_reconciled.reconciliations] == [ + START_OPERATION, + STOP_OPERATION, + ] + assert [item.confirmation_id for item in twice_reconciled.operator_confirmations] == [ + "confirmation-1" + ] + assert twice_reconciled.original_command_outcome == "unknown" + assert twice_reconciled.reconciled_physical_state == "active" + + _prepare_stop( + restarted, + operation_id=STOP_RETRY_OPERATION, + parent_operation_id=STOP_OPERATION, + connection=second_recovery_connection, + operator_confirmation_id="confirmation-2", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + baseline_observed_at="2026-08-07T09:30:00.000Z", + ) + retry = restarted.snapshot().record + assert retry is not None + assert len(retry.reconciliations) == 2 + assert [item.confirmation_id for item in retry.operator_confirmations] == [ + "confirmation-1", + "confirmation-2", + ] + assert retry.operator_confirmations[-1].reconciliation_id == ( + "reconciliation-77777777-7777-4777-8777-777777777777" + ) + + +def test_standby_reconciliation_authorizes_fresh_start_and_carries_audit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + fresh_connection = _reconciliation_connection() + restarted.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-standby-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="only by fresh START"): + _prepare_stop( + restarted, + connection=fresh_connection, + operator_confirmation_id="confirmation-unexpected", + operator_confirmed_at_utc="2026-08-07T09:30:00.000Z", + ) + + next_connection = _connection( + control_session_id="control-66666666-6666-4666-8666-666666666666", + host_path_epoch=9, + producer_generation=13, + ) + restarted.prepare( + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + acquisition_id="acquisition-66666666-6666-4666-8666-666666666666", + action="start", + identity=IDENTITY, + connection=next_connection, + compatibility_profile_id="xgrids-k1-v0.6.0", + payload_sha256="3" * 64, + baseline_status=_status( + "ready", + connection=next_connection, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + prepared = restarted.snapshot().record + assert prepared is not None + assert prepared.action == "start" + assert prepared.reconciliations[0].resolution == "physical-standby-observed" + assert prepared.operator_confirmations == () + + +def test_reconciliation_rejects_mismatched_or_retained_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + fresh_connection = _reconciliation_connection() + active = _status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ) + + with pytest.raises(PhysicalCommandTransitionError, match="new verified"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-same-binding", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=_connection()), + observation=_status( + "scanning", + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="identity does not match"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-wrong-identity", + resolution="physical-active-observed", + verified_binding=_verified_binding( + identity=PhysicalCommandIdentity( + vendor_device_id_sha256="7" * 64, + device_serial_sha256="8" * 64, + ), + connection=fresh_connection, + ), + observation=active, + ) + with pytest.raises(PhysicalCommandTransitionError, match="live-path generation"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-wrong-generation", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=_connection( + control_session_id="different-observation-session", + host_path_epoch=10, + producer_generation=14, + ), + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="retained"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-retained", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + retained=True, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="read-only"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-live-source", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "scanning", + connection=fresh_connection, + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + with pytest.raises(PhysicalCommandTransitionError, match="active reconciliation"): + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-wrong-state", + resolution="physical-active-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + snapshot = ledger.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.reconciliations == () + + +def test_reconciliation_audit_tampering_fails_closed_after_restart( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.mark_dispatching(START_OPERATION) + fresh_connection = _reconciliation_connection() + ledger.reconcile_ambiguous( + START_OPERATION, + reconciliation_id="reconciliation-55555555-5555-4555-8555-555555555555", + resolution="physical-standby-observed", + verified_binding=_verified_binding(connection=fresh_connection), + observation=_status( + "ready", + connection=fresh_connection, + source="explicit-read-only-reconciliation", + observed_at="2026-08-07T09:30:00.000Z", + ), + ) + document = json.loads(ledger.path.read_text(encoding="utf-8")) + reconciliation = document["reconciliations"][0] + reconciliation["original_attempt"]["payload_sha256"] = "7" * 64 + canonical_attempt = json.dumps( + reconciliation["original_attempt"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + reconciliation["original_attempt_sha256"] = hashlib.sha256(canonical_attempt).hexdigest() + ledger.path.write_text( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\n", + encoding="utf-8", + ) + ledger.path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().status == "corrupt" + with pytest.raises(PhysicalCommandLedgerCorrupt): + restarted.require_no_unresolved_attempt() + + +@pytest.mark.parametrize( + "corrupt_payload", + [ + b"{corrupt\n", + ( + b'{"schema_version":"missioncore.xgrids-k1-physical-command/v1",' + b'"schema_version":"missioncore.xgrids-k1-physical-command/v1"}\n' + ), + json.dumps( + { + "schema_version": PHYSICAL_COMMAND_LEDGER_SCHEMA, + "password": "must-never-enter-ledger", + } + ).encode("utf-8"), + ], +) +def test_corrupt_or_noncanonical_record_fails_closed_without_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + corrupt_payload: bytes, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + ledger.path.write_bytes(corrupt_payload) + ledger.path.chmod(0o600) + before = ledger.path.read_bytes() + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + snapshot = restarted.snapshot() + assert snapshot.status == "corrupt" + assert snapshot.reason_code == PhysicalCommandLedgerCorrupt.reason_code + assert snapshot.requires_reconciliation + with pytest.raises(PhysicalCommandLedgerCorrupt): + restarted.require_no_unresolved_attempt() + with pytest.raises(PhysicalCommandLedgerCorrupt): + _prepare_start(restarted) + assert restarted.path.read_bytes() == before + + +@pytest.mark.parametrize("unsafe_kind", ["mode", "hardlink", "symlink", "oversize"]) +def test_unsafe_filesystem_records_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + unsafe_kind: str, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + if unsafe_kind == "mode": + ledger.path.chmod(0o644) + elif unsafe_kind == "hardlink": + os.link(ledger.path, tmp_path / "unexpected-ledger-link") + elif unsafe_kind == "symlink": + target = tmp_path / "attacker-controlled.json" + target.write_text("{}\n", encoding="utf-8") + target.chmod(0o600) + ledger.path.unlink() + ledger.path.symlink_to(target) + else: + ledger.path.write_bytes(b"x" * (ledger_module.PHYSICAL_COMMAND_LEDGER_MAX_BYTES + 1)) + ledger.path.chmod(0o600) + + restarted = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + assert restarted.snapshot().status == "corrupt" + with pytest.raises(PhysicalCommandLedgerCorrupt): + restarted.require_no_unresolved_attempt() + + +def test_unsafe_stable_lock_fails_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + os.link(ledger._process_lock_path, tmp_path / "unexpected-lock-link") + + with pytest.raises(PhysicalCommandLedgerCorrupt, match="lock failed closed"): + PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + + +def test_atomic_publication_fsyncs_file_and_directory_and_cleans_temp( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + real_fsync = ledger_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = ledger_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(ledger_module.os, "fsync", observe_fsync) + _prepare_start(ledger) + + assert "file" in fsync_kinds + assert fsync_kinds[-1] == "directory" + assert not list(ledger.path.parent.glob(".*.tmp")) + + +def test_failed_atomic_replace_preserves_previous_resolved_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ledger = _ledger(tmp_path, monkeypatch) + _prepare_start(ledger) + ledger.resolve(START_OPERATION, resolution="not-dispatched") + before = ledger.path.read_bytes() + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected replace failure") + + monkeypatch.setattr(ledger_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + _prepare_start( + ledger, + operation_id=NEXT_START_OPERATION, + parent_operation_id=START_OPERATION, + ) + assert ledger.path.read_bytes() == before + assert not list(ledger.path.parent.glob(".*.tmp")) + + +def test_two_instances_cannot_prepare_from_the_same_revision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _ledger(tmp_path, monkeypatch) + second = PhysicalCommandLedger(tmp_path / "repository", clock=_clock) + real_write = ledger_module._write_private_json_atomic + first_write_entered = threading.Event() + release_first_write = threading.Event() + second_started = threading.Event() + second_finished = threading.Event() + write_count = 0 + count_lock = threading.Lock() + outcomes: dict[str, BaseException | None] = {} + + def blocked_first_write( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + nonlocal write_count + with count_lock: + write_count += 1 + is_first = write_count == 1 + if is_first: + first_write_entered.set() + assert release_first_write.wait(timeout=5) + real_write(path, payload, data_dir=data_dir) + + monkeypatch.setattr(ledger_module, "_write_private_json_atomic", blocked_first_write) + + def prepare(name: str, ledger: PhysicalCommandLedger, operation_id: str) -> None: + if name == "second": + second_started.set() + try: + _prepare_start(ledger, operation_id=operation_id) + except BaseException as exc: # test captures the exact competing outcome + outcomes[name] = exc + else: + outcomes[name] = None + finally: + if name == "second": + second_finished.set() + + first_thread = threading.Thread( + target=prepare, + args=("first", first, START_OPERATION), + daemon=True, + ) + second_thread = threading.Thread( + target=prepare, + args=("second", second, NEXT_START_OPERATION), + daemon=True, + ) + first_thread.start() + assert first_write_entered.wait(timeout=5) + second_thread.start() + assert second_started.wait(timeout=5) + time.sleep(0.05) + assert not second_finished.is_set() + release_first_write.set() + first_thread.join(timeout=5) + second_thread.join(timeout=5) + + assert outcomes["first"] is None + assert isinstance(outcomes["second"], PhysicalCommandBlocked) + snapshot = first.snapshot() + assert snapshot.status == "unresolved" + assert snapshot.record is not None + assert snapshot.record.operation_id == START_OPERATION + assert write_count == 1 diff --git a/tests/test_xgrids_semantic_topology_store.py b/tests/test_xgrids_semantic_topology_store.py new file mode 100644 index 0000000..41e1212 --- /dev/null +++ b/tests/test_xgrids_semantic_topology_store.py @@ -0,0 +1,538 @@ +from __future__ import annotations + +import json +import multiprocessing +import os +import stat +import threading +import time +from pathlib import Path + +import pytest + +from k1link.device_plugins.xgrids_k1 import semantic_topology_store as store_module +from k1link.device_plugins.xgrids_k1.semantic_topology_store import ( + SEMANTIC_TOPOLOGY_MAX_BYTES, + SEMANTIC_TOPOLOGY_SCHEMA, + SemanticTopologyRecord, + SemanticTopologyStore, + SemanticTopologyStoreCorrupt, + StaleSemanticTopologyObservation, +) + +TRANSPORT_REF = "A161D9D5-C352-1069-D430-5FB0BC13F7F9" +PROFILE_ID = "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2" + + +def _commit_while_holding_process_lock( + data_dir: str, + repository_root: str, + release_path: str, + events: multiprocessing.Queue[tuple[str, int]], +) -> None: + os.environ["MISSIONCORE_DATA_DIR"] = data_dir + store = SemanticTopologyStore(Path(repository_root)) + real_write = store_module._write_private_json_atomic + + def hold_lock( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + events.put(("entered", 0)) + deadline = time.monotonic() + 5 + while not Path(release_path).exists(): + if time.monotonic() >= deadline: + raise TimeoutError("parent did not release cross-process topology commit") + time.sleep(0.01) + real_write(path, payload, data_dir=data_dir) + + store_module._write_private_json_atomic = hold_lock + record = store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:30:00.000Z", + ) + events.put(("committed", record.revision)) + + +def _store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> SemanticTopologyStore: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + return SemanticTopologyStore(tmp_path / "repository") + + +def _commit( + store: SemanticTopologyStore, + *, + ipv4: str = "192.168.68.50", + mode: str = "bridge", + source: str = "ble-read-only-status", + observed_at: str = "2026-08-06T12:30:00.000Z", +) -> None: + store.commit( + transport_ref=TRANSPORT_REF, + connection_mode=mode, # type: ignore[arg-type] + ipv4=ipv4, + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source=source, # type: ignore[arg-type] + observed_at_utc=observed_at, + ) + + +def test_store_is_private_secret_free_and_restart_evidence_is_offline_only( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + record = store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:30:00Z", + ) + + assert record.revision == 1 + assert record.observed_at_utc == "2026-08-06T12:30:00.000Z" + assert record.live_connection_authority is False + assert stat.S_IMODE((tmp_path / "private-data").stat().st_mode) == 0o700 + assert stat.S_IMODE(store.path.parent.stat().st_mode) == 0o700 + assert stat.S_IMODE(store.path.stat().st_mode) == 0o600 + assert store.path.stat().st_nlink == 1 + + document = json.loads(store.path.read_text(encoding="utf-8")) + assert document == record.as_dict() + assert set(document) == { + "schema_version", + "revision", + "transport_ref", + "connection_mode", + "ipv4", + "compatibility_profile_id", + "firmware_version", + "source", + "observed_at_utc", + } + serialized = store.path.read_text(encoding="utf-8").casefold() + assert "ssid" not in serialized + assert "password" not in serialized + assert "credential" not in serialized + assert "secret" not in serialized + + restarted = SemanticTopologyStore(tmp_path / "repository") + snapshot = restarted.snapshot() + assert snapshot.status == "available" + assert snapshot.record == record + assert snapshot.configured_offline_evidence is True + assert snapshot.live_connection_authority is False + assert snapshot.record is not None + assert snapshot.record.live_connection_authority is False + assert snapshot.as_dict() == { + "schema_version": SEMANTIC_TOPOLOGY_SCHEMA, + "status": "available", + "configured_offline_evidence": True, + "live_connection_authority": False, + "reason_code": None, + "record": record.as_dict(), + } + + +def test_serialized_commit_accepts_wall_rollback_and_lineage_rejects_stale_writer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + _commit(store, observed_at="2026-08-06T12:30:01.000Z") + second = store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="quick-connect", + ipv4="192.168.56.1", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-post-write-status", + observed_at_utc="2026-08-06T11:30:02.000Z", + predecessor_revision=1, + ) + + assert second.revision == 2 + assert second.connection_mode == "quick-connect" + assert second.observed_at_utc == "2026-08-06T11:30:02.000Z" + before = store.path.read_bytes() + with pytest.raises(StaleSemanticTopologyObservation): + store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T13:30:01.500Z", + predecessor_revision=1, + ) + assert store.path.read_bytes() == before + + third = store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T10:30:01.000Z", + predecessor_revision=2, + ) + assert third.revision == 3 + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("connection_mode", "automatic"), + ("ipv4", "192.168.068.050"), + ("ipv4", "2001:db8::1"), + ("transport_ref", "../../scanner"), + ("compatibility_profile_id", "profile\nleak"), + ("firmware_version", "3.0.2\npassword=x"), + ("source", "tcp-probe"), + ("observed_at_utc", "2026-08-06T12:30:00+03:00"), + ], +) +def test_commit_rejects_values_outside_the_bounded_schema( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, + value: str, +) -> None: + store = _store(tmp_path, monkeypatch) + arguments: dict[str, object] = { + "transport_ref": TRANSPORT_REF, + "connection_mode": "bridge", + "ipv4": "192.168.68.50", + "compatibility_profile_id": PROFILE_ID, + "firmware_version": "3.0.2", + "source": "ble-read-only-status", + "observed_at_utc": "2026-08-06T12:30:00.000Z", + } + arguments[field] = value + + with pytest.raises(ValueError): + store.commit(**arguments) # type: ignore[arg-type] + assert store.snapshot().status == "empty" + + +@pytest.mark.parametrize( + "payload", + [ + b'{"schema_version":"missioncore.xgrids-k1-semantic-topology/v1",' + b'"schema_version":"missioncore.xgrids-k1-semantic-topology/v1"}\n', + json.dumps( + { + "schema_version": SEMANTIC_TOPOLOGY_SCHEMA, + "revision": 1, + "transport_ref": TRANSPORT_REF, + "connection_mode": "bridge", + "ipv4": "192.168.68.50", + "compatibility_profile_id": PROFILE_ID, + "firmware_version": "3.0.2", + "source": "ble-read-only-status", + "observed_at_utc": "2026-08-06T12:30:00.000Z", + "ssid": "must-not-be-stored", + } + ).encode(), + b"{" + b"x" * SEMANTIC_TOPOLOGY_MAX_BYTES + b"}", + ], +) +def test_duplicate_unknown_or_oversize_json_fails_closed_without_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + payload: bytes, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + path = tmp_path / "private-data" / "xgrids-k1" / "semantic-topology.json" + path.parent.mkdir(mode=0o700, parents=True) + (tmp_path / "private-data").chmod(0o700) + path.parent.chmod(0o700) + path.write_bytes(payload) + path.chmod(0o600) + + store = SemanticTopologyStore(tmp_path / "repository") + assert store.snapshot().status == "corrupt" + before = path.read_bytes() + with pytest.raises(SemanticTopologyStoreCorrupt): + _commit(store) + assert path.read_bytes() == before + + +def test_symlink_hardlink_and_nonprivate_file_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + parent = tmp_path / "private-data" / "xgrids-k1" + parent.mkdir(mode=0o700, parents=True) + (tmp_path / "private-data").chmod(0o700) + parent.chmod(0o700) + path = parent / "semantic-topology.json" + target = tmp_path / "outside.json" + target.write_text("{}", encoding="utf-8") + target.chmod(0o600) + path.symlink_to(target) + + symlink_store = SemanticTopologyStore(tmp_path / "repository") + assert symlink_store.snapshot().status == "corrupt" + path.unlink() + + path.write_text("{}", encoding="utf-8") + path.chmod(0o600) + hardlink = tmp_path / "second-link.json" + os.link(path, hardlink) + hardlink_store = SemanticTopologyStore(tmp_path / "repository") + assert hardlink_store.snapshot().status == "corrupt" + hardlink.unlink() + path.unlink() + + path.write_text("{}", encoding="utf-8") + path.chmod(0o644) + permission_store = SemanticTopologyStore(tmp_path / "repository") + assert permission_store.snapshot().status == "corrupt" + + +def test_nonprivate_directory_and_unsafe_lock_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + data_dir = tmp_path / "private-data" + data_dir.mkdir(mode=0o755) + data_dir.chmod(0o755) + with pytest.raises(SemanticTopologyStoreCorrupt, match="permissions"): + SemanticTopologyStore(tmp_path / "repository") + + data_dir.chmod(0o700) + parent = data_dir / "xgrids-k1" + parent.mkdir(mode=0o700) + lock = parent / ".semantic-topology.lock" + lock.write_bytes(b"not-empty") + lock.chmod(0o600) + with pytest.raises(SemanticTopologyStoreCorrupt, match="stable private"): + SemanticTopologyStore(tmp_path / "repository") + + +def test_symlink_hardlink_and_nonprivate_lock_fail_closed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MISSIONCORE_DATA_DIR", str(tmp_path / "private-data")) + data_dir = tmp_path / "private-data" + parent = data_dir / "xgrids-k1" + parent.mkdir(mode=0o700, parents=True) + data_dir.chmod(0o700) + parent.chmod(0o700) + lock = parent / ".semantic-topology.lock" + target = tmp_path / "outside.lock" + target.touch(mode=0o600) + lock.symlink_to(target) + with pytest.raises(SemanticTopologyStoreCorrupt, match="opened safely"): + SemanticTopologyStore(tmp_path / "repository") + lock.unlink() + + lock.touch(mode=0o600) + lock.chmod(0o600) + hardlink = tmp_path / "second.lock" + os.link(lock, hardlink) + with pytest.raises(SemanticTopologyStoreCorrupt, match="stable private"): + SemanticTopologyStore(tmp_path / "repository") + hardlink.unlink() + lock.unlink() + + lock.touch(mode=0o600) + lock.chmod(0o644) + with pytest.raises(SemanticTopologyStoreCorrupt, match="stable private"): + SemanticTopologyStore(tmp_path / "repository") + + +def test_atomic_publication_fsyncs_file_and_parent_and_cleans_temp_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + real_fsync = store_module.os.fsync + fsync_kinds: list[str] = [] + + def observe_fsync(descriptor: int) -> None: + mode = store_module.os.fstat(descriptor).st_mode + fsync_kinds.append("directory" if stat.S_ISDIR(mode) else "file") + real_fsync(descriptor) + + monkeypatch.setattr(store_module.os, "fsync", observe_fsync) + _commit(store) + + assert "file" in fsync_kinds + assert fsync_kinds[-1] == "directory" + assert not list(store.path.parent.glob(".semantic-topology.json.*.tmp")) + + +def test_failed_atomic_replace_preserves_previous_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + _commit(store) + before = store.path.read_bytes() + + def fail_replace(_source: Path, _destination: Path) -> None: + raise OSError("injected replace failure") + + monkeypatch.setattr(store_module.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected replace failure"): + _commit(store, observed_at="2026-08-06T12:31:00.000Z") + assert store.path.read_bytes() == before + assert not list(store.path.parent.glob(".semantic-topology.json.*.tmp")) + + +def test_two_instances_serialize_reload_and_revision_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + first = _store(tmp_path, monkeypatch) + second = SemanticTopologyStore(tmp_path / "repository") + real_write = store_module._write_private_json_atomic + first_write_entered = threading.Event() + release_first_write = threading.Event() + second_started = threading.Event() + second_finished = threading.Event() + write_count = 0 + count_lock = threading.Lock() + results: dict[str, object] = {} + + def blocked_first_write( + path: Path, + payload: dict[str, object], + *, + data_dir: Path, + ) -> None: + nonlocal write_count + with count_lock: + write_count += 1 + should_block = write_count == 1 + if should_block: + first_write_entered.set() + assert release_first_write.wait(timeout=5) + real_write(path, payload, data_dir=data_dir) + + def run_first() -> None: + results["first"] = first.commit( + transport_ref=TRANSPORT_REF, + connection_mode="bridge", + ipv4="192.168.68.50", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-read-only-status", + observed_at_utc="2026-08-06T12:30:00.000Z", + ) + + def run_second() -> None: + second_started.set() + try: + results["second"] = second.commit( + transport_ref=TRANSPORT_REF, + connection_mode="quick-connect", + ipv4="192.168.56.1", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-post-write-status", + observed_at_utc="2026-08-06T12:30:01.000Z", + ) + finally: + second_finished.set() + + monkeypatch.setattr(store_module, "_write_private_json_atomic", blocked_first_write) + first_thread = threading.Thread(target=run_first, daemon=True) + second_thread = threading.Thread(target=run_second, daemon=True) + first_thread.start() + assert first_write_entered.wait(timeout=5) + second_thread.start() + assert second_started.wait(timeout=5) + try: + assert second_finished.wait(timeout=0.2) is False + finally: + release_first_write.set() + first_thread.join(timeout=5) + second_thread.join(timeout=5) + + assert first_thread.is_alive() is False + assert second_thread.is_alive() is False + assert isinstance(results["first"], SemanticTopologyRecord) + assert isinstance(results["second"], SemanticTopologyRecord) + assert results["first"].revision == 1 + assert results["second"].revision == 2 + snapshot = SemanticTopologyStore(tmp_path / "repository").snapshot() + assert snapshot.record is not None + assert snapshot.record.revision == 2 + assert snapshot.record.connection_mode == "quick-connect" + + +def test_separate_process_commit_holds_stable_flock_for_whole_transaction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _store(tmp_path, monkeypatch) + context = multiprocessing.get_context("spawn") + events = context.Queue() + release_path = tmp_path / "release-child" + child = context.Process( + target=_commit_while_holding_process_lock, + args=( + str(tmp_path / "private-data"), + str(tmp_path / "repository"), + str(release_path), + events, + ), + daemon=True, + ) + parent_finished = threading.Event() + outcome: dict[str, object] = {} + + def commit_from_parent() -> None: + try: + outcome["record"] = store.commit( + transport_ref=TRANSPORT_REF, + connection_mode="quick-connect", + ipv4="192.168.56.1", + compatibility_profile_id=PROFILE_ID, + firmware_version="3.0.2", + source="ble-post-write-status", + observed_at_utc="2026-08-06T12:30:01.000Z", + ) + finally: + parent_finished.set() + + child.start() + parent_thread: threading.Thread | None = None + try: + assert events.get(timeout=5) == ("entered", 0) + parent_thread = threading.Thread(target=commit_from_parent, daemon=True) + parent_thread.start() + assert parent_finished.wait(timeout=0.2) is False + release_path.touch() + assert events.get(timeout=5) == ("committed", 1) + child.join(timeout=5) + parent_thread.join(timeout=5) + finally: + if child.is_alive(): + child.terminate() + child.join(timeout=5) + + assert child.exitcode == 0 + assert parent_thread is not None + assert parent_thread.is_alive() is False + assert isinstance(outcome["record"], SemanticTopologyRecord) + assert outcome["record"].revision == 2