SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

chore(k1): checkpoint connection recovery work

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-21 08:12:22 +03:00
parent a3138f3d71
commit c7843a3c7e
22 changed files with 969 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_frontend_plugin_boundaries.py
+4 -4
View File
@@ -132,16 +132,16 @@ def test_xgrids_live_copy_exposes_response_gated_prepare_and_one_physical_start(
assert "Запустить приём" in connection_source
assert "Запустить K1" not in connection_source
assert "Одно нажатие выполняет каноническую подготовку и один START" in (
assert "Одно нажатие выполняет каноническую подготовку и один START" not in (
connection_source
)
assert "Проверить условия и отправить START" not in connection_source
assert "Физический START отправляется только после отдельного финального окна" not in (
connection_source
)
assert "Имя войдёт в единственный канонический START" in connection_source
assert "один START после подтверждённого READY" in connection_source
assert "Автоматических повторов команд нет" in connection_source
assert "Имя войдёт в единственный канонический START" not in connection_source
assert "один START после подтверждённого READY" not in connection_source
assert "Автоматических повторов команд нет" not in connection_source
assert "Подключить управление K1" not in connection_source
assert "Открыть рабочее пространство K1" not in connection_source
assert "Сохранить проект и подготовить локальный приём" not in connection_source
tests/test_xgrids_acquisition_lifecycle.py
+59 -15
View File
@@ -675,20 +675,6 @@ class FakeInteractiveControlSession:
self.state_revision += 1
return self.snapshot()
def release_inspection_for_operator_dialogue(
self,
*,
expected_session_generation: int,
expected_state_revision: int,
) -> dict[str, object]:
self._accept_checkpoint(
expected_session_generation=expected_session_generation,
expected_state_revision=expected_state_revision,
)
assert self.state == "connection-ready"
self.inspection_promotion_allowed = True
return self.snapshot()
def adopt_reconciled_scanning(
self,
*,
@@ -6553,6 +6539,64 @@ def test_control_session_reuses_reachable_process_owned_connection_lease(
service._release_application_control_process_lease() # noqa: SLF001
def test_operator_control_open_replaces_verify_inspection_with_fresh_dialogue(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
service, _ = service_with_fake_runtime(tmp_path)
transitions: list[str] = []
class RecordingControlSession(FakeInteractiveControlSession):
def close_prestart(self, **kwargs: object) -> dict[str, object]:
transitions.append("inspection-closed")
return super().close_prestart(**kwargs)
def retire_for_network_change(self, **kwargs: object) -> dict[str, object]:
transitions.append("inspection-retired")
return super().retire_for_network_change(**kwargs)
def open(self, *, inspection_only: bool = False, **kwargs: object) -> dict[str, object]:
transitions.append(
"inspection-opened" if inspection_only else "canonical-opened"
)
self.session_generation += 1
return super().open(inspection_only=inspection_only, **kwargs)
control = RecordingControlSession(initial_state="connection-ready")
control.session_generation = 7
control.state_revision = 11
control.inspection_only = True
control.inspection_promotion_allowed = False
service._application_control_session = control # type: ignore[assignment] # noqa: SLF001
binding = _seed_supervised_connection(service)
control.verified_control = _verified_control_for_binding(binding)
service._acquire_application_control_process_lease() # noqa: SLF001
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", lambda _target: True)
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
state = service.open_application_control_session(
OpenApplicationControlSessionRequest(
operator_present=True,
owner_controlled_device=True,
lixelgo_closed=True,
battery_storage_confirmed=True,
expected_physical_state_confirmed=True,
timezone_name="Europe/Moscow",
)
)
assert transitions == [
"inspection-closed",
"inspection-retired",
"canonical-opened",
]
assert state["application_control_session"]["state"] == "connection-ready"
assert state["application_control_session"]["inspection_only"] is False
assert state["application_control_session"]["session_generation"] == 8
service._release_application_control_process_lease() # noqa: SLF001
def test_reachable_connection_lease_supports_repeated_independent_control_sessions(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -27705,7 +27749,7 @@ def test_reset_retired_apply_continuation_settles_without_receiver_rehydrate(
else:
assert control["state"] == "connection-ready"
assert control["inspection_only"] is True
assert control["inspection_promotion_allowed"] is True
assert control["inspection_promotion_allowed"] is False
assert settled["connection_attempt"]["safe_next_action"] == (
"start-acquisition"
)
tests/test_xgrids_active_acquisition_checkpoint_integration.py
+7 -2
View File
@@ -521,7 +521,11 @@ def test_ready_after_dispatched_stop_ceases_active_checkpoint(
)
restarted = _connection(
control_session_id="checkpoint-dispatched-stop-restarted-control",
host_path_epoch=2,
# A read-only reconnect on the same Wi-Fi route legitimately keeps the
# host-path epoch. Fresh control/evidence and producer generations are
# the replay fences; requiring a route change strands a confirmed READY
# behind an ACTIVE checkpoint, as observed in the live Bridge flow.
host_path_epoch=(original.host_path_epoch if same_process else 2),
producer_generation=2,
)
coordinator = service._physical_command_coordinator # noqa: SLF001
@@ -593,7 +597,8 @@ def test_ready_after_dispatched_stop_ceases_active_checkpoint(
# The physical ledger fsync is durable while checkpoint cessation did not
# commit. Recovery may happen after a process restart or, as observed in
# production, later in the same process under a fresh control/path epoch.
# production, later in the same process under a fresh control epoch while
# the host route itself remains unchanged.
if same_process:
monkeypatch.setattr(
service,
tests/test_xgrids_active_acquisition_recovery_checkpoint.py
+98 -1
View File
@@ -87,13 +87,14 @@ def _binding(
host_path_epoch: int = 10,
control_session_id: str = "control-A",
producer_generation: int = 7,
target_ipv4: str = CONNECTION.target_ipv4,
) -> ActiveAcquisitionRecoveryTransportBinding:
return ActiveAcquisitionRecoveryTransportBinding(
runtime_instance_id=runtime_instance_id,
intent_id="intent-001",
transport_ref=CONNECTION.transport_ref,
connection_mode=CONNECTION.connection_mode,
target_ipv4=CONNECTION.target_ipv4,
target_ipv4=target_ipv4,
target_port=CONNECTION.target_port,
host_path_epoch=host_path_epoch,
control_session_id=control_session_id,
@@ -1313,6 +1314,7 @@ def test_cease_active_reconciled_standby_rejects_inexact_restart_proofs(
status.binding,
runtime_instance_id=binding.runtime_instance_id,
host_path_epoch=binding.host_path_epoch,
producer_generation=binding.producer_generation,
),
replace(
status.binding,
@@ -2211,6 +2213,101 @@ def test_prepared_crash_cannot_cease_on_status_alone(
assert ceased.prepared_resolution_proof is not None
def test_prepared_ambiguous_start_standby_accepts_verified_dhcp_target_change(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
prepared_binding = _binding()
store = _store(tmp_path, monkeypatch)
prepared = _prepare(store, prepared_binding)
recovered_binding = _binding(
runtime_instance_id="runtime-recovered-new-address",
host_path_epoch=1,
control_session_id="control-recovered-new-address",
producer_generation=1,
target_ipv4="192.168.43.54",
)
ready = _status(
"ready",
binding=recovered_binding,
evidence_session_id="evidence-recovered-new-address",
observed_at="2026-08-13T12:01:00.000Z",
)
physical = _physical(
"physical-standby-observed",
binding=recovered_binding,
proof_id="physical-ambiguous-ready-new-address",
observed_at=ready.observed_at_utc,
)
ceased = store.cease(
transition_id="transition-cease-ambiguous-new-address",
expected_revision=prepared.revision,
expected_acquisition_id=ACQUISITION_ID,
expected_start_operation_id=START_OPERATION_ID,
status_proof=ready,
physical_proof=physical,
)
assert ceased.state == "ceased"
assert ceased.transport_revision == prepared.transport_revision + 1
assert ceased.prepared_binding == prepared_binding
assert ceased.current_binding == recovered_binding
assert ceased.last_gap_failed_binding == prepared_binding
assert ceased.last_gap_started_transport_revision == prepared.transport_revision
assert ceased.last_gap_recovered_transport_revision == ceased.transport_revision
assert ceased.cessation_status_proof == ready
assert ceased.cessation_physical_proof == physical
assert (
ActiveAcquisitionRecoveryCheckpointStore(tmp_path / "repository")
.snapshot()
.checkpoint
== ceased
)
def test_prepared_dhcp_target_change_rejects_stale_runtime_proof(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
prepared_binding = _binding()
store = _store(tmp_path, monkeypatch)
prepared = _prepare(store, prepared_binding)
stale_binding = _binding(
runtime_instance_id=prepared_binding.runtime_instance_id,
host_path_epoch=prepared_binding.host_path_epoch + 1,
control_session_id=prepared_binding.control_session_id,
producer_generation=prepared_binding.producer_generation + 1,
target_ipv4="192.168.43.54",
)
ready = _status(
"ready",
binding=stale_binding,
evidence_session_id="evidence-recovered-new-address",
observed_at="2026-08-13T12:01:00.000Z",
)
with pytest.raises(
ActiveAcquisitionRecoveryCheckpointTransitionError,
match="target change requires fresh DeviceInfo-bound runtime",
):
store.cease(
transition_id="transition-reject-stale-new-address",
expected_revision=prepared.revision,
expected_acquisition_id=ACQUISITION_ID,
expected_start_operation_id=START_OPERATION_ID,
status_proof=ready,
physical_proof=_physical(
"physical-standby-observed",
binding=stale_binding,
proof_id="physical-stale-ready-new-address",
observed_at=ready.observed_at_utc,
),
)
assert store.snapshot().checkpoint == prepared
def test_active_cease_requires_exact_current_binding_and_physical_stop(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
tests/test_xgrids_active_acquisition_restart_rehydration.py
+96
View File
@@ -828,6 +828,102 @@ def test_ambiguous_prepared_restart_waits_for_first_pcl_before_activation(
assert ledger_record.resolution == "physical-active-observed"
def test_ambiguous_prepared_restart_ready_settles_after_dhcp_target_change(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
first = restart_support._service(tmp_path, monkeypatch)
store = restart_support._seed_matching_prepared_start(first)
ledger = first._physical_command_ledger # noqa: SLF001
original = restart_support._prepared_connection()
ledger.mark_dispatching(restart_support.START_OPERATION_ID)
ledger.mark_observing(
restart_support.START_OPERATION_ID,
publish_call_returned=True,
packet_id=41,
)
ledger.mark_qos2_completed(restart_support.START_OPERATION_ID, packet_id=41)
ledger.record_application_response(
restart_support.START_OPERATION_ID,
restart_support.PhysicalCommandApplicationResponse(
operation_id=restart_support.START_OPERATION_ID,
action="start",
control_session_id=original.control_session_id,
host_path_epoch=original.host_path_epoch,
producer_generation=original.producer_generation,
result_code=restart_support.PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
success=True,
payload_sha256="f" * 64,
observed_at_utc="2026-08-13T12:00:01.000Z",
),
)
restarted = restart_support._service(tmp_path, monkeypatch)
token = restarted._validate_active_acquisition_checkpoint_lineage() # noqa: SLF001
assert token is not None
recovered = restart_support.PhysicalCommandConnectionBinding(
intent_id=original.intent_id,
transport_ref=original.transport_ref,
connection_mode=original.connection_mode,
target_ipv4="192.168.68.54",
target_port=original.target_port,
host_path_epoch=2,
control_session_id="control-restart-safety-new-address",
producer_generation=2,
)
ready = restart_support.PhysicalCommandStatusEvidence(
source="explicit-read-only-reconciliation",
vendor_device_id_sha256=restart_support.VENDOR_SHA256,
device_serial_sha256=restart_support.SERIAL_SHA256,
control_session_id=recovered.control_session_id,
host_path_epoch=recovered.host_path_epoch,
producer_generation=recovered.producer_generation,
session_state="ready",
session_state_code=300,
project_bound=False,
project_id_sha256=None,
init_ready=False,
status_message_sha256="9" * 64,
mqtt_retained=False,
observed_at_utc="2026-08-13T12:20:01.000Z",
)
record = restarted._physical_command_ledger.reconcile_ambiguous( # noqa: SLF001
restart_support.START_OPERATION_ID,
reconciliation_id="reconciliation-ready-new-dhcp-address",
resolution="physical-standby-observed",
verified_binding=restart_support.PhysicalCommandVerifiedBinding(
verification_id="verification-ready-new-dhcp-address",
identity=restart_support.PhysicalCommandIdentity(
vendor_device_id_sha256=restart_support.VENDOR_SHA256,
device_serial_sha256=restart_support.SERIAL_SHA256,
),
connection=recovered,
device_info_message_sha256="8" * 64,
verified_at_utc="2026-08-13T12:20:00.000Z",
),
observation=ready,
)
assert (
restarted._physical_command_coordinator.snapshot()[ # noqa: SLF001
"recovery_requirement"
]
is None
)
result = asyncio.run(
restarted._reconcile_physical_command_after_verify_owned( # noqa: SLF001
verify_operation_id="verify-catches-up-checkpoint-new-address",
)
)
assert result["performed"] is False
checkpoint = store.snapshot().checkpoint
assert checkpoint is not None
assert checkpoint.state == "ceased"
assert checkpoint.prepared_binding.target_ipv4 == "192.168.68.52"
assert checkpoint.current_binding.target_ipv4 == "192.168.68.54"
assert checkpoint.transport_revision == 2
def test_composite_prepared_restart_uses_exact_origin_and_zero_start_replay(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
tests/test_xgrids_application_session.py
+5 -13
View File
@@ -527,6 +527,7 @@ def test_read_only_device_info_open_does_not_require_physical_acceptance(
ready = _wait_phase(session, "connection-ready")
assert ready["verified_control"] is not None
assert FakeExecutor.records[:2] == ["inspection:1", "connection:2-6"]
assert coordinator.prepares == []
assert "start:11-14" not in FakeExecutor.records
assert "stop" not in FakeExecutor.records
@@ -536,7 +537,7 @@ def test_read_only_device_info_open_does_not_require_physical_acceptance(
)
def test_inspection_session_rejects_workspace_until_verify_releases_boundary(
def test_inspection_session_rejects_workspace_and_closes_without_connection_stage(
monkeypatch: pytest.MonkeyPatch,
) -> None:
FakeExecutor.records = []
@@ -561,21 +562,12 @@ def test_inspection_session_rejects_workspace_until_verify_releases_boundary(
)
assert FakeExecutor.records == ["inspection:1", "wait:workspace-entered"]
released = session.release_inspection_for_operator_dialogue(
session.close_prestart(
expected_session_generation=ready["session_generation"], # type: ignore[arg-type]
expected_state_revision=ready["state_revision"], # type: ignore[arg-type]
)
session.enter_workspace(
expected_session_generation=released["session_generation"], # type: ignore[arg-type]
expected_state_revision=released["state_revision"], # type: ignore[arg-type]
)
_wait_phase(session, "workspace-ready")
assert FakeExecutor.records[:4] == [
"inspection:1",
"wait:workspace-entered",
"connection:2-6",
"workspace:7",
]
_wait_phase(session, "closed")
assert FakeExecutor.records == ["inspection:1", "wait:workspace-entered"]
@pytest.mark.parametrize(