NODEDC_MISSION_CORE/tests/test_xgrids_application_ses...

729 lines
24 KiB
Python

from __future__ import annotations
import logging
import threading
import time
from dataclasses import dataclass
from typing import Any
import pytest
from k1link.device_plugins.xgrids_k1.protocol import application_session as session_module
from k1link.device_plugins.xgrids_k1.protocol.application_bootstrap import (
ApplicationControlAuthority,
LiveDeviceControlBinding,
)
from k1link.device_plugins.xgrids_k1.protocol.application_session import (
InteractiveApplicationControlSession,
OperatorPresenceConfirmation,
)
APPLICATION_KEY = "00000000-0000-0000-0000-000000000000"
class FakeAuthorityLoader:
def __init__(self) -> None:
self.calls = 0
def load(self) -> ApplicationControlAuthority:
self.calls += 1
return ApplicationControlAuthority(openapi_key=APPLICATION_KEY)
@dataclass
class FakeTransportSnapshot:
state: str
ready: bool
def as_dict(self) -> dict[str, object]:
return {
"state": self.state,
"publish_attempts": 0,
"qos2_completions": 0,
"correlated_responses": 0,
"ignored_known_responses": 0,
"late_known_responses": 0,
"latest_device_session_state": "ready" if self.ready else "scanning",
"latest_device_project_bound": not self.ready,
"automatic_retry": False,
"automatic_reconnect": False,
}
class FakeTransport:
def __init__(self, host: str) -> None:
self.host = host
self.state = "new"
self.ready = True
def open(self) -> FakeTransportSnapshot:
self.state = "ready"
return self.snapshot()
def close(self) -> None:
self.state = "closed"
def snapshot(self) -> FakeTransportSnapshot:
return FakeTransportSnapshot(self.state, self.ready)
class FakeExecutor:
records: list[str] = []
def __init__(self, transport: FakeTransport) -> None:
self.transport = transport
self.binding = LiveDeviceControlBinding(
vendor_device_id="device-id",
device_serial="serial-id",
software_version="3.0.2",
system_version="3.0.2",
device_model="LixelKity K1",
device_type="A4",
is_activated=True,
)
def run_connection_stage(self, _orchestrator: object) -> LiveDeviceControlBinding:
self.records.append("connection:1-6")
return self.binding
def wait_for_operator_checkpoint(
self,
event: str,
observed: Any,
) -> str:
self.records.append(f"wait:{event}")
deadline = time.monotonic() + 2.0
while not observed():
if time.monotonic() >= deadline:
raise TimeoutError(f"test did not release {event}")
threading.Event().wait(0.005)
return event
def run_workspace_entry_stage(
self,
_orchestrator: object,
_checkpoint: object,
) -> LiveDeviceControlBinding:
self.records.append("workspace:7")
return self.binding
def run_project_prompt_stage(
self,
_orchestrator: object,
_checkpoint: object,
) -> LiveDeviceControlBinding:
self.records.append("project:8-10")
return self.binding
def execute_canonical_start(self, *_args: object, **_kwargs: object) -> object:
self.records.append("start:11-14")
self.transport.ready = False
return object()
def maintain_active_until_stop_requested(self, observed: Any) -> None:
self.records.append("wait:stop")
while not observed():
threading.Event().wait(0.005)
def execute_canonical_stop(self, *_args: object, **_kwargs: object) -> object:
self.records.append("stop")
self.transport.ready = True
return object()
def maintain_post_stop_until_standby(self) -> None:
self.records.append("wait:device-standby")
def snapshot(self) -> dict[str, object]:
return {
"records": list(self.records),
"dialogue_stage": "test-stage",
"start_attempted": False,
"stop_attempted": False,
"automatic_retry": False,
}
def _confirmation() -> OperatorPresenceConfirmation:
return OperatorPresenceConfirmation(
operator_present=True,
owner_controlled_device=True,
lixelgo_closed=True,
battery_storage_confirmed=True,
expected_physical_state_confirmed=True,
)
def _wait_phase(
session: InteractiveApplicationControlSession,
expected: str,
) -> dict[str, object]:
deadline = time.monotonic() + 2.0
while time.monotonic() < deadline:
snapshot = session.snapshot()
if snapshot["state"] == expected:
return snapshot
threading.Event().wait(0.005)
raise AssertionError(f"session did not reach {expected}: {session.snapshot()}")
def test_canonical_stages_require_operator_events_but_device_standby_does_not(
monkeypatch: pytest.MonkeyPatch,
) -> None:
FakeExecutor.records = []
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
FakeExecutor,
)
loader = FakeAuthorityLoader()
transport = FakeTransport("192.168.1.20")
session = InteractiveApplicationControlSession(
loader,
transport_factory=lambda _host: transport, # type: ignore[arg-type]
epoch_seconds=lambda: 1_752_680_000,
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
_wait_phase(session, "connection-ready")
threading.Event().wait(0.02)
assert FakeExecutor.records == ["connection:1-6", "wait:workspace-entered"]
session.enter_workspace()
_wait_phase(session, "workspace-ready")
assert FakeExecutor.records[-1] == "wait:project-prompt-opened"
session.open_project_prompt()
_wait_phase(session, "project-ready")
assert FakeExecutor.records[-1] == "wait:start-confirmed"
session.request_start(project_name="TEST001", confirmation=_confirmation())
_wait_phase(session, "scanning")
assert FakeExecutor.records[-1] == "wait:stop"
session.request_stop(confirmation=_confirmation())
completed = _wait_phase(session, "completed")
assert completed["can_confirm_standby"] is False
assert completed["pending_operator_action"] is None
assert loader.calls == 1
assert completed["automatic_retry"] is False
assert completed["scripted_transitions"] is False
assert FakeExecutor.records == [
"connection:1-6",
"wait:workspace-entered",
"workspace:7",
"wait:project-prompt-opened",
"project:8-10",
"wait:start-confirmed",
"start:11-14",
"wait:stop",
"stop",
"wait:device-standby",
]
def test_prestart_failure_is_reported_and_requires_a_new_operator_click(
monkeypatch: pytest.MonkeyPatch,
caplog: pytest.LogCaptureFixture,
) -> None:
caplog.set_level(logging.ERROR)
class BootstrapFailureExecutor(FakeExecutor):
def run_connection_stage(
self,
_orchestrator: object,
) -> LiveDeviceControlBinding:
raise RuntimeError("bootstrap correlation failed")
def snapshot(self) -> dict[str, object]:
return {
"dialogue_stage": "connection",
"start_attempted": False,
"stop_attempted": False,
"response_evidence": [],
"automatic_retry": False,
}
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
BootstrapFailureExecutor,
)
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
_wait_phase(session, "failed")
failed_thread = session._thread # noqa: SLF001
assert failed_thread is not None
failed_thread.join(timeout=2.0)
assert not failed_thread.is_alive()
failed = session.snapshot()
assert failed["outcome_unknown"] is False
assert failed["can_open"] is True
assert failed["automatic_retry"] is False
assert failed["failure"] == {
"code": "RuntimeError",
"reason_code": "unexpected_runtime_error",
"message": "bootstrap correlation failed",
"failed_phase": "connecting",
"dialogue_stage": "connection",
"transport_state": "ready",
"publish_attempts": 0,
"qos2_completions": 0,
"correlated_responses": 0,
"ignored_known_responses": 0,
"late_known_responses": 0,
"modeling_command_attempted": False,
"diagnostic_snapshot_unavailable": [],
"diagnostic_evidence_unavailable": [],
"correlation_failure": None,
"compatibility_failure": None,
"safe_to_retry": True,
}
assert "reason_code=unexpected_runtime_error" in caplog.text
def test_correlated_read_only_profile_mismatch_allows_only_a_fresh_explicit_attempt(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class ProfileMismatchError(RuntimeError):
reason_code = "compatibility_profile_mismatch"
@dataclass
class CorrelatedReadSnapshot:
state: str
def as_dict(self) -> dict[str, object]:
return {
"state": self.state,
"publish_attempts": 1,
"qos2_completions": 1,
"correlated_responses": 1,
"ignored_known_responses": 0,
"late_known_responses": 0,
"automatic_retry": False,
"automatic_reconnect": False,
}
class CorrelatedReadTransport(FakeTransport):
def snapshot(self) -> CorrelatedReadSnapshot:
return CorrelatedReadSnapshot(self.state)
class ProfileMismatchExecutor(FakeExecutor):
def run_connection_stage(
self,
_orchestrator: object,
) -> LiveDeviceControlBinding:
raise ProfileMismatchError("live DeviceInfo profile mismatch")
def snapshot(self) -> dict[str, object]:
return {
"dialogue_stage": "new",
"start_attempted": False,
"stop_attempted": False,
"response_evidence": [],
"correlation_failure": None,
"compatibility_failure": {
"operation_key": "bootstrap:1:DeviceInfoRequest",
"reason_code": "compatibility_profile_mismatch",
},
"automatic_retry": False,
}
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
ProfileMismatchExecutor,
)
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=lambda host: CorrelatedReadTransport(host), # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
failed_thread = session._thread # noqa: SLF001
assert failed_thread is not None
failed_thread.join(timeout=2.0)
assert not failed_thread.is_alive()
failed = session.snapshot()
assert failed["state"] == "failed"
assert failed["outcome_unknown"] is False
assert failed["automatic_retry"] is False
assert failed["can_open"] is True
failure = failed["failure"]
assert isinstance(failure, dict)
assert failure["reason_code"] == "compatibility_profile_mismatch"
assert failure["modeling_command_attempted"] is False
assert failure["safe_to_retry"] is True
def test_new_explicit_session_waits_for_old_worker_transport_retirement(
monkeypatch: pytest.MonkeyPatch,
) -> None:
old_close_entered = threading.Event()
release_old_close = threading.Event()
transports: list[TaggedTransport] = []
@dataclass
class TaggedTransportSnapshot:
state: str
ready: bool
tag: str
def as_dict(self) -> dict[str, object]:
return {
"state": self.state,
"tag": self.tag,
"publish_attempts": 0,
"qos2_completions": 0,
"correlated_responses": 0,
"ignored_known_responses": 0,
"late_known_responses": 0,
"latest_device_session_state": "ready" if self.ready else "scanning",
"latest_device_project_bound": not self.ready,
"automatic_retry": False,
"automatic_reconnect": False,
}
class TaggedTransport(FakeTransport):
def __init__(self, host: str, tag: str) -> None:
super().__init__(host)
self.tag = tag
self.close_calls = 0
def close(self) -> None:
if self.tag == "generation-1":
old_close_entered.set()
assert release_old_close.wait(timeout=2.0)
self.close_calls += 1
super().close()
def snapshot(self) -> TaggedTransportSnapshot:
return TaggedTransportSnapshot(self.state, self.ready, self.tag)
class RacingExecutor(FakeExecutor):
instances = 0
def __init__(self, transport: TaggedTransport) -> None:
super().__init__(transport)
type(self).instances += 1
self.instance = type(self).instances
def run_connection_stage(
self,
orchestrator: object,
) -> LiveDeviceControlBinding:
if self.instance == 1:
raise RuntimeError("first generation failed before publish")
return super().run_connection_stage(orchestrator)
def transport_factory(host: str) -> TaggedTransport:
transport = TaggedTransport(host, f"generation-{len(transports) + 1}")
transports.append(transport)
return transport
FakeExecutor.records = []
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
RacingExecutor,
)
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=transport_factory, # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
_wait_phase(session, "failed")
assert old_close_entered.wait(timeout=2.0)
old_thread = session._thread # noqa: SLF001
assert old_thread is not None
# The failure is safe to retry at the protocol level, but the old worker
# still owns its socket. A new explicit click must not construct or open a
# second transport until close() completes and the worker exits.
retiring = session.snapshot()
assert retiring["failure"]["safe_to_retry"] is True # type: ignore[index]
assert retiring["can_open"] is False
with pytest.raises(
session_module.ApplicationAcceptanceError,
match="already open or requires manual recovery",
):
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
assert len(transports) == 1
assert transports[0].close_calls == 0
release_old_close.set()
old_thread.join(timeout=2.0)
assert not old_thread.is_alive()
retired = session.snapshot()
assert retired["state"] == "failed"
assert retired["can_open"] is True
assert transports[0].close_calls == 1
# Only a fresh explicit action after retirement creates generation 2.
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
new_thread = session._thread # noqa: SLF001
assert new_thread is not None
current = _wait_phase(session, "connection-ready")
assert current["transport"]["tag"] == "generation-2" # type: ignore[index]
assert session._transport is transports[1] # noqa: SLF001
assert transports[1].close_calls == 0
# Complete the second synthetic session so no background waiter remains.
session.enter_workspace()
_wait_phase(session, "workspace-ready")
session.open_project_prompt()
_wait_phase(session, "project-ready")
session.request_start(project_name="TEST002", confirmation=_confirmation())
_wait_phase(session, "scanning")
session.request_stop(confirmation=_confirmation())
_wait_phase(session, "completed")
new_thread.join(timeout=2.0)
assert not new_thread.is_alive()
assert transports[1].close_calls == 1
def test_start_outcome_unknown_blocks_reopen_even_when_transport_is_closed(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class StartFailureExecutor(FakeExecutor):
start_attempted = False
def execute_canonical_start(self, *_args: object, **_kwargs: object) -> object:
type(self).start_attempted = True
raise session_module.ApplicationCommandOutcomeUnknown(
"START response was not correlated"
)
def snapshot(self) -> dict[str, object]:
return {
"dialogue_stage": "start-attempted",
"start_attempted": type(self).start_attempted,
"stop_attempted": False,
"response_evidence": [],
"automatic_retry": False,
}
FakeExecutor.records = []
StartFailureExecutor.start_attempted = False
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
StartFailureExecutor,
)
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
_wait_phase(session, "connection-ready")
session.enter_workspace()
_wait_phase(session, "workspace-ready")
session.open_project_prompt()
_wait_phase(session, "project-ready")
session.request_start(project_name="TEST001", confirmation=_confirmation())
failed = _wait_phase(session, "failed")
assert failed["outcome_unknown"] is True
assert failed["can_open"] is False
assert failed["failure"]["modeling_command_attempted"] is True # type: ignore[index]
assert failed["failure"]["safe_to_retry"] is False # type: ignore[index]
def test_unavailable_transport_snapshot_after_publish_blocks_reopen(
monkeypatch: pytest.MonkeyPatch,
) -> None:
executor_entered = threading.Event()
release_failure = threading.Event()
class SnapshotFailureTransport(FakeTransport):
def __init__(self, host: str) -> None:
super().__init__(host)
self.simulated_publish_attempts = 0
self.snapshot_unavailable = False
def snapshot(self) -> FakeTransportSnapshot:
if self.snapshot_unavailable:
raise RuntimeError("simulated transport snapshot failure")
return super().snapshot()
class PublishedThenFailedExecutor(FakeExecutor):
def run_connection_stage(
self,
_orchestrator: object,
) -> LiveDeviceControlBinding:
executor_entered.set()
assert release_failure.wait(timeout=2.0)
assert isinstance(self.transport, SnapshotFailureTransport)
self.transport.simulated_publish_attempts = 1
self.transport.snapshot_unavailable = True
raise RuntimeError("bootstrap failed after simulated publish")
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
PublishedThenFailedExecutor,
)
transport = SnapshotFailureTransport("192.168.1.20")
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=lambda _host: transport, # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
assert executor_entered.wait(timeout=2.0)
failed_thread = session._thread # noqa: SLF001
assert failed_thread is not None
release_failure.set()
failed_thread.join(timeout=2.0)
assert not failed_thread.is_alive()
failed = session.snapshot()
assert transport.simulated_publish_attempts == 1
assert failed["state"] == "failed"
assert failed["outcome_unknown"] is True
assert failed["can_open"] is False
failure = failed["failure"]
assert isinstance(failure, dict)
assert failure["transport_state"] is None
assert failure["publish_attempts"] is None
assert failure["diagnostic_snapshot_unavailable"] == ["transport"]
assert failure["diagnostic_evidence_unavailable"] == ["transport.publish_attempts"]
assert failure["modeling_command_attempted"] is False
assert failure["safe_to_retry"] is False
with pytest.raises(
session_module.ApplicationAcceptanceError,
match="already open or requires manual recovery",
):
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
def test_unavailable_dialogue_snapshot_is_outcome_unknown(
monkeypatch: pytest.MonkeyPatch,
) -> None:
class DialogueSnapshotFailureExecutor(FakeExecutor):
def run_connection_stage(
self,
_orchestrator: object,
) -> LiveDeviceControlBinding:
raise RuntimeError("bootstrap failed with unavailable dialogue evidence")
def snapshot(self) -> dict[str, object]:
raise RuntimeError("simulated dialogue snapshot failure")
monkeypatch.setattr(
session_module,
"PhysicalAcceptanceDialogueExecutor",
DialogueSnapshotFailureExecutor,
)
session = InteractiveApplicationControlSession(
FakeAuthorityLoader(),
transport_factory=lambda host: FakeTransport(host), # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
failed_thread = session._thread # noqa: SLF001
assert failed_thread is not None
failed_thread.join(timeout=2.0)
assert not failed_thread.is_alive()
failed = session.snapshot()
assert failed["state"] == "failed"
assert failed["outcome_unknown"] is True
assert failed["can_open"] is False
failure = failed["failure"]
assert isinstance(failure, dict)
assert failure["publish_attempts"] == 0
assert failure["modeling_command_attempted"] is None
assert failure["diagnostic_snapshot_unavailable"] == ["dialogue"]
assert failure["diagnostic_evidence_unavailable"] == ["dialogue.modeling_command_attempted"]
assert failure["safe_to_retry"] is False
def test_pretransport_authority_failure_remains_safe_after_worker_retirement() -> None:
class FailingAuthorityLoader(FakeAuthorityLoader):
def load(self) -> ApplicationControlAuthority:
self.calls += 1
raise RuntimeError("authority unavailable before transport creation")
transport_factory_calls = 0
def transport_factory(host: str) -> FakeTransport:
nonlocal transport_factory_calls
transport_factory_calls += 1
return FakeTransport(host)
session = InteractiveApplicationControlSession(
FailingAuthorityLoader(),
transport_factory=transport_factory, # type: ignore[arg-type]
)
session.open(
host="192.168.1.20",
timezone_name="Europe/Moscow",
confirmation=_confirmation(),
)
failed_thread = session._thread # noqa: SLF001
assert failed_thread is not None
failed_thread.join(timeout=2.0)
assert not failed_thread.is_alive()
failed = session.snapshot()
assert transport_factory_calls == 0
assert failed["state"] == "failed"
assert failed["outcome_unknown"] is False
assert failed["can_open"] is True
failure = failed["failure"]
assert isinstance(failure, dict)
assert failure["publish_attempts"] is None
assert failure["modeling_command_attempted"] is False
assert failure["diagnostic_snapshot_unavailable"] == []
assert failure["diagnostic_evidence_unavailable"] == []
assert failure["safe_to_retry"] is True