SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): stabilize live recovery and media admission

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 00:03:01 +03:00
parent 7217244886
commit eaad9deda1
29 changed files with 1645 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_xgrids_acquisition_lifecycle.py
+480 -24
View File
@@ -5662,7 +5662,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
original_retire = service._retire_application_control_for_network_change # noqa: SLF001
original_apply = service._apply_read_only_device_topology # noqa: SLF001
def retire_with_competing_scan() -> None:
def retire_with_competing_scan(*, allow_terminal_failure: bool = False) -> None:
assert service._provisioning_active is True # noqa: SLF001
scan_outcomes.append(
_attempt_competing_scan_from_sync_boundary(
@@ -5670,7 +5670,7 @@ def test_read_only_verify_scan_is_rejected_after_atomic_fence_before_admission(
operation_id="op-00000000-0000-4000-8000-000000000102",
)
)
original_retire()
original_retire(allow_terminal_failure=allow_terminal_failure)
def apply_with_fence_assertion(**kwargs: Any) -> str:
nonlocal admission_calls
@@ -7692,6 +7692,58 @@ def test_terminal_prepared_start_without_publish_settles_and_stops_local_capture
assert service._acquisition_session_lease is None # noqa: SLF001
def test_live_prepared_start_reduction_does_not_touch_publish_fence(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
service, _ = service_with_fake_runtime(tmp_path)
acquisition_id = "acq-live-prepared"
operation_id = "op-live-prepared"
monkeypatch.setattr(
service._physical_command_coordinator, # noqa: SLF001
"snapshot",
lambda: {
"status": "unresolved",
"record": {
"operation_id": operation_id,
"acquisition_id": acquisition_id,
"action": "start",
"stage": "prepared",
"resolution": None,
"publish_call_returned": None,
"packet_id": None,
"qos2_completed": False,
"application_response": None,
},
},
)
monkeypatch.setattr(
service._application_control_session, # noqa: SLF001
"snapshot",
lambda: {"state": "initializing", "failure": None},
)
class PublishFenceMustStayFree:
def acquire(self, *args: object, **kwargs: object) -> bool:
raise AssertionError("live START reduction touched the publish fence")
def release(self) -> None:
raise AssertionError("live START reduction released an unowned fence")
monkeypatch.setattr(
service,
"_k1_command_dispatch_gate",
PublishFenceMustStayFree(),
)
outcome = service._settle_prepared_start_worker_failure( # noqa: SLF001
acquisition_id=acquisition_id,
start_operation_id=operation_id,
)
assert outcome == "not-applicable"
def _activate_real_checkpoint_for_prepared_stop_fixture(
service: XgridsK1CompatibilityService,
*,
@@ -12273,6 +12325,7 @@ def _stop_response_without_terminal_status_fixture(
tmp_path: Path,
*,
qos2_completed: bool = True,
application_response: bool = True,
) -> SimpleNamespace:
clock_value = [datetime(2026, 8, 10, 8, 10, tzinfo=UTC)]
service, runtime = service_with_fake_runtime(tmp_path)
@@ -12424,20 +12477,21 @@ def _stop_response_without_terminal_status_fixture(
packet_id=82,
)
ledger.mark_qos2_completed(stop_operation_id, packet_id=82)
ledger.record_application_response(
stop_operation_id,
PhysicalCommandApplicationResponse(
operation_id=stop_operation_id,
action="stop",
control_session_id=physical_connection.control_session_id,
host_path_epoch=physical_connection.host_path_epoch,
producer_generation=physical_connection.producer_generation,
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
success=True,
payload_sha256="a" * 64,
observed_at_utc="2026-08-10T08:10:02.000Z",
),
)
if application_response:
ledger.record_application_response(
stop_operation_id,
PhysicalCommandApplicationResponse(
operation_id=stop_operation_id,
action="stop",
control_session_id=physical_connection.control_session_id,
host_path_epoch=physical_connection.host_path_epoch,
producer_generation=physical_connection.producer_generation,
result_code=PHYSICAL_COMMAND_APPLICATION_SUCCESS_CODE,
success=True,
payload_sha256="a" * 64,
observed_at_utc="2026-08-10T08:10:02.000Z",
),
)
return SimpleNamespace(
service=service,
runtime=runtime,
@@ -12450,6 +12504,60 @@ def _stop_response_without_terminal_status_fixture(
)
def test_unknown_stop_retires_only_terminal_control_and_keeps_live_capture(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
fixture = _stop_response_without_terminal_status_fixture(
tmp_path,
qos2_completed=False,
application_response=False,
)
service = fixture.service
control = fixture.control
close_calls = 0
worker_retired = False
original_snapshot = control.snapshot
original_close = control.close
def terminal_snapshot() -> dict[str, object]:
snapshot = original_snapshot()
if control.state == "failed" and not worker_retired:
snapshot["can_open"] = False
return snapshot
def retire_failed_worker() -> None:
nonlocal close_calls, worker_retired
close_calls += 1
worker_retired = True
original_close()
monkeypatch.setattr(control, "snapshot", terminal_snapshot)
monkeypatch.setattr(control, "close", retire_failed_worker)
control.state = "failed"
control.outcome_unknown = True
control.failure = {
"code": "ApplicationCommandOutcomeUnknown",
"reason_code": "mqtt_response_timeout",
"stop_command_attempted": True,
"stop_publish_attempts": 1,
"safe_to_retry": False,
}
service._acquire_application_control_process_lease() # noqa: SLF001
recovered = service.state()
assert close_calls == 1
assert recovered["application_control_session"]["state"] == "idle"
assert recovered["acquisition"]["state"] == "awaiting_external_stop"
assert recovered["source_mode"] == "live"
assert recovered["physical_command"]["status"] == "unresolved"
assert recovered["physical_command"]["requires_reconciliation"] is True
assert fixture.runtime.stop_calls == 0
assert control.stop_calls == 1
assert service._application_control_process_lease_holders == set() # noqa: SLF001
@pytest.mark.parametrize("qos2_completed", [True, False])
def test_stop_success_without_terminal_status_times_out_into_local_only_recovery(
tmp_path: Path,
@@ -13729,7 +13837,18 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
binding=binding,
resolved=True,
)
monkeypatch.setattr(service._physical_command_coordinator, "snapshot", lambda: physical) # noqa: SLF001
physical_snapshot_calls = 0
def snapshot_physical() -> dict[str, object]:
nonlocal physical_snapshot_calls
physical_snapshot_calls += 1
return physical
monkeypatch.setattr(
service._physical_command_coordinator, # noqa: SLF001
"snapshot",
snapshot_physical,
)
runtime.pcl_frames = 1
frame = DecodedPointCloudView(
context=ConsumerFrameContext(
@@ -13818,9 +13937,11 @@ def test_confirmed_scanning_waits_for_first_authoritative_pcl_before_camera(
)
# Every later authoritative PCL for the same lineage is idempotent.
physical_snapshot_calls_before = physical_snapshot_calls
service._observe_published_runtime_envelope(frame, runtime.producer_generation) # noqa: SLF001
time.sleep(0.05)
assert len(events) == 2
assert physical_snapshot_calls == physical_snapshot_calls_before
def test_stale_post_publish_pcl_cannot_activate_camera(
@@ -15316,6 +15437,85 @@ def test_active_stream_recovery_scanning_resumes_same_physical_lineage_without_c
)
def test_active_stream_recovery_privacy_bridge_retains_exact_route_for_read_only_rebind(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
service, runtime = service_with_fake_runtime(tmp_path)
control, physical = _install_composite_active_recovery_fixture(
service,
runtime,
monkeypatch,
)
service.state()
lineage = service._active_stream_recovery_lineage # noqa: SLF001
assert lineage is not None
supervisor = service._connection_supervisor # noqa: SLF001
baseline = supervisor.snapshot()
candidate = replace(
baseline,
device_identity=replace(baseline.device_identity, state="unverified"),
control_plane=replace(
baseline.control_plane,
state="lost",
session_id=None,
),
lease=replace(baseline.lease, state="configured-unverified"),
authority=replace(
baseline.authority,
control_allowed=False,
acquisition_start_allowed=False,
),
)
start_projects_before = list(control.start_projects)
stop_calls_before = control.stop_calls
operation_journal_before = service._operations.snapshot() # noqa: SLF001
physical_ledger_before = service._physical_command_ledger.snapshot() # noqa: SLF001
network_ledger_before = service._network_mutation_ledger.snapshot() # noqa: SLF001
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
monkeypatch.setattr(
service._host_wifi_association_probe, # noqa: SLF001
"observe",
lambda *_args, **_kwargs: {
"schema_version": 1,
"adapter": "CoreWLAN",
"wifi_interface": True,
"association_state": "unavailable",
"evidence_quality": "unavailable",
"continuity_proven": False,
"continuity_token": "d" * 64,
"reason_code": "association-identity-unavailable",
},
)
monkeypatch.setattr(
supervisor,
"association_timeout_retention_candidate",
lambda *, expected_target: (
candidate
if expected_target
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
else None
),
)
sampled = service._sample_host_path(lineage.target_ipv4) # noqa: SLF001
assert sampled.available is True
assert sampled.reason_code is None
assert sampled.fingerprint == baseline.host_path.fingerprint
assert sampled.kernel_route_fingerprint == (
baseline.host_path.kernel_route_fingerprint
)
assert control.start_projects == start_projects_before
assert control.stop_calls == stop_calls_before
assert service._operations.snapshot() == operation_journal_before # noqa: SLF001
assert ( # noqa: SLF001
service._physical_command_ledger.snapshot() == physical_ledger_before
)
assert service._network_mutation_ledger.snapshot() == network_ledger_before # noqa: SLF001
assert service._physical_command_coordinator.snapshot() is physical # noqa: SLF001
def test_active_stream_control_adoption_gets_fresh_budget_after_slow_proof(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -15638,7 +15838,7 @@ def test_active_stream_recovery_wrong_identity_blocks_without_retry_or_camera_re
monkeypatch.setattr(
service,
"_probe_control_endpoint",
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
path=_direct_host_path(lineage.target_ipv4),
reachable=True,
reason_code=None,
@@ -15699,8 +15899,14 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
events.append(("monitor", True))
return True
def probe_exact_target(target_ipv4: str) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
events.append(("probe", target_ipv4))
def probe_exact_target( # noqa: SLF001
target_ipv4: str,
*,
association_timeout_seconds: float,
) -> facade_module._CorrelatedEndpointObservation:
events.append(
("probe", (target_ipv4, association_timeout_seconds))
)
return facade_module._CorrelatedEndpointObservation( # noqa: SLF001
path=_direct_host_path(target_ipv4),
reachable=True,
@@ -15770,7 +15976,13 @@ def test_active_stream_recovery_owned_path_is_read_only_and_resumes_exact_lineag
assert events == [
("monitor", True),
("lease-acquire", "network"),
("probe", lineage.target_ipv4),
(
"probe",
(
lineage.target_ipv4,
facade_module.ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS,
),
),
(
"device-info-status",
(
@@ -15910,6 +16122,7 @@ def test_incident_recovery_retires_epoch_one_control_and_retries_fresh_inspectio
def probe_current_epoch(
target_ipv4: str,
**_kwargs: object,
) -> facade_module._CorrelatedEndpointObservation: # noqa: SLF001
path = next(probe_paths)
service._observe_connection_transport( # noqa: SLF001
@@ -16065,7 +16278,7 @@ def test_active_stream_recovery_waits_for_terminal_control_worker_retirement(
monkeypatch.setattr(
service,
"_probe_control_endpoint",
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
path=_direct_host_path(lineage.target_ipv4),
reachable=True,
reason_code=None,
@@ -16140,7 +16353,7 @@ def test_active_stream_recovery_exact_device_system_fault_is_terminal_without_co
monkeypatch.setattr(
service,
"_probe_control_endpoint",
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
path=_direct_host_path(lineage.target_ipv4),
reachable=True,
reason_code=None,
@@ -16223,7 +16436,7 @@ def test_active_stream_recovery_bootstrap_system_error_is_normalized_terminal_fa
monkeypatch.setattr(
service,
"_probe_control_endpoint",
lambda _target: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
lambda _target, **_kwargs: facade_module._CorrelatedEndpointObservation( # noqa: SLF001
path=_direct_host_path(lineage.target_ipv4),
reachable=True,
reason_code=None,
@@ -16476,6 +16689,111 @@ def test_active_stream_recovery_never_retries_invalid_fmp4_after_media_commit(
assert control.stop_calls == 0
def test_bound_scan_over_without_stop_dominates_late_empty_camera_epoch_failure(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
service, runtime = service_with_fake_runtime(tmp_path)
control, physical = _install_composite_active_recovery_fixture(
service,
runtime,
monkeypatch,
runtime_phase="live",
camera_phase="streaming",
)
camera = service.camera_preview.snapshot()
recording = camera["recording"]
assert isinstance(recording, dict)
recording.update(
{
"active": True,
"active_epoch": 2,
"committed_media_segment_count": 4_833,
"producer_alive": False,
"completed_epochs": 2,
"last_summary": {
"codec_epoch": 2,
"status": "failed",
"media_segment_count": 0,
"failure_code": "invalid-fmp4",
},
}
)
camera.update(
{
"phase": "error",
"error": {
"code": "invalid-fmp4",
"message": "synthetic empty post-power-loss epoch",
},
}
)
physical.update(
{
"runtime_bound": True,
"reconciliation_ready": True,
"observed_session_state": "scan_over",
}
)
control.state = "failed"
control.state_revision += 1
control.failure = {
"reason_code": "application_acceptance_failed",
"failed_phase": "scanning",
"modeling_command_attempted": True,
"stop_command_attempted": False,
"diagnostic_snapshot_unavailable": [],
"diagnostic_evidence_unavailable": [],
"safe_to_retry": False,
}
def scan_over_control_snapshot() -> dict[str, object]:
snapshot = FakeInteractiveControlSession.snapshot(control)
snapshot["transport"] = {
"state": "failed",
"publish_attempts": 7,
"device_status_reports": 9,
"latest_device_session_state": "scan_over",
"latest_device_project_bound": True,
"latest_device_init_ready": False,
"latest_system_error_code": None,
"automatic_retry": False,
"automatic_reconnect": False,
}
return snapshot
monkeypatch.setattr(control, "snapshot", scan_over_control_snapshot)
monkeypatch.setattr(service, "_seal_acquisition_capture_clock", lambda: None)
camera_stop_calls: list[str] = []
monkeypatch.setattr(
service.camera_preview,
"stop_current",
lambda: camera_stop_calls.append("stop") or {},
)
start_projects_before = list(control.start_projects)
stop_calls_before = control.stop_calls
terminal = service.state()
assert terminal["acquisition"]["state"] == "interrupted"
assert terminal["acquisition"]["message_code"] == (
"acquisition.recovery.device_standby_observed"
)
assert terminal["acquisition"]["result"] == {
"receiver_stopped": True,
"device_state": "scan_over",
"device_stop": "not-sent",
"automatic_command_retry": False,
"read_only_recovery": True,
"physical_reconciliation_required": True,
}
assert terminal["acquisition"]["cleanup_pending"] is False
assert camera_stop_calls == ["stop"]
assert runtime.stop_calls == 1
assert control.start_projects == start_projects_before
assert control.stop_calls == stop_calls_before == 0
def test_active_stream_recovery_reopens_exact_camera_epoch_once_and_fences_stale_lineage(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -24955,6 +25273,59 @@ def test_configured_unverified_monitor_keeps_same_route_without_promoting_author
assert "verify-control-device-info" in retained.allowed_actions
def test_monitor_layer_retains_exact_configured_route_across_association_lock_timeout(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
service, _ = service_with_fake_runtime(tmp_path)
binding = _seed_supervised_connection(service, with_control=False)
baseline = service._connection_supervisor.snapshot() # noqa: SLF001
assert baseline.host_path.kernel_route_fingerprint is not None
tcp_calls: list[str] = []
monkeypatch.setattr(
service,
"_sample_host_path",
lambda target, **_kwargs: HostPathProbeResult(
available=False,
fingerprint=None,
interface=baseline.host_path.interface,
source_ipv4=baseline.host_path.source_ipv4,
route_class="unavailable",
reason_code="host-wifi-operation-timeout",
observation_failure_class="association-observer",
kernel_route_fingerprint=baseline.host_path.kernel_route_fingerprint,
),
)
monkeypatch.setattr(
facade_module,
"_probe_control_endpoint_socket",
lambda target: (
tcp_calls.append(target)
or facade_module.TcpReachabilityProbeResult(reachable=True)
),
)
async def poll_three_times() -> list[facade_module.ConnectionSupervisorSnapshot]:
return [
await service._connection_monitor.poll_once() # noqa: SLF001
for _ in range(3)
]
snapshots = asyncio.run(poll_three_times())
assert tcp_calls == [binding.target_ipv4] * 3
for retained in snapshots:
assert retained.host_path.available is True
assert retained.host_path.epoch == baseline.host_path.epoch
assert retained.host_path.fingerprint == baseline.host_path.fingerprint
assert retained.endpoint.tcp_state == "reachable"
assert retained.device_identity.state == "unverified"
assert retained.authority.control_allowed is False
assert retained.authority.acquisition_start_allowed is False
assert "verify-control-device-info" in retained.allowed_actions
def test_association_timeout_cannot_hide_a_real_kernel_route_change(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -29825,6 +30196,91 @@ def test_explicit_verify_reconciles_persisted_start_to_ready_without_device_io(
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
def test_long_first_verify_refreshes_aged_transport_before_exposing_prestart_control(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
"""A long checkpoint reconciliation cannot orphan its new ready socket."""
service, _ = service_with_fake_runtime(tmp_path)
coordinator = _VerifyPhysicalRecoveryCoordinator(
observed_session_state="ready",
reconciliation_ready=True,
)
_install_synthetic_verify_recovery(service, coordinator)
supervisor = service._connection_supervisor # noqa: SLF001
monotonic_now = [100.0]
suspend_aware_now = [1_000.0]
supervisor._monotonic_clock = lambda: monotonic_now[0] # noqa: SLF001
supervisor._suspend_aware_clock = lambda: suspend_aware_now[0] # noqa: SLF001
before = supervisor.snapshot()
assert before.intent is not None
assert before.device_network.target is not None
host_epoch = supervisor.observe_host_path(
_association_bound_direct_host_path(before.device_network.target.ipv4)
)
assert supervisor.observe_endpoint(
target=before.device_network.target,
intent_id=before.intent.intent_id,
host_path_epoch=host_epoch,
reachable=True,
)
monkeypatch.setattr(facade_module, "_inspect_host_path", _direct_host_path)
tcp_probes: list[str] = []
def reachable(target: str) -> bool:
tcp_probes.append(target)
return True
monkeypatch.setattr(facade_module, "_control_endpoint_reachable", reachable)
reconcile = service._reconcile_physical_command_after_verify_owned # noqa: SLF001
async def reconcile_after_transport_ages(
bound_service: XgridsK1CompatibilityService,
*,
verify_operation_id: str,
allow_receiver_rehydrate: bool = True,
) -> dict[str, Any]:
result = await reconcile(
verify_operation_id=verify_operation_id,
allow_receiver_rehydrate=allow_receiver_rehydrate,
)
monotonic_now[0] += 20.0
suspend_aware_now[0] += 20.0
return result
service._reconcile_physical_command_after_verify_owned = MethodType( # type: ignore[method-assign] # noqa: SLF001
reconcile_after_transport_ages,
service,
)
operation_id = "op-00000000-0000-4000-8000-000000001202"
verified = asyncio.run(
service.verify_connection(
_retained_physical_recovery_verify_request(operation_id=operation_id)
)
)
assert tcp_probes == ["192.168.1.20"]
assert verified["last_operation"]["status"] == "succeeded"
assert verified["active_connection_mode"] == "bridge"
assert verified["connection_policy"]["actions"]["start-acquisition"]["allowed"] is True
assert service._application_control_session.snapshot()["state"] == "connection-ready" # noqa: SLF001
assert service._application_control_process_lease_holders == {"control"} # noqa: SLF001
fresh = supervisor.snapshot()
assert fresh.intent is not None
assert fresh.device_network.target is not None
assert supervisor.endpoint_observation_has_remaining_lease(
target=fresh.device_network.target,
intent_id=fresh.intent.intent_id,
host_path_epoch=fresh.host_path.epoch,
minimum_remaining_seconds=(
facade_module.VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
),
)
def test_explicit_verify_reconciliation_does_not_require_normal_command_authority(
tmp_path: Path,
) -> None: