fix(k1): stabilize live recovery and media admission
This commit is contained in:
DCCONSTRUCTIONS
@@ -1868,9 +1868,10 @@ class ReadOnlyConnectionMonitor:
|
||||
return None
|
||||
previous_path = baseline.host_path
|
||||
intent = baseline.intent
|
||||
exact_verified_control = bool(
|
||||
intent is not None
|
||||
and previous_path.available
|
||||
if intent is None:
|
||||
return None
|
||||
exact_reachable_route = bool(
|
||||
previous_path.available
|
||||
and previous_path.fingerprint is not None
|
||||
and previous_path.kernel_route_fingerprint is not None
|
||||
and baseline.device_network.state == "applied"
|
||||
@@ -1880,6 +1881,9 @@ class ReadOnlyConnectionMonitor:
|
||||
and baseline.endpoint.intent_id == intent.intent_id
|
||||
and baseline.endpoint.host_path_epoch == previous_path.epoch
|
||||
and baseline.endpoint.tcp_state == "reachable"
|
||||
)
|
||||
exact_verified_control = bool(
|
||||
exact_reachable_route
|
||||
and baseline.device_identity.state == "verified"
|
||||
and baseline.device_identity.intent_id == intent.intent_id
|
||||
and baseline.device_identity.host_path_epoch == previous_path.epoch
|
||||
@@ -1892,7 +1896,11 @@ class ReadOnlyConnectionMonitor:
|
||||
and baseline.lease.target == target
|
||||
and baseline.authority.control_allowed
|
||||
)
|
||||
if not exact_verified_control:
|
||||
technical_timeout_on_exact_route = bool(
|
||||
result.reason_code == "host-wifi-operation-timeout"
|
||||
and exact_reachable_route
|
||||
)
|
||||
if not (exact_verified_control or technical_timeout_on_exact_route):
|
||||
return None
|
||||
if (
|
||||
result.kernel_route_fingerprint is None
|
||||
|
||||
@@ -278,6 +278,13 @@ HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS = 2.0
|
||||
# without weakening the sub-second association cache or skipping the final
|
||||
# route/association continuity recheck.
|
||||
CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||
# Active-stream recovery has already frozen an exact acquisition/START owner.
|
||||
# Its endpoint probe may therefore use the same short read-only association
|
||||
# budget as the monitor: the exact raw route bridge above preserves continuity
|
||||
# across a technical CoreWLAN timeout, while fresh DeviceInfo still has to
|
||||
# verify the K1 before MQTT publication resumes. Do not inherit the generic
|
||||
# 30-second helper budget on every Wi-Fi return.
|
||||
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS = 3.0
|
||||
# Command checkpoints must not inherit either the monitor's three-second
|
||||
# contact budget or the host helper's generic 30-second budget. One command
|
||||
# may legitimately wait about three seconds for the service-lifetime
|
||||
@@ -303,6 +310,9 @@ CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS = (
|
||||
2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS
|
||||
)
|
||||
CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0
|
||||
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS = (
|
||||
CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS + 1.0
|
||||
)
|
||||
CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"})
|
||||
LIVE_DATA_PLANE_STALL_SECONDS = 5.0
|
||||
LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0
|
||||
@@ -2557,8 +2567,11 @@ class XgridsK1CompatibilityService:
|
||||
A privacy-limited observation has the stricter gate below: exact
|
||||
DeviceInfo, healthy control session, reachable lease and command
|
||||
authority must all still match the same intent/mode/target/epoch. The
|
||||
legacy timeout bridge may retain only its exact reachable route but
|
||||
cannot manufacture any missing command authority.
|
||||
sole exception is an already-admitted active-stream recovery lineage:
|
||||
it may retain only this exact reachable route long enough to perform a
|
||||
fresh read-only DeviceInfo/MQTT rebind. That route bridge grants no
|
||||
control authority and cannot send START, STOP or a network mutation.
|
||||
The legacy timeout bridge has the same route-only limitation.
|
||||
|
||||
A proven association change, a different route/interface/source, a
|
||||
stale or unreachable endpoint, or a different intent/target bypasses
|
||||
@@ -2623,7 +2636,13 @@ class XgridsK1CompatibilityService:
|
||||
and snapshot.lease.target == endpoint_target
|
||||
and snapshot.authority.control_allowed
|
||||
)
|
||||
if not exact_verified_control:
|
||||
exact_active_stream_rebind = (
|
||||
self._exact_active_stream_recovery_route_rebind(
|
||||
snapshot=snapshot,
|
||||
endpoint_target=endpoint_target,
|
||||
)
|
||||
)
|
||||
if not (exact_verified_control or exact_active_stream_rebind):
|
||||
return None
|
||||
return HostPathProbeResult(
|
||||
available=True,
|
||||
@@ -2640,6 +2659,68 @@ class XgridsK1CompatibilityService:
|
||||
kernel_route_fingerprint=path.fingerprint,
|
||||
)
|
||||
|
||||
def _exact_active_stream_recovery_route_rebind(
|
||||
self,
|
||||
*,
|
||||
snapshot: ConnectionSupervisorSnapshot,
|
||||
endpoint_target: EndpointTarget,
|
||||
) -> bool:
|
||||
"""Allow one exact route-only bridge for an admitted live recovery.
|
||||
|
||||
Admission of ``_ActiveStreamRecoveryLineage`` already proved the
|
||||
durable original START owner. This guard rechecks its process-local
|
||||
acquisition/runtime coordinates without consulting or refreshing any
|
||||
device authority. The subsequent recovery path must still establish
|
||||
a fresh DeviceInfo identity and MQTT control proof before it can
|
||||
resume publication.
|
||||
"""
|
||||
|
||||
intent = snapshot.intent
|
||||
with self._lock:
|
||||
lineage = self._active_stream_recovery_lineage
|
||||
acquisition = self._acquisition
|
||||
if lineage is None or acquisition is None:
|
||||
return False
|
||||
exact_local_lineage = bool(
|
||||
self._active_stream_recovery_state == "reconnecting"
|
||||
and self._active_stream_recovery_generation
|
||||
== lineage.recovery_generation
|
||||
and self._snapshot_runtime_id == lineage.snapshot_runtime_id
|
||||
and acquisition.acquisition_id == lineage.acquisition_id
|
||||
and acquisition.device_id == lineage.device_id
|
||||
and acquisition.device_session_id == lineage.device_session_id
|
||||
and acquisition.state == "acquiring"
|
||||
and self._acquisition_start_operation_id
|
||||
in {None, lineage.start_operation_id}
|
||||
and self._acquisition_session_lease is not None
|
||||
and self._acquisition_out_dir is not None
|
||||
and self._acquisition_out_dir.name == lineage.evidence_session_id
|
||||
and self._selected_device_id == lineage.transport_ref
|
||||
and self._device_id == lineage.device_id
|
||||
and self._device_session_id == lineage.device_session_id
|
||||
and self._connection_mode == lineage.connection_mode
|
||||
and self._k1_ip == lineage.target_ipv4
|
||||
)
|
||||
if not exact_local_lineage:
|
||||
return False
|
||||
runtime = self.runtime.snapshot()
|
||||
return bool(
|
||||
runtime.get("phase") == "reconnecting"
|
||||
and runtime.get("source_mode") == "live"
|
||||
and runtime.get("producer_generation")
|
||||
== lineage.runtime_producer_generation
|
||||
and intent is not None
|
||||
and intent.intent_id == lineage.intent_id
|
||||
and intent.requested_mode == lineage.connection_mode
|
||||
and snapshot.device_network.state == "applied"
|
||||
and snapshot.device_network.intent_id == lineage.intent_id
|
||||
and snapshot.device_network.transport_ref == lineage.transport_ref
|
||||
and snapshot.device_network.connection_mode == lineage.connection_mode
|
||||
and snapshot.device_network.target == endpoint_target
|
||||
and endpoint_target
|
||||
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
|
||||
)
|
||||
|
||||
async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult:
|
||||
if not self._connection_monitor_contact_gate.acquire(blocking=False):
|
||||
raise ConnectionMonitorProbeSuperseded(
|
||||
@@ -2715,6 +2796,57 @@ class XgridsK1CompatibilityService:
|
||||
)
|
||||
self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed")
|
||||
|
||||
async def _refresh_aged_verify_transport_owned(
|
||||
self,
|
||||
*,
|
||||
application_control_session: Mapping[str, Any],
|
||||
) -> None:
|
||||
"""Refresh only a nearly expired Verify transport with real I/O.
|
||||
|
||||
The caller owns the Verify lifecycle/network fences, so the background
|
||||
monitor is deliberately superseded. If DeviceInfo or physical
|
||||
reconciliation consumed most of the existing endpoint lease, perform
|
||||
the same route/TCP/route observation before releasing those fences.
|
||||
No cached timestamp, MQTT reconnect, BLE operation, or device command
|
||||
is manufactured here.
|
||||
"""
|
||||
|
||||
verified_binding = application_control_session.get("verified_control")
|
||||
if not isinstance(verified_binding, Mapping):
|
||||
raise ConnectionVerificationError(
|
||||
"Verify lost its exact control proof before transport refresh",
|
||||
reason_code="connection-verify-final-binding-superseded",
|
||||
)
|
||||
target = EndpointTarget(
|
||||
str(verified_binding["target_ipv4"]),
|
||||
int(verified_binding["target_port"]),
|
||||
)
|
||||
intent_id = str(verified_binding["intent_id"])
|
||||
host_path_epoch = int(verified_binding["host_path_epoch"])
|
||||
if self._connection_supervisor.endpoint_observation_has_remaining_lease(
|
||||
target=target,
|
||||
intent_id=intent_id,
|
||||
host_path_epoch=host_path_epoch,
|
||||
minimum_remaining_seconds=(
|
||||
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
|
||||
),
|
||||
):
|
||||
return
|
||||
observation = await _run_blocking_operation_without_abandonment(
|
||||
self._probe_control_endpoint,
|
||||
target.ipv4,
|
||||
association_timeout_seconds=(CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS),
|
||||
)
|
||||
if not observation.reachable:
|
||||
raise ConnectionVerificationError(
|
||||
"K1 endpoint не подтвердил связь после долгой read-only сверки",
|
||||
reason_code="connection-verify-mqtt-unreachable",
|
||||
)
|
||||
self._reconcile_connection_supervisor(
|
||||
application_control_session,
|
||||
self.runtime.snapshot(),
|
||||
)
|
||||
|
||||
def _observe_connection_transport(
|
||||
self,
|
||||
target: str,
|
||||
@@ -2723,9 +2855,17 @@ class XgridsK1CompatibilityService:
|
||||
reachable: bool | None,
|
||||
reason_code: str | None = None,
|
||||
) -> None:
|
||||
supervisor = self._connection_supervisor.snapshot()
|
||||
intent = supervisor.intent
|
||||
endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT)
|
||||
# A completed route sample is fresher than the silence fallback. Read
|
||||
# the exact applied target without expiring the previous observation
|
||||
# between sampling and publication; the CAS below still rejects an
|
||||
# intent/target change and the sample itself decides route continuity.
|
||||
supervisor = self._connection_supervisor.association_timeout_retention_candidate(
|
||||
expected_target=endpoint,
|
||||
)
|
||||
if supervisor is None:
|
||||
return
|
||||
intent = supervisor.intent
|
||||
if (
|
||||
intent is None
|
||||
or supervisor.device_network.state != "applied"
|
||||
@@ -2734,7 +2874,14 @@ class XgridsK1CompatibilityService:
|
||||
or supervisor.device_network.target != endpoint
|
||||
):
|
||||
return
|
||||
epoch = self._connection_supervisor.observe_host_path(path)
|
||||
epoch = self._connection_supervisor.observe_host_path_if_current(
|
||||
expected_intent_id=intent.intent_id,
|
||||
expected_target=endpoint,
|
||||
expected_host_path_observation=supervisor.host_path,
|
||||
result=path,
|
||||
)
|
||||
if epoch is None:
|
||||
return
|
||||
if path.available and reachable is not None:
|
||||
self._connection_supervisor.observe_endpoint(
|
||||
target=endpoint,
|
||||
@@ -3637,8 +3784,15 @@ class XgridsK1CompatibilityService:
|
||||
pending = self._pending_local_control_retirement
|
||||
if not pending:
|
||||
return True
|
||||
control = self._application_control_session.snapshot()
|
||||
try:
|
||||
self._retire_application_control_for_network_change()
|
||||
self._retire_application_control_for_network_change(
|
||||
# A failed worker may finish retiring after the first bounded
|
||||
# close attempt. Retrying that host-only retirement must not
|
||||
# remain permanently blocked by the terminal phase itself.
|
||||
# The durable physical-command row is intentionally untouched.
|
||||
allow_terminal_failure=control.get("state") == "failed",
|
||||
)
|
||||
except (AttributeError, RuntimeError):
|
||||
return False
|
||||
return True
|
||||
@@ -11616,6 +11770,21 @@ class XgridsK1CompatibilityService:
|
||||
and not reconciliation.observation.init_ready
|
||||
and not reconciliation.observation.mqtt_retained
|
||||
)
|
||||
ordinary_terminal_stop = bool(
|
||||
record.stage == "resolved"
|
||||
and record.resolution == "stop-standby-observed"
|
||||
and record.publish_call_returned is True
|
||||
and record.packet_id is not None
|
||||
and record.qos2_completed
|
||||
and record.application_response is not None
|
||||
and record.application_response.success
|
||||
and record.last_status is not None
|
||||
and record.last_status.source == "live-control-session"
|
||||
and record.last_status.session_state in {"ready", "scan_over"}
|
||||
and not record.last_status.project_bound
|
||||
and not record.last_status.init_ready
|
||||
and not record.last_status.mqtt_retained
|
||||
)
|
||||
if not (
|
||||
persisted is not None
|
||||
and status is not None
|
||||
@@ -11624,6 +11793,7 @@ class XgridsK1CompatibilityService:
|
||||
unresolved_stop
|
||||
or classified_terminal_stop
|
||||
or reconciled_ambiguous_terminal_stop
|
||||
or ordinary_terminal_stop
|
||||
)
|
||||
and record.acquisition_id == checkpoint.acquisition_id
|
||||
and record.identity.vendor_device_id_sha256
|
||||
@@ -13107,6 +13277,75 @@ class XgridsK1CompatibilityService:
|
||||
return False
|
||||
ledger_snapshot = self._physical_command_ledger.snapshot()
|
||||
record = ledger_snapshot.record
|
||||
ordinary_terminal_stop = bool(
|
||||
ledger_snapshot.status == "resolved"
|
||||
and record is not None
|
||||
and record.action == "stop"
|
||||
and record.stage == "resolved"
|
||||
and record.resolution == "stop-standby-observed"
|
||||
and record.publish_call_returned is True
|
||||
and record.packet_id is not None
|
||||
and record.qos2_completed
|
||||
and record.application_response is not None
|
||||
and record.application_response.success
|
||||
and record.last_status is not None
|
||||
and record.last_status.source == "live-control-session"
|
||||
and record.last_status.session_state in {"ready", "scan_over"}
|
||||
and not record.last_status.project_bound
|
||||
and not record.last_status.init_ready
|
||||
and not record.last_status.mqtt_retained
|
||||
and self._checkpoint_binding_matches_physical_connection(
|
||||
token.checkpoint.current_binding,
|
||||
record,
|
||||
)
|
||||
)
|
||||
if ordinary_terminal_stop:
|
||||
assert record is not None and record.last_status is not None
|
||||
store = self._active_acquisition_checkpoint
|
||||
if store is None or not self._active_acquisition_checkpoint_trust_token_is_current(
|
||||
token
|
||||
):
|
||||
return False
|
||||
try:
|
||||
binding = token.checkpoint.current_binding
|
||||
status_proof = self._checkpoint_status_proof(
|
||||
status=record.last_status,
|
||||
binding=binding,
|
||||
evidence_session_id=(
|
||||
token.checkpoint.current_evidence_session_id
|
||||
),
|
||||
)
|
||||
physical_proof = self._checkpoint_physical_proof(
|
||||
record=record,
|
||||
binding=binding,
|
||||
checkpoint=token.checkpoint,
|
||||
)
|
||||
store.cease(
|
||||
transition_id=self._active_acquisition_checkpoint_transition_id(
|
||||
"restart-ordinary-stop-cease",
|
||||
record.operation_id,
|
||||
record.revision,
|
||||
token.checkpoint_revision,
|
||||
),
|
||||
expected_revision=token.checkpoint_revision,
|
||||
expected_acquisition_id=token.acquisition_id,
|
||||
expected_start_operation_id=token.root_start_operation_id,
|
||||
physical_proof=physical_proof,
|
||||
status_proof=status_proof,
|
||||
)
|
||||
self._set_active_acquisition_checkpoint_reason(None)
|
||||
return True
|
||||
except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc:
|
||||
self._set_active_acquisition_checkpoint_reason(
|
||||
str(
|
||||
getattr(
|
||||
exc,
|
||||
"reason_code",
|
||||
"restart-ordinary-stop-checkpoint-settlement-failed",
|
||||
)
|
||||
)
|
||||
)
|
||||
return False
|
||||
reconciliation = (
|
||||
record.reconciliations[-1]
|
||||
if record is not None and record.reconciliations
|
||||
@@ -14314,8 +14553,8 @@ class XgridsK1CompatibilityService:
|
||||
self._last_live_data_suspend_aware = time.time()
|
||||
self._last_live_data_session_id = session_id
|
||||
|
||||
physical = self._physical_command_coordinator.snapshot()
|
||||
control = self._application_control_session.snapshot()
|
||||
now = time.monotonic()
|
||||
runtime_recovery = runtime.get("connection_recovery")
|
||||
with self._lock:
|
||||
acquisition = self._acquisition
|
||||
out_dir = self._acquisition_out_dir
|
||||
@@ -14328,6 +14567,35 @@ class XgridsK1CompatibilityService:
|
||||
return
|
||||
acquisition_id = acquisition.acquisition_id
|
||||
control_mode = acquisition.control_mode
|
||||
lineage = (acquisition_id, out_dir.name, producer_generation)
|
||||
active_recovery_lineage = self._active_stream_recovery_lineage
|
||||
post_recovery_restart_candidate = bool(
|
||||
control_mode == "plugin-commanded"
|
||||
and active_recovery_lineage is not None
|
||||
and active_recovery_lineage.acquisition_id == acquisition_id
|
||||
and active_recovery_lineage.evidence_session_id == out_dir.name
|
||||
and active_recovery_lineage.runtime_producer_generation
|
||||
== producer_generation
|
||||
and isinstance(runtime_recovery, Mapping)
|
||||
and runtime_recovery.get("state") == "recovered"
|
||||
)
|
||||
if not post_recovery_restart_candidate and (
|
||||
self._camera_activation_lineage == lineage
|
||||
or (
|
||||
self._camera_activation_retry_lineage == lineage
|
||||
and now < self._camera_activation_retry_not_before_monotonic
|
||||
)
|
||||
):
|
||||
# The exact lineage already owns its one camera activation (or
|
||||
# its bounded local retry delay). Repeating the durable physical
|
||||
# ledger/checkpoint admission for every 10 Hz PCL blocks this
|
||||
# publisher thread and evicts intervening pose frames. The
|
||||
# data-plane liveness edge above remains per-frame; recovery
|
||||
# deliberately bypasses this ordinary idempotence fast path.
|
||||
return
|
||||
|
||||
physical = self._physical_command_coordinator.snapshot()
|
||||
control = self._application_control_session.snapshot()
|
||||
|
||||
if control_mode == "plugin-commanded":
|
||||
start_operation_id = (
|
||||
@@ -14364,21 +14632,10 @@ class XgridsK1CompatibilityService:
|
||||
# this local admission after the durable SCANNING proof arrives.
|
||||
return
|
||||
|
||||
lineage = (acquisition_id, out_dir.name, producer_generation)
|
||||
now = time.monotonic()
|
||||
camera = self.camera_preview.snapshot()
|
||||
with self._lock:
|
||||
active_recovery_lineage = self._active_stream_recovery_lineage
|
||||
runtime_recovery = runtime.get("connection_recovery")
|
||||
if (
|
||||
control_mode == "plugin-commanded"
|
||||
post_recovery_restart_candidate
|
||||
and active_recovery_lineage is not None
|
||||
and active_recovery_lineage.acquisition_id == acquisition_id
|
||||
and active_recovery_lineage.evidence_session_id == out_dir.name
|
||||
and active_recovery_lineage.runtime_producer_generation
|
||||
== producer_generation
|
||||
and isinstance(runtime_recovery, Mapping)
|
||||
and runtime_recovery.get("state") == "recovered"
|
||||
and self._enqueue_post_recovery_camera_restart(
|
||||
active_recovery_lineage,
|
||||
runtime=runtime,
|
||||
@@ -14888,6 +15145,10 @@ class XgridsK1CompatibilityService:
|
||||
or control_after.get("session_generation")
|
||||
!= control_before.get("session_generation")
|
||||
)
|
||||
control_stage = "post-device-info-transport-refresh"
|
||||
await self._refresh_aged_verify_transport_owned(
|
||||
application_control_session=control_after,
|
||||
)
|
||||
control_stage = "physical-reconciliation"
|
||||
physical_reconciliation = await self._reconcile_physical_command_after_verify_owned(
|
||||
verify_operation_id=verify_operation_id,
|
||||
@@ -14895,6 +15156,12 @@ class XgridsK1CompatibilityService:
|
||||
if provisional_topology is not None:
|
||||
control_stage = "semantic-topology-commit"
|
||||
self._commit_provisional_fresh_bridge_topology(provisional_topology)
|
||||
control_stage = "post-reconciliation-transport-refresh"
|
||||
await self._refresh_aged_verify_transport_owned(
|
||||
application_control_session=dict(
|
||||
self._application_control_session.snapshot()
|
||||
),
|
||||
)
|
||||
control_stage = "final-control-binding"
|
||||
ready = self._connection_supervisor.snapshot()
|
||||
if not (
|
||||
@@ -23267,6 +23534,10 @@ class XgridsK1CompatibilityService:
|
||||
)
|
||||
use_reconciliation = bool(
|
||||
reconciliation is not None
|
||||
and self._physical_reconciliation_belongs_to_record(
|
||||
reconciliation,
|
||||
record,
|
||||
)
|
||||
and reconciliation.kind
|
||||
in {
|
||||
"ambiguous-outcome",
|
||||
@@ -24458,6 +24729,9 @@ class XgridsK1CompatibilityService:
|
||||
observation = await _run_blocking_operation_without_abandonment(
|
||||
self._probe_control_endpoint,
|
||||
lineage.target_ipv4,
|
||||
association_timeout_seconds=(
|
||||
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS
|
||||
),
|
||||
)
|
||||
if not observation.path.available or not observation.reachable:
|
||||
with self._lock:
|
||||
@@ -27846,6 +28120,15 @@ class XgridsK1CompatibilityService:
|
||||
self._physical_command_coordinator.snapshot()
|
||||
) is None:
|
||||
return "not-applicable"
|
||||
# A live START worker cannot be settled. Avoid touching the exact
|
||||
# mark_dispatching -> publish fence on every passive state read while
|
||||
# it is still initializing; the terminal branch below re-samples the
|
||||
# worker under that fence before committing any no-dispatch result.
|
||||
if self._application_control_session.snapshot().get("state") not in {
|
||||
"failed",
|
||||
"closed",
|
||||
}:
|
||||
return "not-applicable"
|
||||
if not self._k1_command_dispatch_gate.acquire(blocking=False):
|
||||
return "defer"
|
||||
try:
|
||||
@@ -28371,6 +28654,59 @@ class XgridsK1CompatibilityService:
|
||||
and durable_start_active_confirmed
|
||||
and durable_checkpoint_active_confirmed
|
||||
)
|
||||
terminal_control_failure = (
|
||||
terminal_control_proof.get("failure")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
terminal_control_transport = (
|
||||
terminal_control_proof.get("transport")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
terminal_verified_control = (
|
||||
terminal_control_proof.get("verified_control")
|
||||
if isinstance(terminal_control_proof, Mapping)
|
||||
else None
|
||||
)
|
||||
device_reported_scan_over_without_stop = bool(
|
||||
current is not None
|
||||
and current.control_mode == "plugin-commanded"
|
||||
and current.state in {"starting", "awaiting_external_start", "acquiring"}
|
||||
and current_stop_operation_id is None
|
||||
and isinstance(terminal_control_proof, Mapping)
|
||||
and terminal_control_proof.get("state") == "failed"
|
||||
and isinstance(terminal_control_failure, Mapping)
|
||||
and terminal_control_failure.get("failed_phase") == "scanning"
|
||||
and terminal_control_failure.get("modeling_command_attempted") is True
|
||||
and terminal_control_failure.get("stop_command_attempted") is False
|
||||
and terminal_control_failure.get("diagnostic_snapshot_unavailable") == []
|
||||
and terminal_control_failure.get("diagnostic_evidence_unavailable") == []
|
||||
and isinstance(terminal_control_transport, Mapping)
|
||||
and isinstance(terminal_control_transport.get("device_status_reports"), int)
|
||||
and not isinstance(
|
||||
terminal_control_transport.get("device_status_reports"),
|
||||
bool,
|
||||
)
|
||||
and int(terminal_control_transport["device_status_reports"]) > 0
|
||||
and terminal_control_transport.get("latest_device_session_state")
|
||||
== "scan_over"
|
||||
and terminal_control_transport.get("latest_system_error_code") is None
|
||||
and isinstance(physical_command_proof, Mapping)
|
||||
and physical_command_proof.get("runtime_bound") is True
|
||||
and physical_command_proof.get("reconciliation_ready") is True
|
||||
and physical_command_proof.get("observed_session_state") == "scan_over"
|
||||
and self._matching_start_active_confirmed(
|
||||
physical_command_proof,
|
||||
acquisition_id=current_acquisition_id,
|
||||
start_operation_id=canonical_start_operation_id,
|
||||
verified_control=(
|
||||
terminal_verified_control
|
||||
if isinstance(terminal_verified_control, Mapping)
|
||||
else None
|
||||
),
|
||||
)
|
||||
)
|
||||
# Camera-only transport loss is supervised by the backend producer
|
||||
# watchdog and exact camera CAS. Snapshot polling is observational: it
|
||||
# must not move an otherwise live MQTT/PCL runtime to ``reconnecting``.
|
||||
@@ -28459,6 +28795,21 @@ class XgridsK1CompatibilityService:
|
||||
and stop_response_accepted_without_terminal_status
|
||||
and self._operations.deadline_reached(current_stop_operation_id)
|
||||
)
|
||||
terminal_unknown_stop_control = bool(
|
||||
current is not None
|
||||
and current.control_mode == "plugin-commanded"
|
||||
and current.state == "awaiting_external_stop"
|
||||
and application_control_session.get("state") == "failed"
|
||||
and application_control_session.get("outcome_unknown") is True
|
||||
and isinstance(application_control_session.get("failure"), Mapping)
|
||||
and application_control_session["failure"].get("stop_command_attempted") is True
|
||||
and self._matching_unresolved_stop_stage(
|
||||
physical_command_proof,
|
||||
acquisition_id=current_acquisition_id,
|
||||
stop_operation_id=current_stop_operation_id,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
with self._lock:
|
||||
canonical_stop_standby_confirmed = bool(
|
||||
checkpoint_stop_completion_seen and checkpoint_stop_ceased
|
||||
@@ -28504,7 +28855,11 @@ class XgridsK1CompatibilityService:
|
||||
and stop_response_deadline_reached
|
||||
else None
|
||||
)
|
||||
if stop_response_deadline_reached:
|
||||
if terminal_unknown_stop_control or stop_response_deadline_reached:
|
||||
# Retire only the failed host-owned control socket. In particular,
|
||||
# an UNKNOWN physical STOP keeps the capture, viewer and acquisition
|
||||
# alive so an explicit read-only DeviceInfo/DeviceStatus dialogue
|
||||
# can classify SCANNING or READY without a second command.
|
||||
self._retire_local_control_after_stop_timeout()
|
||||
if terminal_unknown_stop_cleanup:
|
||||
try:
|
||||
@@ -28858,6 +29213,39 @@ class XgridsK1CompatibilityService:
|
||||
# read-only classification and seal the old STOP as
|
||||
# non-replayable/none.
|
||||
pass
|
||||
elif device_reported_scan_over_without_stop:
|
||||
# This is not an inference from a quiet camera or point stream.
|
||||
# The continuously DeviceInfo-bound control socket decoded a
|
||||
# fresh, non-retained SCAN_OVER from this exact K1, and the
|
||||
# physical coordinator independently retained the same bound
|
||||
# status. No canonical STOP was requested or attempted.
|
||||
#
|
||||
# SCAN_OVER proves that this acquisition cannot be resumed. It
|
||||
# does not prove READY and does not authorize a new START; the
|
||||
# durable physical/checkpoint chain remains fenced until a later
|
||||
# explicit read-only reconciliation observes the returned K1.
|
||||
acquisition.transition(
|
||||
"finalizing",
|
||||
message_code="acquisition.recovery.device_standby_observed",
|
||||
)
|
||||
camera_terminal_status = "interrupted"
|
||||
camera_failure_code = "device-reported-scan-over"
|
||||
recovery_standby_after_seal = True
|
||||
if point_frames <= 0:
|
||||
recovery_standby_start_operation_id = (
|
||||
self._acquisition_start_operation_id
|
||||
)
|
||||
completion_message_code = (
|
||||
"acquisition.recovery.device_standby_observed"
|
||||
)
|
||||
completion_result = {
|
||||
"receiver_stopped": True,
|
||||
"device_state": "scan_over",
|
||||
"device_stop": "not-sent",
|
||||
"automatic_command_retry": False,
|
||||
"read_only_recovery": True,
|
||||
"physical_reconciliation_required": True,
|
||||
}
|
||||
elif prepared_stop_recovery_standby:
|
||||
acquisition.transition(
|
||||
"finalizing",
|
||||
@@ -29815,6 +30203,28 @@ class XgridsK1PluginFacade:
|
||||
|
||||
def __init__(self, service: XgridsK1ServicePort) -> None:
|
||||
self.service = service
|
||||
self._state_read_task: asyncio.Task[dict[str, Any]] | None = None
|
||||
|
||||
def _retire_state_read_task(self, task: asyncio.Task[dict[str, Any]]) -> None:
|
||||
if self._state_read_task is task:
|
||||
self._state_read_task = None
|
||||
with suppress(asyncio.CancelledError):
|
||||
task.exception()
|
||||
|
||||
async def _read_state_single_flight(self) -> dict[str, Any]:
|
||||
loop = asyncio.get_running_loop()
|
||||
task = self._state_read_task
|
||||
if task is None or task.done() or task.get_loop() is not loop:
|
||||
task = loop.create_task(
|
||||
asyncio.to_thread(self.service.state),
|
||||
name="xgrids-k1-state-read",
|
||||
)
|
||||
self._state_read_task = task
|
||||
task.add_done_callback(self._retire_state_read_task)
|
||||
# An HTTP disconnect cancels only that request waiter. The shared
|
||||
# read keeps running so a later poll joins it instead of stacking a
|
||||
# second service.state thread behind the lifecycle gate.
|
||||
return await asyncio.shield(task)
|
||||
|
||||
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
|
||||
if invocation.plugin_id != self.plugin_id:
|
||||
@@ -29887,7 +30297,7 @@ class XgridsK1PluginFacade:
|
||||
self.service.bind_runtime_event_loop(asyncio.get_running_loop())
|
||||
if action_id == ACTION_STATE_READ:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_DISCOVERY_SCAN:
|
||||
scan_request = BleScanRequest.model_validate(payload)
|
||||
return await self.service.scan_ble(scan_request)
|
||||
@@ -29899,7 +30309,7 @@ class XgridsK1PluginFacade:
|
||||
EmptyRequest.model_validate(payload)
|
||||
if action_id == ACTION_DEVICE_INSPECT:
|
||||
return await asyncio.to_thread(self.service.inspect_device)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.read_device_calibration_snapshot)
|
||||
@@ -29923,7 +30333,7 @@ class XgridsK1PluginFacade:
|
||||
return await self.service.probe_configured_endpoint(probe_request)
|
||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE:
|
||||
EmptyRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(self.service.state)
|
||||
return await self._read_state_single_flight()
|
||||
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_ARM:
|
||||
arm_request = ShadowApplicationControlArmRequest.model_validate(payload)
|
||||
return await asyncio.to_thread(
|
||||
|
||||
@@ -696,6 +696,7 @@ class VisualizationRuntime:
|
||||
|
||||
def publish_stream(bridge: RerunBridge) -> None:
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
while (
|
||||
not source_done.is_set()
|
||||
or not preview_empty()
|
||||
@@ -708,15 +709,16 @@ class VisualizationRuntime:
|
||||
and preview_empty()
|
||||
):
|
||||
return
|
||||
# A dedicated latest-PCL slot plus this fixed service order
|
||||
# guarantees visible-scene progress even under an unbounded
|
||||
# stream of perception results. Other MQTT preview traffic is
|
||||
# then served before one perception result, so neither class can
|
||||
# monopolize the single publisher thread.
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
continue
|
||||
# The latest-PCL slot preserves visible-scene recovery, but a
|
||||
# continuously full slot must not starve pose. Alternate one
|
||||
# live PCL with one available non-PCL item; when no other item
|
||||
# exists, the fallback below keeps PCL flowing without delay.
|
||||
if not serve_non_point_cloud_next:
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
serve_non_point_cloud_next = True
|
||||
continue
|
||||
if serve_perception_next:
|
||||
try:
|
||||
perception = self._perception_messages.get_nowait()
|
||||
@@ -728,6 +730,7 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
self._perception_messages.task_done()
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
try:
|
||||
message = messages.get_nowait()
|
||||
@@ -739,6 +742,7 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
messages.task_done()
|
||||
serve_perception_next = True
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
try:
|
||||
perception = self._perception_messages.get_nowait()
|
||||
@@ -750,6 +754,12 @@ class VisualizationRuntime:
|
||||
finally:
|
||||
self._perception_messages.task_done()
|
||||
serve_perception_next = False
|
||||
serve_non_point_cloud_next = False
|
||||
continue
|
||||
live_point_cloud = pop_latest_live_point_cloud()
|
||||
if live_point_cloud is not None:
|
||||
publish_message(bridge, live_point_cloud)
|
||||
serve_non_point_cloud_next = True
|
||||
continue
|
||||
live_point_cloud_ready.wait(timeout=0.05)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user