SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): stabilize live recovery and media admission

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 00:03:01 +03:00
parent 7217244886
commit eaad9deda1
29 changed files with 1645 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/device_plugins/xgrids_k1/connection_supervisor.py
+12 -4
View File
@@ -1868,9 +1868,10 @@ class ReadOnlyConnectionMonitor:
return None
previous_path = baseline.host_path
intent = baseline.intent
exact_verified_control = bool(
intent is not None
and previous_path.available
if intent is None:
return None
exact_reachable_route = bool(
previous_path.available
and previous_path.fingerprint is not None
and previous_path.kernel_route_fingerprint is not None
and baseline.device_network.state == "applied"
@@ -1880,6 +1881,9 @@ class ReadOnlyConnectionMonitor:
and baseline.endpoint.intent_id == intent.intent_id
and baseline.endpoint.host_path_epoch == previous_path.epoch
and baseline.endpoint.tcp_state == "reachable"
)
exact_verified_control = bool(
exact_reachable_route
and baseline.device_identity.state == "verified"
and baseline.device_identity.intent_id == intent.intent_id
and baseline.device_identity.host_path_epoch == previous_path.epoch
@@ -1892,7 +1896,11 @@ class ReadOnlyConnectionMonitor:
and baseline.lease.target == target
and baseline.authority.control_allowed
)
if not exact_verified_control:
technical_timeout_on_exact_route = bool(
result.reason_code == "host-wifi-operation-timeout"
and exact_reachable_route
)
if not (exact_verified_control or technical_timeout_on_exact_route):
return None
if (
result.kernel_route_fingerprint is None
src/k1link/device_plugins/xgrids_k1/facade.py
+435 -25
View File
@@ -278,6 +278,13 @@ HOST_ROUTE_INSPECTION_TIMEOUT_SECONDS = 2.0
# without weakening the sub-second association cache or skipping the final
# route/association continuity recheck.
CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS = 3.0
# Active-stream recovery has already frozen an exact acquisition/START owner.
# Its endpoint probe may therefore use the same short read-only association
# budget as the monitor: the exact raw route bridge above preserves continuity
# across a technical CoreWLAN timeout, while fresh DeviceInfo still has to
# verify the K1 before MQTT publication resumes. Do not inherit the generic
# 30-second helper budget on every Wi-Fi return.
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS = 3.0
# Command checkpoints must not inherit either the monitor's three-second
# contact budget or the host helper's generic 30-second budget. One command
# may legitimately wait about three seconds for the service-lifetime
@@ -303,6 +310,9 @@ CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS = (
2 * CONNECTION_MONITOR_HOST_CONTACT_BOUND_SECONDS + CONTROL_ENDPOINT_PROBE_TIMEOUT_SECONDS
)
CONNECTION_MONITOR_QUIESCE_TIMEOUT_SECONDS = 8.0
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS = (
CONNECTION_MONITOR_FULL_PASS_BOUND_SECONDS + 1.0
)
CONNECTION_MONITOR_DEBOUNCEABLE_ASSOCIATION_REASONS = frozenset({"host-wifi-operation-timeout"})
LIVE_DATA_PLANE_STALL_SECONDS = 5.0
LIVE_DATA_PLANE_LOST_SECONDS = LIVE_DATA_PLANE_STALL_SECONDS * 2.0
@@ -2557,8 +2567,11 @@ class XgridsK1CompatibilityService:
A privacy-limited observation has the stricter gate below: exact
DeviceInfo, healthy control session, reachable lease and command
authority must all still match the same intent/mode/target/epoch. The
legacy timeout bridge may retain only its exact reachable route but
cannot manufacture any missing command authority.
sole exception is an already-admitted active-stream recovery lineage:
it may retain only this exact reachable route long enough to perform a
fresh read-only DeviceInfo/MQTT rebind. That route bridge grants no
control authority and cannot send START, STOP or a network mutation.
The legacy timeout bridge has the same route-only limitation.
A proven association change, a different route/interface/source, a
stale or unreachable endpoint, or a different intent/target bypasses
@@ -2623,7 +2636,13 @@ class XgridsK1CompatibilityService:
and snapshot.lease.target == endpoint_target
and snapshot.authority.control_allowed
)
if not exact_verified_control:
exact_active_stream_rebind = (
self._exact_active_stream_recovery_route_rebind(
snapshot=snapshot,
endpoint_target=endpoint_target,
)
)
if not (exact_verified_control or exact_active_stream_rebind):
return None
return HostPathProbeResult(
available=True,
@@ -2640,6 +2659,68 @@ class XgridsK1CompatibilityService:
kernel_route_fingerprint=path.fingerprint,
)
def _exact_active_stream_recovery_route_rebind(
self,
*,
snapshot: ConnectionSupervisorSnapshot,
endpoint_target: EndpointTarget,
) -> bool:
"""Allow one exact route-only bridge for an admitted live recovery.
Admission of ``_ActiveStreamRecoveryLineage`` already proved the
durable original START owner. This guard rechecks its process-local
acquisition/runtime coordinates without consulting or refreshing any
device authority. The subsequent recovery path must still establish
a fresh DeviceInfo identity and MQTT control proof before it can
resume publication.
"""
intent = snapshot.intent
with self._lock:
lineage = self._active_stream_recovery_lineage
acquisition = self._acquisition
if lineage is None or acquisition is None:
return False
exact_local_lineage = bool(
self._active_stream_recovery_state == "reconnecting"
and self._active_stream_recovery_generation
== lineage.recovery_generation
and self._snapshot_runtime_id == lineage.snapshot_runtime_id
and acquisition.acquisition_id == lineage.acquisition_id
and acquisition.device_id == lineage.device_id
and acquisition.device_session_id == lineage.device_session_id
and acquisition.state == "acquiring"
and self._acquisition_start_operation_id
in {None, lineage.start_operation_id}
and self._acquisition_session_lease is not None
and self._acquisition_out_dir is not None
and self._acquisition_out_dir.name == lineage.evidence_session_id
and self._selected_device_id == lineage.transport_ref
and self._device_id == lineage.device_id
and self._device_session_id == lineage.device_session_id
and self._connection_mode == lineage.connection_mode
and self._k1_ip == lineage.target_ipv4
)
if not exact_local_lineage:
return False
runtime = self.runtime.snapshot()
return bool(
runtime.get("phase") == "reconnecting"
and runtime.get("source_mode") == "live"
and runtime.get("producer_generation")
== lineage.runtime_producer_generation
and intent is not None
and intent.intent_id == lineage.intent_id
and intent.requested_mode == lineage.connection_mode
and snapshot.device_network.state == "applied"
and snapshot.device_network.intent_id == lineage.intent_id
and snapshot.device_network.transport_ref == lineage.transport_ref
and snapshot.device_network.connection_mode == lineage.connection_mode
and snapshot.device_network.target == endpoint_target
and endpoint_target
== EndpointTarget(lineage.target_ipv4, lineage.target_port)
)
async def _monitor_host_path(self, target: EndpointTarget) -> HostPathProbeResult:
if not self._connection_monitor_contact_gate.acquire(blocking=False):
raise ConnectionMonitorProbeSuperseded(
@@ -2715,6 +2796,57 @@ class XgridsK1CompatibilityService:
)
self._connection_supervisor.record_monitor_failure("connection-monitor-start-failed")
async def _refresh_aged_verify_transport_owned(
self,
*,
application_control_session: Mapping[str, Any],
) -> None:
"""Refresh only a nearly expired Verify transport with real I/O.
The caller owns the Verify lifecycle/network fences, so the background
monitor is deliberately superseded. If DeviceInfo or physical
reconciliation consumed most of the existing endpoint lease, perform
the same route/TCP/route observation before releasing those fences.
No cached timestamp, MQTT reconnect, BLE operation, or device command
is manufactured here.
"""
verified_binding = application_control_session.get("verified_control")
if not isinstance(verified_binding, Mapping):
raise ConnectionVerificationError(
"Verify lost its exact control proof before transport refresh",
reason_code="connection-verify-final-binding-superseded",
)
target = EndpointTarget(
str(verified_binding["target_ipv4"]),
int(verified_binding["target_port"]),
)
intent_id = str(verified_binding["intent_id"])
host_path_epoch = int(verified_binding["host_path_epoch"])
if self._connection_supervisor.endpoint_observation_has_remaining_lease(
target=target,
intent_id=intent_id,
host_path_epoch=host_path_epoch,
minimum_remaining_seconds=(
VERIFY_POST_TRANSITION_ENDPOINT_MIN_REMAINING_SECONDS
),
):
return
observation = await _run_blocking_operation_without_abandonment(
self._probe_control_endpoint,
target.ipv4,
association_timeout_seconds=(CONNECTION_MONITOR_ASSOCIATION_TIMEOUT_SECONDS),
)
if not observation.reachable:
raise ConnectionVerificationError(
"K1 endpoint не подтвердил связь после долгой read-only сверки",
reason_code="connection-verify-mqtt-unreachable",
)
self._reconcile_connection_supervisor(
application_control_session,
self.runtime.snapshot(),
)
def _observe_connection_transport(
self,
target: str,
@@ -2723,9 +2855,17 @@ class XgridsK1CompatibilityService:
reachable: bool | None,
reason_code: str | None = None,
) -> None:
supervisor = self._connection_supervisor.snapshot()
intent = supervisor.intent
endpoint = EndpointTarget(validate_private_ipv4(target), CONTROL_MQTT_PORT)
# A completed route sample is fresher than the silence fallback. Read
# the exact applied target without expiring the previous observation
# between sampling and publication; the CAS below still rejects an
# intent/target change and the sample itself decides route continuity.
supervisor = self._connection_supervisor.association_timeout_retention_candidate(
expected_target=endpoint,
)
if supervisor is None:
return
intent = supervisor.intent
if (
intent is None
or supervisor.device_network.state != "applied"
@@ -2734,7 +2874,14 @@ class XgridsK1CompatibilityService:
or supervisor.device_network.target != endpoint
):
return
epoch = self._connection_supervisor.observe_host_path(path)
epoch = self._connection_supervisor.observe_host_path_if_current(
expected_intent_id=intent.intent_id,
expected_target=endpoint,
expected_host_path_observation=supervisor.host_path,
result=path,
)
if epoch is None:
return
if path.available and reachable is not None:
self._connection_supervisor.observe_endpoint(
target=endpoint,
@@ -3637,8 +3784,15 @@ class XgridsK1CompatibilityService:
pending = self._pending_local_control_retirement
if not pending:
return True
control = self._application_control_session.snapshot()
try:
self._retire_application_control_for_network_change()
self._retire_application_control_for_network_change(
# A failed worker may finish retiring after the first bounded
# close attempt. Retrying that host-only retirement must not
# remain permanently blocked by the terminal phase itself.
# The durable physical-command row is intentionally untouched.
allow_terminal_failure=control.get("state") == "failed",
)
except (AttributeError, RuntimeError):
return False
return True
@@ -11616,6 +11770,21 @@ class XgridsK1CompatibilityService:
and not reconciliation.observation.init_ready
and not reconciliation.observation.mqtt_retained
)
ordinary_terminal_stop = bool(
record.stage == "resolved"
and record.resolution == "stop-standby-observed"
and record.publish_call_returned is True
and record.packet_id is not None
and record.qos2_completed
and record.application_response is not None
and record.application_response.success
and record.last_status is not None
and record.last_status.source == "live-control-session"
and record.last_status.session_state in {"ready", "scan_over"}
and not record.last_status.project_bound
and not record.last_status.init_ready
and not record.last_status.mqtt_retained
)
if not (
persisted is not None
and status is not None
@@ -11624,6 +11793,7 @@ class XgridsK1CompatibilityService:
unresolved_stop
or classified_terminal_stop
or reconciled_ambiguous_terminal_stop
or ordinary_terminal_stop
)
and record.acquisition_id == checkpoint.acquisition_id
and record.identity.vendor_device_id_sha256
@@ -13107,6 +13277,75 @@ class XgridsK1CompatibilityService:
return False
ledger_snapshot = self._physical_command_ledger.snapshot()
record = ledger_snapshot.record
ordinary_terminal_stop = bool(
ledger_snapshot.status == "resolved"
and record is not None
and record.action == "stop"
and record.stage == "resolved"
and record.resolution == "stop-standby-observed"
and record.publish_call_returned is True
and record.packet_id is not None
and record.qos2_completed
and record.application_response is not None
and record.application_response.success
and record.last_status is not None
and record.last_status.source == "live-control-session"
and record.last_status.session_state in {"ready", "scan_over"}
and not record.last_status.project_bound
and not record.last_status.init_ready
and not record.last_status.mqtt_retained
and self._checkpoint_binding_matches_physical_connection(
token.checkpoint.current_binding,
record,
)
)
if ordinary_terminal_stop:
assert record is not None and record.last_status is not None
store = self._active_acquisition_checkpoint
if store is None or not self._active_acquisition_checkpoint_trust_token_is_current(
token
):
return False
try:
binding = token.checkpoint.current_binding
status_proof = self._checkpoint_status_proof(
status=record.last_status,
binding=binding,
evidence_session_id=(
token.checkpoint.current_evidence_session_id
),
)
physical_proof = self._checkpoint_physical_proof(
record=record,
binding=binding,
checkpoint=token.checkpoint,
)
store.cease(
transition_id=self._active_acquisition_checkpoint_transition_id(
"restart-ordinary-stop-cease",
record.operation_id,
record.revision,
token.checkpoint_revision,
),
expected_revision=token.checkpoint_revision,
expected_acquisition_id=token.acquisition_id,
expected_start_operation_id=token.root_start_operation_id,
physical_proof=physical_proof,
status_proof=status_proof,
)
self._set_active_acquisition_checkpoint_reason(None)
return True
except (ActiveAcquisitionRecoveryCheckpointError, OSError, ValueError) as exc:
self._set_active_acquisition_checkpoint_reason(
str(
getattr(
exc,
"reason_code",
"restart-ordinary-stop-checkpoint-settlement-failed",
)
)
)
return False
reconciliation = (
record.reconciliations[-1]
if record is not None and record.reconciliations
@@ -14314,8 +14553,8 @@ class XgridsK1CompatibilityService:
self._last_live_data_suspend_aware = time.time()
self._last_live_data_session_id = session_id
physical = self._physical_command_coordinator.snapshot()
control = self._application_control_session.snapshot()
now = time.monotonic()
runtime_recovery = runtime.get("connection_recovery")
with self._lock:
acquisition = self._acquisition
out_dir = self._acquisition_out_dir
@@ -14328,6 +14567,35 @@ class XgridsK1CompatibilityService:
return
acquisition_id = acquisition.acquisition_id
control_mode = acquisition.control_mode
lineage = (acquisition_id, out_dir.name, producer_generation)
active_recovery_lineage = self._active_stream_recovery_lineage
post_recovery_restart_candidate = bool(
control_mode == "plugin-commanded"
and active_recovery_lineage is not None
and active_recovery_lineage.acquisition_id == acquisition_id
and active_recovery_lineage.evidence_session_id == out_dir.name
and active_recovery_lineage.runtime_producer_generation
== producer_generation
and isinstance(runtime_recovery, Mapping)
and runtime_recovery.get("state") == "recovered"
)
if not post_recovery_restart_candidate and (
self._camera_activation_lineage == lineage
or (
self._camera_activation_retry_lineage == lineage
and now < self._camera_activation_retry_not_before_monotonic
)
):
# The exact lineage already owns its one camera activation (or
# its bounded local retry delay). Repeating the durable physical
# ledger/checkpoint admission for every 10 Hz PCL blocks this
# publisher thread and evicts intervening pose frames. The
# data-plane liveness edge above remains per-frame; recovery
# deliberately bypasses this ordinary idempotence fast path.
return
physical = self._physical_command_coordinator.snapshot()
control = self._application_control_session.snapshot()
if control_mode == "plugin-commanded":
start_operation_id = (
@@ -14364,21 +14632,10 @@ class XgridsK1CompatibilityService:
# this local admission after the durable SCANNING proof arrives.
return
lineage = (acquisition_id, out_dir.name, producer_generation)
now = time.monotonic()
camera = self.camera_preview.snapshot()
with self._lock:
active_recovery_lineage = self._active_stream_recovery_lineage
runtime_recovery = runtime.get("connection_recovery")
if (
control_mode == "plugin-commanded"
post_recovery_restart_candidate
and active_recovery_lineage is not None
and active_recovery_lineage.acquisition_id == acquisition_id
and active_recovery_lineage.evidence_session_id == out_dir.name
and active_recovery_lineage.runtime_producer_generation
== producer_generation
and isinstance(runtime_recovery, Mapping)
and runtime_recovery.get("state") == "recovered"
and self._enqueue_post_recovery_camera_restart(
active_recovery_lineage,
runtime=runtime,
@@ -14888,6 +15145,10 @@ class XgridsK1CompatibilityService:
or control_after.get("session_generation")
!= control_before.get("session_generation")
)
control_stage = "post-device-info-transport-refresh"
await self._refresh_aged_verify_transport_owned(
application_control_session=control_after,
)
control_stage = "physical-reconciliation"
physical_reconciliation = await self._reconcile_physical_command_after_verify_owned(
verify_operation_id=verify_operation_id,
@@ -14895,6 +15156,12 @@ class XgridsK1CompatibilityService:
if provisional_topology is not None:
control_stage = "semantic-topology-commit"
self._commit_provisional_fresh_bridge_topology(provisional_topology)
control_stage = "post-reconciliation-transport-refresh"
await self._refresh_aged_verify_transport_owned(
application_control_session=dict(
self._application_control_session.snapshot()
),
)
control_stage = "final-control-binding"
ready = self._connection_supervisor.snapshot()
if not (
@@ -23267,6 +23534,10 @@ class XgridsK1CompatibilityService:
)
use_reconciliation = bool(
reconciliation is not None
and self._physical_reconciliation_belongs_to_record(
reconciliation,
record,
)
and reconciliation.kind
in {
"ambiguous-outcome",
@@ -24458,6 +24729,9 @@ class XgridsK1CompatibilityService:
observation = await _run_blocking_operation_without_abandonment(
self._probe_control_endpoint,
lineage.target_ipv4,
association_timeout_seconds=(
ACTIVE_STREAM_RECOVERY_ASSOCIATION_TIMEOUT_SECONDS
),
)
if not observation.path.available or not observation.reachable:
with self._lock:
@@ -27846,6 +28120,15 @@ class XgridsK1CompatibilityService:
self._physical_command_coordinator.snapshot()
) is None:
return "not-applicable"
# A live START worker cannot be settled. Avoid touching the exact
# mark_dispatching -> publish fence on every passive state read while
# it is still initializing; the terminal branch below re-samples the
# worker under that fence before committing any no-dispatch result.
if self._application_control_session.snapshot().get("state") not in {
"failed",
"closed",
}:
return "not-applicable"
if not self._k1_command_dispatch_gate.acquire(blocking=False):
return "defer"
try:
@@ -28371,6 +28654,59 @@ class XgridsK1CompatibilityService:
and durable_start_active_confirmed
and durable_checkpoint_active_confirmed
)
terminal_control_failure = (
terminal_control_proof.get("failure")
if isinstance(terminal_control_proof, Mapping)
else None
)
terminal_control_transport = (
terminal_control_proof.get("transport")
if isinstance(terminal_control_proof, Mapping)
else None
)
terminal_verified_control = (
terminal_control_proof.get("verified_control")
if isinstance(terminal_control_proof, Mapping)
else None
)
device_reported_scan_over_without_stop = bool(
current is not None
and current.control_mode == "plugin-commanded"
and current.state in {"starting", "awaiting_external_start", "acquiring"}
and current_stop_operation_id is None
and isinstance(terminal_control_proof, Mapping)
and terminal_control_proof.get("state") == "failed"
and isinstance(terminal_control_failure, Mapping)
and terminal_control_failure.get("failed_phase") == "scanning"
and terminal_control_failure.get("modeling_command_attempted") is True
and terminal_control_failure.get("stop_command_attempted") is False
and terminal_control_failure.get("diagnostic_snapshot_unavailable") == []
and terminal_control_failure.get("diagnostic_evidence_unavailable") == []
and isinstance(terminal_control_transport, Mapping)
and isinstance(terminal_control_transport.get("device_status_reports"), int)
and not isinstance(
terminal_control_transport.get("device_status_reports"),
bool,
)
and int(terminal_control_transport["device_status_reports"]) > 0
and terminal_control_transport.get("latest_device_session_state")
== "scan_over"
and terminal_control_transport.get("latest_system_error_code") is None
and isinstance(physical_command_proof, Mapping)
and physical_command_proof.get("runtime_bound") is True
and physical_command_proof.get("reconciliation_ready") is True
and physical_command_proof.get("observed_session_state") == "scan_over"
and self._matching_start_active_confirmed(
physical_command_proof,
acquisition_id=current_acquisition_id,
start_operation_id=canonical_start_operation_id,
verified_control=(
terminal_verified_control
if isinstance(terminal_verified_control, Mapping)
else None
),
)
)
# Camera-only transport loss is supervised by the backend producer
# watchdog and exact camera CAS. Snapshot polling is observational: it
# must not move an otherwise live MQTT/PCL runtime to ``reconnecting``.
@@ -28459,6 +28795,21 @@ class XgridsK1CompatibilityService:
and stop_response_accepted_without_terminal_status
and self._operations.deadline_reached(current_stop_operation_id)
)
terminal_unknown_stop_control = bool(
current is not None
and current.control_mode == "plugin-commanded"
and current.state == "awaiting_external_stop"
and application_control_session.get("state") == "failed"
and application_control_session.get("outcome_unknown") is True
and isinstance(application_control_session.get("failure"), Mapping)
and application_control_session["failure"].get("stop_command_attempted") is True
and self._matching_unresolved_stop_stage(
physical_command_proof,
acquisition_id=current_acquisition_id,
stop_operation_id=current_stop_operation_id,
)
is not None
)
with self._lock:
canonical_stop_standby_confirmed = bool(
checkpoint_stop_completion_seen and checkpoint_stop_ceased
@@ -28504,7 +28855,11 @@ class XgridsK1CompatibilityService:
and stop_response_deadline_reached
else None
)
if stop_response_deadline_reached:
if terminal_unknown_stop_control or stop_response_deadline_reached:
# Retire only the failed host-owned control socket. In particular,
# an UNKNOWN physical STOP keeps the capture, viewer and acquisition
# alive so an explicit read-only DeviceInfo/DeviceStatus dialogue
# can classify SCANNING or READY without a second command.
self._retire_local_control_after_stop_timeout()
if terminal_unknown_stop_cleanup:
try:
@@ -28858,6 +29213,39 @@ class XgridsK1CompatibilityService:
# read-only classification and seal the old STOP as
# non-replayable/none.
pass
elif device_reported_scan_over_without_stop:
# This is not an inference from a quiet camera or point stream.
# The continuously DeviceInfo-bound control socket decoded a
# fresh, non-retained SCAN_OVER from this exact K1, and the
# physical coordinator independently retained the same bound
# status. No canonical STOP was requested or attempted.
#
# SCAN_OVER proves that this acquisition cannot be resumed. It
# does not prove READY and does not authorize a new START; the
# durable physical/checkpoint chain remains fenced until a later
# explicit read-only reconciliation observes the returned K1.
acquisition.transition(
"finalizing",
message_code="acquisition.recovery.device_standby_observed",
)
camera_terminal_status = "interrupted"
camera_failure_code = "device-reported-scan-over"
recovery_standby_after_seal = True
if point_frames <= 0:
recovery_standby_start_operation_id = (
self._acquisition_start_operation_id
)
completion_message_code = (
"acquisition.recovery.device_standby_observed"
)
completion_result = {
"receiver_stopped": True,
"device_state": "scan_over",
"device_stop": "not-sent",
"automatic_command_retry": False,
"read_only_recovery": True,
"physical_reconciliation_required": True,
}
elif prepared_stop_recovery_standby:
acquisition.transition(
"finalizing",
@@ -29815,6 +30203,28 @@ class XgridsK1PluginFacade:
def __init__(self, service: XgridsK1ServicePort) -> None:
self.service = service
self._state_read_task: asyncio.Task[dict[str, Any]] | None = None
def _retire_state_read_task(self, task: asyncio.Task[dict[str, Any]]) -> None:
if self._state_read_task is task:
self._state_read_task = None
with suppress(asyncio.CancelledError):
task.exception()
async def _read_state_single_flight(self) -> dict[str, Any]:
loop = asyncio.get_running_loop()
task = self._state_read_task
if task is None or task.done() or task.get_loop() is not loop:
task = loop.create_task(
asyncio.to_thread(self.service.state),
name="xgrids-k1-state-read",
)
self._state_read_task = task
task.add_done_callback(self._retire_state_read_task)
# An HTTP disconnect cancels only that request waiter. The shared
# read keeps running so a later poll joins it instead of stacking a
# second service.state thread behind the lifecycle gate.
return await asyncio.shield(task)
async def invoke(self, invocation: RuntimeActionInvocation) -> dict[str, Any]:
if invocation.plugin_id != self.plugin_id:
@@ -29887,7 +30297,7 @@ class XgridsK1PluginFacade:
self.service.bind_runtime_event_loop(asyncio.get_running_loop())
if action_id == ACTION_STATE_READ:
EmptyRequest.model_validate(payload)
return await asyncio.to_thread(self.service.state)
return await self._read_state_single_flight()
if action_id == ACTION_DISCOVERY_SCAN:
scan_request = BleScanRequest.model_validate(payload)
return await self.service.scan_ble(scan_request)
@@ -29899,7 +30309,7 @@ class XgridsK1PluginFacade:
EmptyRequest.model_validate(payload)
if action_id == ACTION_DEVICE_INSPECT:
return await asyncio.to_thread(self.service.inspect_device)
return await asyncio.to_thread(self.service.state)
return await self._read_state_single_flight()
if action_id == ACTION_DEVICE_CALIBRATION_SNAPSHOT_READ:
EmptyRequest.model_validate(payload)
return await asyncio.to_thread(self.service.read_device_calibration_snapshot)
@@ -29923,7 +30333,7 @@ class XgridsK1PluginFacade:
return await self.service.probe_configured_endpoint(probe_request)
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_STATE:
EmptyRequest.model_validate(payload)
return await asyncio.to_thread(self.service.state)
return await self._read_state_single_flight()
if action_id == ACTION_APPLICATION_CONTROL_SHADOW_ARM:
arm_request = ShadowApplicationControlArmRequest.model_validate(payload)
return await asyncio.to_thread(
src/k1link/device_plugins/xgrids_k1/viewer/runtime.py
+19 -9
View File
@@ -696,6 +696,7 @@ class VisualizationRuntime:
def publish_stream(bridge: RerunBridge) -> None:
serve_perception_next = False
serve_non_point_cloud_next = False
while (
not source_done.is_set()
or not preview_empty()
@@ -708,15 +709,16 @@ class VisualizationRuntime:
and preview_empty()
):
return
# A dedicated latest-PCL slot plus this fixed service order
# guarantees visible-scene progress even under an unbounded
# stream of perception results. Other MQTT preview traffic is
# then served before one perception result, so neither class can
# monopolize the single publisher thread.
live_point_cloud = pop_latest_live_point_cloud()
if live_point_cloud is not None:
publish_message(bridge, live_point_cloud)
continue
# The latest-PCL slot preserves visible-scene recovery, but a
# continuously full slot must not starve pose. Alternate one
# live PCL with one available non-PCL item; when no other item
# exists, the fallback below keeps PCL flowing without delay.
if not serve_non_point_cloud_next:
live_point_cloud = pop_latest_live_point_cloud()
if live_point_cloud is not None:
publish_message(bridge, live_point_cloud)
serve_non_point_cloud_next = True
continue
if serve_perception_next:
try:
perception = self._perception_messages.get_nowait()
@@ -728,6 +730,7 @@ class VisualizationRuntime:
finally:
self._perception_messages.task_done()
serve_perception_next = False
serve_non_point_cloud_next = False
continue
try:
message = messages.get_nowait()
@@ -739,6 +742,7 @@ class VisualizationRuntime:
finally:
messages.task_done()
serve_perception_next = True
serve_non_point_cloud_next = False
continue
try:
perception = self._perception_messages.get_nowait()
@@ -750,6 +754,12 @@ class VisualizationRuntime:
finally:
self._perception_messages.task_done()
serve_perception_next = False
serve_non_point_cloud_next = False
continue
live_point_cloud = pop_latest_live_point_cloud()
if live_point_cloud is not None:
publish_message(bridge, live_point_cloud)
serve_non_point_cloud_next = True
continue
live_point_cloud_ready.wait(timeout=0.05)
src/k1link/web/app.py
+59 -11
View File
@@ -355,24 +355,66 @@ def newly_finalized_recording_ids(
return tuple(sorted(set(current_finalized) - known_finalized))
def observation_archive_revision(roots: Iterable[Path]) -> tuple[tuple[object, ...], ...]:
"""Return a cheap change fence for direct-child observation archives.
The K1 writer creates and retires ``.current_session`` in the archive root,
and every session directory is a direct child of that same root. Those
operations advance the directory metadata, while growing capture/media
files do not. This lets the reconciler notice session start, completion and
removal without recursively reopening tens of thousands of immutable
evidence files every two seconds.
"""
revisions: list[tuple[object, ...]] = []
for configured_root in roots:
root = configured_root.expanduser().resolve(strict=False)
try:
metadata = root.lstat()
except OSError:
revisions.append((str(root), None))
continue
revisions.append(
(
str(root),
metadata.st_dev,
metadata.st_ino,
metadata.st_mtime_ns,
metadata.st_ctime_ns,
)
)
revisions.sort(key=lambda item: str(item[0]))
return tuple(revisions)
async def _recording_preparation_reconciler() -> None:
"""Prepare sessions finalized during this process, never historical rows."""
known_finalized: set[str] | None = None
reconciled_archive_revision: tuple[tuple[object, ...], ...] | None = None
while True:
try:
await asyncio.to_thread(refresh_observation_catalog)
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
newly_finalized = newly_finalized_recording_ids(
known_finalized,
finalized,
archive_revision = await asyncio.to_thread(
observation_archive_revision,
(archive.root for archive in plugin_environment.observation_archives),
)
if newly_finalized:
await asyncio.to_thread(
enqueue_replayable_recordings,
newly_finalized,
if archive_revision != reconciled_archive_revision:
await asyncio.to_thread(refresh_observation_catalog)
finalized = set(await asyncio.to_thread(finalized_replayable_recording_ids))
newly_finalized = newly_finalized_recording_ids(
known_finalized,
finalized,
)
known_finalized = finalized
if newly_finalized:
await asyncio.to_thread(
enqueue_replayable_recordings,
newly_finalized,
)
known_finalized = finalized
# Keep the pre-scan revision. If a writer changed the archive
# during the expensive discovery, the next two-second check
# observes the newer revision and performs one follow-up pass.
reconciled_archive_revision = archive_revision
recording_reconciler_readiness.record_success()
except Exception as exc:
# A transient filesystem/catalog failure must not permanently
@@ -496,6 +538,12 @@ _CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
}
)
# This stream carries the comparatively heavy control-state snapshot, not
# camera or point-cloud media. Action requests return their own result, and the
# frontend also keeps a four-second REST fallback, so a two-second passive
# cadence preserves status recovery without starving the live media sockets.
DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS = 2.0
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
message = " ".join(str(exc).split())
@@ -528,7 +576,7 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
if _is_closed_websocket_send_error(exc):
return
raise
await asyncio.sleep(0.5)
await asyncio.sleep(DEVICE_PLUGIN_EVENT_POLL_INTERVAL_SECONDS)
except (PluginNotFoundError, PluginActionNotFoundError):
await websocket.close(code=1008, reason="Device plugin is not available")
except (PluginExecutionError, PluginRuntimeUnavailableError):