SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): restore low-latency live recovery path

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 16:17:27 +03:00
parent 85035fa07b
commit 1001a31638
12 changed files with 1252 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/device_plugins/xgrids_k1/camera.py
+60 -24
View File
@@ -661,11 +661,13 @@ class XgridsK1CameraGateway:
target_host: str,
session_dir: Path,
*,
pre_prepare_fence: CameraProducerCommitFence,
commit_fence: CameraProducerCommitFence,
committed_before_start: CameraProducerCommittedObserver | None = None,
) -> dict[str, Any]:
"""Prepare the first acquisition camera outside every shared gate."""
"""Reserve authority before Popen, then commit without a dead-pipe gap."""
activation_started = time.monotonic()
if source_id not in CAMERA_SOURCE_PATHS:
raise ValueError("неизвестный camera source")
target = validate_private_ipv4(target_host)
@@ -674,29 +676,41 @@ class XgridsK1CameraGateway:
raise ValueError("observation session directory does not exist")
if not root.is_relative_to(self._repository_root):
raise ValueError("camera recording root must stay inside the repository")
with self._lifecycle_lock: # noqa: SIM117 - lock order is intentional
with self._lock:
self._require_open_locked()
if self._ffmpeg_path is None:
raise RuntimeError("локальный camera adapter FFmpeg не найден")
if self._producer is not None or self._recording_root is not None:
raise RuntimeError("camera acquisition producer уже активен")
if self._source_id is not None and (
self._source_id != source_id or self._target_host != target
):
raise RuntimeError("для camera gateway уже выбран другой source")
if self._source_id is None:
self._generation += 1
self._source_id = source_id
self._target_host = target
self._recording_root = root
self._recording_media_segment_count = 0
self._archive_summaries = []
self._expected_source_end_generation = None
self._phase = "selected"
self._error = None
self._revision += 1
reserved_generation: list[int] = []
def reserve() -> bool:
with self._lifecycle_lock: # noqa: SIM117 - lock order is intentional
with self._lock:
self._require_open_locked()
if self._ffmpeg_path is None:
raise RuntimeError("локальный camera adapter FFmpeg не найден")
if self._producer is not None or self._recording_root is not None:
raise RuntimeError("camera acquisition producer уже активен")
if self._source_id is not None and (
self._source_id != source_id or self._target_host != target
):
raise RuntimeError("для camera gateway уже выбран другой source")
if self._source_id is None:
self._generation += 1
self._source_id = source_id
self._target_host = target
self._recording_root = root
self._recording_media_segment_count = 0
self._archive_summaries = []
self._expected_source_end_generation = None
self._phase = "selected"
self._error = None
self._revision += 1
reserved_generation.append(self._generation)
return True
if not pre_prepare_fence(reserve):
raise ValueError("camera activation authority устарела до запуска adapter")
if len(reserved_generation) != 1:
raise RuntimeError("camera activation fence did not reserve exactly once")
authority_reserved = time.monotonic()
prepared = self._prepare_selected_producer()
ffmpeg_prepared = time.monotonic()
def commit() -> bool:
with self._lifecycle_lock:
@@ -719,7 +733,29 @@ class XgridsK1CameraGateway:
if not committed:
self._discard_prepared_producer(prepared, failure_code="stale-activation-commit")
raise ValueError("camera activation lineage устарела")
return self.snapshot()
readers_started = time.monotonic()
snapshot = self.snapshot()
logger.info(
"K1 camera acquisition producer activation timing",
extra={
"event_code": "k1_camera_activation_timing",
"camera_generation": reserved_generation[0],
"camera_authority_wait_ms": int(round(
(authority_reserved - activation_started) * 1_000,
)),
"camera_ffmpeg_prepare_ms": int(round(
(ffmpeg_prepared - authority_reserved) * 1_000,
)),
"camera_post_spawn_commit_ms": int(round(
(readers_started - ffmpeg_prepared) * 1_000,
)),
"camera_activation_total_ms": int(round(
(readers_started - activation_started) * 1_000,
)),
"device_command_sent": False,
},
)
return snapshot
def stop(self, generation: int) -> dict[str, Any]:
with self._lifecycle_lock:
src/k1link/device_plugins/xgrids_k1/facade.py
+374 -64
View File
@@ -599,6 +599,18 @@ class _ActiveStreamRecoveryLineage:
target_port: int
@dataclass(frozen=True, slots=True)
class _CameraProducerActivationClaim:
"""Pre-Popen authority reduced to a bounded post-Popen local CAS."""
acquisition_id: str
evidence_session_id: str
start_operation_id: str | None
stop_operation_id: str | None
runtime_producer_generation: int
recovery_generation: int
@dataclass(frozen=True, slots=True)
class _PostRecoveryCameraRestartKey:
"""Exact recovered-PCL and camera progress CAS claimed by one worker."""
@@ -3848,7 +3860,7 @@ class XgridsK1CompatibilityService:
def _retire_terminal_prestart_control_failure(self) -> bool:
"""Automatically clear only proven local, pre-START terminal state."""
snapshot = self._application_control_session.snapshot()
snapshot = self._control_session_snapshot_without_physical_command()
self._retire_prepared_acquisition_on_terminal_control_failure(snapshot)
state = str(snapshot.get("state") or "unknown")
failure = snapshot.get("failure")
@@ -3909,6 +3921,13 @@ class XgridsK1CompatibilityService:
or self.runtime.snapshot().get("source_mode") != "idle"
):
return False
control = self._control_session_snapshot_without_physical_command()
if control.get("state") not in {
"connection-ready",
"workspace-ready",
"project-ready",
}:
return False
physical = self._physical_command_coordinator.snapshot()
if (
physical.get("status") not in {"empty", "resolved"}
@@ -3916,14 +3935,6 @@ class XgridsK1CompatibilityService:
or _physical_command_reports_active(physical)
):
return False
control = dict(self._application_control_session.snapshot())
if control.get("state") not in {
"connection-ready",
"workspace-ready",
"project-ready",
}:
return False
verified = control.get("verified_control")
supervisor = self._connection_supervisor.snapshot()
structurally_current = bool(
@@ -5618,6 +5629,14 @@ class XgridsK1CompatibilityService:
intent: Literal["select-device", "change-network"],
transition_gate_owned: bool,
active_binding: Mapping[str, object] | None = None,
runtime_snapshot: Mapping[str, object] | None = None,
physical_command_snapshot: Mapping[str, object] | None = None,
control_session_snapshot: Mapping[str, object] | None = None,
native_runtime_snapshot: Mapping[str, object] | None = None,
semantic_topology_snapshot: Mapping[str, object] | None = None,
identity_pin_snapshot: Mapping[str, object] | None = None,
provisioning_idempotency_snapshot: Mapping[str, object] | None = None,
network_ledger_snapshot: NetworkMutationLedgerSnapshot | None = None,
) -> list[str]:
"""Return one factual Bridge-only pre-START handoff admission policy."""
@@ -5631,14 +5650,44 @@ class XgridsK1CompatibilityService:
configured_mode = self._connection_mode
pending_required_mode = self._connection_reconfiguration_required_connection_mode
lifecycle_holders = set(self._application_control_process_lease_holders)
runtime = self.runtime.snapshot()
physical = self._physical_command_coordinator.snapshot()
control = self._application_control_session.snapshot()
ble_runtime = ble_runtime_snapshot()
semantic = self._semantic_topology_public_snapshot()
identity = self._device_identity_pin_public_snapshot()
idempotency = self._network_provisioning_idempotency_public_snapshot()
network_ledger = self._network_mutation_ledger.snapshot()
runtime = (
runtime_snapshot if runtime_snapshot is not None else self.runtime.snapshot()
)
physical = (
physical_command_snapshot
if physical_command_snapshot is not None
else self._physical_command_coordinator.snapshot()
)
control = (
control_session_snapshot
if control_session_snapshot is not None
else self._application_control_session.snapshot()
)
ble_runtime = (
native_runtime_snapshot
if native_runtime_snapshot is not None
else ble_runtime_snapshot()
)
semantic = (
semantic_topology_snapshot
if semantic_topology_snapshot is not None
else self._semantic_topology_public_snapshot()
)
identity = (
identity_pin_snapshot
if identity_pin_snapshot is not None
else self._device_identity_pin_public_snapshot()
)
idempotency = (
provisioning_idempotency_snapshot
if provisioning_idempotency_snapshot is not None
else self._network_provisioning_idempotency_public_snapshot()
)
network_ledger = (
network_ledger_snapshot
if network_ledger_snapshot is not None
else self._network_mutation_ledger.snapshot()
)
reasons: list[str] = []
if not transition_gate_owned:
@@ -5741,6 +5790,10 @@ class XgridsK1CompatibilityService:
self,
*,
allow_owned_network_holder: bool = False,
physical_command_snapshot: Mapping[str, object] | None = None,
runtime_snapshot: Mapping[str, object] | None = None,
control_session_snapshot: Mapping[str, object] | None = None,
native_runtime_snapshot: Mapping[str, object] | None = None,
) -> dict[str, object]:
"""Project one exact local-only retirement checkpoint.
@@ -5750,7 +5803,11 @@ class XgridsK1CompatibilityService:
ledger performs the final operation/revision/transport CAS.
"""
physical = self._physical_command_coordinator.snapshot()
physical = (
physical_command_snapshot
if physical_command_snapshot is not None
else self._physical_command_coordinator.snapshot()
)
record = physical.get("record")
with self._lock:
provisioning_active = self._provisioning_active
@@ -5761,9 +5818,19 @@ class XgridsK1CompatibilityService:
reconfiguration_active = self._connection_reconfiguration_intent is not None
pending_local_control_retirement = self._pending_local_control_retirement
lifecycle_holders = set(self._application_control_process_lease_holders)
runtime = self.runtime.snapshot()
control = self._application_control_session.snapshot()
native = ble_runtime_snapshot()
runtime = (
runtime_snapshot if runtime_snapshot is not None else self.runtime.snapshot()
)
control = (
control_session_snapshot
if control_session_snapshot is not None
else self._application_control_session.snapshot()
)
native = (
native_runtime_snapshot
if native_runtime_snapshot is not None
else ble_runtime_snapshot()
)
reasons: list[str] = []
retireable_ambiguous = bool(
@@ -5892,10 +5959,23 @@ class XgridsK1CompatibilityService:
self,
*,
allow_owned_network_holder: bool = False,
physical_command_snapshot: Mapping[str, object] | None = None,
runtime_snapshot: Mapping[str, object] | None = None,
control_session_snapshot: Mapping[str, object] | None = None,
native_runtime_snapshot: Mapping[str, object] | None = None,
supervisor_snapshot: ConnectionSupervisorSnapshot | None = None,
network_ledger_snapshot: NetworkMutationLedgerSnapshot | None = None,
provisioning_idempotency_snapshot: Mapping[str, object] | None = None,
semantic_topology_snapshot: Mapping[str, object] | None = None,
identity_pin_snapshot: Mapping[str, object] | None = None,
) -> dict[str, object]:
"""Project one exact fresh UUID checkpoint for local retirement reopen."""
physical = self._physical_command_coordinator.snapshot()
physical = (
physical_command_snapshot
if physical_command_snapshot is not None
else self._physical_command_coordinator.snapshot()
)
record = physical.get("record")
with self._lock:
provisioning_active = self._provisioning_active
@@ -5910,14 +5990,44 @@ class XgridsK1CompatibilityService:
fresh_devices = self._fresh_ble_devices_locked()
discovery_generation = self._ble_discovery_generation
generation_floors = dict(self._physical_retirement_reopen_generation_floors)
runtime = self.runtime.snapshot()
control = self._application_control_session.snapshot()
native = ble_runtime_snapshot()
supervisor = self._connection_supervisor.snapshot()
network_ledger = self._network_mutation_ledger.snapshot()
idempotency = self._network_provisioning_idempotency_public_snapshot()
semantic_topology = self._semantic_topology_public_snapshot()
identity_pins = self._device_identity_pin_public_snapshot()
runtime = (
runtime_snapshot if runtime_snapshot is not None else self.runtime.snapshot()
)
control = (
control_session_snapshot
if control_session_snapshot is not None
else self._application_control_session.snapshot()
)
native = (
native_runtime_snapshot
if native_runtime_snapshot is not None
else ble_runtime_snapshot()
)
supervisor = (
supervisor_snapshot
if supervisor_snapshot is not None
else self._connection_supervisor.snapshot()
)
network_ledger = (
network_ledger_snapshot
if network_ledger_snapshot is not None
else self._network_mutation_ledger.snapshot()
)
idempotency = (
provisioning_idempotency_snapshot
if provisioning_idempotency_snapshot is not None
else self._network_provisioning_idempotency_public_snapshot()
)
semantic_topology = (
semantic_topology_snapshot
if semantic_topology_snapshot is not None
else self._semantic_topology_public_snapshot()
)
identity_pins = (
identity_pin_snapshot
if identity_pin_snapshot is not None
else self._device_identity_pin_public_snapshot()
)
active_retirements = (
self._active_physical_retirement_documents(record)
@@ -6925,6 +7035,45 @@ class XgridsK1CompatibilityService:
binding_key,
)
def _control_session_snapshot_without_physical_command(self) -> dict[str, object]:
"""Read volatile control facts without recursively reloading the ledger."""
snapshotter = getattr(
self._application_control_session,
"snapshot_control_only",
None,
)
snapshot = (
snapshotter()
if callable(snapshotter)
else self._application_control_session.snapshot()
)
if not isinstance(snapshot, Mapping):
raise TypeError("application control session snapshot must be a mapping")
return dict(snapshot)
def _control_session_snapshot_with_physical_command(
self,
physical_command: Mapping[str, object],
*,
fallback_snapshot: Mapping[str, object],
) -> dict[str, object]:
"""Publish one control/physical join without a second ledger reload."""
snapshotter = getattr(
self._application_control_session,
"snapshot_with_physical_command",
None,
)
if callable(snapshotter):
snapshot = snapshotter(physical_command)
if not isinstance(snapshot, Mapping):
raise TypeError("application control session snapshot must be a mapping")
return dict(snapshot)
snapshot = dict(fallback_snapshot)
snapshot["physical_command"] = dict(physical_command)
return snapshot
def require_snapshot_runtime_id(self, expected_snapshot_runtime_id: str) -> None:
"""Reject an action issued by a browser bound to an older service.
@@ -6943,7 +7092,7 @@ class XgridsK1CompatibilityService:
self._retry_pending_local_control_retirement()
application_control = self._application_control.snapshot().as_dict()
application_control_session = _application_control_session_public_snapshot(
self._application_control_session.snapshot()
self._control_session_snapshot_without_physical_command()
)
runtime = self.runtime.snapshot()
camera_preview = self.camera_preview.snapshot()
@@ -6952,7 +7101,7 @@ class XgridsK1CompatibilityService:
self._reconcile_connection_supervisor(application_control_session, runtime)
if self._retire_orphaned_prestart_control_owner():
application_control_session = _application_control_session_public_snapshot(
self._application_control_session.snapshot()
self._control_session_snapshot_without_physical_command()
)
self._reconcile_application_control_process_lease(application_control_session)
# Preserve the terminal pre-START proof long enough to retire a local
@@ -6998,7 +7147,7 @@ class XgridsK1CompatibilityService:
# owner to idle. Continue the same atomic snapshot from that factual
# state instead of returning the stale pre-retirement failure row.
application_control_session = _application_control_session_public_snapshot(
self._application_control_session.snapshot()
self._control_session_snapshot_without_physical_command()
)
self._reconcile_acquisition(
runtime,
@@ -7013,12 +7162,12 @@ class XgridsK1CompatibilityService:
# control owner. Publish that factual post-reduction state in the same
# snapshot so passive BLE/read-only recovery is available immediately.
application_control_session = _application_control_session_public_snapshot(
self._application_control_session.snapshot()
self._control_session_snapshot_without_physical_command()
)
self._reconcile_application_control_process_lease(application_control_session)
if self._retire_terminal_prestart_control_failure():
application_control_session = _application_control_session_public_snapshot(
self._application_control_session.snapshot()
self._control_session_snapshot_without_physical_command()
)
self._reconcile_application_control_process_lease(application_control_session)
runtime = self.runtime.snapshot()
@@ -7038,14 +7187,35 @@ class XgridsK1CompatibilityService:
active_acquisition_checkpoint = (
self._active_acquisition_checkpoint_public_snapshot()
)
operator_retirement = self._physical_operator_retirement_projection()
ble_runtime = ble_runtime_snapshot()
operator_retirement = self._physical_operator_retirement_projection(
physical_command_snapshot=physical_command,
runtime_snapshot=runtime,
control_session_snapshot=application_control_session,
native_runtime_snapshot=ble_runtime,
)
physical_command["operator_retirement"] = operator_retirement
physical_command["operator_retirement_allowed"] = operator_retirement["allowed"]
physical_command["operator_retirement_reason_codes"] = operator_retirement["reason_codes"]
physical_command["operator_reconciliation_reopen"] = (
self._physical_operator_reconciliation_reopen_projection()
self._physical_operator_reconciliation_reopen_projection(
physical_command_snapshot=physical_command,
runtime_snapshot=runtime,
control_session_snapshot=application_control_session,
native_runtime_snapshot=ble_runtime,
supervisor_snapshot=supervisor_snapshot,
network_ledger_snapshot=ledger_snapshot,
provisioning_idempotency_snapshot=idempotency_snapshot,
semantic_topology_snapshot=semantic_topology_store,
identity_pin_snapshot=device_identity_pin_store,
)
)
application_control_session = _application_control_session_public_snapshot(
self._control_session_snapshot_with_physical_command(
physical_command,
fallback_snapshot=application_control_session,
)
)
ble_runtime = ble_runtime_snapshot()
with self._lock:
operation_phase = self._operation_phase
operation_message = self._operation_message
@@ -7318,11 +7488,27 @@ class XgridsK1CompatibilityService:
intent="select-device",
transition_gate_owned=False,
active_binding=active_binding,
runtime_snapshot=runtime,
physical_command_snapshot=physical_command,
control_session_snapshot=application_control_session,
native_runtime_snapshot=ble_runtime,
semantic_topology_snapshot=semantic_topology_store,
identity_pin_snapshot=device_identity_pin_store,
provisioning_idempotency_snapshot=idempotency_snapshot,
network_ledger_snapshot=ledger_snapshot,
)
change_network_reasons = self._connection_reconfiguration_safety_reasons(
intent="change-network",
transition_gate_owned=False,
active_binding=active_binding,
runtime_snapshot=runtime,
physical_command_snapshot=physical_command,
control_session_snapshot=application_control_session,
native_runtime_snapshot=ble_runtime,
semantic_topology_snapshot=semantic_topology_store,
identity_pin_snapshot=device_identity_pin_store,
provisioning_idempotency_snapshot=idempotency_snapshot,
network_ledger_snapshot=ledger_snapshot,
)
def reconfiguration_decision(
@@ -14622,8 +14808,13 @@ class XgridsK1CompatibilityService:
# deliberately bypasses this ordinary idempotence fast path.
return
camera_admission_started = time.monotonic()
physical = self._physical_command_coordinator.snapshot()
control = self._application_control_session.snapshot()
# ``physical`` above is the authoritative durable read for this
# admission attempt. Control facts are process-local; asking the
# public control snapshot for them would recursively reload the same
# physical ledger while this 10 Hz publisher callback is blocked.
control = self._control_session_snapshot_without_physical_command()
if control_mode == "plugin-commanded":
start_operation_id = (
@@ -14753,6 +14944,17 @@ class XgridsK1CompatibilityService:
time.monotonic() + CAMERA_POST_PCL_ACTIVATION_RETRY_SECONDS
)
raise
logger.info(
"K1 first-PCL camera admission timing",
extra={
"event_code": "k1_camera_pcl_admission_timing",
"evidence_session_id": out_dir.name,
"camera_pcl_admission_ms": int(
round((time.monotonic() - camera_admission_started) * 1_000)
),
"device_command_sent": False,
},
)
def _reset_live_data_plane_observation(self) -> None:
"""Make a stopped/replaced producer incapable of authorizing new data."""
@@ -22237,7 +22439,10 @@ class XgridsK1CompatibilityService:
local_start_operation_id=start_operation_id,
)
)
control = self._application_control_session.snapshot()
# Physical authority was loaded explicitly just above. Do not
# recursively reload it through the control projection while the
# first-PCL camera worker is trying to leave the ingress path.
control = self._control_session_snapshot_without_physical_command()
verified_control_value = control.get("verified_control")
verified_control = (
cast(Mapping[str, Any], verified_control_value)
@@ -22299,28 +22504,37 @@ class XgridsK1CompatibilityService:
camera,
evidence_session_id=out_dir.name,
)
activation_claims: list[_CameraProducerActivationClaim] = []
def reserve_activation(reserve: Callable[[], bool]) -> bool:
claim = self._reserve_camera_producer_activation_if_still_active(
acquisition_id=acquisition.acquisition_id,
evidence_session_id=out_dir.name,
start_operation_id=start_operation_id,
runtime_producer_generation=expected_runtime_generation,
reserve=reserve,
)
if claim is None:
return False
activation_claims.append(claim)
return True
def commit_activation(commit: Callable[[], bool]) -> bool:
if len(activation_claims) != 1:
return False
return self._commit_reserved_camera_producer_activation(
activation_claims[0],
commit,
)
if partial_same_session:
camera = self.camera_preview.retry_recording_producer(
DEFAULT_ACQUISITION_CAMERA_SOURCE,
target,
expected_generation=cast(int, camera["generation"]),
expected_recording_session=out_dir.name,
pre_retry_fence=lambda reserve: (
self._reserve_camera_restart_if_still_active(
acquisition_id=acquisition.acquisition_id,
evidence_session_id=out_dir.name,
start_operation_id=start_operation_id,
runtime_producer_generation=expected_runtime_generation,
reserve=reserve,
)
),
commit_fence=lambda commit: self._commit_camera_restart_if_still_active(
acquisition_id=acquisition.acquisition_id,
evidence_session_id=out_dir.name,
start_operation_id=start_operation_id,
runtime_producer_generation=expected_runtime_generation,
commit=commit,
),
pre_retry_fence=reserve_activation,
commit_fence=commit_activation,
committed_before_start=lambda committed: (
self._bind_live_perception_camera(out_dir.name, committed)
),
@@ -22330,13 +22544,8 @@ class XgridsK1CompatibilityService:
DEFAULT_ACQUISITION_CAMERA_SOURCE,
target,
out_dir,
commit_fence=lambda commit: self._commit_camera_restart_if_still_active(
acquisition_id=acquisition.acquisition_id,
evidence_session_id=out_dir.name,
start_operation_id=start_operation_id,
runtime_producer_generation=expected_runtime_generation,
commit=commit,
),
pre_prepare_fence=reserve_activation,
commit_fence=commit_activation,
committed_before_start=lambda committed: (
self._bind_live_perception_camera(out_dir.name, committed)
),
@@ -24378,6 +24587,104 @@ class XgridsK1CompatibilityService:
action=reserve,
)
def _reserve_camera_producer_activation_if_still_active(
self,
*,
acquisition_id: str,
evidence_session_id: str,
start_operation_id: str | None,
runtime_producer_generation: int,
reserve: Callable[[], bool],
) -> _CameraProducerActivationClaim | None:
"""Validate durable authority before Popen and freeze its local CAS."""
claims: list[_CameraProducerActivationClaim] = []
def reserve_and_capture() -> bool:
if not reserve():
return False
with self._lock:
claims.append(
_CameraProducerActivationClaim(
acquisition_id=acquisition_id,
evidence_session_id=evidence_session_id,
start_operation_id=start_operation_id,
stop_operation_id=self._acquisition_stop_operation_id,
runtime_producer_generation=runtime_producer_generation,
recovery_generation=self._active_stream_recovery_generation,
)
)
return True
accepted = self._camera_restart_action_if_still_active(
acquisition_id=acquisition_id,
evidence_session_id=evidence_session_id,
start_operation_id=start_operation_id,
runtime_producer_generation=runtime_producer_generation,
action=reserve_and_capture,
)
if not accepted:
return None
if len(claims) != 1:
raise RuntimeError("camera activation authority was not reserved exactly once")
return claims[0]
def _commit_reserved_camera_producer_activation(
self,
claim: _CameraProducerActivationClaim,
commit: Callable[[], bool],
) -> bool:
"""Commit a prepared FFmpeg candidate without durable reads or device I/O.
Durable physical/control/checkpoint authority was proved before Popen.
After Popen, a new STOP, acquisition replacement, runtime replacement or
recovery generation invalidates that frozen proof. The remaining gate
is deliberately in-memory so stdout/stderr readers start immediately.
"""
with self._camera_restart_commit_gate:
if self._camera_stop_priority_counts.get(claim.acquisition_id, 0) > 0:
return False
with self._acquisition_lifecycle_access():
runtime = self.runtime.snapshot()
with self._lock:
acquisition = self._acquisition
out_dir = self._acquisition_out_dir
current_start_operation_id = self._acquisition_start_operation_id
current_stop_operation_id = self._acquisition_stop_operation_id
recovery_generation = self._active_stream_recovery_generation
recovery_state = self._active_stream_recovery_state
local_current = bool(
acquisition is not None
and acquisition.acquisition_id == claim.acquisition_id
and acquisition.state
in {"starting", "awaiting_external_start", "acquiring"}
and out_dir is not None
and out_dir.name == claim.evidence_session_id
and current_start_operation_id
in {None, claim.start_operation_id}
and current_stop_operation_id == claim.stop_operation_id
and recovery_generation == claim.recovery_generation
and recovery_state
not in {
"reconnecting",
"blocked",
"fault",
"force-finishing",
"force-finished",
}
)
runtime_current = bool(
runtime.get("phase") == "live"
and runtime.get("source_mode") == "live"
and runtime.get("source_ready") is True
and runtime.get("producer_generation")
== claim.runtime_producer_generation
)
if not local_current or not runtime_current:
return False
return commit()
def _commit_camera_restart_if_still_active(
self,
*,
@@ -24421,7 +24728,10 @@ class XgridsK1CompatibilityService:
# only bounded in-process lineage checks and the camera-local commit
# remain.
physical = self._physical_command_coordinator.snapshot()
control = self._application_control_session.snapshot()
# The exact physical proof is already frozen for this pre-Popen
# fence. Only volatile MQTT dialogue facts are needed here; a nested
# physical reload adds no authority and delays camera reader startup.
control = self._control_session_snapshot_without_physical_command()
verified_control_value = control.get("verified_control")
verified_control = (
cast(Mapping[str, Any], verified_control_value)
src/k1link/device_plugins/xgrids_k1/physical_command_ledger.py
+135 -1
View File
@@ -960,6 +960,27 @@ class PhysicalCommandLedgerSnapshot:
return False
_PhysicalCommandStatFingerprint = tuple[int, int, int, int, int, int, int, int, int]
@dataclass(frozen=True, slots=True)
class _PhysicalCommandLedgerFilesystemFingerprint:
"""Cheap identity fence for one already-verified read-only snapshot.
The physical ledger remains fully reloaded for every mutation and proof.
Passive snapshots may reuse the in-memory, cryptographically verified
document only while the ledger directory, main file, archive directory and
every bounded archive entry retain their exact filesystem identities.
``ctime_ns`` deliberately prevents a same-size rewrite with restored mtime
from bypassing the fence.
"""
ledger_directory: _PhysicalCommandStatFingerprint | None
ledger_file: _PhysicalCommandStatFingerprint | None
archive_directory: _PhysicalCommandStatFingerprint | None
archive_entries: tuple[tuple[str, _PhysicalCommandStatFingerprint], ...]
def active_operator_retirements(
record: PhysicalCommandRecord,
) -> tuple[PhysicalCommandOperatorRetirement, ...]:
@@ -1255,12 +1276,24 @@ class PhysicalCommandLedger:
self._record: PhysicalCommandRecord | None = None
self._archive_history = _PhysicalCommandArchiveHistory()
self._corrupt = False
self._snapshot_filesystem_fingerprint: (
_PhysicalCommandLedgerFilesystemFingerprint | None
) = None
with self._lock, self._process_lock_locked():
self._reload_locked()
def snapshot(self) -> PhysicalCommandLedgerSnapshot:
with self._lock, self._process_lock_locked():
self._reload_locked()
current_fingerprint = self._snapshot_fingerprint_locked()
if (
current_fingerprint is None
or current_fingerprint != self._snapshot_filesystem_fingerprint
):
self._reload_locked()
if not self._corrupt:
self._snapshot_filesystem_fingerprint = (
self._snapshot_fingerprint_locked()
)
if self._corrupt:
return PhysicalCommandLedgerSnapshot(
status="corrupt",
@@ -3072,6 +3105,7 @@ class PhysicalCommandLedger:
self._record = plan.record
self._archive_history = plan.history
self._corrupt = False
self._snapshot_filesystem_fingerprint = None
return plan.record
def _publish_archive_plan_locked(
@@ -3157,6 +3191,10 @@ class PhysicalCommandLedger:
)
def _reload_locked(self) -> None:
# Every command/proof path calls the full loader. It must invalidate a
# passive snapshot fingerprint even when the subsequent transition is
# rejected, so the next read establishes a fresh verified generation.
self._snapshot_filesystem_fingerprint = None
try:
parent = self.path.parent.lstat()
_require_private_directory_metadata(parent, label="ledger directory")
@@ -3223,6 +3261,86 @@ class PhysicalCommandLedger:
self._archive_history = archive_history
self._corrupt = False
def _snapshot_fingerprint_locked(
self,
) -> _PhysicalCommandLedgerFilesystemFingerprint | None:
"""Capture a bounded metadata generation or decline cache reuse.
Failure never authorizes stale state: callers fall back to the complete
loader, whose established behavior is to mark unsafe evidence corrupt.
The process-wide flock is already held by every caller, serializing all
cooperating writers across Mission Core processes.
"""
try:
try:
ledger_directory_metadata = self.path.parent.lstat()
except FileNotFoundError:
ledger_directory_metadata = None
if ledger_directory_metadata is not None:
_require_private_directory_metadata(
ledger_directory_metadata,
label="ledger directory",
)
try:
ledger_metadata = self.path.lstat()
except FileNotFoundError:
ledger_metadata = None
if ledger_metadata is not None:
_require_private_regular_file(
ledger_metadata,
label="ledger",
empty=False,
)
if ledger_metadata.st_size > PHYSICAL_COMMAND_LEDGER_MAX_BYTES:
return None
try:
archive_directory_metadata = self._archive_dir.lstat()
except FileNotFoundError:
archive_directory_metadata = None
archive_entries: list[
tuple[str, _PhysicalCommandStatFingerprint]
] = []
if archive_directory_metadata is not None:
_require_private_directory_metadata(
archive_directory_metadata,
label="archive directory",
)
with os.scandir(self._archive_dir) as entries:
for entry in entries:
if len(archive_entries) >= PHYSICAL_COMMAND_ARCHIVE_MAX_SEGMENTS + 2:
return None
metadata = entry.stat(follow_symlinks=False)
if not stat.S_ISREG(metadata.st_mode):
return None
archive_entries.append(
(entry.name, _physical_command_stat_fingerprint(metadata))
)
archive_entries.sort(key=lambda item: item[0])
return _PhysicalCommandLedgerFilesystemFingerprint(
ledger_directory=(
_physical_command_stat_fingerprint(ledger_directory_metadata)
if ledger_directory_metadata is not None
else None
),
ledger_file=(
_physical_command_stat_fingerprint(ledger_metadata)
if ledger_metadata is not None
else None
),
archive_directory=(
_physical_command_stat_fingerprint(archive_directory_metadata)
if archive_directory_metadata is not None
else None
),
archive_entries=tuple(archive_entries),
)
except (OSError, ValueError):
return None
def _load_archive_history_locked(
self,
record: PhysicalCommandRecord,
@@ -3233,6 +3351,22 @@ class PhysicalCommandLedger:
)
def _physical_command_stat_fingerprint(
value: os.stat_result,
) -> _PhysicalCommandStatFingerprint:
return (
value.st_dev,
value.st_ino,
value.st_mode,
value.st_uid,
value.st_gid,
value.st_nlink,
value.st_size,
value.st_mtime_ns,
value.st_ctime_ns,
)
def _require_safe_edge_successor(
previous: PhysicalCommandRecord | None,
*,
src/k1link/device_plugins/xgrids_k1/protocol/application_session.py
+84 -35
View File
@@ -661,6 +661,14 @@ class InteractiveApplicationControlSession:
)
self._set_phase_locked("stop-requested")
self._stop_requested.set()
logger.info(
"K1 STOP timing checkpoint",
extra={
"event_code": "k1_stop_dispatch_timing",
"operation_stage": "prepared-worker-released",
"device_command_sent": False,
},
)
except BaseException as preparation_error:
if coordinator is not None:
try:
@@ -781,41 +789,66 @@ class InteractiveApplicationControlSession:
def snapshot(self) -> dict[str, object]:
with self._lock:
transport_snapshot = self._live_transport_snapshot_locked()
phase = self._phase
return {
"mode": "interactive-canonical",
"inspection_only": self._inspection_only,
"inspection_promotion_allowed": self._inspection_promotion_allowed,
"state": phase,
"session_generation": self._run_generation,
"state_revision": self._state_revision,
"control_socket_open": phase not in {"idle", "completed", "closed", "failed"},
"can_open": self._can_open_locked(),
"can_enter_workspace": phase == "connection-ready",
"can_prepare_project": phase == "workspace-ready",
"can_start": phase == "project-ready",
"can_stop": phase == "scanning",
# Retained for wire compatibility with the v1alpha2 snapshot.
# Standby is now concluded solely from live K1 protocol state.
"can_confirm_standby": False,
"pending_operator_action": self._pending_operator_action_locked(),
"scripted_transitions": False,
"automatic_retry": False,
"outcome_unknown": self._outcome_unknown,
"scanning_observer_errors": self._scanning_observer_errors,
"failure": dict(self._failure) if self._failure is not None else None,
"dialogue": (
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
),
"transport": transport_snapshot,
"verified_control": (self._verified_control_snapshot_locked(transport_snapshot)),
"physical_command": (
self._physical_command_coordinator.snapshot()
if self._physical_command_coordinator is not None
else None
),
}
physical_command = (
self._physical_command_coordinator.snapshot()
if self._physical_command_coordinator is not None
else None
)
return self._snapshot_locked(physical_command=physical_command)
def snapshot_control_only(self) -> dict[str, object]:
"""Return process-local control facts without reloading the durable ledger."""
with self._lock:
return self._snapshot_locked(physical_command=None)
def snapshot_with_physical_command(
self,
physical_command: Mapping[str, object] | None,
) -> dict[str, object]:
"""Join one already-verified physical snapshot to current control facts."""
with self._lock:
return self._snapshot_locked(physical_command=physical_command)
def _snapshot_locked(
self,
*,
physical_command: Mapping[str, object] | None,
) -> dict[str, object]:
transport_snapshot = self._live_transport_snapshot_locked()
phase = self._phase
return {
"mode": "interactive-canonical",
"inspection_only": self._inspection_only,
"inspection_promotion_allowed": self._inspection_promotion_allowed,
"state": phase,
"session_generation": self._run_generation,
"state_revision": self._state_revision,
"control_socket_open": phase not in {"idle", "completed", "closed", "failed"},
"can_open": self._can_open_locked(),
"can_enter_workspace": phase == "connection-ready",
"can_prepare_project": phase == "workspace-ready",
"can_start": phase == "project-ready",
"can_stop": phase == "scanning",
# Retained for wire compatibility with the v1alpha2 snapshot.
# Standby is now concluded solely from live K1 protocol state.
"can_confirm_standby": False,
"pending_operator_action": self._pending_operator_action_locked(),
"scripted_transitions": False,
"automatic_retry": False,
"outcome_unknown": self._outcome_unknown,
"scanning_observer_errors": self._scanning_observer_errors,
"failure": dict(self._failure) if self._failure is not None else None,
"dialogue": (
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
),
"transport": transport_snapshot,
"verified_control": self._verified_control_snapshot_locked(transport_snapshot),
"physical_command": (
dict(physical_command) if physical_command is not None else None
),
}
def _run(self, generation: int) -> None:
executor: PhysicalAcceptanceDialogueExecutor | None = None
@@ -1028,6 +1061,14 @@ class InteractiveApplicationControlSession:
stop_dispatch_admission_deadline_reached
)
self._set_phase("stopping")
logger.info(
"K1 STOP timing checkpoint",
extra={
"event_code": "k1_stop_dispatch_timing",
"operation_stage": "pre-dispatch-validation-complete",
"device_command_sent": False,
},
)
executor.execute_canonical_stop(
stop_command,
stop_permit,
@@ -1038,6 +1079,14 @@ class InteractiveApplicationControlSession:
stop_dispatch_admission_deadline_reached
),
)
logger.info(
"K1 STOP timing checkpoint",
extra={
"event_code": "k1_stop_dispatch_timing",
"operation_stage": "correlated-application-response",
"device_command_sent": True,
},
)
self._validate_connection_binding_snapshot("stop-post-response")
self._set_phase("awaiting-standby-confirmation")
executor.maintain_post_stop_until_standby()
src/k1link/web/runtime_diagnostics.py
+6
View File
@@ -70,6 +70,12 @@ _EXTRA_FIELDS: Final = (
"camera_queue_rejection_reason",
"camera_retry_count",
"camera_generation",
"camera_pcl_admission_ms",
"camera_authority_wait_ms",
"camera_ffmpeg_prepare_ms",
"camera_post_spawn_commit_ms",
"camera_activation_total_ms",
"device_command_sent",
"websocket_close_code",
"transport_epoch",
"camera_append_error_name",