fix(k1): harden live handoff and camera recovery

This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 13:09:24 +03:00
parent eaad9deda1
commit 85035fa07b
26 changed files with 1478 additions and 170 deletions
@@ -1873,6 +1873,7 @@ class ActiveAcquisitionRecoveryCheckpointStore:
validation_checkpoint,
status_proof=cessation_status_proof,
physical_proof=cessation_physical_proof,
allow_reconciled_transport_change=True,
)
_require_active_reconciled_standby_shape(
current,
@@ -2251,9 +2252,9 @@ class ActiveAcquisitionRecoveryCheckpointStore:
"""Cease a proven START that became standby before restart PCL.
This is deliberately distinct from ``cease_prepared_reconciled``:
READY/SCAN_OVER proves that the old successful START is no longer
active, so there is no transient ACTIVE checkpoint, capture promotion,
first-PCL receipt, or invented STOP edge.
READY/SCAN_OVER proves that the previously observed active START is no
longer active, so there is no transient ACTIVE checkpoint, capture
promotion, first-PCL receipt, or invented STOP edge.
"""
_validate_mutation_request(
@@ -2351,7 +2352,11 @@ class ActiveAcquisitionRecoveryCheckpointStore:
self._clock(),
floor=current.updated_at_utc,
)
assert origin_proof.original_project_id_sha256 is not None
active_project_id_sha256 = (
origin_proof.original_project_id_sha256
or origin_proof.reconciled_active_project_id_sha256
)
assert active_project_id_sha256 is not None
candidate = replace(
current,
revision=revision,
@@ -2360,9 +2365,7 @@ class ActiveAcquisitionRecoveryCheckpointStore:
physical_lineage_head_revision=(
cessation_physical_proof.ledger_revision
),
active_project_id_sha256=(
origin_proof.original_project_id_sha256
),
active_project_id_sha256=active_project_id_sha256,
current_evidence_session_id=(
cessation_status_proof.evidence_session_id
),
@@ -2614,6 +2617,7 @@ def _require_binding_matches_checkpoint_values(
compatibility_profile_id: str,
binding: ActiveAcquisitionRecoveryTransportBinding,
allow_target_ipv4_change: bool = False,
allow_connection_mode_change: bool = False,
) -> None:
if (
binding.logical_device_id != identity.logical_device_id
@@ -2621,7 +2625,10 @@ def _require_binding_matches_checkpoint_values(
or binding.device_serial_sha256 != identity.device_serial_sha256
or binding.compatibility_profile_id != compatibility_profile_id
or binding.transport_ref != connection.transport_ref
or binding.connection_mode != connection.connection_mode
or (
not allow_connection_mode_change
and binding.connection_mode != connection.connection_mode
)
or (
not allow_target_ipv4_change
and binding.target_ipv4 != connection.target_ipv4
@@ -2684,6 +2691,7 @@ def _require_physical_lineage_base(
proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
*,
allow_target_ipv4_change: bool = False,
allow_connection_mode_change: bool = False,
) -> None:
if (
proof.acquisition_id != checkpoint.acquisition_id
@@ -2706,6 +2714,7 @@ def _require_physical_lineage_base(
and proof.binding == checkpoint.current_binding
)
),
allow_connection_mode_change=allow_connection_mode_change,
)
@@ -2994,22 +3003,40 @@ def _require_prepared_resolved_start_standby(
physical_proof=cessation_physical_proof,
)
baseline = origin_proof.baseline_status_proof
if not (
exact_origin = bool(
origin_proof.origin_kind == "composite-resolved"
and origin_proof.operation_id == checkpoint.original_start_operation_id
and origin_proof.acquisition_id == checkpoint.acquisition_id
and origin_proof.payload_sha256 == checkpoint.start_payload_sha256
and origin_proof.original_attempt_stage == "resolved"
and origin_proof.original_attempt_resolution == "start-active-observed"
and origin_proof.original_project_id_sha256 is not None
and origin_proof.reconciled_active_project_id_sha256
in {None, origin_proof.original_project_id_sha256}
and origin_proof.project_evidence_strength == "exact-vendor-project-id"
)
ambiguous_origin = bool(
origin_proof.origin_kind == "ambiguous-reconciled"
and origin_proof.original_attempt_stage in {"dispatching", "observing"}
and origin_proof.original_attempt_resolution is None
and origin_proof.original_project_id_sha256 is None
and origin_proof.reconciled_active_project_id_sha256 is not None
and origin_proof.project_evidence_strength
== "edge-correlated-vendor-project-id"
)
settlement_project_id_sha256 = (
origin_proof.original_project_id_sha256
or origin_proof.reconciled_active_project_id_sha256
)
if not (
(exact_origin or ambiguous_origin)
and origin_proof.operation_id == checkpoint.original_start_operation_id
and origin_proof.acquisition_id == checkpoint.acquisition_id
and origin_proof.payload_sha256 == checkpoint.start_payload_sha256
and origin_proof.automatic_replay_allowed is False
and origin_proof.ledger_revision == cessation_physical_proof.ledger_revision
and origin_proof.physical_proof_id == cessation_physical_proof.proof_id
and origin_proof.reconciliation_id == reconciliation_id
and origin_proof.original_attempt_sha256
== reconciliation_original_attempt_sha256
and origin_proof.original_project_id_sha256
and settlement_project_id_sha256
== reconciliation_original_project_id_sha256
and baseline.binding == checkpoint.prepared_binding
and baseline.evidence_session_id
@@ -3029,7 +3056,7 @@ def _require_prepared_resolved_start_standby(
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
"restart standby settlement requires new runtime, control and evidence sessions"
)
if not (
exact_terminal_lineage = bool(
cessation_physical_proof.operation_id
== checkpoint.original_start_operation_id
and cessation_physical_proof.action == "start"
@@ -3044,9 +3071,30 @@ def _require_prepared_resolved_start_standby(
== "physical-standby-observed"
and cessation_physical_proof.composite_complete
and cessation_physical_proof.stop_fence == "none"
)
ambiguous_terminal_lineage = bool(
cessation_physical_proof.operation_id
== checkpoint.original_start_operation_id
and cessation_physical_proof.action == "start"
and cessation_physical_proof.resolution
== "physical-standby-observed"
and cessation_physical_proof.payload_sha256
== checkpoint.start_payload_sha256
and cessation_physical_proof.original_start_payload_sha256
== checkpoint.start_payload_sha256
and cessation_physical_proof.reconciliation_kind
== "resolved-active-cessation"
and cessation_physical_proof.reconciliation_resolution
== "physical-standby-observed"
and not cessation_physical_proof.composite_complete
and cessation_physical_proof.stop_fence == "none"
)
if not (
(exact_origin and exact_terminal_lineage)
or (ambiguous_origin and ambiguous_terminal_lineage)
):
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
"resolved START standby settlement lacks exact terminal lineage"
"reconciled START standby settlement lacks exact terminal lineage"
)
if _validated_timestamp(
cessation_status_proof.observed_at_utc,
@@ -3215,11 +3263,13 @@ def _require_active_cessation(
*,
status_proof: ActiveAcquisitionRecoveryStatusProof,
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
allow_reconciled_transport_change: bool = False,
) -> None:
_require_active_cessation_shape(
checkpoint,
status_proof=status_proof,
physical_proof=physical_proof,
allow_reconciled_transport_change=allow_reconciled_transport_change,
)
if (
checkpoint.physical_lineage_head_revision is None
@@ -3236,12 +3286,23 @@ def _require_active_cessation_shape(
*,
status_proof: ActiveAcquisitionRecoveryStatusProof,
physical_proof: ActiveAcquisitionRecoveryPhysicalLineageProof,
allow_reconciled_transport_change: bool = False,
) -> None:
if status_proof.session_state not in {"ready", "scan_over"}:
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
"active cessation requires fresh READY/SCAN_OVER"
)
_require_status_binding(checkpoint, status_proof)
if allow_reconciled_transport_change:
_require_binding_matches_checkpoint_values(
identity=checkpoint.identity,
connection=checkpoint.connection,
compatibility_profile_id=checkpoint.compatibility_profile_id,
binding=status_proof.binding,
allow_target_ipv4_change=True,
allow_connection_mode_change=True,
)
else:
_require_status_binding(checkpoint, status_proof)
_require_matching_observed_proofs(
status_proof=status_proof,
physical_proof=physical_proof,
@@ -3342,8 +3403,24 @@ def _require_active_reconciled_standby_shape(
raise ActiveAcquisitionRecoveryCheckpointTransitionError(
"restart standby settlement requires fresh READY/SCAN_OVER"
)
_require_status_binding(checkpoint, cessation_status_proof)
_require_physical_lineage_base(checkpoint, cessation_physical_proof)
# This is a terminal, read-only settlement of the same pinned K1. The
# scanner may become observable through Bridge after a Quick Connect STOP
# became ambiguous (or vice versa), so the verified route may change
# without granting START, STOP, replay, or network-mutation authority.
_require_binding_matches_checkpoint_values(
identity=checkpoint.identity,
connection=checkpoint.connection,
compatibility_profile_id=checkpoint.compatibility_profile_id,
binding=cessation_status_proof.binding,
allow_target_ipv4_change=True,
allow_connection_mode_change=True,
)
_require_physical_lineage_base(
checkpoint,
cessation_physical_proof,
allow_target_ipv4_change=True,
allow_connection_mode_change=True,
)
_require_matching_observed_proofs(
status_proof=cessation_status_proof,
physical_proof=cessation_physical_proof,
@@ -4642,6 +4719,12 @@ def _validate_checkpoint_semantics(
checkpoint.state != "prepared"
and checkpoint.last_gap_recovered_at_utc is not None
),
allow_connection_mode_change=(
checkpoint.state == "ceased"
and bool(checkpoint.transition_receipts)
and checkpoint.transition_receipts[-1].kind
== "cease-active-reconciled-standby"
),
)
if checkpoint.last_gap_failed_binding is not None:
_require_binding_matches_checkpoint_values(
@@ -4944,7 +5027,16 @@ def _validate_checkpoint_semantics(
cessation_physical = checkpoint.cessation_physical_proof
if checkpoint.physical_lineage_head_revision != cessation_physical.ledger_revision:
raise ValueError("ceased physical lineage head does not match cessation proof")
_require_physical_lineage_base(checkpoint, cessation_physical)
_require_physical_lineage_base(
checkpoint,
cessation_physical,
allow_target_ipv4_change=(
receipts[-1].kind == "cease-active-reconciled-standby"
),
allow_connection_mode_change=(
receipts[-1].kind == "cease-active-reconciled-standby"
),
)
if resolved_start_reconciliation_id is not None:
if not (
checkpoint.prepared_resolution_proof == cessation_physical
@@ -4983,7 +5075,10 @@ def _validate_checkpoint_semantics(
checkpoint.current_evidence_session_id
!= checkpoint.cessation_status_proof.evidence_session_id
or checkpoint.active_project_id_sha256
!= origin_proof.original_project_id_sha256
!= (
origin_proof.original_project_id_sha256
or origin_proof.reconciled_active_project_id_sha256
)
or receipts[-1].kind
!= "cease-prepared-resolved-start-standby"
):
@@ -5048,6 +5143,9 @@ def _validate_checkpoint_semantics(
checkpoint,
status_proof=status,
physical_proof=cessation_physical,
allow_reconciled_transport_change=(
receipts[-1].kind == "cease-active-reconciled-standby"
),
)
if checkpoint.current_evidence_session_id != status.evidence_session_id:
raise ValueError("active cessation evidence session is inconsistent")
+54 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import logging
import os
import queue
import signal
@@ -27,6 +28,8 @@ from k1link.web.camera_archive import (
CameraSourceId = Literal["sensor.camera.left", "sensor.camera.right"]
logger = logging.getLogger(__name__)
CAMERA_SOURCE_PATHS: Final[dict[CameraSourceId, str]] = {
"sensor.camera.left": "/live/chn_left_main",
"sensor.camera.right": "/live/chn_right_main",
@@ -48,7 +51,11 @@ MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS: Final = 64
# media fragment. This remains a strict per-reader bound and matches the
# frontend's reviewed 12 MiB receive envelope.
MAX_CAMERA_PREVIEW_QUEUED_BYTES: Final = 12 * 1024 * 1024
MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: Final = 3.0
# The browser can pause WebSocket consumption for just over three seconds while
# the live 3D workspace commits a large reactive update. The byte and segment
# caps above remain the hard memory/latency fence; this age fence only avoids
# discarding an otherwise healthy preview at that measured UI pause boundary.
MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS: Final = 6.0
MAX_CAMERA_PREVIEW_CONSUMERS: Final = 8
CAMERA_PREVIEW_SEND_TIMEOUT_SECONDS: Final = 3.0
CAMERA_DRAIN_TIMEOUT_SECONDS: Final = 5.0
@@ -143,6 +150,7 @@ class _CameraPreviewSegmentQueue:
self._queued_bytes = 0
self._closed = False
self._clock = clock
self._last_rejection_reason: str | None = None
@property
def queued_bytes(self) -> int:
@@ -158,14 +166,19 @@ class _CameraPreviewSegmentQueue:
payload_size = len(segment[1])
with self._condition:
if self._closed:
self._last_rejection_reason = "queue-closed"
return False
now = self._clock()
if self._segments and now - self._segments[0][0] > MAX_CAMERA_PREVIEW_QUEUE_AGE_SECONDS:
self._last_rejection_reason = "queue-age"
return False
if len(self._segments) >= MAX_CAMERA_PREVIEW_QUEUED_SEGMENTS:
self._last_rejection_reason = "queue-segments"
return False
if self._queued_bytes + payload_size > MAX_CAMERA_PREVIEW_QUEUED_BYTES:
self._last_rejection_reason = "queue-bytes"
return False
self._last_rejection_reason = None
self._segments.append((now, segment))
self._queued_bytes += payload_size
self._condition.notify()
@@ -188,6 +201,23 @@ class _CameraPreviewSegmentQueue:
return segment
return None
def diagnostic_snapshot(self) -> dict[str, int | str | None]:
"""Return bounded observer facts without changing delivery state."""
with self._condition:
now = self._clock()
oldest_age_ms = (
max(0, int((now - self._segments[0][0]) * 1_000))
if self._segments
else 0
)
return {
"queued_bytes": self._queued_bytes,
"queued_segments": len(self._segments),
"oldest_age_ms": oldest_age_ms,
"rejection_reason": self._last_rejection_reason,
}
def close(self) -> None:
with self._condition:
self._segments.clear()
@@ -1269,11 +1299,34 @@ class XgridsK1CameraGateway:
*,
failure_code: str = "consumer-too-slow",
) -> None:
queue_diagnostic = delivery.segments.diagnostic_snapshot()
with self._lock:
if self._producer is producer and producer.deliveries.get(id(delivery)) is delivery:
producer.deliveries.pop(id(delivery), None)
delivery.failure_code = failure_code
self._revision += 1
logger.warning(
"K1 camera preview browser lease retired: generation=%s "
"failure=%s rejection=%s queued_segments=%s queued_bytes=%s "
"oldest_age_ms=%s",
delivery.generation,
failure_code,
queue_diagnostic["rejection_reason"],
queue_diagnostic["queued_segments"],
queue_diagnostic["queued_bytes"],
queue_diagnostic["oldest_age_ms"],
extra={
"event_code": "k1_camera_preview_delivery_retired",
"failure_code": failure_code,
"camera_generation": delivery.generation,
"camera_queue_rejection_reason": queue_diagnostic["rejection_reason"],
"camera_queue_segments": queue_diagnostic["queued_segments"],
"camera_queue_bytes": queue_diagnostic["queued_bytes"],
"camera_queue_oldest_age_ms": queue_diagnostic["oldest_age_ms"],
"device_write_performed": False,
"automatic_retry": False,
},
)
_close_segment_queue(delivery)
def _mark_producer_failure(
+109 -47
View File
@@ -10019,7 +10019,9 @@ class XgridsK1CompatibilityService:
"retryable": terminal.retryable,
"safe_to_retry": terminal.safe_to_retry,
"side_effect_status": (
"none" if terminal.side_effect_status == "none" else "confirmed"
terminal.side_effect_status
if terminal.side_effect_status in {"none", "unknown"}
else "confirmed"
),
"durable_replay": True,
},
@@ -10748,6 +10750,7 @@ class XgridsK1CompatibilityService:
quick_connect_host_profile_id(selected_device_name) if quick_connect else None
)
operation_stage = "device-ap-activation" if quick_connect else "ble-provisioning-write"
quick_host_association: Mapping[str, Any] | None = None
# Resolve the durable identity expectation while the previous
# control session is still intact. A corrupt/unavailable pin store
@@ -11018,6 +11021,7 @@ class XgridsK1CompatibilityService:
self._duplicate_network_process_fence_descriptor
),
)
quick_host_association = association
except HostWifiProfileError as exc:
write_json_atomic(
session_dir / "host-wifi-association.redacted.json",
@@ -11249,8 +11253,12 @@ class XgridsK1CompatibilityService:
# decide whether that separately authorized host mutation is
# necessary at all.
host_route_class: str | None = None
host_association: Mapping[str, Any] | None = None
host_wifi_association_outcome: str | None = None
host_association: Mapping[str, Any] | None = quick_host_association
host_wifi_association_outcome: str | None = (
str(quick_host_association.get("outcome") or "unknown")
if quick_host_association is not None
else None
)
if request.connection_mode == "bridge" and request.allow_host_wifi_switch:
host_route_class = _host_route_class(ipv4)
if (
@@ -11758,11 +11766,6 @@ class XgridsK1CompatibilityService:
and reconciliation.original_attempt.action == "stop"
and reconciliation.original_attempt.stage == "observing"
and reconciliation.original_attempt.resolution is None
and reconciliation.original_attempt.publish_call_returned is True
and reconciliation.original_attempt.packet_id is not None
and reconciliation.original_attempt.qos2_completed
and reconciliation.original_attempt.application_response is not None
and reconciliation.original_attempt.application_response.success
and reconciliation.observation.source
== "explicit-read-only-reconciliation"
and reconciliation.observation.session_state in {"ready", "scan_over"}
@@ -12932,25 +12935,35 @@ class XgridsK1CompatibilityService:
raise ActiveAcquisitionRecoveryCheckpointError(
"ambiguous START origin has the wrong physical classification"
)
if reconciliation.kind == "resolved-active-rebind":
prior_active_project_id_sha256 = next(
(
item.observation.project_id_sha256
for item in reversed(record.reconciliations[:-1])
if item.original_attempt_sha256
== reconciliation.original_attempt_sha256
and item.resolution == "physical-active-observed"
),
None,
prior_active_project_id_sha256 = next(
(
item.observation.project_id_sha256
for item in reversed(record.reconciliations[:-1])
if item.original_attempt_sha256
== reconciliation.original_attempt_sha256
and item.resolution == "physical-active-observed"
and item.observation.session_state == "scanning"
and item.observation.project_bound
and item.observation.init_ready
and not item.observation.mqtt_retained
),
None,
)
if reconciliation.kind == "resolved-active-rebind" and (
prior_active_project_id_sha256 is None
or reconciliation.observation.project_id_sha256
!= prior_active_project_id_sha256
):
raise ActiveAcquisitionRecoveryCheckpointError(
"ambiguous START rebind changed or lacks its edge-correlated project"
)
if (
reconciliation.kind == "resolved-active-cessation"
and prior_active_project_id_sha256 is None
):
raise ActiveAcquisitionRecoveryCheckpointError(
"ambiguous START cessation lacks its prior active project proof"
)
if (
prior_active_project_id_sha256 is None
or reconciliation.observation.project_id_sha256
!= prior_active_project_id_sha256
):
raise ActiveAcquisitionRecoveryCheckpointError(
"ambiguous START rebind changed or lacks its edge-correlated project"
)
origin_kind = "ambiguous-reconciled"
project_strength = "edge-correlated-vendor-project-id"
original_project_id_sha256 = None
@@ -12980,6 +12993,8 @@ class XgridsK1CompatibilityService:
reconciled_active_project_id_sha256=(
reconciliation.observation.project_id_sha256
if active_observation
else prior_active_project_id_sha256
if ambiguous_origin
else None
),
)
@@ -13151,7 +13166,7 @@ class XgridsK1CompatibilityService:
else status.observed_at_utc
)
if checkpoint.state == "active":
store.cease_active_reconciled_standby(
settled_checkpoint = store.cease_active_reconciled_standby(
transition_id=self._active_acquisition_checkpoint_transition_id(
"cease-active-reconciled-standby",
checkpoint.acquisition_id,
@@ -13167,6 +13182,25 @@ class XgridsK1CompatibilityService:
cessation_status_proof=status_proof,
cessation_physical_proof=physical_proof,
)
logger.info(
"active acquisition checkpoint closed from verified standby",
extra={
"event_code": (
"active_acquisition_checkpoint_reconciled_standby_settled"
),
"acquisition_id": checkpoint.acquisition_id,
"checkpoint_revision_before": checkpoint.revision,
"checkpoint_revision_after": settled_checkpoint.revision,
"connection_mode_before": (
checkpoint.current_binding.connection_mode
),
"connection_mode_after": binding.connection_mode,
"reconciliation_id": reconciliation_id,
"observed_session_state": status.session_state,
"device_write_performed": False,
"automatic_retry": False,
},
)
elif checkpoint.state == "prepared" and (
reconciliation.kind == "resolved-active-cessation"
):
@@ -13176,14 +13210,13 @@ class XgridsK1CompatibilityService:
reconciliation=reconciliation,
physical_proof=physical_proof,
)
original_project_id_sha256 = (
reconciliation.original_attempt.last_status.project_id_sha256
if reconciliation.original_attempt.last_status is not None
else None
settlement_project_id_sha256 = (
origin_proof.original_project_id_sha256
or origin_proof.reconciled_active_project_id_sha256
)
if original_project_id_sha256 is None:
if settlement_project_id_sha256 is None:
raise ActiveAcquisitionRecoveryCheckpointError(
"resolved START standby lacks its original vendor project"
"reconciled START standby lacks its proven vendor project"
)
store.cease_prepared_resolved_start_standby(
transition_id=self._active_acquisition_checkpoint_transition_id(
@@ -13206,7 +13239,7 @@ class XgridsK1CompatibilityService:
reconciliation.original_attempt_sha256
),
reconciliation_original_project_id_sha256=(
original_project_id_sha256
settlement_project_id_sha256
),
)
else:
@@ -13361,7 +13394,7 @@ class XgridsK1CompatibilityService:
)
and reconciliation.kind == "prepared-stop-classification"
)
dispatched_then_observed_standby = bool(
ambiguous_stop_observed_standby = bool(
record is not None
and record.resolution == "physical-standby-observed"
and reconciliation is not None
@@ -13371,11 +13404,6 @@ class XgridsK1CompatibilityService:
and reconciliation.original_attempt.action == "stop"
and reconciliation.original_attempt.stage == "observing"
and reconciliation.original_attempt.resolution is None
and reconciliation.original_attempt.publish_call_returned is True
and reconciliation.original_attempt.packet_id is not None
and reconciliation.original_attempt.qos2_completed
and reconciliation.original_attempt.application_response is not None
and reconciliation.original_attempt.application_response.success
)
if not (
ledger_snapshot.status == "resolved"
@@ -13383,7 +13411,7 @@ class XgridsK1CompatibilityService:
and record.action == "stop"
and record.stage == "resolved"
and reconciliation is not None
and (definitely_undispatched or dispatched_then_observed_standby)
and (definitely_undispatched or ambiguous_stop_observed_standby)
and reconciliation.resolution == "physical-standby-observed"
and reconciliation.observation.source
== "explicit-read-only-reconciliation"
@@ -15573,13 +15601,47 @@ class XgridsK1CompatibilityService:
else:
restart_outcome = None
if allow_receiver_rehydrate:
restart_outcome = (
await self._rehydrate_active_acquisition_after_restart(
token=checkpoint_trust_token,
reconciliation_id=reconciliation_id,
reconciled_record=reconciled_record,
try:
restart_outcome = (
await self._rehydrate_active_acquisition_after_restart(
token=checkpoint_trust_token,
reconciliation_id=reconciliation_id,
reconciled_record=reconciled_record,
)
)
except ActiveAcquisitionRecoveryCheckpointError as exc:
# Receiver resurrection is optional after an exact
# read-only SCANNING proof. A stale/corrupt local
# checkpoint must revoke that broader authority, but
# it must not discard the narrower, already durable
# STOP-only authority derived from DeviceInfo plus a
# fresh non-retained DeviceStatus.
reason_code = str(
getattr(
exc,
"reason_code",
(
"active-acquisition-recovery-"
"checkpoint-rehydrate-failed"
),
)
)
self._mark_active_acquisition_checkpoint_untrusted(
trust="unavailable",
reason_code=reason_code,
)
logger.warning(
"K1 checkpoint rejected receiver rehydration; "
"continuing with proven STOP-only control",
extra={
"event_code": (
"k1_restart_rehydration_checkpoint_rejected"
),
"reason_code": reason_code,
"device_write_performed": False,
"automatic_retry": False,
},
)
)
if restart_outcome is None:
checkpoint_lineage = (
self._restart_stop_only_checkpoint_lineage(
@@ -37,11 +37,11 @@ NetworkProvisioningIdempotencyStage = Literal["prepared", "unresolved", "termina
NetworkProvisioningIdempotencyDisposition = Literal["admitted", "terminal-replay"]
NetworkProvisioningIdempotencyStatus = Literal["empty", "ready", "blocked", "corrupt"]
NetworkProvisioningTerminalOutcome = Literal["succeeded", "failed", "cancelled"]
NetworkProvisioningSideEffectStatus = Literal["none", "applied", "reconciled"]
NetworkProvisioningSideEffectStatus = Literal["none", "applied", "reconciled", "unknown"]
_STAGES = frozenset({"prepared", "unresolved", "terminal"})
_OUTCOMES = frozenset({"succeeded", "failed", "cancelled"})
_SIDE_EFFECT_STATUSES = frozenset({"none", "applied", "reconciled"})
_SIDE_EFFECT_STATUSES = frozenset({"none", "applied", "reconciled", "unknown"})
_SAFE_ACTION = re.compile(r"^[a-z][a-z0-9._-]{0,95}$")
_SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:+-]{0,159}$")
_SAFE_CODE = re.compile(r"^[a-z][a-z0-9._-]{0,127}$")
+12
View File
@@ -56,6 +56,7 @@ _EXTRA_FIELDS: Final = (
"stale_session_retired",
"stream_id",
"ui_build_id",
"expected_ui_build_id",
"document_instance_id",
"viewer_instance_id",
"lifecycle_generation",
@@ -63,6 +64,17 @@ _EXTRA_FIELDS: Final = (
"viewer_range_max_ns",
"stalled_for_ms",
"recovery_attempt",
"camera_queue_bytes",
"camera_queue_segments",
"camera_queue_oldest_age_ms",
"camera_queue_rejection_reason",
"camera_retry_count",
"camera_generation",
"websocket_close_code",
"transport_epoch",
"camera_append_error_name",
"camera_media_source_state",
"camera_video_error_code",
"preferred_port",
"selected_port",
)
+55 -2
View File
@@ -1,10 +1,11 @@
from __future__ import annotations
import logging
import re
from collections.abc import Callable
from typing import Literal
from fastapi import APIRouter, Response
from fastapi import APIRouter, Request, Response
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field
@@ -17,12 +18,27 @@ LiveViewerEventCode = Literal[
"live_receiver_recovery_exhausted",
"live_receiver_active_store_admitted",
"live_receiver_error",
"live_camera_transport_restart_requested",
"live_camera_transport_playing",
]
LiveViewerFailureStage = Literal[
"recording-open-timeout",
"viewer-start",
"module-load",
"receiver-stalled",
"camera-first-media-timeout",
"camera-first-playable-timeout",
"camera-queue-capacity",
"camera-append-quota",
"camera-append-error",
"camera-source-buffer-error",
"camera-fragment-decode",
"camera-websocket-error",
"camera-websocket-close",
"camera-heartbeat-reopen",
"camera-visibility-reopen",
"camera-network-online-reopen",
"camera-page-restore-reopen",
]
LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2"
@@ -61,6 +77,17 @@ class LiveViewerDiagnosticEvent(BaseModel):
viewer_range_max_ns: int | None = Field(default=None, ge=0)
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
camera_queue_bytes: int | None = Field(default=None, ge=0, le=16_777_216)
camera_queue_segments: int | None = Field(default=None, ge=0, le=128)
camera_retry_count: int | None = Field(default=None, ge=1, le=4)
websocket_close_code: int | None = Field(default=None, ge=0, le=4_999)
transport_epoch: int | None = Field(default=None, ge=1)
camera_append_error_name: str | None = Field(
default=None,
pattern=r"^[A-Za-z][A-Za-z0-9]{0,63}$",
)
camera_media_source_state: Literal["closed", "open", "ended"] | None = None
camera_video_error_code: int | None = Field(default=None, ge=0, le=4)
def build_viewer_diagnostics_router(
@@ -70,7 +97,7 @@ def build_viewer_diagnostics_router(
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
@router.get("/client-contract")
def get_live_viewer_client_contract() -> JSONResponse:
def get_live_viewer_client_contract(request: Request) -> JSONResponse:
expected = expected_ui_build_id()
if expected is None:
return JSONResponse(
@@ -81,6 +108,24 @@ def build_viewer_diagnostics_router(
},
headers={"Cache-Control": "no-store"},
)
loaded = request.headers.get(UI_BUILD_HEADER)
if (
loaded not in {None, "development", expected}
and re.fullmatch(_UI_BUILD_ID_PATTERN, loaded) is not None
):
logger.info(
"Mission Core UI build drift observed; automatic page reload suppressed: "
"loaded=%s expected=%s",
loaded,
expected,
extra={
"event_code": "ui_build_drift_reload_suppressed",
"ui_build_id": loaded,
"expected_ui_build_id": expected,
"device_write_performed": False,
"automatic_retry": False,
},
)
return JSONResponse(
content={
"schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA,
@@ -142,6 +187,14 @@ def build_viewer_diagnostics_router(
"viewer_range_max_ns": event.viewer_range_max_ns,
"stalled_for_ms": event.stalled_for_ms,
"recovery_attempt": event.recovery_attempt,
"camera_queue_bytes": event.camera_queue_bytes,
"camera_queue_segments": event.camera_queue_segments,
"camera_retry_count": event.camera_retry_count,
"websocket_close_code": event.websocket_close_code,
"transport_epoch": event.transport_epoch,
"camera_append_error_name": event.camera_append_error_name,
"camera_media_source_state": event.camera_media_source_state,
"camera_video_error_code": event.camera_video_error_code,
},
)
return Response(status_code=204, headers={UI_BUILD_HEADER: expected})