SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): stabilize repeated acquisition and live viewer recovery

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 01:55:12 +03:00
parent 37b8930527
commit 2a97cf28c0
28 changed files with 2776 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/compute/live_perception.py
+9 -11
View File
@@ -396,6 +396,11 @@ class LivePerceptionIngress:
if self._session_id == session_id:
return
raise RuntimeError("another live perception session is active")
# Queued items belong to one acquisition. A worker may be absent
# while a session ends, so clear bounded leftovers before the next
# session can become visible to that worker.
for queue in self._queues.values():
queue.items.clear()
self._session_id = session_id
self._active = True
self._publish_locked(
@@ -681,9 +686,7 @@ class LiveSensorSynchronizer:
or retention_seconds <= 0
):
raise ValueError("live sensor synchronizer bounds are invalid")
self._maximum_lidar_camera_delta_ns = round(
maximum_lidar_camera_delta_ms * 1_000_000
)
self._maximum_lidar_camera_delta_ns = round(maximum_lidar_camera_delta_ms * 1_000_000)
self._maximum_pose_point_delta_ns = round(maximum_pose_point_delta_ms * 1_000_000)
self._capacity = capacity_per_modality
self._retention_ns = round(retention_seconds * 1_000_000_000)
@@ -801,8 +804,7 @@ class LiveSensorSynchronizer:
newest = values[-1].context.captured_at_epoch_ns
oldest_allowed = newest - self._retention_ns
while values and (
len(values) > self._capacity
or values[0].context.captured_at_epoch_ns < oldest_allowed
len(values) > self._capacity or values[0].context.captured_at_epoch_ns < oldest_allowed
):
values.popleft()
removed += 1
@@ -837,9 +839,7 @@ class WorldStateProjector:
if velocity_history_limit_s <= 0:
raise ValueError("velocity history limit must be positive")
self._velocity_history_limit_s = velocity_history_limit_s
self._track_history: dict[
int, deque[tuple[float, tuple[float, float, float]]]
] = {}
self._track_history: dict[int, deque[tuple[float, tuple[float, float, float]]]] = {}
def project(
self,
@@ -950,9 +950,7 @@ class WorldStateProjector:
math.sqrt(
sum(
(observed - expected) ** 2
for observed, expected in zip(
observed_center, predicted, strict=True
)
for observed, expected in zip(observed_center, predicted, strict=True)
)
)
)
src/k1link/device_plugins/xgrids_k1/ble/wifi_provisioning.py
+8 -7
View File
@@ -163,13 +163,18 @@ async def read_wifi_status_once(
device_macos_uuid: str,
*,
timeout_seconds: float = 20.0,
rediscover: bool = False,
) -> WifiStatusReadResult:
"""Read the K1's current DHCP status over BLE without writing a characteristic."""
if timeout_seconds <= 0:
raise ValueError("timeout_seconds must be positive")
async with asyncio.timeout(timeout_seconds + 5.0):
device = discovered_device(device_macos_uuid)
# A CoreBluetooth handle retained by an earlier scan is an optimization,
# not durable connection state. Recovery after sleep, Wi-Fi transition,
# or a completed provisioning GATT session must rediscover the device
# instead of repeatedly opening a stale handle.
device = None if rediscover else discovered_device(device_macos_uuid)
if device is None:
device = await BleakScanner.find_device_by_address(
device_macos_uuid,
@@ -183,9 +188,7 @@ async def read_wifi_status_once(
async with BleakClient(device, timeout=timeout_seconds, pair=False) as client:
service = client.services.get_service(SERVICE_UUID)
status_characteristic = client.services.get_characteristic(
STATUS_CHARACTERISTIC_UUID
)
status_characteristic = client.services.get_characteristic(STATUS_CHARACTERISTIC_UUID)
if service is None:
raise ValueError(f"Reviewed K1 service not found: {SERVICE_UUID}")
if status_characteristic is None:
@@ -193,9 +196,7 @@ async def read_wifi_status_once(
f"Reviewed K1 status characteristic not found: {STATUS_CHARACTERISTIC_UUID}"
)
if status_characteristic.service_uuid != service.uuid:
raise ValueError(
"K1 status characteristic is attached to an unexpected service"
)
raise ValueError("K1 status characteristic is attached to an unexpected service")
if "read" not in status_characteristic.properties:
raise ValueError("Reviewed K1 status characteristic is not readable")
value = bytes(await client.read_gatt_char(status_characteristic))
src/k1link/device_plugins/xgrids_k1/facade.py
+704 -148
View File
File diff suppressed because it is too large Load Diff
src/k1link/device_plugins/xgrids_k1/protocol/application_mqtt.py
+47 -5
View File
@@ -243,6 +243,9 @@ class ApplicationMqttTransportSnapshot:
latest_device_init_ready: bool | None
latest_system_error_code: int | None
latest_system_error_state: str | None
last_loop_result_code: int | None
last_loop_result_name: str | None
last_loop_phase: str | None
operation_keys_consumed: int
clean_session: bool = False
keepalive_seconds: int = CONTROL_KEEPALIVE_SECONDS
@@ -268,6 +271,9 @@ class ApplicationMqttTransportSnapshot:
"latest_device_init_ready": self.latest_device_init_ready,
"latest_system_error_code": self.latest_system_error_code,
"latest_system_error_state": self.latest_system_error_state,
"last_loop_result_code": self.last_loop_result_code,
"last_loop_result_name": self.last_loop_result_name,
"last_loop_phase": self.last_loop_phase,
"operation_keys_consumed": self.operation_keys_consumed,
"clean_session": self.clean_session,
"keepalive_seconds": self.keepalive_seconds,
@@ -348,6 +354,9 @@ class ReviewedApplicationMqttTransport:
self._latest_device_fault = False
self._latest_system_error_code: int | None = None
self._latest_system_error_state: str | None = None
self._last_loop_result_code: int | None = None
self._last_loop_result_name: str | None = None
self._last_loop_phase: str | None = None
def open(self) -> ApplicationMqttTransportSnapshot:
with self._lock:
@@ -562,7 +571,9 @@ class ReviewedApplicationMqttTransport:
except (OSError, RuntimeError, ValueError) as exc:
self._fail_after_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_after_publish(
self._record_loop_failure(result, phase="maintain-open")
)
self._discard_allowed_responses(allowed)
def close(self) -> None:
@@ -649,6 +660,9 @@ class ReviewedApplicationMqttTransport:
latest_device_init_ready=self._latest_device_init_ready,
latest_system_error_code=self._latest_system_error_code,
latest_system_error_state=self._latest_system_error_state,
last_loop_result_code=self._last_loop_result_code,
last_loop_result_name=self._last_loop_result_name,
last_loop_phase=self._last_loop_phase,
operation_keys_consumed=len(self._consumed_operation_keys),
)
@@ -879,9 +893,13 @@ class ReviewedApplicationMqttTransport:
self._fail_after_publish("control MQTT network loop failed", exc)
self._fail_before_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
failure = self._record_loop_failure(
result,
phase="post-publish" if post_publish else "connect-subscribe",
)
if post_publish:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_before_publish("control MQTT network loop returned an error")
self._fail_after_publish(failure)
self._fail_before_publish(failure)
def _service_once(self, *, post_publish: bool) -> None:
with self._lock:
@@ -898,9 +916,33 @@ class ReviewedApplicationMqttTransport:
self._fail_after_publish("control MQTT network loop failed", exc)
self._fail_before_publish("control MQTT network loop failed", exc)
if result != mqtt.MQTT_ERR_SUCCESS:
failure = self._record_loop_failure(
result,
phase="post-publish-drain" if post_publish else "pre-publish-drain",
)
if post_publish:
self._fail_after_publish("control MQTT network loop returned an error")
self._fail_before_publish("control MQTT network loop returned an error")
self._fail_after_publish(failure)
self._fail_before_publish(failure)
def _record_loop_failure(
self,
result: int,
*,
phase: str,
) -> str:
result_code = int(result)
try:
result_name = mqtt.error_string(result_code)
except (TypeError, ValueError):
result_name = "unknown MQTT error"
with self._lock:
self._last_loop_result_code = result_code
self._last_loop_result_name = result_name
self._last_loop_phase = phase
return (
"control MQTT network loop returned an error "
f"(phase={phase}, result={result_code}: {result_name})"
)
def _drain_responses(
self,
src/k1link/device_plugins/xgrids_k1/protocol/application_session.py
+196 -5
View File
@@ -62,6 +62,7 @@ logger = logging.getLogger(__name__)
TransportFactory = Callable[[str], ReviewedApplicationMqttTransport]
ScanningObserver = Callable[[], None]
@dataclass(frozen=True, slots=True)
@@ -97,10 +98,13 @@ class InteractiveApplicationControlSession:
*,
transport_factory: TransportFactory = ReviewedApplicationMqttTransport,
epoch_seconds: Callable[[], int] = lambda: int(time.time()),
scanning_observer: ScanningObserver | None = None,
) -> None:
self._authority_loader = authority_loader
self._transport_factory = transport_factory
self._epoch_seconds = epoch_seconds
self._scanning_observer = scanning_observer
self._scanning_observer_errors = 0
self._lock = threading.RLock()
self._phase: ApplicationControlPhase = "idle"
self._host: str | None = None
@@ -215,6 +219,55 @@ class InteractiveApplicationControlSession:
transport.close()
return self.snapshot()
def retire_for_network_change(self) -> dict[str, object]:
"""Retire local control ownership for one new explicit network action.
A correlated STOP followed by a network-loop loss in SCAN_STOPPING is
not enough evidence to authorize another START. It is enough to retire
the dead local socket when the operator explicitly chooses a new
network route. Pre-START failures already classified safe for a fresh
click are admissible here for the same reason: no modeling command was
attempted.
"""
with self._lock:
failure = self._failure
failed_network_change_admissible = (
self._phase == "failed"
and failure is not None
and (
failure.get("safe_to_retry") is True
or failure.get("network_change_admissible") is True
)
)
if self._phase not in {"idle", "completed", "closed"} and not (
failed_network_change_admissible
):
raise ApplicationAcceptanceError(
"control session cannot be retired for a network change"
)
if not self._worker_retired_locked():
raise ApplicationAcceptanceError(
"control session worker is still retiring"
)
previous_phase = self._phase
previous_reason = (
self._json_string(failure.get("reason_code"))
if failure is not None
else None
)
self._reset_locked()
if previous_phase != "idle":
logger.info(
"K1 local control session retired for an explicit network change",
extra={
"event_code": "k1_control_session_retired_for_network_change",
"reason_code": previous_reason,
"automatic_retry": False,
},
)
return self.snapshot()
def close(self) -> None:
"""Stop the process-owned socket without ever inventing a device STOP."""
@@ -247,6 +300,7 @@ class InteractiveApplicationControlSession:
"scripted_transitions": False,
"automatic_retry": False,
"outcome_unknown": self._outcome_unknown,
"scanning_observer_errors": self._scanning_observer_errors,
"failure": dict(self._failure) if self._failure is not None else None,
"dialogue": (
dict(self._dialogue_snapshot) if self._dialogue_snapshot is not None else None
@@ -310,6 +364,7 @@ class InteractiveApplicationControlSession:
checkpoint=start_checkpoint,
)
self._set_phase("scanning")
self._notify_scanning_observer()
executor.maintain_active_until_stop_requested(self._stop_requested.is_set)
stop_confirmation = self._stop_request()
@@ -412,10 +467,52 @@ class InteractiveApplicationControlSession:
)
== 1
)
safe_to_retry = not outcome_unknown and (
not transport_created
or (transport_snapshot_available and publish_attempts == 0)
or correlated_read_only_profile_mismatch
status_reconciled_prestart_failure = (
outcome_unknown
and modeling_command_attempted is False
and transport_snapshot_available
and not diagnostic_evidence_unavailable
and self._json_int_or_none(
transport_snapshot.get("device_status_reports")
)
not in {None, 0}
and self._json_string(
transport_snapshot.get("latest_device_session_state")
)
== "ready"
and transport_snapshot.get("latest_device_project_bound") is False
and self._json_int_or_none(
transport_snapshot.get("latest_system_error_code")
)
is None
)
stop_acknowledged_network_change = (
outcome_unknown
and failure_reason_code == "mqtt_network_loop_failed"
and failed_phase == "awaiting-standby-confirmation"
and dialogue_snapshot_available
and transport_snapshot_available
and dialogue_snapshot.get("stop_attempted") is True
and dialogue_snapshot.get("stop_complete") is True
and dialogue_stage == "stop-acknowledged"
and modeling_command_attempted is True
and self._json_string(
transport_snapshot.get("latest_device_session_state")
)
== "scan_stopping"
and transport_snapshot.get("latest_device_project_bound") is True
and self._json_int_or_none(
transport_snapshot.get("latest_system_error_code")
)
is None
)
safe_to_retry = status_reconciled_prestart_failure or (
not outcome_unknown
and (
not transport_created
or (transport_snapshot_available and publish_attempts == 0)
or correlated_read_only_profile_mismatch
)
)
self._failure = {
"code": type(exc).__name__,
@@ -451,6 +548,37 @@ class InteractiveApplicationControlSession:
else None
),
"safe_to_retry": safe_to_retry,
**(
{
"status_reconciliation": {
"device_session_state": "ready",
"device_project_bound": False,
"system_error_code": None,
"decision": "safe-explicit-prestart-retry",
"automatic_retry": False,
}
}
if status_reconciled_prestart_failure
else {}
),
**(
{
"network_change_admissible": True,
"network_change_reconciliation": {
"device_session_state": "scan_stopping",
"device_project_bound": True,
"system_error_code": None,
"stop_complete": True,
"standby_confirmed": False,
"decision": (
"explicit-network-change-only-after-acknowledged-stop"
),
"automatic_retry": False,
},
}
if stop_acknowledged_network_change
else {}
),
}
self._outcome_unknown = outcome_unknown
self._dialogue_snapshot = dialogue_snapshot or None
@@ -462,7 +590,8 @@ class InteractiveApplicationControlSession:
"publish_attempts=%s modeling_command_attempted=%s "
"diagnostic_snapshot_unavailable=%s "
"diagnostic_evidence_unavailable=%s outcome_unknown=%s "
"safe_to_retry=%s",
"safe_to_retry=%s status_reconciliation=%s "
"network_change_reconciliation=%s",
type(exc).__name__,
failure_reason_code,
failed_phase,
@@ -474,6 +603,51 @@ class InteractiveApplicationControlSession:
diagnostic_evidence_unavailable,
outcome_unknown,
safe_to_retry,
(
"safe-explicit-prestart-retry"
if status_reconciled_prestart_failure
else None
),
(
"explicit-network-change-only-after-acknowledged-stop"
if stop_acknowledged_network_change
else None
),
extra={
"event_code": "k1_application_control_session_failed",
"reason_code": failure_reason_code,
"failed_phase": failed_phase,
"dialogue_stage": dialogue_stage,
"transport_state": self._json_string(
transport_snapshot.get("state")
),
"publish_attempts": publish_attempts,
"modeling_command_attempted": modeling_command_attempted,
"outcome_unknown": outcome_unknown,
"safe_to_retry": safe_to_retry,
"status_reconciliation": (
"safe-explicit-prestart-retry"
if status_reconciled_prestart_failure
else None
),
"network_change_admissible": (
stop_acknowledged_network_change
),
"network_change_reconciliation": (
"explicit-network-change-only-after-acknowledged-stop"
if stop_acknowledged_network_change
else None
),
"mqtt_loop_result_code": self._json_int_or_none(
transport_snapshot.get("last_loop_result_code")
),
"mqtt_loop_result_name": self._json_string(
transport_snapshot.get("last_loop_result_name")
),
"mqtt_loop_phase": self._json_string(
transport_snapshot.get("last_loop_phase")
),
},
)
finally:
final_dialogue_snapshot: dict[str, object] | None = None
@@ -552,6 +726,23 @@ class InteractiveApplicationControlSession:
with self._lock:
self._set_phase_locked(phase)
def _notify_scanning_observer(self) -> None:
observer = self._scanning_observer
if observer is None:
return
try:
observer()
except Exception:
with self._lock:
self._scanning_observer_errors += 1
logger.exception(
"K1 confirmed SCANNING, but its acquisition observer failed",
extra={
"event_code": "k1_application_scanning_observer_failed",
"automatic_retry": False,
},
)
@staticmethod
def _executor_snapshot_safely(
executor: PhysicalAcceptanceDialogueExecutor | None,
src/k1link/device_plugins/xgrids_k1/viewer/runtime.py
+26 -25
View File
@@ -226,7 +226,7 @@ class VisualizationRuntime:
raise RuntimeError("поток не завершился за отведённое время; повторите остановку")
def close(self, *, wait_seconds: float = 5.0) -> None:
"""Stop the active source and release the process-wide visual bridge."""
"""Stop the active source and release its acquisition-scoped bridge."""
with self._lock:
self._closed = True
thread = self._thread
@@ -429,24 +429,25 @@ class VisualizationRuntime:
bridge: RerunBridge | None = None
try:
with self._lock:
bridge = self._bridge
if self._closed:
publisher_aborted.set()
if publisher_aborted.is_set():
publisher_ready.set()
return
if bridge is None:
candidate = self._bridge_factory(
grpc_port=self._grpc_port,
metrics=self._metrics,
settings_provider=self._current_scene_settings,
)
bridge = candidate
with self._lock:
if self._closed:
publisher_aborted.set()
else:
self._bridge = candidate
candidate = self._bridge_factory(
grpc_port=self._grpc_port,
metrics=self._metrics,
settings_provider=self._current_scene_settings,
)
bridge = candidate
with self._lock:
if self._closed:
publisher_aborted.set()
else:
# Every acquisition owns a fresh Rerun recording/store.
# Reusing a RecordingStream across independent scans can
# leave a later WebViewer without a StoreInfo event.
self._bridge = candidate
if publisher_aborted.is_set():
publisher_ready.set()
return
@@ -520,19 +521,17 @@ class VisualizationRuntime:
finally:
if bridge is not None:
with self._lock:
close_bridge = self._closed and (
self._bridge is bridge or publisher_aborted.is_set()
)
if close_bridge and self._bridge is bridge:
if self._bridge is bridge:
self._bridge = None
self._rerun_grpc_url = None
if close_bridge:
try:
bridge.close()
except BaseException as exc:
# Process shutdown must not become a false successful
# idle state when the native bridge failed to close.
publisher_error.append(exc)
try:
bridge.close()
except BaseException as exc:
# A failed native disconnect must not become a false
# successful idle state or leak port 9876 into the next
# independent acquisition.
publisher_error.append(exc)
self._notify()
def join_publisher() -> None:
# Never orphan a publisher: the session thread remains its owner.
@@ -596,6 +595,7 @@ class VisualizationRuntime:
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
self._rerun_grpc_url = None
self._notify()
def _finish_error(self, message: str) -> None:
@@ -605,6 +605,7 @@ class VisualizationRuntime:
self._message = message
self._foxglove_ws_url = None
self._foxglove_viewer_url = None
self._rerun_grpc_url = None
self._notify()
def _current_scene_settings(self) -> RerunSceneSettings:
src/k1link/viewer/rerun_bridge.py
+46 -2
View File
@@ -1,6 +1,8 @@
from __future__ import annotations
import logging
import math
import socket
import time
from collections.abc import Callable, Mapping
from contextlib import suppress
@@ -31,6 +33,7 @@ TRAJECTORY_MIN_DISTANCE_METERS = 0.02
TRAJECTORY_PUBLISH_INTERVAL_NS = 500_000_000
LIVE_GRPC_BUFFER_LIMIT = "32MiB"
DEFAULT_GRPC_PORT = 9876
GRPC_PORT_SEARCH_SPAN = 128
DEFAULT_CORS_ORIGINS = (
"http://127.0.0.1:5173",
"http://localhost:5173",
@@ -40,6 +43,8 @@ DEFAULT_CORS_ORIGINS = (
"http://localhost:8000",
)
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
@dataclass(frozen=True, slots=True)
class RerunSceneSettings:
@@ -62,6 +67,34 @@ class RerunSceneSettings:
SettingsProvider = Callable[[], RerunSceneSettings]
def _select_available_grpc_port(
preferred_port: int,
*,
search_span: int = GRPC_PORT_SEARCH_SPAN,
) -> int:
"""Select a local Rerun port without reusing a still-served recording."""
if not 1 <= preferred_port <= 65_535:
raise ValueError("Rerun gRPC port must be between 1 and 65535")
if search_span < 1:
raise ValueError("Rerun gRPC port search span must be positive")
last_port = min(preferred_port + search_span, 65_536)
for candidate in range(preferred_port, last_port):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
try:
# Rerun binds all local interfaces. Probe the same address class
# so a previous recording retained by a late viewer is detected.
probe.bind(("0.0.0.0", candidate))
except OSError:
continue
return candidate
raise RuntimeError(
"No local Rerun gRPC port is available in "
f"{preferred_port}..{last_port - 1}"
)
class RerunBridge:
"""Publish transport-neutral canonical envelopes to a Rerun recording."""
@@ -85,9 +118,20 @@ class RerunBridge:
else:
recording = recording_factory("nodedc_mission_core_spatial")
try:
selected_grpc_port = _select_available_grpc_port(grpc_port)
if selected_grpc_port != grpc_port:
logger.info(
"Mission Core selected a new Rerun port because an earlier "
"viewer still owns the preferred listener",
extra={
"event_code": "rerun_grpc_port_rotated",
"preferred_port": grpc_port,
"selected_port": selected_grpc_port,
},
)
blueprint = _blueprint(self._settings)
url = recording.serve_grpc(
grpc_port=grpc_port,
grpc_port=selected_grpc_port,
default_blueprint=blueprint,
# This is a reconnect cushion for the live preview, not the source
# of record. Raw MQTT evidence is persisted independently. A large
@@ -132,7 +176,7 @@ class RerunBridge:
return self._url
def begin_session(self, metrics: BridgeMetrics | None = None) -> None:
"""Reset session state while keeping the process-lifetime server alive."""
"""Initialize the one acquisition owned by this recording server."""
if self._closed:
raise RuntimeError("Rerun bridge is already closed")
if metrics is not None:
src/k1link/web/app.py
+6
View File
@@ -53,8 +53,10 @@ from k1link.web.plugin_runtime import (
PluginRuntimeUnavailableError,
)
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
from k1link.web.session_api import build_session_router
from k1link.web.system_telemetry_api import build_system_telemetry_router
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
@@ -274,6 +276,9 @@ async def _recording_preparation_reconciler() -> None:
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
reconciler: asyncio.Task[None] | None = None
try:
configure_scanner_diagnostics(
REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs"
)
session_recording_preparation_manager.start()
# Recovery is intentionally a one-shot startup phase. The archive
# helper owns a cross-process lease, while ordinary catalog requests
@@ -671,6 +676,7 @@ app.include_router(
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
)
)
app.include_router(build_viewer_diagnostics_router())
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
src/k1link/web/runtime_diagnostics.py
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import json
import logging
import os
from datetime import UTC, datetime
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import Final
SCANNER_LOGGER_NAME: Final = "k1link.device_plugins.xgrids_k1"
SCANNER_DIAGNOSTIC_FILE: Final = "scanner-diagnostics.jsonl"
_HANDLER_MARKER: Final = "_mission_core_scanner_diagnostics_path"
_EXTRA_FIELDS: Final = (
"event_code",
"reason_code",
"failed_phase",
"dialogue_stage",
"transport_state",
"publish_attempts",
"modeling_command_attempted",
"outcome_unknown",
"safe_to_retry",
"status_reconciliation",
"network_change_admissible",
"network_change_reconciliation",
"mqtt_loop_result_code",
"mqtt_loop_result_name",
"mqtt_loop_phase",
"automatic_retry",
"camera_source_id",
"evidence_session_id",
"activation_trigger",
"lease_generation",
"lease_state",
"recovery_strategy",
"endpoint_reachable",
"host_route_class",
"address_changed",
"device_write_performed",
"failure_stage",
"start_checkpoint_released",
"stale_session_retired",
"stream_id",
"backend_activity_sequence",
"viewer_range_max_ns",
"stalled_for_ms",
"recovery_attempt",
"preferred_port",
"selected_port",
)
class ScannerDiagnosticJsonFormatter(logging.Formatter):
"""Serialize a bounded, secret-free scanner diagnostic record."""
def format(self, record: logging.LogRecord) -> str:
document: dict[str, object] = {
"timestamp_utc": datetime.fromtimestamp(record.created, UTC)
.isoformat()
.replace("+00:00", "Z"),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage()[:1000],
}
for field in _EXTRA_FIELDS:
value = getattr(record, field, None)
if value is not None and isinstance(value, (str, int, bool)):
document[field] = value
return json.dumps(document, ensure_ascii=False, separators=(",", ":"))
def configure_scanner_diagnostics(logs_root: Path) -> Path:
"""Install one private rotating JSONL handler for K1 field diagnostics."""
logs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
os.chmod(logs_root, 0o700)
target = (logs_root / SCANNER_DIAGNOSTIC_FILE).resolve()
logger = logging.getLogger(SCANNER_LOGGER_NAME)
for handler in logger.handlers:
if getattr(handler, _HANDLER_MARKER, None) == str(target):
return target
handler = RotatingFileHandler(
target,
maxBytes=5 * 1024 * 1024,
backupCount=5,
encoding="utf-8",
delay=False,
)
os.chmod(target, 0o600)
handler.setLevel(logging.INFO)
handler.setFormatter(ScannerDiagnosticJsonFormatter())
setattr(handler, _HANDLER_MARKER, str(target))
logger.addHandler(handler)
logger.setLevel(logging.INFO)
return target
src/k1link/web/viewer_diagnostics_api.py
+66
View File
@@ -0,0 +1,66 @@
from __future__ import annotations
import logging
from typing import Literal
from fastapi import APIRouter, Response
from pydantic import BaseModel, ConfigDict, Field
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
LiveViewerEventCode = Literal[
"live_receiver_stalled",
"live_receiver_restart_requested",
"live_receiver_recovered",
"live_receiver_recovery_exhausted",
"live_receiver_active_store_admitted",
"live_receiver_error",
]
LiveViewerFailureStage = Literal[
"recording-open-timeout",
"viewer-start",
"module-load",
"receiver-stalled",
]
class LiveViewerDiagnosticEvent(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
event_code: LiveViewerEventCode
failure_stage: LiveViewerFailureStage | None = None
stream_id: str | None = Field(
default=None,
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$",
)
backend_activity_sequence: int | None = Field(default=None, ge=0)
viewer_range_max_ns: int | None = Field(default=None, ge=0)
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
def build_viewer_diagnostics_router() -> APIRouter:
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
@router.post("/live-diagnostics", status_code=204)
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
logger.info(
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
event.event_code,
event.stream_id,
extra={
"event_code": event.event_code,
"failure_stage": event.failure_stage,
"stream_id": event.stream_id,
"backend_activity_sequence": event.backend_activity_sequence,
"viewer_range_max_ns": event.viewer_range_max_ns,
"stalled_for_ms": event.stalled_for_ms,
"recovery_attempt": event.recovery_attempt,
},
)
return Response(status_code=204)
return router