fix(k1): stabilize repeated acquisition and live viewer recovery

This commit is contained in:
DCCONSTRUCTIONS
2026-07-29 01:55:12 +03:00
parent 37b8930527
commit 2a97cf28c0
28 changed files with 2776 additions and 286 deletions
+6
View File
@@ -53,8 +53,10 @@ from k1link.web.plugin_runtime import (
PluginRuntimeUnavailableError,
)
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
from k1link.web.session_api import build_session_router
from k1link.web.system_telemetry_api import build_system_telemetry_router
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
@@ -274,6 +276,9 @@ async def _recording_preparation_reconciler() -> None:
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
reconciler: asyncio.Task[None] | None = None
try:
configure_scanner_diagnostics(
REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs"
)
session_recording_preparation_manager.start()
# Recovery is intentionally a one-shot startup phase. The archive
# helper owns a cross-process lease, while ordinary catalog requests
@@ -671,6 +676,7 @@ app.include_router(
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
)
)
app.include_router(build_viewer_diagnostics_router())
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import json
import logging
import os
from datetime import UTC, datetime
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import Final
SCANNER_LOGGER_NAME: Final = "k1link.device_plugins.xgrids_k1"
SCANNER_DIAGNOSTIC_FILE: Final = "scanner-diagnostics.jsonl"
_HANDLER_MARKER: Final = "_mission_core_scanner_diagnostics_path"
_EXTRA_FIELDS: Final = (
"event_code",
"reason_code",
"failed_phase",
"dialogue_stage",
"transport_state",
"publish_attempts",
"modeling_command_attempted",
"outcome_unknown",
"safe_to_retry",
"status_reconciliation",
"network_change_admissible",
"network_change_reconciliation",
"mqtt_loop_result_code",
"mqtt_loop_result_name",
"mqtt_loop_phase",
"automatic_retry",
"camera_source_id",
"evidence_session_id",
"activation_trigger",
"lease_generation",
"lease_state",
"recovery_strategy",
"endpoint_reachable",
"host_route_class",
"address_changed",
"device_write_performed",
"failure_stage",
"start_checkpoint_released",
"stale_session_retired",
"stream_id",
"backend_activity_sequence",
"viewer_range_max_ns",
"stalled_for_ms",
"recovery_attempt",
"preferred_port",
"selected_port",
)
class ScannerDiagnosticJsonFormatter(logging.Formatter):
"""Serialize a bounded, secret-free scanner diagnostic record."""
def format(self, record: logging.LogRecord) -> str:
document: dict[str, object] = {
"timestamp_utc": datetime.fromtimestamp(record.created, UTC)
.isoformat()
.replace("+00:00", "Z"),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage()[:1000],
}
for field in _EXTRA_FIELDS:
value = getattr(record, field, None)
if value is not None and isinstance(value, (str, int, bool)):
document[field] = value
return json.dumps(document, ensure_ascii=False, separators=(",", ":"))
def configure_scanner_diagnostics(logs_root: Path) -> Path:
"""Install one private rotating JSONL handler for K1 field diagnostics."""
logs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
os.chmod(logs_root, 0o700)
target = (logs_root / SCANNER_DIAGNOSTIC_FILE).resolve()
logger = logging.getLogger(SCANNER_LOGGER_NAME)
for handler in logger.handlers:
if getattr(handler, _HANDLER_MARKER, None) == str(target):
return target
handler = RotatingFileHandler(
target,
maxBytes=5 * 1024 * 1024,
backupCount=5,
encoding="utf-8",
delay=False,
)
os.chmod(target, 0o600)
handler.setLevel(logging.INFO)
handler.setFormatter(ScannerDiagnosticJsonFormatter())
setattr(handler, _HANDLER_MARKER, str(target))
logger.addHandler(handler)
logger.setLevel(logging.INFO)
return target
+66
View File
@@ -0,0 +1,66 @@
from __future__ import annotations
import logging
from typing import Literal
from fastapi import APIRouter, Response
from pydantic import BaseModel, ConfigDict, Field
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
LiveViewerEventCode = Literal[
"live_receiver_stalled",
"live_receiver_restart_requested",
"live_receiver_recovered",
"live_receiver_recovery_exhausted",
"live_receiver_active_store_admitted",
"live_receiver_error",
]
LiveViewerFailureStage = Literal[
"recording-open-timeout",
"viewer-start",
"module-load",
"receiver-stalled",
]
class LiveViewerDiagnosticEvent(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
event_code: LiveViewerEventCode
failure_stage: LiveViewerFailureStage | None = None
stream_id: str | None = Field(
default=None,
min_length=1,
max_length=128,
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$",
)
backend_activity_sequence: int | None = Field(default=None, ge=0)
viewer_range_max_ns: int | None = Field(default=None, ge=0)
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
def build_viewer_diagnostics_router() -> APIRouter:
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
@router.post("/live-diagnostics", status_code=204)
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
logger.info(
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
event.event_code,
event.stream_id,
extra={
"event_code": event.event_code,
"failure_stage": event.failure_stage,
"stream_id": event.stream_id,
"backend_activity_sequence": event.backend_activity_sequence,
"viewer_range_max_ns": event.viewer_range_max_ns,
"stalled_for_ms": event.stalled_for_ms,
"recovery_attempt": event.recovery_attempt,
},
)
return Response(status_code=204)
return router