fix(k1): stabilize repeated acquisition and live viewer recovery
This commit is contained in:
@@ -53,8 +53,10 @@ from k1link.web.plugin_runtime import (
|
||||
PluginRuntimeUnavailableError,
|
||||
)
|
||||
from k1link.web.polygon_api import build_polygon_router, configured_polygon_runs_root
|
||||
from k1link.web.runtime_diagnostics import configure_scanner_diagnostics
|
||||
from k1link.web.session_api import build_session_router
|
||||
from k1link.web.system_telemetry_api import build_system_telemetry_router
|
||||
from k1link.web.viewer_diagnostics_api import build_viewer_diagnostics_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
|
||||
INVALID_REQUEST_DETAIL = "Некорректные параметры запроса."
|
||||
@@ -274,6 +276,9 @@ async def _recording_preparation_reconciler() -> None:
|
||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler: asyncio.Task[None] | None = None
|
||||
try:
|
||||
configure_scanner_diagnostics(
|
||||
REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs"
|
||||
)
|
||||
session_recording_preparation_manager.start()
|
||||
# Recovery is intentionally a one-shot startup phase. The archive
|
||||
# helper owns a cross-process lease, while ordinary catalog requests
|
||||
@@ -671,6 +676,7 @@ app.include_router(
|
||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||
)
|
||||
)
|
||||
app.include_router(build_viewer_diagnostics_router())
|
||||
|
||||
|
||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
SCANNER_LOGGER_NAME: Final = "k1link.device_plugins.xgrids_k1"
|
||||
SCANNER_DIAGNOSTIC_FILE: Final = "scanner-diagnostics.jsonl"
|
||||
_HANDLER_MARKER: Final = "_mission_core_scanner_diagnostics_path"
|
||||
_EXTRA_FIELDS: Final = (
|
||||
"event_code",
|
||||
"reason_code",
|
||||
"failed_phase",
|
||||
"dialogue_stage",
|
||||
"transport_state",
|
||||
"publish_attempts",
|
||||
"modeling_command_attempted",
|
||||
"outcome_unknown",
|
||||
"safe_to_retry",
|
||||
"status_reconciliation",
|
||||
"network_change_admissible",
|
||||
"network_change_reconciliation",
|
||||
"mqtt_loop_result_code",
|
||||
"mqtt_loop_result_name",
|
||||
"mqtt_loop_phase",
|
||||
"automatic_retry",
|
||||
"camera_source_id",
|
||||
"evidence_session_id",
|
||||
"activation_trigger",
|
||||
"lease_generation",
|
||||
"lease_state",
|
||||
"recovery_strategy",
|
||||
"endpoint_reachable",
|
||||
"host_route_class",
|
||||
"address_changed",
|
||||
"device_write_performed",
|
||||
"failure_stage",
|
||||
"start_checkpoint_released",
|
||||
"stale_session_retired",
|
||||
"stream_id",
|
||||
"backend_activity_sequence",
|
||||
"viewer_range_max_ns",
|
||||
"stalled_for_ms",
|
||||
"recovery_attempt",
|
||||
"preferred_port",
|
||||
"selected_port",
|
||||
)
|
||||
|
||||
|
||||
class ScannerDiagnosticJsonFormatter(logging.Formatter):
|
||||
"""Serialize a bounded, secret-free scanner diagnostic record."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
document: dict[str, object] = {
|
||||
"timestamp_utc": datetime.fromtimestamp(record.created, UTC)
|
||||
.isoformat()
|
||||
.replace("+00:00", "Z"),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage()[:1000],
|
||||
}
|
||||
for field in _EXTRA_FIELDS:
|
||||
value = getattr(record, field, None)
|
||||
if value is not None and isinstance(value, (str, int, bool)):
|
||||
document[field] = value
|
||||
return json.dumps(document, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def configure_scanner_diagnostics(logs_root: Path) -> Path:
|
||||
"""Install one private rotating JSONL handler for K1 field diagnostics."""
|
||||
|
||||
logs_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
os.chmod(logs_root, 0o700)
|
||||
target = (logs_root / SCANNER_DIAGNOSTIC_FILE).resolve()
|
||||
logger = logging.getLogger(SCANNER_LOGGER_NAME)
|
||||
for handler in logger.handlers:
|
||||
if getattr(handler, _HANDLER_MARKER, None) == str(target):
|
||||
return target
|
||||
|
||||
handler = RotatingFileHandler(
|
||||
target,
|
||||
maxBytes=5 * 1024 * 1024,
|
||||
backupCount=5,
|
||||
encoding="utf-8",
|
||||
delay=False,
|
||||
)
|
||||
os.chmod(target, 0o600)
|
||||
handler.setLevel(logging.INFO)
|
||||
handler.setFormatter(ScannerDiagnosticJsonFormatter())
|
||||
setattr(handler, _HANDLER_MARKER, str(target))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
return target
|
||||
@@ -0,0 +1,66 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Response
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
|
||||
|
||||
LiveViewerEventCode = Literal[
|
||||
"live_receiver_stalled",
|
||||
"live_receiver_restart_requested",
|
||||
"live_receiver_recovered",
|
||||
"live_receiver_recovery_exhausted",
|
||||
"live_receiver_active_store_admitted",
|
||||
"live_receiver_error",
|
||||
]
|
||||
LiveViewerFailureStage = Literal[
|
||||
"recording-open-timeout",
|
||||
"viewer-start",
|
||||
"module-load",
|
||||
"receiver-stalled",
|
||||
]
|
||||
|
||||
|
||||
class LiveViewerDiagnosticEvent(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", strict=True)
|
||||
|
||||
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
|
||||
event_code: LiveViewerEventCode
|
||||
failure_stage: LiveViewerFailureStage | None = None
|
||||
stream_id: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=128,
|
||||
pattern=r"^[A-Za-z0-9][A-Za-z0-9._:-]*$",
|
||||
)
|
||||
backend_activity_sequence: int | None = Field(default=None, ge=0)
|
||||
viewer_range_max_ns: int | None = Field(default=None, ge=0)
|
||||
stalled_for_ms: int | None = Field(default=None, ge=0, le=600_000)
|
||||
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
|
||||
|
||||
|
||||
def build_viewer_diagnostics_router() -> APIRouter:
|
||||
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
|
||||
|
||||
@router.post("/live-diagnostics", status_code=204)
|
||||
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
|
||||
logger.info(
|
||||
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
|
||||
event.event_code,
|
||||
event.stream_id,
|
||||
extra={
|
||||
"event_code": event.event_code,
|
||||
"failure_stage": event.failure_stage,
|
||||
"stream_id": event.stream_id,
|
||||
"backend_activity_sequence": event.backend_activity_sequence,
|
||||
"viewer_range_max_ns": event.viewer_range_max_ns,
|
||||
"stalled_for_ms": event.stalled_for_ms,
|
||||
"recovery_attempt": event.recovery_attempt,
|
||||
},
|
||||
)
|
||||
return Response(status_code=204)
|
||||
|
||||
return router
|
||||
Reference in New Issue
Block a user