427 lines
18 KiB
Python
427 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import stat
|
|
from collections.abc import Callable
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from fastapi import APIRouter
|
|
from fastapi.routing import APIRoute
|
|
from pydantic import ValidationError
|
|
from starlette.requests import Request
|
|
|
|
from k1link.web.runtime_diagnostics import (
|
|
SCANNER_LOGGER_NAME,
|
|
configure_scanner_diagnostics,
|
|
)
|
|
from k1link.web.viewer_diagnostics_api import (
|
|
LiveViewerDiagnosticEvent,
|
|
build_viewer_diagnostics_router,
|
|
)
|
|
|
|
|
|
def _endpoint(router: APIRouter, path: str, method: str) -> Callable[..., Any]:
|
|
for route in router.routes:
|
|
if (
|
|
isinstance(route, APIRoute)
|
|
and route.path == path
|
|
and route.methods is not None
|
|
and method in route.methods
|
|
):
|
|
return route.endpoint
|
|
raise AssertionError(f"{method} {path} route is missing")
|
|
|
|
|
|
def _request(*, ui_build_id: str | None = None) -> Request:
|
|
headers = []
|
|
if ui_build_id is not None:
|
|
headers.append((b"x-missioncore-ui-build", ui_build_id.encode("ascii")))
|
|
return Request({"type": "http", "method": "GET", "path": "/", "headers": headers})
|
|
|
|
|
|
def test_scanner_exception_keeps_location_without_private_message_or_locals(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
target = configure_scanner_diagnostics(tmp_path / "logs")
|
|
logger = logging.getLogger(f"{SCANNER_LOGGER_NAME}.test")
|
|
private_detail = "synthetic-private-runtime-detail"
|
|
try:
|
|
raise ValueError(private_detail)
|
|
except ValueError:
|
|
logger.exception("Camera activation failed")
|
|
for handler in logging.getLogger(SCANNER_LOGGER_NAME).handlers:
|
|
handler.flush()
|
|
serialized = target.read_text(encoding="utf-8").splitlines()[-1]
|
|
document = json.loads(serialized)
|
|
assert document["exception_type"] == "ValueError"
|
|
assert document["exception_site"].startswith("test_viewer_diagnostics_api.py:")
|
|
assert document["exception_site"].endswith(
|
|
":test_scanner_exception_keeps_location_without_private_message_or_locals"
|
|
)
|
|
assert private_detail not in serialized
|
|
assert str(Path(__file__).parent) not in serialized
|
|
|
|
|
|
def test_private_scanner_diagnostics_are_durable_structured_and_bounded(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
target = configure_scanner_diagnostics(tmp_path / "logs")
|
|
logger = logging.getLogger(f"{SCANNER_LOGGER_NAME}.test")
|
|
logger.error(
|
|
"field control failure",
|
|
extra={
|
|
"event_code": "k1_application_control_session_failed",
|
|
"operation_id": "operation-test-01",
|
|
"operation_stage": "ble-provisioning-write",
|
|
"connection_mode": "bridge",
|
|
"error_category": "device",
|
|
"error_code": "BleakGATTProtocolError",
|
|
"reason_code": "mqtt_network_loop_failed",
|
|
"mqtt_loop_result_code": 7,
|
|
"mqtt_loop_result_name": "The connection was lost.",
|
|
"mqtt_loop_phase": "post-publish-drain",
|
|
"automatic_retry": False,
|
|
"side_effect_status": "unknown",
|
|
"network_change_attempted": True,
|
|
"device_write_attempted": True,
|
|
"device_write_confirmed": False,
|
|
"ble_att_error_code": 4,
|
|
"ble_att_error_name": "INVALID_PDU",
|
|
"resolved_write_mode": "with_response",
|
|
"max_write_without_response_size": 253,
|
|
"mtu_size": 256,
|
|
"frame_length": 99,
|
|
"write_characteristic_properties": ["read", "write"],
|
|
"scan_elapsed_ms": 7300,
|
|
"scanner_start_ms": 31,
|
|
"initial_window_ms": 6000,
|
|
"first_candidate_ms": 7240,
|
|
"scan_extended": True,
|
|
"candidate_count": 4,
|
|
"likely_k1_candidate_count": 1,
|
|
"discovery_generation": 2,
|
|
"helper_stage": "compile",
|
|
"helper_elapsed_ms": 34720,
|
|
"camera_source_id": "sensor.camera.right",
|
|
"evidence_session_id": "20260728T163450Z_viewer_live",
|
|
"activation_trigger": "application-control-scanning",
|
|
"network_change_admissible": True,
|
|
"network_change_reconciliation": (
|
|
"explicit-network-change-only-after-acknowledged-stop"
|
|
),
|
|
"lease_generation": 3,
|
|
"lease_state": "reachable",
|
|
"recovery_strategy": "existing-mqtt-endpoint",
|
|
"endpoint_reachable": True,
|
|
"address_changed": False,
|
|
"device_write_performed": False,
|
|
"preferred_port": 9876,
|
|
"selected_port": 9877,
|
|
"camera_queue_bytes": 12_000_000,
|
|
"camera_queue_segments": 96,
|
|
"camera_retry_count": 3,
|
|
"camera_pcl_admission_ms": 184,
|
|
"camera_authority_wait_ms": 91,
|
|
"camera_ffmpeg_prepare_ms": 42,
|
|
"camera_post_spawn_commit_ms": 7,
|
|
"camera_activation_total_ms": 140,
|
|
"device_command_sent": False,
|
|
"websocket_close_code": 4_008,
|
|
"transport_epoch": 7,
|
|
"unapproved_secret_field": "must-not-be-written",
|
|
},
|
|
)
|
|
for handler in logging.getLogger(SCANNER_LOGGER_NAME).handlers:
|
|
handler.flush()
|
|
|
|
document = json.loads(target.read_text(encoding="utf-8").splitlines()[-1])
|
|
assert stat.S_IMODE(target.parent.stat().st_mode) == 0o700
|
|
assert stat.S_IMODE(target.stat().st_mode) == 0o600
|
|
assert document["event_code"] == "k1_application_control_session_failed"
|
|
assert document["operation_id"] == "operation-test-01"
|
|
assert document["operation_stage"] == "ble-provisioning-write"
|
|
assert document["connection_mode"] == "bridge"
|
|
assert document["error_category"] == "device"
|
|
assert document["error_code"] == "BleakGATTProtocolError"
|
|
assert document["reason_code"] == "mqtt_network_loop_failed"
|
|
assert document["mqtt_loop_result_code"] == 7
|
|
assert document["mqtt_loop_phase"] == "post-publish-drain"
|
|
assert document["automatic_retry"] is False
|
|
assert document["camera_pcl_admission_ms"] == 184
|
|
assert document["camera_authority_wait_ms"] == 91
|
|
assert document["camera_ffmpeg_prepare_ms"] == 42
|
|
assert document["camera_post_spawn_commit_ms"] == 7
|
|
assert document["camera_activation_total_ms"] == 140
|
|
assert document["device_command_sent"] is False
|
|
assert document["side_effect_status"] == "unknown"
|
|
assert document["network_change_attempted"] is True
|
|
assert document["device_write_attempted"] is True
|
|
assert document["device_write_confirmed"] is False
|
|
assert document["ble_att_error_code"] == 4
|
|
assert document["ble_att_error_name"] == "INVALID_PDU"
|
|
assert document["resolved_write_mode"] == "with_response"
|
|
assert document["max_write_without_response_size"] == 253
|
|
assert document["mtu_size"] == 256
|
|
assert document["frame_length"] == 99
|
|
assert document["write_characteristic_properties"] == ["read", "write"]
|
|
assert document["scan_elapsed_ms"] == 7300
|
|
assert document["scanner_start_ms"] == 31
|
|
assert document["initial_window_ms"] == 6000
|
|
assert document["first_candidate_ms"] == 7240
|
|
assert document["scan_extended"] is True
|
|
assert document["candidate_count"] == 4
|
|
assert document["likely_k1_candidate_count"] == 1
|
|
assert document["discovery_generation"] == 2
|
|
assert document["helper_stage"] == "compile"
|
|
assert document["helper_elapsed_ms"] == 34720
|
|
assert document["camera_source_id"] == "sensor.camera.right"
|
|
assert document["evidence_session_id"] == "20260728T163450Z_viewer_live"
|
|
assert document["activation_trigger"] == "application-control-scanning"
|
|
assert document["network_change_admissible"] is True
|
|
assert document["network_change_reconciliation"] == (
|
|
"explicit-network-change-only-after-acknowledged-stop"
|
|
)
|
|
assert document["lease_generation"] == 3
|
|
assert document["lease_state"] == "reachable"
|
|
assert document["recovery_strategy"] == "existing-mqtt-endpoint"
|
|
assert document["endpoint_reachable"] is True
|
|
assert document["address_changed"] is False
|
|
assert document["device_write_performed"] is False
|
|
assert document["preferred_port"] == 9876
|
|
assert document["selected_port"] == 9877
|
|
assert document["camera_queue_bytes"] == 12_000_000
|
|
assert document["camera_queue_segments"] == 96
|
|
assert document["camera_retry_count"] == 3
|
|
assert document["websocket_close_code"] == 4_008
|
|
assert document["transport_epoch"] == 7
|
|
assert "unapproved_secret_field" not in document
|
|
|
|
parent = logging.getLogger(SCANNER_LOGGER_NAME)
|
|
for handler in list(parent.handlers):
|
|
if getattr(handler, "baseFilename", None) == str(target):
|
|
parent.removeHandler(handler)
|
|
handler.close()
|
|
|
|
|
|
def test_live_viewer_diagnostic_endpoint_accepts_only_bounded_events(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
expected_build = "/assets/index-abcdefgh.js"
|
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST")
|
|
event = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code="live_receiver_stalled",
|
|
ui_build_id=expected_build,
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000002",
|
|
lifecycle_generation=4,
|
|
failure_stage="receiver-stalled",
|
|
stream_id="acquisition-123",
|
|
backend_activity_sequence=8_572,
|
|
viewer_range_max_ns=231_000_000_000,
|
|
stalled_for_ms=5_500,
|
|
recovery_attempt=1,
|
|
)
|
|
|
|
with caplog.at_level(
|
|
logging.INFO,
|
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
|
):
|
|
response = endpoint(event)
|
|
|
|
assert response.status_code == 204
|
|
assert response.headers["x-missioncore-ui-build"] == expected_build
|
|
assert "event=live_receiver_stalled" in caplog.text
|
|
assert caplog.records[-1].failure_stage == "receiver-stalled"
|
|
assert caplog.records[-1].document_instance_id == event.document_instance_id
|
|
assert caplog.records[-1].viewer_instance_id == event.viewer_instance_id
|
|
assert caplog.records[-1].lifecycle_generation == 4
|
|
with pytest.raises(ValidationError):
|
|
LiveViewerDiagnosticEvent.model_validate(
|
|
{
|
|
**event.model_dump(),
|
|
"source_url": "http://192.168.56.1/private",
|
|
}
|
|
)
|
|
fallback = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code="live_receiver_active_store_admitted",
|
|
ui_build_id=expected_build,
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000002",
|
|
lifecycle_generation=4,
|
|
stream_id="acquisition-123",
|
|
backend_activity_sequence=8_573,
|
|
)
|
|
assert fallback.failure_stage is None
|
|
|
|
camera_restart = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code="live_camera_transport_restart_requested",
|
|
ui_build_id=expected_build,
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000003",
|
|
lifecycle_generation=2,
|
|
failure_stage="camera-queue-capacity",
|
|
stream_id="camera-preview-2",
|
|
camera_queue_bytes=12_000_000,
|
|
camera_queue_segments=96,
|
|
camera_retry_count=3,
|
|
websocket_close_code=4_008,
|
|
transport_epoch=7,
|
|
camera_append_error_name="InvalidStateError",
|
|
camera_media_source_state="open",
|
|
camera_video_error_code=3,
|
|
)
|
|
with caplog.at_level(
|
|
logging.INFO,
|
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
|
):
|
|
camera_response = endpoint(camera_restart)
|
|
assert camera_response.status_code == 204
|
|
assert caplog.records[-1].failure_stage == "camera-queue-capacity"
|
|
assert caplog.records[-1].camera_queue_bytes == 12_000_000
|
|
assert caplog.records[-1].camera_queue_segments == 96
|
|
assert caplog.records[-1].camera_retry_count == 3
|
|
assert caplog.records[-1].websocket_close_code == 4_008
|
|
assert caplog.records[-1].transport_epoch == 7
|
|
assert caplog.records[-1].camera_append_error_name == "InvalidStateError"
|
|
assert caplog.records[-1].camera_media_source_state == "open"
|
|
assert caplog.records[-1].camera_video_error_code == 3
|
|
|
|
for event_code in (
|
|
"live_camera_source_projected",
|
|
"live_camera_window_admitted",
|
|
"live_camera_player_effect_started",
|
|
"live_camera_playback_authority_missing",
|
|
"live_camera_media_source_open",
|
|
"live_camera_websocket_open",
|
|
):
|
|
boundary_event = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code=event_code,
|
|
ui_build_id=expected_build,
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000004",
|
|
lifecycle_generation=1,
|
|
stream_id="camera-preview-2",
|
|
transport_epoch=(
|
|
1
|
|
if event_code
|
|
not in {"live_camera_source_projected", "live_camera_window_admitted"}
|
|
else None
|
|
),
|
|
)
|
|
with caplog.at_level(
|
|
logging.INFO,
|
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
|
):
|
|
boundary_response = endpoint(boundary_event)
|
|
assert boundary_response.status_code == 204
|
|
assert caplog.records[-1].event_code == event_code
|
|
assert caplog.records[-1].stream_id == "camera-preview-2"
|
|
|
|
|
|
def test_live_viewer_diagnostic_rejects_stale_build_before_logging(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
expected_build = "/assets/index-ijklmnop.js"
|
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST")
|
|
event = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code="live_receiver_error",
|
|
ui_build_id="/assets/index-abcdefgh.js",
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000002",
|
|
lifecycle_generation=1,
|
|
)
|
|
|
|
with caplog.at_level(
|
|
logging.INFO,
|
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
|
):
|
|
response = endpoint(event)
|
|
|
|
assert response.status_code == 409
|
|
assert response.headers["x-missioncore-ui-build"] == expected_build
|
|
assert "Mission Core live Rerun receiver diagnostic" not in caplog.text
|
|
|
|
|
|
def test_live_viewer_client_contract_is_no_store_and_exact_build() -> None:
|
|
expected_build = "/assets/index-abcdefgh.js"
|
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
|
|
|
response = endpoint(_request(ui_build_id=expected_build))
|
|
|
|
assert response.status_code == 200
|
|
assert response.headers["cache-control"] == "no-store"
|
|
assert response.headers["x-missioncore-ui-build"] == expected_build
|
|
assert json.loads(response.body) == {
|
|
"schema_version": "missioncore.live-viewer-client-contract/v1",
|
|
"status": "ready",
|
|
"ui_build_id": expected_build,
|
|
"diagnostic_schema_version": "missioncore.live-viewer-diagnostic/v2",
|
|
}
|
|
|
|
|
|
def test_live_viewer_client_contract_logs_suppressed_stale_build_reload(
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
loaded_build = "/assets/index-abcdefgh.js"
|
|
expected_build = "/assets/index-ijklmnop.js"
|
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: expected_build)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
|
|
|
with caplog.at_level(
|
|
logging.INFO,
|
|
logger="k1link.device_plugins.xgrids_k1.viewer_receiver",
|
|
):
|
|
response = endpoint(_request(ui_build_id=loaded_build))
|
|
|
|
assert response.status_code == 200
|
|
assert caplog.records[-1].event_code == "ui_build_drift_reload_suppressed"
|
|
assert caplog.records[-1].ui_build_id == loaded_build
|
|
assert caplog.records[-1].expected_ui_build_id == expected_build
|
|
assert caplog.records[-1].device_write_performed is False
|
|
assert caplog.records[-1].automatic_retry is False
|
|
|
|
|
|
def test_live_viewer_client_contract_no_dist_is_retryable_without_reload_header() -> None:
|
|
router = build_viewer_diagnostics_router(expected_ui_build_id=lambda: None)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/client-contract", "GET")
|
|
|
|
response = endpoint(_request())
|
|
|
|
assert response.status_code == 503
|
|
assert response.headers["cache-control"] == "no-store"
|
|
assert "x-missioncore-ui-build" not in response.headers
|
|
assert json.loads(response.body) == {
|
|
"schema_version": "missioncore.live-viewer-client-contract/v1",
|
|
"status": "frontend-build-unavailable",
|
|
}
|
|
|
|
|
|
def test_development_viewer_diagnostics_do_not_reload_against_dist_build() -> None:
|
|
router = build_viewer_diagnostics_router(
|
|
expected_ui_build_id=lambda: "/assets/index-abcdefgh.js",
|
|
)
|
|
endpoint = _endpoint(router, "/api/v1/viewer/live-diagnostics", "POST")
|
|
event = LiveViewerDiagnosticEvent(
|
|
schema_version="missioncore.live-viewer-diagnostic/v2",
|
|
event_code="live_receiver_active_store_admitted",
|
|
ui_build_id="development",
|
|
document_instance_id="00000000-0000-4000-8000-000000000001",
|
|
viewer_instance_id="00000000-0000-4000-8000-000000000002",
|
|
lifecycle_generation=1,
|
|
)
|
|
|
|
response = endpoint(event)
|
|
|
|
assert response.status_code == 204
|