wip(k1): checkpoint connection recovery rewrite
Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
+51
-15
@@ -11,7 +11,6 @@ from typing import Any
|
||||
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import ValidationError
|
||||
from starlette.middleware.gzip import GZipMiddleware
|
||||
|
||||
@@ -76,6 +75,7 @@ from k1link.web.e46j_raw_fisheye_realtime_api import (
|
||||
)
|
||||
from k1link.web.e47_semantic_slam_api import build_e47_semantic_slam_router
|
||||
from k1link.web.environment_api import build_environment_router
|
||||
from k1link.web.frontend_assets import ControlStationStaticFiles, frontend_build_id
|
||||
from k1link.web.l3_pointpillars_visual_api import (
|
||||
build_l3_pointpillars_visual_router,
|
||||
)
|
||||
@@ -385,7 +385,7 @@ async def _recording_preparation_reconciler() -> None:
|
||||
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
reconciler: asyncio.Task[None] | None = None
|
||||
try:
|
||||
configure_scanner_diagnostics(REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs")
|
||||
configure_scanner_diagnostics(session_store.data_dir / "logs")
|
||||
session_recording_preparation_manager.start()
|
||||
# Recovery is intentionally a one-shot startup phase. The archive
|
||||
# helper owns a cross-process lease, while ordinary catalog requests
|
||||
@@ -480,11 +480,36 @@ async def invoke_device_plugin_action(
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except PluginExecutionError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
raise HTTPException(status_code=exc.http_status_code, detail=str(exc)) from exc
|
||||
except PluginRuntimeUnavailableError as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
||||
|
||||
|
||||
_CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
|
||||
{
|
||||
"handler is closed",
|
||||
'Cannot call "send" once a close message has been sent.',
|
||||
(
|
||||
"Unexpected ASGI message 'websocket.send', after sending "
|
||||
"'websocket.close' or response already completed."
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
|
||||
message = " ".join(str(exc).split())
|
||||
if message in _CLOSED_WEBSOCKET_SEND_ERRORS:
|
||||
return True
|
||||
# uvloop raises this after the browser has already torn down the TCP
|
||||
# transport, before Starlette can translate the failed send into a
|
||||
# WebSocketDisconnect. Match both stable parts so an unrelated RuntimeError
|
||||
# containing only "closed" is never swallowed.
|
||||
return message.startswith("unable to perform operation on ") and message.endswith(
|
||||
"; the handler is closed"
|
||||
)
|
||||
|
||||
|
||||
@app.websocket("/api/v1/device-plugins/{plugin_id}/events")
|
||||
async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
|
||||
await websocket.accept()
|
||||
@@ -493,10 +518,17 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
|
||||
while True:
|
||||
state = await plugin_dispatcher.invoke(plugin_id, STATE_READ_ACTION_ID, {})
|
||||
sequence += 1
|
||||
await websocket.send_json({"pluginId": plugin_id, "sequence": sequence, "state": state})
|
||||
try:
|
||||
await websocket.send_json(
|
||||
{"pluginId": plugin_id, "sequence": sequence, "state": state}
|
||||
)
|
||||
except WebSocketDisconnect:
|
||||
return
|
||||
except RuntimeError as exc:
|
||||
if _is_closed_websocket_send_error(exc):
|
||||
return
|
||||
raise
|
||||
await asyncio.sleep(0.5)
|
||||
except WebSocketDisconnect:
|
||||
return
|
||||
except (PluginNotFoundError, PluginActionNotFoundError):
|
||||
await websocket.close(code=1008, reason="Device plugin is not available")
|
||||
except (PluginExecutionError, PluginRuntimeUnavailableError):
|
||||
@@ -781,11 +813,7 @@ app.include_router(
|
||||
app.include_router(
|
||||
build_e47_semantic_slam_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "e47"
|
||||
/ "semantic-slam-results"
|
||||
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e47" / "semantic-slam-results"
|
||||
),
|
||||
)
|
||||
)
|
||||
@@ -1114,9 +1142,17 @@ app.include_router(
|
||||
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
|
||||
)
|
||||
)
|
||||
app.include_router(build_viewer_diagnostics_router())
|
||||
|
||||
|
||||
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
|
||||
app.include_router(
|
||||
build_viewer_diagnostics_router(
|
||||
expected_ui_build_id=lambda: frontend_build_id(frontend_dist),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if frontend_dist.is_dir():
|
||||
app.mount("/", StaticFiles(directory=frontend_dist, html=True), name="frontend")
|
||||
app.mount(
|
||||
"/",
|
||||
ControlStationStaticFiles(directory=frontend_dist, html=True),
|
||||
name="frontend",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user