SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wip(k1): checkpoint connection recovery rewrite

Browse Source 
Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
DCCONSTRUCTIONS
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/k1link/web/app.py
+51 -15
View File
@@ -11,7 +11,6 @@ from typing import Any
from fastapi import FastAPI, HTTPException, Request, WebSocket, WebSocketDisconnect
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from fastapi.staticfiles import StaticFiles
from pydantic import ValidationError
from starlette.middleware.gzip import GZipMiddleware
@@ -76,6 +75,7 @@ from k1link.web.e46j_raw_fisheye_realtime_api import (
)
from k1link.web.e47_semantic_slam_api import build_e47_semantic_slam_router
from k1link.web.environment_api import build_environment_router
from k1link.web.frontend_assets import ControlStationStaticFiles, frontend_build_id
from k1link.web.l3_pointpillars_visual_api import (
build_l3_pointpillars_visual_router,
)
@@ -385,7 +385,7 @@ async def _recording_preparation_reconciler() -> None:
async def app_lifespan(_: FastAPI) -> AsyncIterator[None]:
reconciler: asyncio.Task[None] | None = None
try:
configure_scanner_diagnostics(REPOSITORY_ROOT / ".runtime" / "mission-core" / "logs")
configure_scanner_diagnostics(session_store.data_dir / "logs")
session_recording_preparation_manager.start()
# Recovery is intentionally a one-shot startup phase. The archive
# helper owns a cross-process lease, while ordinary catalog requests
@@ -480,11 +480,36 @@ async def invoke_device_plugin_action(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except PluginExecutionError as exc:
raise HTTPException(status_code=502, detail=str(exc)) from exc
raise HTTPException(status_code=exc.http_status_code, detail=str(exc)) from exc
except PluginRuntimeUnavailableError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
_CLOSED_WEBSOCKET_SEND_ERRORS = frozenset(
{
"handler is closed",
'Cannot call "send" once a close message has been sent.',
(
"Unexpected ASGI message 'websocket.send', after sending "
"'websocket.close' or response already completed."
),
}
)
def _is_closed_websocket_send_error(exc: RuntimeError) -> bool:
message = " ".join(str(exc).split())
if message in _CLOSED_WEBSOCKET_SEND_ERRORS:
return True
# uvloop raises this after the browser has already torn down the TCP
# transport, before Starlette can translate the failed send into a
# WebSocketDisconnect. Match both stable parts so an unrelated RuntimeError
# containing only "closed" is never swallowed.
return message.startswith("unable to perform operation on ") and message.endswith(
"; the handler is closed"
)
@app.websocket("/api/v1/device-plugins/{plugin_id}/events")
async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
await websocket.accept()
@@ -493,10 +518,17 @@ async def device_plugin_events(websocket: WebSocket, plugin_id: str) -> None:
while True:
state = await plugin_dispatcher.invoke(plugin_id, STATE_READ_ACTION_ID, {})
sequence += 1
await websocket.send_json({"pluginId": plugin_id, "sequence": sequence, "state": state})
try:
await websocket.send_json(
{"pluginId": plugin_id, "sequence": sequence, "state": state}
)
except WebSocketDisconnect:
return
except RuntimeError as exc:
if _is_closed_websocket_send_error(exc):
return
raise
await asyncio.sleep(0.5)
except WebSocketDisconnect:
return
except (PluginNotFoundError, PluginActionNotFoundError):
await websocket.close(code=1008, reason="Device plugin is not available")
except (PluginExecutionError, PluginRuntimeUnavailableError):
@@ -781,11 +813,7 @@ app.include_router(
app.include_router(
build_e47_semantic_slam_router(
root_provider=lambda: (
REPOSITORY_ROOT
/ ".runtime"
/ "compute-experiments"
/ "e47"
/ "semantic-slam-results"
REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "e47" / "semantic-slam-results"
),
)
)
@@ -1114,9 +1142,17 @@ app.include_router(
root_provider=lambda: REPOSITORY_ROOT / ".runtime" / "system",
)
)
app.include_router(build_viewer_diagnostics_router())
frontend_dist = REPOSITORY_ROOT / "apps" / "control-station" / "dist"
app.include_router(
build_viewer_diagnostics_router(
expected_ui_build_id=lambda: frontend_build_id(frontend_dist),
)
)
if frontend_dist.is_dir():
app.mount("/", StaticFiles(directory=frontend_dist, html=True), name="frontend")
app.mount(
"/",
ControlStationStaticFiles(directory=frontend_dist, html=True),
name="frontend",
)
src/k1link/web/device_lifecycle.py
+55
View File
@@ -82,6 +82,8 @@ class OperationRecord:
result: dict[str, Any] | None = None
error: dict[str, Any] | None = None
evidence_refs: tuple[str, ...] = ()
context: dict[str, Any] = field(default_factory=dict)
events: list[dict[str, Any]] = field(default_factory=list)
# A keyed, non-reversible digest supplied by the service. It is deliberately
# excluded from API snapshots: callers only need mismatch detection, while
# the journal must never retain action inputs or secret material.
@@ -108,6 +110,8 @@ class OperationRecord:
"result": dict(self.result) if self.result is not None else None,
"error": dict(self.error) if self.error is not None else None,
"evidence_refs": list(self.evidence_refs),
"context": dict(self.context),
"events": [dict(event) for event in self.events],
}
@@ -144,6 +148,7 @@ class OperationJournal:
deadline_seconds: float | None = None,
cancellable: bool = False,
request_fingerprint: str | None = None,
context: Mapping[str, Any] | None = None,
) -> tuple[OperationRecord, bool]:
action = action.strip()
if not action:
@@ -189,7 +194,9 @@ class OperationJournal:
),
cancellable=cancellable,
request_fingerprint=request_fingerprint,
context=dict(context or {}),
)
self._append_event_locked(record)
self._records[resolved_id] = record
self._order.append(resolved_id)
if idempotency_key is not None:
@@ -225,6 +232,7 @@ class OperationJournal:
if status in TERMINAL_OPERATION_STATUSES:
record.completed_at = self._clock()
self._trim_locked()
self._append_event_locked(record)
return record
def request_cancel(self, operation_id: str) -> OperationRecord:
@@ -239,6 +247,7 @@ class OperationJournal:
record.state_revision += 1
record.stage_code = "cancellation-requested"
record.message_code = "operation.cancellation_requested"
self._append_event_locked(record)
return record
def transition_if_pending(
@@ -276,12 +285,32 @@ class OperationJournal:
if status in TERMINAL_OPERATION_STATUSES:
record.completed_at = self._clock()
self._trim_locked()
self._append_event_locked(record)
return record
def get(self, operation_id: str) -> OperationRecord:
with self._lock:
return self._require_locked(operation_id)
def deadline_reached(self, operation_id: str | None) -> bool:
"""Evaluate one operation deadline on the journal-owned server clock.
Device lifecycle reducers use this instead of browser timers or a
second wall-clock source. Terminal records deliberately retain the
same answer so an idempotent local-cleanup retry can continue after the
operation outcome itself has already been sealed.
"""
if operation_id is None:
return False
with self._lock:
record = self._records.get(operation_id)
return bool(
record is not None
and record.deadline_at is not None
and self._clock() >= record.deadline_at
)
def latest(self) -> OperationRecord | None:
with self._lock:
return self._records[self._order[-1]] if self._order else None
@@ -298,6 +327,32 @@ class OperationJournal:
except KeyError as exc:
raise KeyError(f"unknown operation: {operation_id}") from exc
def _append_event_locked(self, record: OperationRecord) -> None:
"""Append one bounded, secret-free stage fact for operator diagnosis."""
error = record.error or {}
result = record.result or {}
side_effect_status = error.get("side_effect_status", result.get("side_effect_status"))
event: dict[str, Any] = {
"schema_version": "missioncore.operation-event/v1",
"sequence": record.sequence,
"status": record.status,
"stage_code": record.stage_code,
"message_code": record.message_code,
"observed_at": _iso(self._clock()),
"side_effect_status": (
side_effect_status if isinstance(side_effect_status, str) else None
),
"error_code": error.get("code") if isinstance(error.get("code"), str) else None,
"safe_to_retry": (
error.get("safe_to_retry")
if isinstance(error.get("safe_to_retry"), bool)
else None
),
"automatic_retry": False,
}
record.events.append(event)
def _trim_locked(self) -> None:
while len(self._order) > self._max_records:
oldest_id = next(
src/k1link/web/frontend_assets.py
+49
View File
@@ -0,0 +1,49 @@
from __future__ import annotations
import re
from pathlib import Path
from fastapi.staticfiles import StaticFiles
from starlette.responses import Response
from starlette.types import Scope
HTML_NO_STORE = "no-store"
HASHED_ASSET_IMMUTABLE = "public, max-age=31536000, immutable"
_MODULE_SCRIPT = re.compile(
r'<script\b[^>]*\bsrc=["\'](?P<src>/assets/[^"\']+)["\'][^>]*>',
re.IGNORECASE,
)
_HASHED_ASSET = re.compile(
r"^assets/(?:.+)-[A-Za-z0-9_-]{8,}\.[A-Za-z0-9]+$",
)
def frontend_build_id(frontend_root: Path) -> str | None:
"""Return the exact content-hashed module loaded by the current index."""
try:
index = (frontend_root / "index.html").read_text(encoding="utf-8")
except OSError:
return None
for match in _MODULE_SCRIPT.finditer(index):
source = match.group("src")
if _HASHED_ASSET.fullmatch(source.removeprefix("/")):
return source
return None
class ControlStationStaticFiles(StaticFiles):
"""Serve the SPA shell fresh while retaining immutable hashed assets."""
async def get_response(self, path: str, scope: Scope) -> Response:
response = await super().get_response(path, scope)
if response.status_code >= 400:
return response
content_type = response.headers.get("content-type", "").lower()
normalized = path.lstrip("/")
if content_type.startswith("text/html"):
response.headers["Cache-Control"] = HTML_NO_STORE
elif _HASHED_ASSET.fullmatch(normalized):
response.headers["Cache-Control"] = HASHED_ASSET_IMMUTABLE
return response
src/k1link/web/plugin_runtime.py
+15 -6
View File
@@ -68,6 +68,19 @@ class PluginActionNotFoundError(LookupError):
class PluginExecutionError(RuntimeError):
"""A validated plugin action failed while talking to its device/runtime."""
def __init__(
self,
message: str,
*,
http_status_code: int = 502,
reason_code: str | None = None,
) -> None:
super().__init__(message)
if not 400 <= http_status_code <= 599:
raise ValueError("plugin execution HTTP status must be an error response")
self.http_status_code = http_status_code
self.reason_code = reason_code
class PluginRuntimeCompatibilityError(RuntimeError):
"""A plugin runtime cannot satisfy the reviewed manifest/host contract."""
@@ -124,9 +137,7 @@ class InProcessDevicePluginRuntime:
raise PluginRuntimeCompatibilityError(
"Runtime does not support the manifest host API version"
)
if frozenset(request.required_action_ids) != frozenset(
self.descriptor.action_ids
):
if frozenset(request.required_action_ids) != frozenset(self.descriptor.action_ids):
raise PluginRuntimeCompatibilityError(
"Runtime handshake actions do not match the manifest"
)
@@ -225,9 +236,7 @@ class DevicePluginDispatcher:
@property
def health_snapshots(self) -> dict[str, RuntimeHealthSnapshot]:
return {
plugin_id: runtime.health() for plugin_id, runtime in self._runtimes.items()
}
return {plugin_id: runtime.health() for plugin_id, runtime in self._runtimes.items()}
async def invoke(
self,
src/k1link/web/runtime_diagnostics.py
+4
View File
@@ -55,6 +55,10 @@ _EXTRA_FIELDS: Final = (
"start_checkpoint_released",
"stale_session_retired",
"stream_id",
"ui_build_id",
"document_instance_id",
"viewer_instance_id",
"lifecycle_generation",
"backend_activity_sequence",
"viewer_range_max_ns",
"stalled_for_ms",
src/k1link/web/viewer_diagnostics_api.py
+87 -4
View File
@@ -1,9 +1,11 @@
from __future__ import annotations
import logging
from collections.abc import Callable
from typing import Literal
from fastapi import APIRouter, Response
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field
logger = logging.getLogger("k1link.device_plugins.xgrids_k1.viewer_receiver")
@@ -23,12 +25,31 @@ LiveViewerFailureStage = Literal[
"receiver-stalled",
]
LIVE_VIEWER_DIAGNOSTIC_SCHEMA = "missioncore.live-viewer-diagnostic/v2"
LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA = "missioncore.live-viewer-client-contract/v1"
UI_BUILD_HEADER = "X-MissionCore-UI-Build"
_INSTANCE_ID_PATTERN = r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
_UI_BUILD_ID_PATTERN = r"^(?:development|/assets/[A-Za-z0-9._/-]+)$"
class LiveViewerDiagnosticEvent(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal["missioncore.live-viewer-diagnostic/v1"]
schema_version: Literal["missioncore.live-viewer-diagnostic/v2"]
event_code: LiveViewerEventCode
ui_build_id: str = Field(min_length=1, max_length=256, pattern=_UI_BUILD_ID_PATTERN)
document_instance_id: str = Field(
min_length=36,
max_length=36,
pattern=_INSTANCE_ID_PATTERN,
)
viewer_instance_id: str = Field(
min_length=36,
max_length=36,
pattern=_INSTANCE_ID_PATTERN,
)
lifecycle_generation: int = Field(ge=1)
failure_stage: LiveViewerFailureStage | None = None
stream_id: str | None = Field(
default=None,
@@ -42,17 +63,79 @@ class LiveViewerDiagnosticEvent(BaseModel):
recovery_attempt: int | None = Field(default=None, ge=1, le=3)
def build_viewer_diagnostics_router() -> APIRouter:
def build_viewer_diagnostics_router(
*,
expected_ui_build_id: Callable[[], str | None] = lambda: "development",
) -> APIRouter:
router = APIRouter(prefix="/api/v1/viewer", tags=["viewer"])
@router.get("/client-contract")
def get_live_viewer_client_contract() -> JSONResponse:
expected = expected_ui_build_id()
if expected is None:
return JSONResponse(
status_code=503,
content={
"schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA,
"status": "frontend-build-unavailable",
},
headers={"Cache-Control": "no-store"},
)
return JSONResponse(
content={
"schema_version": LIVE_VIEWER_CLIENT_CONTRACT_SCHEMA,
"status": "ready",
"ui_build_id": expected,
"diagnostic_schema_version": LIVE_VIEWER_DIAGNOSTIC_SCHEMA,
},
headers={
"Cache-Control": "no-store",
UI_BUILD_HEADER: expected,
},
)
@router.post("/live-diagnostics", status_code=204)
def record_live_diagnostic(event: LiveViewerDiagnosticEvent) -> Response:
expected = expected_ui_build_id()
if expected is None:
return JSONResponse(
status_code=503,
content={"detail": "frontend-build-unavailable"},
headers={"Cache-Control": "no-store"},
)
# Vite's unhashed development entry intentionally has no deploy
# identity and the browser-side monitor is disabled for it. Keep local
# diagnostics usable without turning a dist build into a reload loop;
# every production document carries the exact hashed entry below.
if event.ui_build_id not in {expected, "development"}:
return JSONResponse(
status_code=409,
content={
"detail": "stale-ui-build",
"expected_ui_build_id": expected,
},
headers={
"Cache-Control": "no-store",
UI_BUILD_HEADER: expected,
},
)
logger.info(
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s",
(
"Mission Core live Rerun receiver diagnostic: event=%s stream=%s "
"document=%s viewer=%s lifecycle=%s build=%s"
),
event.event_code,
event.stream_id,
event.document_instance_id,
event.viewer_instance_id,
event.lifecycle_generation,
event.ui_build_id,
extra={
"event_code": event.event_code,
"ui_build_id": event.ui_build_id,
"document_instance_id": event.document_instance_id,
"viewer_instance_id": event.viewer_instance_id,
"lifecycle_generation": event.lifecycle_generation,
"failure_stage": event.failure_stage,
"stream_id": event.stream_id,
"backend_activity_sequence": event.backend_activity_sequence,
@@ -61,6 +144,6 @@ def build_viewer_diagnostics_router() -> APIRouter:
"recovery_attempt": event.recovery_attempt,
},
)
return Response(status_code=204)
return Response(status_code=204, headers={UI_BUILD_HEADER: expected})
return router