fix(k1): stabilize repeated acquisition and live viewer recovery

apps/control-station/src/core/observation/liveReceiverWatchdog.ts

@@ -0,0 +1,156 @@
+export interface LiveReceiverWatchdogState {
+  lastBackendActivitySequence: number | null;
+  lastViewerRangeMaxNs: number | null;
+  staticRangeBackendAdvances: number;
+  stalledSinceMs: number | null;
+  stallReported: boolean;
+}
+
+export type LiveReceiverWatchdogSignal = "none" | "receiver-advanced" | "stalled";
+
+export interface LiveReceiverWatchdogResult {
+  state: LiveReceiverWatchdogState;
+  signal: LiveReceiverWatchdogSignal;
+  stalledForMs: number;
+}
+
+export interface LiveReceiverRecoveryState {
+  attempts: number;
+  awaitingRecovery: boolean;
+}
+
+export type LiveReceiverRecoverySignal = "retry" | "exhausted";
+
+export interface LiveReceiverRecoveryResult {
+  state: LiveReceiverRecoveryState;
+  signal: LiveReceiverRecoverySignal;
+  attempt: number;
+}
+
+export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
+export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
+
+export function initialLiveReceiverWatchdogState(): LiveReceiverWatchdogState {
+  return {
+    lastBackendActivitySequence: null,
+    lastViewerRangeMaxNs: null,
+    staticRangeBackendAdvances: 0,
+    stalledSinceMs: null,
+    stallReported: false,
+  };
+}
+
+export function initialLiveReceiverRecoveryState(): LiveReceiverRecoveryState {
+  return {
+    attempts: 0,
+    awaitingRecovery: false,
+  };
+}
+
+/**
+ * Bound viewer-only restarts independently from scanner and acquisition
+ * lifecycle. The caller may dispose and recreate the browser receiver, but
+ * must never issue START/STOP or reconnect the physical device.
+ */
+export function requestLiveReceiverRecovery(
+  current: LiveReceiverRecoveryState,
+  maxAttempts = LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
+): LiveReceiverRecoveryResult {
+  if (current.attempts >= maxAttempts) {
+    return {
+      state: {
+        attempts: current.attempts,
+        awaitingRecovery: false,
+      },
+      signal: "exhausted",
+      attempt: current.attempts,
+    };
+  }
+  const attempt = current.attempts + 1;
+  return {
+    state: {
+      attempts: attempt,
+      awaitingRecovery: true,
+    },
+    signal: "retry",
+    attempt,
+  };
+}
+
+/**
+ * Detect a disposable receiver stall only when the backend frame counter keeps
+ * advancing while Rerun's visible time range does not. Scanner commands are
+ * deliberately absent from this state machine.
+ */
+export function advanceLiveReceiverWatchdog(
+  current: LiveReceiverWatchdogState,
+  sample: {
+    nowMs: number;
+    backendActivitySequence: number | null;
+    viewerRangeMaxNs: number | null;
+  },
+  thresholdMs = LIVE_RECEIVER_STALL_THRESHOLD_MS,
+): LiveReceiverWatchdogResult {
+  const backendSequence = Number.isSafeInteger(sample.backendActivitySequence) &&
+      (sample.backendActivitySequence ?? -1) >= 0
+    ? sample.backendActivitySequence
+    : null;
+  const viewerRange = Number.isFinite(sample.viewerRangeMaxNs) &&
+      (sample.viewerRangeMaxNs ?? -1) >= 0
+    ? sample.viewerRangeMaxNs
+    : null;
+  const state = { ...current };
+
+  if (state.lastBackendActivitySequence === null && state.lastViewerRangeMaxNs === null) {
+    state.lastBackendActivitySequence = backendSequence;
+    state.lastViewerRangeMaxNs = viewerRange;
+    return { state, signal: "none", stalledForMs: 0 };
+  }
+
+  const backendAdvanced = backendSequence !== null &&
+    (state.lastBackendActivitySequence === null ||
+      backendSequence > state.lastBackendActivitySequence);
+  const receiverAdvanced = viewerRange !== null &&
+    (state.lastViewerRangeMaxNs === null || viewerRange > state.lastViewerRangeMaxNs);
+
+  if (backendSequence !== null) {
+    state.lastBackendActivitySequence = Math.max(
+      backendSequence,
+      state.lastBackendActivitySequence ?? 0,
+    );
+  }
+  if (viewerRange !== null) {
+    state.lastViewerRangeMaxNs = Math.max(viewerRange, state.lastViewerRangeMaxNs ?? 0);
+  }
+
+  if (receiverAdvanced) {
+    const hadStaticBackendProgress = state.stalledSinceMs !== null;
+    state.staticRangeBackendAdvances = 0;
+    state.stalledSinceMs = null;
+    state.stallReported = false;
+    return {
+      state,
+      signal: hadStaticBackendProgress ? "receiver-advanced" : "none",
+      stalledForMs: 0,
+    };
+  }
+
+  if (backendAdvanced) {
+    state.staticRangeBackendAdvances += 1;
+    if (state.staticRangeBackendAdvances >= 2 && state.stalledSinceMs === null) {
+      state.stalledSinceMs = sample.nowMs;
+    }
+  }
+  const stalledForMs = state.stalledSinceMs === null
+    ? 0
+    : Math.max(0, sample.nowMs - state.stalledSinceMs);
+  if (
+    !state.stallReported &&
+    state.staticRangeBackendAdvances >= 2 &&
+    stalledForMs >= thresholdMs
+  ) {
+    state.stallReported = true;
+    return { state, signal: "stalled", stalledForMs };
+  }
+  return { state, signal: "none", stalledForMs };
+}

apps/control-station/src/core/observation/liveViewerDiagnostics.ts

@@ -0,0 +1,61 @@
+export type LiveViewerDiagnosticEventCode =
+  | "live_receiver_stalled"
+  | "live_receiver_restart_requested"
+  | "live_receiver_recovered"
+  | "live_receiver_recovery_exhausted"
+  | "live_receiver_active_store_admitted"
+  | "live_receiver_error";
+
+export type LiveViewerFailureStage =
+  | "recording-open-timeout"
+  | "viewer-start"
+  | "module-load"
+  | "receiver-stalled";
+
+export interface LiveViewerDiagnostic {
+  eventCode: LiveViewerDiagnosticEventCode;
+  failureStage?: LiveViewerFailureStage | null;
+  streamId?: string | null;
+  backendActivitySequence?: number | null;
+  viewerRangeMaxNs?: number | null;
+  stalledForMs?: number | null;
+  recoveryAttempt?: number | null;
+}
+
+const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+
+function safeInteger(value: number | null | undefined): number | undefined {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0
+    ? value
+    : undefined;
+}
+
+export function postLiveViewerDiagnostic(event: LiveViewerDiagnostic): void {
+  const streamId = event.streamId?.trim();
+  const body = {
+    schema_version: "missioncore.live-viewer-diagnostic/v1",
+    event_code: event.eventCode,
+    ...(event.failureStage ? { failure_stage: event.failureStage } : {}),
+    ...(streamId && SAFE_STREAM_ID.test(streamId) ? { stream_id: streamId } : {}),
+    ...(safeInteger(event.backendActivitySequence) === undefined
+      ? {}
+      : { backend_activity_sequence: safeInteger(event.backendActivitySequence) }),
+    ...(safeInteger(event.viewerRangeMaxNs) === undefined
+      ? {}
+      : { viewer_range_max_ns: safeInteger(event.viewerRangeMaxNs) }),
+    ...(safeInteger(event.stalledForMs) === undefined
+      ? {}
+      : { stalled_for_ms: safeInteger(event.stalledForMs) }),
+    ...(safeInteger(event.recoveryAttempt) === undefined
+      ? {}
+      : { recovery_attempt: safeInteger(event.recoveryAttempt) }),
+  };
+  void fetch("/api/v1/viewer/live-diagnostics", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(body),
+    keepalive: true,
+  }).catch(() => {
+    // Diagnostics must never interfere with the live receiver recovery path.
+  });
+}

apps/control-station/src/core/runtime/contracts.ts

@@ -28,6 +28,7 @@ export interface ViewerSettings {
 }
 
 export interface StreamMetrics {
+  publishedFrameCount?: number | null;
   latencyMs?: number | null;
   frameRateHz?: number | null;
   pointCount?: number | null;