fix(k1): stabilize repeated acquisition and live viewer recovery

2026-07-29 01:55:12 +03:00
parent 37b8930527
commit 2a97cf28c0
28 changed files with 2776 additions and 286 deletions
@@ -1,6 +1,15 @@
 import { useEffect, useRef, useState } from "react";

 import type { SceneSettings } from "../sceneSettings";
+import {
+  advanceLiveReceiverWatchdog,
+  initialLiveReceiverRecoveryState,
+  initialLiveReceiverWatchdogState,
+  LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
+  requestLiveReceiverRecovery,
+} from "../core/observation/liveReceiverWatchdog";
+import { postLiveViewerDiagnostic } from "../core/observation/liveViewerDiagnostics";
+import type { LiveViewerFailureStage } from "../core/observation/liveViewerDiagnostics";
 import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";

 export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
@@ -66,6 +75,8 @@ export interface RerunViewportProps {
  sourceUrl: string;
  recordedArtifact?: RecordedRrdArtifactDescriptor | null;
  followLive?: boolean;
+  liveActivitySequence?: number | null;
+  liveStreamId?: string | null;
  autoplayWhenReady?: boolean;
  presentationGate?: RecordedAdmissionPhase;
  expectedTimelineStartSeconds?: number;
@@ -796,6 +807,8 @@ export function RerunViewport({
  sourceUrl,
  recordedArtifact = null,
  followLive = false,
+  liveActivitySequence = null,
+  liveStreamId = null,
  autoplayWhenReady = false,
  presentationGate = "ready",
  expectedTimelineStartSeconds,
@@ -824,6 +837,11 @@ export function RerunViewport({
  const [status, setStatus] = useState<RerunViewportStatus>(sourceUrl ? "loading" : "idle");
  const [recordingBufferProgress, setRecordingBufferProgress] = useState<number | null>(null);
  const [retryNonce, setRetryNonce] = useState(0);
+  const liveActivitySequenceRef = useRef<number | null>(liveActivitySequence);
+  liveActivitySequenceRef.current = liveActivitySequence;
+  const liveStreamIdRef = useRef<string | null>(liveStreamId);
+  liveStreamIdRef.current = liveStreamId;
+  const liveRecoveryRef = useRef(initialLiveReceiverRecoveryState());
  const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
  const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
  const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
@@ -849,6 +867,10 @@ export function RerunViewport({
    recordedArtifact !== null,
  );

+  useEffect(() => {
+    liveRecoveryRef.current = initialLiveReceiverRecoveryState();
+  }, [followLive, liveStreamId, sourceUrl]);
+
  useEffect(() => {
    const normalizedSource = sourceUrl.trim();
    const host = hostRef.current;
@@ -889,6 +911,7 @@ export function RerunViewport({
    let disposed = false;
    let disposeViewer: (() => void) | undefined;
    let recordingOpenTimer: number | undefined;
+    let liveRecordingDiscoveryTimer: number | undefined;
    let recordedOpenWatchdog: {
      arm: () => void;
      clear: () => void;
@@ -898,6 +921,8 @@ export function RerunViewport({
    let recordingOpened = false;
    let recordingOpenTimedOut = false;
    let viewerStartResolved = false;
+    let latestLiveRangeMaxNs: number | null = null;
+    let liveWatchdog = initialLiveReceiverWatchdogState();
    // The backend serves this exact digest-bound URL or rejects it with 412.
    // Presentation still waits for Rerun to decode the complete declared range.
    const artifactVerified = true;
@@ -925,9 +950,16 @@ export function RerunViewport({
        recordingOpenTimer = undefined;
      }
    };
+    const clearLiveRecordingDiscoveryTimer = () => {
+      if (liveRecordingDiscoveryTimer !== undefined) {
+        window.clearInterval(liveRecordingDiscoveryTimer);
+        liveRecordingDiscoveryTimer = undefined;
+      }
+    };
    const clearRecordingTimers = () => {
      clearRecordedAdmissionWatchdog();
      clearLiveRecordingOpenTimer();
+      clearLiveRecordingDiscoveryTimer();
    };
    const clearPlaybackRangeTimer = () => {
      if (playbackRangeTimer === undefined) return;
@@ -950,11 +982,118 @@ export function RerunViewport({
      requestFrame: (callback) => window.requestAnimationFrame(callback),
      cancelFrame: (handle) => window.cancelAnimationFrame(handle),
    });
-    const reportError = (message: string) => {
+    const reportError = (message: string, failureStage?: LiveViewerFailureStage) => {
      if (disposed) return;
+      if (followLive) {
+        postLiveViewerDiagnostic({
+          eventCode: "live_receiver_error",
+          failureStage,
+          streamId: liveStreamIdRef.current,
+          backendActivitySequence: liveActivitySequenceRef.current,
+          viewerRangeMaxNs: latestLiveRangeMaxNs,
+          recoveryAttempt: liveRecoveryRef.current.attempts || null,
+        });
+      }
      setStatus("error");
      onStatusChange?.("error", message);
    };
+    const requestLiveRecovery = (
+      failureStage: LiveViewerFailureStage,
+      stalledForMs = 0,
+      emitErrorEvent = true,
+    ) => {
+      if (!followLive || disposed) return false;
+      if (emitErrorEvent) {
+        postLiveViewerDiagnostic({
+          eventCode: "live_receiver_error",
+          failureStage,
+          streamId: liveStreamIdRef.current,
+          backendActivitySequence: liveActivitySequenceRef.current,
+          viewerRangeMaxNs: latestLiveRangeMaxNs,
+          stalledForMs: Math.round(stalledForMs) || null,
+          recoveryAttempt: liveRecoveryRef.current.attempts || null,
+        });
+      }
+      const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current);
+      liveRecoveryRef.current = recovery.state;
+      if (recovery.signal === "exhausted") {
+        postLiveViewerDiagnostic({
+          eventCode: "live_receiver_recovery_exhausted",
+          failureStage,
+          streamId: liveStreamIdRef.current,
+          backendActivitySequence: liveActivitySequenceRef.current,
+          viewerRangeMaxNs: latestLiveRangeMaxNs,
+          stalledForMs: Math.round(stalledForMs) || null,
+          recoveryAttempt: LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
+        });
+        setStatus("error");
+        onStatusChange?.(
+          "error",
+          "Живой поток продолжает поступать, но визуализатор не восстановился после трёх переподключений.",
+        );
+        return true;
+      }
+      postLiveViewerDiagnostic({
+        eventCode: "live_receiver_restart_requested",
+        failureStage,
+        streamId: liveStreamIdRef.current,
+        backendActivitySequence: liveActivitySequenceRef.current,
+        viewerRangeMaxNs: latestLiveRangeMaxNs,
+        stalledForMs: Math.round(stalledForMs) || null,
+        recoveryAttempt: recovery.attempt,
+      });
+      setStatus("loading");
+      onStatusChange?.(
+        "loading",
+        "Живой визуализатор переподключается к продолжающемуся потоку.",
+      );
+      disposeViewer?.();
+      setRetryNonce((nonce) => nonce + 1);
+      return true;
+    };
+    const observeLiveReceiver = (viewerRangeMaxNs: number | null) => {
+      if (!followLive || disposed) return;
+      latestLiveRangeMaxNs = viewerRangeMaxNs;
+      const receiverOpenedAfterRecovery =
+        liveRecoveryRef.current.awaitingRecovery &&
+        liveWatchdog.lastViewerRangeMaxNs === null &&
+        viewerRangeMaxNs !== null;
+      const observed = advanceLiveReceiverWatchdog(liveWatchdog, {
+        nowMs: Date.now(),
+        backendActivitySequence: liveActivitySequenceRef.current,
+        viewerRangeMaxNs,
+      });
+      liveWatchdog = observed.state;
+      if (
+        (receiverOpenedAfterRecovery || observed.signal === "receiver-advanced") &&
+        liveRecoveryRef.current.awaitingRecovery
+      ) {
+        postLiveViewerDiagnostic({
+          eventCode: "live_receiver_recovered",
+          streamId: liveStreamIdRef.current,
+          backendActivitySequence: liveActivitySequenceRef.current,
+          viewerRangeMaxNs,
+          recoveryAttempt: liveRecoveryRef.current.attempts,
+        });
+        liveRecoveryRef.current = initialLiveReceiverRecoveryState();
+        return;
+      }
+      if (observed.signal !== "stalled") return;
+
+      postLiveViewerDiagnostic({
+        eventCode: "live_receiver_stalled",
+        failureStage: "receiver-stalled",
+        streamId: liveStreamIdRef.current,
+        backendActivitySequence: liveActivitySequenceRef.current,
+        viewerRangeMaxNs,
+        stalledForMs: Math.round(observed.stalledForMs),
+        recoveryAttempt: Math.min(
+          liveRecoveryRef.current.attempts + 1,
+          LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
+        ),
+      });
+      requestLiveRecovery("receiver-stalled", observed.stalledForMs, false);
+    };

    host.replaceChildren();
    appliedPointColorKeyRef.current = null;
@@ -1016,7 +1155,7 @@ export function RerunViewport({
            viewer.stop();
          } catch {
            // A partially initialized WASM handle can already be gone after a
-            // startup failure. The host still has to be cleared below.
+            // startup failure.
          }
          host.replaceChildren();
        };
@@ -1034,14 +1173,20 @@ export function RerunViewport({
          });
        }

-        unsubscribers.push(
-          viewer.on("recording_open", (event) => {
+        const admitRecording = (event: {
+          application_id: string;
+          recording_id: string;
+        }) => {
            if (
              disposed ||
              recordingOpened ||
              (isRecordedSource && event.application_id !== "nodedc_mission_core_recorded")
            ) return;
            recordingOpened = true;
+            if (!isRecordedSource) {
+              clearLiveRecordingOpenTimer();
+              clearLiveRecordingDiscoveryTimer();
+            }
            if (
              recordedBlueprintUrl &&
              event.application_id === "nodedc_mission_core_recorded" &&
@@ -1060,7 +1205,6 @@ export function RerunViewport({
                setPerceptionChannelRevision((revision) => revision + 1);
              }
            }
-            if (!isRecordedSource) clearLiveRecordingOpenTimer();
            // Durable Mission Core recordings use a zero-based session clock.
            // Keeping wall-clock capture_time as a secondary RRD timeline is
            // useful for audit, but the operator scrubber must remain a small,
@@ -1144,6 +1288,7 @@ export function RerunViewport({
              } catch {
                return;
              }
+              observeLiveReceiver(rangeNs?.max ?? null);
              const recordedBuffer = recordedPlaybackBufferState(
                rangeNs,
                expectedTimelineEndSeconds,
@@ -1225,8 +1370,31 @@ export function RerunViewport({
            clearPlaybackRangeTimer();
            publishBufferedState();
            playbackRangeTimer = window.setInterval(publishBufferedState, 500);
-          }),
+        };
+        unsubscribers.push(
+          viewer.on("recording_open", admitRecording),
        );
+        const discoverActiveLiveRecording = () => {
+          if (isRecordedSource || disposed || recordingOpened || !viewer.ready) return;
+          try {
+            const recordingId = viewer.get_active_recording_id();
+            if (!recordingId) return;
+            // Rerun 0.34.1 may ingest an SDK gRPC store without forwarding its
+            // recording_open event to the JavaScript wrapper. The active store
+            // is the authoritative fallback and avoids hiding a ready canvas.
+            postLiveViewerDiagnostic({
+              eventCode: "live_receiver_active_store_admitted",
+              streamId: liveStreamIdRef.current,
+              backendActivitySequence: liveActivitySequenceRef.current,
+            });
+            admitRecording({
+              application_id: "nodedc_mission_core_spatial",
+              recording_id: recordingId,
+            });
+          } catch {
+            // The native receiver is still opening; the bounded timer retries.
+          }
+        };

        unsubscribers.push(
          viewer.on("time_update", (event) => {
@@ -1316,25 +1484,34 @@ export function RerunViewport({
            setPerceptionChannelRevision((revision) => revision + 1);
          }

-          if (!recordingOpened && !isRecordedSource) {
-            recordingOpenTimer = window.setTimeout(() => {
-              recordingOpenTimedOut = true;
-              disposeViewer?.();
-              reportError("Визуализатор запущен, но поток записи не открылся.");
-            }, 12_000);
+          if (!isRecordedSource) {
+            discoverActiveLiveRecording();
+            if (!recordingOpened) {
+              liveRecordingDiscoveryTimer = window.setInterval(
+                discoverActiveLiveRecording,
+                100,
+              );
+              recordingOpenTimer = window.setTimeout(() => {
+                recordingOpenTimedOut = true;
+                requestLiveRecovery("recording-open-timeout");
+              }, 12_000);
+            }
          }
        } catch {
          disposeViewer();
          if (!recordingOpenTimedOut && !disposed) {
-            reportError(isRecordedSource
+            const message = isRecordedSource
              ? "Запись не прошла атомарную проверку целостности."
-              : "Не удалось запустить встроенный визуализатор.");
+              : "Не удалось запустить встроенный визуализатор.";
+            if (!isRecordedSource && requestLiveRecovery("viewer-start")) return;
+            reportError(message);
          }
        }
      })
      .catch(() => {
        if (disposed) return;
        disposeViewer?.();
+        if (requestLiveRecovery("module-load")) return;
        reportError("Не удалось загрузить модуль визуализатора.");
      });

@@ -1355,6 +1532,7 @@ export function RerunViewport({
    expectedTimelineStartSeconds,
      followLive,
    initialPlaybackStartSeconds,
+    liveStreamId,
    onPlaybackChange,
    onPlaybackControllerChange,
    onSelectionChange,