wip(k1): checkpoint connection recovery rewrite

Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
DCCONSTRUCTIONS
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
@@ -2,6 +2,14 @@ import { useEffect, useRef, useState } from "react";
import { Icon } from "@nodedc/ui-react";
import type { ObservationSourceDelivery } from "../core/runtime/contracts";
import {
cameraBrowserTransportIdentity,
cameraTransportCallbackIsCurrent,
initialCameraPlaybackRecoveryState,
reduceCameraPlaybackRecovery,
type CameraPlaybackRecoveryEvent,
} from "../core/observation/liveCameraRecovery";
import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics";
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
@@ -10,7 +18,92 @@ export interface CameraLeaseRetryBudget {
count: number;
}
const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000] as const;
const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000, 5_000] as const;
export const CAMERA_FIRST_MEDIA_TIMEOUT_MS = 8_000;
export const CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS = 8_000;
// The gateway may release an 8 MiB / 64-fragment slow-reader backlog after a
// main-thread stall. Keep one bounded append margin above that complete batch;
// crossing either limit replaces the MSE epoch instead of dropping fragments.
export const CAMERA_PENDING_QUEUE_MAX_BYTES = 12 * 1024 * 1024;
export const CAMERA_PENDING_QUEUE_MAX_SEGMENTS = 96;
export type CameraStartupWatchdogStage = "first-media" | "first-playable-frame";
export interface CameraStartupWatchdog {
armFirstMedia: () => void;
markMediaReceived: () => void;
markPlaying: () => void;
clear: () => void;
pendingStage: () => CameraStartupWatchdogStage | null;
}
/**
* Supervise only the disposable browser transport. One fixed first-media
* deadline covers both an MSE that never opens and an open-but-silent socket.
* The first non-empty fragment then starts a separate first-playable-frame
* deadline; later fragments deliberately do not extend it.
*/
export function createCameraStartupWatchdog({
schedule,
cancel,
onTimeout,
firstMediaTimeoutMs = CAMERA_FIRST_MEDIA_TIMEOUT_MS,
firstPlayableFrameTimeoutMs = CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS,
}: {
schedule: (callback: () => void, timeoutMs: number) => number;
cancel: (handle: number) => void;
onTimeout: (stage: CameraStartupWatchdogStage) => void;
firstMediaTimeoutMs?: number;
firstPlayableFrameTimeoutMs?: number;
}): CameraStartupWatchdog {
let timer: number | undefined;
let stage: CameraStartupWatchdogStage | null = null;
let mediaReceived = false;
let playing = false;
const clearTimer = () => {
if (timer !== undefined) cancel(timer);
timer = undefined;
stage = null;
};
const arm = (nextStage: CameraStartupWatchdogStage, timeoutMs: number) => {
clearTimer();
stage = nextStage;
timer = schedule(() => {
if (stage !== nextStage || playing) return;
timer = undefined;
stage = null;
onTimeout(nextStage);
}, timeoutMs);
};
return {
armFirstMedia() {
if (mediaReceived || playing || stage !== null) return;
arm("first-media", firstMediaTimeoutMs);
},
markMediaReceived() {
if (mediaReceived || playing) return;
mediaReceived = true;
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
},
markPlaying() {
if (playing) return;
playing = true;
clearTimer();
},
clear: clearTimer,
pendingStage: () => stage,
};
}
export function cameraStartupWatchdogRecoveryMessage(
stage: CameraStartupWatchdogStage,
): string {
return stage === "first-media"
? "Камера подключена, но не передаёт медиаданные; восстанавливаем browser-preview."
: "Медиаданные поступают, но первый кадр не воспроизводится; пересоздаём decoder.";
}
export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetryBudget {
return { deliveryId, count: 0 };
@@ -19,15 +112,60 @@ export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetr
export function consumeCameraLeaseRetry(
current: CameraLeaseRetryBudget,
deliveryId: string,
): { budget: CameraLeaseRetryBudget; delay: number | null } {
): { budget: CameraLeaseRetryBudget; delay: number } {
const count = current.deliveryId === deliveryId ? current.count : 0;
const delay = CAMERA_LEASE_RETRY_DELAYS[count] ?? null;
const delayIndex = Math.min(count, CAMERA_LEASE_RETRY_DELAYS.length - 1);
const delay = CAMERA_LEASE_RETRY_DELAYS[delayIndex];
return {
budget: { deliveryId, count: delay === null ? count : count + 1 },
budget: {
deliveryId,
count: Math.min(count + 1, CAMERA_LEASE_RETRY_DELAYS.length),
},
delay,
};
}
export function cameraTransportRecoveryIsCurrent(
activeAuthorityIdentity: string | null,
expectedAuthorityIdentity: string | null,
activeEpoch: number,
expectedEpoch: number,
disposed: boolean,
): boolean {
return Boolean(
expectedAuthorityIdentity
&& activeAuthorityIdentity === expectedAuthorityIdentity
&& cameraTransportCallbackIsCurrent(activeEpoch, expectedEpoch, disposed),
);
}
export function cameraTransportCanOpen(uiBuildStale: boolean): boolean {
return !uiBuildStale;
}
export function cameraPendingQueueCanAccept(
queuedBytes: number,
queuedSegments: number,
incomingBytes: number,
): boolean {
return incomingBytes > 0
&& queuedBytes + incomingBytes <= CAMERA_PENDING_QUEUE_MAX_BYTES
&& queuedSegments + 1 <= CAMERA_PENDING_QUEUE_MAX_SEGMENTS;
}
export function cameraTransportCloseRecoveryMessage(code: number): string {
if (code === 4_008) {
return "Browser-reader отстал от эфира; восстанавливаем текущую камеру.";
}
if (code === 1_008) {
return "Camera adapter освобождает прежний browser-reader; переподключаемся.";
}
if (code === 1_000) {
return "Browser-preview завершился; восстанавливаем текущую камеру.";
}
return "Browser-preview прерван; восстанавливаем текущую камеру.";
}
function websocketUrl(path: string): string {
const url = new URL(path, window.location.href);
if (url.protocol === "http:") url.protocol = "ws:";
@@ -41,61 +179,149 @@ function websocketUrl(path: string): string {
export function MseFmp4WebSocketPlayer({
delivery,
label,
recoveryAuthorityIdentity,
}: {
delivery: ObservationSourceDelivery & { kind: "mse-fmp4-websocket" };
label: string;
recoveryAuthorityIdentity: string | null;
}) {
const videoRef = useRef<HTMLVideoElement>(null);
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
const recoveryPendingAuthorityRef = useRef<string | null>(null);
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
const uiBuildStaleRef = useRef(false);
const [attempt, setAttempt] = useState(0);
const [uiBuildStale, setUiBuildStale] = useState(false);
const [status, setStatus] = useState<PlayerStatus>("connecting");
const [message, setMessage] = useState("Подключение к локальному видеопотоку");
activeAuthorityRef.current = recoveryAuthorityIdentity;
const transportIdentity = cameraBrowserTransportIdentity(
delivery,
recoveryAuthorityIdentity,
);
useEffect(() => subscribeToLiveViewerBuildFence(() => {
// Setting this local fence tears down the effect-owned WebSocket and MSE
// buffer. It intentionally never invokes a plugin or device command.
uiBuildStaleRef.current = true;
activeTransportDisposeRef.current?.();
setUiBuildStale(true);
}), []);
useEffect(() => {
const video = videoRef.current;
if (!video) return;
if (uiBuildStale || !cameraTransportCanOpen(uiBuildStaleRef.current)) {
video.pause();
video.removeAttribute("src");
video.load();
return;
}
if (transportAuthorityRef.current !== recoveryAuthorityIdentity) {
transportAuthorityRef.current = recoveryAuthorityIdentity;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
}
let disposed = false;
const transportEpoch = transportEpochRef.current + 1;
transportEpochRef.current = transportEpoch;
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
);
let socket: WebSocket | null = null;
let sourceBuffer: SourceBuffer | null = null;
let onBufferUpdateEnd: (() => void) | null = null;
let onBufferError: (() => void) | null = null;
let objectUrl = "";
let retryTimer: number | undefined;
let receivedMedia = false;
let failed = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
const recovering = Boolean(
activeAuthorityRef.current
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
);
const queue: ArrayBuffer[] = [];
let queuedBytes = 0;
const fail = (copy: string) => {
if (disposed || failed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.clear();
failed = true;
recoveryPendingAuthorityRef.current = null;
setStatus("error");
setMessage(copy);
try {
if (socket && socket.readyState < WebSocket.CLOSING) {
socket.close(1011, "Live video buffer reset");
socket.close(4_000, "Live video buffer reset");
}
} catch {
// The manual reconnect button will create a fresh transport and MSE buffer.
}
};
const retryLease = () => {
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
if (retry.delay === null) {
fail("Camera adapter ещё занят предыдущим окном. Подключитесь повторно.");
const retryTransport = (copy: string) => {
if (!transportIsCurrent() || failed) return;
if (!cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
recoveryAuthorityIdentity,
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
)) {
fail(copy);
return;
}
startupWatchdog?.clear();
failed = true;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage("Освобождение предыдущего окна камеры");
setMessage(copy);
try {
if (socket && socket.readyState < WebSocket.CLOSING) {
socket.close(4_001, "Live video transport recovery");
}
} catch {
// The replacement effect still fences and releases this transport.
}
retryTimer = window.setTimeout(() => {
if (!disposed) setAttempt((value) => value + 1);
if (cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
recoveryAuthorityIdentity,
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
)) {
setAttempt((value) => value + 1);
}
}, retry.delay);
};
if (recoveryAuthorityIdentity) {
startupWatchdog = createCameraStartupWatchdog({
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
cancel: (handle) => window.clearTimeout(handle),
onTimeout: (stage) => {
retryTransport(cameraStartupWatchdogRecoveryMessage(stage));
},
});
}
const onPlaying = () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.markPlaying();
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
setMessage("");
};
@@ -103,25 +329,32 @@ export function MseFmp4WebSocketPlayer({
video.addEventListener("playing", onPlaying);
const appendNext = () => {
if (disposed || !sourceBuffer || sourceBuffer.updating || queue.length === 0) return;
if (
!transportIsCurrent()
|| failed
|| !sourceBuffer
|| sourceBuffer.updating
|| queue.length === 0
) return;
const chunk = queue.shift();
if (!chunk) return;
queuedBytes -= chunk.byteLength;
try {
sourceBuffer.appendBuffer(chunk);
} catch (error) {
fail(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен и сброшен, чтобы не накапливать задержку."
: "Не удалось добавить видеосегмент. Повторите подключение.");
retryTransport(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.");
}
};
const enqueue = (chunk: ArrayBuffer) => {
if (disposed || chunk.byteLength === 0) return;
if (!transportIsCurrent() || failed || chunk.byteLength === 0) return;
startupWatchdog?.markMediaReceived();
// Never drop arbitrary fMP4 fragments: the following samples may depend
// on them. A bounded reset is safer and keeps live latency deterministic.
if (queuedBytes + chunk.byteLength > 2 * 1024 * 1024) {
fail("Видеодекодер не успевает за эфиром. Live-буфер сброшен.");
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер.");
return;
}
queue.push(chunk);
@@ -142,13 +375,16 @@ export function MseFmp4WebSocketPlayer({
}
setStatus("connecting");
setMessage("Подключение к локальному видеопотоку");
setMessage(recovering
? "Восстановление камеры после разрыва браузерного канала"
: "Подключение к локальному видеопотоку");
const mediaSource = new MediaSource();
objectUrl = URL.createObjectURL(mediaSource);
video.src = objectUrl;
startupWatchdog?.armFirstMedia();
const onSourceOpen = () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
try {
sourceBuffer = mediaSource.addSourceBuffer(mediaType);
} catch {
@@ -156,8 +392,8 @@ export function MseFmp4WebSocketPlayer({
return;
}
sourceBuffer.addEventListener("updateend", () => {
if (disposed || !sourceBuffer) return;
onBufferUpdateEnd = () => {
if (!transportIsCurrent() || failed || !sourceBuffer) return;
const buffered = sourceBuffer.buffered;
if (buffered.length > 0) {
const end = buffered.end(buffered.length - 1);
@@ -166,7 +402,9 @@ export function MseFmp4WebSocketPlayer({
if (!receivedMedia) {
receivedMedia = true;
setStatus("buffering");
setMessage("Запуск первого декодированного кадра");
setMessage(recovering
? "Запуск первого кадра восстановленной камеры"
: "Запуск первого декодированного кадра");
void video.play().catch(() => undefined);
}
const removeBefore = end - 3;
@@ -180,10 +418,12 @@ export function MseFmp4WebSocketPlayer({
}
}
appendNext();
});
sourceBuffer.addEventListener("error", () => {
fail("MSE сообщил об ошибке декодирования видеосегмента.");
});
};
onBufferError = () => {
retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder.");
};
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
sourceBuffer.addEventListener("error", onBufferError);
try {
socket = new WebSocket(websocketUrl(delivery.url));
@@ -193,41 +433,49 @@ export function MseFmp4WebSocketPlayer({
}
socket.binaryType = "arraybuffer";
socket.addEventListener("open", () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.armFirstMedia();
setStatus("buffering");
setMessage("Ожидание первого видеокадра");
setMessage(recovering
? "Ожидание первого кадра после восстановления"
: "Ожидание первого видеокадра");
});
socket.addEventListener("message", (event) => {
if (!transportIsCurrent() || failed) return;
if (event.data instanceof ArrayBuffer) {
enqueue(event.data);
} else if (event.data instanceof Blob) {
void event.data.arrayBuffer().then(enqueue).catch(() => {
fail("Получен повреждённый видеосегмент.");
retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал.");
});
}
});
socket.addEventListener("error", () => {
fail("Соединение с локальным video adapter потеряно.");
retryTransport("Связь с локальным video adapter потеряна; переподключаемся.");
});
socket.addEventListener("close", (event) => {
if (!disposed && !failed && event.code === 1008) {
retryLease();
} else if (!disposed) {
fail(event.code === 1000
? "Видеопоток завершён. Можно подключиться повторно."
: "Видеопоток прерван. Проверьте устройство и повторите подключение.");
}
if (!transportIsCurrent() || failed) return;
retryTransport(cameraTransportCloseRecoveryMessage(event.code));
});
};
mediaSource.addEventListener("sourceopen", onSourceOpen, { once: true });
return () => {
const disposeTransport = () => {
if (disposed) return;
disposed = true;
startupWatchdog?.clear();
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
queue.length = 0;
socket?.close(1000, "Источник скрыт оператором");
socket?.close(1000, "Browser preview transport replaced or hidden");
video.removeEventListener("playing", onPlaying);
mediaSource.removeEventListener("sourceopen", onSourceOpen);
if (sourceBuffer && onBufferUpdateEnd) {
sourceBuffer.removeEventListener("updateend", onBufferUpdateEnd);
}
if (sourceBuffer && onBufferError) {
sourceBuffer.removeEventListener("error", onBufferError);
}
try {
if (sourceBuffer?.updating) sourceBuffer.abort();
} catch {
@@ -243,7 +491,69 @@ export function MseFmp4WebSocketPlayer({
video.load();
if (objectUrl) URL.revokeObjectURL(objectUrl);
};
}, [attempt, delivery.id, delivery.mediaType, delivery.url]);
activeTransportDisposeRef.current = disposeTransport;
return () => {
if (activeTransportDisposeRef.current === disposeTransport) {
activeTransportDisposeRef.current = null;
}
disposeTransport();
};
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
useEffect(() => {
if (
!recoveryAuthorityIdentity
|| uiBuildStale
|| !cameraTransportCanOpen(uiBuildStaleRef.current)
) return;
let recovery = initialCameraPlaybackRecoveryState(
recoveryAuthorityIdentity,
Date.now(),
);
const dispatchRecovery = (event: CameraPlaybackRecoveryEvent) => {
if (uiBuildStaleRef.current) return;
const decision = reduceCameraPlaybackRecovery(recovery, event, {
activeAuthorityIdentity: activeAuthorityRef.current,
now: Date.now(),
documentVisible: document.visibilityState === "visible",
networkOnline: navigator.onLine !== false,
});
recovery = decision.state;
if (!decision.reopen) return;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage("Восстановление камеры после разрыва браузерного канала");
// The transport effect cleans up the old WebSocket and MSE object before
// opening their replacement. No plugin action or device command occurs.
setAttempt((value) => value + 1);
};
const onVisibilityChange = () => {
dispatchRecovery({
type: document.visibilityState === "hidden"
? "document-hidden"
: "document-visible",
});
};
const onOnline = () => dispatchRecovery({ type: "network-online" });
const onPageShow = (event: PageTransitionEvent) => {
dispatchRecovery({ type: "page-restore", persisted: event.persisted });
};
const heartbeat = window.setInterval(() => {
dispatchRecovery({ type: "heartbeat" });
}, 1_000);
document.addEventListener("visibilitychange", onVisibilityChange);
window.addEventListener("online", onOnline);
window.addEventListener("pageshow", onPageShow);
return () => {
window.clearInterval(heartbeat);
document.removeEventListener("visibilitychange", onVisibilityChange);
window.removeEventListener("online", onOnline);
window.removeEventListener("pageshow", onPageShow);
};
}, [delivery.id, recoveryAuthorityIdentity, uiBuildStale]);
return (
<div className="mse-fmp4-player" data-status={status}>
@@ -265,6 +575,7 @@ export function MseFmp4WebSocketPlayer({
type="button"
onClick={() => {
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setAttempt((value) => value + 1);
}}
>
@@ -14,6 +14,7 @@ import type {
RecordedAdmissionPhase,
RecordedCameraAdmissionState,
} from "../core/observation/recordedSessionAdmission";
import { liveCameraPlaybackAuthorityIdentity } from "../core/observation/liveCameraRecovery";
const sourceIcon: Record<ObservationSourceModality, IconName> = {
"point-cloud": "globe",
@@ -63,7 +64,13 @@ export function ObservationMedia({
source.delivery?.kind === "mse-fmp4-websocket" &&
source.modality === "video"
) {
return <MseFmp4WebSocketPlayer delivery={source.delivery} label={source.label} />;
return (
<MseFmp4WebSocketPlayer
delivery={source.delivery}
label={source.label}
recoveryAuthorityIdentity={liveCameraPlaybackAuthorityIdentity(source)}
/>
);
}
if (
@@ -2,14 +2,22 @@ import { useEffect, useRef, useState } from "react";
import type { SceneSettings } from "../sceneSettings";
import {
advanceLiveReceiverOpenWatchdog,
advanceLiveReceiverWatchdog,
initialLiveReceiverOpenWatchdogState,
initialLiveReceiverRecoveryState,
initialLiveReceiverWatchdogState,
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
LIVE_RECEIVER_OPEN_MAX_AGE_MS,
requestLiveReceiverRecovery,
} from "../core/observation/liveReceiverWatchdog";
import { postLiveViewerDiagnostic } from "../core/observation/liveViewerDiagnostics";
import type { LiveViewerFailureStage } from "../core/observation/liveViewerDiagnostics";
import {
createLiveViewerDiagnosticLifecycle,
createLiveViewerInstanceId,
createLiveViewerLineage,
subscribeToLiveViewerBuildFence,
type LiveViewerFailureStage,
} from "../core/observation/liveViewerDiagnostics";
import {
fetchPerceptionPreparationStatus,
perceptionPreparationMessage,
@@ -81,6 +89,7 @@ export interface RerunViewportProps {
followLive?: boolean;
liveActivitySequence?: number | null;
liveStreamId?: string | null;
liveRecoveryAuthorityIdentity?: string | null;
autoplayWhenReady?: boolean;
presentationGate?: RecordedAdmissionPhase;
expectedTimelineStartSeconds?: number;
@@ -197,6 +206,51 @@ export function createRecordedOpenWatchdog<T>({
};
}
export function createReentrantViewerDisposer(
cleanupOnce: () => void,
releaseNativeViewer: () => void,
): () => void {
let cleanupComplete = false;
return () => {
try {
if (!cleanupComplete) {
cleanupComplete = true;
cleanupOnce();
}
} finally {
// `viewer.start()` can resolve after an earlier pre-ready stop. Reapply
// native release on every disposal boundary so that a stale viewer can
// never reopen after React and diagnostics have already unmounted it.
releaseNativeViewer();
}
};
}
interface ActiveLiveViewerOwner {
release: () => void;
}
let activeLiveViewerOwner: ActiveLiveViewerOwner | null = null;
/**
* Own exactly one native live receiver per application document.
*
* React route/StrictMode transitions can overlap two mounted workspaces for a
* render turn. Rerun keeps each native gRPC receiver alive independently, so
* the overlap used to consume the bounded live replay slots and leave the
* operator's visible canvas black. Claiming the next owner synchronously
* retires the previous native receiver before the next one starts.
*/
export function claimExclusiveLiveViewer(release: () => void): () => void {
const owner = { release };
const previous = activeLiveViewerOwner;
activeLiveViewerOwner = owner;
previous?.release();
return () => {
if (activeLiveViewerOwner === owner) activeLiveViewerOwner = null;
};
}
export interface RecordedPlaybackBufferState {
bufferedEndNs: number | null;
expectedStartNs: number | null;
@@ -813,6 +867,7 @@ export function RerunViewport({
followLive = false,
liveActivitySequence = null,
liveStreamId = null,
liveRecoveryAuthorityIdentity = null,
autoplayWhenReady = false,
presentationGate = "ready",
expectedTimelineStartSeconds,
@@ -845,7 +900,14 @@ export function RerunViewport({
liveActivitySequenceRef.current = liveActivitySequence;
const liveStreamIdRef = useRef<string | null>(liveStreamId);
liveStreamIdRef.current = liveStreamId;
const liveRecoveryAuthorityRef = useRef<string | null>(liveRecoveryAuthorityIdentity);
liveRecoveryAuthorityRef.current = liveRecoveryAuthorityIdentity;
const liveRecoveryRef = useRef(initialLiveReceiverRecoveryState());
const liveViewerInstanceIdRef = useRef<string | null>(null);
liveViewerInstanceIdRef.current ??= createLiveViewerInstanceId();
const liveViewerLifecycleGenerationRef = useRef(0);
const activeViewerLifecycleRef = useRef<(() => void) | null>(null);
const uiBuildStaleRef = useRef(false);
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
@@ -871,14 +933,22 @@ export function RerunViewport({
recordedArtifact !== null,
);
useEffect(() => subscribeToLiveViewerBuildFence(() => {
uiBuildStaleRef.current = true;
// A stale document may only release its own browser transports. Device
// START/STOP remains owned by the acquisition authority while the fresh
// application document is loaded.
activeViewerLifecycleRef.current?.();
}), []);
useEffect(() => {
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
}, [followLive, liveStreamId, sourceUrl]);
}, [followLive, liveRecoveryAuthorityIdentity, liveStreamId, sourceUrl]);
useEffect(() => {
const normalizedSource = sourceUrl.trim();
const host = hostRef.current;
if (!normalizedSource || !host) {
if (!normalizedSource || !host || uiBuildStaleRef.current) {
setStatus("idle");
setRecordingBufferProgress(null);
onStatusChange?.("idle");
@@ -887,6 +957,7 @@ export function RerunViewport({
onPlaybackControllerChange?.(null);
return;
}
const isRecordedSource = RECORDED_RRD_PATH.test(normalizedSource);
let resolvedSource: string;
try {
@@ -912,18 +983,29 @@ export function RerunViewport({
return;
}
liveViewerLifecycleGenerationRef.current += 1;
const diagnosticLifecycle = createLiveViewerDiagnosticLifecycle({
lineage: createLiveViewerLineage(
liveViewerInstanceIdRef.current!,
liveViewerLifecycleGenerationRef.current,
),
});
diagnosticLifecycle.verifyBuild();
let disposed = false;
let disposeViewer: (() => void) | undefined;
let recordingOpenTimer: number | undefined;
let liveRecordingDiscoveryTimer: number | undefined;
let recordedOpenWatchdog: {
arm: () => void;
clear: () => void;
pending: () => boolean;
} | null = null;
let playbackRangeTimer: number | undefined;
let liveRecoveryRetryTimer: number | undefined;
let recordingOpened = false;
let recordingOpenTimedOut = false;
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
);
let viewerStartResolved = false;
let latestLiveRangeMaxNs: number | null = null;
let liveWatchdog = initialLiveReceiverWatchdogState();
@@ -948,22 +1030,68 @@ export function RerunViewport({
}
};
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
const clearLiveRecordingOpenTimer = () => {
if (recordingOpenTimer !== undefined) {
window.clearTimeout(recordingOpenTimer);
recordingOpenTimer = undefined;
}
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
function refreshOpeningLiveReceiver(openForMs: number) {
if (disposed || recordingOpened) return;
diagnosticLifecycle.post({
eventCode: "live_receiver_restart_requested",
failureStage: "recording-open-timeout",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
viewerRangeMaxNs: latestLiveRangeMaxNs,
stalledForMs: Math.round(openForMs),
recoveryAttempt: liveRecoveryRef.current.attempts || null,
});
setStatus("loading");
onStatusChange?.(
"loading",
"Живой визуализатор обновляет приёмник продолжающегося потока.",
);
disposeViewer?.();
setRetryNonce((nonce) => nonce + 1);
}
const armLiveRecordingOpenTimer = () => {
clearLiveRecordingOpenTimer();
diagnosticLifecycle.armAdmissionTimeout(() => {
if (disposed || recordingOpened) return;
const observed = advanceLiveReceiverOpenWatchdog(
liveOpenWatchdog,
liveRecoveryRef.current,
liveActivitySequenceRef.current,
Date.now(),
);
liveOpenWatchdog = observed.state;
liveRecoveryRef.current = observed.recoveryState;
if (observed.signal === "wait-for-store") {
// The backend is still publishing this exact acquisition. Preserve
// the receiver and its partially replayed store instead of throwing
// away startup work on every fixed timeout.
armLiveRecordingOpenTimer();
return;
}
if (observed.signal === "refresh-receiver") {
// Publication is healthy, so this is presentation-only maintenance:
// refresh the aged native receiver without spending (or clearing)
// recovery debt.
recordingOpenTimedOut = true;
refreshOpeningLiveReceiver(observed.openForMs);
return;
}
recordingOpenTimedOut = true;
requestLiveRecovery("recording-open-timeout");
}, LIVE_RECEIVER_OPEN_MAX_AGE_MS);
};
const clearLiveRecordingDiscoveryTimer = () => {
if (liveRecordingDiscoveryTimer !== undefined) {
window.clearInterval(liveRecordingDiscoveryTimer);
liveRecordingDiscoveryTimer = undefined;
}
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
const clearLiveRecoveryRetryTimer = () => {
if (liveRecoveryRetryTimer === undefined) return;
window.clearTimeout(liveRecoveryRetryTimer);
liveRecoveryRetryTimer = undefined;
};
const clearRecordingTimers = () => {
clearRecordedAdmissionWatchdog();
clearLiveRecordingOpenTimer();
clearLiveRecordingDiscoveryTimer();
clearLiveRecoveryRetryTimer();
};
const clearPlaybackRangeTimer = () => {
if (playbackRangeTimer === undefined) return;
@@ -989,7 +1117,7 @@ export function RerunViewport({
const reportError = (message: string, failureStage?: LiveViewerFailureStage) => {
if (disposed) return;
if (followLive) {
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_error",
failureStage,
streamId: liveStreamIdRef.current,
@@ -1008,7 +1136,7 @@ export function RerunViewport({
) => {
if (!followLive || disposed) return false;
if (emitErrorEvent) {
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_error",
failureStage,
streamId: liveStreamIdRef.current,
@@ -1018,10 +1146,20 @@ export function RerunViewport({
recoveryAttempt: liveRecoveryRef.current.attempts || null,
});
}
const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current);
const recovery = requestLiveReceiverRecovery(liveRecoveryRef.current, {
activeAuthorityIdentity: liveRecoveryAuthorityRef.current,
expectedAuthorityIdentity: liveRecoveryAuthorityIdentity,
disposed,
});
liveRecoveryRef.current = recovery.state;
if (recovery.signal === "stale") {
// A newer runtime snapshot owns the next effect. This retired viewer
// may release itself but cannot schedule a receiver for that lineage.
disposeViewer?.();
return true;
}
if (recovery.signal === "exhausted") {
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_recovery_exhausted",
failureStage,
streamId: liveStreamIdRef.current,
@@ -1037,7 +1175,7 @@ export function RerunViewport({
);
return true;
}
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_restart_requested",
failureStage,
streamId: liveStreamIdRef.current,
@@ -1052,7 +1190,15 @@ export function RerunViewport({
"Живой визуализатор переподключается к продолжающемуся потоку.",
);
disposeViewer?.();
setRetryNonce((nonce) => nonce + 1);
clearLiveRecoveryRetryTimer();
liveRecoveryRetryTimer = window.setTimeout(() => {
liveRecoveryRetryTimer = undefined;
if (
disposed
|| liveRecoveryAuthorityRef.current !== liveRecoveryAuthorityIdentity
) return;
setRetryNonce((nonce) => nonce + 1);
}, recovery.delayMs ?? 0);
return true;
};
const observeLiveReceiver = (viewerRangeMaxNs: number | null) => {
@@ -1072,7 +1218,7 @@ export function RerunViewport({
(receiverOpenedAfterRecovery || observed.signal === "receiver-advanced") &&
liveRecoveryRef.current.awaitingRecovery
) {
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_recovered",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
@@ -1084,7 +1230,7 @@ export function RerunViewport({
}
if (observed.signal !== "stalled") return;
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_stalled",
failureStage: "receiver-stalled",
streamId: liveStreamIdRef.current,
@@ -1093,12 +1239,30 @@ export function RerunViewport({
stalledForMs: Math.round(observed.stalledForMs),
recoveryAttempt: Math.min(
liveRecoveryRef.current.attempts + 1,
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
liveRecoveryAuthorityIdentity
? Number.MAX_SAFE_INTEGER
: LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
),
});
requestLiveRecovery("receiver-stalled", observed.stalledForMs, false);
};
let relinquishLiveViewerOwnership: () => void = () => {};
const disposeActiveViewerLifecycle = () => {
if (disposed) return;
disposed = true;
relinquishLiveViewerOwnership();
diagnosticLifecycle.dispose();
disposeViewer?.();
host.replaceChildren();
};
if (followLive) {
relinquishLiveViewerOwnership = claimExclusiveLiveViewer(
disposeActiveViewerLifecycle,
);
}
activeViewerLifecycleRef.current = disposeActiveViewerLifecycle;
host.replaceChildren();
appliedPointColorKeyRef.current = null;
setStatus("loading");
@@ -1113,10 +1277,7 @@ export function RerunViewport({
if (disposed) return;
const viewer = new WebViewer();
let viewerDisposed = false;
disposeViewer = () => {
if (viewerDisposed) return;
viewerDisposed = true;
disposeViewer = createReentrantViewerDisposer(() => {
clearRecordingTimers();
clearPlaybackRangeTimer();
playbackTimeUpdates.cancel();
@@ -1150,6 +1311,7 @@ export function RerunViewport({
} catch {
// The viewer may already have closed all auxiliary channels.
}
}, () => {
try {
if (viewer.ready) viewer.close(resolvedSource);
} catch {
@@ -1162,7 +1324,7 @@ export function RerunViewport({
// startup failure.
}
host.replaceChildren();
};
});
if (isRecordedSource && recordedArtifact) {
recordedOpenWatchdog = createRecordedOpenWatchdog({
byteLength: recordedArtifact.byteLength,
@@ -1188,8 +1350,17 @@ export function RerunViewport({
) return;
recordingOpened = true;
if (!isRecordedSource) {
clearLiveRecordingOpenTimer();
clearLiveRecordingDiscoveryTimer();
diagnosticLifecycle.markAdmitted();
if (liveRecoveryRef.current.awaitingRecovery) {
diagnosticLifecycle.post({
eventCode: "live_receiver_recovered",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
viewerRangeMaxNs: latestLiveRangeMaxNs,
recoveryAttempt: liveRecoveryRef.current.attempts,
});
}
liveRecoveryRef.current = initialLiveReceiverRecoveryState();
}
if (
recordedBlueprintUrl &&
@@ -1386,7 +1557,7 @@ export function RerunViewport({
// Rerun 0.34.1 may ingest an SDK gRPC store without forwarding its
// recording_open event to the JavaScript wrapper. The active store
// is the authoritative fallback and avoids hiding a ready canvas.
postLiveViewerDiagnostic({
diagnosticLifecycle.post({
eventCode: "live_receiver_active_store_admitted",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
@@ -1489,16 +1660,19 @@ export function RerunViewport({
}
if (!isRecordedSource) {
// Measure native store admission from the resolved viewer start,
// not from dynamic module import or React effect setup time.
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
Date.now(),
);
discoverActiveLiveRecording();
if (!recordingOpened) {
liveRecordingDiscoveryTimer = window.setInterval(
diagnosticLifecycle.armAdmissionInterval(
discoverActiveLiveRecording,
100,
);
recordingOpenTimer = window.setTimeout(() => {
recordingOpenTimedOut = true;
requestLiveRecovery("recording-open-timeout");
}, 12_000);
armLiveRecordingOpenTimer();
}
}
} catch {
@@ -1520,12 +1694,14 @@ export function RerunViewport({
});
return () => {
disposed = true;
if (activeViewerLifecycleRef.current === disposeActiveViewerLifecycle) {
activeViewerLifecycleRef.current = null;
}
disposeActiveViewerLifecycle();
clearRecordingTimers();
clearPlaybackRangeTimer();
playbackTimeUpdates.cancel();
unsubscribeAll();
disposeViewer?.();
onSelectionChange?.(null);
onPlaybackControllerChange?.(null);
onPlaybackChange?.(null);
@@ -1537,6 +1713,7 @@ export function RerunViewport({
followLive,
initialPlaybackStartSeconds,
liveStreamId,
liveRecoveryAuthorityIdentity,
onPlaybackChange,
onPlaybackControllerChange,
onSelectionChange,
@@ -6,14 +6,46 @@ import type { WorkspaceDefinition } from "../productModel";
interface ApplicationPanelActionsOptions {
definition: WorkspaceDefinition | null;
refreshRuntime: () => void;
resetConnectionScenario?: () => Promise<boolean>;
connectionScenarioResetting: boolean;
saveWorkspaceLayout: () => Promise<void>;
workspaceLayoutSaving: boolean;
systemUtilityActions: readonly ApplicationPanelUtilityAction[];
}
export function deviceRuntimeUtilityAction({
refreshRuntime,
resetConnectionScenario,
connectionScenarioResetting,
}: Pick<
ApplicationPanelActionsOptions,
"refreshRuntime" | "resetConnectionScenario" | "connectionScenarioResetting"
>): ApplicationPanelUtilityAction {
return {
label: resetConnectionScenario
? connectionScenarioResetting
? "Сбрасываем подключение"
: "Сбросить подключение"
: "Обновить состояние локального контура",
icon: resetConnectionScenario && connectionScenarioResetting
? "activity"
: "refresh",
disabled: resetConnectionScenario && connectionScenarioResetting
? true
: undefined,
onClick: resetConnectionScenario
? connectionScenarioResetting
? () => undefined
: () => void resetConnectionScenario()
: refreshRuntime,
};
}
export function useApplicationPanelActions({
definition,
refreshRuntime,
resetConnectionScenario,
connectionScenarioResetting,
saveWorkspaceLayout,
workspaceLayoutSaving,
systemUtilityActions,
@@ -21,11 +53,11 @@ export function useApplicationPanelActions({
return useMemo(() => {
const actions: ApplicationPanelUtilityAction[] = [];
if (definition?.kind === "device") {
actions.push({
label: "Обновить состояние локального контура",
icon: "refresh",
onClick: refreshRuntime,
});
actions.push(deviceRuntimeUtilityAction({
refreshRuntime,
resetConnectionScenario,
connectionScenarioResetting,
}));
}
if (definition && ["spatial", "recordings"].includes(definition.kind)) {
actions.push({
@@ -40,6 +72,8 @@ export function useApplicationPanelActions({
}, [
definition,
refreshRuntime,
resetConnectionScenario,
connectionScenarioResetting,
saveWorkspaceLayout,
systemUtilityActions,
workspaceLayoutSaving,