SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

wip(k1): checkpoint connection recovery rewrite

Browse Source 
Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
This commit is contained in:
DCCONSTRUCTIONS
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx
+355 -44
View File
@@ -2,6 +2,14 @@ import { useEffect, useRef, useState } from "react";
import { Icon } from "@nodedc/ui-react";
import type { ObservationSourceDelivery } from "../core/runtime/contracts";
import {
cameraBrowserTransportIdentity,
cameraTransportCallbackIsCurrent,
initialCameraPlaybackRecoveryState,
reduceCameraPlaybackRecovery,
type CameraPlaybackRecoveryEvent,
} from "../core/observation/liveCameraRecovery";
import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics";
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
@@ -10,7 +18,92 @@ export interface CameraLeaseRetryBudget {
count: number;
}
const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000] as const;
const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000, 5_000] as const;
export const CAMERA_FIRST_MEDIA_TIMEOUT_MS = 8_000;
export const CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS = 8_000;
// The gateway may release an 8 MiB / 64-fragment slow-reader backlog after a
// main-thread stall. Keep one bounded append margin above that complete batch;
// crossing either limit replaces the MSE epoch instead of dropping fragments.
export const CAMERA_PENDING_QUEUE_MAX_BYTES = 12 * 1024 * 1024;
export const CAMERA_PENDING_QUEUE_MAX_SEGMENTS = 96;
export type CameraStartupWatchdogStage = "first-media" | "first-playable-frame";
export interface CameraStartupWatchdog {
armFirstMedia: () => void;
markMediaReceived: () => void;
markPlaying: () => void;
clear: () => void;
pendingStage: () => CameraStartupWatchdogStage | null;
}
/**
* Supervise only the disposable browser transport. One fixed first-media
* deadline covers both an MSE that never opens and an open-but-silent socket.
* The first non-empty fragment then starts a separate first-playable-frame
* deadline; later fragments deliberately do not extend it.
*/
export function createCameraStartupWatchdog({
schedule,
cancel,
onTimeout,
firstMediaTimeoutMs = CAMERA_FIRST_MEDIA_TIMEOUT_MS,
firstPlayableFrameTimeoutMs = CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS,
}: {
schedule: (callback: () => void, timeoutMs: number) => number;
cancel: (handle: number) => void;
onTimeout: (stage: CameraStartupWatchdogStage) => void;
firstMediaTimeoutMs?: number;
firstPlayableFrameTimeoutMs?: number;
}): CameraStartupWatchdog {
let timer: number | undefined;
let stage: CameraStartupWatchdogStage | null = null;
let mediaReceived = false;
let playing = false;
const clearTimer = () => {
if (timer !== undefined) cancel(timer);
timer = undefined;
stage = null;
};
const arm = (nextStage: CameraStartupWatchdogStage, timeoutMs: number) => {
clearTimer();
stage = nextStage;
timer = schedule(() => {
if (stage !== nextStage || playing) return;
timer = undefined;
stage = null;
onTimeout(nextStage);
}, timeoutMs);
};
return {
armFirstMedia() {
if (mediaReceived || playing || stage !== null) return;
arm("first-media", firstMediaTimeoutMs);
},
markMediaReceived() {
if (mediaReceived || playing) return;
mediaReceived = true;
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
},
markPlaying() {
if (playing) return;
playing = true;
clearTimer();
},
clear: clearTimer,
pendingStage: () => stage,
};
}
export function cameraStartupWatchdogRecoveryMessage(
stage: CameraStartupWatchdogStage,
): string {
return stage === "first-media"
? "Камера подключена, но не передаёт медиаданные; восстанавливаем browser-preview."
: "Медиаданные поступают, но первый кадр не воспроизводится; пересоздаём decoder.";
}
export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetryBudget {
return { deliveryId, count: 0 };
@@ -19,15 +112,60 @@ export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetr
export function consumeCameraLeaseRetry(
current: CameraLeaseRetryBudget,
deliveryId: string,
): { budget: CameraLeaseRetryBudget; delay: number | null } {
): { budget: CameraLeaseRetryBudget; delay: number } {
const count = current.deliveryId === deliveryId ? current.count : 0;
const delay = CAMERA_LEASE_RETRY_DELAYS[count] ?? null;
const delayIndex = Math.min(count, CAMERA_LEASE_RETRY_DELAYS.length - 1);
const delay = CAMERA_LEASE_RETRY_DELAYS[delayIndex];
return {
budget: { deliveryId, count: delay === null ? count : count + 1 },
budget: {
deliveryId,
count: Math.min(count + 1, CAMERA_LEASE_RETRY_DELAYS.length),
},
delay,
};
}
export function cameraTransportRecoveryIsCurrent(
activeAuthorityIdentity: string | null,
expectedAuthorityIdentity: string | null,
activeEpoch: number,
expectedEpoch: number,
disposed: boolean,
): boolean {
return Boolean(
expectedAuthorityIdentity
&& activeAuthorityIdentity === expectedAuthorityIdentity
&& cameraTransportCallbackIsCurrent(activeEpoch, expectedEpoch, disposed),
);
}
export function cameraTransportCanOpen(uiBuildStale: boolean): boolean {
return !uiBuildStale;
}
export function cameraPendingQueueCanAccept(
queuedBytes: number,
queuedSegments: number,
incomingBytes: number,
): boolean {
return incomingBytes > 0
&& queuedBytes + incomingBytes <= CAMERA_PENDING_QUEUE_MAX_BYTES
&& queuedSegments + 1 <= CAMERA_PENDING_QUEUE_MAX_SEGMENTS;
}
export function cameraTransportCloseRecoveryMessage(code: number): string {
if (code === 4_008) {
return "Browser-reader отстал от эфира; восстанавливаем текущую камеру.";
}
if (code === 1_008) {
return "Camera adapter освобождает прежний browser-reader; переподключаемся.";
}
if (code === 1_000) {
return "Browser-preview завершился; восстанавливаем текущую камеру.";
}
return "Browser-preview прерван; восстанавливаем текущую камеру.";
}
function websocketUrl(path: string): string {
const url = new URL(path, window.location.href);
if (url.protocol === "http:") url.protocol = "ws:";
@@ -41,61 +179,149 @@ function websocketUrl(path: string): string {
export function MseFmp4WebSocketPlayer({
delivery,
label,
recoveryAuthorityIdentity,
}: {
delivery: ObservationSourceDelivery & { kind: "mse-fmp4-websocket" };
label: string;
recoveryAuthorityIdentity: string | null;
}) {
const videoRef = useRef<HTMLVideoElement>(null);
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
const recoveryPendingAuthorityRef = useRef<string | null>(null);
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
const uiBuildStaleRef = useRef(false);
const [attempt, setAttempt] = useState(0);
const [uiBuildStale, setUiBuildStale] = useState(false);
const [status, setStatus] = useState<PlayerStatus>("connecting");
const [message, setMessage] = useState("Подключение к локальному видеопотоку");
activeAuthorityRef.current = recoveryAuthorityIdentity;
const transportIdentity = cameraBrowserTransportIdentity(
delivery,
recoveryAuthorityIdentity,
);
useEffect(() => subscribeToLiveViewerBuildFence(() => {
// Setting this local fence tears down the effect-owned WebSocket and MSE
// buffer. It intentionally never invokes a plugin or device command.
uiBuildStaleRef.current = true;
activeTransportDisposeRef.current?.();
setUiBuildStale(true);
}), []);
useEffect(() => {
const video = videoRef.current;
if (!video) return;
if (uiBuildStale || !cameraTransportCanOpen(uiBuildStaleRef.current)) {
video.pause();
video.removeAttribute("src");
video.load();
return;
}
if (transportAuthorityRef.current !== recoveryAuthorityIdentity) {
transportAuthorityRef.current = recoveryAuthorityIdentity;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
}
let disposed = false;
const transportEpoch = transportEpochRef.current + 1;
transportEpochRef.current = transportEpoch;
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
);
let socket: WebSocket | null = null;
let sourceBuffer: SourceBuffer | null = null;
let onBufferUpdateEnd: (() => void) | null = null;
let onBufferError: (() => void) | null = null;
let objectUrl = "";
let retryTimer: number | undefined;
let receivedMedia = false;
let failed = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
const recovering = Boolean(
activeAuthorityRef.current
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
);
const queue: ArrayBuffer[] = [];
let queuedBytes = 0;
const fail = (copy: string) => {
if (disposed || failed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.clear();
failed = true;
recoveryPendingAuthorityRef.current = null;
setStatus("error");
setMessage(copy);
try {
if (socket && socket.readyState < WebSocket.CLOSING) {
socket.close(1011, "Live video buffer reset");
socket.close(4_000, "Live video buffer reset");
}
} catch {
// The manual reconnect button will create a fresh transport and MSE buffer.
}
};
const retryLease = () => {
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
if (retry.delay === null) {
fail("Camera adapter ещё занят предыдущим окном. Подключитесь повторно.");
const retryTransport = (copy: string) => {
if (!transportIsCurrent() || failed) return;
if (!cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
recoveryAuthorityIdentity,
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
)) {
fail(copy);
return;
}
startupWatchdog?.clear();
failed = true;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage("Освобождение предыдущего окна камеры");
setMessage(copy);
try {
if (socket && socket.readyState < WebSocket.CLOSING) {
socket.close(4_001, "Live video transport recovery");
}
} catch {
// The replacement effect still fences and releases this transport.
}
retryTimer = window.setTimeout(() => {
if (!disposed) setAttempt((value) => value + 1);
if (cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
recoveryAuthorityIdentity,
transportEpochRef.current,
transportEpoch,
disposed || uiBuildStaleRef.current,
)) {
setAttempt((value) => value + 1);
}
}, retry.delay);
};
if (recoveryAuthorityIdentity) {
startupWatchdog = createCameraStartupWatchdog({
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
cancel: (handle) => window.clearTimeout(handle),
onTimeout: (stage) => {
retryTransport(cameraStartupWatchdogRecoveryMessage(stage));
},
});
}
const onPlaying = () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.markPlaying();
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
setMessage("");
};
@@ -103,25 +329,32 @@ export function MseFmp4WebSocketPlayer({
video.addEventListener("playing", onPlaying);
const appendNext = () => {
if (disposed || !sourceBuffer || sourceBuffer.updating || queue.length === 0) return;
if (
!transportIsCurrent()
|| failed
|| !sourceBuffer
|| sourceBuffer.updating
|| queue.length === 0
) return;
const chunk = queue.shift();
if (!chunk) return;
queuedBytes -= chunk.byteLength;
try {
sourceBuffer.appendBuffer(chunk);
} catch (error) {
fail(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен и сброшен, чтобы не накапливать задержку."
: "Не удалось добавить видеосегмент. Повторите подключение.");
retryTransport(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.");
}
};
const enqueue = (chunk: ArrayBuffer) => {
if (disposed || chunk.byteLength === 0) return;
if (!transportIsCurrent() || failed || chunk.byteLength === 0) return;
startupWatchdog?.markMediaReceived();
// Never drop arbitrary fMP4 fragments: the following samples may depend
// on them. A bounded reset is safer and keeps live latency deterministic.
if (queuedBytes + chunk.byteLength > 2 * 1024 * 1024) {
fail("Видеодекодер не успевает за эфиром. Live-буфер сброшен.");
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер.");
return;
}
queue.push(chunk);
@@ -142,13 +375,16 @@ export function MseFmp4WebSocketPlayer({
}
setStatus("connecting");
setMessage("Подключение к локальному видеопотоку");
setMessage(recovering
? "Восстановление камеры после разрыва браузерного канала"
: "Подключение к локальному видеопотоку");
const mediaSource = new MediaSource();
objectUrl = URL.createObjectURL(mediaSource);
video.src = objectUrl;
startupWatchdog?.armFirstMedia();
const onSourceOpen = () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
try {
sourceBuffer = mediaSource.addSourceBuffer(mediaType);
} catch {
@@ -156,8 +392,8 @@ export function MseFmp4WebSocketPlayer({
return;
}
sourceBuffer.addEventListener("updateend", () => {
if (disposed || !sourceBuffer) return;
onBufferUpdateEnd = () => {
if (!transportIsCurrent() || failed || !sourceBuffer) return;
const buffered = sourceBuffer.buffered;
if (buffered.length > 0) {
const end = buffered.end(buffered.length - 1);
@@ -166,7 +402,9 @@ export function MseFmp4WebSocketPlayer({
if (!receivedMedia) {
receivedMedia = true;
setStatus("buffering");
setMessage("Запуск первого декодированного кадра");
setMessage(recovering
? "Запуск первого кадра восстановленной камеры"
: "Запуск первого декодированного кадра");
void video.play().catch(() => undefined);
}
const removeBefore = end - 3;
@@ -180,10 +418,12 @@ export function MseFmp4WebSocketPlayer({
}
}
appendNext();
});
sourceBuffer.addEventListener("error", () => {
fail("MSE сообщил об ошибке декодирования видеосегмента.");
});
};
onBufferError = () => {
retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder.");
};
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
sourceBuffer.addEventListener("error", onBufferError);
try {
socket = new WebSocket(websocketUrl(delivery.url));
@@ -193,41 +433,49 @@ export function MseFmp4WebSocketPlayer({
}
socket.binaryType = "arraybuffer";
socket.addEventListener("open", () => {
if (disposed) return;
if (!transportIsCurrent() || failed) return;
startupWatchdog?.armFirstMedia();
setStatus("buffering");
setMessage("Ожидание первого видеокадра");
setMessage(recovering
? "Ожидание первого кадра после восстановления"
: "Ожидание первого видеокадра");
});
socket.addEventListener("message", (event) => {
if (!transportIsCurrent() || failed) return;
if (event.data instanceof ArrayBuffer) {
enqueue(event.data);
} else if (event.data instanceof Blob) {
void event.data.arrayBuffer().then(enqueue).catch(() => {
fail("Получен повреждённый видеосегмент.");
retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал.");
});
}
});
socket.addEventListener("error", () => {
fail("Соединение с локальным video adapter потеряно.");
retryTransport("Связь с локальным video adapter потеряна; переподключаемся.");
});
socket.addEventListener("close", (event) => {
if (!disposed && !failed && event.code === 1008) {
retryLease();
} else if (!disposed) {
fail(event.code === 1000
? "Видеопоток завершён. Можно подключиться повторно."
: "Видеопоток прерван. Проверьте устройство и повторите подключение.");
}
if (!transportIsCurrent() || failed) return;
retryTransport(cameraTransportCloseRecoveryMessage(event.code));
});
};
mediaSource.addEventListener("sourceopen", onSourceOpen, { once: true });
return () => {
const disposeTransport = () => {
if (disposed) return;
disposed = true;
startupWatchdog?.clear();
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
queue.length = 0;
socket?.close(1000, "Источник скрыт оператором");
socket?.close(1000, "Browser preview transport replaced or hidden");
video.removeEventListener("playing", onPlaying);
mediaSource.removeEventListener("sourceopen", onSourceOpen);
if (sourceBuffer && onBufferUpdateEnd) {
sourceBuffer.removeEventListener("updateend", onBufferUpdateEnd);
}
if (sourceBuffer && onBufferError) {
sourceBuffer.removeEventListener("error", onBufferError);
}
try {
if (sourceBuffer?.updating) sourceBuffer.abort();
} catch {
@@ -243,7 +491,69 @@ export function MseFmp4WebSocketPlayer({
video.load();
if (objectUrl) URL.revokeObjectURL(objectUrl);
};
}, [attempt, delivery.id, delivery.mediaType, delivery.url]);
activeTransportDisposeRef.current = disposeTransport;
return () => {
if (activeTransportDisposeRef.current === disposeTransport) {
activeTransportDisposeRef.current = null;
}
disposeTransport();
};
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
useEffect(() => {
if (
!recoveryAuthorityIdentity
|| uiBuildStale
|| !cameraTransportCanOpen(uiBuildStaleRef.current)
) return;
let recovery = initialCameraPlaybackRecoveryState(
recoveryAuthorityIdentity,
Date.now(),
);
const dispatchRecovery = (event: CameraPlaybackRecoveryEvent) => {
if (uiBuildStaleRef.current) return;
const decision = reduceCameraPlaybackRecovery(recovery, event, {
activeAuthorityIdentity: activeAuthorityRef.current,
now: Date.now(),
documentVisible: document.visibilityState === "visible",
networkOnline: navigator.onLine !== false,
});
recovery = decision.state;
if (!decision.reopen) return;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage("Восстановление камеры после разрыва браузерного канала");
// The transport effect cleans up the old WebSocket and MSE object before
// opening their replacement. No plugin action or device command occurs.
setAttempt((value) => value + 1);
};
const onVisibilityChange = () => {
dispatchRecovery({
type: document.visibilityState === "hidden"
? "document-hidden"
: "document-visible",
});
};
const onOnline = () => dispatchRecovery({ type: "network-online" });
const onPageShow = (event: PageTransitionEvent) => {
dispatchRecovery({ type: "page-restore", persisted: event.persisted });
};
const heartbeat = window.setInterval(() => {
dispatchRecovery({ type: "heartbeat" });
}, 1_000);
document.addEventListener("visibilitychange", onVisibilityChange);
window.addEventListener("online", onOnline);
window.addEventListener("pageshow", onPageShow);
return () => {
window.clearInterval(heartbeat);
document.removeEventListener("visibilitychange", onVisibilityChange);
window.removeEventListener("online", onOnline);
window.removeEventListener("pageshow", onPageShow);
};
}, [delivery.id, recoveryAuthorityIdentity, uiBuildStale]);
return (
<div className="mse-fmp4-player" data-status={status}>
@@ -265,6 +575,7 @@ export function MseFmp4WebSocketPlayer({
type="button"
onClick={() => {
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setAttempt((value) => value + 1);
}}
>