diff --git a/.gitignore b/.gitignore index f8e0715..2261aa9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ build/ !.env.example config/local.toml private/ +.runtime/ # Real laboratory captures and decoded artifacts are sensitive and large. captures/ diff --git a/apps/control-station/src/App.tsx b/apps/control-station/src/App.tsx index 5d75cd7..98f49a9 100644 --- a/apps/control-station/src/App.tsx +++ b/apps/control-station/src/App.tsx @@ -133,7 +133,10 @@ export default function App() { const activeSceneWindow = sceneWindowOrder[sceneWindowOrder.length - 1] ?? null; const automaticSourceUrl = runtime.state?.spatialSource?.url.trim() ?? ""; const effectiveSourceUrl = sourceUrl || automaticSourceUrl; - const observationLayout = useObservationLayout(runtime.state?.observationSources ?? []); + const observationLayout = useObservationLayout( + runtime.state?.observationSources ?? [], + runtime.setObservationSourceActive, + ); useEffect(() => { const remote = runtime.state?.viewerSettings; diff --git a/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx b/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx new file mode 100644 index 0000000..3588e62 --- /dev/null +++ b/apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx @@ -0,0 +1,279 @@ +import { useEffect, useRef, useState } from "react"; +import { Icon } from "@nodedc/ui-react"; + +import type { ObservationSourceDelivery } from "../core/runtime/contracts"; + +type PlayerStatus = "connecting" | "buffering" | "playing" | "error"; + +export interface CameraLeaseRetryBudget { + deliveryId: string; + count: number; +} + +const CAMERA_LEASE_RETRY_DELAYS = [400, 1_000, 2_000] as const; + +export function resetCameraLeaseRetryBudget(deliveryId: string): CameraLeaseRetryBudget { + return { deliveryId, count: 0 }; +} + +export function consumeCameraLeaseRetry( + current: CameraLeaseRetryBudget, + deliveryId: string, +): { budget: CameraLeaseRetryBudget; delay: number | null } { + const count = current.deliveryId === deliveryId ? current.count : 0; + const delay = CAMERA_LEASE_RETRY_DELAYS[count] ?? null; + return { + budget: { deliveryId, count: delay === null ? count : count + 1 }, + delay, + }; +} + +function websocketUrl(path: string): string { + const url = new URL(path, window.location.href); + if (url.protocol === "http:") url.protocol = "ws:"; + if (url.protocol === "https:") url.protocol = "wss:"; + if (url.protocol !== "ws:" && url.protocol !== "wss:") { + throw new Error("Адаптер вернул неподдерживаемый адрес видеопотока."); + } + return url.toString(); +} + +export function MseFmp4WebSocketPlayer({ + delivery, + label, +}: { + delivery: ObservationSourceDelivery & { kind: "mse-fmp4-websocket" }; + label: string; +}) { + const videoRef = useRef(null); + const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id)); + const [attempt, setAttempt] = useState(0); + const [status, setStatus] = useState("connecting"); + const [message, setMessage] = useState("Подключение к локальному видеопотоку"); + + useEffect(() => { + const video = videoRef.current; + if (!video) return; + + let disposed = false; + let socket: WebSocket | null = null; + let sourceBuffer: SourceBuffer | null = null; + let objectUrl = ""; + let retryTimer: number | undefined; + let receivedMedia = false; + let failed = false; + const queue: ArrayBuffer[] = []; + let queuedBytes = 0; + + const fail = (copy: string) => { + if (disposed || failed) return; + failed = true; + setStatus("error"); + setMessage(copy); + try { + if (socket && socket.readyState < WebSocket.CLOSING) { + socket.close(1011, "Live video buffer reset"); + } + } catch { + // The manual reconnect button will create a fresh transport and MSE buffer. + } + }; + + const retryLease = () => { + const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id); + leaseRetryRef.current = retry.budget; + if (retry.delay === null) { + fail("Camera adapter ещё занят предыдущим окном. Подключитесь повторно."); + return; + } + setStatus("connecting"); + setMessage("Освобождение предыдущего окна камеры"); + retryTimer = window.setTimeout(() => { + if (!disposed) setAttempt((value) => value + 1); + }, retry.delay); + }; + + const onPlaying = () => { + if (disposed) return; + leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id); + setStatus("playing"); + setMessage(""); + }; + + video.addEventListener("playing", onPlaying); + + const appendNext = () => { + if (disposed || !sourceBuffer || sourceBuffer.updating || queue.length === 0) return; + const chunk = queue.shift(); + if (!chunk) return; + queuedBytes -= chunk.byteLength; + try { + sourceBuffer.appendBuffer(chunk); + } catch (error) { + fail(error instanceof DOMException && error.name === "QuotaExceededError" + ? "Live-буфер переполнен и сброшен, чтобы не накапливать задержку." + : "Не удалось добавить видеосегмент. Повторите подключение."); + } + }; + + const enqueue = (chunk: ArrayBuffer) => { + if (disposed || chunk.byteLength === 0) return; + // Never drop arbitrary fMP4 fragments: the following samples may depend + // on them. A bounded reset is safer and keeps live latency deterministic. + if (queuedBytes + chunk.byteLength > 2 * 1024 * 1024) { + fail("Видеодекодер не успевает за эфиром. Live-буфер сброшен."); + return; + } + queue.push(chunk); + queuedBytes += chunk.byteLength; + appendNext(); + }; + + const mediaType = delivery.mediaType?.trim(); + if (typeof MediaSource === "undefined") { + fail("Этот браузер не поддерживает Media Source Extensions."); + return; + } + if (!mediaType || !MediaSource.isTypeSupported(mediaType)) { + fail(mediaType + ? `Браузер не поддерживает ${mediaType}` + : "Адаптер не сообщил MIME/codec fMP4-потока."); + return; + } + + setStatus("connecting"); + setMessage("Подключение к локальному видеопотоку"); + const mediaSource = new MediaSource(); + objectUrl = URL.createObjectURL(mediaSource); + video.src = objectUrl; + + const onSourceOpen = () => { + if (disposed) return; + try { + sourceBuffer = mediaSource.addSourceBuffer(mediaType); + } catch { + fail("Не удалось создать MSE-буфер для указанного кодека."); + return; + } + + sourceBuffer.addEventListener("updateend", () => { + if (disposed || !sourceBuffer) return; + const buffered = sourceBuffer.buffered; + if (buffered.length > 0) { + const end = buffered.end(buffered.length - 1); + const start = buffered.start(0); + if (end - video.currentTime > 1) video.currentTime = Math.max(0, end - 0.1); + if (!receivedMedia) { + receivedMedia = true; + setStatus("buffering"); + setMessage("Запуск первого декодированного кадра"); + void video.play().catch(() => undefined); + } + const removeBefore = end - 3; + if (removeBefore > start && !sourceBuffer.updating) { + try { + sourceBuffer.remove(0, removeBefore); + return; + } catch { + // Continue appending; quota handling performs a clean reset. + } + } + } + appendNext(); + }); + sourceBuffer.addEventListener("error", () => { + fail("MSE сообщил об ошибке декодирования видеосегмента."); + }); + + try { + socket = new WebSocket(websocketUrl(delivery.url)); + } catch (error) { + fail(error instanceof Error ? error.message : "Некорректный адрес видеопотока."); + return; + } + socket.binaryType = "arraybuffer"; + socket.addEventListener("open", () => { + if (disposed) return; + setStatus("buffering"); + setMessage("Ожидание первого видеокадра"); + }); + socket.addEventListener("message", (event) => { + if (event.data instanceof ArrayBuffer) { + enqueue(event.data); + } else if (event.data instanceof Blob) { + void event.data.arrayBuffer().then(enqueue).catch(() => { + fail("Получен повреждённый видеосегмент."); + }); + } + }); + socket.addEventListener("error", () => { + fail("Соединение с локальным video adapter потеряно."); + }); + socket.addEventListener("close", (event) => { + if (!disposed && !failed && event.code === 1008) { + retryLease(); + } else if (!disposed) { + fail(event.code === 1000 + ? "Видеопоток завершён. Можно подключиться повторно." + : "Видеопоток прерван. Проверьте устройство и повторите подключение."); + } + }); + }; + + mediaSource.addEventListener("sourceopen", onSourceOpen, { once: true }); + + return () => { + disposed = true; + if (retryTimer !== undefined) window.clearTimeout(retryTimer); + queue.length = 0; + socket?.close(1000, "Источник скрыт оператором"); + video.removeEventListener("playing", onPlaying); + try { + if (sourceBuffer?.updating) sourceBuffer.abort(); + } catch { + // The MediaSource can already be closing. + } + try { + if (mediaSource.readyState === "open") mediaSource.endOfStream(); + } catch { + // Cleanup must continue even if the browser has already detached MSE. + } + video.pause(); + video.removeAttribute("src"); + video.load(); + if (objectUrl) URL.revokeObjectURL(objectUrl); + }; + }, [attempt, delivery.id, delivery.mediaType, delivery.url]); + + return ( +
+
+ ); +} diff --git a/apps/control-station/src/components/ObservationSources.tsx b/apps/control-station/src/components/ObservationSources.tsx index 659b243..8ff5b9b 100644 --- a/apps/control-station/src/components/ObservationSources.tsx +++ b/apps/control-station/src/components/ObservationSources.tsx @@ -5,6 +5,7 @@ import type { ObservationSourceDescriptor, ObservationSourceModality, } from "../core/runtime/contracts"; +import { MseFmp4WebSocketPlayer } from "./MseFmp4WebSocketPlayer"; const sourceIcon: Record = { "point-cloud": "globe", @@ -29,7 +30,38 @@ export function observationSourceStatusLabel(source: ObservationSourceDescriptor } export function ObservationMedia({ source }: { source: ObservationSourceDescriptor }) { - if (source.previewUrl && source.modality === "video") { + const sourceSelected = source.activation ? source.activation.selected : true; + const deliveryActive = Boolean(source.delivery && sourceSelected); + + if ( + deliveryActive && + source.delivery?.kind === "mse-fmp4-websocket" && + source.modality === "video" + ) { + return ; + } + + if (deliveryActive && source.delivery?.kind === "video-url" && source.modality === "video") { + return ( +