SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): harden live handoff and camera recovery

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 13:09:24 +03:00
parent eaad9deda1
commit 85035fa07b
26 changed files with 1478 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/App.tsx
+4 -10
View File
@@ -50,6 +50,7 @@ import type {
} from "./core/observation/sessionArchive";
import { useRecordedSessionAdmission } from "./core/observation/useRecordedSessionAdmission";
import { useWorkspaceLayoutProfile } from "./core/observation/useWorkspaceLayoutProfile";
import { viewerSettingsTargetIdentity } from "./core/observation/viewerSettingsTarget";
import { resolvePolygonRunRoute } from "./core/polygon/runArchive";
import {
OBSERVATION_WORKSPACE_ID,
@@ -302,14 +303,7 @@ export default function App() {
},
};
}, [recordedReplay, replayPresented, replaySources, runtime.state]);
const viewerSettingsTargetIdentity = [
runtime.state?.activeDevice?.pluginId,
runtime.state?.activeDevice?.modelId,
runtime.state?.activeDevice?.instanceId,
runtime.state?.deviceSession?.sessionId,
runtime.state?.deviceSession?.deviceId,
runtime.state?.acquisition?.acquisitionId,
].filter(Boolean).join(":") || "local-runtime";
const viewerSettingsTarget = viewerSettingsTargetIdentity(runtime.state);
const observationLayout = useObservationLayout(
activeObservationSources,
replayPresented ? undefined : runtime.setObservationSourceActive,
@@ -389,7 +383,7 @@ export default function App() {
useEffect(() => {
const profile = workspaceLayoutProfile.profile;
const applicationKey = profile
? `${profile.revision}:${viewerSettingsTargetIdentity}`
? `${profile.revision}:${viewerSettingsTarget}`
: null;
if (
!profile ||
@@ -410,7 +404,7 @@ export default function App() {
}, [
runtime.backendStatus,
runtime.updateViewerSettings,
viewerSettingsTargetIdentity,
viewerSettingsTarget,
workspaceLayoutProfile.profile,
]);
apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx
+119 -13
View File
@@ -9,7 +9,14 @@ import {
reduceCameraPlaybackRecovery,
type CameraPlaybackRecoveryEvent,
} from "../core/observation/liveCameraRecovery";
import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics";
import {
createLiveViewerDiagnosticLifecycle,
createLiveViewerInstanceId,
createLiveViewerLineage,
subscribeToLiveViewerBuildFence,
type LiveViewerDiagnosticLifecycle,
type LiveViewerFailureStage,
} from "../core/observation/liveViewerDiagnostics";
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
@@ -168,6 +175,15 @@ export function cameraTransportCloseRecoveryMessage(code: number): string {
return "Browser-preview прерван; восстанавливаем текущую камеру.";
}
export function cameraPlaybackRecoveryFailureStage(
event: CameraPlaybackRecoveryEvent,
): LiveViewerFailureStage {
if (event.type === "heartbeat") return "camera-heartbeat-reopen";
if (event.type === "network-online") return "camera-network-online-reopen";
if (event.type === "page-restore") return "camera-page-restore-reopen";
return "camera-visibility-reopen";
}
function websocketUrl(path: string): string {
const url = new URL(path, window.location.href);
if (url.protocol === "http:") url.protocol = "ws:";
@@ -195,6 +211,9 @@ export function MseFmp4WebSocketPlayer({
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
const activeDiagnosticRef = useRef<LiveViewerDiagnosticLifecycle | null>(null);
const diagnosticInstanceIdRef = useRef<string | null>(null);
const diagnosticGenerationRef = useRef(0);
const uiBuildStaleRef = useRef(false);
const [attempt, setAttempt] = useState(0);
const [uiBuildStale, setUiBuildStale] = useState(false);
@@ -232,6 +251,16 @@ export function MseFmp4WebSocketPlayer({
let disposed = false;
const transportEpoch = transportEpochRef.current + 1;
transportEpochRef.current = transportEpoch;
diagnosticInstanceIdRef.current ??= createLiveViewerInstanceId();
diagnosticGenerationRef.current += 1;
const diagnosticLifecycle = createLiveViewerDiagnosticLifecycle({
lineage: createLiveViewerLineage(
diagnosticInstanceIdRef.current,
diagnosticGenerationRef.current,
),
});
activeDiagnosticRef.current = diagnosticLifecycle;
diagnosticLifecycle.verifyBuild();
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
transportEpochRef.current,
transportEpoch,
@@ -245,6 +274,7 @@ export function MseFmp4WebSocketPlayer({
let retryTimer: number | undefined;
let receivedMedia = false;
let failed = false;
let playingReported = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
transportHealthyRef.current = false;
const recovering = Boolean(
@@ -271,7 +301,16 @@ export function MseFmp4WebSocketPlayer({
}
};
const retryTransport = (copy: string) => {
const retryTransport = (
copy: string,
failureStage: LiveViewerFailureStage,
websocketCloseCode?: number,
appendFailure?: {
cameraAppendErrorName: string;
cameraMediaSourceState: "closed" | "open" | "ended";
cameraVideoErrorCode: number | null;
},
) => {
if (!transportIsCurrent() || failed) return;
if (!cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
@@ -286,10 +325,21 @@ export function MseFmp4WebSocketPlayer({
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
diagnosticLifecycle.post({
eventCode: "live_camera_transport_restart_requested",
failureStage,
streamId: delivery.id,
cameraQueueBytes: queuedBytes,
cameraQueueSegments: queue.length,
cameraRetryCount: retry.budget.count,
websocketCloseCode,
transportEpoch,
...appendFailure,
});
queue.length = 0;
queuedBytes = 0;
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage(copy);
@@ -318,7 +368,12 @@ export function MseFmp4WebSocketPlayer({
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
cancel: (handle) => window.clearTimeout(handle),
onTimeout: (stage) => {
retryTransport(cameraStartupWatchdogRecoveryMessage(stage));
retryTransport(
cameraStartupWatchdogRecoveryMessage(stage),
stage === "first-media"
? "camera-first-media-timeout"
: "camera-first-playable-timeout",
);
},
});
}
@@ -331,6 +386,16 @@ export function MseFmp4WebSocketPlayer({
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
setMessage("");
if (!playingReported) {
playingReported = true;
diagnosticLifecycle.post({
eventCode: "live_camera_transport_playing",
streamId: delivery.id,
cameraQueueBytes: queuedBytes,
cameraQueueSegments: queue.length,
transportEpoch,
});
}
};
video.addEventListener("playing", onPlaying);
@@ -349,9 +414,24 @@ export function MseFmp4WebSocketPlayer({
try {
sourceBuffer.appendBuffer(chunk);
} catch (error) {
retryTransport(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.");
const quotaExceeded = error instanceof DOMException
&& error.name === "QuotaExceededError";
retryTransport(
quotaExceeded
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.",
quotaExceeded ? "camera-append-quota" : "camera-append-error",
undefined,
{
cameraAppendErrorName: error instanceof DOMException
? error.name
: error instanceof Error
? error.name
: "UnknownError",
cameraMediaSourceState: mediaSource.readyState,
cameraVideoErrorCode: video.error?.code ?? null,
},
);
}
};
@@ -361,7 +441,10 @@ export function MseFmp4WebSocketPlayer({
// Never drop arbitrary fMP4 fragments: the following samples may depend
// on them. A bounded reset is safer and keeps live latency deterministic.
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер.");
retryTransport(
"Видеодекодер отстал от эфира; восстанавливаем live-буфер.",
"camera-queue-capacity",
);
return;
}
queue.push(chunk);
@@ -427,7 +510,10 @@ export function MseFmp4WebSocketPlayer({
appendNext();
};
onBufferError = () => {
retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder.");
retryTransport(
"MSE сбросил видеосегмент; восстанавливаем decoder.",
"camera-source-buffer-error",
);
};
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
sourceBuffer.addEventListener("error", onBufferError);
@@ -453,16 +539,26 @@ export function MseFmp4WebSocketPlayer({
enqueue(event.data);
} else if (event.data instanceof Blob) {
void event.data.arrayBuffer().then(enqueue).catch(() => {
retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал.");
retryTransport(
"Получен повреждённый видеосегмент; восстанавливаем канал.",
"camera-fragment-decode",
);
});
}
});
socket.addEventListener("error", () => {
retryTransport("Связь с локальным video adapter потеряна; переподключаемся.");
retryTransport(
"Связь с локальным video adapter потеряна; переподключаемся.",
"camera-websocket-error",
);
});
socket.addEventListener("close", (event) => {
if (!transportIsCurrent() || failed) return;
retryTransport(cameraTransportCloseRecoveryMessage(event.code));
retryTransport(
cameraTransportCloseRecoveryMessage(event.code),
"camera-websocket-close",
event.code,
);
});
};
@@ -504,6 +600,10 @@ export function MseFmp4WebSocketPlayer({
if (activeTransportDisposeRef.current === disposeTransport) {
activeTransportDisposeRef.current = null;
}
if (activeDiagnosticRef.current === diagnosticLifecycle) {
activeDiagnosticRef.current = null;
}
diagnosticLifecycle.dispose();
disposeTransport();
};
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
@@ -530,6 +630,12 @@ export function MseFmp4WebSocketPlayer({
});
recovery = decision.state;
if (!decision.reopen) return;
activeDiagnosticRef.current?.post({
eventCode: "live_camera_transport_restart_requested",
failureStage: cameraPlaybackRecoveryFailureStage(event),
streamId: delivery.id,
transportEpoch: transportEpochRef.current,
});
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
apps/control-station/src/components/RerunViewport.tsx
+5 -7
View File
@@ -1063,9 +1063,10 @@ export function RerunViewport({
let liveRecoveryRetryTimer: number | undefined;
let recordingOpened = false;
let recordingOpenTimedOut = false;
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
);
// A new browser receiver has not observed any backend publication yet.
// Starting from the already-consumed runtime sequence makes the first
// four-second check look stalled and destroys a healthy partial replay.
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null);
let viewerStartResolved = false;
let latestLiveRangeMaxNs: number | null = null;
let liveWatchdog = initialLiveReceiverWatchdogState();
@@ -1718,10 +1719,7 @@ export function RerunViewport({
if (!isRecordedSource) {
// Measure native store admission from the resolved viewer start,
// not from dynamic module import or React effect setup time.
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
Date.now(),
);
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null, Date.now());
discoverActiveLiveRecording();
if (!recordingOpened) {
diagnosticLifecycle.armAdmissionInterval(
apps/control-station/src/core/observation/liveViewerDiagnostics.ts
+65 -28
View File
@@ -4,13 +4,28 @@ export type LiveViewerDiagnosticEventCode =
| "live_receiver_recovered"
| "live_receiver_recovery_exhausted"
| "live_receiver_active_store_admitted"
| "live_receiver_error";
| "live_receiver_error"
| "live_camera_transport_restart_requested"
| "live_camera_transport_playing";
export type LiveViewerFailureStage =
| "recording-open-timeout"
| "viewer-start"
| "module-load"
| "receiver-stalled";
| "receiver-stalled"
| "camera-first-media-timeout"
| "camera-first-playable-timeout"
| "camera-queue-capacity"
| "camera-append-quota"
| "camera-append-error"
| "camera-source-buffer-error"
| "camera-fragment-decode"
| "camera-websocket-error"
| "camera-websocket-close"
| "camera-heartbeat-reopen"
| "camera-visibility-reopen"
| "camera-network-online-reopen"
| "camera-page-restore-reopen";
export interface LiveViewerDiagnostic {
eventCode: LiveViewerDiagnosticEventCode;
@@ -20,6 +35,14 @@ export interface LiveViewerDiagnostic {
viewerRangeMaxNs?: number | null;
stalledForMs?: number | null;
recoveryAttempt?: number | null;
cameraQueueBytes?: number | null;
cameraQueueSegments?: number | null;
cameraRetryCount?: number | null;
websocketCloseCode?: number | null;
transportEpoch?: number | null;
cameraAppendErrorName?: string | null;
cameraMediaSourceState?: "closed" | "open" | "ended" | null;
cameraVideoErrorCode?: number | null;
}
export interface LiveViewerLineage {
@@ -69,7 +92,6 @@ const DEVELOPMENT_UI_BUILD_ID = "development";
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const HASHED_UI_BUILD_ID = /^\/assets\/[A-Za-z0-9._/-]+-[A-Za-z0-9_-]{8,}\.js$/;
const UI_BUILD_CHECK_INTERVAL_MILLISECONDS = 15_000;
const UI_BUILD_RELOAD_DELAY_MILLISECONDS = 50;
let documentInstanceId: string | null = null;
let sharedUiBuildCoordinator: UiBuildStaleCoordinator | null = null;
@@ -172,32 +194,45 @@ export function liveViewerDiagnosticBody(
...(safeInteger(event.recoveryAttempt) === undefined
? {}
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
...(safeInteger(event.cameraQueueBytes) === undefined
? {}
: { camera_queue_bytes: safeInteger(event.cameraQueueBytes) }),
...(safeInteger(event.cameraQueueSegments) === undefined
? {}
: { camera_queue_segments: safeInteger(event.cameraQueueSegments) }),
...(safeInteger(event.cameraRetryCount) === undefined
? {}
: { camera_retry_count: safeInteger(event.cameraRetryCount) }),
...(safeInteger(event.websocketCloseCode) === undefined
? {}
: { websocket_close_code: safeInteger(event.websocketCloseCode) }),
...(safeInteger(event.transportEpoch) === undefined
? {}
: { transport_epoch: safeInteger(event.transportEpoch) }),
...(event.cameraAppendErrorName
? { camera_append_error_name: event.cameraAppendErrorName }
: {}),
...(event.cameraMediaSourceState
? { camera_media_source_state: event.cameraMediaSourceState }
: {}),
...(safeInteger(event.cameraVideoErrorCode) === undefined
? {}
: { camera_video_error_code: safeInteger(event.cameraVideoErrorCode) }),
};
}
export function createUiBuildStaleCoordinator({
scheduleReload,
reload,
}: {
scheduleReload: (callback: () => void, delayMilliseconds: number) => void;
reload: () => void;
}): UiBuildStaleCoordinator {
const listeners = new Set<(event: StaleUiBuild) => void>();
export function createUiBuildStaleCoordinator(): UiBuildStaleCoordinator {
let staleEvent: StaleUiBuild | null = null;
let reloadScheduled = false;
return {
subscribe(listener) {
listeners.add(listener);
if (staleEvent) listener(staleEvent);
return () => listeners.delete(listener);
subscribe(_listener) {
// Build drift is diagnostic-only. The loaded document and its local
// Rerun/camera transports remain authoritative until normal unmount or
// their existing bounded recovery replaces them.
return () => undefined;
},
report(event) {
if (event.loadedUiBuildId === event.expectedUiBuildId || staleEvent) return;
staleEvent = event;
for (const listener of [...listeners]) listener(event);
if (reloadScheduled) return;
reloadScheduled = true;
scheduleReload(reload, UI_BUILD_RELOAD_DELAY_MILLISECONDS);
},
stale() {
return staleEvent !== null;
@@ -206,12 +241,7 @@ export function createUiBuildStaleCoordinator({
}
function browserUiBuildCoordinator(): UiBuildStaleCoordinator {
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator({
scheduleReload: (callback, delayMilliseconds) => {
window.setTimeout(callback, delayMilliseconds);
},
reload: () => window.location.reload(),
});
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator();
return sharedUiBuildCoordinator;
}
@@ -254,10 +284,17 @@ export function verifyLiveViewerClientBuild(
lineage: Pick<LiveViewerLineage, "uiBuildId">,
signal?: AbortSignal,
): void {
if (signal?.aborted || lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID) return;
if (
signal?.aborted ||
lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID ||
browserUiBuildCoordinator().stale()
) return;
void fetch("/api/v1/viewer/client-contract", {
method: "GET",
headers: { Accept: "application/json" },
headers: {
Accept: "application/json",
"X-MissionCore-UI-Build": lineage.uiBuildId,
},
cache: "no-store",
signal,
}).then(async (response) => {
apps/control-station/src/core/observation/viewerSettingsTarget.ts
+19
View File
@@ -0,0 +1,19 @@
import type { MissionRuntimeState } from "../runtime/contracts";
export function viewerSettingsTargetIdentity(
state: MissionRuntimeState | null,
): string {
const deviceSessionId = state?.deviceSession?.sessionId?.trim();
const sessionDeviceId = state?.deviceSession?.deviceId?.trim();
if (deviceSessionId || sessionDeviceId) {
return ["device-session", deviceSessionId, sessionDeviceId]
.filter(Boolean)
.join(":");
}
return [
state?.activeDevice?.pluginId,
state?.activeDevice?.modelId,
state?.activeDevice?.instanceId,
].filter(Boolean).join(":") || "local-runtime";
}