fix(k1): stabilize live recovery and media admission

This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 00:03:01 +03:00
parent 7217244886
commit eaad9deda1
29 changed files with 1645 additions and 199 deletions
@@ -41,7 +41,9 @@ export interface CameraStartupWatchdog {
* Supervise only the disposable browser transport. One fixed first-media
* deadline covers both an MSE that never opens and an open-but-silent socket.
* The first non-empty fragment then starts a separate first-playable-frame
* deadline; later fragments deliberately do not extend it.
* deadline. Every subsequent fragment extends that deadline: a browser that is
* still receiving the authoritative stream must not destroy its partial MSE
* decode solely because the main thread or decoder needed longer to start.
*/
export function createCameraStartupWatchdog({
schedule,
@@ -83,7 +85,7 @@ export function createCameraStartupWatchdog({
arm("first-media", firstMediaTimeoutMs);
},
markMediaReceived() {
if (mediaReceived || playing) return;
if (playing) return;
mediaReceived = true;
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
},
@@ -189,6 +191,7 @@ export function MseFmp4WebSocketPlayer({
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
const recoveryPendingAuthorityRef = useRef<string | null>(null);
const transportHealthyRef = useRef(false);
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
@@ -243,6 +246,7 @@ export function MseFmp4WebSocketPlayer({
let receivedMedia = false;
let failed = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
transportHealthyRef.current = false;
const recovering = Boolean(
activeAuthorityRef.current
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
@@ -254,6 +258,7 @@ export function MseFmp4WebSocketPlayer({
if (!transportIsCurrent() || failed) return;
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
recoveryPendingAuthorityRef.current = null;
setStatus("error");
setMessage(copy);
@@ -280,6 +285,7 @@ export function MseFmp4WebSocketPlayer({
}
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
@@ -320,6 +326,7 @@ export function MseFmp4WebSocketPlayer({
const onPlaying = () => {
if (!transportIsCurrent() || failed) return;
startupWatchdog?.markPlaying();
transportHealthyRef.current = true;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
@@ -464,6 +471,7 @@ export function MseFmp4WebSocketPlayer({
const disposeTransport = () => {
if (disposed) return;
disposed = true;
transportHealthyRef.current = false;
startupWatchdog?.clear();
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
queue.length = 0;
@@ -518,6 +526,7 @@ export function MseFmp4WebSocketPlayer({
now: Date.now(),
documentVisible: document.visibilityState === "visible",
networkOnline: navigator.onLine !== false,
transportHealthy: transportHealthyRef.current,
});
recovery = decision.state;
if (!decision.reopen) return;
@@ -8,7 +8,7 @@ import {
initialLiveReceiverRecoveryState,
initialLiveReceiverWatchdogState,
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
LIVE_RECEIVER_OPEN_MAX_AGE_MS,
LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS,
requestLiveReceiverRecovery,
} from "../core/observation/liveReceiverWatchdog";
import {
@@ -1091,25 +1091,6 @@ export function RerunViewport({
};
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
function refreshOpeningLiveReceiver(openForMs: number) {
if (disposed || recordingOpened) return;
diagnosticLifecycle.post({
eventCode: "live_receiver_restart_requested",
failureStage: "recording-open-timeout",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
viewerRangeMaxNs: latestLiveRangeMaxNs,
stalledForMs: Math.round(openForMs),
recoveryAttempt: liveRecoveryRef.current.attempts || null,
});
setStatus("loading");
onStatusChange?.(
"loading",
"Живой визуализатор обновляет приёмник продолжающегося потока.",
);
disposeViewer?.();
setRetryNonce((nonce) => nonce + 1);
}
const armLiveRecordingOpenTimer = () => {
clearLiveRecordingOpenTimer();
diagnosticLifecycle.armAdmissionTimeout(() => {
@@ -1129,17 +1110,9 @@ export function RerunViewport({
armLiveRecordingOpenTimer();
return;
}
if (observed.signal === "refresh-receiver") {
// Publication is healthy, so this is presentation-only maintenance:
// refresh the aged native receiver without spending (or clearing)
// recovery debt.
recordingOpenTimedOut = true;
refreshOpeningLiveReceiver(observed.openForMs);
return;
}
recordingOpenTimedOut = true;
requestLiveRecovery("recording-open-timeout");
}, LIVE_RECEIVER_OPEN_MAX_AGE_MS);
}, LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS);
};
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
const clearLiveRecoveryRetryTimer = () => {
@@ -26,6 +26,7 @@ export interface CameraPlaybackRecoveryContext {
now: number;
documentVisible: boolean;
networkOnline: boolean;
transportHealthy?: boolean;
}
export interface CameraPlaybackRecoveryDecision {
@@ -186,7 +187,8 @@ export function reduceCameraPlaybackRecovery(
} else if (event.type === "page-restore") {
candidate = event.persisted;
} else if (event.type === "heartbeat") {
candidate = now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
candidate = context.transportHealthy !== true
&& now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
}
const outsideCooldown = current.lastReopenAt === null
@@ -31,7 +31,6 @@ export interface LiveReceiverOpenWatchdogState {
export type LiveReceiverOpenWatchdogSignal =
| "wait-for-store"
| "refresh-receiver"
| "restart-receiver";
export interface LiveReceiverOpenWatchdogResult {
@@ -60,10 +59,11 @@ export interface LiveReceiverRecoveryRequest {
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
// The bridge publishes its URL only after StoreInfo, blueprint and static
// scene data have been flushed. A receiver that still has not admitted that
// store after one operator-visible four-second window is wedged, not merely
// slow; keeping it for 48 seconds made a healthy live scan look blank.
export const LIVE_RECEIVER_OPEN_MAX_AGE_MS = 4_000;
// scene data have been flushed. Poll at the original four-second boundary,
// but never discard a receiver while this exact acquisition is still proving
// fresh publication progress: doing so throws away its partial store replay and
// can keep a healthy long-running stream blank indefinitely.
export const LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS = 4_000;
const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const;
export function liveReceiverRecoveryRetryDelay(attempt: number): number {
@@ -216,11 +216,10 @@ export function initialLiveReceiverOpenWatchdogState(
/**
* Keep one still-opening Rerun receiver alive while the backend is proving
* fresh publication progress. Recreating the WASM receiver on a fixed timer
* can repeatedly discard an otherwise healthy late StoreInfo replay. Rolling
* patience is nevertheless bounded: a receiver that has not admitted a store
* by the absolute open-age limit is refreshed without consuming the recovery
* budget. A true lack of backend progress delegates to the bounded restart
* policy. Recovery debt is cleared only after viewer admission, never merely
* can repeatedly discard an otherwise healthy late StoreInfo replay. Preserve
* that receiver for as long as the backend proves fresh publication. A true
* lack of backend progress delegates to the bounded restart policy on the next
* check. Recovery debt is cleared only after viewer admission, never merely
* because the backend counter advanced.
*/
export function advanceLiveReceiverOpenWatchdog(
@@ -228,7 +227,6 @@ export function advanceLiveReceiverOpenWatchdog(
recoveryState: LiveReceiverRecoveryState,
backendActivitySequence: number | null,
nowMs = Date.now(),
maxOpenAgeMs = LIVE_RECEIVER_OPEN_MAX_AGE_MS,
): LiveReceiverOpenWatchdogResult {
const sequence = validBackendActivitySequence(backendActivitySequence);
const previous = current.lastBackendActivitySequence;
@@ -247,9 +245,7 @@ export function advanceLiveReceiverOpenWatchdog(
return {
state,
recoveryState,
signal: openForMs >= maxOpenAgeMs
? "refresh-receiver"
: "wait-for-store",
signal: "wait-for-store",
openForMs,
};
}
@@ -129,6 +129,26 @@ export function admitLiveDefaultPresentations(
};
}
/**
* A restored workspace may hide cameras that the device plugin has not selected.
* It must still admit the exact selected delivery in a fresh browser document,
* or a replacement delivery for a camera already presented in this acquisition.
* An explicit close remains a separate acquisition-scoped fence in
* `admitLiveDefaultPresentations` and de-selects the source in the plugin first.
*/
export function restoredLayoutMayAdmitLiveDefault(
sources: readonly ObservationSourceDescriptor[],
presentedAcquisitionSources: ReadonlySet<string>,
): boolean {
const selectedSources = sources.filter(automaticLivePresentationIdentity);
if (selectedSources.length === 0) return false;
if (presentedAcquisitionSources.size === 0) return true;
return selectedSources.some((source) => {
const lineage = livePresentationCloseFence(source);
return Boolean(lineage && presentedAcquisitionSources.has(lineage));
});
}
function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string {
return sources
.map((source) => [
@@ -170,6 +190,7 @@ export function useObservationLayout(
const restoredLayoutAuthorityRef = useRef(false);
const admittedLivePresentationIdentitiesRef = useRef(new Set<string>());
const closedLiveAcquisitionSourcesRef = useRef(new Set<string>());
const presentedLiveAcquisitionSourcesRef = useRef(new Set<string>());
const initializedCatalog = useRef<string | null>(null);
const sourceIdList = sources.map((source) => source.id).sort();
const sourceIdsIdentity = sourceIdList.join("\u0000");
@@ -182,6 +203,12 @@ export function useObservationLayout(
const commitVisibleIds = useCallback((next: readonly string[]) => {
const unique = [...new Set(next)];
const visible = new Set(unique);
for (const source of sourcesRef.current) {
if (!visible.has(source.id) || !automaticLivePresentationIdentity(source)) continue;
const lineage = livePresentationCloseFence(source);
if (lineage) presentedLiveAcquisitionSourcesRef.current.add(lineage);
}
visibleIdsRef.current = unique;
setVisibleIds(unique);
}, []);
@@ -352,11 +379,18 @@ export function useObservationLayout(
.filter((candidate): candidate is string => candidate !== null)
.sort());
useEffect(() => {
if (restoredLayoutAuthorityRef.current) return;
const admitAutomaticLivePresentations = useCallback(() => {
const currentSources = sourcesRef.current;
if (
restoredLayoutAuthorityRef.current
&& !restoredLayoutMayAdmitLiveDefault(
currentSources,
presentedLiveAcquisitionSourcesRef.current,
)
) return;
const admission = admitLiveDefaultPresentations(
visibleIdsRef.current,
sources,
currentSources,
admittedLivePresentationIdentitiesRef.current,
closedLiveAcquisitionSourcesRef.current,
);
@@ -367,7 +401,11 @@ export function useObservationLayout(
commitVisibleIds(admission.visibleIds);
clearPresentation(admission.removedIds, false);
persistLiveLayout();
}, [clearPresentation, commitVisibleIds, persistLiveLayout, selectedDeliveryIdentity]);
}, [clearPresentation, commitVisibleIds, persistLiveLayout]);
useEffect(() => {
admitAutomaticLivePresentations();
}, [admitAutomaticLivePresentations, selectedDeliveryIdentity]);
const markPending = useCallback((source: ObservationSourceDescriptor, pending: boolean) => {
const groupId = source.activation?.groupId;
@@ -545,7 +583,10 @@ export function useObservationLayout(
});
restoredLayoutAuthorityRef.current = true;
applyDesiredSnapshot(desiredSnapshotRef.current, "reset");
}, [applyDesiredSnapshot]);
// The selected delivery may already have arrived before the saved layout.
// Re-run admission here so restore ordering cannot strand its camera window.
admitAutomaticLivePresentations();
}, [admitAutomaticLivePresentations, applyDesiredSnapshot]);
const visibleSourceIds = useMemo(() => new Set(visibleIds), [visibleIds]);
const pendingSourceIds = useMemo(() => new Set(pendingIds), [pendingIds]);
@@ -235,12 +235,7 @@ function SpatialWorkspace({
Boolean(observationLayout.maximizedFloatingSourceId);
const timeline = state?.observationTimeline;
const viewportRef = useRef<HTMLDivElement>(null);
const intentionalSourceEnd = !recordedSource && [
"awaiting_external_stop",
"stopping",
"finalizing",
"completed",
].includes(state?.acquisition?.state ?? "");
const intentionalSourceEnd = !recordedSource && state?.sourceMode === "idle" && ["awaiting_external_stop", "stopping", "finalizing", "completed",].includes(state?.acquisition?.state ?? "");
const presentedViewerStatus = intentionalSourceEnd
? "idle"
: rerunPresentationStatus(