fix(k1): stabilize live recovery and media admission

This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 00:03:01 +03:00
parent 7217244886
commit eaad9deda1
29 changed files with 1645 additions and 199 deletions
@@ -41,7 +41,9 @@ export interface CameraStartupWatchdog {
* Supervise only the disposable browser transport. One fixed first-media
* deadline covers both an MSE that never opens and an open-but-silent socket.
* The first non-empty fragment then starts a separate first-playable-frame
* deadline; later fragments deliberately do not extend it.
* deadline. Every subsequent fragment extends that deadline: a browser that is
* still receiving the authoritative stream must not destroy its partial MSE
* decode solely because the main thread or decoder needed longer to start.
*/
export function createCameraStartupWatchdog({
schedule,
@@ -83,7 +85,7 @@ export function createCameraStartupWatchdog({
arm("first-media", firstMediaTimeoutMs);
},
markMediaReceived() {
if (mediaReceived || playing) return;
if (playing) return;
mediaReceived = true;
arm("first-playable-frame", firstPlayableFrameTimeoutMs);
},
@@ -189,6 +191,7 @@ export function MseFmp4WebSocketPlayer({
const leaseRetryRef = useRef(resetCameraLeaseRetryBudget(delivery.id));
const activeAuthorityRef = useRef(recoveryAuthorityIdentity);
const recoveryPendingAuthorityRef = useRef<string | null>(null);
const transportHealthyRef = useRef(false);
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
@@ -243,6 +246,7 @@ export function MseFmp4WebSocketPlayer({
let receivedMedia = false;
let failed = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
transportHealthyRef.current = false;
const recovering = Boolean(
activeAuthorityRef.current
&& recoveryPendingAuthorityRef.current === activeAuthorityRef.current,
@@ -254,6 +258,7 @@ export function MseFmp4WebSocketPlayer({
if (!transportIsCurrent() || failed) return;
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
recoveryPendingAuthorityRef.current = null;
setStatus("error");
setMessage(copy);
@@ -280,6 +285,7 @@ export function MseFmp4WebSocketPlayer({
}
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
@@ -320,6 +326,7 @@ export function MseFmp4WebSocketPlayer({
const onPlaying = () => {
if (!transportIsCurrent() || failed) return;
startupWatchdog?.markPlaying();
transportHealthyRef.current = true;
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
@@ -464,6 +471,7 @@ export function MseFmp4WebSocketPlayer({
const disposeTransport = () => {
if (disposed) return;
disposed = true;
transportHealthyRef.current = false;
startupWatchdog?.clear();
if (retryTimer !== undefined) window.clearTimeout(retryTimer);
queue.length = 0;
@@ -518,6 +526,7 @@ export function MseFmp4WebSocketPlayer({
now: Date.now(),
documentVisible: document.visibilityState === "visible",
networkOnline: navigator.onLine !== false,
transportHealthy: transportHealthyRef.current,
});
recovery = decision.state;
if (!decision.reopen) return;
@@ -8,7 +8,7 @@ import {
initialLiveReceiverRecoveryState,
initialLiveReceiverWatchdogState,
LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS,
LIVE_RECEIVER_OPEN_MAX_AGE_MS,
LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS,
requestLiveReceiverRecovery,
} from "../core/observation/liveReceiverWatchdog";
import {
@@ -1091,25 +1091,6 @@ export function RerunViewport({
};
const clearRecordedAdmissionWatchdog = () => recordedOpenWatchdog?.clear();
const clearLiveRecordingOpenTimer = diagnosticLifecycle.clearAdmissionTimeout;
function refreshOpeningLiveReceiver(openForMs: number) {
if (disposed || recordingOpened) return;
diagnosticLifecycle.post({
eventCode: "live_receiver_restart_requested",
failureStage: "recording-open-timeout",
streamId: liveStreamIdRef.current,
backendActivitySequence: liveActivitySequenceRef.current,
viewerRangeMaxNs: latestLiveRangeMaxNs,
stalledForMs: Math.round(openForMs),
recoveryAttempt: liveRecoveryRef.current.attempts || null,
});
setStatus("loading");
onStatusChange?.(
"loading",
"Живой визуализатор обновляет приёмник продолжающегося потока.",
);
disposeViewer?.();
setRetryNonce((nonce) => nonce + 1);
}
const armLiveRecordingOpenTimer = () => {
clearLiveRecordingOpenTimer();
diagnosticLifecycle.armAdmissionTimeout(() => {
@@ -1129,17 +1110,9 @@ export function RerunViewport({
armLiveRecordingOpenTimer();
return;
}
if (observed.signal === "refresh-receiver") {
// Publication is healthy, so this is presentation-only maintenance:
// refresh the aged native receiver without spending (or clearing)
// recovery debt.
recordingOpenTimedOut = true;
refreshOpeningLiveReceiver(observed.openForMs);
return;
}
recordingOpenTimedOut = true;
requestLiveRecovery("recording-open-timeout");
}, LIVE_RECEIVER_OPEN_MAX_AGE_MS);
}, LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS);
};
const clearLiveRecordingDiscoveryTimer = diagnosticLifecycle.clearAdmissionInterval;
const clearLiveRecoveryRetryTimer = () => {
@@ -26,6 +26,7 @@ export interface CameraPlaybackRecoveryContext {
now: number;
documentVisible: boolean;
networkOnline: boolean;
transportHealthy?: boolean;
}
export interface CameraPlaybackRecoveryDecision {
@@ -186,7 +187,8 @@ export function reduceCameraPlaybackRecovery(
} else if (event.type === "page-restore") {
candidate = event.persisted;
} else if (event.type === "heartbeat") {
candidate = now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
candidate = context.transportHealthy !== true
&& now - current.lastObservedAt >= CAMERA_WAKE_GAP_MS;
}
const outsideCooldown = current.lastReopenAt === null
@@ -31,7 +31,6 @@ export interface LiveReceiverOpenWatchdogState {
export type LiveReceiverOpenWatchdogSignal =
| "wait-for-store"
| "refresh-receiver"
| "restart-receiver";
export interface LiveReceiverOpenWatchdogResult {
@@ -60,10 +59,11 @@ export interface LiveReceiverRecoveryRequest {
export const LIVE_RECEIVER_STALL_THRESHOLD_MS = 5_000;
export const LIVE_RECEIVER_MAX_RECOVERY_ATTEMPTS = 3;
// The bridge publishes its URL only after StoreInfo, blueprint and static
// scene data have been flushed. A receiver that still has not admitted that
// store after one operator-visible four-second window is wedged, not merely
// slow; keeping it for 48 seconds made a healthy live scan look blank.
export const LIVE_RECEIVER_OPEN_MAX_AGE_MS = 4_000;
// scene data have been flushed. Poll at the original four-second boundary,
// but never discard a receiver while this exact acquisition is still proving
// fresh publication progress: doing so throws away its partial store replay and
// can keep a healthy long-running stream blank indefinitely.
export const LIVE_RECEIVER_OPEN_CHECK_INTERVAL_MS = 4_000;
const LIVE_RECEIVER_RECOVERY_DELAYS_MS = [400, 1_000, 2_000, 5_000] as const;
export function liveReceiverRecoveryRetryDelay(attempt: number): number {
@@ -216,11 +216,10 @@ export function initialLiveReceiverOpenWatchdogState(
/**
* Keep one still-opening Rerun receiver alive while the backend is proving
* fresh publication progress. Recreating the WASM receiver on a fixed timer
* can repeatedly discard an otherwise healthy late StoreInfo replay. Rolling
* patience is nevertheless bounded: a receiver that has not admitted a store
* by the absolute open-age limit is refreshed without consuming the recovery
* budget. A true lack of backend progress delegates to the bounded restart
* policy. Recovery debt is cleared only after viewer admission, never merely
* can repeatedly discard an otherwise healthy late StoreInfo replay. Preserve
* that receiver for as long as the backend proves fresh publication. A true
* lack of backend progress delegates to the bounded restart policy on the next
* check. Recovery debt is cleared only after viewer admission, never merely
* because the backend counter advanced.
*/
export function advanceLiveReceiverOpenWatchdog(
@@ -228,7 +227,6 @@ export function advanceLiveReceiverOpenWatchdog(
recoveryState: LiveReceiverRecoveryState,
backendActivitySequence: number | null,
nowMs = Date.now(),
maxOpenAgeMs = LIVE_RECEIVER_OPEN_MAX_AGE_MS,
): LiveReceiverOpenWatchdogResult {
const sequence = validBackendActivitySequence(backendActivitySequence);
const previous = current.lastBackendActivitySequence;
@@ -247,9 +245,7 @@ export function advanceLiveReceiverOpenWatchdog(
return {
state,
recoveryState,
signal: openForMs >= maxOpenAgeMs
? "refresh-receiver"
: "wait-for-store",
signal: "wait-for-store",
openForMs,
};
}
@@ -129,6 +129,26 @@ export function admitLiveDefaultPresentations(
};
}
/**
* A restored workspace may hide cameras that the device plugin has not selected.
* It must still admit the exact selected delivery in a fresh browser document,
* or a replacement delivery for a camera already presented in this acquisition.
* An explicit close remains a separate acquisition-scoped fence in
* `admitLiveDefaultPresentations` and de-selects the source in the plugin first.
*/
export function restoredLayoutMayAdmitLiveDefault(
sources: readonly ObservationSourceDescriptor[],
presentedAcquisitionSources: ReadonlySet<string>,
): boolean {
const selectedSources = sources.filter(automaticLivePresentationIdentity);
if (selectedSources.length === 0) return false;
if (presentedAcquisitionSources.size === 0) return true;
return selectedSources.some((source) => {
const lineage = livePresentationCloseFence(source);
return Boolean(lineage && presentedAcquisitionSources.has(lineage));
});
}
function catalogIdentity(sources: readonly ObservationSourceDescriptor[]): string {
return sources
.map((source) => [
@@ -170,6 +190,7 @@ export function useObservationLayout(
const restoredLayoutAuthorityRef = useRef(false);
const admittedLivePresentationIdentitiesRef = useRef(new Set<string>());
const closedLiveAcquisitionSourcesRef = useRef(new Set<string>());
const presentedLiveAcquisitionSourcesRef = useRef(new Set<string>());
const initializedCatalog = useRef<string | null>(null);
const sourceIdList = sources.map((source) => source.id).sort();
const sourceIdsIdentity = sourceIdList.join("\u0000");
@@ -182,6 +203,12 @@ export function useObservationLayout(
const commitVisibleIds = useCallback((next: readonly string[]) => {
const unique = [...new Set(next)];
const visible = new Set(unique);
for (const source of sourcesRef.current) {
if (!visible.has(source.id) || !automaticLivePresentationIdentity(source)) continue;
const lineage = livePresentationCloseFence(source);
if (lineage) presentedLiveAcquisitionSourcesRef.current.add(lineage);
}
visibleIdsRef.current = unique;
setVisibleIds(unique);
}, []);
@@ -352,11 +379,18 @@ export function useObservationLayout(
.filter((candidate): candidate is string => candidate !== null)
.sort());
useEffect(() => {
if (restoredLayoutAuthorityRef.current) return;
const admitAutomaticLivePresentations = useCallback(() => {
const currentSources = sourcesRef.current;
if (
restoredLayoutAuthorityRef.current
&& !restoredLayoutMayAdmitLiveDefault(
currentSources,
presentedLiveAcquisitionSourcesRef.current,
)
) return;
const admission = admitLiveDefaultPresentations(
visibleIdsRef.current,
sources,
currentSources,
admittedLivePresentationIdentitiesRef.current,
closedLiveAcquisitionSourcesRef.current,
);
@@ -367,7 +401,11 @@ export function useObservationLayout(
commitVisibleIds(admission.visibleIds);
clearPresentation(admission.removedIds, false);
persistLiveLayout();
}, [clearPresentation, commitVisibleIds, persistLiveLayout, selectedDeliveryIdentity]);
}, [clearPresentation, commitVisibleIds, persistLiveLayout]);
useEffect(() => {
admitAutomaticLivePresentations();
}, [admitAutomaticLivePresentations, selectedDeliveryIdentity]);
const markPending = useCallback((source: ObservationSourceDescriptor, pending: boolean) => {
const groupId = source.activation?.groupId;
@@ -545,7 +583,10 @@ export function useObservationLayout(
});
restoredLayoutAuthorityRef.current = true;
applyDesiredSnapshot(desiredSnapshotRef.current, "reset");
}, [applyDesiredSnapshot]);
// The selected delivery may already have arrived before the saved layout.
// Re-run admission here so restore ordering cannot strand its camera window.
admitAutomaticLivePresentations();
}, [admitAutomaticLivePresentations, applyDesiredSnapshot]);
const visibleSourceIds = useMemo(() => new Set(visibleIds), [visibleIds]);
const pendingSourceIds = useMemo(() => new Set(pendingIds), [pendingIds]);
@@ -235,12 +235,7 @@ function SpatialWorkspace({
Boolean(observationLayout.maximizedFloatingSourceId);
const timeline = state?.observationTimeline;
const viewportRef = useRef<HTMLDivElement>(null);
const intentionalSourceEnd = !recordedSource && [
"awaiting_external_stop",
"stopping",
"finalizing",
"completed",
].includes(state?.acquisition?.state ?? "");
const intentionalSourceEnd = !recordedSource && state?.sourceMode === "idle" && ["awaiting_external_stop", "stopping", "finalizing", "completed",].includes(state?.acquisition?.state ?? "");
const presentedViewerStatus = intentionalSourceEnd
? "idle"
: rerunPresentationStatus(
@@ -620,6 +620,53 @@ test("canonical K1 preparation stops before START and guards every async stage",
);
});
test("K1 control-phase polling fails closed instead of stacking timed-out state reads", async () => {
const hookSource = await readFile(
new URL(
"../../../plugins/xgrids-k1/frontend/src/useXgridsK1Runtime.ts",
import.meta.url,
),
"utf8",
);
const controlPollingLoop = hookSource.slice(
hookSource.indexOf("async function waitForControlPhase"),
hookSource.indexOf("async function waitForPhysicalReconciliationProof"),
);
assert.match(hookSource, /const CONTROL_PHASE_WAIT_TIMEOUT_MS = 120_000/);
assert.match(controlPollingLoop, /xgridsK1Api\.getState\(\)/);
assert.doesNotMatch(controlPollingLoop, /ApiRequestTimeoutError/);
assert.doesNotMatch(controlPollingLoop, /catch \(error\)/);
assert.doesNotMatch(
controlPollingLoop,
/(?:openApplicationControlSession|enterApplicationWorkspace|prepareAcquisition|startAcquisition|stopAcquisition)\(/,
);
});
test("K1 frontend coalesces concurrent read-only state requests without caching them", async () => {
const apiSource = await readFile(
new URL(
"../../../plugins/xgrids-k1/frontend/src/api.ts",
import.meta.url,
),
"utf8",
);
const singleFlight = apiSource.slice(
apiSource.indexOf("let stateReadInFlight"),
apiSource.indexOf("export const xgridsK1Api"),
);
const getState = apiSource.slice(
apiSource.indexOf("async getState"),
apiSource.indexOf("scanBle("),
);
assert.match(singleFlight, /if \(stateReadInFlight\) return stateReadInFlight/);
assert.match(singleFlight, /invokeState\(xgridsK1Actions\.stateRead\)/);
assert.match(singleFlight, /stateReadInFlight === request[\s\S]*?stateReadInFlight = null/);
assert.match(getState, /return readStateSingleFlight\(\)/);
assert.doesNotMatch(singleFlight, /setTimeout|setInterval|retry/i);
});
test("K1 control-session CAS is exact, integer-only and mapped for each mutation family", () => {
const state = {
application_control_session: {
@@ -244,13 +244,16 @@ test("connecting Rerun authority requires an exact recovery generation lease", (
);
});
test("opening receiver gets bounded rolling patience while backend publication advances", () => {
test("opening receiver preserves partial store replay while backend publication advances", () => {
let openState = initialLiveReceiverOpenWatchdogState(0, 0);
let recoveryState = initialLiveReceiverRecoveryState();
const samples = [
[134, 3_999, "wait-for-store"],
[266, 4_000, "refresh-receiver"],
[266, 4_000, "wait-for-store"],
[380, 8_000, "wait-for-store"],
[486, 12_000, "wait-for-store"],
[700, 120_000, "wait-for-store"],
];
for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
const observed = advanceLiveReceiverOpenWatchdog(
@@ -302,17 +305,17 @@ test("fresh backend progress preserves earlier restart debt until viewer admissi
});
});
test("aged active receiver refresh does not spend or erase restart debt", () => {
test("aged active receiver remains intact and preserves restart debt", () => {
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
const observed = advanceLiveReceiverOpenWatchdog(
openState,
consumedRestart.state,
900,
4_000,
12_000,
);
assert.equal(observed.signal, "refresh-receiver");
assert.equal(observed.signal, "wait-for-store");
assert.deepEqual(observed.recoveryState, consumedRestart.state);
assert.equal(observed.openForMs, 4_000);
assert.equal(observed.openForMs, 12_000);
});
@@ -1199,7 +1199,7 @@ test("camera startup watchdog replaces an open-but-silent MSE or WebSocket", ()
);
});
test("first media starts one non-sliding first-playable-frame deadline", () => {
test("continuing media extends the first-playable-frame deadline", () => {
const scheduled = new Map();
const cancelled = [];
const timeouts = [];
@@ -1228,8 +1228,14 @@ test("first media starts one non-sliding first-playable-frame deadline", () => {
assert.equal(scheduled.get(playableHandle).timeoutMs, CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS);
watchdog.markMediaReceived();
assert.equal(nextHandle - 1, playableHandle, "later fragments must not extend the deadline");
scheduled.get(playableHandle).callback();
const extendedPlayableHandle = nextHandle - 1;
assert.notEqual(extendedPlayableHandle, playableHandle);
assert.deepEqual(cancelled, [10, playableHandle]);
assert.equal(
scheduled.get(extendedPlayableHandle).timeoutMs,
CAMERA_FIRST_PLAYABLE_FRAME_TIMEOUT_MS,
);
scheduled.get(extendedPlayableHandle).callback();
assert.deepEqual(timeouts, ["first-playable-frame"]);
assert.match(
cameraStartupWatchdogRecoveryMessage("first-playable-frame"),
@@ -1306,6 +1312,7 @@ test("a laptop sleep gap reopens only the exact authoritative live camera transp
now: 7_000,
documentVisible: true,
networkOnline: true,
transportHealthy: false,
});
assert.equal(wake.reopen, true);
assert.equal(wake.state.lastReopenAt, 7_000);
@@ -1369,15 +1376,16 @@ test("visibility, pageshow and online wake burst owns one replacement decoder",
assert.equal(secondWake.state.lastReopenAt, 18_000);
});
test("healthy visible camera heartbeats do not churn its WebSocket or decoder", () => {
test("healthy camera survives a long main-thread heartbeat gap without decoder churn", () => {
const authority = "camera-authority-acquisition-healthy";
let state = initialCameraPlaybackRecoveryState(authority, 1_000);
for (const now of [2_000, 3_000, 4_000, 5_000]) {
for (const now of [2_000, 3_000, 4_000, 12_000, 25_000]) {
const heartbeat = reduceCameraPlaybackRecovery(state, { type: "heartbeat" }, {
activeAuthorityIdentity: authority,
now,
documentVisible: true,
networkOnline: true,
transportHealthy: true,
});
assert.equal(heartbeat.reopen, false);
state = heartbeat.state;
@@ -295,6 +295,25 @@ test("raw replay exercises the same streaming receiver lifecycle as a live scan"
);
});
test("pending K1 STOP keeps the live Rerun source mounted until local capture ends", async () => {
const source = await readFile(
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
"utf8",
);
assert.match(
source,
/const intentionalSourceEnd = !recordedSource &&\s*state\?\.sourceMode === "idle" && \[/,
);
assert.match(
source,
/"awaiting_external_stop",\s*"stopping",\s*"finalizing",\s*"completed",/,
);
assert.doesNotMatch(
source,
/const intentionalSourceEnd = !recordedSource && \[\s*"awaiting_external_stop"/,
);
});
test("the spatial header reports raw replay as an active source", async () => {
const source = await readFile(
new URL("../src/App.tsx", import.meta.url),
@@ -16,6 +16,7 @@ let WorkspaceLayoutContractError;
let admitLiveDefaultPresentations;
let automaticLivePresentationIdentity;
let livePresentationCloseFence;
let restoredLayoutMayAdmitLiveDefault;
let observationPresentationSourceAfterLayoutApply;
let visibleSourceIdsAfterRecordedCatalogActivation;
@@ -39,6 +40,7 @@ before(async () => {
admitLiveDefaultPresentations,
automaticLivePresentationIdentity,
livePresentationCloseFence,
restoredLayoutMayAdmitLiveDefault,
observationPresentationSourceAfterLayoutApply,
visibleSourceIdsAfterRecordedCatalogActivation,
} = await server.ssrLoadModule("/src/core/observation/useObservationLayout.ts"));
@@ -308,6 +310,67 @@ test("a sequential live acquisition re-arms the same camera without reopening a
]);
});
test("a restored layout admits a selected delivery after reload and only same-acquisition successors", () => {
const camera = (acquisitionId, deliveryId) => ({
id: "k1:sensor.camera.right",
sourceId: "sensor.camera.right",
modality: "video",
availability: "streaming",
transport: "websocket",
previewUrl: null,
delivery: {
id: deliveryId,
kind: "mse-fmp4-websocket",
url: `/camera-preview/${deliveryId}`,
mediaType: 'video/mp4; codecs="avc1.641028"',
},
activation: {
groupId: "k1:device-session-reused:camera.preview.decoder",
maxActive: 1,
selected: true,
controllable: true,
},
binding: {
deviceId: "k1-a",
deviceSessionId: "device-session-reused",
acquisitionId,
},
capabilities: { defaultVisible: true, overlay: true },
});
const original = camera("acquisition-a", "camera-preview-2");
const successor = camera("acquisition-a", "camera-preview-3");
const unrelated = camera("acquisition-b", "camera-preview-3");
const presented = new Set([livePresentationCloseFence(original)]);
assert.equal(restoredLayoutMayAdmitLiveDefault([original], new Set()), true);
assert.equal(restoredLayoutMayAdmitLiveDefault([successor], presented), true);
assert.equal(restoredLayoutMayAdmitLiveDefault([unrelated], presented), false);
assert.equal(restoredLayoutMayAdmitLiveDefault([{
...original,
activation: { ...original.activation, selected: false },
}], new Set()), false);
const admitted = admitLiveDefaultPresentations(
[],
[successor],
new Set([automaticLivePresentationIdentity(original)]),
new Set(),
);
assert.deepEqual(admitted.visibleIds, [successor.id]);
assert.deepEqual(admitted.admittedIdentities, [
automaticLivePresentationIdentity(successor),
]);
const deliberatelyClosed = admitLiveDefaultPresentations(
[],
[successor],
new Set([automaticLivePresentationIdentity(original)]),
presented,
);
assert.deepEqual(deliberatelyClosed.visibleIds, []);
assert.deepEqual(deliberatelyClosed.admittedIdentities, []);
});
test("workspace layout API uses the canonical endpoint and optimistic revision", async () => {
const calls = [];
const current = decodeObservationWorkspaceLayoutProfile(wireProfile());