SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix(k1): harden live handoff and camera recovery

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-22 13:09:24 +03:00
parent eaad9deda1
commit 85035fa07b
26 changed files with 1478 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/App.tsx
+4 -10
View File
@@ -50,6 +50,7 @@ import type {
} from "./core/observation/sessionArchive";
import { useRecordedSessionAdmission } from "./core/observation/useRecordedSessionAdmission";
import { useWorkspaceLayoutProfile } from "./core/observation/useWorkspaceLayoutProfile";
import { viewerSettingsTargetIdentity } from "./core/observation/viewerSettingsTarget";
import { resolvePolygonRunRoute } from "./core/polygon/runArchive";
import {
OBSERVATION_WORKSPACE_ID,
@@ -302,14 +303,7 @@ export default function App() {
},
};
}, [recordedReplay, replayPresented, replaySources, runtime.state]);
const viewerSettingsTargetIdentity = [
runtime.state?.activeDevice?.pluginId,
runtime.state?.activeDevice?.modelId,
runtime.state?.activeDevice?.instanceId,
runtime.state?.deviceSession?.sessionId,
runtime.state?.deviceSession?.deviceId,
runtime.state?.acquisition?.acquisitionId,
].filter(Boolean).join(":") || "local-runtime";
const viewerSettingsTarget = viewerSettingsTargetIdentity(runtime.state);
const observationLayout = useObservationLayout(
activeObservationSources,
replayPresented ? undefined : runtime.setObservationSourceActive,
@@ -389,7 +383,7 @@ export default function App() {
useEffect(() => {
const profile = workspaceLayoutProfile.profile;
const applicationKey = profile
? `${profile.revision}:${viewerSettingsTargetIdentity}`
? `${profile.revision}:${viewerSettingsTarget}`
: null;
if (
!profile ||
@@ -410,7 +404,7 @@ export default function App() {
}, [
runtime.backendStatus,
runtime.updateViewerSettings,
viewerSettingsTargetIdentity,
viewerSettingsTarget,
workspaceLayoutProfile.profile,
]);
apps/control-station/src/components/MseFmp4WebSocketPlayer.tsx
+119 -13
View File
@@ -9,7 +9,14 @@ import {
reduceCameraPlaybackRecovery,
type CameraPlaybackRecoveryEvent,
} from "../core/observation/liveCameraRecovery";
import { subscribeToLiveViewerBuildFence } from "../core/observation/liveViewerDiagnostics";
import {
createLiveViewerDiagnosticLifecycle,
createLiveViewerInstanceId,
createLiveViewerLineage,
subscribeToLiveViewerBuildFence,
type LiveViewerDiagnosticLifecycle,
type LiveViewerFailureStage,
} from "../core/observation/liveViewerDiagnostics";
type PlayerStatus = "connecting" | "buffering" | "playing" | "error";
@@ -168,6 +175,15 @@ export function cameraTransportCloseRecoveryMessage(code: number): string {
return "Browser-preview прерван; восстанавливаем текущую камеру.";
}
export function cameraPlaybackRecoveryFailureStage(
event: CameraPlaybackRecoveryEvent,
): LiveViewerFailureStage {
if (event.type === "heartbeat") return "camera-heartbeat-reopen";
if (event.type === "network-online") return "camera-network-online-reopen";
if (event.type === "page-restore") return "camera-page-restore-reopen";
return "camera-visibility-reopen";
}
function websocketUrl(path: string): string {
const url = new URL(path, window.location.href);
if (url.protocol === "http:") url.protocol = "ws:";
@@ -195,6 +211,9 @@ export function MseFmp4WebSocketPlayer({
const transportEpochRef = useRef(0);
const transportAuthorityRef = useRef(recoveryAuthorityIdentity);
const activeTransportDisposeRef = useRef<(() => void) | null>(null);
const activeDiagnosticRef = useRef<LiveViewerDiagnosticLifecycle | null>(null);
const diagnosticInstanceIdRef = useRef<string | null>(null);
const diagnosticGenerationRef = useRef(0);
const uiBuildStaleRef = useRef(false);
const [attempt, setAttempt] = useState(0);
const [uiBuildStale, setUiBuildStale] = useState(false);
@@ -232,6 +251,16 @@ export function MseFmp4WebSocketPlayer({
let disposed = false;
const transportEpoch = transportEpochRef.current + 1;
transportEpochRef.current = transportEpoch;
diagnosticInstanceIdRef.current ??= createLiveViewerInstanceId();
diagnosticGenerationRef.current += 1;
const diagnosticLifecycle = createLiveViewerDiagnosticLifecycle({
lineage: createLiveViewerLineage(
diagnosticInstanceIdRef.current,
diagnosticGenerationRef.current,
),
});
activeDiagnosticRef.current = diagnosticLifecycle;
diagnosticLifecycle.verifyBuild();
const transportIsCurrent = () => cameraTransportCallbackIsCurrent(
transportEpochRef.current,
transportEpoch,
@@ -245,6 +274,7 @@ export function MseFmp4WebSocketPlayer({
let retryTimer: number | undefined;
let receivedMedia = false;
let failed = false;
let playingReported = false;
let startupWatchdog: CameraStartupWatchdog | null = null;
transportHealthyRef.current = false;
const recovering = Boolean(
@@ -271,7 +301,16 @@ export function MseFmp4WebSocketPlayer({
}
};
const retryTransport = (copy: string) => {
const retryTransport = (
copy: string,
failureStage: LiveViewerFailureStage,
websocketCloseCode?: number,
appendFailure?: {
cameraAppendErrorName: string;
cameraMediaSourceState: "closed" | "open" | "ended";
cameraVideoErrorCode: number | null;
},
) => {
if (!transportIsCurrent() || failed) return;
if (!cameraTransportRecoveryIsCurrent(
activeAuthorityRef.current,
@@ -286,10 +325,21 @@ export function MseFmp4WebSocketPlayer({
startupWatchdog?.clear();
failed = true;
transportHealthyRef.current = false;
queue.length = 0;
queuedBytes = 0;
const retry = consumeCameraLeaseRetry(leaseRetryRef.current, delivery.id);
leaseRetryRef.current = retry.budget;
diagnosticLifecycle.post({
eventCode: "live_camera_transport_restart_requested",
failureStage,
streamId: delivery.id,
cameraQueueBytes: queuedBytes,
cameraQueueSegments: queue.length,
cameraRetryCount: retry.budget.count,
websocketCloseCode,
transportEpoch,
...appendFailure,
});
queue.length = 0;
queuedBytes = 0;
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
setMessage(copy);
@@ -318,7 +368,12 @@ export function MseFmp4WebSocketPlayer({
schedule: (callback, timeoutMs) => window.setTimeout(callback, timeoutMs),
cancel: (handle) => window.clearTimeout(handle),
onTimeout: (stage) => {
retryTransport(cameraStartupWatchdogRecoveryMessage(stage));
retryTransport(
cameraStartupWatchdogRecoveryMessage(stage),
stage === "first-media"
? "camera-first-media-timeout"
: "camera-first-playable-timeout",
);
},
});
}
@@ -331,6 +386,16 @@ export function MseFmp4WebSocketPlayer({
recoveryPendingAuthorityRef.current = null;
setStatus("playing");
setMessage("");
if (!playingReported) {
playingReported = true;
diagnosticLifecycle.post({
eventCode: "live_camera_transport_playing",
streamId: delivery.id,
cameraQueueBytes: queuedBytes,
cameraQueueSegments: queue.length,
transportEpoch,
});
}
};
video.addEventListener("playing", onPlaying);
@@ -349,9 +414,24 @@ export function MseFmp4WebSocketPlayer({
try {
sourceBuffer.appendBuffer(chunk);
} catch (error) {
retryTransport(error instanceof DOMException && error.name === "QuotaExceededError"
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.");
const quotaExceeded = error instanceof DOMException
&& error.name === "QuotaExceededError";
retryTransport(
quotaExceeded
? "Live-буфер переполнен; восстанавливаем канал без накопленной задержки."
: "Не удалось добавить видеосегмент; восстанавливаем browser-preview.",
quotaExceeded ? "camera-append-quota" : "camera-append-error",
undefined,
{
cameraAppendErrorName: error instanceof DOMException
? error.name
: error instanceof Error
? error.name
: "UnknownError",
cameraMediaSourceState: mediaSource.readyState,
cameraVideoErrorCode: video.error?.code ?? null,
},
);
}
};
@@ -361,7 +441,10 @@ export function MseFmp4WebSocketPlayer({
// Never drop arbitrary fMP4 fragments: the following samples may depend
// on them. A bounded reset is safer and keeps live latency deterministic.
if (!cameraPendingQueueCanAccept(queuedBytes, queue.length, chunk.byteLength)) {
retryTransport("Видеодекодер отстал от эфира; восстанавливаем live-буфер.");
retryTransport(
"Видеодекодер отстал от эфира; восстанавливаем live-буфер.",
"camera-queue-capacity",
);
return;
}
queue.push(chunk);
@@ -427,7 +510,10 @@ export function MseFmp4WebSocketPlayer({
appendNext();
};
onBufferError = () => {
retryTransport("MSE сбросил видеосегмент; восстанавливаем decoder.");
retryTransport(
"MSE сбросил видеосегмент; восстанавливаем decoder.",
"camera-source-buffer-error",
);
};
sourceBuffer.addEventListener("updateend", onBufferUpdateEnd);
sourceBuffer.addEventListener("error", onBufferError);
@@ -453,16 +539,26 @@ export function MseFmp4WebSocketPlayer({
enqueue(event.data);
} else if (event.data instanceof Blob) {
void event.data.arrayBuffer().then(enqueue).catch(() => {
retryTransport("Получен повреждённый видеосегмент; восстанавливаем канал.");
retryTransport(
"Получен повреждённый видеосегмент; восстанавливаем канал.",
"camera-fragment-decode",
);
});
}
});
socket.addEventListener("error", () => {
retryTransport("Связь с локальным video adapter потеряна; переподключаемся.");
retryTransport(
"Связь с локальным video adapter потеряна; переподключаемся.",
"camera-websocket-error",
);
});
socket.addEventListener("close", (event) => {
if (!transportIsCurrent() || failed) return;
retryTransport(cameraTransportCloseRecoveryMessage(event.code));
retryTransport(
cameraTransportCloseRecoveryMessage(event.code),
"camera-websocket-close",
event.code,
);
});
};
@@ -504,6 +600,10 @@ export function MseFmp4WebSocketPlayer({
if (activeTransportDisposeRef.current === disposeTransport) {
activeTransportDisposeRef.current = null;
}
if (activeDiagnosticRef.current === diagnosticLifecycle) {
activeDiagnosticRef.current = null;
}
diagnosticLifecycle.dispose();
disposeTransport();
};
}, [attempt, recoveryAuthorityIdentity, transportIdentity, uiBuildStale]);
@@ -530,6 +630,12 @@ export function MseFmp4WebSocketPlayer({
});
recovery = decision.state;
if (!decision.reopen) return;
activeDiagnosticRef.current?.post({
eventCode: "live_camera_transport_restart_requested",
failureStage: cameraPlaybackRecoveryFailureStage(event),
streamId: delivery.id,
transportEpoch: transportEpochRef.current,
});
leaseRetryRef.current = resetCameraLeaseRetryBudget(delivery.id);
recoveryPendingAuthorityRef.current = recoveryAuthorityIdentity;
setStatus("connecting");
apps/control-station/src/components/RerunViewport.tsx
+5 -7
View File
@@ -1063,9 +1063,10 @@ export function RerunViewport({
let liveRecoveryRetryTimer: number | undefined;
let recordingOpened = false;
let recordingOpenTimedOut = false;
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
);
// A new browser receiver has not observed any backend publication yet.
// Starting from the already-consumed runtime sequence makes the first
// four-second check look stalled and destroys a healthy partial replay.
let liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null);
let viewerStartResolved = false;
let latestLiveRangeMaxNs: number | null = null;
let liveWatchdog = initialLiveReceiverWatchdogState();
@@ -1718,10 +1719,7 @@ export function RerunViewport({
if (!isRecordedSource) {
// Measure native store admission from the resolved viewer start,
// not from dynamic module import or React effect setup time.
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(
liveActivitySequenceRef.current,
Date.now(),
);
liveOpenWatchdog = initialLiveReceiverOpenWatchdogState(null, Date.now());
discoverActiveLiveRecording();
if (!recordingOpened) {
diagnosticLifecycle.armAdmissionInterval(
apps/control-station/src/core/observation/liveViewerDiagnostics.ts
+65 -28
View File
@@ -4,13 +4,28 @@ export type LiveViewerDiagnosticEventCode =
| "live_receiver_recovered"
| "live_receiver_recovery_exhausted"
| "live_receiver_active_store_admitted"
| "live_receiver_error";
| "live_receiver_error"
| "live_camera_transport_restart_requested"
| "live_camera_transport_playing";
export type LiveViewerFailureStage =
| "recording-open-timeout"
| "viewer-start"
| "module-load"
| "receiver-stalled";
| "receiver-stalled"
| "camera-first-media-timeout"
| "camera-first-playable-timeout"
| "camera-queue-capacity"
| "camera-append-quota"
| "camera-append-error"
| "camera-source-buffer-error"
| "camera-fragment-decode"
| "camera-websocket-error"
| "camera-websocket-close"
| "camera-heartbeat-reopen"
| "camera-visibility-reopen"
| "camera-network-online-reopen"
| "camera-page-restore-reopen";
export interface LiveViewerDiagnostic {
eventCode: LiveViewerDiagnosticEventCode;
@@ -20,6 +35,14 @@ export interface LiveViewerDiagnostic {
viewerRangeMaxNs?: number | null;
stalledForMs?: number | null;
recoveryAttempt?: number | null;
cameraQueueBytes?: number | null;
cameraQueueSegments?: number | null;
cameraRetryCount?: number | null;
websocketCloseCode?: number | null;
transportEpoch?: number | null;
cameraAppendErrorName?: string | null;
cameraMediaSourceState?: "closed" | "open" | "ended" | null;
cameraVideoErrorCode?: number | null;
}
export interface LiveViewerLineage {
@@ -69,7 +92,6 @@ const DEVELOPMENT_UI_BUILD_ID = "development";
const SAFE_STREAM_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const HASHED_UI_BUILD_ID = /^\/assets\/[A-Za-z0-9._/-]+-[A-Za-z0-9_-]{8,}\.js$/;
const UI_BUILD_CHECK_INTERVAL_MILLISECONDS = 15_000;
const UI_BUILD_RELOAD_DELAY_MILLISECONDS = 50;
let documentInstanceId: string | null = null;
let sharedUiBuildCoordinator: UiBuildStaleCoordinator | null = null;
@@ -172,32 +194,45 @@ export function liveViewerDiagnosticBody(
...(safeInteger(event.recoveryAttempt) === undefined
? {}
: { recovery_attempt: safeInteger(event.recoveryAttempt) }),
...(safeInteger(event.cameraQueueBytes) === undefined
? {}
: { camera_queue_bytes: safeInteger(event.cameraQueueBytes) }),
...(safeInteger(event.cameraQueueSegments) === undefined
? {}
: { camera_queue_segments: safeInteger(event.cameraQueueSegments) }),
...(safeInteger(event.cameraRetryCount) === undefined
? {}
: { camera_retry_count: safeInteger(event.cameraRetryCount) }),
...(safeInteger(event.websocketCloseCode) === undefined
? {}
: { websocket_close_code: safeInteger(event.websocketCloseCode) }),
...(safeInteger(event.transportEpoch) === undefined
? {}
: { transport_epoch: safeInteger(event.transportEpoch) }),
...(event.cameraAppendErrorName
? { camera_append_error_name: event.cameraAppendErrorName }
: {}),
...(event.cameraMediaSourceState
? { camera_media_source_state: event.cameraMediaSourceState }
: {}),
...(safeInteger(event.cameraVideoErrorCode) === undefined
? {}
: { camera_video_error_code: safeInteger(event.cameraVideoErrorCode) }),
};
}
export function createUiBuildStaleCoordinator({
scheduleReload,
reload,
}: {
scheduleReload: (callback: () => void, delayMilliseconds: number) => void;
reload: () => void;
}): UiBuildStaleCoordinator {
const listeners = new Set<(event: StaleUiBuild) => void>();
export function createUiBuildStaleCoordinator(): UiBuildStaleCoordinator {
let staleEvent: StaleUiBuild | null = null;
let reloadScheduled = false;
return {
subscribe(listener) {
listeners.add(listener);
if (staleEvent) listener(staleEvent);
return () => listeners.delete(listener);
subscribe(_listener) {
// Build drift is diagnostic-only. The loaded document and its local
// Rerun/camera transports remain authoritative until normal unmount or
// their existing bounded recovery replaces them.
return () => undefined;
},
report(event) {
if (event.loadedUiBuildId === event.expectedUiBuildId || staleEvent) return;
staleEvent = event;
for (const listener of [...listeners]) listener(event);
if (reloadScheduled) return;
reloadScheduled = true;
scheduleReload(reload, UI_BUILD_RELOAD_DELAY_MILLISECONDS);
},
stale() {
return staleEvent !== null;
@@ -206,12 +241,7 @@ export function createUiBuildStaleCoordinator({
}
function browserUiBuildCoordinator(): UiBuildStaleCoordinator {
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator({
scheduleReload: (callback, delayMilliseconds) => {
window.setTimeout(callback, delayMilliseconds);
},
reload: () => window.location.reload(),
});
sharedUiBuildCoordinator ??= createUiBuildStaleCoordinator();
return sharedUiBuildCoordinator;
}
@@ -254,10 +284,17 @@ export function verifyLiveViewerClientBuild(
lineage: Pick<LiveViewerLineage, "uiBuildId">,
signal?: AbortSignal,
): void {
if (signal?.aborted || lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID) return;
if (
signal?.aborted ||
lineage.uiBuildId === DEVELOPMENT_UI_BUILD_ID ||
browserUiBuildCoordinator().stale()
) return;
void fetch("/api/v1/viewer/client-contract", {
method: "GET",
headers: { Accept: "application/json" },
headers: {
Accept: "application/json",
"X-MissionCore-UI-Build": lineage.uiBuildId,
},
cache: "no-store",
signal,
}).then(async (response) => {
apps/control-station/src/core/observation/viewerSettingsTarget.ts
+19
View File
@@ -0,0 +1,19 @@
import type { MissionRuntimeState } from "../runtime/contracts";
export function viewerSettingsTargetIdentity(
state: MissionRuntimeState | null,
): string {
const deviceSessionId = state?.deviceSession?.sessionId?.trim();
const sessionDeviceId = state?.deviceSession?.deviceId?.trim();
if (deviceSessionId || sessionDeviceId) {
return ["device-session", deviceSessionId, sessionDeviceId]
.filter(Boolean)
.join(":");
}
return [
state?.activeDevice?.pluginId,
state?.activeDevice?.modelId,
state?.activeDevice?.instanceId,
].filter(Boolean).join(":") || "local-runtime";
}
apps/control-station/test/devicePluginContracts.test.mjs
+52
View File
@@ -20,6 +20,7 @@ let operatorIntentGeneration;
let configuration;
let compatibility;
let networkProvisionFailureMessage;
let awaitNetworkProvisionSettlementAfterLostResponse;
let discoveryScanFailureMessage;
let connectionVerificationFailureMessage;
let operationById;
@@ -99,6 +100,7 @@ before(async () => {
));
({
networkProvisionFailureMessage,
awaitNetworkProvisionSettlementAfterLostResponse,
discoveryScanFailureMessage,
connectionVerificationFailureMessage,
operationById,
@@ -1043,6 +1045,56 @@ test("K1 API uses bounded no-retry timeout classes per operation family", async
assert.doesNotMatch(apiSource, /automaticRetry\s*=\s*true/);
});
test("lost Quick response follows the admitted Apply by journal reads only", async () => {
const idempotencyKey = "network-provision:quick-lost-response";
let nowMs = 1_000;
let reads = 0;
const accepted = [];
const operation = {
operation_id: "op-quick-lost-response",
action: "network.provision",
status: "running",
idempotency_key: idempotencyKey,
deadline_at: new Date(5_000).toISOString(),
};
const initialState = {
snapshot_runtime_id: "runtime-quick-lost-response",
operations: [operation],
};
const terminalState = {
...initialState,
operations: [{
...operation,
status: "succeeded",
result: { phase: "network_applied" },
}],
};
const settled = await awaitNetworkProvisionSettlementAfterLostResponse(
initialState,
idempotencyKey,
async () => {
reads += 1;
if (reads === 1) {
throw new ApiError("transient localhost handoff", 0, true);
}
return terminalState;
},
(state) => accepted.push(state),
() => {},
{
now: () => nowMs,
wait: async (delayMs) => {
nowMs += delayMs;
},
},
);
assert.equal(reads, 2);
assert.deepEqual(accepted, [terminalState]);
assert.equal(settled.operations[0].status, "succeeded");
});
test("one explicit K1 operator action supplies the backend physical acceptance", () => {
assert.deepEqual(physicalCommandConfirmation.operatorActionPhysicalAcceptance(), {
operator_present: true,
apps/control-station/test/k1SupervisorPresentation.test.mjs
+23
View File
@@ -72,6 +72,7 @@ let transportRefEquivalenceKey;
let shouldRevealProvisioningNetworkStep;
let retiredPhysicalReopenAuthority;
let SnapshotRuntimeActionArbiter;
let runtimeActionResponseAlreadyAccepted;
let connectionActionAuthoritySnapshot;
let exactAppliedNetworkIntentCompleted;
let selectMonotonicXgridsState;
@@ -106,6 +107,7 @@ before(async () => {
SnapshotRuntimeActionArbiter,
connectionActionAuthoritySnapshot,
exactAppliedNetworkIntentCompleted,
runtimeActionResponseAlreadyAccepted,
} = await server.ssrLoadModule(
"@xgrids-k1/frontend/useXgridsK1Runtime.ts",
));
@@ -2026,6 +2028,27 @@ test("K1 runtime replacement retires A and isolates a new B action from late set
assert.equal(pendingAction, null);
});
test("successful action response may be superseded only by the same runtime revision", () => {
const response = {
snapshot_runtime_id: "runtime-a",
snapshot_revision: 41,
};
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
snapshot_runtime_id: "runtime-a",
snapshot_revision: 42,
}), true);
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
snapshot_runtime_id: "runtime-a",
snapshot_revision: 40,
}), false);
assert.equal(runtimeActionResponseAlreadyAccepted(response, {
snapshot_runtime_id: "runtime-b",
snapshot_revision: 99,
}), false);
assert.equal(runtimeActionResponseAlreadyAccepted(response, null), false);
});
test("explicit scenario reset B supersedes pending callback A in one runtime", () => {
const arbiter = new SnapshotRuntimeActionArbiter();
const actionA = arbiter.begin(12);
apps/control-station/test/liveReceiverWatchdog.test.mjs
+15
View File
@@ -273,6 +273,21 @@ test("opening receiver preserves partial store replay while backend publication
}
});
test("new receiver grants its first confirmed backend publication a full open window", () => {
const openState = initialLiveReceiverOpenWatchdogState(null, 0);
const recoveryState = initialLiveReceiverRecoveryState();
const firstObservedPublication = advanceLiveReceiverOpenWatchdog(
openState,
recoveryState,
1,
4_000,
);
assert.equal(firstObservedPublication.signal, "wait-for-store");
assert.equal(firstObservedPublication.state.lastBackendActivitySequence, 1);
assert.deepEqual(firstObservedPublication.recoveryState, recoveryState);
});
test("unchanged opening sequence delegates to bounded receiver restart", () => {
const openState = initialLiveReceiverOpenWatchdogState(486);
const recoveryState = initialLiveReceiverRecoveryState();
apps/control-station/test/liveViewerDiagnostics.test.mjs
+84 -13
View File
@@ -9,6 +9,7 @@ let createUiBuildStaleCoordinator;
let liveViewerDiagnosticBody;
let server;
let uiBuildIdFromModuleScripts;
let verifyLiveViewerClientBuild;
before(async () => {
server = await createServer({
@@ -22,6 +23,7 @@ before(async () => {
createUiBuildStaleCoordinator,
liveViewerDiagnosticBody,
uiBuildIdFromModuleScripts,
verifyLiveViewerClientBuild,
} = await server.ssrLoadModule("/src/core/observation/liveViewerDiagnostics.ts"));
});
@@ -129,9 +131,9 @@ test("two mounted viewers keep timer and diagnostic lineage isolated", () => {
assert.equal(posts[0].eventLineage.lifecycleGeneration, 7);
});
test("stale-build and unmount fence callbacks before one reload", () => {
test("stale build drift neither fences transports nor reloads the document", () => {
const clock = createFakeScheduler();
const order = [];
const sideEffects = [];
const posts = [];
const lifecycle = createLiveViewerDiagnosticLifecycle({
lineage: lineage("00000000-0000-4000-8000-000000000031"),
@@ -142,15 +144,9 @@ test("stale-build and unmount fence callbacks before one reload", () => {
lifecycle.armAdmissionTimeout(() => {
lifecycle.post({ eventCode: "live_receiver_error" });
}, 12_000);
const coordinator = createUiBuildStaleCoordinator({
scheduleReload: (callback, delay) => {
order.push(`scheduled:${delay}`);
clock.scheduler.setTimeout(callback, delay);
},
reload: () => order.push("reload"),
});
const coordinator = createUiBuildStaleCoordinator();
coordinator.subscribe(() => {
order.push("local-transports-closed");
sideEffects.push("local-transports-closed");
lifecycle.dispose();
});
@@ -165,9 +161,47 @@ test("stale-build and unmount fence callbacks before one reload", () => {
clock.advance(60_000);
lifecycle.post({ eventCode: "live_receiver_error" });
assert.deepEqual(order, ["local-transports-closed", "scheduled:50", "reload"]);
assert.deepEqual(posts, []);
assert.equal(lifecycle.active(), false);
assert.deepEqual(sideEffects, []);
assert.deepEqual(posts, [
{ eventCode: "live_receiver_error" },
{ eventCode: "live_receiver_error" },
]);
assert.equal(lifecycle.active(), true);
assert.equal(coordinator.stale(), true);
});
test("build drift is reported once with the exact loaded build", async () => {
const originalFetch = globalThis.fetch;
const calls = [];
globalThis.fetch = async (url, options) => {
calls.push({ url, options });
return new Response(JSON.stringify({
schema_version: "missioncore.live-viewer-client-contract/v1",
status: "ready",
ui_build_id: "/assets/index-ijklmnop.js",
diagnostic_schema_version: "missioncore.live-viewer-diagnostic/v2",
}), {
status: 200,
headers: { "X-MissionCore-UI-Build": "/assets/index-ijklmnop.js" },
});
};
try {
const eventLineage = lineage("00000000-0000-4000-8000-000000000032");
verifyLiveViewerClientBuild(eventLineage);
await new Promise((resolve) => setTimeout(resolve, 0));
verifyLiveViewerClientBuild(eventLineage);
await new Promise((resolve) => setTimeout(resolve, 0));
} finally {
globalThis.fetch = originalFetch;
}
assert.equal(calls.length, 1);
assert.equal(calls[0].url, "/api/v1/viewer/client-contract");
assert.equal(
calls[0].options.headers["X-MissionCore-UI-Build"],
"/assets/index-abcdefgh.js",
);
});
test("last unsubscribe fences an already queued build verification callback", () => {
@@ -213,4 +247,41 @@ test("diagnostic body and build id retain exact document/viewer/build lineage",
),
"/assets/index-dT7dN-y4.js",
);
assert.deepEqual(
liveViewerDiagnosticBody(
{
eventCode: "live_camera_transport_restart_requested",
failureStage: "camera-queue-capacity",
streamId: "camera-preview-2",
cameraQueueBytes: 12_000_000,
cameraQueueSegments: 96,
cameraRetryCount: 3,
websocketCloseCode: 4_008,
transportEpoch: 7,
cameraAppendErrorName: "InvalidStateError",
cameraMediaSourceState: "open",
cameraVideoErrorCode: 3,
},
eventLineage,
),
{
schema_version: "missioncore.live-viewer-diagnostic/v2",
event_code: "live_camera_transport_restart_requested",
ui_build_id: "/assets/index-abcdefgh.js",
document_instance_id: "00000000-0000-4000-8000-000000000001",
viewer_instance_id: "00000000-0000-4000-8000-000000000041",
lifecycle_generation: 9,
failure_stage: "camera-queue-capacity",
stream_id: "camera-preview-2",
camera_queue_bytes: 12_000_000,
camera_queue_segments: 96,
camera_retry_count: 3,
websocket_close_code: 4_008,
transport_epoch: 7,
camera_append_error_name: "InvalidStateError",
camera_media_source_state: "open",
camera_video_error_code: 3,
},
);
});
apps/control-station/test/observationSources.test.mjs
+21
View File
@@ -16,6 +16,7 @@ let cameraTransportRecoveryIsCurrent;
let cameraTransportCanOpen;
let cameraPendingQueueCanAccept;
let cameraTransportCloseRecoveryMessage;
let cameraPlaybackRecoveryFailureStage;
let createCameraStartupWatchdog;
let cameraStartupWatchdogRecoveryMessage;
let CAMERA_FIRST_MEDIA_TIMEOUT_MS;
@@ -69,6 +70,7 @@ before(async () => {
cameraTransportCanOpen,
cameraPendingQueueCanAccept,
cameraTransportCloseRecoveryMessage,
cameraPlaybackRecoveryFailureStage,
createCameraStartupWatchdog,
cameraStartupWatchdogRecoveryMessage,
CAMERA_FIRST_MEDIA_TIMEOUT_MS,
@@ -1297,6 +1299,25 @@ test("server slow-reader close is an automatic browser-only camera recovery", ()
assert.match(cameraTransportCloseRecoveryMessage(1_011), /восстанавливаем/);
});
test("camera transport recovery events keep exact diagnostic causes", () => {
assert.equal(
cameraPlaybackRecoveryFailureStage({ type: "heartbeat" }),
"camera-heartbeat-reopen",
);
assert.equal(
cameraPlaybackRecoveryFailureStage({ type: "document-visible" }),
"camera-visibility-reopen",
);
assert.equal(
cameraPlaybackRecoveryFailureStage({ type: "network-online" }),
"camera-network-online-reopen",
);
assert.equal(
cameraPlaybackRecoveryFailureStage({ type: "page-restore", persisted: true }),
"camera-page-restore-reopen",
);
});
test("a laptop sleep gap reopens only the exact authoritative live camera transport", () => {
const source = xgridsK1ObservationSources(
cameraStreamingState("sensor.camera.right"),
apps/control-station/test/rerunViewportAtomicAdmission.test.mjs
+8
View File
@@ -237,6 +237,14 @@ test("recorded RRD bytes are never split across LogChannel.send_rrd calls", asyn
source,
/if \(readyToRender && !readyPublished\)[\s\S]*eventCode: "live_receiver_active_store_admitted"[\s\S]*diagnosticLifecycle\.markAdmitted\(\)/,
);
assert.match(
source,
/liveOpenWatchdog = initialLiveReceiverOpenWatchdogState\(null, Date\.now\(\)\)/,
);
assert.doesNotMatch(
source,
/initialLiveReceiverOpenWatchdogState\(\s*liveActivitySequenceRef\.current/,
);
assert.doesNotMatch(
source,
/\], \[followLive, liveRecoveryAuthorityIdentity, liveStreamId, sourceUrl\]\);/,
apps/control-station/test/viewerSettingsTarget.test.mjs
+88
View File
@@ -0,0 +1,88 @@
import assert from "node:assert/strict";
import { after, before, test } from "node:test";
import { createServer } from "vite";
let server;
let viewerSettingsTargetIdentity;
before(async () => {
server = await createServer({
appType: "custom",
logLevel: "silent",
server: { middlewareMode: true },
});
({ viewerSettingsTargetIdentity } = await server.ssrLoadModule(
"/src/core/observation/viewerSettingsTarget.ts",
));
});
after(async () => {
await server?.close();
});
function runtimeState({ acquisitionId, sessionId = "session-a", deviceId = "device-a" }) {
return {
phase: "connected",
sourceMode: "idle",
activeDevice: {
pluginId: "xgrids-k1",
modelId: "lixel-k1",
instanceId: "instance-a",
displayName: "K1",
},
deviceSession: { sessionId, deviceId },
acquisition: acquisitionId
? {
acquisitionId,
deviceId,
deviceSessionId: sessionId,
compatibilityProfileId: "profile-a",
controlMode: "live",
state: "streaming",
stateRevision: 1,
operatorInstructions: [],
}
: null,
};
}
test("new acquisition on the same control session keeps the viewer settings target stable", () => {
const beforeStart = viewerSettingsTargetIdentity(runtimeState({}));
const firstRun = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-a" }));
const secondRun = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-b" }));
assert.equal(firstRun, beforeStart);
assert.equal(secondRun, beforeStart);
});
test("temporary loss of control authority does not reapply settings within one device session", () => {
const authoritative = runtimeState({ acquisitionId: "acquisition-a" });
const authorityUnavailable = {
...authoritative,
activeDevice: null,
phase: "starting",
};
assert.equal(
viewerSettingsTargetIdentity(authorityUnavailable),
viewerSettingsTargetIdentity(authoritative),
);
});
test("a new control session or device produces a new viewer settings target", () => {
const original = viewerSettingsTargetIdentity(runtimeState({ acquisitionId: "acquisition-a" }));
assert.notEqual(
viewerSettingsTargetIdentity(runtimeState({ sessionId: "session-b" })),
original,
);
assert.notEqual(
viewerSettingsTargetIdentity(runtimeState({ deviceId: "device-b" })),
original,
);
});
test("an unconfigured runtime uses the local target", () => {
assert.equal(viewerSettingsTargetIdentity(null), "local-runtime");
});