337 lines
10 KiB
JavaScript
337 lines
10 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { after, before, test } from "node:test";
|
|
|
|
import { createServer } from "vite";
|
|
|
|
let server;
|
|
let advanceLiveReceiverOpenWatchdog;
|
|
let advanceLiveReceiverWatchdog;
|
|
let initialLiveReceiverOpenWatchdogState;
|
|
let initialLiveReceiverWatchdogState;
|
|
let initialLiveReceiverRecoveryState;
|
|
let liveReceiverRecoveryAuthorityIsCurrent;
|
|
let liveReceiverRecoveryRetryDelay;
|
|
let liveRerunRecoveryAuthorityIdentity;
|
|
let requestLiveReceiverRecovery;
|
|
|
|
before(async () => {
|
|
server = await createServer({
|
|
appType: "custom",
|
|
logLevel: "silent",
|
|
server: { middlewareMode: true },
|
|
});
|
|
({
|
|
advanceLiveReceiverOpenWatchdog,
|
|
advanceLiveReceiverWatchdog,
|
|
initialLiveReceiverOpenWatchdogState,
|
|
initialLiveReceiverRecoveryState,
|
|
initialLiveReceiverWatchdogState,
|
|
liveReceiverRecoveryAuthorityIsCurrent,
|
|
liveReceiverRecoveryRetryDelay,
|
|
liveRerunRecoveryAuthorityIdentity,
|
|
requestLiveReceiverRecovery,
|
|
} = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts"));
|
|
});
|
|
|
|
after(async () => {
|
|
await server?.close();
|
|
});
|
|
|
|
test("watchdog ignores healthy live progress", () => {
|
|
let state = initialLiveReceiverWatchdogState();
|
|
for (let index = 0; index < 20; index += 1) {
|
|
const observed = advanceLiveReceiverWatchdog(state, {
|
|
nowMs: index * 500,
|
|
backendActivitySequence: index * 2,
|
|
viewerRangeMaxNs: index * 1_000_000_000,
|
|
});
|
|
state = observed.state;
|
|
assert.notEqual(observed.signal, "stalled");
|
|
}
|
|
});
|
|
|
|
test("watchdog detects a frozen receiver while backend publication advances", () => {
|
|
let state = initialLiveReceiverWatchdogState();
|
|
let signal = "none";
|
|
let stalledForMs = 0;
|
|
for (let index = 0; index <= 13; index += 1) {
|
|
const observed = advanceLiveReceiverWatchdog(state, {
|
|
nowMs: index * 500,
|
|
backendActivitySequence: 100 + index,
|
|
viewerRangeMaxNs: 42_000_000_000,
|
|
});
|
|
state = observed.state;
|
|
signal = observed.signal === "stalled" ? observed.signal : signal;
|
|
stalledForMs = Math.max(stalledForMs, observed.stalledForMs);
|
|
}
|
|
assert.equal(signal, "stalled");
|
|
assert.ok(stalledForMs >= 5_000);
|
|
});
|
|
|
|
test("one backend increment at stream finalization does not create a false stall", () => {
|
|
let state = initialLiveReceiverWatchdogState();
|
|
state = advanceLiveReceiverWatchdog(state, {
|
|
nowMs: 0,
|
|
backendActivitySequence: 10,
|
|
viewerRangeMaxNs: 20_000_000_000,
|
|
}).state;
|
|
state = advanceLiveReceiverWatchdog(state, {
|
|
nowMs: 500,
|
|
backendActivitySequence: 11,
|
|
viewerRangeMaxNs: 20_000_000_000,
|
|
}).state;
|
|
const observed = advanceLiveReceiverWatchdog(state, {
|
|
nowMs: 10_000,
|
|
backendActivitySequence: 11,
|
|
viewerRangeMaxNs: 20_000_000_000,
|
|
});
|
|
assert.equal(observed.signal, "none");
|
|
});
|
|
|
|
test("startup failures request only three bounded viewer restarts", () => {
|
|
let state = initialLiveReceiverRecoveryState();
|
|
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
|
const recovery = requestLiveReceiverRecovery(state);
|
|
assert.equal(recovery.signal, "retry");
|
|
assert.equal(recovery.attempt, attempt);
|
|
assert.equal(recovery.state.awaitingRecovery, true);
|
|
state = recovery.state;
|
|
}
|
|
const exhausted = requestLiveReceiverRecovery(state);
|
|
assert.equal(exhausted.signal, "exhausted");
|
|
assert.equal(exhausted.attempt, 3);
|
|
assert.equal(exhausted.state.awaitingRecovery, false);
|
|
});
|
|
|
|
function livePointCloudDescriptor(overrides = {}) {
|
|
return {
|
|
id: "xgrids-k1:lixelkity-k1:sensor.lidar.primary",
|
|
sourceId: "sensor.lidar.primary",
|
|
semanticChannelId: "spatial.point-cloud.live",
|
|
label: "K1 point cloud",
|
|
description: "live",
|
|
modality: "point-cloud",
|
|
role: "primary",
|
|
availability: "streaming",
|
|
transport: "rerun-grpc",
|
|
endpointLabel: "Rerun gRPC",
|
|
previewUrl: "rerun+http://127.0.0.1:9877/proxy",
|
|
delivery: null,
|
|
activation: null,
|
|
presentationLease: null,
|
|
provider: {
|
|
pluginId: "xgrids-k1",
|
|
pluginVersion: "0.1.0",
|
|
modelId: "lixelkity-k1",
|
|
compatibilityProfileId: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2",
|
|
},
|
|
binding: {
|
|
deviceId: "device-k1-001",
|
|
deviceSessionId: "device-session-001",
|
|
acquisitionId: "acquisition-001",
|
|
},
|
|
capabilities: {
|
|
overlay: false,
|
|
fullscreen: true,
|
|
resizable: false,
|
|
defaultVisible: true,
|
|
timelineMode: "live-only",
|
|
seekable: false,
|
|
sessionRecording: false,
|
|
clockId: "acquisition-001",
|
|
spatialRegistration: "native",
|
|
},
|
|
...overrides,
|
|
};
|
|
}
|
|
|
|
const liveSpatialSource = {
|
|
id: "acquisition-001",
|
|
url: "rerun+http://127.0.0.1:9877/proxy",
|
|
label: "Live",
|
|
kind: "rerun-grpc",
|
|
};
|
|
|
|
test("exact live Rerun authority gets durable retries with capped delay", () => {
|
|
const authority = liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor(),
|
|
liveSpatialSource,
|
|
);
|
|
assert.ok(authority);
|
|
assert.equal(liveReceiverRecoveryAuthorityIsCurrent(authority, authority), true);
|
|
|
|
let state = initialLiveReceiverRecoveryState();
|
|
const delays = [];
|
|
for (let attempt = 1; attempt <= 8; attempt += 1) {
|
|
const recovery = requestLiveReceiverRecovery(state, {
|
|
activeAuthorityIdentity: authority,
|
|
expectedAuthorityIdentity: authority,
|
|
});
|
|
assert.equal(recovery.signal, "retry");
|
|
assert.equal(recovery.attempt, attempt);
|
|
delays.push(recovery.delayMs);
|
|
state = recovery.state;
|
|
}
|
|
assert.deepEqual(delays, [400, 1_000, 2_000, 5_000, 5_000, 5_000, 5_000, 5_000]);
|
|
assert.equal(state.attempts, 8);
|
|
assert.equal(liveReceiverRecoveryRetryDelay(100), 5_000);
|
|
});
|
|
|
|
test("Rerun durable retry fails closed when exact authority is replaced", () => {
|
|
const authority = liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor(),
|
|
liveSpatialSource,
|
|
);
|
|
assert.ok(authority);
|
|
const replacement = liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor({
|
|
binding: {
|
|
deviceId: "device-k1-001",
|
|
deviceSessionId: "device-session-002",
|
|
acquisitionId: "acquisition-002",
|
|
},
|
|
capabilities: {
|
|
...livePointCloudDescriptor().capabilities,
|
|
clockId: "acquisition-002",
|
|
},
|
|
}),
|
|
{ ...liveSpatialSource, id: "acquisition-002" },
|
|
);
|
|
assert.ok(replacement);
|
|
|
|
const stale = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState(), {
|
|
activeAuthorityIdentity: replacement,
|
|
expectedAuthorityIdentity: authority,
|
|
});
|
|
assert.equal(stale.signal, "stale");
|
|
assert.equal(stale.delayMs, null);
|
|
assert.deepEqual(stale.state, initialLiveReceiverRecoveryState());
|
|
});
|
|
|
|
test("connecting Rerun authority requires an exact recovery generation lease", () => {
|
|
const recoveryLease = {
|
|
kind: "active-stream-recovery",
|
|
runtimeId: "runtime-recovery-001",
|
|
acquisitionId: "acquisition-001",
|
|
acquisitionStateRevision: 4,
|
|
producerGeneration: 17,
|
|
recoveryGeneration: 6,
|
|
};
|
|
const recoveredAuthority = liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor({
|
|
availability: "connecting",
|
|
presentationLease: recoveryLease,
|
|
}),
|
|
liveSpatialSource,
|
|
);
|
|
assert.ok(recoveredAuthority);
|
|
assert.equal(
|
|
liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor({ availability: "connecting" }),
|
|
liveSpatialSource,
|
|
),
|
|
null,
|
|
);
|
|
assert.equal(
|
|
liveRerunRecoveryAuthorityIdentity(
|
|
livePointCloudDescriptor({
|
|
availability: "connecting",
|
|
presentationLease: { ...recoveryLease, producerGeneration: 0 },
|
|
}),
|
|
liveSpatialSource,
|
|
),
|
|
null,
|
|
);
|
|
});
|
|
|
|
test("opening receiver preserves partial store replay while backend publication advances", () => {
|
|
let openState = initialLiveReceiverOpenWatchdogState(0, 0);
|
|
let recoveryState = initialLiveReceiverRecoveryState();
|
|
|
|
const samples = [
|
|
[134, 3_999, "wait-for-store"],
|
|
[266, 4_000, "wait-for-store"],
|
|
[380, 8_000, "wait-for-store"],
|
|
[486, 12_000, "wait-for-store"],
|
|
[700, 120_000, "wait-for-store"],
|
|
];
|
|
for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
|
|
const observed = advanceLiveReceiverOpenWatchdog(
|
|
openState,
|
|
recoveryState,
|
|
backendActivitySequence,
|
|
nowMs,
|
|
);
|
|
assert.equal(observed.signal, expectedSignal);
|
|
assert.equal(observed.state.lastBackendActivitySequence, backendActivitySequence);
|
|
assert.deepEqual(observed.recoveryState, {
|
|
attempts: 0,
|
|
awaitingRecovery: false,
|
|
});
|
|
openState = observed.state;
|
|
recoveryState = observed.recoveryState;
|
|
}
|
|
});
|
|
|
|
test("new receiver grants its first confirmed backend publication a full open window", () => {
|
|
const openState = initialLiveReceiverOpenWatchdogState(null, 0);
|
|
const recoveryState = initialLiveReceiverRecoveryState();
|
|
const firstObservedPublication = advanceLiveReceiverOpenWatchdog(
|
|
openState,
|
|
recoveryState,
|
|
1,
|
|
4_000,
|
|
);
|
|
|
|
assert.equal(firstObservedPublication.signal, "wait-for-store");
|
|
assert.equal(firstObservedPublication.state.lastBackendActivitySequence, 1);
|
|
assert.deepEqual(firstObservedPublication.recoveryState, recoveryState);
|
|
});
|
|
|
|
test("unchanged opening sequence delegates to bounded receiver restart", () => {
|
|
const openState = initialLiveReceiverOpenWatchdogState(486);
|
|
const recoveryState = initialLiveReceiverRecoveryState();
|
|
const unchanged = advanceLiveReceiverOpenWatchdog(
|
|
openState,
|
|
recoveryState,
|
|
486,
|
|
);
|
|
|
|
assert.equal(unchanged.signal, "restart-receiver");
|
|
const restart = requestLiveReceiverRecovery(unchanged.recoveryState);
|
|
assert.equal(restart.signal, "retry");
|
|
assert.equal(restart.attempt, 1);
|
|
});
|
|
|
|
test("fresh backend progress preserves earlier restart debt until viewer admission", () => {
|
|
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
|
|
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
|
|
const observed = advanceLiveReceiverOpenWatchdog(
|
|
openState,
|
|
consumedRestart.state,
|
|
600,
|
|
3_999,
|
|
);
|
|
|
|
assert.equal(observed.signal, "wait-for-store");
|
|
assert.deepEqual(observed.recoveryState, {
|
|
attempts: 1,
|
|
awaitingRecovery: true,
|
|
});
|
|
});
|
|
|
|
test("aged active receiver remains intact and preserves restart debt", () => {
|
|
const openState = initialLiveReceiverOpenWatchdogState(486, 0);
|
|
const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
|
|
const observed = advanceLiveReceiverOpenWatchdog(
|
|
openState,
|
|
consumedRestart.state,
|
|
900,
|
|
12_000,
|
|
);
|
|
|
|
assert.equal(observed.signal, "wait-for-store");
|
|
assert.deepEqual(observed.recoveryState, consumedRestart.state);
|
|
assert.equal(observed.openForMs, 12_000);
|
|
});
|