import assert from "node:assert/strict"; import { after, before, test } from "node:test"; import { createServer } from "vite"; let server; let advanceLiveReceiverOpenWatchdog; let advanceLiveReceiverWatchdog; let initialLiveReceiverOpenWatchdogState; let initialLiveReceiverWatchdogState; let initialLiveReceiverRecoveryState; let liveReceiverRecoveryAuthorityIsCurrent; let liveReceiverRecoveryRetryDelay; let liveRerunRecoveryAuthorityIdentity; let requestLiveReceiverRecovery; before(async () => { server = await createServer({ appType: "custom", logLevel: "silent", server: { middlewareMode: true }, }); ({ advanceLiveReceiverOpenWatchdog, advanceLiveReceiverWatchdog, initialLiveReceiverOpenWatchdogState, initialLiveReceiverRecoveryState, initialLiveReceiverWatchdogState, liveReceiverRecoveryAuthorityIsCurrent, liveReceiverRecoveryRetryDelay, liveRerunRecoveryAuthorityIdentity, requestLiveReceiverRecovery, } = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts")); }); after(async () => { await server?.close(); }); test("watchdog ignores healthy live progress", () => { let state = initialLiveReceiverWatchdogState(); for (let index = 0; index < 20; index += 1) { const observed = advanceLiveReceiverWatchdog(state, { nowMs: index * 500, backendActivitySequence: index * 2, viewerRangeMaxNs: index * 1_000_000_000, }); state = observed.state; assert.notEqual(observed.signal, "stalled"); } }); test("watchdog detects a frozen receiver while backend publication advances", () => { let state = initialLiveReceiverWatchdogState(); let signal = "none"; let stalledForMs = 0; for (let index = 0; index <= 13; index += 1) { const observed = advanceLiveReceiverWatchdog(state, { nowMs: index * 500, backendActivitySequence: 100 + index, viewerRangeMaxNs: 42_000_000_000, }); state = observed.state; signal = observed.signal === "stalled" ? observed.signal : signal; stalledForMs = Math.max(stalledForMs, observed.stalledForMs); } assert.equal(signal, "stalled"); assert.ok(stalledForMs >= 5_000); }); test("one backend increment at stream finalization does not create a false stall", () => { let state = initialLiveReceiverWatchdogState(); state = advanceLiveReceiverWatchdog(state, { nowMs: 0, backendActivitySequence: 10, viewerRangeMaxNs: 20_000_000_000, }).state; state = advanceLiveReceiverWatchdog(state, { nowMs: 500, backendActivitySequence: 11, viewerRangeMaxNs: 20_000_000_000, }).state; const observed = advanceLiveReceiverWatchdog(state, { nowMs: 10_000, backendActivitySequence: 11, viewerRangeMaxNs: 20_000_000_000, }); assert.equal(observed.signal, "none"); }); test("startup failures request only three bounded viewer restarts", () => { let state = initialLiveReceiverRecoveryState(); for (let attempt = 1; attempt <= 3; attempt += 1) { const recovery = requestLiveReceiverRecovery(state); assert.equal(recovery.signal, "retry"); assert.equal(recovery.attempt, attempt); assert.equal(recovery.state.awaitingRecovery, true); state = recovery.state; } const exhausted = requestLiveReceiverRecovery(state); assert.equal(exhausted.signal, "exhausted"); assert.equal(exhausted.attempt, 3); assert.equal(exhausted.state.awaitingRecovery, false); }); function livePointCloudDescriptor(overrides = {}) { return { id: "xgrids-k1:lixelkity-k1:sensor.lidar.primary", sourceId: "sensor.lidar.primary", semanticChannelId: "spatial.point-cloud.live", label: "K1 point cloud", description: "live", modality: "point-cloud", role: "primary", availability: "streaming", transport: "rerun-grpc", endpointLabel: "Rerun gRPC", previewUrl: "rerun+http://127.0.0.1:9877/proxy", delivery: null, activation: null, presentationLease: null, provider: { pluginId: "xgrids-k1", pluginVersion: "0.1.0", modelId: "lixelkity-k1", compatibilityProfileId: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2", }, binding: { deviceId: "device-k1-001", deviceSessionId: "device-session-001", acquisitionId: "acquisition-001", }, capabilities: { overlay: false, fullscreen: true, resizable: false, defaultVisible: true, timelineMode: "live-only", seekable: false, sessionRecording: false, clockId: "acquisition-001", spatialRegistration: "native", }, ...overrides, }; } const liveSpatialSource = { id: "acquisition-001", url: "rerun+http://127.0.0.1:9877/proxy", label: "Live", kind: "rerun-grpc", }; test("exact live Rerun authority gets durable retries with capped delay", () => { const authority = liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor(), liveSpatialSource, ); assert.ok(authority); assert.equal(liveReceiverRecoveryAuthorityIsCurrent(authority, authority), true); let state = initialLiveReceiverRecoveryState(); const delays = []; for (let attempt = 1; attempt <= 8; attempt += 1) { const recovery = requestLiveReceiverRecovery(state, { activeAuthorityIdentity: authority, expectedAuthorityIdentity: authority, }); assert.equal(recovery.signal, "retry"); assert.equal(recovery.attempt, attempt); delays.push(recovery.delayMs); state = recovery.state; } assert.deepEqual(delays, [400, 1_000, 2_000, 5_000, 5_000, 5_000, 5_000, 5_000]); assert.equal(state.attempts, 8); assert.equal(liveReceiverRecoveryRetryDelay(100), 5_000); }); test("Rerun durable retry fails closed when exact authority is replaced", () => { const authority = liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor(), liveSpatialSource, ); assert.ok(authority); const replacement = liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor({ binding: { deviceId: "device-k1-001", deviceSessionId: "device-session-002", acquisitionId: "acquisition-002", }, capabilities: { ...livePointCloudDescriptor().capabilities, clockId: "acquisition-002", }, }), { ...liveSpatialSource, id: "acquisition-002" }, ); assert.ok(replacement); const stale = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState(), { activeAuthorityIdentity: replacement, expectedAuthorityIdentity: authority, }); assert.equal(stale.signal, "stale"); assert.equal(stale.delayMs, null); assert.deepEqual(stale.state, initialLiveReceiverRecoveryState()); }); test("connecting Rerun authority requires an exact recovery generation lease", () => { const recoveryLease = { kind: "active-stream-recovery", runtimeId: "runtime-recovery-001", acquisitionId: "acquisition-001", acquisitionStateRevision: 4, producerGeneration: 17, recoveryGeneration: 6, }; const recoveredAuthority = liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor({ availability: "connecting", presentationLease: recoveryLease, }), liveSpatialSource, ); assert.ok(recoveredAuthority); assert.equal( liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor({ availability: "connecting" }), liveSpatialSource, ), null, ); assert.equal( liveRerunRecoveryAuthorityIdentity( livePointCloudDescriptor({ availability: "connecting", presentationLease: { ...recoveryLease, producerGeneration: 0 }, }), liveSpatialSource, ), null, ); }); test("opening receiver preserves partial store replay while backend publication advances", () => { let openState = initialLiveReceiverOpenWatchdogState(0, 0); let recoveryState = initialLiveReceiverRecoveryState(); const samples = [ [134, 3_999, "wait-for-store"], [266, 4_000, "wait-for-store"], [380, 8_000, "wait-for-store"], [486, 12_000, "wait-for-store"], [700, 120_000, "wait-for-store"], ]; for (const [backendActivitySequence, nowMs, expectedSignal] of samples) { const observed = advanceLiveReceiverOpenWatchdog( openState, recoveryState, backendActivitySequence, nowMs, ); assert.equal(observed.signal, expectedSignal); assert.equal(observed.state.lastBackendActivitySequence, backendActivitySequence); assert.deepEqual(observed.recoveryState, { attempts: 0, awaitingRecovery: false, }); openState = observed.state; recoveryState = observed.recoveryState; } }); test("unchanged opening sequence delegates to bounded receiver restart", () => { const openState = initialLiveReceiverOpenWatchdogState(486); const recoveryState = initialLiveReceiverRecoveryState(); const unchanged = advanceLiveReceiverOpenWatchdog( openState, recoveryState, 486, ); assert.equal(unchanged.signal, "restart-receiver"); const restart = requestLiveReceiverRecovery(unchanged.recoveryState); assert.equal(restart.signal, "retry"); assert.equal(restart.attempt, 1); }); test("fresh backend progress preserves earlier restart debt until viewer admission", () => { const openState = initialLiveReceiverOpenWatchdogState(486, 0); const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState()); const observed = advanceLiveReceiverOpenWatchdog( openState, consumedRestart.state, 600, 3_999, ); assert.equal(observed.signal, "wait-for-store"); assert.deepEqual(observed.recoveryState, { attempts: 1, awaitingRecovery: true, }); }); test("aged active receiver remains intact and preserves restart debt", () => { const openState = initialLiveReceiverOpenWatchdogState(486, 0); const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState()); const observed = advanceLiveReceiverOpenWatchdog( openState, consumedRestart.state, 900, 12_000, ); assert.equal(observed.signal, "wait-for-store"); assert.deepEqual(observed.recoveryState, consumedRestart.state); assert.equal(observed.openForMs, 12_000); });