fix(k1): stabilize repeated acquisition and live viewer recovery

apps/control-station/test/liveReceiverWatchdog.test.mjs

@@ -0,0 +1,94 @@
+import assert from "node:assert/strict";
+import { after, before, test } from "node:test";
+
+import { createServer } from "vite";
+
+let server;
+let advanceLiveReceiverWatchdog;
+let initialLiveReceiverWatchdogState;
+let initialLiveReceiverRecoveryState;
+let requestLiveReceiverRecovery;
+
+before(async () => {
+  server = await createServer({
+    appType: "custom",
+    logLevel: "silent",
+    server: { middlewareMode: true },
+  });
+  ({
+    advanceLiveReceiverWatchdog,
+    initialLiveReceiverRecoveryState,
+    initialLiveReceiverWatchdogState,
+    requestLiveReceiverRecovery,
+  } = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts"));
+});
+
+after(async () => {
+  await server?.close();
+});
+
+test("watchdog ignores healthy live progress", () => {
+  let state = initialLiveReceiverWatchdogState();
+  for (let index = 0; index < 20; index += 1) {
+    const observed = advanceLiveReceiverWatchdog(state, {
+      nowMs: index * 500,
+      backendActivitySequence: index * 2,
+      viewerRangeMaxNs: index * 1_000_000_000,
+    });
+    state = observed.state;
+    assert.notEqual(observed.signal, "stalled");
+  }
+});
+
+test("watchdog detects a frozen receiver while backend publication advances", () => {
+  let state = initialLiveReceiverWatchdogState();
+  let signal = "none";
+  let stalledForMs = 0;
+  for (let index = 0; index <= 13; index += 1) {
+    const observed = advanceLiveReceiverWatchdog(state, {
+      nowMs: index * 500,
+      backendActivitySequence: 100 + index,
+      viewerRangeMaxNs: 42_000_000_000,
+    });
+    state = observed.state;
+    signal = observed.signal === "stalled" ? observed.signal : signal;
+    stalledForMs = Math.max(stalledForMs, observed.stalledForMs);
+  }
+  assert.equal(signal, "stalled");
+  assert.ok(stalledForMs >= 5_000);
+});
+
+test("one backend increment at stream finalization does not create a false stall", () => {
+  let state = initialLiveReceiverWatchdogState();
+  state = advanceLiveReceiverWatchdog(state, {
+    nowMs: 0,
+    backendActivitySequence: 10,
+    viewerRangeMaxNs: 20_000_000_000,
+  }).state;
+  state = advanceLiveReceiverWatchdog(state, {
+    nowMs: 500,
+    backendActivitySequence: 11,
+    viewerRangeMaxNs: 20_000_000_000,
+  }).state;
+  const observed = advanceLiveReceiverWatchdog(state, {
+    nowMs: 10_000,
+    backendActivitySequence: 11,
+    viewerRangeMaxNs: 20_000_000_000,
+  });
+  assert.equal(observed.signal, "none");
+});
+
+test("startup failures request only three bounded viewer restarts", () => {
+  let state = initialLiveReceiverRecoveryState();
+  for (let attempt = 1; attempt <= 3; attempt += 1) {
+    const recovery = requestLiveReceiverRecovery(state);
+    assert.equal(recovery.signal, "retry");
+    assert.equal(recovery.attempt, attempt);
+    assert.equal(recovery.state.awaitingRecovery, true);
+    state = recovery.state;
+  }
+  const exhausted = requestLiveReceiverRecovery(state);
+  assert.equal(exhausted.signal, "exhausted");
+  assert.equal(exhausted.attempt, 3);
+  assert.equal(exhausted.state.awaitingRecovery, false);
+});