wip(k1): checkpoint connection recovery rewrite

Capture the current unreleased K1 connection, recovery, lifecycle, viewer, and test work as a single known-bad baseline for subsequent fixes.
2026-08-14 14:57:50 +03:00
parent aff331082f
commit 0ca7316a24
157 changed files with 152962 additions and 4036 deletions
@@ -4,9 +4,14 @@ import { after, before, test } from "node:test";
 import { createServer } from "vite";

 let server;
+let advanceLiveReceiverOpenWatchdog;
 let advanceLiveReceiverWatchdog;
+let initialLiveReceiverOpenWatchdogState;
 let initialLiveReceiverWatchdogState;
 let initialLiveReceiverRecoveryState;
+let liveReceiverRecoveryAuthorityIsCurrent;
+let liveReceiverRecoveryRetryDelay;
+let liveRerunRecoveryAuthorityIdentity;
 let requestLiveReceiverRecovery;

 before(async () => {
@@ -16,9 +21,14 @@ before(async () => {
    server: { middlewareMode: true },
  });
  ({
+    advanceLiveReceiverOpenWatchdog,
    advanceLiveReceiverWatchdog,
+    initialLiveReceiverOpenWatchdogState,
    initialLiveReceiverRecoveryState,
    initialLiveReceiverWatchdogState,
+    liveReceiverRecoveryAuthorityIsCurrent,
+    liveReceiverRecoveryRetryDelay,
+    liveRerunRecoveryAuthorityIdentity,
    requestLiveReceiverRecovery,
  } = await server.ssrLoadModule("/src/core/observation/liveReceiverWatchdog.ts"));
 });
@@ -92,3 +102,217 @@ test("startup failures request only three bounded viewer restarts", () => {
  assert.equal(exhausted.attempt, 3);
  assert.equal(exhausted.state.awaitingRecovery, false);
 });
+
+function livePointCloudDescriptor(overrides = {}) {
+  return {
+    id: "xgrids-k1:lixelkity-k1:sensor.lidar.primary",
+    sourceId: "sensor.lidar.primary",
+    semanticChannelId: "spatial.point-cloud.live",
+    label: "K1 point cloud",
+    description: "live",
+    modality: "point-cloud",
+    role: "primary",
+    availability: "streaming",
+    transport: "rerun-grpc",
+    endpointLabel: "Rerun gRPC",
+    previewUrl: "rerun+http://127.0.0.1:9877/proxy",
+    delivery: null,
+    activation: null,
+    presentationLease: null,
+    provider: {
+      pluginId: "xgrids-k1",
+      pluginVersion: "0.1.0",
+      modelId: "lixelkity-k1",
+      compatibilityProfileId: "xgrids.lixelkity-k1.fw-3.0.2.local-network.v2",
+    },
+    binding: {
+      deviceId: "device-k1-001",
+      deviceSessionId: "device-session-001",
+      acquisitionId: "acquisition-001",
+    },
+    capabilities: {
+      overlay: false,
+      fullscreen: true,
+      resizable: false,
+      defaultVisible: true,
+      timelineMode: "live-only",
+      seekable: false,
+      sessionRecording: false,
+      clockId: "acquisition-001",
+      spatialRegistration: "native",
+    },
+    ...overrides,
+  };
+}
+
+const liveSpatialSource = {
+  id: "acquisition-001",
+  url: "rerun+http://127.0.0.1:9877/proxy",
+  label: "Live",
+  kind: "rerun-grpc",
+};
+
+test("exact live Rerun authority gets durable retries with capped delay", () => {
+  const authority = liveRerunRecoveryAuthorityIdentity(
+    livePointCloudDescriptor(),
+    liveSpatialSource,
+  );
+  assert.ok(authority);
+  assert.equal(liveReceiverRecoveryAuthorityIsCurrent(authority, authority), true);
+
+  let state = initialLiveReceiverRecoveryState();
+  const delays = [];
+  for (let attempt = 1; attempt <= 8; attempt += 1) {
+    const recovery = requestLiveReceiverRecovery(state, {
+      activeAuthorityIdentity: authority,
+      expectedAuthorityIdentity: authority,
+    });
+    assert.equal(recovery.signal, "retry");
+    assert.equal(recovery.attempt, attempt);
+    delays.push(recovery.delayMs);
+    state = recovery.state;
+  }
+  assert.deepEqual(delays, [400, 1_000, 2_000, 5_000, 5_000, 5_000, 5_000, 5_000]);
+  assert.equal(state.attempts, 8);
+  assert.equal(liveReceiverRecoveryRetryDelay(100), 5_000);
+});
+
+test("Rerun durable retry fails closed when exact authority is replaced", () => {
+  const authority = liveRerunRecoveryAuthorityIdentity(
+    livePointCloudDescriptor(),
+    liveSpatialSource,
+  );
+  assert.ok(authority);
+  const replacement = liveRerunRecoveryAuthorityIdentity(
+    livePointCloudDescriptor({
+      binding: {
+        deviceId: "device-k1-001",
+        deviceSessionId: "device-session-002",
+        acquisitionId: "acquisition-002",
+      },
+      capabilities: {
+        ...livePointCloudDescriptor().capabilities,
+        clockId: "acquisition-002",
+      },
+    }),
+    { ...liveSpatialSource, id: "acquisition-002" },
+  );
+  assert.ok(replacement);
+
+  const stale = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState(), {
+    activeAuthorityIdentity: replacement,
+    expectedAuthorityIdentity: authority,
+  });
+  assert.equal(stale.signal, "stale");
+  assert.equal(stale.delayMs, null);
+  assert.deepEqual(stale.state, initialLiveReceiverRecoveryState());
+});
+
+test("connecting Rerun authority requires an exact recovery generation lease", () => {
+  const recoveryLease = {
+    kind: "active-stream-recovery",
+    runtimeId: "runtime-recovery-001",
+    acquisitionId: "acquisition-001",
+    acquisitionStateRevision: 4,
+    producerGeneration: 17,
+    recoveryGeneration: 6,
+  };
+  const recoveredAuthority = liveRerunRecoveryAuthorityIdentity(
+    livePointCloudDescriptor({
+      availability: "connecting",
+      presentationLease: recoveryLease,
+    }),
+    liveSpatialSource,
+  );
+  assert.ok(recoveredAuthority);
+  assert.equal(
+    liveRerunRecoveryAuthorityIdentity(
+      livePointCloudDescriptor({ availability: "connecting" }),
+      liveSpatialSource,
+    ),
+    null,
+  );
+  assert.equal(
+    liveRerunRecoveryAuthorityIdentity(
+      livePointCloudDescriptor({
+        availability: "connecting",
+        presentationLease: { ...recoveryLease, producerGeneration: 0 },
+      }),
+      liveSpatialSource,
+    ),
+    null,
+  );
+});
+
+test("opening receiver gets bounded rolling patience while backend publication advances", () => {
+  let openState = initialLiveReceiverOpenWatchdogState(0, 0);
+  let recoveryState = initialLiveReceiverRecoveryState();
+
+  const samples = [
+    [134, 3_999, "wait-for-store"],
+    [266, 4_000, "refresh-receiver"],
+  ];
+  for (const [backendActivitySequence, nowMs, expectedSignal] of samples) {
+    const observed = advanceLiveReceiverOpenWatchdog(
+      openState,
+      recoveryState,
+      backendActivitySequence,
+      nowMs,
+    );
+    assert.equal(observed.signal, expectedSignal);
+    assert.equal(observed.state.lastBackendActivitySequence, backendActivitySequence);
+    assert.deepEqual(observed.recoveryState, {
+      attempts: 0,
+      awaitingRecovery: false,
+    });
+    openState = observed.state;
+    recoveryState = observed.recoveryState;
+  }
+});
+
+test("unchanged opening sequence delegates to bounded receiver restart", () => {
+  const openState = initialLiveReceiverOpenWatchdogState(486);
+  const recoveryState = initialLiveReceiverRecoveryState();
+  const unchanged = advanceLiveReceiverOpenWatchdog(
+    openState,
+    recoveryState,
+    486,
+  );
+
+  assert.equal(unchanged.signal, "restart-receiver");
+  const restart = requestLiveReceiverRecovery(unchanged.recoveryState);
+  assert.equal(restart.signal, "retry");
+  assert.equal(restart.attempt, 1);
+});
+
+test("fresh backend progress preserves earlier restart debt until viewer admission", () => {
+  const openState = initialLiveReceiverOpenWatchdogState(486, 0);
+  const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
+  const observed = advanceLiveReceiverOpenWatchdog(
+    openState,
+    consumedRestart.state,
+    600,
+    3_999,
+  );
+
+  assert.equal(observed.signal, "wait-for-store");
+  assert.deepEqual(observed.recoveryState, {
+    attempts: 1,
+    awaitingRecovery: true,
+  });
+});
+
+test("aged active receiver refresh does not spend or erase restart debt", () => {
+  const openState = initialLiveReceiverOpenWatchdogState(486, 0);
+  const consumedRestart = requestLiveReceiverRecovery(initialLiveReceiverRecoveryState());
+  const observed = advanceLiveReceiverOpenWatchdog(
+    openState,
+    consumedRestart.state,
+    900,
+    4_000,
+  );
+
+  assert.equal(observed.signal, "refresh-receiver");
+  assert.deepEqual(observed.recoveryState, consumedRestart.state);
+  assert.equal(observed.openForMs, 4_000);
+});