fix(observatory): preserve camera across follow transitions

2026-09-05 08:46:35 +03:00
parent cada687173
commit b2a1b23131
8 changed files with 268 additions and 76 deletions
@@ -146,6 +146,8 @@ export interface RerunViewportProps {
 interface RerunBlueprintChannel {
  endpointUrl: string;
  cameraContract?: string | null;
+  appliedFollowTrajectory?: boolean | null;
+  pendingFollowCameraEye?: RecordedRerunCameraEye | null;
  configureCameraJournal?: (
    eye: RecordedRerunCameraEye,
    spatialViewportStart: number,
@@ -415,7 +417,9 @@ export async function fetchRecordedBlueprintRrd(
    unifiedCameraShare = 0.46,
    planView = false,
    cameraEye,
+    eyeRelativeToTracking = false,
    currentTimeNs,
+    reactivateUpdates = false,
    onCameraMaxOrbitalRadius,
    perceptionLayers = {
      enabled: false,
@@ -436,7 +440,9 @@ export async function fetchRecordedBlueprintRrd(
    unifiedCameraShare?: number;
    planView?: boolean;
    cameraEye?: RecordedRerunCameraEye;
+    eyeRelativeToTracking?: boolean;
    currentTimeNs?: number | null;
+    reactivateUpdates?: boolean;
    onCameraMaxOrbitalRadius?: (maxOrbitalRadius: number) => void;
    perceptionLayers?: RecordedPerceptionLayers;
    fetcher?: typeof globalThis.fetch;
@@ -466,6 +472,9 @@ export async function fetchRecordedBlueprintRrd(
    unifiedCameraShare < 0.1 ||
    unifiedCameraShare > 0.9 ||
    typeof planView !== "boolean" ||
+    typeof eyeRelativeToTracking !== "boolean" ||
+    typeof reactivateUpdates !== "boolean" ||
+    (eyeRelativeToTracking && (cameraEye === undefined || currentTimeNs == null)) ||
    (cameraEye !== undefined && [
      ...cameraEye.position,
      ...cameraEye.lookTarget,
@@ -517,6 +526,7 @@ export async function fetchRecordedBlueprintRrd(
      eye_position: cameraEye?.position ?? null,
      eye_look_target: cameraEye?.lookTarget ?? null,
      eye_up: cameraEye?.eyeUp ?? null,
+      ...(eyeRelativeToTracking ? { eye_relative_to_tracking: true } : {}),
      ...(currentTimeNs === undefined || currentTimeNs === null ? {} : {
        current_time_ns: currentTimeNs,
      }),
@@ -528,6 +538,9 @@ export async function fetchRecordedBlueprintRrd(
        show_costmap: perceptionLayers.costmap,
        reactivate_updates: true,
      }),
+      ...(reactivateUpdates && perceptionLayers.costmap === undefined
+        ? { reactivate_updates: true }
+        : {}),
    }),
    signal,
  });
@@ -1615,6 +1628,8 @@ export function RerunViewport({
            blueprintChannel = {
              endpointUrl: recordedBlueprintUrl,
              cameraContract: null,
+              appliedFollowTrajectory: null,
+              pendingFollowCameraEye: null,
              configureCameraJournal: (eye, spatialViewportStart) => {
                if ("configure_camera_journal" in viewer) {
                  viewer.configure_camera_journal(eye, spatialViewportStart);
@@ -2081,7 +2096,6 @@ export function RerunViewport({
      active.endpointUrl !== recordedBlueprintUrl ||
      !active.channel.ready
    ) return;
-    const abort = new AbortController();
    const cameraContract = recordedCameraJournalContract(
      {
        activeView: recordedView,
@@ -2090,47 +2104,108 @@ export function RerunViewport({
        followTrajectory: recordedFollowTrajectory,
      },
    );
-    if (active.cameraContract !== cameraContract) {
+    const cameraContractChanged = active.cameraContract !== cameraContract;
+    if (cameraContractChanged) {
      active.configureCameraJournal?.(
        recordedPlanView ? RECORDED_RERUN_PLAN_EYE : RECORDED_RERUN_ORBITAL_EYE,
        recordedUnifiedPerception ? recordedUnifiedCameraShare : 0,
      );
-      active.cameraContract = cameraContract;
    }
-    const cameraEye = active.getCameraEye?.();
-    void fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, {
-      origin: window.location.origin,
-      blueprintSessionId: blueprintSessionIdRef.current,
-      signal: abort.signal,
-      activeView: recordedView,
-      viewResetGeneration: recordedViewResetGeneration,
-      followTrajectory: recordedFollowTrajectory,
-      perceptionLayers: recordedPerceptionLayers,
-      semanticLayer: recordedSemanticLayer,
-      unifiedPerception: recordedUnifiedPerception,
-      unifiedCameraShare: recordedUnifiedCameraShare,
-      planView: recordedPlanView,
-      cameraEye,
-      currentTimeNs: active.getCurrentTimeNs?.(),
-      onCameraMaxOrbitalRadius: (maxOrbitalRadius) => {
-        if (blueprintChannelRef.current === active) {
-          active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius);
-        }
-      },
-    }).then((payload) => {
-      if (
-        abort.signal.aborted ||
-        blueprintChannelRef.current !== active ||
-        recordedIdentityRef.current !== identity ||
-        !active.channel.ready
-      ) {
-        return;
+    const abort = new AbortController();
+    const previousFollow = active.appliedFollowTrajectory ?? false;
+    const enablingFollow = recordedFollowTrajectory && !previousFollow;
+    const disablingFollow = !recordedFollowTrajectory && previousFollow;
+    const transitionEye = (enablingFollow || disablingFollow)
+      ? active.getCameraEye?.()
+      : null;
+    if (enablingFollow && transitionEye) {
+      active.pendingFollowCameraEye = transitionEye;
+    } else if (disablingFollow) {
+      active.pendingFollowCameraEye = null;
+    }
+    const pendingFollowEye = recordedFollowTrajectory
+      ? active.pendingFollowCameraEye ?? null
+      : null;
+    const requestBlueprint = async (
+      cameraEye?: RecordedRerunCameraEye,
+      eyeRelativeToTracking = false,
+      reactivateUpdates = false,
+    ) => {
+      const currentTimeNs = active.getCurrentTimeNs?.();
+      if (eyeRelativeToTracking && currentTimeNs == null) {
+        throw new Error("Recorded tracking cursor is unavailable");
      }
+      return fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, {
+        origin: window.location.origin,
+        blueprintSessionId: blueprintSessionIdRef.current,
+        signal: abort.signal,
+        activeView: recordedView,
+        viewResetGeneration: recordedViewResetGeneration,
+        followTrajectory: recordedFollowTrajectory,
+        perceptionLayers: recordedPerceptionLayers,
+        semanticLayer: recordedSemanticLayer,
+        unifiedPerception: recordedUnifiedPerception,
+        unifiedCameraShare: recordedUnifiedCameraShare,
+        planView: recordedPlanView,
+        cameraEye,
+        eyeRelativeToTracking,
+        currentTimeNs,
+        reactivateUpdates,
+        onCameraMaxOrbitalRadius: (maxOrbitalRadius) => {
+          if (blueprintChannelRef.current === active) {
+            active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius);
+          }
+        },
+      });
+    };
+    const canApply = () => (
+      !abort.signal.aborted &&
+      blueprintChannelRef.current === active &&
+      recordedIdentityRef.current === identity &&
+      active.channel.ready
+    );
+    const applyPayload = (payload: Uint8Array) => {
+      if (!canApply()) return false;
      active.channel.send_rrd(payload);
      active.setCameraViewportStart?.(
        recordedUnifiedPerception ? recordedUnifiedCameraShare : 0,
      );
-    }).catch(() => {
+      return true;
+    };
+    void (async () => {
+      const firstEye = disablingFollow
+        ? transitionEye ?? undefined
+        : !enablingFollow && (cameraContractChanged || pendingFollowEye)
+          ? pendingFollowEye ?? active.getCameraEye?.()
+          : undefined;
+      const firstEyeIsTrackingRelative = Boolean(firstEye) && (
+        disablingFollow || recordedFollowTrajectory
+      );
+      const firstPayload = await requestBlueprint(
+        firstEye,
+        firstEyeIsTrackingRelative,
+        enablingFollow || disablingFollow || Boolean(pendingFollowEye),
+      );
+      if (!applyPayload(firstPayload)) return;
+      active.cameraContract = cameraContract;
+      active.appliedFollowTrajectory = recordedFollowTrajectory;
+      if (!enablingFollow || !transitionEye) {
+        if (pendingFollowEye && firstEye === pendingFollowEye) {
+          active.pendingFollowCameraEye = null;
+        }
+        return;
+      }
+
+      // Rerun 0.36.3 intentionally fits a newly tracked transform to its
+      // bounding box. Apply the operator's relative eye on the next native
+      // frame, after tracking is established, to retain direction and zoom.
+      await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
+      await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
+      if (!canApply()) return;
+      const stabilizedPayload = await requestBlueprint(transitionEye, true, true);
+      if (!applyPayload(stabilizedPayload)) return;
+      active.pendingFollowCameraEye = null;
+    })().catch(() => {
      // The recording remains usable with its embedded default blueprint.
      // A later settings change retries through the same small channel.
    });
@@ -486,6 +486,12 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
      activeView: "perception3d",
      viewResetGeneration: 1,
      followTrajectory: true,
+      cameraEye: {
+        position: [3, 4, 5],
+        lookTarget: [1, 2, 0],
+        eyeUp: [0, 0, 1],
+      },
+      eyeRelativeToTracking: true,
      currentTimeNs: 39_215_263_458,
      onCameraMaxOrbitalRadius: value => cameraLimits.push(value),
      unifiedCameraShare: 0.73,
@@ -530,9 +536,10 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
    unified_camera_share: 0.73,
    semantic_layer: null,
    plan_view: false,
-    eye_position: null,
-    eye_look_target: null,
-    eye_up: null,
+    eye_position: [3, 4, 5],
+    eye_look_target: [1, 2, 0],
+    eye_up: [0, 0, 1],
+    eye_relative_to_tracking: true,
    show_camera_image: true,
    show_detections_2d: true,
    show_segmentation: false,