diff --git a/apps/control-station/src/components/RerunViewport.tsx b/apps/control-station/src/components/RerunViewport.tsx index 22481ae..4857bd8 100644 --- a/apps/control-station/src/components/RerunViewport.tsx +++ b/apps/control-station/src/components/RerunViewport.tsx @@ -146,6 +146,8 @@ export interface RerunViewportProps { interface RerunBlueprintChannel { endpointUrl: string; cameraContract?: string | null; + appliedFollowTrajectory?: boolean | null; + pendingFollowCameraEye?: RecordedRerunCameraEye | null; configureCameraJournal?: ( eye: RecordedRerunCameraEye, spatialViewportStart: number, @@ -415,7 +417,9 @@ export async function fetchRecordedBlueprintRrd( unifiedCameraShare = 0.46, planView = false, cameraEye, + eyeRelativeToTracking = false, currentTimeNs, + reactivateUpdates = false, onCameraMaxOrbitalRadius, perceptionLayers = { enabled: false, @@ -436,7 +440,9 @@ export async function fetchRecordedBlueprintRrd( unifiedCameraShare?: number; planView?: boolean; cameraEye?: RecordedRerunCameraEye; + eyeRelativeToTracking?: boolean; currentTimeNs?: number | null; + reactivateUpdates?: boolean; onCameraMaxOrbitalRadius?: (maxOrbitalRadius: number) => void; perceptionLayers?: RecordedPerceptionLayers; fetcher?: typeof globalThis.fetch; @@ -466,6 +472,9 @@ export async function fetchRecordedBlueprintRrd( unifiedCameraShare < 0.1 || unifiedCameraShare > 0.9 || typeof planView !== "boolean" || + typeof eyeRelativeToTracking !== "boolean" || + typeof reactivateUpdates !== "boolean" || + (eyeRelativeToTracking && (cameraEye === undefined || currentTimeNs == null)) || (cameraEye !== undefined && [ ...cameraEye.position, ...cameraEye.lookTarget, @@ -517,6 +526,7 @@ export async function fetchRecordedBlueprintRrd( eye_position: cameraEye?.position ?? null, eye_look_target: cameraEye?.lookTarget ?? null, eye_up: cameraEye?.eyeUp ?? null, + ...(eyeRelativeToTracking ? { eye_relative_to_tracking: true } : {}), ...(currentTimeNs === undefined || currentTimeNs === null ? {} : { current_time_ns: currentTimeNs, }), @@ -528,6 +538,9 @@ export async function fetchRecordedBlueprintRrd( show_costmap: perceptionLayers.costmap, reactivate_updates: true, }), + ...(reactivateUpdates && perceptionLayers.costmap === undefined + ? { reactivate_updates: true } + : {}), }), signal, }); @@ -1615,6 +1628,8 @@ export function RerunViewport({ blueprintChannel = { endpointUrl: recordedBlueprintUrl, cameraContract: null, + appliedFollowTrajectory: null, + pendingFollowCameraEye: null, configureCameraJournal: (eye, spatialViewportStart) => { if ("configure_camera_journal" in viewer) { viewer.configure_camera_journal(eye, spatialViewportStart); @@ -2081,7 +2096,6 @@ export function RerunViewport({ active.endpointUrl !== recordedBlueprintUrl || !active.channel.ready ) return; - const abort = new AbortController(); const cameraContract = recordedCameraJournalContract( { activeView: recordedView, @@ -2090,47 +2104,108 @@ export function RerunViewport({ followTrajectory: recordedFollowTrajectory, }, ); - if (active.cameraContract !== cameraContract) { + const cameraContractChanged = active.cameraContract !== cameraContract; + if (cameraContractChanged) { active.configureCameraJournal?.( recordedPlanView ? RECORDED_RERUN_PLAN_EYE : RECORDED_RERUN_ORBITAL_EYE, recordedUnifiedPerception ? recordedUnifiedCameraShare : 0, ); - active.cameraContract = cameraContract; } - const cameraEye = active.getCameraEye?.(); - void fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, { - origin: window.location.origin, - blueprintSessionId: blueprintSessionIdRef.current, - signal: abort.signal, - activeView: recordedView, - viewResetGeneration: recordedViewResetGeneration, - followTrajectory: recordedFollowTrajectory, - perceptionLayers: recordedPerceptionLayers, - semanticLayer: recordedSemanticLayer, - unifiedPerception: recordedUnifiedPerception, - unifiedCameraShare: recordedUnifiedCameraShare, - planView: recordedPlanView, - cameraEye, - currentTimeNs: active.getCurrentTimeNs?.(), - onCameraMaxOrbitalRadius: (maxOrbitalRadius) => { - if (blueprintChannelRef.current === active) { - active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius); - } - }, - }).then((payload) => { - if ( - abort.signal.aborted || - blueprintChannelRef.current !== active || - recordedIdentityRef.current !== identity || - !active.channel.ready - ) { - return; + const abort = new AbortController(); + const previousFollow = active.appliedFollowTrajectory ?? false; + const enablingFollow = recordedFollowTrajectory && !previousFollow; + const disablingFollow = !recordedFollowTrajectory && previousFollow; + const transitionEye = (enablingFollow || disablingFollow) + ? active.getCameraEye?.() + : null; + if (enablingFollow && transitionEye) { + active.pendingFollowCameraEye = transitionEye; + } else if (disablingFollow) { + active.pendingFollowCameraEye = null; + } + const pendingFollowEye = recordedFollowTrajectory + ? active.pendingFollowCameraEye ?? null + : null; + const requestBlueprint = async ( + cameraEye?: RecordedRerunCameraEye, + eyeRelativeToTracking = false, + reactivateUpdates = false, + ) => { + const currentTimeNs = active.getCurrentTimeNs?.(); + if (eyeRelativeToTracking && currentTimeNs == null) { + throw new Error("Recorded tracking cursor is unavailable"); } + return fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, { + origin: window.location.origin, + blueprintSessionId: blueprintSessionIdRef.current, + signal: abort.signal, + activeView: recordedView, + viewResetGeneration: recordedViewResetGeneration, + followTrajectory: recordedFollowTrajectory, + perceptionLayers: recordedPerceptionLayers, + semanticLayer: recordedSemanticLayer, + unifiedPerception: recordedUnifiedPerception, + unifiedCameraShare: recordedUnifiedCameraShare, + planView: recordedPlanView, + cameraEye, + eyeRelativeToTracking, + currentTimeNs, + reactivateUpdates, + onCameraMaxOrbitalRadius: (maxOrbitalRadius) => { + if (blueprintChannelRef.current === active) { + active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius); + } + }, + }); + }; + const canApply = () => ( + !abort.signal.aborted && + blueprintChannelRef.current === active && + recordedIdentityRef.current === identity && + active.channel.ready + ); + const applyPayload = (payload: Uint8Array) => { + if (!canApply()) return false; active.channel.send_rrd(payload); active.setCameraViewportStart?.( recordedUnifiedPerception ? recordedUnifiedCameraShare : 0, ); - }).catch(() => { + return true; + }; + void (async () => { + const firstEye = disablingFollow + ? transitionEye ?? undefined + : !enablingFollow && (cameraContractChanged || pendingFollowEye) + ? pendingFollowEye ?? active.getCameraEye?.() + : undefined; + const firstEyeIsTrackingRelative = Boolean(firstEye) && ( + disablingFollow || recordedFollowTrajectory + ); + const firstPayload = await requestBlueprint( + firstEye, + firstEyeIsTrackingRelative, + enablingFollow || disablingFollow || Boolean(pendingFollowEye), + ); + if (!applyPayload(firstPayload)) return; + active.cameraContract = cameraContract; + active.appliedFollowTrajectory = recordedFollowTrajectory; + if (!enablingFollow || !transitionEye) { + if (pendingFollowEye && firstEye === pendingFollowEye) { + active.pendingFollowCameraEye = null; + } + return; + } + + // Rerun 0.36.3 intentionally fits a newly tracked transform to its + // bounding box. Apply the operator's relative eye on the next native + // frame, after tracking is established, to retain direction and zoom. + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + if (!canApply()) return; + const stabilizedPayload = await requestBlueprint(transitionEye, true, true); + if (!applyPayload(stabilizedPayload)) return; + active.pendingFollowCameraEye = null; + })().catch(() => { // The recording remains usable with its embedded default blueprint. // A later settings change retries through the same small channel. }); diff --git a/apps/control-station/test/observationSources.test.mjs b/apps/control-station/test/observationSources.test.mjs index bf849be..d1cc100 100644 --- a/apps/control-station/test/observationSources.test.mjs +++ b/apps/control-station/test/observationSources.test.mjs @@ -486,6 +486,12 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting activeView: "perception3d", viewResetGeneration: 1, followTrajectory: true, + cameraEye: { + position: [3, 4, 5], + lookTarget: [1, 2, 0], + eyeUp: [0, 0, 1], + }, + eyeRelativeToTracking: true, currentTimeNs: 39_215_263_458, onCameraMaxOrbitalRadius: value => cameraLimits.push(value), unifiedCameraShare: 0.73, @@ -530,9 +536,10 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting unified_camera_share: 0.73, semantic_layer: null, plan_view: false, - eye_position: null, - eye_look_target: null, - eye_up: null, + eye_position: [3, 4, 5], + eye_look_target: [1, 2, 0], + eye_up: [0, 0, 1], + eye_relative_to_tracking: true, show_camera_image: true, show_detections_2d: true, show_segmentation: false, diff --git a/src/k1link/viewer/recorded.py b/src/k1link/viewer/recorded.py index fbccdf7..cf532b5 100644 --- a/src/k1link/viewer/recorded.py +++ b/src/k1link/viewer/recorded.py @@ -53,7 +53,7 @@ class _RecordedBlueprintStream: Upstream 0.36.3 activates a clone, not this source store. Explicit refresh makes layer changes visible. Ordinary layer updates deliberately omit eye components so the active clone retains the operator's camera. Only an - explicit 3D/plan/follow transition may write a new spatial eye preset. + explicit 3D/plan/reset transition may write a new spatial eye preset. """ def __init__( @@ -98,6 +98,9 @@ class _RecordedBlueprintStream: raise RecordedBlueprintError("stable blueprint stream is closed") eye_contract = (follow_trajectory, plan_view) update_eye_controls = self._eye_contract != eye_contract + use_spatial_preset = ( + self._eye_contract is not None and self._eye_contract[1] != plan_view + ) # Initial admission/reset uses native framing. Explicit presets # apply to mode transitions, after the viewer has a source cursor. # Keep one view identity for the lifetime of this browser owner. @@ -108,7 +111,7 @@ class _RecordedBlueprintStream: view_instance_token = self.blueprint_session_id blueprint = blueprint_factory( update_eye_controls, - self._eye_contract is not None, + use_spatial_preset, view_instance_token, ) # Appending rows alone does not refresh upstream's active clone. diff --git a/src/k1link/viewer/recorded_camera_bounds.py b/src/k1link/viewer/recorded_camera_bounds.py index b925e98..025aec2 100644 --- a/src/k1link/viewer/recorded_camera_bounds.py +++ b/src/k1link/viewer/recorded_camera_bounds.py @@ -50,6 +50,7 @@ def _recorded_spatial_bounds_index( rows: dict[str, list[tuple[int, np.ndarray, np.ndarray]]] = { "/world/points": [], "/world/trajectory": [], + "/world/sensor_pose": [], } for chunk in reader.stream(recording): entity_path = str(chunk.entity_path) @@ -59,9 +60,11 @@ def _recorded_spatial_bounds_index( names = batch.column_names if SESSION_TIMELINE not in names: continue - component = ( - "Points3D:positions" if entity_path == "/world/points" else "LineStrips3D:strips" - ) + component = { + "/world/points": "Points3D:positions", + "/world/trajectory": "LineStrips3D:strips", + "/world/sensor_pose": "Transform3D:translation", + }[entity_path] if component not in names: continue times = np.asarray( @@ -100,6 +103,34 @@ def _recorded_spatial_bounds_index( return result +def recorded_tracking_position( + recording_path: Path, + *, + current_time_ns: int, +) -> tuple[float, float, float] | None: + """Return the latest scanner pivot at or before the playback cursor.""" + + if current_time_ns < 0: + raise ValueError("invalid recorded camera query") + path = recording_path.expanduser().absolute() + if path.is_symlink() or not path.is_file(): + raise ValueError("recorded camera source is unavailable") + + stat = path.stat() + series = _recorded_spatial_bounds_index( + str(path), + stat.st_size, + stat.st_mtime_ns, + ).get("/world/sensor_pose") + if series is None: + return None + eligible_end = int(np.searchsorted(series.times_ns, current_time_ns, side="right")) + if eligible_end == 0: + return None + position = series.lower[eligible_end - 1] + return tuple(float(value) for value in position) + + def recorded_orbital_radius_limit( recording_path: Path, *, diff --git a/src/k1link/web/session_api.py b/src/k1link/web/session_api.py index 226f2da..70ea016 100644 --- a/src/k1link/web/session_api.py +++ b/src/k1link/web/session_api.py @@ -52,7 +52,10 @@ from k1link.viewer.recorded import ( recorded_blueprint_sessions, ) from k1link.viewer.recorded_blueprint_lifecycle import BlueprintSessionReleased -from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit +from k1link.viewer.recorded_camera_bounds import ( + recorded_orbital_radius_limit, + recorded_tracking_position, +) from k1link.viewer.rerun_bridge import RerunSceneSettings DEFAULT_REPLAY_ACTION_ID = "stream.start-replay" @@ -152,6 +155,7 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity): eye_position: EyeVector | None = None eye_look_target: EyeVector | None = None eye_up: EyeVector | None = None + eye_relative_to_tracking: StrictBool = False current_time_ns: int | None = Field(default=None, strict=True, ge=0, le=MAX_SAFE_INTEGER) @model_validator(mode="after") @@ -160,6 +164,8 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity): if any(vector is None for vector in vectors): if not all(vector is None for vector in vectors): raise ValueError("all eye vectors must be supplied together") + if self.eye_relative_to_tracking: + raise ValueError("a tracking-relative eye requires all eye vectors") return self assert self.eye_position is not None assert self.eye_look_target is not None @@ -168,6 +174,8 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity): raise ValueError("eye position and look target must differ") if sum(value * value for value in self.eye_up) <= 1.0e-12: raise ValueError("eye up vector must be non-zero") + if self.eye_relative_to_tracking and self.current_time_ns is None: + raise ValueError("a tracking-relative eye requires the playback cursor") return self @@ -1017,6 +1025,56 @@ def build_session_router( False, False, ) + camera_recording = None + if request.current_time_ns is not None: + if recording_preparation_manager is not None: + snapshot = recording_preparation_manager.status(session_id) + if ( + snapshot is not None + and snapshot.state == "ready" + and snapshot.recording is not None + ): + camera_recording = snapshot.recording + # A composition replay consumes the immutable base launch but + # does not GET its recording. Its short launch reservation can + # therefore expire while the combined RRD remains open. Restore + # the already-published base descriptor with bounded stat checks + # so later layer toggles still receive the native zoom limit. + if camera_recording is None and recording_materializer is not None: + camera_recording = await run_in_threadpool( + recording_materializer.restore_published, + command, + ) + eye_position = request.eye_position + eye_look_target = request.eye_look_target + if request.eye_relative_to_tracking: + if camera_recording is None or request.current_time_ns is None: + raise HTTPException( + status_code=409, + detail="Позиция сканера для текущего кадра недоступна.", + ) + tracking_position = await run_in_threadpool( + recorded_tracking_position, + camera_recording.path, + current_time_ns=request.current_time_ns, + ) + if tracking_position is None: + raise HTTPException( + status_code=409, + detail="Позиция сканера для текущего кадра недоступна.", + ) + assert eye_position is not None + assert eye_look_target is not None + eye_position = tuple( + tracking + position - target + for tracking, position, target in zip( + tracking_position, + eye_position, + eye_look_target, + strict=True, + ) + ) + eye_look_target = tracking_position payload = await run_in_threadpool( recorded_blueprint_rrd, RerunSceneSettings( @@ -1045,39 +1103,19 @@ def build_session_router( show_costmap=request.show_costmap, reactivate_updates=request.reactivate_updates, follow_trajectory=request.follow_trajectory, - eye_position=request.eye_position, - eye_look_target=request.eye_look_target, + eye_position=eye_position, + eye_look_target=eye_look_target, eye_up=request.eye_up, ) - if request.current_time_ns is not None: - camera_recording = None - if recording_preparation_manager is not None: - snapshot = recording_preparation_manager.status(session_id) - if ( - snapshot is not None - and snapshot.state == "ready" - and snapshot.recording is not None - ): - camera_recording = snapshot.recording - # A composition replay consumes the immutable base launch but - # does not GET its recording. Its short launch reservation can - # therefore expire while the combined RRD remains open. Restore - # the already-published base descriptor with bounded stat checks - # so later layer toggles still receive the native zoom limit. - if camera_recording is None and recording_materializer is not None: - camera_recording = await run_in_threadpool( - recording_materializer.restore_published, - command, - ) - if camera_recording is not None: - camera_max_orbital_radius = await run_in_threadpool( - recorded_orbital_radius_limit, - camera_recording.path, - current_time_ns=request.current_time_ns, - accumulation_seconds=request.accumulation_seconds, - show_points=request.show_points, - show_trajectory=request.show_trajectory, - ) + if request.current_time_ns is not None and camera_recording is not None: + camera_max_orbital_radius = await run_in_threadpool( + recorded_orbital_radius_limit, + camera_recording.path, + current_time_ns=request.current_time_ns, + accumulation_seconds=request.accumulation_seconds, + show_points=request.show_points, + show_trajectory=request.show_trajectory, + ) except SessionNotFoundError as exc: raise HTTPException(status_code=404, detail=str(exc)) from exc except (SessionNotReplayableError, SessionIntegrityError) as exc: diff --git a/tests/test_recorded_camera_bounds.py b/tests/test_recorded_camera_bounds.py index 52c284d..bf24b90 100644 --- a/tests/test_recorded_camera_bounds.py +++ b/tests/test_recorded_camera_bounds.py @@ -5,7 +5,10 @@ import pytest import rerun as rr import k1link.viewer.recorded_camera_bounds as camera_bounds -from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit +from k1link.viewer.recorded_camera_bounds import ( + recorded_orbital_radius_limit, + recorded_tracking_position, +) def _recording(path: Path) -> None: @@ -14,8 +17,10 @@ def _recording(path: Path) -> None: try: recording.set_time("session_time", duration=np.timedelta64(10, "s")) recording.log("/world/points", rr.Points3D([[0, 0, 0], [3, 4, 0]])) + recording.log("/world/sensor_pose", rr.Transform3D(translation=[1, 2, 3])) recording.set_time("session_time", duration=np.timedelta64(20, "s")) recording.log("/world/points", rr.Points3D([[10, 0, 0], [10, 0, 12]])) + recording.log("/world/sensor_pose", rr.Transform3D(translation=[10, 20, 30])) recording.flush(timeout_sec=5) finally: recording.disconnect() @@ -84,3 +89,18 @@ def test_recorded_camera_queries_reuse_the_generation_bounds_index(tmp_path: Pat assert second == pytest.approx(60, rel=1e-6) assert cache.misses == 1 assert cache.hits == 1 + + +def test_recorded_tracking_position_uses_latest_pose_at_cursor(tmp_path: Path) -> None: + path = tmp_path / "recording.rrd" + _recording(path) + + assert recorded_tracking_position(path, current_time_ns=9_000_000_000) is None + assert recorded_tracking_position( + path, + current_time_ns=15_000_000_000, + ) == pytest.approx((1, 2, 3)) + assert recorded_tracking_position( + path, + current_time_ns=20_000_000_000, + ) == pytest.approx((10, 20, 30)) diff --git a/tests/test_rrd_export.py b/tests/test_rrd_export.py index d80763d..8939a0a 100644 --- a/tests/test_rrd_export.py +++ b/tests/test_rrd_export.py @@ -627,8 +627,9 @@ def test_recorded_blueprint_updates_reuse_source_store_and_opt_in_to_activation( assert eye_control_updates == [True, False, True, False, True] # Reactivating a blueprint clone is required for visible layer changes, but # those changes must not write position/look-target/eye-up and reset the - # operator's camera. Only the follow transition writes a spatial preset. - assert explicit_presets == [False, False, True, False, False] + # operator's camera. Follow only changes tracking; it must not write the + # startup eye. 3D/plan remains the explicit spatial preset transition. + assert explicit_presets == [False, False, False, False, False] assert [activation[1:] for activation in activations] == [ (True, False), (reactivate_updates, False), diff --git a/tests/test_session_api.py b/tests/test_session_api.py index 62354b5..0c67534 100644 --- a/tests/test_session_api.py +++ b/tests/test_session_api.py @@ -1796,6 +1796,7 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir store.reconcile_archive(xgrids_k1_archive_source(sessions)) recording_path = tmp_path / "published.rrd" calls: list[tuple[Path, dict[str, object]]] = [] + blueprint_calls: list[dict[str, object]] = [] class PublishedMaterializer: def restore_published(self, command: ReplayCommand) -> object: @@ -1806,11 +1807,19 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir calls.append((path, kwargs)) return 123.5 + def blueprint_payload(*_args: object, **kwargs: object) -> bytes: + blueprint_calls.append(kwargs) + return b"RRF2" + monkeypatch.setattr( "k1link.web.session_api.recorded_blueprint_rrd", - lambda *_args, **_kwargs: b"RRF2", + blueprint_payload, ) monkeypatch.setattr("k1link.web.session_api.recorded_orbital_radius_limit", camera_bounds) + monkeypatch.setattr( + "k1link.web.session_api.recorded_tracking_position", + lambda *_args, **_kwargs: (100.0, 200.0, 300.0), + ) route = endpoint( build_session_router(store, recording_materializer=PublishedMaterializer()), # type: ignore[arg-type] "/api/v1/observation-sessions/{session_id}/blueprint.rrd", @@ -1828,6 +1837,11 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir show_points=True, show_trajectory=False, show_grid=True, + follow_trajectory=False, + eye_position=(3.0, 4.0, 5.0), + eye_look_target=(1.0, 2.0, 0.0), + eye_up=(0.0, 0.0, 1.0), + eye_relative_to_tracking=True, current_time_ns=39_215_000_000, ), ) @@ -1845,6 +1859,9 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir }, ) ] + assert blueprint_calls[0]["eye_position"] == pytest.approx((102.0, 202.0, 305.0)) + assert blueprint_calls[0]["eye_look_target"] == pytest.approx((100.0, 200.0, 300.0)) + assert blueprint_calls[0]["eye_up"] == (0.0, 0.0, 1.0) def test_recorded_perception_endpoint_returns_one_complete_optional_overlay(