fix(observatory): preserve camera across follow transitions

This commit is contained in:
DCCONSTRUCTIONS
2026-09-05 08:46:35 +03:00
parent cada687173
commit b2a1b23131
8 changed files with 268 additions and 76 deletions
@@ -146,6 +146,8 @@ export interface RerunViewportProps {
interface RerunBlueprintChannel {
endpointUrl: string;
cameraContract?: string | null;
appliedFollowTrajectory?: boolean | null;
pendingFollowCameraEye?: RecordedRerunCameraEye | null;
configureCameraJournal?: (
eye: RecordedRerunCameraEye,
spatialViewportStart: number,
@@ -415,7 +417,9 @@ export async function fetchRecordedBlueprintRrd(
unifiedCameraShare = 0.46,
planView = false,
cameraEye,
eyeRelativeToTracking = false,
currentTimeNs,
reactivateUpdates = false,
onCameraMaxOrbitalRadius,
perceptionLayers = {
enabled: false,
@@ -436,7 +440,9 @@ export async function fetchRecordedBlueprintRrd(
unifiedCameraShare?: number;
planView?: boolean;
cameraEye?: RecordedRerunCameraEye;
eyeRelativeToTracking?: boolean;
currentTimeNs?: number | null;
reactivateUpdates?: boolean;
onCameraMaxOrbitalRadius?: (maxOrbitalRadius: number) => void;
perceptionLayers?: RecordedPerceptionLayers;
fetcher?: typeof globalThis.fetch;
@@ -466,6 +472,9 @@ export async function fetchRecordedBlueprintRrd(
unifiedCameraShare < 0.1 ||
unifiedCameraShare > 0.9 ||
typeof planView !== "boolean" ||
typeof eyeRelativeToTracking !== "boolean" ||
typeof reactivateUpdates !== "boolean" ||
(eyeRelativeToTracking && (cameraEye === undefined || currentTimeNs == null)) ||
(cameraEye !== undefined && [
...cameraEye.position,
...cameraEye.lookTarget,
@@ -517,6 +526,7 @@ export async function fetchRecordedBlueprintRrd(
eye_position: cameraEye?.position ?? null,
eye_look_target: cameraEye?.lookTarget ?? null,
eye_up: cameraEye?.eyeUp ?? null,
...(eyeRelativeToTracking ? { eye_relative_to_tracking: true } : {}),
...(currentTimeNs === undefined || currentTimeNs === null ? {} : {
current_time_ns: currentTimeNs,
}),
@@ -528,6 +538,9 @@ export async function fetchRecordedBlueprintRrd(
show_costmap: perceptionLayers.costmap,
reactivate_updates: true,
}),
...(reactivateUpdates && perceptionLayers.costmap === undefined
? { reactivate_updates: true }
: {}),
}),
signal,
});
@@ -1615,6 +1628,8 @@ export function RerunViewport({
blueprintChannel = {
endpointUrl: recordedBlueprintUrl,
cameraContract: null,
appliedFollowTrajectory: null,
pendingFollowCameraEye: null,
configureCameraJournal: (eye, spatialViewportStart) => {
if ("configure_camera_journal" in viewer) {
viewer.configure_camera_journal(eye, spatialViewportStart);
@@ -2081,7 +2096,6 @@ export function RerunViewport({
active.endpointUrl !== recordedBlueprintUrl ||
!active.channel.ready
) return;
const abort = new AbortController();
const cameraContract = recordedCameraJournalContract(
{
activeView: recordedView,
@@ -2090,47 +2104,108 @@ export function RerunViewport({
followTrajectory: recordedFollowTrajectory,
},
);
if (active.cameraContract !== cameraContract) {
const cameraContractChanged = active.cameraContract !== cameraContract;
if (cameraContractChanged) {
active.configureCameraJournal?.(
recordedPlanView ? RECORDED_RERUN_PLAN_EYE : RECORDED_RERUN_ORBITAL_EYE,
recordedUnifiedPerception ? recordedUnifiedCameraShare : 0,
);
active.cameraContract = cameraContract;
}
const cameraEye = active.getCameraEye?.();
void fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, {
origin: window.location.origin,
blueprintSessionId: blueprintSessionIdRef.current,
signal: abort.signal,
activeView: recordedView,
viewResetGeneration: recordedViewResetGeneration,
followTrajectory: recordedFollowTrajectory,
perceptionLayers: recordedPerceptionLayers,
semanticLayer: recordedSemanticLayer,
unifiedPerception: recordedUnifiedPerception,
unifiedCameraShare: recordedUnifiedCameraShare,
planView: recordedPlanView,
cameraEye,
currentTimeNs: active.getCurrentTimeNs?.(),
onCameraMaxOrbitalRadius: (maxOrbitalRadius) => {
if (blueprintChannelRef.current === active) {
active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius);
}
},
}).then((payload) => {
if (
abort.signal.aborted ||
blueprintChannelRef.current !== active ||
recordedIdentityRef.current !== identity ||
!active.channel.ready
) {
return;
const abort = new AbortController();
const previousFollow = active.appliedFollowTrajectory ?? false;
const enablingFollow = recordedFollowTrajectory && !previousFollow;
const disablingFollow = !recordedFollowTrajectory && previousFollow;
const transitionEye = (enablingFollow || disablingFollow)
? active.getCameraEye?.()
: null;
if (enablingFollow && transitionEye) {
active.pendingFollowCameraEye = transitionEye;
} else if (disablingFollow) {
active.pendingFollowCameraEye = null;
}
const pendingFollowEye = recordedFollowTrajectory
? active.pendingFollowCameraEye ?? null
: null;
const requestBlueprint = async (
cameraEye?: RecordedRerunCameraEye,
eyeRelativeToTracking = false,
reactivateUpdates = false,
) => {
const currentTimeNs = active.getCurrentTimeNs?.();
if (eyeRelativeToTracking && currentTimeNs == null) {
throw new Error("Recorded tracking cursor is unavailable");
}
return fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, {
origin: window.location.origin,
blueprintSessionId: blueprintSessionIdRef.current,
signal: abort.signal,
activeView: recordedView,
viewResetGeneration: recordedViewResetGeneration,
followTrajectory: recordedFollowTrajectory,
perceptionLayers: recordedPerceptionLayers,
semanticLayer: recordedSemanticLayer,
unifiedPerception: recordedUnifiedPerception,
unifiedCameraShare: recordedUnifiedCameraShare,
planView: recordedPlanView,
cameraEye,
eyeRelativeToTracking,
currentTimeNs,
reactivateUpdates,
onCameraMaxOrbitalRadius: (maxOrbitalRadius) => {
if (blueprintChannelRef.current === active) {
active.setCameraMaxOrbitalRadius?.(maxOrbitalRadius);
}
},
});
};
const canApply = () => (
!abort.signal.aborted &&
blueprintChannelRef.current === active &&
recordedIdentityRef.current === identity &&
active.channel.ready
);
const applyPayload = (payload: Uint8Array) => {
if (!canApply()) return false;
active.channel.send_rrd(payload);
active.setCameraViewportStart?.(
recordedUnifiedPerception ? recordedUnifiedCameraShare : 0,
);
}).catch(() => {
return true;
};
void (async () => {
const firstEye = disablingFollow
? transitionEye ?? undefined
: !enablingFollow && (cameraContractChanged || pendingFollowEye)
? pendingFollowEye ?? active.getCameraEye?.()
: undefined;
const firstEyeIsTrackingRelative = Boolean(firstEye) && (
disablingFollow || recordedFollowTrajectory
);
const firstPayload = await requestBlueprint(
firstEye,
firstEyeIsTrackingRelative,
enablingFollow || disablingFollow || Boolean(pendingFollowEye),
);
if (!applyPayload(firstPayload)) return;
active.cameraContract = cameraContract;
active.appliedFollowTrajectory = recordedFollowTrajectory;
if (!enablingFollow || !transitionEye) {
if (pendingFollowEye && firstEye === pendingFollowEye) {
active.pendingFollowCameraEye = null;
}
return;
}
// Rerun 0.36.3 intentionally fits a newly tracked transform to its
// bounding box. Apply the operator's relative eye on the next native
// frame, after tracking is established, to retain direction and zoom.
await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
await new Promise<void>((resolve) => window.requestAnimationFrame(() => resolve()));
if (!canApply()) return;
const stabilizedPayload = await requestBlueprint(transitionEye, true, true);
if (!applyPayload(stabilizedPayload)) return;
active.pendingFollowCameraEye = null;
})().catch(() => {
// The recording remains usable with its embedded default blueprint.
// A later settings change retries through the same small channel.
});
@@ -486,6 +486,12 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
activeView: "perception3d",
viewResetGeneration: 1,
followTrajectory: true,
cameraEye: {
position: [3, 4, 5],
lookTarget: [1, 2, 0],
eyeUp: [0, 0, 1],
},
eyeRelativeToTracking: true,
currentTimeNs: 39_215_263_458,
onCameraMaxOrbitalRadius: value => cameraLimits.push(value),
unifiedCameraShare: 0.73,
@@ -530,9 +536,10 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
unified_camera_share: 0.73,
semantic_layer: null,
plan_view: false,
eye_position: null,
eye_look_target: null,
eye_up: null,
eye_position: [3, 4, 5],
eye_look_target: [1, 2, 0],
eye_up: [0, 0, 1],
eye_relative_to_tracking: true,
show_camera_image: true,
show_detections_2d: true,
show_segmentation: false,
+5 -2
View File
@@ -53,7 +53,7 @@ class _RecordedBlueprintStream:
Upstream 0.36.3 activates a clone, not this source store. Explicit refresh
makes layer changes visible. Ordinary layer updates deliberately omit eye
components so the active clone retains the operator's camera. Only an
explicit 3D/plan/follow transition may write a new spatial eye preset.
explicit 3D/plan/reset transition may write a new spatial eye preset.
"""
def __init__(
@@ -98,6 +98,9 @@ class _RecordedBlueprintStream:
raise RecordedBlueprintError("stable blueprint stream is closed")
eye_contract = (follow_trajectory, plan_view)
update_eye_controls = self._eye_contract != eye_contract
use_spatial_preset = (
self._eye_contract is not None and self._eye_contract[1] != plan_view
)
# Initial admission/reset uses native framing. Explicit presets
# apply to mode transitions, after the viewer has a source cursor.
# Keep one view identity for the lifetime of this browser owner.
@@ -108,7 +111,7 @@ class _RecordedBlueprintStream:
view_instance_token = self.blueprint_session_id
blueprint = blueprint_factory(
update_eye_controls,
self._eye_contract is not None,
use_spatial_preset,
view_instance_token,
)
# Appending rows alone does not refresh upstream's active clone.
+34 -3
View File
@@ -50,6 +50,7 @@ def _recorded_spatial_bounds_index(
rows: dict[str, list[tuple[int, np.ndarray, np.ndarray]]] = {
"/world/points": [],
"/world/trajectory": [],
"/world/sensor_pose": [],
}
for chunk in reader.stream(recording):
entity_path = str(chunk.entity_path)
@@ -59,9 +60,11 @@ def _recorded_spatial_bounds_index(
names = batch.column_names
if SESSION_TIMELINE not in names:
continue
component = (
"Points3D:positions" if entity_path == "/world/points" else "LineStrips3D:strips"
)
component = {
"/world/points": "Points3D:positions",
"/world/trajectory": "LineStrips3D:strips",
"/world/sensor_pose": "Transform3D:translation",
}[entity_path]
if component not in names:
continue
times = np.asarray(
@@ -100,6 +103,34 @@ def _recorded_spatial_bounds_index(
return result
def recorded_tracking_position(
recording_path: Path,
*,
current_time_ns: int,
) -> tuple[float, float, float] | None:
"""Return the latest scanner pivot at or before the playback cursor."""
if current_time_ns < 0:
raise ValueError("invalid recorded camera query")
path = recording_path.expanduser().absolute()
if path.is_symlink() or not path.is_file():
raise ValueError("recorded camera source is unavailable")
stat = path.stat()
series = _recorded_spatial_bounds_index(
str(path),
stat.st_size,
stat.st_mtime_ns,
).get("/world/sensor_pose")
if series is None:
return None
eligible_end = int(np.searchsorted(series.times_ns, current_time_ns, side="right"))
if eligible_end == 0:
return None
position = series.lower[eligible_end - 1]
return tuple(float(value) for value in position)
def recorded_orbital_radius_limit(
recording_path: Path,
*,
+70 -32
View File
@@ -52,7 +52,10 @@ from k1link.viewer.recorded import (
recorded_blueprint_sessions,
)
from k1link.viewer.recorded_blueprint_lifecycle import BlueprintSessionReleased
from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit
from k1link.viewer.recorded_camera_bounds import (
recorded_orbital_radius_limit,
recorded_tracking_position,
)
from k1link.viewer.rerun_bridge import RerunSceneSettings
DEFAULT_REPLAY_ACTION_ID = "stream.start-replay"
@@ -152,6 +155,7 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity):
eye_position: EyeVector | None = None
eye_look_target: EyeVector | None = None
eye_up: EyeVector | None = None
eye_relative_to_tracking: StrictBool = False
current_time_ns: int | None = Field(default=None, strict=True, ge=0, le=MAX_SAFE_INTEGER)
@model_validator(mode="after")
@@ -160,6 +164,8 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity):
if any(vector is None for vector in vectors):
if not all(vector is None for vector in vectors):
raise ValueError("all eye vectors must be supplied together")
if self.eye_relative_to_tracking:
raise ValueError("a tracking-relative eye requires all eye vectors")
return self
assert self.eye_position is not None
assert self.eye_look_target is not None
@@ -168,6 +174,8 @@ class RecordedBlueprintRequest(RecordedBlueprintIdentity):
raise ValueError("eye position and look target must differ")
if sum(value * value for value in self.eye_up) <= 1.0e-12:
raise ValueError("eye up vector must be non-zero")
if self.eye_relative_to_tracking and self.current_time_ns is None:
raise ValueError("a tracking-relative eye requires the playback cursor")
return self
@@ -1017,6 +1025,56 @@ def build_session_router(
False,
False,
)
camera_recording = None
if request.current_time_ns is not None:
if recording_preparation_manager is not None:
snapshot = recording_preparation_manager.status(session_id)
if (
snapshot is not None
and snapshot.state == "ready"
and snapshot.recording is not None
):
camera_recording = snapshot.recording
# A composition replay consumes the immutable base launch but
# does not GET its recording. Its short launch reservation can
# therefore expire while the combined RRD remains open. Restore
# the already-published base descriptor with bounded stat checks
# so later layer toggles still receive the native zoom limit.
if camera_recording is None and recording_materializer is not None:
camera_recording = await run_in_threadpool(
recording_materializer.restore_published,
command,
)
eye_position = request.eye_position
eye_look_target = request.eye_look_target
if request.eye_relative_to_tracking:
if camera_recording is None or request.current_time_ns is None:
raise HTTPException(
status_code=409,
detail="Позиция сканера для текущего кадра недоступна.",
)
tracking_position = await run_in_threadpool(
recorded_tracking_position,
camera_recording.path,
current_time_ns=request.current_time_ns,
)
if tracking_position is None:
raise HTTPException(
status_code=409,
detail="Позиция сканера для текущего кадра недоступна.",
)
assert eye_position is not None
assert eye_look_target is not None
eye_position = tuple(
tracking + position - target
for tracking, position, target in zip(
tracking_position,
eye_position,
eye_look_target,
strict=True,
)
)
eye_look_target = tracking_position
payload = await run_in_threadpool(
recorded_blueprint_rrd,
RerunSceneSettings(
@@ -1045,39 +1103,19 @@ def build_session_router(
show_costmap=request.show_costmap,
reactivate_updates=request.reactivate_updates,
follow_trajectory=request.follow_trajectory,
eye_position=request.eye_position,
eye_look_target=request.eye_look_target,
eye_position=eye_position,
eye_look_target=eye_look_target,
eye_up=request.eye_up,
)
if request.current_time_ns is not None:
camera_recording = None
if recording_preparation_manager is not None:
snapshot = recording_preparation_manager.status(session_id)
if (
snapshot is not None
and snapshot.state == "ready"
and snapshot.recording is not None
):
camera_recording = snapshot.recording
# A composition replay consumes the immutable base launch but
# does not GET its recording. Its short launch reservation can
# therefore expire while the combined RRD remains open. Restore
# the already-published base descriptor with bounded stat checks
# so later layer toggles still receive the native zoom limit.
if camera_recording is None and recording_materializer is not None:
camera_recording = await run_in_threadpool(
recording_materializer.restore_published,
command,
)
if camera_recording is not None:
camera_max_orbital_radius = await run_in_threadpool(
recorded_orbital_radius_limit,
camera_recording.path,
current_time_ns=request.current_time_ns,
accumulation_seconds=request.accumulation_seconds,
show_points=request.show_points,
show_trajectory=request.show_trajectory,
)
if request.current_time_ns is not None and camera_recording is not None:
camera_max_orbital_radius = await run_in_threadpool(
recorded_orbital_radius_limit,
camera_recording.path,
current_time_ns=request.current_time_ns,
accumulation_seconds=request.accumulation_seconds,
show_points=request.show_points,
show_trajectory=request.show_trajectory,
)
except SessionNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except (SessionNotReplayableError, SessionIntegrityError) as exc:
+21 -1
View File
@@ -5,7 +5,10 @@ import pytest
import rerun as rr
import k1link.viewer.recorded_camera_bounds as camera_bounds
from k1link.viewer.recorded_camera_bounds import recorded_orbital_radius_limit
from k1link.viewer.recorded_camera_bounds import (
recorded_orbital_radius_limit,
recorded_tracking_position,
)
def _recording(path: Path) -> None:
@@ -14,8 +17,10 @@ def _recording(path: Path) -> None:
try:
recording.set_time("session_time", duration=np.timedelta64(10, "s"))
recording.log("/world/points", rr.Points3D([[0, 0, 0], [3, 4, 0]]))
recording.log("/world/sensor_pose", rr.Transform3D(translation=[1, 2, 3]))
recording.set_time("session_time", duration=np.timedelta64(20, "s"))
recording.log("/world/points", rr.Points3D([[10, 0, 0], [10, 0, 12]]))
recording.log("/world/sensor_pose", rr.Transform3D(translation=[10, 20, 30]))
recording.flush(timeout_sec=5)
finally:
recording.disconnect()
@@ -84,3 +89,18 @@ def test_recorded_camera_queries_reuse_the_generation_bounds_index(tmp_path: Pat
assert second == pytest.approx(60, rel=1e-6)
assert cache.misses == 1
assert cache.hits == 1
def test_recorded_tracking_position_uses_latest_pose_at_cursor(tmp_path: Path) -> None:
path = tmp_path / "recording.rrd"
_recording(path)
assert recorded_tracking_position(path, current_time_ns=9_000_000_000) is None
assert recorded_tracking_position(
path,
current_time_ns=15_000_000_000,
) == pytest.approx((1, 2, 3))
assert recorded_tracking_position(
path,
current_time_ns=20_000_000_000,
) == pytest.approx((10, 20, 30))
+3 -2
View File
@@ -627,8 +627,9 @@ def test_recorded_blueprint_updates_reuse_source_store_and_opt_in_to_activation(
assert eye_control_updates == [True, False, True, False, True]
# Reactivating a blueprint clone is required for visible layer changes, but
# those changes must not write position/look-target/eye-up and reset the
# operator's camera. Only the follow transition writes a spatial preset.
assert explicit_presets == [False, False, True, False, False]
# operator's camera. Follow only changes tracking; it must not write the
# startup eye. 3D/plan remains the explicit spatial preset transition.
assert explicit_presets == [False, False, False, False, False]
assert [activation[1:] for activation in activations] == [
(True, False),
(reactivate_updates, False),
+18 -1
View File
@@ -1796,6 +1796,7 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir
store.reconcile_archive(xgrids_k1_archive_source(sessions))
recording_path = tmp_path / "published.rrd"
calls: list[tuple[Path, dict[str, object]]] = []
blueprint_calls: list[dict[str, object]] = []
class PublishedMaterializer:
def restore_published(self, command: ReplayCommand) -> object:
@@ -1806,11 +1807,19 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir
calls.append((path, kwargs))
return 123.5
def blueprint_payload(*_args: object, **kwargs: object) -> bytes:
blueprint_calls.append(kwargs)
return b"RRF2"
monkeypatch.setattr(
"k1link.web.session_api.recorded_blueprint_rrd",
lambda *_args, **_kwargs: b"RRF2",
blueprint_payload,
)
monkeypatch.setattr("k1link.web.session_api.recorded_orbital_radius_limit", camera_bounds)
monkeypatch.setattr(
"k1link.web.session_api.recorded_tracking_position",
lambda *_args, **_kwargs: (100.0, 200.0, 300.0),
)
route = endpoint(
build_session_router(store, recording_materializer=PublishedMaterializer()), # type: ignore[arg-type]
"/api/v1/observation-sessions/{session_id}/blueprint.rrd",
@@ -1828,6 +1837,11 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir
show_points=True,
show_trajectory=False,
show_grid=True,
follow_trajectory=False,
eye_position=(3.0, 4.0, 5.0),
eye_look_target=(1.0, 2.0, 0.0),
eye_up=(0.0, 0.0, 1.0),
eye_relative_to_tracking=True,
current_time_ns=39_215_000_000,
),
)
@@ -1845,6 +1859,9 @@ def test_recorded_blueprint_restores_camera_bounds_after_base_launch_lease_expir
},
)
]
assert blueprint_calls[0]["eye_position"] == pytest.approx((102.0, 202.0, 305.0))
assert blueprint_calls[0]["eye_look_target"] == pytest.approx((100.0, 200.0, 300.0))
assert blueprint_calls[0]["eye_up"] == (0.0, 0.0, 1.0)
def test_recorded_perception_endpoint_returns_one_complete_optional_overlay(