feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
@@ -1,10 +1,15 @@
import { readFileSync, writeFileSync } from "node:fs";
import { createHash } from "node:crypto";
import { copyFileSync, readFileSync, writeFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const packageRoot = resolve(root, "node_modules/@rerun-io/web-viewer");
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "package.json"), "utf8"));
const vendorRoot = resolve(root, "vendor/rerun-web-viewer-0.34.1");
const sha256File = (path) =>
createHash("sha256").update(readFileSync(path)).digest("hex");
if (manifest.version !== "0.34.1") {
throw new Error(
@@ -12,6 +17,51 @@ if (manifest.version !== "0.34.1") {
);
}
const vendorRuntime = [
{
label: "wasm runtime",
packagePath: resolve(packageRoot, "re_viewer_bg.wasm"),
vendorPath: resolve(vendorRoot, "re_viewer_bg.nodedc.wasm"),
publishedSha256: "3fe7aab8ea6bb0fd03c3ef694932943411ee174029e398c539c390beb0824d35",
previousNodedcSha256: "1c25e8cecd7641e6f8044d00a6a1cf9d08a615b6f8737026c7d93623b3e06ee7",
nodedcSha256: "38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb",
},
{
label: "wasm JavaScript glue",
packagePath: resolve(packageRoot, "re_viewer.js"),
vendorPath: resolve(vendorRoot, "re_viewer.nodedc.js"),
publishedSha256: "7c4ba900820137a6ba23e0f7f57d56e3b007db0de3825d5c8bf0d7684b3cc6a6",
nodedcSha256: "0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339",
},
];
for (const runtime of vendorRuntime) {
const vendorSha256 = sha256File(runtime.vendorPath);
if (vendorSha256 !== runtime.nodedcSha256) {
throw new Error(
`Refusing corrupt NODE.DC Rerun ${runtime.label}: ${vendorSha256} != ${runtime.nodedcSha256}`,
);
}
const installedSha256 = sha256File(runtime.packagePath);
if (
![
runtime.publishedSha256,
runtime.previousNodedcSha256,
runtime.nodedcSha256,
].includes(installedSha256)
) {
throw new Error(
`Refusing to replace unexpected Rerun ${runtime.label}: ${installedSha256}`,
);
}
copyFileSync(runtime.vendorPath, runtime.packagePath);
if (sha256File(runtime.packagePath) !== runtime.nodedcSha256) {
throw new Error(`NODE.DC Rerun ${runtime.label} verification failed after copy`);
}
}
const patches = [
{
label: "compiled watchdog",
@@ -57,5 +107,5 @@ for (const path of [resolve(packageRoot, "index.js"), resolve(packageRoot, "inde
}
console.log(
"Patched @rerun-io/web-viewer 0.34.1 watchdog teardown and singleton global keyup listener.",
"Patched @rerun-io/web-viewer 0.34.1 native zoom-to-cursor, watchdog teardown, and singleton global keyup listener.",
);
@@ -0,0 +1,128 @@
import { createHash } from "node:crypto";
import { readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
const [inputArgument, outputArgument] = process.argv.slice(2);
if (!inputArgument || !outputArgument) {
throw new Error("Usage: node transform-rerun-web-viewer-glue.mjs INPUT OUTPUT");
}
const inputPath = resolve(inputArgument);
const outputPath = resolve(outputArgument);
const expectedGeneratedSha256 =
"cc196a93c5be972c801d46be4dc9934f7f042eb62941f0aa0678f1c8416c6874";
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
let code = readFileSync(inputPath, "utf8");
if (sha256(code) !== expectedGeneratedSha256) {
throw new Error(`Refusing to transform unexpected wasm-bindgen output: ${inputPath}`);
}
// This is the same no-modules-base transformation used by Rerun 0.34.1's
// rerun_js/web-viewer/build-wasm.mjs. Each factory call gets isolated closure state.
const wrapperStart = "let wasm_bindgen = (function(exports) {";
const wrapperEnd = `return Object.assign(__wbg_init, { initSync }, exports);
})({ __proto__: null });`;
if (!code.includes(wrapperStart) || !code.includes(wrapperEnd)) {
throw new Error("Rerun wasm-bindgen wrapper markers no longer match");
}
code = code.replace(wrapperStart, "").replace(wrapperEnd, "");
code = `
export default function() {
const exports = { __proto__: null };
${code}
function deinit() {
__wbg_init.__wbindgen_wasm_module = null;
wasmModule = null;
wasm = null;
cachedDataViewMemory0 = null;
cachedFloat32ArrayMemory0 = null;
cachedInt16ArrayMemory0 = null;
cachedInt32ArrayMemory0 = null;
cachedInt8ArrayMemory0 = null;
cachedUint16ArrayMemory0 = null;
cachedUint32ArrayMemory0 = null;
cachedUint8ArrayMemory0 = null;
}
return Object.assign(__wbg_init, { initSync, deinit }, exports);
}
`;
const closureDtorsOriginal = `const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(state => wasm.__wbindgen_destroy_closure(state.a, state.b));`;
const closureDtorsPatch = `const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined')
? { register: () => {}, unregister: () => {} }
: new FinalizationRegistry(state => {
if (wasm) wasm.__wbindgen_destroy_closure(state.a, state.b);
});`;
if (!code.includes(closureDtorsOriginal)) {
throw new Error("Rerun CLOSURE_DTORS block no longer matches");
}
code = code.replace(closureDtorsOriginal, closureDtorsPatch);
const makeMutClosureOriginal = `function makeMutClosure(arg0, arg1, f) {
const state = { a: arg0, b: arg1, cnt: 1 };
const real = (...args) => {
// First up with a closure we increment the internal reference
// count. This ensures that the Rust closure environment won't
// be deallocated while we're invoking it.
state.cnt++;
const a = state.a;
state.a = 0;
try {
return f(a, state.b, ...args);
} finally {
state.a = a;
real._wbg_cb_unref();
}
};
real._wbg_cb_unref = () => {
if (--state.cnt === 0) {
wasm.__wbindgen_destroy_closure(state.a, state.b);
state.a = 0;
CLOSURE_DTORS.unregister(state);
}
};
CLOSURE_DTORS.register(real, state, state);
return real;
}`;
const makeMutClosurePatch = `function makeMutClosure(arg0, arg1, f) {
const state = { a: arg0, b: arg1, cnt: 1 };
const real = (...args) => {
state.cnt++;
const a = state.a;
state.a = 0;
try {
if (!wasm) return;
return f(a, state.b, ...args);
} finally {
state.a = a;
real._wbg_cb_unref();
}
};
real._wbg_cb_unref = () => {
if (--state.cnt === 0) {
if (wasm) wasm.__wbindgen_destroy_closure(state.a, state.b);
state.a = 0;
CLOSURE_DTORS.unregister(state);
}
};
CLOSURE_DTORS.register(real, state, state);
return real;
}`;
if (!code.includes(makeMutClosureOriginal)) {
throw new Error("Rerun makeMutClosure block no longer matches");
}
code = code.replace(makeMutClosureOriginal, makeMutClosurePatch);
writeFileSync(outputPath, code);
console.log(`Transformed Rerun wasm glue: ${sha256(code)} ${outputPath}`);
+54 -3
View File
@@ -88,7 +88,22 @@ const paletteOptions: Array<{ value: PointPalette; label: string; description: s
{ value: "custom", label: "Свой цвет", description: "Один назначенный цвет" },
];
function toViewerSettings(settings: SceneSettings): ViewerSettings {
interface LivePerceptionLayers {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}
const defaultLivePerceptionLayers: LivePerceptionLayers = {
detections2d: false,
segmentation: false,
cuboids3d: false,
};
function toViewerSettings(
settings: SceneSettings,
perception: LivePerceptionLayers = defaultLivePerceptionLayers,
): ViewerSettings {
return {
point_size: settings.pointSize,
color_mode: settings.colorMode,
@@ -98,6 +113,9 @@ function toViewerSettings(settings: SceneSettings): ViewerSettings {
show_points: settings.showPoints,
show_trajectory: settings.showTrajectory,
show_grid: settings.showGrid,
show_detections_2d: perception.detections2d,
show_segmentation: perception.segmentation,
show_cuboids_3d: perception.cuboids3d,
};
}
@@ -149,6 +167,9 @@ export default function App() {
const [layoutSaveNotice, setLayoutSaveNotice] = useState<string | null>(null);
const [sceneSettings, setSceneSettings] = useState<SceneSettings>(defaultSceneSettings);
const [displayDraft, setDisplayDraft] = useState<SceneSettings>(defaultSceneSettings);
const [livePerceptionLayers, setLivePerceptionLayers] = useState<LivePerceptionLayers>(
defaultLivePerceptionLayers,
);
const sourceSwitchBlocked = isSpatialSourceSwitchBlocked(runtime.state);
const sourceSwitchBlockedReason = sourceSwitchBlocked
? SPATIAL_SOURCE_SWITCH_BLOCKED_REASON
@@ -161,6 +182,8 @@ export default function App() {
const confirmedSceneSettingsRef = useRef<SceneSettings>(defaultSceneSettings);
const viewerSettingsCommitTimerRef = useRef<number | null>(null);
const runtimeUpdateViewerSettingsRef = useRef(runtime.updateViewerSettings);
const livePerceptionLayersRef = useRef<LivePerceptionLayers>(defaultLivePerceptionLayers);
const livePerceptionRevisionRef = useRef(0);
const replayActiveRef = useRef(false);
const sceneSettingsCommitterActiveRef = useRef(true);
const sceneSettingsCommitterRef = useRef<LatestAsyncCommitter<SceneSettings> | null>(null);
@@ -197,7 +220,9 @@ export default function App() {
sceneSettingsCommitterRef.current = createLatestAsyncCommitter<SceneSettings>({
commit: (settings) => replayActiveRef.current
? Promise.resolve(true)
: runtimeUpdateViewerSettingsRef.current(toViewerSettings(settings)),
: runtimeUpdateViewerSettingsRef.current(
toViewerSettings(settings, livePerceptionLayersRef.current),
),
onSettled: ({ value, applied, superseded }) => {
if (!sceneSettingsCommitterActiveRef.current) return;
if (applied) confirmedSceneSettingsRef.current = value;
@@ -275,6 +300,13 @@ export default function App() {
sceneSettingsCommitterRef.current?.isBusy()
) return;
const merged = mergeViewerSettings(sceneSettingsRef.current, remote);
const remoteLayers = {
detections2d: remote.show_detections_2d ?? false,
segmentation: remote.show_segmentation ?? false,
cuboids3d: remote.show_cuboids_3d ?? false,
};
livePerceptionLayersRef.current = remoteLayers;
setLivePerceptionLayers(remoteLayers);
sceneSettingsRef.current = merged;
confirmedSceneSettingsRef.current = merged;
setSceneSettings(merged);
@@ -334,7 +366,9 @@ export default function App() {
return;
}
appliedProfileKeyRef.current = applicationKey;
void runtime.updateViewerSettings(toViewerSettings(profile.sceneSettings)).then((applied) => {
void runtime.updateViewerSettings(
toViewerSettings(profile.sceneSettings, livePerceptionLayersRef.current),
).then((applied) => {
if (!applied && appliedProfileKeyRef.current === applicationKey) {
appliedProfileKeyRef.current = null;
}
@@ -441,6 +475,21 @@ export default function App() {
activateSceneWindow("layers");
};
const changeLivePerceptionLayers = useCallback((next: LivePerceptionLayers) => {
const previous = livePerceptionLayersRef.current;
const revision = ++livePerceptionRevisionRef.current;
livePerceptionLayersRef.current = next;
setLivePerceptionLayers(next);
if (replayActiveRef.current) return;
void runtimeUpdateViewerSettingsRef.current(
toViewerSettings(sceneSettingsRef.current, next),
).then((applied) => {
if (applied || livePerceptionRevisionRef.current !== revision) return;
livePerceptionLayersRef.current = previous;
setLivePerceptionLayers(previous);
});
}, []);
const beginRecordedReplaySwitch = useCallback(async () => {
if (sourceSwitchBlockedRef.current) {
throw new Error(SPATIAL_SOURCE_SWITCH_BLOCKED_REASON);
@@ -706,6 +755,8 @@ export default function App() {
onAccumulationChange={(accumulationSeconds) =>
stageDisplayPatch({ accumulationSeconds })}
onAccumulationCommit={flushDisplaySettings}
livePerceptionLayers={livePerceptionLayers}
onLivePerceptionLayersChange={changeLivePerceptionLayers}
observationLayout={observationLayout}
spatialControls={selection?.SpatialControlsView
? {
@@ -72,6 +72,7 @@ export function FloatingObservationWindow({
rect,
maximized,
active,
hidden = false,
onRectChange,
onMaximizedChange,
onActivate,
@@ -89,6 +90,7 @@ export function FloatingObservationWindow({
rect?: ObservationWindowRect;
maximized: boolean;
active: boolean;
hidden?: boolean;
onRectChange: (rect: ObservationWindowRect) => void;
onMaximizedChange: (maximized: boolean) => void;
onActivate: () => void;
@@ -153,7 +155,7 @@ export function FloatingObservationWindow({
resizable={source.capabilities.resizable}
active={active}
zIndex={maximized ? 15 : active ? 9 : 7}
className="floating-observation-window"
className={`floating-observation-window${hidden ? " floating-observation-window--hidden" : ""}`}
onPointerDownCapture={(event) => {
if (!shouldCaptureWorkspacePointer(event.button, event.target as HTMLElement)) return;
try {
@@ -52,18 +52,17 @@ const preparationLabel: Record<ObservationPreparationPhase, string> = {
requesting: "Запрашиваем подготовку",
queued: "В очереди",
validating: "Проверяем запись",
exporting: "Готовим облако точек",
exporting: "Готовим операторскую сцену",
finalizing: "Завершаем подготовку",
failed: "Подготовка не выполнена",
cancelled: "Подготовка отменена",
};
function progressCopy(
phase: ObservationPreparationPhase,
progress: number | null,
): string {
const label = preparationLabel[phase];
return progress === null ? label : `${label} · ${Math.round(progress * 100)}%`;
function progressCopy(phase: ObservationPreparationPhase): string {
// Backend progress values are phase markers and heartbeat revisions, not a
// measured fraction of bytes or frames. Presenting 0.5 as "50%" made a
// healthy long export look stalled, especially after a browser reload.
return preparationLabel[phase];
}
function formatStartedAt(value: string): string {
@@ -126,7 +125,7 @@ export function ObservationSessionSelect({
onReplaySettled,
});
const triggerCopy = sessions.replayProgress
? progressCopy(sessions.replayProgress.phase, sessions.replayProgress.progress)
? progressCopy(sessions.replayProgress.phase)
: sessions.state === "loading"
? "Загружаем сессии…"
: "Сохранённые сессии";
@@ -1,5 +1,3 @@
import { sha256 } from "@noble/hashes/sha2.js";
import { bytesToHex } from "@noble/hashes/utils.js";
import { useEffect, useMemo, useRef, useState } from "react";
import {
@@ -11,7 +9,6 @@ import {
type ObservationSessionFetch,
} from "../core/observation/sessionArchive";
import {
MAX_RECORDED_MEDIA_SOURCE_BYTES,
type RecordedAdmissionPhase,
type RecordedCameraAdmissionState,
} from "../core/observation/recordedSessionAdmission";
@@ -22,32 +19,11 @@ export interface RecordedObservationPlayback {
playing: boolean;
}
export interface RecordedMediaLoadProgress {
loadedBytes: number;
totalBytes: number;
loadedParts: number;
totalParts: number;
}
interface VerifiedRecordedMediaEpoch {
descriptor: ObservationRecordedMediaEpoch;
readonly init: ArrayBuffer;
readonly segments: readonly ArrayBuffer[];
}
export interface VerifiedRecordedMediaArchive {
export interface RecordedMediaArchive {
manifest: ObservationRecordedMediaManifest;
epochs: readonly VerifiedRecordedMediaEpoch[];
byteLength: number;
}
interface RecordedMediaBinaryDescriptor {
url: string;
byteLength: number;
sha256: string;
accept: string;
}
export type RecordedMediaPresentationState = "loading" | "ready" | "waiting" | "error";
export const RECORDED_MEDIA_DURATION_TOLERANCE_SECONDS = 1;
@@ -134,205 +110,33 @@ function sourceContract(source: ObservationSourceDescriptor): ObservationRecorde
};
}
function expectedPayloadEtag(digest: string): string {
return `"sha256:${digest}"`;
}
export async function fetchVerifiedRecordedMediaBytes(
descriptor: RecordedMediaBinaryDescriptor,
{
signal,
fetcher = globalThis.fetch,
}: { signal?: AbortSignal; fetcher?: ObservationSessionFetch } = {},
): Promise<ArrayBuffer> {
let response: Response;
try {
response = await fetcher(descriptor.url, {
method: "GET",
headers: {
Accept: descriptor.accept,
"If-Match": expectedPayloadEtag(descriptor.sha256),
},
signal,
});
} catch (error) {
if (error instanceof DOMException && error.name === "AbortError") throw error;
throw new ObservationSessionContractError(
"Не удалось загрузить канонический фрагмент записанного видео.",
);
}
if (!response.ok || response.status !== 200) {
throw new ObservationSessionContractError(
`Фрагмент записанного видео вернул HTTP ${response.status}.`,
);
}
if (response.headers.get("ETag") !== expectedPayloadEtag(descriptor.sha256)) {
throw new ObservationSessionContractError(
"Фрагмент записанного видео не соответствует immutable manifest.",
);
}
const contentLength = response.headers.get("Content-Length");
if (
contentLength === null ||
!/^[1-9][0-9]*$/.test(contentLength) ||
Number(contentLength) !== descriptor.byteLength
) {
throw new ObservationSessionContractError(
"Длина фрагмента записанного видео не соответствует immutable manifest.",
);
}
const payload = await response.arrayBuffer();
if (payload.byteLength !== descriptor.byteLength) {
throw new ObservationSessionContractError(
"Фрагмент записанного видео был усечён во время передачи.",
);
}
const digest = bytesToHex(sha256(new Uint8Array(payload)));
if (digest !== descriptor.sha256) {
throw new ObservationSessionContractError(
"SHA-256 фрагмента записанного видео не совпадает с immutable manifest.",
);
}
return payload;
}
function manifestIdentity(manifest: ObservationRecordedMediaManifest): string {
return JSON.stringify(manifest);
}
export async function fetchVerifiedRecordedMediaArchive(
export async function fetchRecordedMediaArchive(
source: ObservationRecordedMediaSource,
{
signal,
fetcher = globalThis.fetch,
onProgress,
}: {
signal?: AbortSignal;
fetcher?: ObservationSessionFetch;
onProgress?: (progress: RecordedMediaLoadProgress) => void;
} = {},
): Promise<VerifiedRecordedMediaArchive> {
): Promise<RecordedMediaArchive> {
const manifest = await fetchObservationRecordedMediaManifest(source, {
signal,
fetcher,
expectedGenerationSha256: source.manifestGenerationSha256,
});
const totalBytes = manifest.epochs.reduce(
(archiveTotal, epoch) => archiveTotal + epoch.initByteLength + epoch.segments.reduce(
(epochTotal, segment) => epochTotal + segment.byteLength,
0,
),
0,
);
const totalParts = manifest.epochs.reduce(
(total, epoch) => total + 1 + epoch.segments.length,
0,
);
const totalBytes = manifest.epochs.reduce((total, epoch) => total + epoch.byteLength, 0);
if (
!Number.isSafeInteger(totalBytes) ||
totalBytes < 1 ||
totalBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES ||
totalBytes !== manifest.byteLength ||
totalBytes !== source.byteLength
) {
throw new ObservationSessionContractError(
"Размер записанного медиаканала выходит за безопасный лимит браузера.",
"Размеры потоков записанного медиаканала не совпадают с manifest.",
);
}
let loadedBytes = 0;
let loadedParts = 0;
const publishProgress = () => onProgress?.({
loadedBytes,
totalBytes,
loadedParts,
totalParts,
});
publishProgress();
const epochs: VerifiedRecordedMediaEpoch[] = [];
for (const epoch of manifest.epochs) {
const init = await fetchVerifiedRecordedMediaBytes({
url: epoch.initUrl,
byteLength: epoch.initByteLength,
sha256: epoch.initSha256,
accept: "video/mp4",
}, { signal, fetcher });
loadedBytes += init.byteLength;
loadedParts += 1;
publishProgress();
const segments: ArrayBuffer[] = [];
for (const segment of epoch.segments) {
const payload = await fetchVerifiedRecordedMediaBytes({
url: segment.url,
byteLength: segment.byteLength,
sha256: segment.sha256,
accept: "video/iso.segment",
}, { signal, fetcher });
segments.push(payload);
loadedBytes += payload.byteLength;
loadedParts += 1;
publishProgress();
}
epochs.push({ descriptor: epoch, init, segments });
}
// Bind the complete byte set to the same manifest generation at both ends
// of the transfer. A replacement during a long camera download fails closed.
const confirmed = await fetchObservationRecordedMediaManifest(source, {
signal,
fetcher,
expectedGenerationSha256: manifest.generationSha256,
});
if (manifestIdentity(confirmed) !== manifestIdentity(manifest)) {
throw new ObservationSessionContractError(
"Manifest записанного видео изменился во время полной загрузки.",
);
}
if (loadedBytes !== source.byteLength) {
throw new ObservationSessionContractError(
"Полная загрузка камеры не совпала с launch byte_length.",
);
}
return { manifest, epochs, byteLength: loadedBytes };
}
export function appendRecordedMediaBuffer(
sourceBuffer: SourceBuffer,
payload: ArrayBuffer,
signal: AbortSignal,
): Promise<void> {
return new Promise((resolve, reject) => {
const onUpdateEnd = () => {
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error("SourceBuffer rejected archived fMP4 data"));
};
const onAbort = () => {
cleanup();
reject(new DOMException("Aborted", "AbortError"));
};
const cleanup = () => {
sourceBuffer.removeEventListener("updateend", onUpdateEnd);
sourceBuffer.removeEventListener("error", onError);
signal.removeEventListener("abort", onAbort);
};
if (signal.aborted) {
onAbort();
return;
}
sourceBuffer.addEventListener("updateend", onUpdateEnd, { once: true });
sourceBuffer.addEventListener("error", onError, { once: true });
signal.addEventListener("abort", onAbort, { once: true });
try {
sourceBuffer.appendBuffer(payload);
} catch (error) {
cleanup();
reject(error);
}
});
return { manifest, byteLength: totalBytes };
}
function videoHasSeekableArchive(
@@ -358,17 +162,22 @@ function waitForSeekableArchive(
if (videoHasSeekableArchive(video, declaredDurationSeconds)) return Promise.resolve();
return new Promise((resolve, reject) => {
const events = ["loadedmetadata", "durationchange", "progress", "canplay"] as const;
const timeout = globalThis.setTimeout(() => {
cleanup();
reject(new Error("Archived camera did not become seekable"));
}, 20_000);
let stallTimer: ReturnType<typeof globalThis.setTimeout> | undefined;
const armStallTimer = () => {
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
stallTimer = globalThis.setTimeout(() => {
cleanup();
reject(new Error("Archived camera stream stalled"));
}, 45_000);
};
const cleanup = () => {
globalThis.clearTimeout(timeout);
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
for (const event of events) video.removeEventListener(event, onProgress);
video.removeEventListener("error", onError);
signal.removeEventListener("abort", onAbort);
};
const onProgress = () => {
armStallTimer();
if (!videoHasSeekableArchive(video, declaredDurationSeconds)) return;
cleanup();
resolve();
@@ -384,43 +193,30 @@ function waitForSeekableArchive(
for (const event of events) video.addEventListener(event, onProgress);
video.addEventListener("error", onError, { once: true });
signal.addEventListener("abort", onAbort, { once: true });
armStallTimer();
});
}
async function mountVerifiedRecordedEpoch(
async function mountRecordedEpochStream(
video: HTMLVideoElement,
epoch: VerifiedRecordedMediaEpoch,
descriptor: ObservationRecordedMediaEpoch,
signal: AbortSignal,
): Promise<() => void> {
const descriptor = epoch.descriptor;
if (!descriptor.mediaType.startsWith("video/mp4;") || !video.canPlayType(descriptor.mediaType)) {
throw new Error("Archived camera codec is not supported");
}
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
// A sealed fragmented-MP4 generation is already complete and immutable.
// Present it as one Blob so Chromium can index and seek the whole archive
// without retaining the same 166+ MiB epoch in a quota-limited MSE
// SourceBuffer. Every constituent byte was fetched and SHA-256 verified
// before this boundary, and Blob preserves their canonical order.
const payload = new Blob([epoch.init, ...epoch.segments], {
type: descriptor.mediaType,
});
const expectedByteLength = epoch.init.byteLength + epoch.segments.reduce(
(total, segment) => total + segment.byteLength,
0,
);
if (payload.size !== expectedByteLength) {
throw new Error("Archived camera Blob is incomplete");
}
const objectUrl = URL.createObjectURL(payload);
// The generation token binds the native media request to the exact manifest.
// The browser range-streams the virtual init+fragment file from the server;
// no complete camera archive is copied into JavaScript memory.
const cleanup = () => {
video.pause();
video.removeAttribute("src");
video.load();
URL.revokeObjectURL(objectUrl);
};
video.src = objectUrl;
video.preload = "auto";
video.src = descriptor.streamUrl;
video.load();
try {
await waitForSeekableArchive(
@@ -483,22 +279,15 @@ export function RecordedFmp4Player({
recordedDelivery?.timelineEndSeconds,
],
);
const [archive, setArchive] = useState<VerifiedRecordedMediaArchive | null>(null);
const [archive, setArchive] = useState<RecordedMediaArchive | null>(null);
const [state, setState] = useState<"loading" | "ready" | "error">("loading");
const [readyGeneration, setReadyGeneration] = useState<string | null>(null);
const [progress, setProgress] = useState<RecordedMediaLoadProgress | null>(null);
const [bufferRevision, setBufferRevision] = useState(0);
const currentSeconds = playback?.currentSeconds ?? contract?.timelineStartSeconds ?? 0;
const epoch = useMemo(
() => selectRecordedMediaEpoch(archive?.manifest.epochs ?? [], currentSeconds),
[archive?.manifest.epochs, currentSeconds],
);
const verifiedEpoch = useMemo(
() => epoch
? archive?.epochs.find(({ descriptor }) => descriptor.ordinal === epoch.ordinal) ?? null
: null,
[archive?.epochs, epoch],
);
const waitingForEpoch = Boolean(archive && !epoch);
const selectedGeneration = contract && epoch
? `${contract.manifestGenerationSha256}:${epoch.ordinal}:${epoch.timelineStartSeconds}:${epoch.timelineEndSeconds}`
@@ -514,7 +303,6 @@ export function RecordedFmp4Player({
useEffect(() => {
if (!contract) {
setArchive(null);
setProgress(null);
setReadyGeneration(null);
setState("error");
reportAdmission({
@@ -527,7 +315,6 @@ export function RecordedFmp4Player({
if (!prepare) return;
const abort = new AbortController();
setArchive(null);
setProgress(null);
setReadyGeneration(null);
setState("loading");
reportAdmission({
@@ -535,12 +322,7 @@ export function RecordedFmp4Player({
byteLength: contract.byteLength,
message: null,
});
void fetchVerifiedRecordedMediaArchive(contract, {
signal: abort.signal,
onProgress: (next) => {
if (!abort.signal.aborted) setProgress(next);
},
})
void fetchRecordedMediaArchive(contract, { signal: abort.signal })
.then((loaded) => {
if (abort.signal.aborted) return;
setArchive(loaded);
@@ -566,11 +348,11 @@ export function RecordedFmp4Player({
const abort = new AbortController();
let disposed = false;
void (async () => {
for (const candidate of archive.epochs) {
for (const candidate of archive.manifest.epochs) {
const probe = document.createElement("video");
probe.muted = true;
probe.playsInline = true;
const cleanup = await mountVerifiedRecordedEpoch(probe, candidate, abort.signal);
const cleanup = await mountRecordedEpochStream(probe, candidate, abort.signal);
cleanup();
if (disposed || abort.signal.aborted) return;
}
@@ -602,8 +384,8 @@ export function RecordedFmp4Player({
useEffect(() => {
const video = videoRef.current;
if (!video || !verifiedEpoch) return;
const epochDescriptor = verifiedEpoch.descriptor;
if (!video || !epoch) return;
const epochDescriptor = epoch;
const generation = contract
? `${contract.manifestGenerationSha256}:${epochDescriptor.ordinal}:${epochDescriptor.timelineStartSeconds}:${epochDescriptor.timelineEndSeconds}`
: null;
@@ -615,7 +397,7 @@ export function RecordedFmp4Player({
const loadEpoch = async () => {
try {
cleanup = await mountVerifiedRecordedEpoch(video, verifiedEpoch, abort.signal);
cleanup = await mountRecordedEpochStream(video, epochDescriptor, abort.signal);
if (disposed || abort.signal.aborted) {
cleanup();
cleanup = null;
@@ -648,7 +430,7 @@ export function RecordedFmp4Player({
abort.abort();
cleanup?.();
};
}, [archive?.byteLength, contract, verifiedEpoch]);
}, [archive?.byteLength, contract, epoch]);
useEffect(() => {
const video = videoRef.current;
@@ -679,10 +461,6 @@ export function RecordedFmp4Player({
}
}, [archive?.byteLength, bufferRevision, currentSeconds, epoch, playback?.playing, visualState]);
const progressPercent = progress && progress.totalBytes > 0
? Math.min(100, Math.floor((progress.loadedBytes / progress.totalBytes) * 100))
: 0;
return (
<div
className="recorded-media-player"
@@ -707,8 +485,8 @@ export function RecordedFmp4Player({
: visualState === "error"
? "Записанное видео недоступно"
: archive
? "Проверяем полную готовность записанного видео…"
: `Загружаем и проверяем записанное видео · ${progressPercent}%`}
? "Проверяем seek и codec записанного видео…"
: "Читаем manifest записанного видео…"}
</div>
) : null}
</div>
@@ -4,7 +4,14 @@ import type { SceneSettings } from "../sceneSettings";
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
export type RecordedRerunView = "spatial" | "perception" | "metrics";
export type RecordedRerunView = "spatial" | "perception" | "perception3d" | "metrics";
export interface RecordedPerceptionLayers {
enabled: boolean;
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}
export interface RerunSelection {
entityPath: string;
@@ -57,6 +64,8 @@ export interface RerunViewportProps {
| "customColor"
>;
recordedView?: RecordedRerunView;
recordedViewResetGeneration?: 0 | 1;
recordedPerceptionLayers?: RecordedPerceptionLayers;
onPerceptionAvailabilityChange?: (available: boolean) => void;
}
@@ -438,11 +447,20 @@ export async function fetchRecordedBlueprintRrd(
origin,
signal,
activeView = "spatial",
viewResetGeneration = 0,
perceptionLayers = {
enabled: false,
detections2d: false,
segmentation: false,
cuboids3d: false,
},
fetcher = globalThis.fetch,
}: {
origin: string;
signal?: AbortSignal;
activeView?: RecordedRerunView;
viewResetGeneration?: 0 | 1;
perceptionLayers?: RecordedPerceptionLayers;
fetcher?: typeof globalThis.fetch;
},
): Promise<Uint8Array> {
@@ -461,7 +479,14 @@ export async function fetchRecordedBlueprintRrd(
settings.pointSize > 32 ||
!["turbo", "viridis", "plasma", "grayscale", "custom"].includes(settings.palette) ||
!/^#[0-9A-Fa-f]{6}$/.test(settings.customColor) ||
!["spatial", "perception", "metrics"].includes(activeView) ||
!["spatial", "perception", "perception3d", "metrics"].includes(activeView) ||
![0, 1].includes(viewResetGeneration) ||
[
perceptionLayers.enabled,
perceptionLayers.detections2d,
perceptionLayers.segmentation,
perceptionLayers.cuboids3d,
].some((value) => typeof value !== "boolean") ||
identity.applicationId !== "nodedc_mission_core_recorded" ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(identity.recordingId)
) {
@@ -485,6 +510,11 @@ export async function fetchRecordedBlueprintRrd(
palette: settings.palette,
custom_color: settings.customColor,
active_view: activeView,
view_reset_generation: viewResetGeneration,
unified_perception: perceptionLayers.enabled,
show_detections_2d: perceptionLayers.detections2d,
show_segmentation: perceptionLayers.segmentation,
show_cuboids_3d: perceptionLayers.cuboids3d,
}),
signal,
});
@@ -588,6 +618,13 @@ export function RerunViewport({
onPlaybackControllerChange,
sceneSettings,
recordedView = "spatial",
recordedViewResetGeneration = 0,
recordedPerceptionLayers = {
enabled: false,
detections2d: false,
segmentation: false,
cuboids3d: false,
},
onPerceptionAvailabilityChange,
}: RerunViewportProps) {
const hostRef = useRef<HTMLDivElement>(null);
@@ -596,6 +633,7 @@ export function RerunViewport({
const [retryNonce, setRetryNonce] = useState(0);
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const recordedIdentityRef = useRef<RecordedRerunIdentity | null>(null);
const presentationGateRef = useRef(presentationGate);
presentationGateRef.current = presentationGate;
@@ -1123,11 +1161,11 @@ export function RerunViewport({
]);
useEffect(() => {
onPerceptionAvailabilityChange?.(false);
}, [onPerceptionAvailabilityChange, recordedPerceptionUrl]);
loadedPerceptionChannelRef.current = null;
}, [recordedPerceptionUrl]);
useEffect(() => {
if (!recordedPerceptionUrl) return;
if (!recordedPerceptionUrl || !recordedPerceptionLayers.enabled) return;
const active = perceptionChannelRef.current;
const identity = recordedIdentityRef.current;
if (
@@ -1136,6 +1174,10 @@ export function RerunViewport({
active.endpointUrl !== recordedPerceptionUrl ||
!active.channel.ready
) return;
if (loadedPerceptionChannelRef.current === active) {
onPerceptionAvailabilityChange?.(true);
return;
}
const abort = new AbortController();
void fetchRecordedPerceptionRrd(recordedPerceptionUrl, identity, {
origin: window.location.origin,
@@ -1154,6 +1196,7 @@ export function RerunViewport({
return;
}
active.channel.send_rrd(payload);
loadedPerceptionChannelRef.current = active;
onPerceptionAvailabilityChange?.(true);
}).catch(() => {
onPerceptionAvailabilityChange?.(false);
@@ -1163,6 +1206,7 @@ export function RerunViewport({
}, [
onPerceptionAvailabilityChange,
perceptionChannelRevision,
recordedPerceptionLayers.enabled,
recordedPerceptionUrl,
]);
@@ -1181,6 +1225,8 @@ export function RerunViewport({
origin: window.location.origin,
signal: abort.signal,
activeView: recordedView,
viewResetGeneration: recordedViewResetGeneration,
perceptionLayers: recordedPerceptionLayers,
}).then((payload) => {
if (
abort.signal.aborted ||
@@ -1200,6 +1246,11 @@ export function RerunViewport({
blueprintChannelRevision,
recordedBlueprintUrl,
recordedView,
recordedViewResetGeneration,
recordedPerceptionLayers.enabled,
recordedPerceptionLayers.detections2d,
recordedPerceptionLayers.segmentation,
recordedPerceptionLayers.cuboids3d,
sceneSettings?.accumulationSeconds,
sceneSettings?.customColor,
sceneSettings?.palette,
@@ -38,47 +38,57 @@ export function recordedObservationSources(
spatialRegistration: "native",
},
};
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => ({
id: source.id,
sourceId: source.id,
semanticChannelId: "camera.video.recorded",
label: source.label,
description: "Сохранённый видеоканал на общей временной шкале сессии",
modality: "video",
role: "auxiliary",
availability: "available",
transport: "recording",
endpointLabel: "Сохранённая сессия",
previewUrl: null,
delivery: {
id: `${launch.sessionId}:${source.id}`,
kind: "recorded-fmp4-manifest",
url: source.manifestUrl,
mediaType: source.mediaType,
manifestGenerationSha256: source.manifestGenerationSha256,
byteLength: source.byteLength,
timelineStartSeconds: source.timelineStartSeconds,
timelineEndSeconds: source.timelineEndSeconds,
},
activation: null,
provider: {
pluginId: "missioncore.session-archive",
pluginVersion: "1",
modelId: "recorded-media",
compatibilityProfileId: null,
},
binding: {},
capabilities: {
overlay: true,
fullscreen: true,
resizable: true,
defaultVisible: index < 2,
timelineMode: "recorded",
seekable: true,
sessionRecording: true,
clockId: "session_time",
spatialRegistration: "unresolved",
},
}));
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => {
const perception = source.id.startsWith("recorded.perception.");
return {
id: source.id,
sourceId: source.id,
semanticChannelId: perception
? "camera.perception.panoptic.recorded"
: "camera.video.recorded",
label: source.label,
description: perception
? "Покадровая instance + semantic сегментация на общей временной шкале"
: "Сохранённый видеоканал на общей временной шкале сессии",
modality: "video",
role: "auxiliary",
availability: "available",
transport: "recording",
endpointLabel: "Сохранённая сессия",
previewUrl: null,
delivery: {
id: `${launch.sessionId}:${source.id}`,
kind: "recorded-fmp4-manifest",
url: source.manifestUrl,
mediaType: source.mediaType,
manifestGenerationSha256: source.manifestGenerationSha256,
byteLength: source.byteLength,
timelineStartSeconds: source.timelineStartSeconds,
timelineEndSeconds: source.timelineEndSeconds,
},
activation: null,
provider: {
pluginId: "missioncore.session-archive",
pluginVersion: "1",
modelId: perception ? "recorded-panoptic-perception" : "recorded-media",
compatibilityProfileId: null,
},
binding: {},
capabilities: {
overlay: true,
fullscreen: true,
resizable: true,
// AI presentation belongs to the unified Rerun composition. Keep the
// pre-rendered perception video as an explicit fallback instead of
// opening it as a second, independently controlled stream.
defaultVisible: !perception && index < 2,
timelineMode: "recorded",
seekable: true,
sessionRecording: true,
clockId: "session_time",
spatialRegistration: perception ? "calibrated" : "unresolved",
},
};
});
return [spatial, ...media];
}
@@ -1,12 +1,8 @@
// A device-agnostic frontend safety policy. Sixteen channels covers multi-rig
// vehicles while the independent byte/concurrency limits keep admission
// bounded. One real accepted archive is ~163 MiB for a single camera, so the
// old 128 MiB laboratory ceiling rejected a valid sealed generation before the
// first manifest request. OPFS-backed sealed generations remain the scaling
// path beyond this in-memory policy.
// Camera count and preparation concurrency remain device-agnostic scheduling
// policy. Recorded duration and aggregate bytes are deliberately not admission
// criteria: sealed media is presented through a generation-bound HTTP stream,
// so a one-, three- or ten-hour recording never has to fit in browser memory.
export const MAX_RECORDED_CAMERA_SOURCES = 16;
export const MAX_RECORDED_MEDIA_SOURCE_BYTES = 256 * 1024 * 1024;
export const MAX_RECORDED_SESSION_CAMERA_BYTES = 512 * 1024 * 1024;
export const MAX_CONCURRENT_RECORDED_CAMERA_PREPARATIONS = 1;
export type RecordedAdmissionPhase = "loading" | "ready" | "error";
@@ -32,18 +28,12 @@ export function recordedCameraDescriptorPreflight(
): RecordedAdmissionPhase {
if (sources.length > MAX_RECORDED_CAMERA_SOURCES) return "error";
if (new Set(sources.map(({ id }) => id)).size !== sources.length) return "error";
let totalBytes = 0;
for (const source of sources) {
if (
!source.id ||
!Number.isSafeInteger(source.byteLength) ||
source.byteLength < 1 ||
source.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
source.byteLength < 1
) return "error";
totalBytes += source.byteLength;
if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) {
return "error";
}
}
return "ready";
}
@@ -67,7 +57,6 @@ export function recordedSessionAdmissionPhase(
if (sourceIds.length > MAX_RECORDED_CAMERA_SOURCES || spatialPhase === "error") {
return "error";
}
let totalBytes = 0;
for (const sourceId of sourceIds) {
const camera = cameras[sourceId];
if (!camera || camera.phase === "error") return "error";
@@ -75,13 +64,10 @@ export function recordedSessionAdmissionPhase(
if (camera.byteLength !== null) {
if (
!Number.isSafeInteger(camera.byteLength) ||
camera.byteLength < 1 ||
camera.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
camera.byteLength < 1
) return "error";
totalBytes += camera.byteLength;
}
}
if (totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) return "error";
if (spatialPhase !== "ready") return "loading";
return sourceIds.every((sourceId) => cameras[sourceId]?.phase === "ready")
? "ready"
@@ -1,7 +1,5 @@
import {
MAX_RECORDED_CAMERA_SOURCES,
MAX_RECORDED_MEDIA_SOURCE_BYTES,
MAX_RECORDED_SESSION_CAMERA_BYTES,
} from "./recordedSessionAdmission";
export type ObservationSessionStatus =
@@ -99,19 +97,8 @@ export interface ObservationRecordedMediaEpoch {
timelineStartSeconds: number;
timelineEndSeconds: number;
mediaType: string;
initUrl: string;
initByteLength: number;
initSha256: string;
segmentCount: number;
segmentUrlPrefix: string;
segments: readonly ObservationRecordedMediaSegment[];
}
export interface ObservationRecordedMediaSegment {
sequence: number;
url: string;
byteLength: number;
sha256: string;
streamUrl: string;
}
export interface ObservationRecordedMediaManifest {
@@ -215,18 +202,8 @@ const RECORDED_MEDIA_EPOCH_KEYS = new Set([
"timeline_start_seconds",
"timeline_end_seconds",
"media_type",
"init_url",
"init_byte_length",
"init_sha256",
"segment_count",
"segment_url_prefix",
"segments",
]);
const RECORDED_MEDIA_SEGMENT_KEYS = new Set([
"sequence",
"url",
"byte_length",
"sha256",
"stream_url",
]);
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const SAFE_MODALITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
@@ -234,10 +211,9 @@ const ISO_WITH_TIMEZONE = /^\d{4}-\d{2}-\d{2}T.+(?:Z|[+-]\d{2}:\d{2})$/;
const SHA256 = /^[a-f0-9]{64}$/;
const SAFE_RECORDING_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording\.rrd$/;
const SAFE_PREPARATION_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording-preparation$/;
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/media\/[A-Za-z0-9._%-]+\/manifest$/;
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+|perception-media\/result-[a-f0-9]{64})\/manifest$/;
const SAFE_MEDIA_STREAM_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+\/epochs\/[1-9][0-9]*|perception-media\/result-[a-f0-9]{64})\/recording\.mp4\?generation=[a-f0-9]{64}$/;
const SAFE_MP4_MEDIA_TYPE = /^video\/mp4(?:; codecs="[A-Za-z0-9.,_-]+")?$/;
const MAX_RECORDED_INIT_BYTES = 8 * 1024 * 1024;
const MAX_RECORDED_SEGMENT_BYTES = 64 * 1024 * 1024;
export class ObservationSessionContractError extends Error {
constructor(message: string) {
@@ -570,15 +546,6 @@ export function decodeObservationSessionReplay(
"Descriptor записи содержит повторяющиеся медиаканалы.",
);
}
const mediaByteLength = mediaSources.reduce((total, source) => total + source.byteLength, 0);
if (
!Number.isSafeInteger(mediaByteLength) ||
mediaByteLength > MAX_RECORDED_SESSION_CAMERA_BYTES
) {
throw new ObservationSessionContractError(
"Совокупный объём записанных медиаканалов превышает безопасный лимит браузера.",
);
}
return {
kind: "rerun-recording",
sessionId,
@@ -709,7 +676,10 @@ function decodeRecordedMediaSource(
}
assertExactKeys(value, RECORDED_MEDIA_SOURCE_KEYS, `Медиаканал ${index}`);
const id = requireString(value.id, `media_sources[${index}].id`, 128);
if (!SAFE_ID.test(id) || !id.startsWith("recorded.camera.")) {
if (
!SAFE_ID.test(id) ||
(!id.startsWith("recorded.camera.") && !id.startsWith("recorded.perception."))
) {
throw new ObservationSessionContractError("Медиаканал содержит небезопасный opaque id.");
}
if (value.modality !== "video" || value.media_type !== "video/mp4") {
@@ -728,10 +698,10 @@ function decodeRecordedMediaSource(
"Медиаканал не содержит immutable generation SHA-256.",
);
}
const sessionPrefix = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/media/`;
const sessionBase = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/`;
if (
!SAFE_MEDIA_MANIFEST_URL.test(manifestUrl) ||
!manifestUrl.startsWith(sessionPrefix) ||
!manifestUrl.startsWith(sessionBase) ||
manifestUrl.includes("..")
) {
throw new ObservationSessionContractError(
@@ -765,7 +735,7 @@ function decodeRecordedMediaSource(
byteLength: requireFiniteNumber(
value.byte_length,
`media_sources[${index}].byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES, integer: true },
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
),
mediaType: "video/mp4",
timelineStartSeconds,
@@ -784,22 +754,21 @@ export function decodeObservationRecordedMediaManifest(
}
assertExactKeys(payload, RECORDED_MEDIA_MANIFEST_KEYS, "Manifest записанного видео");
if (
payload.schema_version !== "missioncore.observation-recorded-media/v2" ||
payload.schema_version !== "missioncore.observation-recorded-media/v3" ||
payload.source_id !== source.id ||
typeof payload.generation_sha256 !== "string" ||
!SHA256.test(payload.generation_sha256) ||
payload.generation_sha256 !== source.manifestGenerationSha256 ||
payload.synchronization !== "host-arrival-best-effort" ||
!Array.isArray(payload.epochs) ||
payload.epochs.length < 1 ||
payload.epochs.length > 1_000
payload.epochs.length < 1
) {
throw new ObservationSessionContractError("Manifest записанного видео несовместим.");
}
const manifestBase = source.manifestUrl.slice(0, -"/manifest".length);
const manifestByteLength = requireFiniteNumber(payload.byte_length, "manifest.byte_length", {
minimum: 1,
maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES,
maximum: Number.MAX_SAFE_INTEGER,
integer: true,
});
if (manifestByteLength !== source.byteLength) {
@@ -833,7 +802,7 @@ export function decodeObservationRecordedMediaManifest(
assertExactKeys(entry, RECORDED_MEDIA_EPOCH_KEYS, `Codec epoch ${index}`);
const ordinal = requireFiniteNumber(entry.ordinal, `epochs[${index}].ordinal`, {
minimum: 1,
maximum: 1_000,
maximum: Number.MAX_SAFE_INTEGER,
integer: true,
});
if (ordinal !== index + 1) {
@@ -860,87 +829,28 @@ export function decodeObservationRecordedMediaManifest(
if (!SAFE_MP4_MEDIA_TYPE.test(mediaType)) {
throw new ObservationSessionContractError("Codec epoch содержит небезопасный media type.");
}
const initUrl = requireString(entry.init_url, `epochs[${index}].init_url`, 512);
const segmentUrlPrefix = requireString(
entry.segment_url_prefix,
`epochs[${index}].segment_url_prefix`,
512,
const byteLength = requireFiniteNumber(
entry.byte_length,
`epochs[${index}].byte_length`,
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
);
const expectedEpochBase = `${manifestBase}/epochs/${ordinal}`;
const streamUrl = requireString(entry.stream_url, `epochs[${index}].stream_url`, 768);
const expectedEpochBase = source.id.startsWith("recorded.perception.")
? manifestBase
: `${manifestBase}/epochs/${ordinal}`;
const expectedStreamUrl =
`${expectedEpochBase}/recording.mp4?generation=${payload.generation_sha256}`;
if (
initUrl !== `${expectedEpochBase}/init.mp4` ||
segmentUrlPrefix !== `${expectedEpochBase}/segments/` ||
initUrl.includes("..") ||
segmentUrlPrefix.includes("..")
!SAFE_MEDIA_STREAM_URL.test(streamUrl) ||
streamUrl !== expectedStreamUrl ||
streamUrl.includes("..")
) {
throw new ObservationSessionContractError("Codec epoch содержит небезопасный API URL.");
}
const initByteLength = requireFiniteNumber(
entry.init_byte_length,
`epochs[${index}].init_byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_INIT_BYTES, integer: true },
);
if (typeof entry.init_sha256 !== "string" || !SHA256.test(entry.init_sha256)) {
throw new ObservationSessionContractError("Codec epoch не содержит SHA-256 init-сегмента.");
}
const segmentCount = requireFiniteNumber(
entry.segment_count,
`epochs[${index}].segment_count`,
{ minimum: 1, maximum: 500_000, integer: true },
);
if (!Array.isArray(entry.segments) || entry.segments.length !== segmentCount) {
declaredBytes += byteLength;
if (!Number.isSafeInteger(declaredBytes)) {
throw new ObservationSessionContractError(
"Codec epoch содержит неполный список канонических сегментов.",
);
}
const segments = entry.segments.map((segment, segmentIndex): ObservationRecordedMediaSegment => {
if (!isRecord(segment)) {
throw new ObservationSessionContractError(
`Сегмент epochs[${index}].segments[${segmentIndex}] должен быть объектом.`,
);
}
assertExactKeys(
segment,
RECORDED_MEDIA_SEGMENT_KEYS,
`Сегмент epochs[${index}].segments[${segmentIndex}]`,
);
const sequence = requireFiniteNumber(
segment.sequence,
`epochs[${index}].segments[${segmentIndex}].sequence`,
{ minimum: 1, maximum: 500_000, integer: true },
);
if (sequence !== segmentIndex + 1) {
throw new ObservationSessionContractError(
"Codec epoch содержит непоследовательный сегмент.",
);
}
const url = requireString(
segment.url,
`epochs[${index}].segments[${segmentIndex}].url`,
512,
);
if (url !== `${segmentUrlPrefix}${sequence}.m4s` || url.includes("..")) {
throw new ObservationSessionContractError(
"Codec epoch содержит небезопасный URL сегмента.",
);
}
const byteLength = requireFiniteNumber(
segment.byte_length,
`epochs[${index}].segments[${segmentIndex}].byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_SEGMENT_BYTES, integer: true },
);
if (typeof segment.sha256 !== "string" || !SHA256.test(segment.sha256)) {
throw new ObservationSessionContractError(
"Codec epoch содержит сегмент без SHA-256.",
);
}
declaredBytes += byteLength;
return { sequence, url, byteLength, sha256: segment.sha256 };
});
declaredBytes += initByteLength;
if (declaredBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES) {
throw new ObservationSessionContractError(
"Записанный медиаканал превышает безопасный лимит браузера.",
"Суммарный размер codec epoch выходит за точный числовой диапазон клиента.",
);
}
return {
@@ -948,17 +858,13 @@ export function decodeObservationRecordedMediaManifest(
timelineStartSeconds,
timelineEndSeconds,
mediaType,
initUrl,
initByteLength,
initSha256: entry.init_sha256,
segmentCount,
segmentUrlPrefix,
segments,
byteLength,
streamUrl,
};
});
if (declaredBytes !== manifestByteLength) {
throw new ObservationSessionContractError(
"Сумма init- и media-сегментов не совпадает с размером immutable manifest.",
"Сумма codec epoch не совпадает с размером immutable manifest.",
);
}
if (
@@ -285,11 +285,16 @@ export async function resolveObservationSessionReplay(
sessionId: string,
options: ObservationPreparationPollingOptions,
): Promise<ObservationSessionReplayLaunch> {
const response = await withRequestTimeout(
options.signal,
Math.max(100, options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS),
(signal) => replayObservationSession(sessionId, { signal, fetcher: options.fetcher }),
);
// The initial replay request may restore and integrity-check large, already
// published artifacts before it can return either a launch descriptor or a
// background-preparation handle. Its duration therefore scales with the
// recording package and must not be confused with a stalled status poll.
// Keep it cancellable by the owning UI attempt, but do not impose the short
// per-poll timeout used once the server has returned a preparation handle.
const response = await replayObservationSession(sessionId, {
signal: options.signal,
fetcher: options.fetcher,
});
if (response.kind === "ready") return response.launch;
if (!pendingPreparation(response.preparation)) {
options.onUpdate?.(response.preparation);
@@ -22,6 +22,9 @@ export interface ViewerSettings {
show_points: boolean;
show_trajectory: boolean;
show_grid: boolean;
show_detections_2d: boolean;
show_segmentation: boolean;
show_cuboids_3d: boolean;
}
export interface StreamMetrics {
@@ -29,6 +32,10 @@ export interface StreamMetrics {
frameRateHz?: number | null;
pointCount?: number | null;
droppedPreviewFrames?: number | null;
aiLatencyMs?: number | null;
aiFrameRateHz?: number | null;
aiDroppedFrames?: number | null;
aiStaleMs?: number | null;
elapsedSeconds?: number | null;
routeDistanceMeters?: number | null;
speedMetersPerSecond?: number | null;
@@ -32,6 +32,25 @@
backdrop-filter: blur(18px);
}
.scene-navigation-hint {
position: absolute;
z-index: 11;
right: 0.85rem;
bottom: 4.9rem;
display: flex;
align-items: center;
gap: 0.75rem;
border: 1px solid rgb(255 255 255 / 0.07);
border-radius: 999px;
background: rgb(9 10 13 / 0.7);
color: var(--nodedc-text-muted);
padding: 0.42rem 0.65rem;
font-size: 0.52rem;
font-weight: 680;
pointer-events: none;
backdrop-filter: blur(14px);
}
.scene-source-picker__trigger {
position: relative;
}
@@ -350,6 +369,11 @@ i[data-availability="error"] {
box-shadow: 0 1.5rem 4rem rgb(0 0 0 / 0.34);
}
.floating-observation-window--hidden {
visibility: hidden;
pointer-events: none;
}
.observation-timeline {
display: block;
min-width: 0;
@@ -25,6 +25,20 @@
gap: 0.45rem;
}
.spatial-toolbar__view-switch {
display: inline-flex;
align-items: center;
gap: 0.25rem;
border: 1px solid rgb(255 255 255 / 0.07);
border-radius: 999px;
background: rgb(255 255 255 / 0.025);
padding: 0.2rem;
}
.spatial-toolbar__view-switch .nodedc-button {
border-radius: 999px;
}
.spatial-viewport-shell {
position: relative;
min-width: 0;
@@ -45,7 +45,6 @@ import {
type RerunPlaybackState,
type RerunSelection,
type RerunViewportStatus,
type RecordedRerunView,
} from "../components/RerunViewport";
import {
capabilityStatusLabel,
@@ -124,6 +123,16 @@ export interface WorkspaceRendererProps {
accumulationSeconds: number;
onAccumulationChange: (value: number) => void;
onAccumulationCommit: () => void;
livePerceptionLayers: {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
};
onLivePerceptionLayersChange: (next: {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}) => void;
observationLayout: ObservationLayoutController;
navigation: WorkspaceNavigation;
spatialControls: {
@@ -286,6 +295,8 @@ function SpatialWorkspace({
accumulationSeconds,
onAccumulationChange,
onAccumulationCommit,
livePerceptionLayers,
onLivePerceptionLayersChange,
observationLayout,
navigation,
spatialControls,
@@ -295,8 +306,13 @@ function SpatialWorkspace({
const [selection, setSelection] = useState<RerunSelection | null>(null);
const [playbackState, setPlaybackState] = useState<RerunPlaybackState | null>(null);
const [playbackController, setPlaybackController] = useState<RerunPlaybackController | null>(null);
const [recordedRerunView, setRecordedRerunView] = useState<RecordedRerunView>("spatial");
const [perceptionAvailable, setPerceptionAvailable] = useState(false);
const [recordedViewResetGeneration, setRecordedViewResetGeneration] = useState<0 | 1>(0);
const [perceptionAvailability, setPerceptionAvailability] = useState<
"unknown" | "available" | "unavailable"
>("unknown");
const [showDetections2d, setShowDetections2d] = useState(false);
const [showSegmentation, setShowSegmentation] = useState(false);
const [showCuboids3d, setShowCuboids3d] = useState(false);
const recordedSource = state?.sourceMode === "replay" || /\.rrd(?:$|[?#])/i.test(sourceUrl);
const recordedSessionGate: RecordedAdmissionPhase = recordedSource
? recordedSessionAdmission?.phase ?? "loading"
@@ -309,6 +325,8 @@ function SpatialWorkspace({
const latency = pipelineLatency(metrics);
const frameRate = finiteMetric(metrics?.frameRateHz);
const points = finiteMetric(metrics?.pointCount);
const aiLatency = finiteMetric(metrics?.aiLatencyMs);
const aiFrameRate = finiteMetric(metrics?.aiFrameRateHz);
const observationSources = state?.observationSources ?? [];
const pointCloudSource = observationSources.find((source) => source.modality === "point-cloud");
const pointCloudVisible = pointCloudSource
@@ -317,13 +335,29 @@ function SpatialWorkspace({
const mediaSources = observationSources.filter(
(source) => source.capabilities.overlay && source.modality !== "point-cloud",
);
const recordedPerceptionSupported = recordedSource && perceptionAvailability !== "unavailable";
const recordedPerceptionEnabled = showDetections2d || showSegmentation || showCuboids3d;
const unifiedPerception = recordedPerceptionSupported && recordedPerceptionEnabled;
const livePerceptionAvailable = !recordedSource && streamActive;
const detections2dActive = recordedSource
? showDetections2d
: livePerceptionLayers.detections2d;
const segmentationActive = recordedSource
? showSegmentation
: livePerceptionLayers.segmentation;
const cuboids3dActive = recordedSource
? showCuboids3d
: livePerceptionLayers.cuboids3d;
const visibleMediaSources = mediaSources.filter((source) =>
observationLayout.visibleSourceIds.has(source.id),
observationLayout.visibleSourceIds.has(source.id) &&
!source.id.startsWith("recorded.perception."),
);
const presentedMediaSourceCount = unifiedPerception ? 0 : visibleMediaSources.length;
const pointCloudFocused = Boolean(
pointCloudSource && observationLayout.focusedSourceId === pointCloudSource.id,
);
const floatingSourceMaximized = Boolean(observationLayout.maximizedFloatingSourceId);
const floatingSourceMaximized = !unifiedPerception &&
Boolean(observationLayout.maximizedFloatingSourceId);
const timeline = state?.observationTimeline;
const viewportRef = useRef<HTMLDivElement>(null);
const intentionalSourceEnd = !recordedSource && [
@@ -373,13 +407,20 @@ function SpatialWorkspace({
[],
);
const onPerceptionAvailabilityChange = useCallback((available: boolean) => {
setPerceptionAvailable(available);
if (!available) setRecordedRerunView("spatial");
setPerceptionAvailability(available ? "available" : "unavailable");
if (!available) {
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
}
}, []);
useEffect(() => {
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
setPerceptionAvailability("unknown");
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
setRecordedViewResetGeneration(0);
}, [sourceUrl]);
useEffect(() => {
@@ -389,8 +430,10 @@ function SpatialWorkspace({
setSelection(null);
setPlaybackState(null);
setPlaybackController(null);
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
setPerceptionAvailability("unknown");
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
}, [pointCloudVisible, sourceUrl]);
useEffect(() => {
@@ -428,20 +471,74 @@ function SpatialWorkspace({
className="spatial-workspace"
data-focused={pointCloudFocused || floatingSourceMaximized ? "true" : undefined}
>
<div className="spatial-toolbar">
<div className="spatial-toolbar" data-viewer-controls="v1">
<div className="spatial-toolbar__mode">
<span className="section-eyebrow">СЦЕНА 3D · RERUN</span>
</div>
<div className="spatial-toolbar__actions">
{recordedSource && perceptionAvailable ? (
{recordedPerceptionSupported || livePerceptionAvailable ? (
<div className="spatial-toolbar__view-switch" role="group" aria-label="Слои распознавания сцены">
<Button
size="compact"
variant="primary"
icon={<Icon name="video" />}
aria-pressed="true"
disabled
>
Оригинал
</Button>
<Button
size="compact"
variant={detections2dActive ? "primary" : "secondary"}
icon={<Icon name="target" />}
aria-pressed={detections2dActive}
onClick={() => recordedSource
? setShowDetections2d((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
detections2d: !livePerceptionLayers.detections2d,
})}
>
Объекты 2D
</Button>
<Button
size="compact"
variant={segmentationActive ? "primary" : "secondary"}
icon={<Icon name="image" />}
aria-pressed={segmentationActive}
onClick={() => recordedSource
? setShowSegmentation((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
segmentation: !livePerceptionLayers.segmentation,
})}
>
Сегментация
</Button>
<Button
size="compact"
variant={cuboids3dActive ? "primary" : "secondary"}
icon={<Icon name="apps" />}
aria-pressed={cuboids3dActive}
onClick={() => recordedSource
? setShowCuboids3d((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
cuboids3d: !livePerceptionLayers.cuboids3d,
})}
>
Кубы 3D
</Button>
</div>
) : null}
{recordedSource && presentedViewerStatus === "ready" ? (
<Button
size="compact"
variant={recordedRerunView === "perception" ? "primary" : "secondary"}
icon={<Icon name={recordedRerunView === "perception" ? "globe" : "image"} />}
onClick={() => setRecordedRerunView((current) =>
current === "perception" ? "spatial" : "perception")}
variant="secondary"
icon={<Icon name="refresh" />}
onClick={() => setRecordedViewResetGeneration((current) => current === 0 ? 1 : 0)}
>
{recordedRerunView === "perception" ? "Облако точек" : "Распознавание"}
Сброс вида
</Button>
) : null}
<Button size="compact" variant="secondary" icon={<Icon name="network" />} onClick={navigation.openSource}>
@@ -476,7 +573,13 @@ function SpatialWorkspace({
? state?.observationTimeline?.range?.endSeconds
: undefined}
sceneSettings={sceneSettings}
recordedView={recordedRerunView}
recordedViewResetGeneration={recordedViewResetGeneration}
recordedPerceptionLayers={{
enabled: unifiedPerception,
detections2d: showDetections2d,
segmentation: showSegmentation,
cuboids3d: showCuboids3d,
}}
onPerceptionAvailabilityChange={onPerceptionAvailabilityChange}
onStatusChange={onStatusChange}
onSelectionChange={onSelectionChange}
@@ -511,7 +614,9 @@ function SpatialWorkspace({
) : !floatingSourceMaximized ? (
<div className="scene-source-controls">
<ObservationSourcePicker
sources={observationSources}
sources={unifiedPerception
? observationSources.filter((source) => source.modality === "point-cloud")
: observationSources}
visibleSourceIds={observationLayout.visibleSourceIds}
pendingSourceIds={observationLayout.pendingSourceIds}
onToggle={observationLayout.toggleSource}
@@ -555,6 +660,16 @@ function SpatialWorkspace({
<span>До публикации</span>
<strong>{formatNumber(latency)}<small> мс</small></strong>
</div>
{streamActive ? (
<div>
<span>AI</span>
<strong>
{aiLatency === null ? "—" : formatNumber(aiLatency)}
<small>{aiLatency === null ? "" : " мс"}</small>
{aiFrameRate === null ? null : <small> · {formatNumber(aiFrameRate)} Гц</small>}
</strong>
</div>
) : null}
</div>
{!pointCloudFocused && !floatingSourceMaximized && state?.sourceMode && state.sourceMode !== "idle" && !sourceUrl.trim() ? (
@@ -575,10 +690,17 @@ function SpatialWorkspace({
</div>
) : null}
{presentedViewerStatus === "ready" && !floatingSourceMaximized ? (
<div className="scene-navigation-hint" aria-label="Навигация по 3D-сцене">
<span>Колесо · зум к курсору</span>
<span>WASD · свободный проход</span>
</div>
) : null}
{!pointCloudFocused && !floatingSourceMaximized && recordedPlaybackReady ? (
<ObservationTimeline
active={presentedViewerStatus === "ready"}
sourceCount={Math.max(1, 1 + visibleMediaSources.length)}
sourceCount={Math.max(1, (unifiedPerception ? 2 : 1) + presentedMediaSourceCount)}
mode={recordedSource && playbackState?.rangeNs
? "recorded"
: timeline?.mode}
@@ -599,7 +721,7 @@ function SpatialWorkspace({
/>
) : null}
{!pointCloudFocused ? visibleMediaSources.map((source, index) => (
{visibleMediaSources.map((source, index) => (
<FloatingObservationWindow
key={source.id}
source={source}
@@ -609,6 +731,7 @@ function SpatialWorkspace({
rect={observationLayout.windowRects[source.id]}
maximized={observationLayout.maximizedFloatingSourceId === source.id}
active={observationLayout.activeFloatingSourceId === source.id}
hidden={pointCloudFocused || unifiedPerception}
onRectChange={(rect) => observationLayout.setWindowRect(source.id, rect)}
onMaximizedChange={(maximized) =>
observationLayout.setFloatingMaximized(source.id, maximized)}
@@ -627,12 +750,12 @@ function SpatialWorkspace({
void observationLayout.hideSource(source.id);
}}
/>
)) : null}
))}
{recordedSource ? (
<div className="recorded-session-preloaders" aria-hidden="true">
{mediaSources.filter((source) => (
source.delivery?.kind === "recorded-fmp4-manifest" &&
(pointCloudFocused || !observationLayout.visibleSourceIds.has(source.id)) &&
!observationLayout.visibleSourceIds.has(source.id) &&
shouldPrepareRecordedSource(source.id)
)).map((source) => (
<ObservationMedia
@@ -657,7 +780,9 @@ function SpatialWorkspace({
<span><i data-state="ready" />Траектория</span>
<span><i data-state="ready" />Преобразования</span>
<span><i data-state="contract" />Камеры в 3D</span>
<span><i data-state={perceptionAvailable ? "ready" : "contract"} />Объекты / рамки</span>
<span><i data-state={detections2dActive ? "ready" : "contract"} />Объекты 2D</span>
<span><i data-state={segmentationActive ? "ready" : "contract"} />Сегментация</span>
<span><i data-state={cuboids3dActive ? "ready" : "contract"} />Кубы 3D</span>
<span><i data-state="contract" />Компоновка</span>
</div>
</div>
@@ -484,6 +484,31 @@ test("202 preparation polls through explicit phases and reveals launch only when
assert.deepEqual(calls.map((entry) => entry[2]), [null, preparationEtag, preparationEtag]);
});
test("initial replay restore is not mistaken for a stalled preparation poll", async () => {
const sessionId = "session-20260716T205632Z";
let requestSignal;
const launch = await resolveObservationSessionReplay(sessionId, {
signal: new AbortController().signal,
requestTimeoutMs: 100,
fetcher: async (_input, init) => {
requestSignal = init.signal;
await new Promise((resolve, reject) => {
const timer = setTimeout(resolve, 150);
init.signal.addEventListener("abort", () => {
clearTimeout(timer);
reject(new DOMException("cancelled", "AbortError"));
}, { once: true });
});
return new Response(JSON.stringify(replay({ session_id: sessionId })), {
status: 200,
headers: { "Content-Type": "application/json" },
});
},
});
assert.equal(requestSignal.aborted, false);
assert.equal(launch.sessionId, sessionId);
});
test("status-ready response must match the preparation ETag before launch is accepted", async () => {
const decoded = decodeObservationSessionPreparation(
preparation({ state: "finalizing", progress: 0.95 }),
@@ -704,6 +729,29 @@ test("switching saved sessions aborts only local polling and never cancels share
assert.doesNotMatch(selectorSource, /cancelReplay|>\s*Отменить\s*</);
});
test("recording preparation never presents phase heartbeats as fake percentages", async () => {
const selectorSource = await readFile(
new URL("../src/components/ObservationSessionSelect.tsx", import.meta.url),
"utf8",
);
assert.match(selectorSource, /exporting:\s*["']Готовим операторскую сцену["']/);
assert.doesNotMatch(selectorSource, /Math\.round\(progress\s*\*\s*100\)/);
});
test("unified recorded AI view keeps the raw camera mounted but does not cover overlays", async () => {
const workspaceSource = await readFile(
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
"utf8",
);
assert.match(workspaceSource, /hidden=\{pointCloudFocused \|\| unifiedPerception\}/);
assert.match(
workspaceSource,
/presentedMediaSourceCount = unifiedPerception \? 0 : visibleMediaSources\.length/,
);
});
test("replay descriptor rejects path leaks, mismatched sessions and non-seekable data", () => {
assert.throws(
() => decodeObservationSessionReplay(replay({ source_url: "file:///private/session.rrd" })),
@@ -776,7 +824,7 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
);
const manifest = decodeObservationRecordedMediaManifest({
schema_version: "missioncore.observation-recorded-media/v2",
schema_version: "missioncore.observation-recorded-media/v3",
source_id: source.id,
generation_sha256: "c".repeat(64),
byte_length: 3_266,
@@ -788,21 +836,15 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
timeline_start_seconds: 0.25,
timeline_end_seconds: 20,
media_type: 'video/mp4; codecs="avc1.640028"',
init_url: source.manifest_url.replace("/manifest", "/epochs/1/init.mp4"),
init_byte_length: 128,
init_sha256: "a".repeat(64),
segment_count: 12,
segment_url_prefix: source.manifest_url.replace("/manifest", "/epochs/1/segments/"),
segments: Array.from({ length: 12 }, (_, index) => ({
sequence: index + 1,
url: source.manifest_url.replace("/manifest", `/epochs/1/segments/${index + 1}.m4s`),
byte_length: 256 + index,
sha256: "b".repeat(64),
})),
byte_length: 3_266,
stream_url: source.manifest_url.replace(
"/manifest",
`/epochs/1/recording.mp4?generation=${"c".repeat(64)}`,
),
}],
}, decoded.mediaSources[0]);
assert.equal(manifest.epochs[0].segmentCount, 12);
assert.equal(manifest.epochs[0].segments.length, 12);
assert.equal(manifest.epochs[0].byteLength, 3_266);
assert.match(manifest.epochs[0].streamUrl, /recording\.mp4\?generation=/);
assert.equal(manifest.epochs[0].timelineStartSeconds, 0.25);
assert.throws(
() => decodeObservationRecordedMediaManifest({
@@ -821,7 +863,7 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
);
assert.throws(
() => decodeObservationRecordedMediaManifest({
schema_version: "missioncore.observation-recorded-media/v2",
schema_version: "missioncore.observation-recorded-media/v3",
source_id: source.id,
generation_sha256: "c".repeat(64),
byte_length: 3_267,
@@ -834,6 +876,53 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
);
});
test("replay decodes a synchronized full-epoch perception video", () => {
const sessionId = "20260720T065719Z_viewer_live";
const resultId = `result-${"d".repeat(64)}`;
const generation = "e".repeat(64);
const source = {
id: "recorded.perception.right",
label: "Сегментация · камера right",
modality: "video",
manifest_url: `/api/v1/observation-sessions/${sessionId}/perception-media/${resultId}/manifest`,
manifest_generation_sha256: generation,
byte_length: 12_345_678,
media_type: "video/mp4",
timeline_start_seconds: 35.421857292,
timeline_end_seconds: 484.144857292,
seekable: true,
synchronization: "host-arrival-best-effort",
};
const decoded = decodeObservationSessionReplay(replay({
session_id: sessionId,
timeline_end_seconds: 500,
media_sources: [source],
}));
assert.equal(decoded.mediaSources[0].id, "recorded.perception.right");
assert.equal(decoded.mediaSources[0].manifestUrl, source.manifest_url);
const manifest = decodeObservationRecordedMediaManifest({
schema_version: "missioncore.observation-recorded-media/v3",
source_id: source.id,
generation_sha256: generation,
byte_length: source.byte_length,
timeline_start_seconds: source.timeline_start_seconds,
timeline_end_seconds: source.timeline_end_seconds,
synchronization: "host-arrival-best-effort",
epochs: [{
ordinal: 1,
timeline_start_seconds: source.timeline_start_seconds,
timeline_end_seconds: source.timeline_end_seconds,
media_type: 'video/mp4; codecs="avc1.640028"',
byte_length: source.byte_length,
stream_url: `/api/v1/observation-sessions/${sessionId}/perception-media/${resultId}/recording.mp4?generation=${generation}`,
}],
}, decoded.mediaSources[0]);
assert.equal(manifest.epochs.length, 1);
assert.equal(manifest.epochs[0].timelineStartSeconds, source.timeline_start_seconds);
assert.equal(manifest.epochs[0].timelineEndSeconds, source.timeline_end_seconds);
});
test("recorded camera contracts reject foreign origins, path escapes and unknown fields", () => {
const sessionId = "session-20260716T205632Z";
const base = {
@@ -888,7 +977,7 @@ test("recorded camera contracts reject foreign origins, path escapes and unknown
}));
assert.throws(
() => decodeObservationRecordedMediaManifest({
schema_version: "missioncore.observation-recorded-media/v2",
schema_version: "missioncore.observation-recorded-media/v3",
source_id: base.id,
generation_sha256: "c".repeat(64),
byte_length: 384,
@@ -900,17 +989,8 @@ test("recorded camera contracts reject foreign origins, path escapes and unknown
timeline_start_seconds: 0,
timeline_end_seconds: 20,
media_type: 'video/mp4; codecs="avc1.640028"',
init_url: "file:///private/init.mp4",
init_byte_length: 128,
init_sha256: "a".repeat(64),
segment_count: 1,
segment_url_prefix: "file:///private/segments/",
segments: [{
sequence: 1,
url: "file:///private/segments/1.m4s",
byte_length: 256,
sha256: "b".repeat(64),
}],
byte_length: 384,
stream_url: "file:///private/recording.mp4",
}],
}, decoded.mediaSources[0]),
/небезопасный API URL/,
@@ -269,7 +269,14 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
{ applicationId: "nodedc_mission_core_recorded", recordingId: "recording-001" },
{
origin: "http://127.0.0.1:5174",
activeView: "perception",
activeView: "perception3d",
viewResetGeneration: 1,
perceptionLayers: {
enabled: true,
detections2d: true,
segmentation: false,
cuboids3d: true,
},
fetcher: async (input, init) => {
calls.push({ input: String(input), init, body: JSON.parse(String(init.body)) });
return new Response(payload, {
@@ -292,7 +299,12 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
point_size: 4.5,
palette: "custom",
custom_color: "#35d7c1",
active_view: "perception",
active_view: "perception3d",
view_reset_generation: 1,
unified_perception: true,
show_detections_2d: true,
show_segmentation: false,
show_cuboids_3d: true,
});
await assert.rejects(
@@ -665,7 +677,7 @@ test("camera delivery rejects literal and encoded endpoint or credential leaks",
"/api/preview?upstream=rtsp://camera.local/live",
"/api/preview?upstream=rtsp%3A%2F%2Fcamera.local%2Flive",
"/api/preview?upstream=rtsp%253A%252F%252Fcamera.local%252Flive",
"/api/preview?endpoint=192.168.68.52:8554",
"/api/preview?endpoint=192.0.2.52:8554",
"/api/preview?endpoint=192%2E168%2E68%2E52",
"/api/preview?password=not-for-the-browser",
"/api/preview?%70%61%73%73%77%6f%72%64=not-for-the-browser",
@@ -1,15 +1,13 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { after, before, test } from "node:test";
import { createServer } from "vite";
let server;
let fetchVerifiedRecordedMediaArchive;
let fetchRecordedMediaArchive;
let recordedMediaPresentationState;
let recordedMediaSeekableCoverage;
let appendRecordedMediaBuffer;
before(async () => {
server = await createServer({
@@ -18,10 +16,9 @@ before(async () => {
server: { middlewareMode: true },
});
({
fetchVerifiedRecordedMediaArchive,
fetchRecordedMediaArchive,
recordedMediaPresentationState,
recordedMediaSeekableCoverage,
appendRecordedMediaBuffer,
} = await server.ssrLoadModule("/src/components/RecordedFmp4Player.tsx"));
});
@@ -29,41 +26,28 @@ after(async () => {
await server?.close();
});
function digest(payload) {
return createHash("sha256").update(payload).digest("hex");
}
function fixture() {
function fixture({ byteLength = 36_000_000_000 } = {}) {
const manifestUrl = "/api/v1/observation-sessions/session-1/media/camera-1/manifest";
const init = Buffer.from("canonical-init");
const first = Buffer.from("canonical-first-fragment");
const second = Buffer.from("canonical-second-fragment");
const generation = "a".repeat(64);
const segmentPrefix = manifestUrl.replace("/manifest", "/epochs/1/segments/");
const streamUrl = manifestUrl.replace(
"/manifest",
`/epochs/1/recording.mp4?generation=${generation}`,
);
const manifest = {
schema_version: "missioncore.observation-recorded-media/v2",
schema_version: "missioncore.observation-recorded-media/v3",
source_id: "recorded.camera.camera-1",
generation_sha256: generation,
byte_length: init.byteLength + first.byteLength + second.byteLength,
byte_length: byteLength,
timeline_start_seconds: 0,
timeline_end_seconds: 20,
timeline_end_seconds: 36_000,
synchronization: "host-arrival-best-effort",
epochs: [{
ordinal: 1,
timeline_start_seconds: 0,
timeline_end_seconds: 20,
timeline_end_seconds: 36_000,
media_type: 'video/mp4; codecs="avc1.640028"',
init_url: manifestUrl.replace("/manifest", "/epochs/1/init.mp4"),
init_byte_length: init.byteLength,
init_sha256: digest(init),
segment_count: 2,
segment_url_prefix: segmentPrefix,
segments: [first, second].map((payload, index) => ({
sequence: index + 1,
url: `${segmentPrefix}${index + 1}.m4s`,
byte_length: payload.byteLength,
sha256: digest(payload),
})),
byte_length: byteLength,
stream_url: streamUrl,
}],
};
const source = {
@@ -72,14 +56,14 @@ function fixture() {
modality: "video",
manifestUrl,
manifestGenerationSha256: generation,
byteLength: manifest.byte_length,
byteLength,
mediaType: "video/mp4",
timelineStartSeconds: 0,
timelineEndSeconds: 20,
timelineEndSeconds: 36_000,
seekable: true,
synchronization: "host-arrival-best-effort",
};
return { source, manifest, generation, init, first, second };
return { source, manifest, generation, streamUrl };
}
function jsonResponse(payload, generation) {
@@ -92,145 +76,54 @@ function jsonResponse(payload, generation) {
});
}
function mediaResponse(payload, sha = digest(payload), length = payload.byteLength) {
return new Response(payload, {
status: 200,
headers: {
"Content-Length": String(length),
ETag: `"sha256:${sha}"`,
test("multi-hour camera admission fetches only its compact generation-bound manifest", async () => {
const { source, manifest, generation, streamUrl } = fixture();
const requested = [];
const archive = await fetchRecordedMediaArchive(source, {
fetcher: async (input, request = {}) => {
requested.push(String(input));
assert.equal(new Headers(request.headers).get("If-Match"), `"sha256:${generation}"`);
return jsonResponse(manifest, generation);
},
});
}
function deferred() {
let resolve;
let reject;
const promise = new Promise((resolvePromise, rejectPromise) => {
resolve = resolvePromise;
reject = rejectPromise;
});
return { promise, resolve, reject };
}
async function flushAsync() {
await new Promise((resolve) => setTimeout(resolve, 0));
await new Promise((resolve) => setTimeout(resolve, 0));
}
test("recorded camera archive stays pending until every canonical byte is verified", async () => {
const { source, manifest, generation, init, first, second } = fixture();
const finalSegment = deferred();
const requested = [];
let manifestRequests = 0;
const fetcher = async (input, request = {}) => {
const url = String(input);
requested.push(url);
const headers = new Headers(request.headers);
if (url === source.manifestUrl) {
manifestRequests += 1;
assert.equal(headers.get("If-Match"), `"sha256:${generation}"`);
return jsonResponse(manifest, generation);
}
const epoch = manifest.epochs[0];
if (url === epoch.init_url) {
assert.equal(headers.get("If-Match"), `"sha256:${epoch.init_sha256}"`);
return mediaResponse(init);
}
if (url === epoch.segments[0].url) return mediaResponse(first);
if (url === epoch.segments[1].url) return finalSegment.promise;
throw new Error(`Unexpected URL ${url}`);
};
let settled = false;
const archivePromise = fetchVerifiedRecordedMediaArchive(source, { fetcher })
.then((archive) => {
settled = true;
return archive;
});
await flushAsync();
assert.equal(settled, false, "init and a partial segment prefix must not publish the camera");
assert.deepEqual(requested.slice(0, 4), [
source.manifestUrl,
manifest.epochs[0].init_url,
manifest.epochs[0].segments[0].url,
manifest.epochs[0].segments[1].url,
]);
finalSegment.resolve(mediaResponse(second));
const archive = await archivePromise;
assert.equal(manifestRequests, 2, "the immutable generation is revalidated after transfer");
assert.equal(archive.byteLength, init.byteLength + first.byteLength + second.byteLength);
assert.equal(archive.epochs[0].segments.length, 2);
assert.deepEqual(requested, [source.manifestUrl]);
assert.equal(archive.byteLength, 36_000_000_000);
assert.equal(archive.manifest.epochs[0].streamUrl, streamUrl);
});
test("first camera manifest request is launch-generation bound and rejects replacement", async () => {
test("first camera manifest request rejects a replaced generation", async () => {
const { source, manifest, generation } = fixture();
const replacementGeneration = "d".repeat(64);
let requests = 0;
const fetcher = async (input, request = {}) => {
requests += 1;
assert.equal(String(input), source.manifestUrl);
assert.equal(
new Headers(request.headers).get("If-Match"),
`"sha256:${generation}"`,
);
return jsonResponse(
{ ...manifest, generation_sha256: replacementGeneration },
replacementGeneration,
);
};
await assert.rejects(
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
fetchRecordedMediaArchive(source, {
fetcher: async (_input, request = {}) => {
assert.equal(new Headers(request.headers).get("If-Match"), `"sha256:${generation}"`);
return jsonResponse(
{ ...manifest, generation_sha256: replacementGeneration },
replacementGeneration,
);
},
}),
/несовместим|заменён/,
);
assert.equal(requests, 1);
});
test("recorded camera archive fails closed on truncation despite plausible headers", async () => {
const { source, manifest, generation, init, first, second } = fixture();
const fetcher = async (input) => {
const url = String(input);
if (url === source.manifestUrl) return jsonResponse(manifest, generation);
const epoch = manifest.epochs[0];
if (url === epoch.init_url) return mediaResponse(init);
if (url === epoch.segments[0].url) return mediaResponse(first);
if (url === epoch.segments[1].url) {
return mediaResponse(second.subarray(0, second.byteLength - 1), digest(second), second.byteLength);
}
throw new Error(`Unexpected URL ${url}`);
test("camera manifest fails closed when epoch bytes do not match launch bytes", async () => {
const { source, manifest, generation } = fixture();
const mismatched = {
...manifest,
epochs: [{ ...manifest.epochs[0], byte_length: manifest.byte_length - 1 }],
};
await assert.rejects(
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
/усечён/,
fetchRecordedMediaArchive(source, {
fetcher: async () => jsonResponse(mismatched, generation),
}),
/не совпадает/,
);
});
test("recorded camera archive fails closed when bytes are replaced under an old ETag", async () => {
const { source, manifest, generation, init, first, second } = fixture();
const replacement = Buffer.from(second.map((value) => value ^ 0xff));
assert.equal(replacement.byteLength, second.byteLength);
const fetcher = async (input) => {
const url = String(input);
if (url === source.manifestUrl) return jsonResponse(manifest, generation);
const epoch = manifest.epochs[0];
if (url === epoch.init_url) return mediaResponse(init);
if (url === epoch.segments[0].url) return mediaResponse(first);
if (url === epoch.segments[1].url) {
return mediaResponse(replacement, digest(second), second.byteLength);
}
throw new Error(`Unexpected URL ${url}`);
};
await assert.rejects(
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
/SHA-256/,
);
});
test("camera presentation gate opens only for the completely appended selected epoch", () => {
test("camera presentation gate opens only for the seekable selected epoch", () => {
assert.equal(recordedMediaPresentationState("loading", null, "g1", false), "loading");
assert.equal(recordedMediaPresentationState("ready", null, "g1", false), "loading");
assert.equal(recordedMediaPresentationState("ready", "g2", "g1", false), "loading");
@@ -251,40 +144,13 @@ test("decoded duration and seekable range cover the complete declared epoch", ()
assert.equal(recordedMediaSeekableCoverage(20, 20, 20, 1, 1.01), false);
});
test("aborting SourceBuffer append removes every temporary listener", async () => {
const listeners = new Map();
const sourceBuffer = {
addEventListener(type, listener) {
const bucket = listeners.get(type) ?? new Set();
bucket.add(listener);
listeners.set(type, bucket);
},
removeEventListener(type, listener) {
listeners.get(type)?.delete(listener);
},
appendBuffer() {},
};
const abort = new AbortController();
const pending = appendRecordedMediaBuffer(
sourceBuffer,
new ArrayBuffer(8),
abort.signal,
);
assert.equal(listeners.get("updateend")?.size, 1);
assert.equal(listeners.get("error")?.size, 1);
abort.abort();
await assert.rejects(pending, (error) => error?.name === "AbortError");
assert.equal(listeners.get("updateend")?.size, 0);
assert.equal(listeners.get("error")?.size, 0);
});
test("verified camera archives remain immutable for safe player remount", async () => {
test("recorded player range-streams and never builds a whole-video RAM Blob", async () => {
const source = await readFile(
new URL("../src/components/RecordedFmp4Player.tsx", import.meta.url),
"utf8",
);
assert.doesNotMatch(source, /\.init\s*=\s*new ArrayBuffer/);
assert.doesNotMatch(source, /\.segments\.length\s*=\s*0/);
assert.match(source, /video\.src\s*=\s*descriptor\.streamUrl/);
assert.doesNotMatch(source, /new Blob\(|response\.arrayBuffer\(|SourceBuffer/);
});
test("loading and error overlays fully conceal recorded camera pixels", async () => {
@@ -301,3 +167,20 @@ test("loading and error overlays fully conceal recorded camera pixels", async ()
/\.recorded-media-player__notice\s*\{[^}]*inset:\s*0;[^}]*background:\s*#070809/s,
);
});
test("point-cloud fullscreen keeps the admitted recorded camera worker mounted", async () => {
const source = await readFile(
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
"utf8",
);
assert.match(source, /\{visibleMediaSources\.map\(\(source, index\) => \(/);
assert.match(source, /hidden=\{pointCloudFocused \|\| unifiedPerception\}/);
assert.doesNotMatch(
source,
/\{!pointCloudFocused \? visibleMediaSources\.map/,
);
assert.doesNotMatch(
source,
/\(pointCloudFocused \|\| !observationLayout\.visibleSourceIds\.has\(source\.id\)\)/,
);
});
@@ -78,34 +78,27 @@ test("any RRD or camera failure closes the complete recorded session", () => {
);
});
test("camera admission enforces independent 16/256/512 MiB limits before fetching", () => {
test("camera admission keeps source scheduling bounded but has no duration or byte ceiling", () => {
const {
MAX_RECORDED_CAMERA_SOURCES,
MAX_RECORDED_MEDIA_SOURCE_BYTES,
MAX_RECORDED_SESSION_CAMERA_BYTES,
recordedCameraDescriptorPreflight,
} = admission;
assert.equal(MAX_RECORDED_CAMERA_SOURCES, 16);
assert.equal(MAX_RECORDED_MEDIA_SOURCE_BYTES, 256 * 1024 * 1024);
assert.equal(MAX_RECORDED_SESSION_CAMERA_BYTES, 512 * 1024 * 1024);
assert.equal(recordedCameraDescriptorPreflight(
Array.from({ length: 2 }, (_, index) => ({
id: `camera.${index}`,
byteLength: MAX_RECORDED_MEDIA_SOURCE_BYTES,
byteLength: 36_000_000_000,
})),
), "ready");
assert.equal(recordedCameraDescriptorPreflight(
Array.from({ length: 17 }, (_, index) => ({ id: `camera.${index}`, byteLength: 1 })),
), "error");
assert.equal(recordedCameraDescriptorPreflight([
{ id: "camera.large", byteLength: MAX_RECORDED_MEDIA_SOURCE_BYTES + 1 },
{ id: "camera.ten-hours", byteLength: 36_000_000_000 },
]), "ready");
assert.equal(recordedCameraDescriptorPreflight([
{ id: "camera.invalid", byteLength: Number.MAX_SAFE_INTEGER + 1 },
]), "error");
assert.equal(recordedCameraDescriptorPreflight(
Array.from({ length: 5 }, (_, index) => ({
id: `camera.${index}`,
byteLength: 110 * 1024 * 1024,
})),
), "error");
assert.equal(recordedCameraDescriptorPreflight([
{ id: "camera.duplicate", byteLength: 1 },
{ id: "camera.duplicate", byteLength: 1 },
@@ -0,0 +1,67 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import test from "node:test";
import makeRerunRuntime from "../vendor/rerun-web-viewer-0.34.1/re_viewer.nodedc.js";
const root = resolve(import.meta.dirname, "..");
const packageRoot = resolve(root, "node_modules/@rerun-io/web-viewer");
const vendorRoot = resolve(root, "vendor/rerun-web-viewer-0.34.1");
const sha256 = (path) =>
createHash("sha256").update(readFileSync(path)).digest("hex");
test("NODE.DC Rerun runtime is the audited 0.34.1 zoom-to-cursor build", () => {
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "package.json"), "utf8"));
assert.equal(manifest.version, "0.34.1");
const expectedWasm = "38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb";
const expectedGlue = "0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339";
assert.equal(sha256(resolve(vendorRoot, "re_viewer_bg.nodedc.wasm")), expectedWasm);
assert.equal(sha256(resolve(vendorRoot, "re_viewer.nodedc.js")), expectedGlue);
assert.equal(sha256(resolve(packageRoot, "re_viewer_bg.wasm")), expectedWasm);
assert.equal(sha256(resolve(packageRoot, "re_viewer.js")), expectedGlue);
});
test("custom JavaScript glue references only exports present in its paired WASM", () => {
const wasmPath = resolve(vendorRoot, "re_viewer_bg.nodedc.wasm");
const gluePath = resolve(vendorRoot, "re_viewer.nodedc.js");
const module = new WebAssembly.Module(readFileSync(wasmPath));
const exports = new Set(WebAssembly.Module.exports(module).map(({ name }) => name));
const imports = WebAssembly.Module.imports(module);
const glue = readFileSync(gluePath, "utf8");
const referencedExports = new Set(
[...glue.matchAll(/\bwasm\.([A-Za-z_$][\w$]*)/g)].map((match) => match[1]),
);
const missingExports = [...referencedExports].filter((name) => !exports.has(name));
assert.equal(imports.length, 927);
assert.equal(exports.size, 79);
assert.deepEqual(missingExports, []);
assert.match(glue, /export default function\(\)/);
assert.match(glue, /if \(!wasm\) return;/);
});
test("custom Rerun WASM initializes and grows its externref table", () => {
const runtime = makeRerunRuntime();
runtime.initSync({
module: readFileSync(resolve(vendorRoot, "re_viewer_bg.nodedc.wasm")),
});
assert.equal(typeof runtime.WebHandle, "function");
runtime.deinit();
});
test("source patch carries cursor pivot, minimum-radius handoff, and geometry tests", () => {
const patch = readFileSync(resolve(vendorRoot, "NODEDC_ZOOM_TO_CURSOR.patch"), "utf8");
assert.match(patch, /fn pointer_ray_direction/);
assert.match(patch, /fn zoom_orbit_towards_pointer/);
assert.match(patch, /near_limit_hands_excess_zoom_to_cursor_directed_dolly/);
assert.match(patch, /crossing_near_limit_preserves_unconsumed_scene_scaled_zoom/);
assert.match(patch, /remaining_zoom_factor\.ln\(\) \* self\.speed/);
assert.match(patch, /off_center_pointer_stays_on_the_same_view_ray/);
});
@@ -0,0 +1,68 @@
# NODE.DC Rerun web viewer 0.34.1
This directory contains the audited Mission Core camera-controller override for
`@rerun-io/web-viewer` 0.34.1. It changes only the native orbital zoom behavior:
- the pointer ray selects an anchor on the current focus plane;
- eye position and look target scale around that anchor, so the point under the
cursor remains under the cursor;
- after Rerun's `0.02 m` near-plane safety radius is reached, excess zoom becomes
a cursor-directed dolly scaled by the scene's navigation speed instead of
silently ignoring the wheel or moving by imperceptible millimeters;
- first-person movement, orbit rotation, panning, WASD and Rerun's zoom-out cap
are unchanged.
## Source identity
- Upstream: `rerun-io/rerun`
- Tag: `0.34.1`
- Commit: `4efb18f17f6f0e41985cda99a2bdcd012febc8d5`
- Patched file: `crates/viewer/re_view_spatial/src/eye.rs`
- Patch: `NODEDC_ZOOM_TO_CURSOR.patch`
- Rust: `1.92.0`
- Binaryen / `wasm-opt`: `117` (the version pinned by Rerun's `pixi.lock`)
- Build image: `rust:1.92-bookworm`
- Build image digest:
`sha256:e90e846de4124376164ddfbaab4b0774c7bdeef5e738866295e5a90a34a307a2`
- Build date: `2026-07-22` (`Europe/Moscow`)
## Reproduction
Apply the patch to the exact upstream commit, then run Rerun's own builder:
```sh
git apply NODEDC_ZOOM_TO_CURSOR.patch
cargo test -p re_view_spatial --lib eye::tests:: -- --nocapture
cargo run -p re_dev_tools -- build-web-viewer \
--release -g \
--target no-modules-base \
--no-default-features \
--features map_view \
-o rerun_js/web-viewer
```
The container also needs Binaryen `117` for Rerun's final `wasm-opt -O2` step.
Binaryen `108` from Debian 12 must not be used: it produced a module whose
`externref` table could not grow during initialization. The
generated `re_viewer.js` is transformed with
`scripts/transform-rerun-web-viewer-glue.mjs`, which mirrors Rerun 0.34.1's
`rerun_js/web-viewer/build-wasm.mjs` no-modules wrapper and teardown guards.
## Verified artifacts
| Artifact | SHA-256 |
| --- | --- |
| `re_viewer_bg.nodedc.wasm` | `38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb` |
| raw generated `re_viewer.js` | `cc196a93c5be972c801d46be4dc9934f7f042eb62941f0aa0678f1c8416c6874` |
| `re_viewer.nodedc.js` | `0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339` |
The focused Rust suite completed with `5 passed, 0 failed`, including the
orbit-to-dolly boundary case. A Node `initSync`
smoke test completed successfully, including the `externref` table-growth step.
The Mission Core Node suite also checks both artifact hashes and verifies that
every `wasm.*` reference in the JavaScript glue exists in the paired WASM
exports.
`scripts/patch-rerun-web-viewer.mjs` installs the pair after npm extracts the
official package. It accepts only the known published or NODE.DC hashes and
fails closed on any other package contents.
@@ -0,0 +1,230 @@
diff --git a/crates/viewer/re_view_spatial/src/eye.rs b/crates/viewer/re_view_spatial/src/eye.rs
index e59b311..3f33259 100644
--- a/crates/viewer/re_view_spatial/src/eye.rs
+++ b/crates/viewer/re_view_spatial/src/eye.rs
@@ -512,8 +512,85 @@ impl EyeController {
}
}
+ /// Returns the world-space ray under the pointer.
+ ///
+ /// Keeping this calculation local to the eye controller lets orbital zoom use the pointer
+ /// synchronously. GPU picking arrives a frame later and would make the zoom pivot visibly lag.
+ fn pointer_ray_direction(&self, rect: Rect, pointer: egui::Pos2) -> Option<Vec3> {
+ if !rect.contains(pointer) || rect.width() <= 0.0 || rect.height() <= 0.0 {
+ return None;
+ }
+
+ let fov_y = self.fov_y.unwrap_or(Eye::DEFAULT_FOV_Y);
+ let aspect_ratio = rect.width() / rect.height();
+ let focal_scale = (fov_y * 0.5).tan();
+ let x = (2.0 * (pointer.x - rect.left()) / rect.width() - 1.0)
+ * focal_scale
+ * aspect_ratio;
+ let y = (1.0 - 2.0 * (pointer.y - rect.top()) / rect.height()) * focal_scale;
+
+ (self.rotation() * vec3(x, y, -1.0)).try_normalize()
+ }
+
+ /// Zoom an orbital eye around the point under the pointer on the current focus plane.
+ ///
+ /// The position and look target are scaled around the same anchor. This preserves the
+ /// projected pointer position while retaining the existing orbit direction and controls.
+ /// Once the near-plane safety radius is reached, excess zoom becomes a cursor-directed dolly
+ /// measured in the scene's navigation speed instead of near-plane millimeters.
+ fn zoom_orbit_towards_pointer(
+ &mut self,
+ zoom_factor: f32,
+ max_radius: f32,
+ rect: Rect,
+ pointer: Option<egui::Pos2>,
+ ) {
+ let radius = self.radius();
+ if !radius.is_finite() || radius <= f32::MIN_POSITIVE {
+ return;
+ }
+
+ let requested_radius = radius / zoom_factor;
+ let new_radius = requested_radius.clamp(Self::MIN_ORBIT_DISTANCE, max_radius);
+ let scale = new_radius / radius;
+
+ let ray_direction = pointer.and_then(|pointer| self.pointer_ray_direction(rect, pointer));
+ if let Some(ray_direction) = ray_direction {
+ let forward = self.fwd();
+ let denominator = ray_direction.dot(forward);
+ if denominator > 1.0e-4 {
+ let anchor = self.pos + ray_direction * (radius / denominator);
+ self.pos = anchor + (self.pos - anchor) * scale;
+ self.look_target = anchor + (self.look_target - anchor) * scale;
+
+ if requested_radius < Self::MIN_ORBIT_DISTANCE {
+ // Shrinking the remaining 2 cm orbit radius consumes only part of this input.
+ // Hand the logarithmic remainder to the same scene-scaled speed used by WASD
+ // and first-person scroll. Basing this on the near-plane remainder itself made
+ // each wheel event move by millimeters and felt indistinguishable from a hard
+ // zoom limit on building- and map-scale recordings.
+ let orbit_zoom_factor = (radius / Self::MIN_ORBIT_DISTANCE).max(1.0);
+ let remaining_zoom_factor = zoom_factor / orbit_zoom_factor;
+ if remaining_zoom_factor > 1.0 && remaining_zoom_factor.is_finite() {
+ let dolly = remaining_zoom_factor.ln() * self.speed as f32;
+ self.pos += ray_direction * dolly;
+ self.look_target += ray_direction * dolly;
+ }
+ }
+ self.did_interact = true;
+ return;
+ }
+ }
+
+ // Pointer data can be absent for synthetic zoom events. Preserve Rerun's centered zoom
+ // behavior in that case instead of dropping the input.
+ self.pos = self.look_target - self.fwd() * new_radius;
+ self.did_interact = true;
+ }
+
/// Handle zoom/scroll input.
- fn handle_zoom(&mut self, egui_ctx: &egui::Context, scene_bounding_box: &macaw::BoundingBox) {
+ fn handle_zoom(&mut self, response: &egui::Response, scene_bounding_box: &macaw::BoundingBox) {
+ let egui_ctx = &response.ctx;
let zoom_factor = egui_ctx.input(|input| {
// egui's default horizontal_scroll_modifier is shift, which is also our speed-up modifier.
// This means that a user who wants to speed up scroll-to-zoom will generate a horizontal scroll delta.
@@ -528,22 +605,12 @@ impl EyeController {
match self.kind {
Eye3DKind::Orbital => {
- let radius = self.pos.distance(self.look_target);
-
// Cap zoom-out against the scene bounding box. If we're already past the cap
// (e.g. right after loading) use the current radius instead — no snap-back.
+ let radius = self.radius();
let max_radius = max_orbital_radius(scene_bounding_box).max(radius);
- let new_radius = (radius / zoom_factor).clamp(Self::MIN_ORBIT_DISTANCE, max_radius);
-
- // The user may be scrolling to move the camera closer, but are not realizing
- // the radius is now tiny.
- // TODO(emilk): inform the users somehow that scrolling won't help, and that they should use WSAD instead.
- // It might be tempting to start moving the camera here on scroll, but that would is bad for other reasons.
-
- if f32::MIN_POSITIVE < new_radius {
- self.pos = self.look_target - self.fwd() * new_radius;
- self.did_interact = true;
- }
+ let pointer = response.ctx.pointer_latest_pos();
+ self.zoom_orbit_towards_pointer(zoom_factor, max_radius, response.rect, pointer);
}
Eye3DKind::FirstPerson => {
// Move along the forward axis when zooming in first person mode.
@@ -687,7 +754,7 @@ impl EyeController {
self.handle_drag(response, drag_threshold);
if response.hovered() {
- self.handle_zoom(&response.ctx, scene_bounding_box);
+ self.handle_zoom(response, scene_bounding_box);
}
if response.has_focus() {
@@ -1249,3 +1316,103 @@ impl EyeState {
Ok(eye)
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn orbital_controller(pos: Vec3, look_target: Vec3) -> EyeController {
+ EyeController {
+ pos,
+ look_target,
+ kind: Eye3DKind::Orbital,
+ speed: 1.0,
+ eye_up: Vec3::Z,
+ fov_y: Some(Eye::DEFAULT_FOV_Y),
+ did_interact: false,
+ }
+ }
+
+ fn test_rect() -> Rect {
+ Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(800.0, 600.0))
+ }
+
+ fn assert_vec3_close(actual: Vec3, expected: Vec3) {
+ assert!(
+ actual.abs_diff_eq(expected, 1.0e-5),
+ "actual={actual:?}, expected={expected:?}"
+ );
+ }
+
+ #[test]
+ fn centered_pointer_keeps_the_orbit_target() {
+ let rect = test_rect();
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
+
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(rect.center()));
+
+ assert_vec3_close(controller.pos, vec3(0.0, -5.0, 0.0));
+ assert_vec3_close(controller.look_target, Vec3::ZERO);
+ assert!(controller.did_interact);
+ }
+
+ #[test]
+ fn off_center_pointer_stays_on_the_same_view_ray() {
+ let rect = test_rect();
+ let pointer = egui::pos2(650.0, 240.0);
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
+ let original_ray = controller
+ .pointer_ray_direction(rect, pointer)
+ .expect("test pointer must produce a ray");
+ let forward = controller.fwd();
+ let anchor = controller.pos
+ + original_ray * (controller.radius() / original_ray.dot(forward));
+
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(pointer));
+
+ assert!((controller.radius() - 5.0).abs() < 1.0e-5);
+ let ray_after = (anchor - controller.pos).normalize();
+ assert!(ray_after.dot(original_ray) > 0.99999);
+ }
+
+ #[test]
+ fn near_limit_hands_excess_zoom_to_cursor_directed_dolly() {
+ let rect = test_rect();
+ let radius = EyeController::MIN_ORBIT_DISTANCE;
+ let mut controller = orbital_controller(vec3(0.0, -radius, 0.0), Vec3::ZERO);
+ let old_pos = controller.pos;
+ let old_target = controller.look_target;
+
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(rect.center()));
+
+ assert!((controller.radius() - radius).abs() < 1.0e-6);
+ assert_vec3_close(controller.pos, old_pos + Vec3::Y * 2.0_f32.ln());
+ assert_vec3_close(
+ controller.look_target,
+ old_target + Vec3::Y * 2.0_f32.ln(),
+ );
+ assert_vec3_close(controller.look_target - controller.pos, Vec3::Y * radius);
+ }
+
+ #[test]
+ fn crossing_near_limit_preserves_unconsumed_scene_scaled_zoom() {
+ let rect = test_rect();
+ let mut controller = orbital_controller(vec3(0.0, -0.03, 0.0), Vec3::ZERO);
+
+ controller.zoom_orbit_towards_pointer(3.0, 100.0, rect, Some(rect.center()));
+
+ assert!((controller.radius() - EyeController::MIN_ORBIT_DISTANCE).abs() < 1.0e-6);
+ assert_vec3_close(controller.pos, vec3(0.0, 2.0_f32.ln() - 0.02, 0.0));
+ assert_vec3_close(controller.look_target, vec3(0.0, 2.0_f32.ln(), 0.0));
+ }
+
+ #[test]
+ fn zoom_out_respects_scene_radius_cap() {
+ let rect = test_rect();
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
+
+ controller.zoom_orbit_towards_pointer(0.1, 25.0, rect, Some(rect.center()));
+
+ assert!((controller.radius() - 25.0).abs() < 1.0e-5);
+ }
+}
File diff suppressed because it is too large Load Diff
+7
View File
@@ -53,6 +53,13 @@ export default defineConfig(({ mode }) => {
host: "127.0.0.1",
port: 4173,
strictPort: true,
proxy: {
"/api": {
target: apiTarget,
changeOrigin: false,
ws: true,
},
},
},
};
});