feat(perception): integrate calibrated operator pipeline
Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
@@ -1,10 +1,15 @@
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { copyFileSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const packageRoot = resolve(root, "node_modules/@rerun-io/web-viewer");
|
||||
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "package.json"), "utf8"));
|
||||
const vendorRoot = resolve(root, "vendor/rerun-web-viewer-0.34.1");
|
||||
|
||||
const sha256File = (path) =>
|
||||
createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
|
||||
if (manifest.version !== "0.34.1") {
|
||||
throw new Error(
|
||||
@@ -12,6 +17,51 @@ if (manifest.version !== "0.34.1") {
|
||||
);
|
||||
}
|
||||
|
||||
const vendorRuntime = [
|
||||
{
|
||||
label: "wasm runtime",
|
||||
packagePath: resolve(packageRoot, "re_viewer_bg.wasm"),
|
||||
vendorPath: resolve(vendorRoot, "re_viewer_bg.nodedc.wasm"),
|
||||
publishedSha256: "3fe7aab8ea6bb0fd03c3ef694932943411ee174029e398c539c390beb0824d35",
|
||||
previousNodedcSha256: "1c25e8cecd7641e6f8044d00a6a1cf9d08a615b6f8737026c7d93623b3e06ee7",
|
||||
nodedcSha256: "38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb",
|
||||
},
|
||||
{
|
||||
label: "wasm JavaScript glue",
|
||||
packagePath: resolve(packageRoot, "re_viewer.js"),
|
||||
vendorPath: resolve(vendorRoot, "re_viewer.nodedc.js"),
|
||||
publishedSha256: "7c4ba900820137a6ba23e0f7f57d56e3b007db0de3825d5c8bf0d7684b3cc6a6",
|
||||
nodedcSha256: "0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339",
|
||||
},
|
||||
];
|
||||
|
||||
for (const runtime of vendorRuntime) {
|
||||
const vendorSha256 = sha256File(runtime.vendorPath);
|
||||
if (vendorSha256 !== runtime.nodedcSha256) {
|
||||
throw new Error(
|
||||
`Refusing corrupt NODE.DC Rerun ${runtime.label}: ${vendorSha256} != ${runtime.nodedcSha256}`,
|
||||
);
|
||||
}
|
||||
|
||||
const installedSha256 = sha256File(runtime.packagePath);
|
||||
if (
|
||||
![
|
||||
runtime.publishedSha256,
|
||||
runtime.previousNodedcSha256,
|
||||
runtime.nodedcSha256,
|
||||
].includes(installedSha256)
|
||||
) {
|
||||
throw new Error(
|
||||
`Refusing to replace unexpected Rerun ${runtime.label}: ${installedSha256}`,
|
||||
);
|
||||
}
|
||||
|
||||
copyFileSync(runtime.vendorPath, runtime.packagePath);
|
||||
if (sha256File(runtime.packagePath) !== runtime.nodedcSha256) {
|
||||
throw new Error(`NODE.DC Rerun ${runtime.label} verification failed after copy`);
|
||||
}
|
||||
}
|
||||
|
||||
const patches = [
|
||||
{
|
||||
label: "compiled watchdog",
|
||||
@@ -57,5 +107,5 @@ for (const path of [resolve(packageRoot, "index.js"), resolve(packageRoot, "inde
|
||||
}
|
||||
|
||||
console.log(
|
||||
"Patched @rerun-io/web-viewer 0.34.1 watchdog teardown and singleton global keyup listener.",
|
||||
"Patched @rerun-io/web-viewer 0.34.1 native zoom-to-cursor, watchdog teardown, and singleton global keyup listener.",
|
||||
);
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const [inputArgument, outputArgument] = process.argv.slice(2);
|
||||
if (!inputArgument || !outputArgument) {
|
||||
throw new Error("Usage: node transform-rerun-web-viewer-glue.mjs INPUT OUTPUT");
|
||||
}
|
||||
|
||||
const inputPath = resolve(inputArgument);
|
||||
const outputPath = resolve(outputArgument);
|
||||
const expectedGeneratedSha256 =
|
||||
"cc196a93c5be972c801d46be4dc9934f7f042eb62941f0aa0678f1c8416c6874";
|
||||
|
||||
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
|
||||
let code = readFileSync(inputPath, "utf8");
|
||||
|
||||
if (sha256(code) !== expectedGeneratedSha256) {
|
||||
throw new Error(`Refusing to transform unexpected wasm-bindgen output: ${inputPath}`);
|
||||
}
|
||||
|
||||
// This is the same no-modules-base transformation used by Rerun 0.34.1's
|
||||
// rerun_js/web-viewer/build-wasm.mjs. Each factory call gets isolated closure state.
|
||||
const wrapperStart = "let wasm_bindgen = (function(exports) {";
|
||||
const wrapperEnd = `return Object.assign(__wbg_init, { initSync }, exports);
|
||||
})({ __proto__: null });`;
|
||||
|
||||
if (!code.includes(wrapperStart) || !code.includes(wrapperEnd)) {
|
||||
throw new Error("Rerun wasm-bindgen wrapper markers no longer match");
|
||||
}
|
||||
code = code.replace(wrapperStart, "").replace(wrapperEnd, "");
|
||||
|
||||
code = `
|
||||
export default function() {
|
||||
const exports = { __proto__: null };
|
||||
${code}
|
||||
|
||||
function deinit() {
|
||||
__wbg_init.__wbindgen_wasm_module = null;
|
||||
wasmModule = null;
|
||||
wasm = null;
|
||||
cachedDataViewMemory0 = null;
|
||||
cachedFloat32ArrayMemory0 = null;
|
||||
cachedInt16ArrayMemory0 = null;
|
||||
cachedInt32ArrayMemory0 = null;
|
||||
cachedInt8ArrayMemory0 = null;
|
||||
cachedUint16ArrayMemory0 = null;
|
||||
cachedUint32ArrayMemory0 = null;
|
||||
cachedUint8ArrayMemory0 = null;
|
||||
}
|
||||
|
||||
return Object.assign(__wbg_init, { initSync, deinit }, exports);
|
||||
}
|
||||
`;
|
||||
|
||||
const closureDtorsOriginal = `const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined')
|
||||
? { register: () => {}, unregister: () => {} }
|
||||
: new FinalizationRegistry(state => wasm.__wbindgen_destroy_closure(state.a, state.b));`;
|
||||
const closureDtorsPatch = `const CLOSURE_DTORS = (typeof FinalizationRegistry === 'undefined')
|
||||
? { register: () => {}, unregister: () => {} }
|
||||
: new FinalizationRegistry(state => {
|
||||
if (wasm) wasm.__wbindgen_destroy_closure(state.a, state.b);
|
||||
});`;
|
||||
|
||||
if (!code.includes(closureDtorsOriginal)) {
|
||||
throw new Error("Rerun CLOSURE_DTORS block no longer matches");
|
||||
}
|
||||
code = code.replace(closureDtorsOriginal, closureDtorsPatch);
|
||||
|
||||
const makeMutClosureOriginal = `function makeMutClosure(arg0, arg1, f) {
|
||||
const state = { a: arg0, b: arg1, cnt: 1 };
|
||||
const real = (...args) => {
|
||||
|
||||
// First up with a closure we increment the internal reference
|
||||
// count. This ensures that the Rust closure environment won't
|
||||
// be deallocated while we're invoking it.
|
||||
state.cnt++;
|
||||
const a = state.a;
|
||||
state.a = 0;
|
||||
try {
|
||||
return f(a, state.b, ...args);
|
||||
} finally {
|
||||
state.a = a;
|
||||
real._wbg_cb_unref();
|
||||
}
|
||||
};
|
||||
real._wbg_cb_unref = () => {
|
||||
if (--state.cnt === 0) {
|
||||
wasm.__wbindgen_destroy_closure(state.a, state.b);
|
||||
state.a = 0;
|
||||
CLOSURE_DTORS.unregister(state);
|
||||
}
|
||||
};
|
||||
CLOSURE_DTORS.register(real, state, state);
|
||||
return real;
|
||||
}`;
|
||||
const makeMutClosurePatch = `function makeMutClosure(arg0, arg1, f) {
|
||||
const state = { a: arg0, b: arg1, cnt: 1 };
|
||||
const real = (...args) => {
|
||||
state.cnt++;
|
||||
const a = state.a;
|
||||
state.a = 0;
|
||||
try {
|
||||
if (!wasm) return;
|
||||
return f(a, state.b, ...args);
|
||||
} finally {
|
||||
state.a = a;
|
||||
real._wbg_cb_unref();
|
||||
}
|
||||
};
|
||||
real._wbg_cb_unref = () => {
|
||||
if (--state.cnt === 0) {
|
||||
if (wasm) wasm.__wbindgen_destroy_closure(state.a, state.b);
|
||||
state.a = 0;
|
||||
CLOSURE_DTORS.unregister(state);
|
||||
}
|
||||
};
|
||||
CLOSURE_DTORS.register(real, state, state);
|
||||
return real;
|
||||
}`;
|
||||
|
||||
if (!code.includes(makeMutClosureOriginal)) {
|
||||
throw new Error("Rerun makeMutClosure block no longer matches");
|
||||
}
|
||||
code = code.replace(makeMutClosureOriginal, makeMutClosurePatch);
|
||||
|
||||
writeFileSync(outputPath, code);
|
||||
console.log(`Transformed Rerun wasm glue: ${sha256(code)} ${outputPath}`);
|
||||
@@ -88,7 +88,22 @@ const paletteOptions: Array<{ value: PointPalette; label: string; description: s
|
||||
{ value: "custom", label: "Свой цвет", description: "Один назначенный цвет" },
|
||||
];
|
||||
|
||||
function toViewerSettings(settings: SceneSettings): ViewerSettings {
|
||||
interface LivePerceptionLayers {
|
||||
detections2d: boolean;
|
||||
segmentation: boolean;
|
||||
cuboids3d: boolean;
|
||||
}
|
||||
|
||||
const defaultLivePerceptionLayers: LivePerceptionLayers = {
|
||||
detections2d: false,
|
||||
segmentation: false,
|
||||
cuboids3d: false,
|
||||
};
|
||||
|
||||
function toViewerSettings(
|
||||
settings: SceneSettings,
|
||||
perception: LivePerceptionLayers = defaultLivePerceptionLayers,
|
||||
): ViewerSettings {
|
||||
return {
|
||||
point_size: settings.pointSize,
|
||||
color_mode: settings.colorMode,
|
||||
@@ -98,6 +113,9 @@ function toViewerSettings(settings: SceneSettings): ViewerSettings {
|
||||
show_points: settings.showPoints,
|
||||
show_trajectory: settings.showTrajectory,
|
||||
show_grid: settings.showGrid,
|
||||
show_detections_2d: perception.detections2d,
|
||||
show_segmentation: perception.segmentation,
|
||||
show_cuboids_3d: perception.cuboids3d,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -149,6 +167,9 @@ export default function App() {
|
||||
const [layoutSaveNotice, setLayoutSaveNotice] = useState<string | null>(null);
|
||||
const [sceneSettings, setSceneSettings] = useState<SceneSettings>(defaultSceneSettings);
|
||||
const [displayDraft, setDisplayDraft] = useState<SceneSettings>(defaultSceneSettings);
|
||||
const [livePerceptionLayers, setLivePerceptionLayers] = useState<LivePerceptionLayers>(
|
||||
defaultLivePerceptionLayers,
|
||||
);
|
||||
const sourceSwitchBlocked = isSpatialSourceSwitchBlocked(runtime.state);
|
||||
const sourceSwitchBlockedReason = sourceSwitchBlocked
|
||||
? SPATIAL_SOURCE_SWITCH_BLOCKED_REASON
|
||||
@@ -161,6 +182,8 @@ export default function App() {
|
||||
const confirmedSceneSettingsRef = useRef<SceneSettings>(defaultSceneSettings);
|
||||
const viewerSettingsCommitTimerRef = useRef<number | null>(null);
|
||||
const runtimeUpdateViewerSettingsRef = useRef(runtime.updateViewerSettings);
|
||||
const livePerceptionLayersRef = useRef<LivePerceptionLayers>(defaultLivePerceptionLayers);
|
||||
const livePerceptionRevisionRef = useRef(0);
|
||||
const replayActiveRef = useRef(false);
|
||||
const sceneSettingsCommitterActiveRef = useRef(true);
|
||||
const sceneSettingsCommitterRef = useRef<LatestAsyncCommitter<SceneSettings> | null>(null);
|
||||
@@ -197,7 +220,9 @@ export default function App() {
|
||||
sceneSettingsCommitterRef.current = createLatestAsyncCommitter<SceneSettings>({
|
||||
commit: (settings) => replayActiveRef.current
|
||||
? Promise.resolve(true)
|
||||
: runtimeUpdateViewerSettingsRef.current(toViewerSettings(settings)),
|
||||
: runtimeUpdateViewerSettingsRef.current(
|
||||
toViewerSettings(settings, livePerceptionLayersRef.current),
|
||||
),
|
||||
onSettled: ({ value, applied, superseded }) => {
|
||||
if (!sceneSettingsCommitterActiveRef.current) return;
|
||||
if (applied) confirmedSceneSettingsRef.current = value;
|
||||
@@ -275,6 +300,13 @@ export default function App() {
|
||||
sceneSettingsCommitterRef.current?.isBusy()
|
||||
) return;
|
||||
const merged = mergeViewerSettings(sceneSettingsRef.current, remote);
|
||||
const remoteLayers = {
|
||||
detections2d: remote.show_detections_2d ?? false,
|
||||
segmentation: remote.show_segmentation ?? false,
|
||||
cuboids3d: remote.show_cuboids_3d ?? false,
|
||||
};
|
||||
livePerceptionLayersRef.current = remoteLayers;
|
||||
setLivePerceptionLayers(remoteLayers);
|
||||
sceneSettingsRef.current = merged;
|
||||
confirmedSceneSettingsRef.current = merged;
|
||||
setSceneSettings(merged);
|
||||
@@ -334,7 +366,9 @@ export default function App() {
|
||||
return;
|
||||
}
|
||||
appliedProfileKeyRef.current = applicationKey;
|
||||
void runtime.updateViewerSettings(toViewerSettings(profile.sceneSettings)).then((applied) => {
|
||||
void runtime.updateViewerSettings(
|
||||
toViewerSettings(profile.sceneSettings, livePerceptionLayersRef.current),
|
||||
).then((applied) => {
|
||||
if (!applied && appliedProfileKeyRef.current === applicationKey) {
|
||||
appliedProfileKeyRef.current = null;
|
||||
}
|
||||
@@ -441,6 +475,21 @@ export default function App() {
|
||||
activateSceneWindow("layers");
|
||||
};
|
||||
|
||||
const changeLivePerceptionLayers = useCallback((next: LivePerceptionLayers) => {
|
||||
const previous = livePerceptionLayersRef.current;
|
||||
const revision = ++livePerceptionRevisionRef.current;
|
||||
livePerceptionLayersRef.current = next;
|
||||
setLivePerceptionLayers(next);
|
||||
if (replayActiveRef.current) return;
|
||||
void runtimeUpdateViewerSettingsRef.current(
|
||||
toViewerSettings(sceneSettingsRef.current, next),
|
||||
).then((applied) => {
|
||||
if (applied || livePerceptionRevisionRef.current !== revision) return;
|
||||
livePerceptionLayersRef.current = previous;
|
||||
setLivePerceptionLayers(previous);
|
||||
});
|
||||
}, []);
|
||||
|
||||
const beginRecordedReplaySwitch = useCallback(async () => {
|
||||
if (sourceSwitchBlockedRef.current) {
|
||||
throw new Error(SPATIAL_SOURCE_SWITCH_BLOCKED_REASON);
|
||||
@@ -706,6 +755,8 @@ export default function App() {
|
||||
onAccumulationChange={(accumulationSeconds) =>
|
||||
stageDisplayPatch({ accumulationSeconds })}
|
||||
onAccumulationCommit={flushDisplaySettings}
|
||||
livePerceptionLayers={livePerceptionLayers}
|
||||
onLivePerceptionLayersChange={changeLivePerceptionLayers}
|
||||
observationLayout={observationLayout}
|
||||
spatialControls={selection?.SpatialControlsView
|
||||
? {
|
||||
|
||||
@@ -72,6 +72,7 @@ export function FloatingObservationWindow({
|
||||
rect,
|
||||
maximized,
|
||||
active,
|
||||
hidden = false,
|
||||
onRectChange,
|
||||
onMaximizedChange,
|
||||
onActivate,
|
||||
@@ -89,6 +90,7 @@ export function FloatingObservationWindow({
|
||||
rect?: ObservationWindowRect;
|
||||
maximized: boolean;
|
||||
active: boolean;
|
||||
hidden?: boolean;
|
||||
onRectChange: (rect: ObservationWindowRect) => void;
|
||||
onMaximizedChange: (maximized: boolean) => void;
|
||||
onActivate: () => void;
|
||||
@@ -153,7 +155,7 @@ export function FloatingObservationWindow({
|
||||
resizable={source.capabilities.resizable}
|
||||
active={active}
|
||||
zIndex={maximized ? 15 : active ? 9 : 7}
|
||||
className="floating-observation-window"
|
||||
className={`floating-observation-window${hidden ? " floating-observation-window--hidden" : ""}`}
|
||||
onPointerDownCapture={(event) => {
|
||||
if (!shouldCaptureWorkspacePointer(event.button, event.target as HTMLElement)) return;
|
||||
try {
|
||||
|
||||
@@ -52,18 +52,17 @@ const preparationLabel: Record<ObservationPreparationPhase, string> = {
|
||||
requesting: "Запрашиваем подготовку",
|
||||
queued: "В очереди",
|
||||
validating: "Проверяем запись",
|
||||
exporting: "Готовим облако точек",
|
||||
exporting: "Готовим операторскую сцену",
|
||||
finalizing: "Завершаем подготовку",
|
||||
failed: "Подготовка не выполнена",
|
||||
cancelled: "Подготовка отменена",
|
||||
};
|
||||
|
||||
function progressCopy(
|
||||
phase: ObservationPreparationPhase,
|
||||
progress: number | null,
|
||||
): string {
|
||||
const label = preparationLabel[phase];
|
||||
return progress === null ? label : `${label} · ${Math.round(progress * 100)}%`;
|
||||
function progressCopy(phase: ObservationPreparationPhase): string {
|
||||
// Backend progress values are phase markers and heartbeat revisions, not a
|
||||
// measured fraction of bytes or frames. Presenting 0.5 as "50%" made a
|
||||
// healthy long export look stalled, especially after a browser reload.
|
||||
return preparationLabel[phase];
|
||||
}
|
||||
|
||||
function formatStartedAt(value: string): string {
|
||||
@@ -126,7 +125,7 @@ export function ObservationSessionSelect({
|
||||
onReplaySettled,
|
||||
});
|
||||
const triggerCopy = sessions.replayProgress
|
||||
? progressCopy(sessions.replayProgress.phase, sessions.replayProgress.progress)
|
||||
? progressCopy(sessions.replayProgress.phase)
|
||||
: sessions.state === "loading"
|
||||
? "Загружаем сессии…"
|
||||
: "Сохранённые сессии";
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { sha256 } from "@noble/hashes/sha2.js";
|
||||
import { bytesToHex } from "@noble/hashes/utils.js";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
|
||||
import {
|
||||
@@ -11,7 +9,6 @@ import {
|
||||
type ObservationSessionFetch,
|
||||
} from "../core/observation/sessionArchive";
|
||||
import {
|
||||
MAX_RECORDED_MEDIA_SOURCE_BYTES,
|
||||
type RecordedAdmissionPhase,
|
||||
type RecordedCameraAdmissionState,
|
||||
} from "../core/observation/recordedSessionAdmission";
|
||||
@@ -22,32 +19,11 @@ export interface RecordedObservationPlayback {
|
||||
playing: boolean;
|
||||
}
|
||||
|
||||
export interface RecordedMediaLoadProgress {
|
||||
loadedBytes: number;
|
||||
totalBytes: number;
|
||||
loadedParts: number;
|
||||
totalParts: number;
|
||||
}
|
||||
|
||||
interface VerifiedRecordedMediaEpoch {
|
||||
descriptor: ObservationRecordedMediaEpoch;
|
||||
readonly init: ArrayBuffer;
|
||||
readonly segments: readonly ArrayBuffer[];
|
||||
}
|
||||
|
||||
export interface VerifiedRecordedMediaArchive {
|
||||
export interface RecordedMediaArchive {
|
||||
manifest: ObservationRecordedMediaManifest;
|
||||
epochs: readonly VerifiedRecordedMediaEpoch[];
|
||||
byteLength: number;
|
||||
}
|
||||
|
||||
interface RecordedMediaBinaryDescriptor {
|
||||
url: string;
|
||||
byteLength: number;
|
||||
sha256: string;
|
||||
accept: string;
|
||||
}
|
||||
|
||||
export type RecordedMediaPresentationState = "loading" | "ready" | "waiting" | "error";
|
||||
|
||||
export const RECORDED_MEDIA_DURATION_TOLERANCE_SECONDS = 1;
|
||||
@@ -134,205 +110,33 @@ function sourceContract(source: ObservationSourceDescriptor): ObservationRecorde
|
||||
};
|
||||
}
|
||||
|
||||
function expectedPayloadEtag(digest: string): string {
|
||||
return `"sha256:${digest}"`;
|
||||
}
|
||||
|
||||
export async function fetchVerifiedRecordedMediaBytes(
|
||||
descriptor: RecordedMediaBinaryDescriptor,
|
||||
{
|
||||
signal,
|
||||
fetcher = globalThis.fetch,
|
||||
}: { signal?: AbortSignal; fetcher?: ObservationSessionFetch } = {},
|
||||
): Promise<ArrayBuffer> {
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetcher(descriptor.url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: descriptor.accept,
|
||||
"If-Match": expectedPayloadEtag(descriptor.sha256),
|
||||
},
|
||||
signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof DOMException && error.name === "AbortError") throw error;
|
||||
throw new ObservationSessionContractError(
|
||||
"Не удалось загрузить канонический фрагмент записанного видео.",
|
||||
);
|
||||
}
|
||||
if (!response.ok || response.status !== 200) {
|
||||
throw new ObservationSessionContractError(
|
||||
`Фрагмент записанного видео вернул HTTP ${response.status}.`,
|
||||
);
|
||||
}
|
||||
if (response.headers.get("ETag") !== expectedPayloadEtag(descriptor.sha256)) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Фрагмент записанного видео не соответствует immutable manifest.",
|
||||
);
|
||||
}
|
||||
const contentLength = response.headers.get("Content-Length");
|
||||
if (
|
||||
contentLength === null ||
|
||||
!/^[1-9][0-9]*$/.test(contentLength) ||
|
||||
Number(contentLength) !== descriptor.byteLength
|
||||
) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Длина фрагмента записанного видео не соответствует immutable manifest.",
|
||||
);
|
||||
}
|
||||
const payload = await response.arrayBuffer();
|
||||
if (payload.byteLength !== descriptor.byteLength) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Фрагмент записанного видео был усечён во время передачи.",
|
||||
);
|
||||
}
|
||||
const digest = bytesToHex(sha256(new Uint8Array(payload)));
|
||||
if (digest !== descriptor.sha256) {
|
||||
throw new ObservationSessionContractError(
|
||||
"SHA-256 фрагмента записанного видео не совпадает с immutable manifest.",
|
||||
);
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
function manifestIdentity(manifest: ObservationRecordedMediaManifest): string {
|
||||
return JSON.stringify(manifest);
|
||||
}
|
||||
|
||||
export async function fetchVerifiedRecordedMediaArchive(
|
||||
export async function fetchRecordedMediaArchive(
|
||||
source: ObservationRecordedMediaSource,
|
||||
{
|
||||
signal,
|
||||
fetcher = globalThis.fetch,
|
||||
onProgress,
|
||||
}: {
|
||||
signal?: AbortSignal;
|
||||
fetcher?: ObservationSessionFetch;
|
||||
onProgress?: (progress: RecordedMediaLoadProgress) => void;
|
||||
} = {},
|
||||
): Promise<VerifiedRecordedMediaArchive> {
|
||||
): Promise<RecordedMediaArchive> {
|
||||
const manifest = await fetchObservationRecordedMediaManifest(source, {
|
||||
signal,
|
||||
fetcher,
|
||||
expectedGenerationSha256: source.manifestGenerationSha256,
|
||||
});
|
||||
const totalBytes = manifest.epochs.reduce(
|
||||
(archiveTotal, epoch) => archiveTotal + epoch.initByteLength + epoch.segments.reduce(
|
||||
(epochTotal, segment) => epochTotal + segment.byteLength,
|
||||
0,
|
||||
),
|
||||
0,
|
||||
);
|
||||
const totalParts = manifest.epochs.reduce(
|
||||
(total, epoch) => total + 1 + epoch.segments.length,
|
||||
0,
|
||||
);
|
||||
const totalBytes = manifest.epochs.reduce((total, epoch) => total + epoch.byteLength, 0);
|
||||
if (
|
||||
!Number.isSafeInteger(totalBytes) ||
|
||||
totalBytes < 1 ||
|
||||
totalBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES ||
|
||||
totalBytes !== manifest.byteLength ||
|
||||
totalBytes !== source.byteLength
|
||||
) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Размер записанного медиаканала выходит за безопасный лимит браузера.",
|
||||
"Размеры потоков записанного медиаканала не совпадают с manifest.",
|
||||
);
|
||||
}
|
||||
let loadedBytes = 0;
|
||||
let loadedParts = 0;
|
||||
const publishProgress = () => onProgress?.({
|
||||
loadedBytes,
|
||||
totalBytes,
|
||||
loadedParts,
|
||||
totalParts,
|
||||
});
|
||||
publishProgress();
|
||||
|
||||
const epochs: VerifiedRecordedMediaEpoch[] = [];
|
||||
for (const epoch of manifest.epochs) {
|
||||
const init = await fetchVerifiedRecordedMediaBytes({
|
||||
url: epoch.initUrl,
|
||||
byteLength: epoch.initByteLength,
|
||||
sha256: epoch.initSha256,
|
||||
accept: "video/mp4",
|
||||
}, { signal, fetcher });
|
||||
loadedBytes += init.byteLength;
|
||||
loadedParts += 1;
|
||||
publishProgress();
|
||||
|
||||
const segments: ArrayBuffer[] = [];
|
||||
for (const segment of epoch.segments) {
|
||||
const payload = await fetchVerifiedRecordedMediaBytes({
|
||||
url: segment.url,
|
||||
byteLength: segment.byteLength,
|
||||
sha256: segment.sha256,
|
||||
accept: "video/iso.segment",
|
||||
}, { signal, fetcher });
|
||||
segments.push(payload);
|
||||
loadedBytes += payload.byteLength;
|
||||
loadedParts += 1;
|
||||
publishProgress();
|
||||
}
|
||||
epochs.push({ descriptor: epoch, init, segments });
|
||||
}
|
||||
|
||||
// Bind the complete byte set to the same manifest generation at both ends
|
||||
// of the transfer. A replacement during a long camera download fails closed.
|
||||
const confirmed = await fetchObservationRecordedMediaManifest(source, {
|
||||
signal,
|
||||
fetcher,
|
||||
expectedGenerationSha256: manifest.generationSha256,
|
||||
});
|
||||
if (manifestIdentity(confirmed) !== manifestIdentity(manifest)) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Manifest записанного видео изменился во время полной загрузки.",
|
||||
);
|
||||
}
|
||||
if (loadedBytes !== source.byteLength) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Полная загрузка камеры не совпала с launch byte_length.",
|
||||
);
|
||||
}
|
||||
return { manifest, epochs, byteLength: loadedBytes };
|
||||
}
|
||||
|
||||
export function appendRecordedMediaBuffer(
|
||||
sourceBuffer: SourceBuffer,
|
||||
payload: ArrayBuffer,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const onUpdateEnd = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onError = () => {
|
||||
cleanup();
|
||||
reject(new Error("SourceBuffer rejected archived fMP4 data"));
|
||||
};
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
reject(new DOMException("Aborted", "AbortError"));
|
||||
};
|
||||
const cleanup = () => {
|
||||
sourceBuffer.removeEventListener("updateend", onUpdateEnd);
|
||||
sourceBuffer.removeEventListener("error", onError);
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
};
|
||||
if (signal.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
sourceBuffer.addEventListener("updateend", onUpdateEnd, { once: true });
|
||||
sourceBuffer.addEventListener("error", onError, { once: true });
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
try {
|
||||
sourceBuffer.appendBuffer(payload);
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
return { manifest, byteLength: totalBytes };
|
||||
}
|
||||
|
||||
function videoHasSeekableArchive(
|
||||
@@ -358,17 +162,22 @@ function waitForSeekableArchive(
|
||||
if (videoHasSeekableArchive(video, declaredDurationSeconds)) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const events = ["loadedmetadata", "durationchange", "progress", "canplay"] as const;
|
||||
const timeout = globalThis.setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("Archived camera did not become seekable"));
|
||||
}, 20_000);
|
||||
let stallTimer: ReturnType<typeof globalThis.setTimeout> | undefined;
|
||||
const armStallTimer = () => {
|
||||
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
|
||||
stallTimer = globalThis.setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("Archived camera stream stalled"));
|
||||
}, 45_000);
|
||||
};
|
||||
const cleanup = () => {
|
||||
globalThis.clearTimeout(timeout);
|
||||
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
|
||||
for (const event of events) video.removeEventListener(event, onProgress);
|
||||
video.removeEventListener("error", onError);
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
};
|
||||
const onProgress = () => {
|
||||
armStallTimer();
|
||||
if (!videoHasSeekableArchive(video, declaredDurationSeconds)) return;
|
||||
cleanup();
|
||||
resolve();
|
||||
@@ -384,43 +193,30 @@ function waitForSeekableArchive(
|
||||
for (const event of events) video.addEventListener(event, onProgress);
|
||||
video.addEventListener("error", onError, { once: true });
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
armStallTimer();
|
||||
});
|
||||
}
|
||||
|
||||
async function mountVerifiedRecordedEpoch(
|
||||
async function mountRecordedEpochStream(
|
||||
video: HTMLVideoElement,
|
||||
epoch: VerifiedRecordedMediaEpoch,
|
||||
descriptor: ObservationRecordedMediaEpoch,
|
||||
signal: AbortSignal,
|
||||
): Promise<() => void> {
|
||||
const descriptor = epoch.descriptor;
|
||||
if (!descriptor.mediaType.startsWith("video/mp4;") || !video.canPlayType(descriptor.mediaType)) {
|
||||
throw new Error("Archived camera codec is not supported");
|
||||
}
|
||||
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
|
||||
|
||||
// A sealed fragmented-MP4 generation is already complete and immutable.
|
||||
// Present it as one Blob so Chromium can index and seek the whole archive
|
||||
// without retaining the same 166+ MiB epoch in a quota-limited MSE
|
||||
// SourceBuffer. Every constituent byte was fetched and SHA-256 verified
|
||||
// before this boundary, and Blob preserves their canonical order.
|
||||
const payload = new Blob([epoch.init, ...epoch.segments], {
|
||||
type: descriptor.mediaType,
|
||||
});
|
||||
const expectedByteLength = epoch.init.byteLength + epoch.segments.reduce(
|
||||
(total, segment) => total + segment.byteLength,
|
||||
0,
|
||||
);
|
||||
if (payload.size !== expectedByteLength) {
|
||||
throw new Error("Archived camera Blob is incomplete");
|
||||
}
|
||||
const objectUrl = URL.createObjectURL(payload);
|
||||
// The generation token binds the native media request to the exact manifest.
|
||||
// The browser range-streams the virtual init+fragment file from the server;
|
||||
// no complete camera archive is copied into JavaScript memory.
|
||||
const cleanup = () => {
|
||||
video.pause();
|
||||
video.removeAttribute("src");
|
||||
video.load();
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
};
|
||||
video.src = objectUrl;
|
||||
video.preload = "auto";
|
||||
video.src = descriptor.streamUrl;
|
||||
video.load();
|
||||
try {
|
||||
await waitForSeekableArchive(
|
||||
@@ -483,22 +279,15 @@ export function RecordedFmp4Player({
|
||||
recordedDelivery?.timelineEndSeconds,
|
||||
],
|
||||
);
|
||||
const [archive, setArchive] = useState<VerifiedRecordedMediaArchive | null>(null);
|
||||
const [archive, setArchive] = useState<RecordedMediaArchive | null>(null);
|
||||
const [state, setState] = useState<"loading" | "ready" | "error">("loading");
|
||||
const [readyGeneration, setReadyGeneration] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<RecordedMediaLoadProgress | null>(null);
|
||||
const [bufferRevision, setBufferRevision] = useState(0);
|
||||
const currentSeconds = playback?.currentSeconds ?? contract?.timelineStartSeconds ?? 0;
|
||||
const epoch = useMemo(
|
||||
() => selectRecordedMediaEpoch(archive?.manifest.epochs ?? [], currentSeconds),
|
||||
[archive?.manifest.epochs, currentSeconds],
|
||||
);
|
||||
const verifiedEpoch = useMemo(
|
||||
() => epoch
|
||||
? archive?.epochs.find(({ descriptor }) => descriptor.ordinal === epoch.ordinal) ?? null
|
||||
: null,
|
||||
[archive?.epochs, epoch],
|
||||
);
|
||||
const waitingForEpoch = Boolean(archive && !epoch);
|
||||
const selectedGeneration = contract && epoch
|
||||
? `${contract.manifestGenerationSha256}:${epoch.ordinal}:${epoch.timelineStartSeconds}:${epoch.timelineEndSeconds}`
|
||||
@@ -514,7 +303,6 @@ export function RecordedFmp4Player({
|
||||
useEffect(() => {
|
||||
if (!contract) {
|
||||
setArchive(null);
|
||||
setProgress(null);
|
||||
setReadyGeneration(null);
|
||||
setState("error");
|
||||
reportAdmission({
|
||||
@@ -527,7 +315,6 @@ export function RecordedFmp4Player({
|
||||
if (!prepare) return;
|
||||
const abort = new AbortController();
|
||||
setArchive(null);
|
||||
setProgress(null);
|
||||
setReadyGeneration(null);
|
||||
setState("loading");
|
||||
reportAdmission({
|
||||
@@ -535,12 +322,7 @@ export function RecordedFmp4Player({
|
||||
byteLength: contract.byteLength,
|
||||
message: null,
|
||||
});
|
||||
void fetchVerifiedRecordedMediaArchive(contract, {
|
||||
signal: abort.signal,
|
||||
onProgress: (next) => {
|
||||
if (!abort.signal.aborted) setProgress(next);
|
||||
},
|
||||
})
|
||||
void fetchRecordedMediaArchive(contract, { signal: abort.signal })
|
||||
.then((loaded) => {
|
||||
if (abort.signal.aborted) return;
|
||||
setArchive(loaded);
|
||||
@@ -566,11 +348,11 @@ export function RecordedFmp4Player({
|
||||
const abort = new AbortController();
|
||||
let disposed = false;
|
||||
void (async () => {
|
||||
for (const candidate of archive.epochs) {
|
||||
for (const candidate of archive.manifest.epochs) {
|
||||
const probe = document.createElement("video");
|
||||
probe.muted = true;
|
||||
probe.playsInline = true;
|
||||
const cleanup = await mountVerifiedRecordedEpoch(probe, candidate, abort.signal);
|
||||
const cleanup = await mountRecordedEpochStream(probe, candidate, abort.signal);
|
||||
cleanup();
|
||||
if (disposed || abort.signal.aborted) return;
|
||||
}
|
||||
@@ -602,8 +384,8 @@ export function RecordedFmp4Player({
|
||||
|
||||
useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
if (!video || !verifiedEpoch) return;
|
||||
const epochDescriptor = verifiedEpoch.descriptor;
|
||||
if (!video || !epoch) return;
|
||||
const epochDescriptor = epoch;
|
||||
const generation = contract
|
||||
? `${contract.manifestGenerationSha256}:${epochDescriptor.ordinal}:${epochDescriptor.timelineStartSeconds}:${epochDescriptor.timelineEndSeconds}`
|
||||
: null;
|
||||
@@ -615,7 +397,7 @@ export function RecordedFmp4Player({
|
||||
|
||||
const loadEpoch = async () => {
|
||||
try {
|
||||
cleanup = await mountVerifiedRecordedEpoch(video, verifiedEpoch, abort.signal);
|
||||
cleanup = await mountRecordedEpochStream(video, epochDescriptor, abort.signal);
|
||||
if (disposed || abort.signal.aborted) {
|
||||
cleanup();
|
||||
cleanup = null;
|
||||
@@ -648,7 +430,7 @@ export function RecordedFmp4Player({
|
||||
abort.abort();
|
||||
cleanup?.();
|
||||
};
|
||||
}, [archive?.byteLength, contract, verifiedEpoch]);
|
||||
}, [archive?.byteLength, contract, epoch]);
|
||||
|
||||
useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
@@ -679,10 +461,6 @@ export function RecordedFmp4Player({
|
||||
}
|
||||
}, [archive?.byteLength, bufferRevision, currentSeconds, epoch, playback?.playing, visualState]);
|
||||
|
||||
const progressPercent = progress && progress.totalBytes > 0
|
||||
? Math.min(100, Math.floor((progress.loadedBytes / progress.totalBytes) * 100))
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div
|
||||
className="recorded-media-player"
|
||||
@@ -707,8 +485,8 @@ export function RecordedFmp4Player({
|
||||
: visualState === "error"
|
||||
? "Записанное видео недоступно"
|
||||
: archive
|
||||
? "Проверяем полную готовность записанного видео…"
|
||||
: `Загружаем и проверяем записанное видео · ${progressPercent}%`}
|
||||
? "Проверяем seek и codec записанного видео…"
|
||||
: "Читаем manifest записанного видео…"}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
@@ -4,7 +4,14 @@ import type { SceneSettings } from "../sceneSettings";
|
||||
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
|
||||
|
||||
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
|
||||
export type RecordedRerunView = "spatial" | "perception" | "metrics";
|
||||
export type RecordedRerunView = "spatial" | "perception" | "perception3d" | "metrics";
|
||||
|
||||
export interface RecordedPerceptionLayers {
|
||||
enabled: boolean;
|
||||
detections2d: boolean;
|
||||
segmentation: boolean;
|
||||
cuboids3d: boolean;
|
||||
}
|
||||
|
||||
export interface RerunSelection {
|
||||
entityPath: string;
|
||||
@@ -57,6 +64,8 @@ export interface RerunViewportProps {
|
||||
| "customColor"
|
||||
>;
|
||||
recordedView?: RecordedRerunView;
|
||||
recordedViewResetGeneration?: 0 | 1;
|
||||
recordedPerceptionLayers?: RecordedPerceptionLayers;
|
||||
onPerceptionAvailabilityChange?: (available: boolean) => void;
|
||||
}
|
||||
|
||||
@@ -438,11 +447,20 @@ export async function fetchRecordedBlueprintRrd(
|
||||
origin,
|
||||
signal,
|
||||
activeView = "spatial",
|
||||
viewResetGeneration = 0,
|
||||
perceptionLayers = {
|
||||
enabled: false,
|
||||
detections2d: false,
|
||||
segmentation: false,
|
||||
cuboids3d: false,
|
||||
},
|
||||
fetcher = globalThis.fetch,
|
||||
}: {
|
||||
origin: string;
|
||||
signal?: AbortSignal;
|
||||
activeView?: RecordedRerunView;
|
||||
viewResetGeneration?: 0 | 1;
|
||||
perceptionLayers?: RecordedPerceptionLayers;
|
||||
fetcher?: typeof globalThis.fetch;
|
||||
},
|
||||
): Promise<Uint8Array> {
|
||||
@@ -461,7 +479,14 @@ export async function fetchRecordedBlueprintRrd(
|
||||
settings.pointSize > 32 ||
|
||||
!["turbo", "viridis", "plasma", "grayscale", "custom"].includes(settings.palette) ||
|
||||
!/^#[0-9A-Fa-f]{6}$/.test(settings.customColor) ||
|
||||
!["spatial", "perception", "metrics"].includes(activeView) ||
|
||||
!["spatial", "perception", "perception3d", "metrics"].includes(activeView) ||
|
||||
![0, 1].includes(viewResetGeneration) ||
|
||||
[
|
||||
perceptionLayers.enabled,
|
||||
perceptionLayers.detections2d,
|
||||
perceptionLayers.segmentation,
|
||||
perceptionLayers.cuboids3d,
|
||||
].some((value) => typeof value !== "boolean") ||
|
||||
identity.applicationId !== "nodedc_mission_core_recorded" ||
|
||||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(identity.recordingId)
|
||||
) {
|
||||
@@ -485,6 +510,11 @@ export async function fetchRecordedBlueprintRrd(
|
||||
palette: settings.palette,
|
||||
custom_color: settings.customColor,
|
||||
active_view: activeView,
|
||||
view_reset_generation: viewResetGeneration,
|
||||
unified_perception: perceptionLayers.enabled,
|
||||
show_detections_2d: perceptionLayers.detections2d,
|
||||
show_segmentation: perceptionLayers.segmentation,
|
||||
show_cuboids_3d: perceptionLayers.cuboids3d,
|
||||
}),
|
||||
signal,
|
||||
});
|
||||
@@ -588,6 +618,13 @@ export function RerunViewport({
|
||||
onPlaybackControllerChange,
|
||||
sceneSettings,
|
||||
recordedView = "spatial",
|
||||
recordedViewResetGeneration = 0,
|
||||
recordedPerceptionLayers = {
|
||||
enabled: false,
|
||||
detections2d: false,
|
||||
segmentation: false,
|
||||
cuboids3d: false,
|
||||
},
|
||||
onPerceptionAvailabilityChange,
|
||||
}: RerunViewportProps) {
|
||||
const hostRef = useRef<HTMLDivElement>(null);
|
||||
@@ -596,6 +633,7 @@ export function RerunViewport({
|
||||
const [retryNonce, setRetryNonce] = useState(0);
|
||||
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
|
||||
const recordedIdentityRef = useRef<RecordedRerunIdentity | null>(null);
|
||||
const presentationGateRef = useRef(presentationGate);
|
||||
presentationGateRef.current = presentationGate;
|
||||
@@ -1123,11 +1161,11 @@ export function RerunViewport({
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
onPerceptionAvailabilityChange?.(false);
|
||||
}, [onPerceptionAvailabilityChange, recordedPerceptionUrl]);
|
||||
loadedPerceptionChannelRef.current = null;
|
||||
}, [recordedPerceptionUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!recordedPerceptionUrl) return;
|
||||
if (!recordedPerceptionUrl || !recordedPerceptionLayers.enabled) return;
|
||||
const active = perceptionChannelRef.current;
|
||||
const identity = recordedIdentityRef.current;
|
||||
if (
|
||||
@@ -1136,6 +1174,10 @@ export function RerunViewport({
|
||||
active.endpointUrl !== recordedPerceptionUrl ||
|
||||
!active.channel.ready
|
||||
) return;
|
||||
if (loadedPerceptionChannelRef.current === active) {
|
||||
onPerceptionAvailabilityChange?.(true);
|
||||
return;
|
||||
}
|
||||
const abort = new AbortController();
|
||||
void fetchRecordedPerceptionRrd(recordedPerceptionUrl, identity, {
|
||||
origin: window.location.origin,
|
||||
@@ -1154,6 +1196,7 @@ export function RerunViewport({
|
||||
return;
|
||||
}
|
||||
active.channel.send_rrd(payload);
|
||||
loadedPerceptionChannelRef.current = active;
|
||||
onPerceptionAvailabilityChange?.(true);
|
||||
}).catch(() => {
|
||||
onPerceptionAvailabilityChange?.(false);
|
||||
@@ -1163,6 +1206,7 @@ export function RerunViewport({
|
||||
}, [
|
||||
onPerceptionAvailabilityChange,
|
||||
perceptionChannelRevision,
|
||||
recordedPerceptionLayers.enabled,
|
||||
recordedPerceptionUrl,
|
||||
]);
|
||||
|
||||
@@ -1181,6 +1225,8 @@ export function RerunViewport({
|
||||
origin: window.location.origin,
|
||||
signal: abort.signal,
|
||||
activeView: recordedView,
|
||||
viewResetGeneration: recordedViewResetGeneration,
|
||||
perceptionLayers: recordedPerceptionLayers,
|
||||
}).then((payload) => {
|
||||
if (
|
||||
abort.signal.aborted ||
|
||||
@@ -1200,6 +1246,11 @@ export function RerunViewport({
|
||||
blueprintChannelRevision,
|
||||
recordedBlueprintUrl,
|
||||
recordedView,
|
||||
recordedViewResetGeneration,
|
||||
recordedPerceptionLayers.enabled,
|
||||
recordedPerceptionLayers.detections2d,
|
||||
recordedPerceptionLayers.segmentation,
|
||||
recordedPerceptionLayers.cuboids3d,
|
||||
sceneSettings?.accumulationSeconds,
|
||||
sceneSettings?.customColor,
|
||||
sceneSettings?.palette,
|
||||
|
||||
@@ -38,47 +38,57 @@ export function recordedObservationSources(
|
||||
spatialRegistration: "native",
|
||||
},
|
||||
};
|
||||
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => ({
|
||||
id: source.id,
|
||||
sourceId: source.id,
|
||||
semanticChannelId: "camera.video.recorded",
|
||||
label: source.label,
|
||||
description: "Сохранённый видеоканал на общей временной шкале сессии",
|
||||
modality: "video",
|
||||
role: "auxiliary",
|
||||
availability: "available",
|
||||
transport: "recording",
|
||||
endpointLabel: "Сохранённая сессия",
|
||||
previewUrl: null,
|
||||
delivery: {
|
||||
id: `${launch.sessionId}:${source.id}`,
|
||||
kind: "recorded-fmp4-manifest",
|
||||
url: source.manifestUrl,
|
||||
mediaType: source.mediaType,
|
||||
manifestGenerationSha256: source.manifestGenerationSha256,
|
||||
byteLength: source.byteLength,
|
||||
timelineStartSeconds: source.timelineStartSeconds,
|
||||
timelineEndSeconds: source.timelineEndSeconds,
|
||||
},
|
||||
activation: null,
|
||||
provider: {
|
||||
pluginId: "missioncore.session-archive",
|
||||
pluginVersion: "1",
|
||||
modelId: "recorded-media",
|
||||
compatibilityProfileId: null,
|
||||
},
|
||||
binding: {},
|
||||
capabilities: {
|
||||
overlay: true,
|
||||
fullscreen: true,
|
||||
resizable: true,
|
||||
defaultVisible: index < 2,
|
||||
timelineMode: "recorded",
|
||||
seekable: true,
|
||||
sessionRecording: true,
|
||||
clockId: "session_time",
|
||||
spatialRegistration: "unresolved",
|
||||
},
|
||||
}));
|
||||
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => {
|
||||
const perception = source.id.startsWith("recorded.perception.");
|
||||
return {
|
||||
id: source.id,
|
||||
sourceId: source.id,
|
||||
semanticChannelId: perception
|
||||
? "camera.perception.panoptic.recorded"
|
||||
: "camera.video.recorded",
|
||||
label: source.label,
|
||||
description: perception
|
||||
? "Покадровая instance + semantic сегментация на общей временной шкале"
|
||||
: "Сохранённый видеоканал на общей временной шкале сессии",
|
||||
modality: "video",
|
||||
role: "auxiliary",
|
||||
availability: "available",
|
||||
transport: "recording",
|
||||
endpointLabel: "Сохранённая сессия",
|
||||
previewUrl: null,
|
||||
delivery: {
|
||||
id: `${launch.sessionId}:${source.id}`,
|
||||
kind: "recorded-fmp4-manifest",
|
||||
url: source.manifestUrl,
|
||||
mediaType: source.mediaType,
|
||||
manifestGenerationSha256: source.manifestGenerationSha256,
|
||||
byteLength: source.byteLength,
|
||||
timelineStartSeconds: source.timelineStartSeconds,
|
||||
timelineEndSeconds: source.timelineEndSeconds,
|
||||
},
|
||||
activation: null,
|
||||
provider: {
|
||||
pluginId: "missioncore.session-archive",
|
||||
pluginVersion: "1",
|
||||
modelId: perception ? "recorded-panoptic-perception" : "recorded-media",
|
||||
compatibilityProfileId: null,
|
||||
},
|
||||
binding: {},
|
||||
capabilities: {
|
||||
overlay: true,
|
||||
fullscreen: true,
|
||||
resizable: true,
|
||||
// AI presentation belongs to the unified Rerun composition. Keep the
|
||||
// pre-rendered perception video as an explicit fallback instead of
|
||||
// opening it as a second, independently controlled stream.
|
||||
defaultVisible: !perception && index < 2,
|
||||
timelineMode: "recorded",
|
||||
seekable: true,
|
||||
sessionRecording: true,
|
||||
clockId: "session_time",
|
||||
spatialRegistration: perception ? "calibrated" : "unresolved",
|
||||
},
|
||||
};
|
||||
});
|
||||
return [spatial, ...media];
|
||||
}
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
// A device-agnostic frontend safety policy. Sixteen channels covers multi-rig
|
||||
// vehicles while the independent byte/concurrency limits keep admission
|
||||
// bounded. One real accepted archive is ~163 MiB for a single camera, so the
|
||||
// old 128 MiB laboratory ceiling rejected a valid sealed generation before the
|
||||
// first manifest request. OPFS-backed sealed generations remain the scaling
|
||||
// path beyond this in-memory policy.
|
||||
// Camera count and preparation concurrency remain device-agnostic scheduling
|
||||
// policy. Recorded duration and aggregate bytes are deliberately not admission
|
||||
// criteria: sealed media is presented through a generation-bound HTTP stream,
|
||||
// so a one-, three- or ten-hour recording never has to fit in browser memory.
|
||||
export const MAX_RECORDED_CAMERA_SOURCES = 16;
|
||||
export const MAX_RECORDED_MEDIA_SOURCE_BYTES = 256 * 1024 * 1024;
|
||||
export const MAX_RECORDED_SESSION_CAMERA_BYTES = 512 * 1024 * 1024;
|
||||
export const MAX_CONCURRENT_RECORDED_CAMERA_PREPARATIONS = 1;
|
||||
|
||||
export type RecordedAdmissionPhase = "loading" | "ready" | "error";
|
||||
@@ -32,18 +28,12 @@ export function recordedCameraDescriptorPreflight(
|
||||
): RecordedAdmissionPhase {
|
||||
if (sources.length > MAX_RECORDED_CAMERA_SOURCES) return "error";
|
||||
if (new Set(sources.map(({ id }) => id)).size !== sources.length) return "error";
|
||||
let totalBytes = 0;
|
||||
for (const source of sources) {
|
||||
if (
|
||||
!source.id ||
|
||||
!Number.isSafeInteger(source.byteLength) ||
|
||||
source.byteLength < 1 ||
|
||||
source.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
|
||||
source.byteLength < 1
|
||||
) return "error";
|
||||
totalBytes += source.byteLength;
|
||||
if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) {
|
||||
return "error";
|
||||
}
|
||||
}
|
||||
return "ready";
|
||||
}
|
||||
@@ -67,7 +57,6 @@ export function recordedSessionAdmissionPhase(
|
||||
if (sourceIds.length > MAX_RECORDED_CAMERA_SOURCES || spatialPhase === "error") {
|
||||
return "error";
|
||||
}
|
||||
let totalBytes = 0;
|
||||
for (const sourceId of sourceIds) {
|
||||
const camera = cameras[sourceId];
|
||||
if (!camera || camera.phase === "error") return "error";
|
||||
@@ -75,13 +64,10 @@ export function recordedSessionAdmissionPhase(
|
||||
if (camera.byteLength !== null) {
|
||||
if (
|
||||
!Number.isSafeInteger(camera.byteLength) ||
|
||||
camera.byteLength < 1 ||
|
||||
camera.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
|
||||
camera.byteLength < 1
|
||||
) return "error";
|
||||
totalBytes += camera.byteLength;
|
||||
}
|
||||
}
|
||||
if (totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) return "error";
|
||||
if (spatialPhase !== "ready") return "loading";
|
||||
return sourceIds.every((sourceId) => cameras[sourceId]?.phase === "ready")
|
||||
? "ready"
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import {
|
||||
MAX_RECORDED_CAMERA_SOURCES,
|
||||
MAX_RECORDED_MEDIA_SOURCE_BYTES,
|
||||
MAX_RECORDED_SESSION_CAMERA_BYTES,
|
||||
} from "./recordedSessionAdmission";
|
||||
|
||||
export type ObservationSessionStatus =
|
||||
@@ -99,19 +97,8 @@ export interface ObservationRecordedMediaEpoch {
|
||||
timelineStartSeconds: number;
|
||||
timelineEndSeconds: number;
|
||||
mediaType: string;
|
||||
initUrl: string;
|
||||
initByteLength: number;
|
||||
initSha256: string;
|
||||
segmentCount: number;
|
||||
segmentUrlPrefix: string;
|
||||
segments: readonly ObservationRecordedMediaSegment[];
|
||||
}
|
||||
|
||||
export interface ObservationRecordedMediaSegment {
|
||||
sequence: number;
|
||||
url: string;
|
||||
byteLength: number;
|
||||
sha256: string;
|
||||
streamUrl: string;
|
||||
}
|
||||
|
||||
export interface ObservationRecordedMediaManifest {
|
||||
@@ -215,18 +202,8 @@ const RECORDED_MEDIA_EPOCH_KEYS = new Set([
|
||||
"timeline_start_seconds",
|
||||
"timeline_end_seconds",
|
||||
"media_type",
|
||||
"init_url",
|
||||
"init_byte_length",
|
||||
"init_sha256",
|
||||
"segment_count",
|
||||
"segment_url_prefix",
|
||||
"segments",
|
||||
]);
|
||||
const RECORDED_MEDIA_SEGMENT_KEYS = new Set([
|
||||
"sequence",
|
||||
"url",
|
||||
"byte_length",
|
||||
"sha256",
|
||||
"stream_url",
|
||||
]);
|
||||
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
||||
const SAFE_MODALITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
|
||||
@@ -234,10 +211,9 @@ const ISO_WITH_TIMEZONE = /^\d{4}-\d{2}-\d{2}T.+(?:Z|[+-]\d{2}:\d{2})$/;
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const SAFE_RECORDING_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording\.rrd$/;
|
||||
const SAFE_PREPARATION_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording-preparation$/;
|
||||
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/media\/[A-Za-z0-9._%-]+\/manifest$/;
|
||||
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+|perception-media\/result-[a-f0-9]{64})\/manifest$/;
|
||||
const SAFE_MEDIA_STREAM_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+\/epochs\/[1-9][0-9]*|perception-media\/result-[a-f0-9]{64})\/recording\.mp4\?generation=[a-f0-9]{64}$/;
|
||||
const SAFE_MP4_MEDIA_TYPE = /^video\/mp4(?:; codecs="[A-Za-z0-9.,_-]+")?$/;
|
||||
const MAX_RECORDED_INIT_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_RECORDED_SEGMENT_BYTES = 64 * 1024 * 1024;
|
||||
|
||||
export class ObservationSessionContractError extends Error {
|
||||
constructor(message: string) {
|
||||
@@ -570,15 +546,6 @@ export function decodeObservationSessionReplay(
|
||||
"Descriptor записи содержит повторяющиеся медиаканалы.",
|
||||
);
|
||||
}
|
||||
const mediaByteLength = mediaSources.reduce((total, source) => total + source.byteLength, 0);
|
||||
if (
|
||||
!Number.isSafeInteger(mediaByteLength) ||
|
||||
mediaByteLength > MAX_RECORDED_SESSION_CAMERA_BYTES
|
||||
) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Совокупный объём записанных медиаканалов превышает безопасный лимит браузера.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
kind: "rerun-recording",
|
||||
sessionId,
|
||||
@@ -709,7 +676,10 @@ function decodeRecordedMediaSource(
|
||||
}
|
||||
assertExactKeys(value, RECORDED_MEDIA_SOURCE_KEYS, `Медиаканал ${index}`);
|
||||
const id = requireString(value.id, `media_sources[${index}].id`, 128);
|
||||
if (!SAFE_ID.test(id) || !id.startsWith("recorded.camera.")) {
|
||||
if (
|
||||
!SAFE_ID.test(id) ||
|
||||
(!id.startsWith("recorded.camera.") && !id.startsWith("recorded.perception."))
|
||||
) {
|
||||
throw new ObservationSessionContractError("Медиаканал содержит небезопасный opaque id.");
|
||||
}
|
||||
if (value.modality !== "video" || value.media_type !== "video/mp4") {
|
||||
@@ -728,10 +698,10 @@ function decodeRecordedMediaSource(
|
||||
"Медиаканал не содержит immutable generation SHA-256.",
|
||||
);
|
||||
}
|
||||
const sessionPrefix = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/media/`;
|
||||
const sessionBase = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/`;
|
||||
if (
|
||||
!SAFE_MEDIA_MANIFEST_URL.test(manifestUrl) ||
|
||||
!manifestUrl.startsWith(sessionPrefix) ||
|
||||
!manifestUrl.startsWith(sessionBase) ||
|
||||
manifestUrl.includes("..")
|
||||
) {
|
||||
throw new ObservationSessionContractError(
|
||||
@@ -765,7 +735,7 @@ function decodeRecordedMediaSource(
|
||||
byteLength: requireFiniteNumber(
|
||||
value.byte_length,
|
||||
`media_sources[${index}].byte_length`,
|
||||
{ minimum: 1, maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES, integer: true },
|
||||
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
|
||||
),
|
||||
mediaType: "video/mp4",
|
||||
timelineStartSeconds,
|
||||
@@ -784,22 +754,21 @@ export function decodeObservationRecordedMediaManifest(
|
||||
}
|
||||
assertExactKeys(payload, RECORDED_MEDIA_MANIFEST_KEYS, "Manifest записанного видео");
|
||||
if (
|
||||
payload.schema_version !== "missioncore.observation-recorded-media/v2" ||
|
||||
payload.schema_version !== "missioncore.observation-recorded-media/v3" ||
|
||||
payload.source_id !== source.id ||
|
||||
typeof payload.generation_sha256 !== "string" ||
|
||||
!SHA256.test(payload.generation_sha256) ||
|
||||
payload.generation_sha256 !== source.manifestGenerationSha256 ||
|
||||
payload.synchronization !== "host-arrival-best-effort" ||
|
||||
!Array.isArray(payload.epochs) ||
|
||||
payload.epochs.length < 1 ||
|
||||
payload.epochs.length > 1_000
|
||||
payload.epochs.length < 1
|
||||
) {
|
||||
throw new ObservationSessionContractError("Manifest записанного видео несовместим.");
|
||||
}
|
||||
const manifestBase = source.manifestUrl.slice(0, -"/manifest".length);
|
||||
const manifestByteLength = requireFiniteNumber(payload.byte_length, "manifest.byte_length", {
|
||||
minimum: 1,
|
||||
maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES,
|
||||
maximum: Number.MAX_SAFE_INTEGER,
|
||||
integer: true,
|
||||
});
|
||||
if (manifestByteLength !== source.byteLength) {
|
||||
@@ -833,7 +802,7 @@ export function decodeObservationRecordedMediaManifest(
|
||||
assertExactKeys(entry, RECORDED_MEDIA_EPOCH_KEYS, `Codec epoch ${index}`);
|
||||
const ordinal = requireFiniteNumber(entry.ordinal, `epochs[${index}].ordinal`, {
|
||||
minimum: 1,
|
||||
maximum: 1_000,
|
||||
maximum: Number.MAX_SAFE_INTEGER,
|
||||
integer: true,
|
||||
});
|
||||
if (ordinal !== index + 1) {
|
||||
@@ -860,87 +829,28 @@ export function decodeObservationRecordedMediaManifest(
|
||||
if (!SAFE_MP4_MEDIA_TYPE.test(mediaType)) {
|
||||
throw new ObservationSessionContractError("Codec epoch содержит небезопасный media type.");
|
||||
}
|
||||
const initUrl = requireString(entry.init_url, `epochs[${index}].init_url`, 512);
|
||||
const segmentUrlPrefix = requireString(
|
||||
entry.segment_url_prefix,
|
||||
`epochs[${index}].segment_url_prefix`,
|
||||
512,
|
||||
const byteLength = requireFiniteNumber(
|
||||
entry.byte_length,
|
||||
`epochs[${index}].byte_length`,
|
||||
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
|
||||
);
|
||||
const expectedEpochBase = `${manifestBase}/epochs/${ordinal}`;
|
||||
const streamUrl = requireString(entry.stream_url, `epochs[${index}].stream_url`, 768);
|
||||
const expectedEpochBase = source.id.startsWith("recorded.perception.")
|
||||
? manifestBase
|
||||
: `${manifestBase}/epochs/${ordinal}`;
|
||||
const expectedStreamUrl =
|
||||
`${expectedEpochBase}/recording.mp4?generation=${payload.generation_sha256}`;
|
||||
if (
|
||||
initUrl !== `${expectedEpochBase}/init.mp4` ||
|
||||
segmentUrlPrefix !== `${expectedEpochBase}/segments/` ||
|
||||
initUrl.includes("..") ||
|
||||
segmentUrlPrefix.includes("..")
|
||||
!SAFE_MEDIA_STREAM_URL.test(streamUrl) ||
|
||||
streamUrl !== expectedStreamUrl ||
|
||||
streamUrl.includes("..")
|
||||
) {
|
||||
throw new ObservationSessionContractError("Codec epoch содержит небезопасный API URL.");
|
||||
}
|
||||
const initByteLength = requireFiniteNumber(
|
||||
entry.init_byte_length,
|
||||
`epochs[${index}].init_byte_length`,
|
||||
{ minimum: 1, maximum: MAX_RECORDED_INIT_BYTES, integer: true },
|
||||
);
|
||||
if (typeof entry.init_sha256 !== "string" || !SHA256.test(entry.init_sha256)) {
|
||||
throw new ObservationSessionContractError("Codec epoch не содержит SHA-256 init-сегмента.");
|
||||
}
|
||||
const segmentCount = requireFiniteNumber(
|
||||
entry.segment_count,
|
||||
`epochs[${index}].segment_count`,
|
||||
{ minimum: 1, maximum: 500_000, integer: true },
|
||||
);
|
||||
if (!Array.isArray(entry.segments) || entry.segments.length !== segmentCount) {
|
||||
declaredBytes += byteLength;
|
||||
if (!Number.isSafeInteger(declaredBytes)) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Codec epoch содержит неполный список канонических сегментов.",
|
||||
);
|
||||
}
|
||||
const segments = entry.segments.map((segment, segmentIndex): ObservationRecordedMediaSegment => {
|
||||
if (!isRecord(segment)) {
|
||||
throw new ObservationSessionContractError(
|
||||
`Сегмент epochs[${index}].segments[${segmentIndex}] должен быть объектом.`,
|
||||
);
|
||||
}
|
||||
assertExactKeys(
|
||||
segment,
|
||||
RECORDED_MEDIA_SEGMENT_KEYS,
|
||||
`Сегмент epochs[${index}].segments[${segmentIndex}]`,
|
||||
);
|
||||
const sequence = requireFiniteNumber(
|
||||
segment.sequence,
|
||||
`epochs[${index}].segments[${segmentIndex}].sequence`,
|
||||
{ minimum: 1, maximum: 500_000, integer: true },
|
||||
);
|
||||
if (sequence !== segmentIndex + 1) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Codec epoch содержит непоследовательный сегмент.",
|
||||
);
|
||||
}
|
||||
const url = requireString(
|
||||
segment.url,
|
||||
`epochs[${index}].segments[${segmentIndex}].url`,
|
||||
512,
|
||||
);
|
||||
if (url !== `${segmentUrlPrefix}${sequence}.m4s` || url.includes("..")) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Codec epoch содержит небезопасный URL сегмента.",
|
||||
);
|
||||
}
|
||||
const byteLength = requireFiniteNumber(
|
||||
segment.byte_length,
|
||||
`epochs[${index}].segments[${segmentIndex}].byte_length`,
|
||||
{ minimum: 1, maximum: MAX_RECORDED_SEGMENT_BYTES, integer: true },
|
||||
);
|
||||
if (typeof segment.sha256 !== "string" || !SHA256.test(segment.sha256)) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Codec epoch содержит сегмент без SHA-256.",
|
||||
);
|
||||
}
|
||||
declaredBytes += byteLength;
|
||||
return { sequence, url, byteLength, sha256: segment.sha256 };
|
||||
});
|
||||
declaredBytes += initByteLength;
|
||||
if (declaredBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Записанный медиаканал превышает безопасный лимит браузера.",
|
||||
"Суммарный размер codec epoch выходит за точный числовой диапазон клиента.",
|
||||
);
|
||||
}
|
||||
return {
|
||||
@@ -948,17 +858,13 @@ export function decodeObservationRecordedMediaManifest(
|
||||
timelineStartSeconds,
|
||||
timelineEndSeconds,
|
||||
mediaType,
|
||||
initUrl,
|
||||
initByteLength,
|
||||
initSha256: entry.init_sha256,
|
||||
segmentCount,
|
||||
segmentUrlPrefix,
|
||||
segments,
|
||||
byteLength,
|
||||
streamUrl,
|
||||
};
|
||||
});
|
||||
if (declaredBytes !== manifestByteLength) {
|
||||
throw new ObservationSessionContractError(
|
||||
"Сумма init- и media-сегментов не совпадает с размером immutable manifest.",
|
||||
"Сумма codec epoch не совпадает с размером immutable manifest.",
|
||||
);
|
||||
}
|
||||
if (
|
||||
|
||||
@@ -285,11 +285,16 @@ export async function resolveObservationSessionReplay(
|
||||
sessionId: string,
|
||||
options: ObservationPreparationPollingOptions,
|
||||
): Promise<ObservationSessionReplayLaunch> {
|
||||
const response = await withRequestTimeout(
|
||||
options.signal,
|
||||
Math.max(100, options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS),
|
||||
(signal) => replayObservationSession(sessionId, { signal, fetcher: options.fetcher }),
|
||||
);
|
||||
// The initial replay request may restore and integrity-check large, already
|
||||
// published artifacts before it can return either a launch descriptor or a
|
||||
// background-preparation handle. Its duration therefore scales with the
|
||||
// recording package and must not be confused with a stalled status poll.
|
||||
// Keep it cancellable by the owning UI attempt, but do not impose the short
|
||||
// per-poll timeout used once the server has returned a preparation handle.
|
||||
const response = await replayObservationSession(sessionId, {
|
||||
signal: options.signal,
|
||||
fetcher: options.fetcher,
|
||||
});
|
||||
if (response.kind === "ready") return response.launch;
|
||||
if (!pendingPreparation(response.preparation)) {
|
||||
options.onUpdate?.(response.preparation);
|
||||
|
||||
@@ -22,6 +22,9 @@ export interface ViewerSettings {
|
||||
show_points: boolean;
|
||||
show_trajectory: boolean;
|
||||
show_grid: boolean;
|
||||
show_detections_2d: boolean;
|
||||
show_segmentation: boolean;
|
||||
show_cuboids_3d: boolean;
|
||||
}
|
||||
|
||||
export interface StreamMetrics {
|
||||
@@ -29,6 +32,10 @@ export interface StreamMetrics {
|
||||
frameRateHz?: number | null;
|
||||
pointCount?: number | null;
|
||||
droppedPreviewFrames?: number | null;
|
||||
aiLatencyMs?: number | null;
|
||||
aiFrameRateHz?: number | null;
|
||||
aiDroppedFrames?: number | null;
|
||||
aiStaleMs?: number | null;
|
||||
elapsedSeconds?: number | null;
|
||||
routeDistanceMeters?: number | null;
|
||||
speedMetersPerSecond?: number | null;
|
||||
|
||||
@@ -32,6 +32,25 @@
|
||||
backdrop-filter: blur(18px);
|
||||
}
|
||||
|
||||
.scene-navigation-hint {
|
||||
position: absolute;
|
||||
z-index: 11;
|
||||
right: 0.85rem;
|
||||
bottom: 4.9rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
border: 1px solid rgb(255 255 255 / 0.07);
|
||||
border-radius: 999px;
|
||||
background: rgb(9 10 13 / 0.7);
|
||||
color: var(--nodedc-text-muted);
|
||||
padding: 0.42rem 0.65rem;
|
||||
font-size: 0.52rem;
|
||||
font-weight: 680;
|
||||
pointer-events: none;
|
||||
backdrop-filter: blur(14px);
|
||||
}
|
||||
|
||||
.scene-source-picker__trigger {
|
||||
position: relative;
|
||||
}
|
||||
@@ -350,6 +369,11 @@ i[data-availability="error"] {
|
||||
box-shadow: 0 1.5rem 4rem rgb(0 0 0 / 0.34);
|
||||
}
|
||||
|
||||
.floating-observation-window--hidden {
|
||||
visibility: hidden;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.observation-timeline {
|
||||
display: block;
|
||||
min-width: 0;
|
||||
|
||||
@@ -25,6 +25,20 @@
|
||||
gap: 0.45rem;
|
||||
}
|
||||
|
||||
.spatial-toolbar__view-switch {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.25rem;
|
||||
border: 1px solid rgb(255 255 255 / 0.07);
|
||||
border-radius: 999px;
|
||||
background: rgb(255 255 255 / 0.025);
|
||||
padding: 0.2rem;
|
||||
}
|
||||
|
||||
.spatial-toolbar__view-switch .nodedc-button {
|
||||
border-radius: 999px;
|
||||
}
|
||||
|
||||
.spatial-viewport-shell {
|
||||
position: relative;
|
||||
min-width: 0;
|
||||
|
||||
@@ -45,7 +45,6 @@ import {
|
||||
type RerunPlaybackState,
|
||||
type RerunSelection,
|
||||
type RerunViewportStatus,
|
||||
type RecordedRerunView,
|
||||
} from "../components/RerunViewport";
|
||||
import {
|
||||
capabilityStatusLabel,
|
||||
@@ -124,6 +123,16 @@ export interface WorkspaceRendererProps {
|
||||
accumulationSeconds: number;
|
||||
onAccumulationChange: (value: number) => void;
|
||||
onAccumulationCommit: () => void;
|
||||
livePerceptionLayers: {
|
||||
detections2d: boolean;
|
||||
segmentation: boolean;
|
||||
cuboids3d: boolean;
|
||||
};
|
||||
onLivePerceptionLayersChange: (next: {
|
||||
detections2d: boolean;
|
||||
segmentation: boolean;
|
||||
cuboids3d: boolean;
|
||||
}) => void;
|
||||
observationLayout: ObservationLayoutController;
|
||||
navigation: WorkspaceNavigation;
|
||||
spatialControls: {
|
||||
@@ -286,6 +295,8 @@ function SpatialWorkspace({
|
||||
accumulationSeconds,
|
||||
onAccumulationChange,
|
||||
onAccumulationCommit,
|
||||
livePerceptionLayers,
|
||||
onLivePerceptionLayersChange,
|
||||
observationLayout,
|
||||
navigation,
|
||||
spatialControls,
|
||||
@@ -295,8 +306,13 @@ function SpatialWorkspace({
|
||||
const [selection, setSelection] = useState<RerunSelection | null>(null);
|
||||
const [playbackState, setPlaybackState] = useState<RerunPlaybackState | null>(null);
|
||||
const [playbackController, setPlaybackController] = useState<RerunPlaybackController | null>(null);
|
||||
const [recordedRerunView, setRecordedRerunView] = useState<RecordedRerunView>("spatial");
|
||||
const [perceptionAvailable, setPerceptionAvailable] = useState(false);
|
||||
const [recordedViewResetGeneration, setRecordedViewResetGeneration] = useState<0 | 1>(0);
|
||||
const [perceptionAvailability, setPerceptionAvailability] = useState<
|
||||
"unknown" | "available" | "unavailable"
|
||||
>("unknown");
|
||||
const [showDetections2d, setShowDetections2d] = useState(false);
|
||||
const [showSegmentation, setShowSegmentation] = useState(false);
|
||||
const [showCuboids3d, setShowCuboids3d] = useState(false);
|
||||
const recordedSource = state?.sourceMode === "replay" || /\.rrd(?:$|[?#])/i.test(sourceUrl);
|
||||
const recordedSessionGate: RecordedAdmissionPhase = recordedSource
|
||||
? recordedSessionAdmission?.phase ?? "loading"
|
||||
@@ -309,6 +325,8 @@ function SpatialWorkspace({
|
||||
const latency = pipelineLatency(metrics);
|
||||
const frameRate = finiteMetric(metrics?.frameRateHz);
|
||||
const points = finiteMetric(metrics?.pointCount);
|
||||
const aiLatency = finiteMetric(metrics?.aiLatencyMs);
|
||||
const aiFrameRate = finiteMetric(metrics?.aiFrameRateHz);
|
||||
const observationSources = state?.observationSources ?? [];
|
||||
const pointCloudSource = observationSources.find((source) => source.modality === "point-cloud");
|
||||
const pointCloudVisible = pointCloudSource
|
||||
@@ -317,13 +335,29 @@ function SpatialWorkspace({
|
||||
const mediaSources = observationSources.filter(
|
||||
(source) => source.capabilities.overlay && source.modality !== "point-cloud",
|
||||
);
|
||||
const recordedPerceptionSupported = recordedSource && perceptionAvailability !== "unavailable";
|
||||
const recordedPerceptionEnabled = showDetections2d || showSegmentation || showCuboids3d;
|
||||
const unifiedPerception = recordedPerceptionSupported && recordedPerceptionEnabled;
|
||||
const livePerceptionAvailable = !recordedSource && streamActive;
|
||||
const detections2dActive = recordedSource
|
||||
? showDetections2d
|
||||
: livePerceptionLayers.detections2d;
|
||||
const segmentationActive = recordedSource
|
||||
? showSegmentation
|
||||
: livePerceptionLayers.segmentation;
|
||||
const cuboids3dActive = recordedSource
|
||||
? showCuboids3d
|
||||
: livePerceptionLayers.cuboids3d;
|
||||
const visibleMediaSources = mediaSources.filter((source) =>
|
||||
observationLayout.visibleSourceIds.has(source.id),
|
||||
observationLayout.visibleSourceIds.has(source.id) &&
|
||||
!source.id.startsWith("recorded.perception."),
|
||||
);
|
||||
const presentedMediaSourceCount = unifiedPerception ? 0 : visibleMediaSources.length;
|
||||
const pointCloudFocused = Boolean(
|
||||
pointCloudSource && observationLayout.focusedSourceId === pointCloudSource.id,
|
||||
);
|
||||
const floatingSourceMaximized = Boolean(observationLayout.maximizedFloatingSourceId);
|
||||
const floatingSourceMaximized = !unifiedPerception &&
|
||||
Boolean(observationLayout.maximizedFloatingSourceId);
|
||||
const timeline = state?.observationTimeline;
|
||||
const viewportRef = useRef<HTMLDivElement>(null);
|
||||
const intentionalSourceEnd = !recordedSource && [
|
||||
@@ -373,13 +407,20 @@ function SpatialWorkspace({
|
||||
[],
|
||||
);
|
||||
const onPerceptionAvailabilityChange = useCallback((available: boolean) => {
|
||||
setPerceptionAvailable(available);
|
||||
if (!available) setRecordedRerunView("spatial");
|
||||
setPerceptionAvailability(available ? "available" : "unavailable");
|
||||
if (!available) {
|
||||
setShowDetections2d(false);
|
||||
setShowSegmentation(false);
|
||||
setShowCuboids3d(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setPerceptionAvailable(false);
|
||||
setRecordedRerunView("spatial");
|
||||
setPerceptionAvailability("unknown");
|
||||
setShowDetections2d(false);
|
||||
setShowSegmentation(false);
|
||||
setShowCuboids3d(false);
|
||||
setRecordedViewResetGeneration(0);
|
||||
}, [sourceUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -389,8 +430,10 @@ function SpatialWorkspace({
|
||||
setSelection(null);
|
||||
setPlaybackState(null);
|
||||
setPlaybackController(null);
|
||||
setPerceptionAvailable(false);
|
||||
setRecordedRerunView("spatial");
|
||||
setPerceptionAvailability("unknown");
|
||||
setShowDetections2d(false);
|
||||
setShowSegmentation(false);
|
||||
setShowCuboids3d(false);
|
||||
}, [pointCloudVisible, sourceUrl]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -428,20 +471,74 @@ function SpatialWorkspace({
|
||||
className="spatial-workspace"
|
||||
data-focused={pointCloudFocused || floatingSourceMaximized ? "true" : undefined}
|
||||
>
|
||||
<div className="spatial-toolbar">
|
||||
<div className="spatial-toolbar" data-viewer-controls="v1">
|
||||
<div className="spatial-toolbar__mode">
|
||||
<span className="section-eyebrow">СЦЕНА 3D · RERUN</span>
|
||||
</div>
|
||||
<div className="spatial-toolbar__actions">
|
||||
{recordedSource && perceptionAvailable ? (
|
||||
{recordedPerceptionSupported || livePerceptionAvailable ? (
|
||||
<div className="spatial-toolbar__view-switch" role="group" aria-label="Слои распознавания сцены">
|
||||
<Button
|
||||
size="compact"
|
||||
variant="primary"
|
||||
icon={<Icon name="video" />}
|
||||
aria-pressed="true"
|
||||
disabled
|
||||
>
|
||||
Оригинал
|
||||
</Button>
|
||||
<Button
|
||||
size="compact"
|
||||
variant={detections2dActive ? "primary" : "secondary"}
|
||||
icon={<Icon name="target" />}
|
||||
aria-pressed={detections2dActive}
|
||||
onClick={() => recordedSource
|
||||
? setShowDetections2d((current) => !current)
|
||||
: onLivePerceptionLayersChange({
|
||||
...livePerceptionLayers,
|
||||
detections2d: !livePerceptionLayers.detections2d,
|
||||
})}
|
||||
>
|
||||
Объекты 2D
|
||||
</Button>
|
||||
<Button
|
||||
size="compact"
|
||||
variant={segmentationActive ? "primary" : "secondary"}
|
||||
icon={<Icon name="image" />}
|
||||
aria-pressed={segmentationActive}
|
||||
onClick={() => recordedSource
|
||||
? setShowSegmentation((current) => !current)
|
||||
: onLivePerceptionLayersChange({
|
||||
...livePerceptionLayers,
|
||||
segmentation: !livePerceptionLayers.segmentation,
|
||||
})}
|
||||
>
|
||||
Сегментация
|
||||
</Button>
|
||||
<Button
|
||||
size="compact"
|
||||
variant={cuboids3dActive ? "primary" : "secondary"}
|
||||
icon={<Icon name="apps" />}
|
||||
aria-pressed={cuboids3dActive}
|
||||
onClick={() => recordedSource
|
||||
? setShowCuboids3d((current) => !current)
|
||||
: onLivePerceptionLayersChange({
|
||||
...livePerceptionLayers,
|
||||
cuboids3d: !livePerceptionLayers.cuboids3d,
|
||||
})}
|
||||
>
|
||||
Кубы 3D
|
||||
</Button>
|
||||
</div>
|
||||
) : null}
|
||||
{recordedSource && presentedViewerStatus === "ready" ? (
|
||||
<Button
|
||||
size="compact"
|
||||
variant={recordedRerunView === "perception" ? "primary" : "secondary"}
|
||||
icon={<Icon name={recordedRerunView === "perception" ? "globe" : "image"} />}
|
||||
onClick={() => setRecordedRerunView((current) =>
|
||||
current === "perception" ? "spatial" : "perception")}
|
||||
variant="secondary"
|
||||
icon={<Icon name="refresh" />}
|
||||
onClick={() => setRecordedViewResetGeneration((current) => current === 0 ? 1 : 0)}
|
||||
>
|
||||
{recordedRerunView === "perception" ? "Облако точек" : "Распознавание"}
|
||||
Сброс вида
|
||||
</Button>
|
||||
) : null}
|
||||
<Button size="compact" variant="secondary" icon={<Icon name="network" />} onClick={navigation.openSource}>
|
||||
@@ -476,7 +573,13 @@ function SpatialWorkspace({
|
||||
? state?.observationTimeline?.range?.endSeconds
|
||||
: undefined}
|
||||
sceneSettings={sceneSettings}
|
||||
recordedView={recordedRerunView}
|
||||
recordedViewResetGeneration={recordedViewResetGeneration}
|
||||
recordedPerceptionLayers={{
|
||||
enabled: unifiedPerception,
|
||||
detections2d: showDetections2d,
|
||||
segmentation: showSegmentation,
|
||||
cuboids3d: showCuboids3d,
|
||||
}}
|
||||
onPerceptionAvailabilityChange={onPerceptionAvailabilityChange}
|
||||
onStatusChange={onStatusChange}
|
||||
onSelectionChange={onSelectionChange}
|
||||
@@ -511,7 +614,9 @@ function SpatialWorkspace({
|
||||
) : !floatingSourceMaximized ? (
|
||||
<div className="scene-source-controls">
|
||||
<ObservationSourcePicker
|
||||
sources={observationSources}
|
||||
sources={unifiedPerception
|
||||
? observationSources.filter((source) => source.modality === "point-cloud")
|
||||
: observationSources}
|
||||
visibleSourceIds={observationLayout.visibleSourceIds}
|
||||
pendingSourceIds={observationLayout.pendingSourceIds}
|
||||
onToggle={observationLayout.toggleSource}
|
||||
@@ -555,6 +660,16 @@ function SpatialWorkspace({
|
||||
<span>До публикации</span>
|
||||
<strong>{formatNumber(latency)}<small> мс</small></strong>
|
||||
</div>
|
||||
{streamActive ? (
|
||||
<div>
|
||||
<span>AI</span>
|
||||
<strong>
|
||||
{aiLatency === null ? "—" : formatNumber(aiLatency)}
|
||||
<small>{aiLatency === null ? "" : " мс"}</small>
|
||||
{aiFrameRate === null ? null : <small> · {formatNumber(aiFrameRate)} Гц</small>}
|
||||
</strong>
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{!pointCloudFocused && !floatingSourceMaximized && state?.sourceMode && state.sourceMode !== "idle" && !sourceUrl.trim() ? (
|
||||
@@ -575,10 +690,17 @@ function SpatialWorkspace({
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{presentedViewerStatus === "ready" && !floatingSourceMaximized ? (
|
||||
<div className="scene-navigation-hint" aria-label="Навигация по 3D-сцене">
|
||||
<span>Колесо · зум к курсору</span>
|
||||
<span>WASD · свободный проход</span>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{!pointCloudFocused && !floatingSourceMaximized && recordedPlaybackReady ? (
|
||||
<ObservationTimeline
|
||||
active={presentedViewerStatus === "ready"}
|
||||
sourceCount={Math.max(1, 1 + visibleMediaSources.length)}
|
||||
sourceCount={Math.max(1, (unifiedPerception ? 2 : 1) + presentedMediaSourceCount)}
|
||||
mode={recordedSource && playbackState?.rangeNs
|
||||
? "recorded"
|
||||
: timeline?.mode}
|
||||
@@ -599,7 +721,7 @@ function SpatialWorkspace({
|
||||
/>
|
||||
) : null}
|
||||
|
||||
{!pointCloudFocused ? visibleMediaSources.map((source, index) => (
|
||||
{visibleMediaSources.map((source, index) => (
|
||||
<FloatingObservationWindow
|
||||
key={source.id}
|
||||
source={source}
|
||||
@@ -609,6 +731,7 @@ function SpatialWorkspace({
|
||||
rect={observationLayout.windowRects[source.id]}
|
||||
maximized={observationLayout.maximizedFloatingSourceId === source.id}
|
||||
active={observationLayout.activeFloatingSourceId === source.id}
|
||||
hidden={pointCloudFocused || unifiedPerception}
|
||||
onRectChange={(rect) => observationLayout.setWindowRect(source.id, rect)}
|
||||
onMaximizedChange={(maximized) =>
|
||||
observationLayout.setFloatingMaximized(source.id, maximized)}
|
||||
@@ -627,12 +750,12 @@ function SpatialWorkspace({
|
||||
void observationLayout.hideSource(source.id);
|
||||
}}
|
||||
/>
|
||||
)) : null}
|
||||
))}
|
||||
{recordedSource ? (
|
||||
<div className="recorded-session-preloaders" aria-hidden="true">
|
||||
{mediaSources.filter((source) => (
|
||||
source.delivery?.kind === "recorded-fmp4-manifest" &&
|
||||
(pointCloudFocused || !observationLayout.visibleSourceIds.has(source.id)) &&
|
||||
!observationLayout.visibleSourceIds.has(source.id) &&
|
||||
shouldPrepareRecordedSource(source.id)
|
||||
)).map((source) => (
|
||||
<ObservationMedia
|
||||
@@ -657,7 +780,9 @@ function SpatialWorkspace({
|
||||
<span><i data-state="ready" />Траектория</span>
|
||||
<span><i data-state="ready" />Преобразования</span>
|
||||
<span><i data-state="contract" />Камеры в 3D</span>
|
||||
<span><i data-state={perceptionAvailable ? "ready" : "contract"} />Объекты / рамки</span>
|
||||
<span><i data-state={detections2dActive ? "ready" : "contract"} />Объекты 2D</span>
|
||||
<span><i data-state={segmentationActive ? "ready" : "contract"} />Сегментация</span>
|
||||
<span><i data-state={cuboids3dActive ? "ready" : "contract"} />Кубы 3D</span>
|
||||
<span><i data-state="contract" />Компоновка</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -484,6 +484,31 @@ test("202 preparation polls through explicit phases and reveals launch only when
|
||||
assert.deepEqual(calls.map((entry) => entry[2]), [null, preparationEtag, preparationEtag]);
|
||||
});
|
||||
|
||||
test("initial replay restore is not mistaken for a stalled preparation poll", async () => {
|
||||
const sessionId = "session-20260716T205632Z";
|
||||
let requestSignal;
|
||||
const launch = await resolveObservationSessionReplay(sessionId, {
|
||||
signal: new AbortController().signal,
|
||||
requestTimeoutMs: 100,
|
||||
fetcher: async (_input, init) => {
|
||||
requestSignal = init.signal;
|
||||
await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(resolve, 150);
|
||||
init.signal.addEventListener("abort", () => {
|
||||
clearTimeout(timer);
|
||||
reject(new DOMException("cancelled", "AbortError"));
|
||||
}, { once: true });
|
||||
});
|
||||
return new Response(JSON.stringify(replay({ session_id: sessionId })), {
|
||||
status: 200,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
});
|
||||
},
|
||||
});
|
||||
assert.equal(requestSignal.aborted, false);
|
||||
assert.equal(launch.sessionId, sessionId);
|
||||
});
|
||||
|
||||
test("status-ready response must match the preparation ETag before launch is accepted", async () => {
|
||||
const decoded = decodeObservationSessionPreparation(
|
||||
preparation({ state: "finalizing", progress: 0.95 }),
|
||||
@@ -704,6 +729,29 @@ test("switching saved sessions aborts only local polling and never cancels share
|
||||
assert.doesNotMatch(selectorSource, /cancelReplay|>\s*Отменить\s*</);
|
||||
});
|
||||
|
||||
test("recording preparation never presents phase heartbeats as fake percentages", async () => {
|
||||
const selectorSource = await readFile(
|
||||
new URL("../src/components/ObservationSessionSelect.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
assert.match(selectorSource, /exporting:\s*["']Готовим операторскую сцену["']/);
|
||||
assert.doesNotMatch(selectorSource, /Math\.round\(progress\s*\*\s*100\)/);
|
||||
});
|
||||
|
||||
test("unified recorded AI view keeps the raw camera mounted but does not cover overlays", async () => {
|
||||
const workspaceSource = await readFile(
|
||||
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
assert.match(workspaceSource, /hidden=\{pointCloudFocused \|\| unifiedPerception\}/);
|
||||
assert.match(
|
||||
workspaceSource,
|
||||
/presentedMediaSourceCount = unifiedPerception \? 0 : visibleMediaSources\.length/,
|
||||
);
|
||||
});
|
||||
|
||||
test("replay descriptor rejects path leaks, mismatched sessions and non-seekable data", () => {
|
||||
assert.throws(
|
||||
() => decodeObservationSessionReplay(replay({ source_url: "file:///private/session.rrd" })),
|
||||
@@ -776,7 +824,7 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
|
||||
);
|
||||
|
||||
const manifest = decodeObservationRecordedMediaManifest({
|
||||
schema_version: "missioncore.observation-recorded-media/v2",
|
||||
schema_version: "missioncore.observation-recorded-media/v3",
|
||||
source_id: source.id,
|
||||
generation_sha256: "c".repeat(64),
|
||||
byte_length: 3_266,
|
||||
@@ -788,21 +836,15 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
|
||||
timeline_start_seconds: 0.25,
|
||||
timeline_end_seconds: 20,
|
||||
media_type: 'video/mp4; codecs="avc1.640028"',
|
||||
init_url: source.manifest_url.replace("/manifest", "/epochs/1/init.mp4"),
|
||||
init_byte_length: 128,
|
||||
init_sha256: "a".repeat(64),
|
||||
segment_count: 12,
|
||||
segment_url_prefix: source.manifest_url.replace("/manifest", "/epochs/1/segments/"),
|
||||
segments: Array.from({ length: 12 }, (_, index) => ({
|
||||
sequence: index + 1,
|
||||
url: source.manifest_url.replace("/manifest", `/epochs/1/segments/${index + 1}.m4s`),
|
||||
byte_length: 256 + index,
|
||||
sha256: "b".repeat(64),
|
||||
})),
|
||||
byte_length: 3_266,
|
||||
stream_url: source.manifest_url.replace(
|
||||
"/manifest",
|
||||
`/epochs/1/recording.mp4?generation=${"c".repeat(64)}`,
|
||||
),
|
||||
}],
|
||||
}, decoded.mediaSources[0]);
|
||||
assert.equal(manifest.epochs[0].segmentCount, 12);
|
||||
assert.equal(manifest.epochs[0].segments.length, 12);
|
||||
assert.equal(manifest.epochs[0].byteLength, 3_266);
|
||||
assert.match(manifest.epochs[0].streamUrl, /recording\.mp4\?generation=/);
|
||||
assert.equal(manifest.epochs[0].timelineStartSeconds, 0.25);
|
||||
assert.throws(
|
||||
() => decodeObservationRecordedMediaManifest({
|
||||
@@ -821,7 +863,7 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
|
||||
);
|
||||
assert.throws(
|
||||
() => decodeObservationRecordedMediaManifest({
|
||||
schema_version: "missioncore.observation-recorded-media/v2",
|
||||
schema_version: "missioncore.observation-recorded-media/v3",
|
||||
source_id: source.id,
|
||||
generation_sha256: "c".repeat(64),
|
||||
byte_length: 3_267,
|
||||
@@ -834,6 +876,53 @@ test("replay decodes opaque recorded cameras and their same-origin fMP4 manifest
|
||||
);
|
||||
});
|
||||
|
||||
test("replay decodes a synchronized full-epoch perception video", () => {
|
||||
const sessionId = "20260720T065719Z_viewer_live";
|
||||
const resultId = `result-${"d".repeat(64)}`;
|
||||
const generation = "e".repeat(64);
|
||||
const source = {
|
||||
id: "recorded.perception.right",
|
||||
label: "Сегментация · камера right",
|
||||
modality: "video",
|
||||
manifest_url: `/api/v1/observation-sessions/${sessionId}/perception-media/${resultId}/manifest`,
|
||||
manifest_generation_sha256: generation,
|
||||
byte_length: 12_345_678,
|
||||
media_type: "video/mp4",
|
||||
timeline_start_seconds: 35.421857292,
|
||||
timeline_end_seconds: 484.144857292,
|
||||
seekable: true,
|
||||
synchronization: "host-arrival-best-effort",
|
||||
};
|
||||
const decoded = decodeObservationSessionReplay(replay({
|
||||
session_id: sessionId,
|
||||
timeline_end_seconds: 500,
|
||||
media_sources: [source],
|
||||
}));
|
||||
assert.equal(decoded.mediaSources[0].id, "recorded.perception.right");
|
||||
assert.equal(decoded.mediaSources[0].manifestUrl, source.manifest_url);
|
||||
|
||||
const manifest = decodeObservationRecordedMediaManifest({
|
||||
schema_version: "missioncore.observation-recorded-media/v3",
|
||||
source_id: source.id,
|
||||
generation_sha256: generation,
|
||||
byte_length: source.byte_length,
|
||||
timeline_start_seconds: source.timeline_start_seconds,
|
||||
timeline_end_seconds: source.timeline_end_seconds,
|
||||
synchronization: "host-arrival-best-effort",
|
||||
epochs: [{
|
||||
ordinal: 1,
|
||||
timeline_start_seconds: source.timeline_start_seconds,
|
||||
timeline_end_seconds: source.timeline_end_seconds,
|
||||
media_type: 'video/mp4; codecs="avc1.640028"',
|
||||
byte_length: source.byte_length,
|
||||
stream_url: `/api/v1/observation-sessions/${sessionId}/perception-media/${resultId}/recording.mp4?generation=${generation}`,
|
||||
}],
|
||||
}, decoded.mediaSources[0]);
|
||||
assert.equal(manifest.epochs.length, 1);
|
||||
assert.equal(manifest.epochs[0].timelineStartSeconds, source.timeline_start_seconds);
|
||||
assert.equal(manifest.epochs[0].timelineEndSeconds, source.timeline_end_seconds);
|
||||
});
|
||||
|
||||
test("recorded camera contracts reject foreign origins, path escapes and unknown fields", () => {
|
||||
const sessionId = "session-20260716T205632Z";
|
||||
const base = {
|
||||
@@ -888,7 +977,7 @@ test("recorded camera contracts reject foreign origins, path escapes and unknown
|
||||
}));
|
||||
assert.throws(
|
||||
() => decodeObservationRecordedMediaManifest({
|
||||
schema_version: "missioncore.observation-recorded-media/v2",
|
||||
schema_version: "missioncore.observation-recorded-media/v3",
|
||||
source_id: base.id,
|
||||
generation_sha256: "c".repeat(64),
|
||||
byte_length: 384,
|
||||
@@ -900,17 +989,8 @@ test("recorded camera contracts reject foreign origins, path escapes and unknown
|
||||
timeline_start_seconds: 0,
|
||||
timeline_end_seconds: 20,
|
||||
media_type: 'video/mp4; codecs="avc1.640028"',
|
||||
init_url: "file:///private/init.mp4",
|
||||
init_byte_length: 128,
|
||||
init_sha256: "a".repeat(64),
|
||||
segment_count: 1,
|
||||
segment_url_prefix: "file:///private/segments/",
|
||||
segments: [{
|
||||
sequence: 1,
|
||||
url: "file:///private/segments/1.m4s",
|
||||
byte_length: 256,
|
||||
sha256: "b".repeat(64),
|
||||
}],
|
||||
byte_length: 384,
|
||||
stream_url: "file:///private/recording.mp4",
|
||||
}],
|
||||
}, decoded.mediaSources[0]),
|
||||
/небезопасный API URL/,
|
||||
|
||||
@@ -269,7 +269,14 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
|
||||
{ applicationId: "nodedc_mission_core_recorded", recordingId: "recording-001" },
|
||||
{
|
||||
origin: "http://127.0.0.1:5174",
|
||||
activeView: "perception",
|
||||
activeView: "perception3d",
|
||||
viewResetGeneration: 1,
|
||||
perceptionLayers: {
|
||||
enabled: true,
|
||||
detections2d: true,
|
||||
segmentation: false,
|
||||
cuboids3d: true,
|
||||
},
|
||||
fetcher: async (input, init) => {
|
||||
calls.push({ input: String(input), init, body: JSON.parse(String(init.body)) });
|
||||
return new Response(payload, {
|
||||
@@ -292,7 +299,12 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
|
||||
point_size: 4.5,
|
||||
palette: "custom",
|
||||
custom_color: "#35d7c1",
|
||||
active_view: "perception",
|
||||
active_view: "perception3d",
|
||||
view_reset_generation: 1,
|
||||
unified_perception: true,
|
||||
show_detections_2d: true,
|
||||
show_segmentation: false,
|
||||
show_cuboids_3d: true,
|
||||
});
|
||||
|
||||
await assert.rejects(
|
||||
@@ -665,7 +677,7 @@ test("camera delivery rejects literal and encoded endpoint or credential leaks",
|
||||
"/api/preview?upstream=rtsp://camera.local/live",
|
||||
"/api/preview?upstream=rtsp%3A%2F%2Fcamera.local%2Flive",
|
||||
"/api/preview?upstream=rtsp%253A%252F%252Fcamera.local%252Flive",
|
||||
"/api/preview?endpoint=192.168.68.52:8554",
|
||||
"/api/preview?endpoint=192.0.2.52:8554",
|
||||
"/api/preview?endpoint=192%2E168%2E68%2E52",
|
||||
"/api/preview?password=not-for-the-browser",
|
||||
"/api/preview?%70%61%73%73%77%6f%72%64=not-for-the-browser",
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { after, before, test } from "node:test";
|
||||
|
||||
import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let fetchVerifiedRecordedMediaArchive;
|
||||
let fetchRecordedMediaArchive;
|
||||
let recordedMediaPresentationState;
|
||||
let recordedMediaSeekableCoverage;
|
||||
let appendRecordedMediaBuffer;
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
@@ -18,10 +16,9 @@ before(async () => {
|
||||
server: { middlewareMode: true },
|
||||
});
|
||||
({
|
||||
fetchVerifiedRecordedMediaArchive,
|
||||
fetchRecordedMediaArchive,
|
||||
recordedMediaPresentationState,
|
||||
recordedMediaSeekableCoverage,
|
||||
appendRecordedMediaBuffer,
|
||||
} = await server.ssrLoadModule("/src/components/RecordedFmp4Player.tsx"));
|
||||
});
|
||||
|
||||
@@ -29,41 +26,28 @@ after(async () => {
|
||||
await server?.close();
|
||||
});
|
||||
|
||||
function digest(payload) {
|
||||
return createHash("sha256").update(payload).digest("hex");
|
||||
}
|
||||
|
||||
function fixture() {
|
||||
function fixture({ byteLength = 36_000_000_000 } = {}) {
|
||||
const manifestUrl = "/api/v1/observation-sessions/session-1/media/camera-1/manifest";
|
||||
const init = Buffer.from("canonical-init");
|
||||
const first = Buffer.from("canonical-first-fragment");
|
||||
const second = Buffer.from("canonical-second-fragment");
|
||||
const generation = "a".repeat(64);
|
||||
const segmentPrefix = manifestUrl.replace("/manifest", "/epochs/1/segments/");
|
||||
const streamUrl = manifestUrl.replace(
|
||||
"/manifest",
|
||||
`/epochs/1/recording.mp4?generation=${generation}`,
|
||||
);
|
||||
const manifest = {
|
||||
schema_version: "missioncore.observation-recorded-media/v2",
|
||||
schema_version: "missioncore.observation-recorded-media/v3",
|
||||
source_id: "recorded.camera.camera-1",
|
||||
generation_sha256: generation,
|
||||
byte_length: init.byteLength + first.byteLength + second.byteLength,
|
||||
byte_length: byteLength,
|
||||
timeline_start_seconds: 0,
|
||||
timeline_end_seconds: 20,
|
||||
timeline_end_seconds: 36_000,
|
||||
synchronization: "host-arrival-best-effort",
|
||||
epochs: [{
|
||||
ordinal: 1,
|
||||
timeline_start_seconds: 0,
|
||||
timeline_end_seconds: 20,
|
||||
timeline_end_seconds: 36_000,
|
||||
media_type: 'video/mp4; codecs="avc1.640028"',
|
||||
init_url: manifestUrl.replace("/manifest", "/epochs/1/init.mp4"),
|
||||
init_byte_length: init.byteLength,
|
||||
init_sha256: digest(init),
|
||||
segment_count: 2,
|
||||
segment_url_prefix: segmentPrefix,
|
||||
segments: [first, second].map((payload, index) => ({
|
||||
sequence: index + 1,
|
||||
url: `${segmentPrefix}${index + 1}.m4s`,
|
||||
byte_length: payload.byteLength,
|
||||
sha256: digest(payload),
|
||||
})),
|
||||
byte_length: byteLength,
|
||||
stream_url: streamUrl,
|
||||
}],
|
||||
};
|
||||
const source = {
|
||||
@@ -72,14 +56,14 @@ function fixture() {
|
||||
modality: "video",
|
||||
manifestUrl,
|
||||
manifestGenerationSha256: generation,
|
||||
byteLength: manifest.byte_length,
|
||||
byteLength,
|
||||
mediaType: "video/mp4",
|
||||
timelineStartSeconds: 0,
|
||||
timelineEndSeconds: 20,
|
||||
timelineEndSeconds: 36_000,
|
||||
seekable: true,
|
||||
synchronization: "host-arrival-best-effort",
|
||||
};
|
||||
return { source, manifest, generation, init, first, second };
|
||||
return { source, manifest, generation, streamUrl };
|
||||
}
|
||||
|
||||
function jsonResponse(payload, generation) {
|
||||
@@ -92,145 +76,54 @@ function jsonResponse(payload, generation) {
|
||||
});
|
||||
}
|
||||
|
||||
function mediaResponse(payload, sha = digest(payload), length = payload.byteLength) {
|
||||
return new Response(payload, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Length": String(length),
|
||||
ETag: `"sha256:${sha}"`,
|
||||
test("multi-hour camera admission fetches only its compact generation-bound manifest", async () => {
|
||||
const { source, manifest, generation, streamUrl } = fixture();
|
||||
const requested = [];
|
||||
const archive = await fetchRecordedMediaArchive(source, {
|
||||
fetcher: async (input, request = {}) => {
|
||||
requested.push(String(input));
|
||||
assert.equal(new Headers(request.headers).get("If-Match"), `"sha256:${generation}"`);
|
||||
return jsonResponse(manifest, generation);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function deferred() {
|
||||
let resolve;
|
||||
let reject;
|
||||
const promise = new Promise((resolvePromise, rejectPromise) => {
|
||||
resolve = resolvePromise;
|
||||
reject = rejectPromise;
|
||||
});
|
||||
return { promise, resolve, reject };
|
||||
}
|
||||
|
||||
async function flushAsync() {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
test("recorded camera archive stays pending until every canonical byte is verified", async () => {
|
||||
const { source, manifest, generation, init, first, second } = fixture();
|
||||
const finalSegment = deferred();
|
||||
const requested = [];
|
||||
let manifestRequests = 0;
|
||||
const fetcher = async (input, request = {}) => {
|
||||
const url = String(input);
|
||||
requested.push(url);
|
||||
const headers = new Headers(request.headers);
|
||||
if (url === source.manifestUrl) {
|
||||
manifestRequests += 1;
|
||||
assert.equal(headers.get("If-Match"), `"sha256:${generation}"`);
|
||||
return jsonResponse(manifest, generation);
|
||||
}
|
||||
const epoch = manifest.epochs[0];
|
||||
if (url === epoch.init_url) {
|
||||
assert.equal(headers.get("If-Match"), `"sha256:${epoch.init_sha256}"`);
|
||||
return mediaResponse(init);
|
||||
}
|
||||
if (url === epoch.segments[0].url) return mediaResponse(first);
|
||||
if (url === epoch.segments[1].url) return finalSegment.promise;
|
||||
throw new Error(`Unexpected URL ${url}`);
|
||||
};
|
||||
|
||||
let settled = false;
|
||||
const archivePromise = fetchVerifiedRecordedMediaArchive(source, { fetcher })
|
||||
.then((archive) => {
|
||||
settled = true;
|
||||
return archive;
|
||||
});
|
||||
await flushAsync();
|
||||
|
||||
assert.equal(settled, false, "init and a partial segment prefix must not publish the camera");
|
||||
assert.deepEqual(requested.slice(0, 4), [
|
||||
source.manifestUrl,
|
||||
manifest.epochs[0].init_url,
|
||||
manifest.epochs[0].segments[0].url,
|
||||
manifest.epochs[0].segments[1].url,
|
||||
]);
|
||||
|
||||
finalSegment.resolve(mediaResponse(second));
|
||||
const archive = await archivePromise;
|
||||
assert.equal(manifestRequests, 2, "the immutable generation is revalidated after transfer");
|
||||
assert.equal(archive.byteLength, init.byteLength + first.byteLength + second.byteLength);
|
||||
assert.equal(archive.epochs[0].segments.length, 2);
|
||||
assert.deepEqual(requested, [source.manifestUrl]);
|
||||
assert.equal(archive.byteLength, 36_000_000_000);
|
||||
assert.equal(archive.manifest.epochs[0].streamUrl, streamUrl);
|
||||
});
|
||||
|
||||
test("first camera manifest request is launch-generation bound and rejects replacement", async () => {
|
||||
test("first camera manifest request rejects a replaced generation", async () => {
|
||||
const { source, manifest, generation } = fixture();
|
||||
const replacementGeneration = "d".repeat(64);
|
||||
let requests = 0;
|
||||
const fetcher = async (input, request = {}) => {
|
||||
requests += 1;
|
||||
assert.equal(String(input), source.manifestUrl);
|
||||
assert.equal(
|
||||
new Headers(request.headers).get("If-Match"),
|
||||
`"sha256:${generation}"`,
|
||||
);
|
||||
return jsonResponse(
|
||||
{ ...manifest, generation_sha256: replacementGeneration },
|
||||
replacementGeneration,
|
||||
);
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
|
||||
fetchRecordedMediaArchive(source, {
|
||||
fetcher: async (_input, request = {}) => {
|
||||
assert.equal(new Headers(request.headers).get("If-Match"), `"sha256:${generation}"`);
|
||||
return jsonResponse(
|
||||
{ ...manifest, generation_sha256: replacementGeneration },
|
||||
replacementGeneration,
|
||||
);
|
||||
},
|
||||
}),
|
||||
/несовместим|заменён/,
|
||||
);
|
||||
assert.equal(requests, 1);
|
||||
});
|
||||
|
||||
test("recorded camera archive fails closed on truncation despite plausible headers", async () => {
|
||||
const { source, manifest, generation, init, first, second } = fixture();
|
||||
const fetcher = async (input) => {
|
||||
const url = String(input);
|
||||
if (url === source.manifestUrl) return jsonResponse(manifest, generation);
|
||||
const epoch = manifest.epochs[0];
|
||||
if (url === epoch.init_url) return mediaResponse(init);
|
||||
if (url === epoch.segments[0].url) return mediaResponse(first);
|
||||
if (url === epoch.segments[1].url) {
|
||||
return mediaResponse(second.subarray(0, second.byteLength - 1), digest(second), second.byteLength);
|
||||
}
|
||||
throw new Error(`Unexpected URL ${url}`);
|
||||
test("camera manifest fails closed when epoch bytes do not match launch bytes", async () => {
|
||||
const { source, manifest, generation } = fixture();
|
||||
const mismatched = {
|
||||
...manifest,
|
||||
epochs: [{ ...manifest.epochs[0], byte_length: manifest.byte_length - 1 }],
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
|
||||
/усечён/,
|
||||
fetchRecordedMediaArchive(source, {
|
||||
fetcher: async () => jsonResponse(mismatched, generation),
|
||||
}),
|
||||
/не совпадает/,
|
||||
);
|
||||
});
|
||||
|
||||
test("recorded camera archive fails closed when bytes are replaced under an old ETag", async () => {
|
||||
const { source, manifest, generation, init, first, second } = fixture();
|
||||
const replacement = Buffer.from(second.map((value) => value ^ 0xff));
|
||||
assert.equal(replacement.byteLength, second.byteLength);
|
||||
const fetcher = async (input) => {
|
||||
const url = String(input);
|
||||
if (url === source.manifestUrl) return jsonResponse(manifest, generation);
|
||||
const epoch = manifest.epochs[0];
|
||||
if (url === epoch.init_url) return mediaResponse(init);
|
||||
if (url === epoch.segments[0].url) return mediaResponse(first);
|
||||
if (url === epoch.segments[1].url) {
|
||||
return mediaResponse(replacement, digest(second), second.byteLength);
|
||||
}
|
||||
throw new Error(`Unexpected URL ${url}`);
|
||||
};
|
||||
|
||||
await assert.rejects(
|
||||
fetchVerifiedRecordedMediaArchive(source, { fetcher }),
|
||||
/SHA-256/,
|
||||
);
|
||||
});
|
||||
|
||||
test("camera presentation gate opens only for the completely appended selected epoch", () => {
|
||||
test("camera presentation gate opens only for the seekable selected epoch", () => {
|
||||
assert.equal(recordedMediaPresentationState("loading", null, "g1", false), "loading");
|
||||
assert.equal(recordedMediaPresentationState("ready", null, "g1", false), "loading");
|
||||
assert.equal(recordedMediaPresentationState("ready", "g2", "g1", false), "loading");
|
||||
@@ -251,40 +144,13 @@ test("decoded duration and seekable range cover the complete declared epoch", ()
|
||||
assert.equal(recordedMediaSeekableCoverage(20, 20, 20, 1, 1.01), false);
|
||||
});
|
||||
|
||||
test("aborting SourceBuffer append removes every temporary listener", async () => {
|
||||
const listeners = new Map();
|
||||
const sourceBuffer = {
|
||||
addEventListener(type, listener) {
|
||||
const bucket = listeners.get(type) ?? new Set();
|
||||
bucket.add(listener);
|
||||
listeners.set(type, bucket);
|
||||
},
|
||||
removeEventListener(type, listener) {
|
||||
listeners.get(type)?.delete(listener);
|
||||
},
|
||||
appendBuffer() {},
|
||||
};
|
||||
const abort = new AbortController();
|
||||
const pending = appendRecordedMediaBuffer(
|
||||
sourceBuffer,
|
||||
new ArrayBuffer(8),
|
||||
abort.signal,
|
||||
);
|
||||
assert.equal(listeners.get("updateend")?.size, 1);
|
||||
assert.equal(listeners.get("error")?.size, 1);
|
||||
abort.abort();
|
||||
await assert.rejects(pending, (error) => error?.name === "AbortError");
|
||||
assert.equal(listeners.get("updateend")?.size, 0);
|
||||
assert.equal(listeners.get("error")?.size, 0);
|
||||
});
|
||||
|
||||
test("verified camera archives remain immutable for safe player remount", async () => {
|
||||
test("recorded player range-streams and never builds a whole-video RAM Blob", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/components/RecordedFmp4Player.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.doesNotMatch(source, /\.init\s*=\s*new ArrayBuffer/);
|
||||
assert.doesNotMatch(source, /\.segments\.length\s*=\s*0/);
|
||||
assert.match(source, /video\.src\s*=\s*descriptor\.streamUrl/);
|
||||
assert.doesNotMatch(source, /new Blob\(|response\.arrayBuffer\(|SourceBuffer/);
|
||||
});
|
||||
|
||||
test("loading and error overlays fully conceal recorded camera pixels", async () => {
|
||||
@@ -301,3 +167,20 @@ test("loading and error overlays fully conceal recorded camera pixels", async ()
|
||||
/\.recorded-media-player__notice\s*\{[^}]*inset:\s*0;[^}]*background:\s*#070809/s,
|
||||
);
|
||||
});
|
||||
|
||||
test("point-cloud fullscreen keeps the admitted recorded camera worker mounted", async () => {
|
||||
const source = await readFile(
|
||||
new URL("../src/workspaces/Workspaces.tsx", import.meta.url),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(source, /\{visibleMediaSources\.map\(\(source, index\) => \(/);
|
||||
assert.match(source, /hidden=\{pointCloudFocused \|\| unifiedPerception\}/);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
/\{!pointCloudFocused \? visibleMediaSources\.map/,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
source,
|
||||
/\(pointCloudFocused \|\| !observationLayout\.visibleSourceIds\.has\(source\.id\)\)/,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -78,34 +78,27 @@ test("any RRD or camera failure closes the complete recorded session", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("camera admission enforces independent 16/256/512 MiB limits before fetching", () => {
|
||||
test("camera admission keeps source scheduling bounded but has no duration or byte ceiling", () => {
|
||||
const {
|
||||
MAX_RECORDED_CAMERA_SOURCES,
|
||||
MAX_RECORDED_MEDIA_SOURCE_BYTES,
|
||||
MAX_RECORDED_SESSION_CAMERA_BYTES,
|
||||
recordedCameraDescriptorPreflight,
|
||||
} = admission;
|
||||
assert.equal(MAX_RECORDED_CAMERA_SOURCES, 16);
|
||||
assert.equal(MAX_RECORDED_MEDIA_SOURCE_BYTES, 256 * 1024 * 1024);
|
||||
assert.equal(MAX_RECORDED_SESSION_CAMERA_BYTES, 512 * 1024 * 1024);
|
||||
assert.equal(recordedCameraDescriptorPreflight(
|
||||
Array.from({ length: 2 }, (_, index) => ({
|
||||
id: `camera.${index}`,
|
||||
byteLength: MAX_RECORDED_MEDIA_SOURCE_BYTES,
|
||||
byteLength: 36_000_000_000,
|
||||
})),
|
||||
), "ready");
|
||||
assert.equal(recordedCameraDescriptorPreflight(
|
||||
Array.from({ length: 17 }, (_, index) => ({ id: `camera.${index}`, byteLength: 1 })),
|
||||
), "error");
|
||||
assert.equal(recordedCameraDescriptorPreflight([
|
||||
{ id: "camera.large", byteLength: MAX_RECORDED_MEDIA_SOURCE_BYTES + 1 },
|
||||
{ id: "camera.ten-hours", byteLength: 36_000_000_000 },
|
||||
]), "ready");
|
||||
assert.equal(recordedCameraDescriptorPreflight([
|
||||
{ id: "camera.invalid", byteLength: Number.MAX_SAFE_INTEGER + 1 },
|
||||
]), "error");
|
||||
assert.equal(recordedCameraDescriptorPreflight(
|
||||
Array.from({ length: 5 }, (_, index) => ({
|
||||
id: `camera.${index}`,
|
||||
byteLength: 110 * 1024 * 1024,
|
||||
})),
|
||||
), "error");
|
||||
assert.equal(recordedCameraDescriptorPreflight([
|
||||
{ id: "camera.duplicate", byteLength: 1 },
|
||||
{ id: "camera.duplicate", byteLength: 1 },
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import test from "node:test";
|
||||
|
||||
import makeRerunRuntime from "../vendor/rerun-web-viewer-0.34.1/re_viewer.nodedc.js";
|
||||
|
||||
const root = resolve(import.meta.dirname, "..");
|
||||
const packageRoot = resolve(root, "node_modules/@rerun-io/web-viewer");
|
||||
const vendorRoot = resolve(root, "vendor/rerun-web-viewer-0.34.1");
|
||||
|
||||
const sha256 = (path) =>
|
||||
createHash("sha256").update(readFileSync(path)).digest("hex");
|
||||
|
||||
test("NODE.DC Rerun runtime is the audited 0.34.1 zoom-to-cursor build", () => {
|
||||
const manifest = JSON.parse(readFileSync(resolve(packageRoot, "package.json"), "utf8"));
|
||||
assert.equal(manifest.version, "0.34.1");
|
||||
|
||||
const expectedWasm = "38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb";
|
||||
const expectedGlue = "0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339";
|
||||
|
||||
assert.equal(sha256(resolve(vendorRoot, "re_viewer_bg.nodedc.wasm")), expectedWasm);
|
||||
assert.equal(sha256(resolve(vendorRoot, "re_viewer.nodedc.js")), expectedGlue);
|
||||
assert.equal(sha256(resolve(packageRoot, "re_viewer_bg.wasm")), expectedWasm);
|
||||
assert.equal(sha256(resolve(packageRoot, "re_viewer.js")), expectedGlue);
|
||||
});
|
||||
|
||||
test("custom JavaScript glue references only exports present in its paired WASM", () => {
|
||||
const wasmPath = resolve(vendorRoot, "re_viewer_bg.nodedc.wasm");
|
||||
const gluePath = resolve(vendorRoot, "re_viewer.nodedc.js");
|
||||
const module = new WebAssembly.Module(readFileSync(wasmPath));
|
||||
const exports = new Set(WebAssembly.Module.exports(module).map(({ name }) => name));
|
||||
const imports = WebAssembly.Module.imports(module);
|
||||
const glue = readFileSync(gluePath, "utf8");
|
||||
const referencedExports = new Set(
|
||||
[...glue.matchAll(/\bwasm\.([A-Za-z_$][\w$]*)/g)].map((match) => match[1]),
|
||||
);
|
||||
const missingExports = [...referencedExports].filter((name) => !exports.has(name));
|
||||
|
||||
assert.equal(imports.length, 927);
|
||||
assert.equal(exports.size, 79);
|
||||
assert.deepEqual(missingExports, []);
|
||||
assert.match(glue, /export default function\(\)/);
|
||||
assert.match(glue, /if \(!wasm\) return;/);
|
||||
});
|
||||
|
||||
test("custom Rerun WASM initializes and grows its externref table", () => {
|
||||
const runtime = makeRerunRuntime();
|
||||
runtime.initSync({
|
||||
module: readFileSync(resolve(vendorRoot, "re_viewer_bg.nodedc.wasm")),
|
||||
});
|
||||
|
||||
assert.equal(typeof runtime.WebHandle, "function");
|
||||
runtime.deinit();
|
||||
});
|
||||
|
||||
test("source patch carries cursor pivot, minimum-radius handoff, and geometry tests", () => {
|
||||
const patch = readFileSync(resolve(vendorRoot, "NODEDC_ZOOM_TO_CURSOR.patch"), "utf8");
|
||||
|
||||
assert.match(patch, /fn pointer_ray_direction/);
|
||||
assert.match(patch, /fn zoom_orbit_towards_pointer/);
|
||||
assert.match(patch, /near_limit_hands_excess_zoom_to_cursor_directed_dolly/);
|
||||
assert.match(patch, /crossing_near_limit_preserves_unconsumed_scene_scaled_zoom/);
|
||||
assert.match(patch, /remaining_zoom_factor\.ln\(\) \* self\.speed/);
|
||||
assert.match(patch, /off_center_pointer_stays_on_the_same_view_ray/);
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
# NODE.DC Rerun web viewer 0.34.1
|
||||
|
||||
This directory contains the audited Mission Core camera-controller override for
|
||||
`@rerun-io/web-viewer` 0.34.1. It changes only the native orbital zoom behavior:
|
||||
|
||||
- the pointer ray selects an anchor on the current focus plane;
|
||||
- eye position and look target scale around that anchor, so the point under the
|
||||
cursor remains under the cursor;
|
||||
- after Rerun's `0.02 m` near-plane safety radius is reached, excess zoom becomes
|
||||
a cursor-directed dolly scaled by the scene's navigation speed instead of
|
||||
silently ignoring the wheel or moving by imperceptible millimeters;
|
||||
- first-person movement, orbit rotation, panning, WASD and Rerun's zoom-out cap
|
||||
are unchanged.
|
||||
|
||||
## Source identity
|
||||
|
||||
- Upstream: `rerun-io/rerun`
|
||||
- Tag: `0.34.1`
|
||||
- Commit: `4efb18f17f6f0e41985cda99a2bdcd012febc8d5`
|
||||
- Patched file: `crates/viewer/re_view_spatial/src/eye.rs`
|
||||
- Patch: `NODEDC_ZOOM_TO_CURSOR.patch`
|
||||
- Rust: `1.92.0`
|
||||
- Binaryen / `wasm-opt`: `117` (the version pinned by Rerun's `pixi.lock`)
|
||||
- Build image: `rust:1.92-bookworm`
|
||||
- Build image digest:
|
||||
`sha256:e90e846de4124376164ddfbaab4b0774c7bdeef5e738866295e5a90a34a307a2`
|
||||
- Build date: `2026-07-22` (`Europe/Moscow`)
|
||||
|
||||
## Reproduction
|
||||
|
||||
Apply the patch to the exact upstream commit, then run Rerun's own builder:
|
||||
|
||||
```sh
|
||||
git apply NODEDC_ZOOM_TO_CURSOR.patch
|
||||
cargo test -p re_view_spatial --lib eye::tests:: -- --nocapture
|
||||
cargo run -p re_dev_tools -- build-web-viewer \
|
||||
--release -g \
|
||||
--target no-modules-base \
|
||||
--no-default-features \
|
||||
--features map_view \
|
||||
-o rerun_js/web-viewer
|
||||
```
|
||||
|
||||
The container also needs Binaryen `117` for Rerun's final `wasm-opt -O2` step.
|
||||
Binaryen `108` from Debian 12 must not be used: it produced a module whose
|
||||
`externref` table could not grow during initialization. The
|
||||
generated `re_viewer.js` is transformed with
|
||||
`scripts/transform-rerun-web-viewer-glue.mjs`, which mirrors Rerun 0.34.1's
|
||||
`rerun_js/web-viewer/build-wasm.mjs` no-modules wrapper and teardown guards.
|
||||
|
||||
## Verified artifacts
|
||||
|
||||
| Artifact | SHA-256 |
|
||||
| --- | --- |
|
||||
| `re_viewer_bg.nodedc.wasm` | `38d19bac06b7c3b8e549489469cf7c4a24f319ec953849e4656b5827a6c105bb` |
|
||||
| raw generated `re_viewer.js` | `cc196a93c5be972c801d46be4dc9934f7f042eb62941f0aa0678f1c8416c6874` |
|
||||
| `re_viewer.nodedc.js` | `0f7b76c9f24cbd8437021b5d37499894aeadc586183e422ebc82ef556d7b8339` |
|
||||
|
||||
The focused Rust suite completed with `5 passed, 0 failed`, including the
|
||||
orbit-to-dolly boundary case. A Node `initSync`
|
||||
smoke test completed successfully, including the `externref` table-growth step.
|
||||
The Mission Core Node suite also checks both artifact hashes and verifies that
|
||||
every `wasm.*` reference in the JavaScript glue exists in the paired WASM
|
||||
exports.
|
||||
|
||||
`scripts/patch-rerun-web-viewer.mjs` installs the pair after npm extracts the
|
||||
official package. It accepts only the known published or NODE.DC hashes and
|
||||
fails closed on any other package contents.
|
||||
+230
@@ -0,0 +1,230 @@
|
||||
diff --git a/crates/viewer/re_view_spatial/src/eye.rs b/crates/viewer/re_view_spatial/src/eye.rs
|
||||
index e59b311..3f33259 100644
|
||||
--- a/crates/viewer/re_view_spatial/src/eye.rs
|
||||
+++ b/crates/viewer/re_view_spatial/src/eye.rs
|
||||
@@ -512,8 +512,85 @@ impl EyeController {
|
||||
}
|
||||
}
|
||||
|
||||
+ /// Returns the world-space ray under the pointer.
|
||||
+ ///
|
||||
+ /// Keeping this calculation local to the eye controller lets orbital zoom use the pointer
|
||||
+ /// synchronously. GPU picking arrives a frame later and would make the zoom pivot visibly lag.
|
||||
+ fn pointer_ray_direction(&self, rect: Rect, pointer: egui::Pos2) -> Option<Vec3> {
|
||||
+ if !rect.contains(pointer) || rect.width() <= 0.0 || rect.height() <= 0.0 {
|
||||
+ return None;
|
||||
+ }
|
||||
+
|
||||
+ let fov_y = self.fov_y.unwrap_or(Eye::DEFAULT_FOV_Y);
|
||||
+ let aspect_ratio = rect.width() / rect.height();
|
||||
+ let focal_scale = (fov_y * 0.5).tan();
|
||||
+ let x = (2.0 * (pointer.x - rect.left()) / rect.width() - 1.0)
|
||||
+ * focal_scale
|
||||
+ * aspect_ratio;
|
||||
+ let y = (1.0 - 2.0 * (pointer.y - rect.top()) / rect.height()) * focal_scale;
|
||||
+
|
||||
+ (self.rotation() * vec3(x, y, -1.0)).try_normalize()
|
||||
+ }
|
||||
+
|
||||
+ /// Zoom an orbital eye around the point under the pointer on the current focus plane.
|
||||
+ ///
|
||||
+ /// The position and look target are scaled around the same anchor. This preserves the
|
||||
+ /// projected pointer position while retaining the existing orbit direction and controls.
|
||||
+ /// Once the near-plane safety radius is reached, excess zoom becomes a cursor-directed dolly
|
||||
+ /// measured in the scene's navigation speed instead of near-plane millimeters.
|
||||
+ fn zoom_orbit_towards_pointer(
|
||||
+ &mut self,
|
||||
+ zoom_factor: f32,
|
||||
+ max_radius: f32,
|
||||
+ rect: Rect,
|
||||
+ pointer: Option<egui::Pos2>,
|
||||
+ ) {
|
||||
+ let radius = self.radius();
|
||||
+ if !radius.is_finite() || radius <= f32::MIN_POSITIVE {
|
||||
+ return;
|
||||
+ }
|
||||
+
|
||||
+ let requested_radius = radius / zoom_factor;
|
||||
+ let new_radius = requested_radius.clamp(Self::MIN_ORBIT_DISTANCE, max_radius);
|
||||
+ let scale = new_radius / radius;
|
||||
+
|
||||
+ let ray_direction = pointer.and_then(|pointer| self.pointer_ray_direction(rect, pointer));
|
||||
+ if let Some(ray_direction) = ray_direction {
|
||||
+ let forward = self.fwd();
|
||||
+ let denominator = ray_direction.dot(forward);
|
||||
+ if denominator > 1.0e-4 {
|
||||
+ let anchor = self.pos + ray_direction * (radius / denominator);
|
||||
+ self.pos = anchor + (self.pos - anchor) * scale;
|
||||
+ self.look_target = anchor + (self.look_target - anchor) * scale;
|
||||
+
|
||||
+ if requested_radius < Self::MIN_ORBIT_DISTANCE {
|
||||
+ // Shrinking the remaining 2 cm orbit radius consumes only part of this input.
|
||||
+ // Hand the logarithmic remainder to the same scene-scaled speed used by WASD
|
||||
+ // and first-person scroll. Basing this on the near-plane remainder itself made
|
||||
+ // each wheel event move by millimeters and felt indistinguishable from a hard
|
||||
+ // zoom limit on building- and map-scale recordings.
|
||||
+ let orbit_zoom_factor = (radius / Self::MIN_ORBIT_DISTANCE).max(1.0);
|
||||
+ let remaining_zoom_factor = zoom_factor / orbit_zoom_factor;
|
||||
+ if remaining_zoom_factor > 1.0 && remaining_zoom_factor.is_finite() {
|
||||
+ let dolly = remaining_zoom_factor.ln() * self.speed as f32;
|
||||
+ self.pos += ray_direction * dolly;
|
||||
+ self.look_target += ray_direction * dolly;
|
||||
+ }
|
||||
+ }
|
||||
+ self.did_interact = true;
|
||||
+ return;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ // Pointer data can be absent for synthetic zoom events. Preserve Rerun's centered zoom
|
||||
+ // behavior in that case instead of dropping the input.
|
||||
+ self.pos = self.look_target - self.fwd() * new_radius;
|
||||
+ self.did_interact = true;
|
||||
+ }
|
||||
+
|
||||
/// Handle zoom/scroll input.
|
||||
- fn handle_zoom(&mut self, egui_ctx: &egui::Context, scene_bounding_box: &macaw::BoundingBox) {
|
||||
+ fn handle_zoom(&mut self, response: &egui::Response, scene_bounding_box: &macaw::BoundingBox) {
|
||||
+ let egui_ctx = &response.ctx;
|
||||
let zoom_factor = egui_ctx.input(|input| {
|
||||
// egui's default horizontal_scroll_modifier is shift, which is also our speed-up modifier.
|
||||
// This means that a user who wants to speed up scroll-to-zoom will generate a horizontal scroll delta.
|
||||
@@ -528,22 +605,12 @@ impl EyeController {
|
||||
|
||||
match self.kind {
|
||||
Eye3DKind::Orbital => {
|
||||
- let radius = self.pos.distance(self.look_target);
|
||||
-
|
||||
// Cap zoom-out against the scene bounding box. If we're already past the cap
|
||||
// (e.g. right after loading) use the current radius instead — no snap-back.
|
||||
+ let radius = self.radius();
|
||||
let max_radius = max_orbital_radius(scene_bounding_box).max(radius);
|
||||
- let new_radius = (radius / zoom_factor).clamp(Self::MIN_ORBIT_DISTANCE, max_radius);
|
||||
-
|
||||
- // The user may be scrolling to move the camera closer, but are not realizing
|
||||
- // the radius is now tiny.
|
||||
- // TODO(emilk): inform the users somehow that scrolling won't help, and that they should use WSAD instead.
|
||||
- // It might be tempting to start moving the camera here on scroll, but that would is bad for other reasons.
|
||||
-
|
||||
- if f32::MIN_POSITIVE < new_radius {
|
||||
- self.pos = self.look_target - self.fwd() * new_radius;
|
||||
- self.did_interact = true;
|
||||
- }
|
||||
+ let pointer = response.ctx.pointer_latest_pos();
|
||||
+ self.zoom_orbit_towards_pointer(zoom_factor, max_radius, response.rect, pointer);
|
||||
}
|
||||
Eye3DKind::FirstPerson => {
|
||||
// Move along the forward axis when zooming in first person mode.
|
||||
@@ -687,7 +754,7 @@ impl EyeController {
|
||||
self.handle_drag(response, drag_threshold);
|
||||
|
||||
if response.hovered() {
|
||||
- self.handle_zoom(&response.ctx, scene_bounding_box);
|
||||
+ self.handle_zoom(response, scene_bounding_box);
|
||||
}
|
||||
|
||||
if response.has_focus() {
|
||||
@@ -1249,3 +1316,103 @@ impl EyeState {
|
||||
Ok(eye)
|
||||
}
|
||||
}
|
||||
+
|
||||
+#[cfg(test)]
|
||||
+mod tests {
|
||||
+ use super::*;
|
||||
+
|
||||
+ fn orbital_controller(pos: Vec3, look_target: Vec3) -> EyeController {
|
||||
+ EyeController {
|
||||
+ pos,
|
||||
+ look_target,
|
||||
+ kind: Eye3DKind::Orbital,
|
||||
+ speed: 1.0,
|
||||
+ eye_up: Vec3::Z,
|
||||
+ fov_y: Some(Eye::DEFAULT_FOV_Y),
|
||||
+ did_interact: false,
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ fn test_rect() -> Rect {
|
||||
+ Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(800.0, 600.0))
|
||||
+ }
|
||||
+
|
||||
+ fn assert_vec3_close(actual: Vec3, expected: Vec3) {
|
||||
+ assert!(
|
||||
+ actual.abs_diff_eq(expected, 1.0e-5),
|
||||
+ "actual={actual:?}, expected={expected:?}"
|
||||
+ );
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn centered_pointer_keeps_the_orbit_target() {
|
||||
+ let rect = test_rect();
|
||||
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
|
||||
+
|
||||
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(rect.center()));
|
||||
+
|
||||
+ assert_vec3_close(controller.pos, vec3(0.0, -5.0, 0.0));
|
||||
+ assert_vec3_close(controller.look_target, Vec3::ZERO);
|
||||
+ assert!(controller.did_interact);
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn off_center_pointer_stays_on_the_same_view_ray() {
|
||||
+ let rect = test_rect();
|
||||
+ let pointer = egui::pos2(650.0, 240.0);
|
||||
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
|
||||
+ let original_ray = controller
|
||||
+ .pointer_ray_direction(rect, pointer)
|
||||
+ .expect("test pointer must produce a ray");
|
||||
+ let forward = controller.fwd();
|
||||
+ let anchor = controller.pos
|
||||
+ + original_ray * (controller.radius() / original_ray.dot(forward));
|
||||
+
|
||||
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(pointer));
|
||||
+
|
||||
+ assert!((controller.radius() - 5.0).abs() < 1.0e-5);
|
||||
+ let ray_after = (anchor - controller.pos).normalize();
|
||||
+ assert!(ray_after.dot(original_ray) > 0.99999);
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn near_limit_hands_excess_zoom_to_cursor_directed_dolly() {
|
||||
+ let rect = test_rect();
|
||||
+ let radius = EyeController::MIN_ORBIT_DISTANCE;
|
||||
+ let mut controller = orbital_controller(vec3(0.0, -radius, 0.0), Vec3::ZERO);
|
||||
+ let old_pos = controller.pos;
|
||||
+ let old_target = controller.look_target;
|
||||
+
|
||||
+ controller.zoom_orbit_towards_pointer(2.0, 100.0, rect, Some(rect.center()));
|
||||
+
|
||||
+ assert!((controller.radius() - radius).abs() < 1.0e-6);
|
||||
+ assert_vec3_close(controller.pos, old_pos + Vec3::Y * 2.0_f32.ln());
|
||||
+ assert_vec3_close(
|
||||
+ controller.look_target,
|
||||
+ old_target + Vec3::Y * 2.0_f32.ln(),
|
||||
+ );
|
||||
+ assert_vec3_close(controller.look_target - controller.pos, Vec3::Y * radius);
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn crossing_near_limit_preserves_unconsumed_scene_scaled_zoom() {
|
||||
+ let rect = test_rect();
|
||||
+ let mut controller = orbital_controller(vec3(0.0, -0.03, 0.0), Vec3::ZERO);
|
||||
+
|
||||
+ controller.zoom_orbit_towards_pointer(3.0, 100.0, rect, Some(rect.center()));
|
||||
+
|
||||
+ assert!((controller.radius() - EyeController::MIN_ORBIT_DISTANCE).abs() < 1.0e-6);
|
||||
+ assert_vec3_close(controller.pos, vec3(0.0, 2.0_f32.ln() - 0.02, 0.0));
|
||||
+ assert_vec3_close(controller.look_target, vec3(0.0, 2.0_f32.ln(), 0.0));
|
||||
+ }
|
||||
+
|
||||
+ #[test]
|
||||
+ fn zoom_out_respects_scene_radius_cap() {
|
||||
+ let rect = test_rect();
|
||||
+ let mut controller = orbital_controller(vec3(0.0, -10.0, 0.0), Vec3::ZERO);
|
||||
+
|
||||
+ controller.zoom_orbit_towards_pointer(0.1, 25.0, rect, Some(rect.center()));
|
||||
+
|
||||
+ assert!((controller.radius() - 25.0).abs() < 1.0e-5);
|
||||
+ }
|
||||
+}
|
||||
File diff suppressed because it is too large
Load Diff
BIN
Binary file not shown.
@@ -53,6 +53,13 @@ export default defineConfig(({ mode }) => {
|
||||
host: "127.0.0.1",
|
||||
port: 4173,
|
||||
strictPort: true,
|
||||
proxy: {
|
||||
"/api": {
|
||||
target: apiTarget,
|
||||
changeOrigin: false,
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user