feat(perception): integrate calibrated operator pipeline

Add calibrated K1 projection, recorded and near-live perception qualification, unified Rerun operator layers, bounded replay admission, audited viewer controls, worker experiments, and lab evidence.
This commit is contained in:
DCCONSTRUCTIONS
2026-07-23 00:23:28 +03:00
parent ada2a55ee6
commit b53d6d5a45
221 changed files with 55923 additions and 1357 deletions
+54 -3
View File
@@ -88,7 +88,22 @@ const paletteOptions: Array<{ value: PointPalette; label: string; description: s
{ value: "custom", label: "Свой цвет", description: "Один назначенный цвет" },
];
function toViewerSettings(settings: SceneSettings): ViewerSettings {
interface LivePerceptionLayers {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}
const defaultLivePerceptionLayers: LivePerceptionLayers = {
detections2d: false,
segmentation: false,
cuboids3d: false,
};
function toViewerSettings(
settings: SceneSettings,
perception: LivePerceptionLayers = defaultLivePerceptionLayers,
): ViewerSettings {
return {
point_size: settings.pointSize,
color_mode: settings.colorMode,
@@ -98,6 +113,9 @@ function toViewerSettings(settings: SceneSettings): ViewerSettings {
show_points: settings.showPoints,
show_trajectory: settings.showTrajectory,
show_grid: settings.showGrid,
show_detections_2d: perception.detections2d,
show_segmentation: perception.segmentation,
show_cuboids_3d: perception.cuboids3d,
};
}
@@ -149,6 +167,9 @@ export default function App() {
const [layoutSaveNotice, setLayoutSaveNotice] = useState<string | null>(null);
const [sceneSettings, setSceneSettings] = useState<SceneSettings>(defaultSceneSettings);
const [displayDraft, setDisplayDraft] = useState<SceneSettings>(defaultSceneSettings);
const [livePerceptionLayers, setLivePerceptionLayers] = useState<LivePerceptionLayers>(
defaultLivePerceptionLayers,
);
const sourceSwitchBlocked = isSpatialSourceSwitchBlocked(runtime.state);
const sourceSwitchBlockedReason = sourceSwitchBlocked
? SPATIAL_SOURCE_SWITCH_BLOCKED_REASON
@@ -161,6 +182,8 @@ export default function App() {
const confirmedSceneSettingsRef = useRef<SceneSettings>(defaultSceneSettings);
const viewerSettingsCommitTimerRef = useRef<number | null>(null);
const runtimeUpdateViewerSettingsRef = useRef(runtime.updateViewerSettings);
const livePerceptionLayersRef = useRef<LivePerceptionLayers>(defaultLivePerceptionLayers);
const livePerceptionRevisionRef = useRef(0);
const replayActiveRef = useRef(false);
const sceneSettingsCommitterActiveRef = useRef(true);
const sceneSettingsCommitterRef = useRef<LatestAsyncCommitter<SceneSettings> | null>(null);
@@ -197,7 +220,9 @@ export default function App() {
sceneSettingsCommitterRef.current = createLatestAsyncCommitter<SceneSettings>({
commit: (settings) => replayActiveRef.current
? Promise.resolve(true)
: runtimeUpdateViewerSettingsRef.current(toViewerSettings(settings)),
: runtimeUpdateViewerSettingsRef.current(
toViewerSettings(settings, livePerceptionLayersRef.current),
),
onSettled: ({ value, applied, superseded }) => {
if (!sceneSettingsCommitterActiveRef.current) return;
if (applied) confirmedSceneSettingsRef.current = value;
@@ -275,6 +300,13 @@ export default function App() {
sceneSettingsCommitterRef.current?.isBusy()
) return;
const merged = mergeViewerSettings(sceneSettingsRef.current, remote);
const remoteLayers = {
detections2d: remote.show_detections_2d ?? false,
segmentation: remote.show_segmentation ?? false,
cuboids3d: remote.show_cuboids_3d ?? false,
};
livePerceptionLayersRef.current = remoteLayers;
setLivePerceptionLayers(remoteLayers);
sceneSettingsRef.current = merged;
confirmedSceneSettingsRef.current = merged;
setSceneSettings(merged);
@@ -334,7 +366,9 @@ export default function App() {
return;
}
appliedProfileKeyRef.current = applicationKey;
void runtime.updateViewerSettings(toViewerSettings(profile.sceneSettings)).then((applied) => {
void runtime.updateViewerSettings(
toViewerSettings(profile.sceneSettings, livePerceptionLayersRef.current),
).then((applied) => {
if (!applied && appliedProfileKeyRef.current === applicationKey) {
appliedProfileKeyRef.current = null;
}
@@ -441,6 +475,21 @@ export default function App() {
activateSceneWindow("layers");
};
const changeLivePerceptionLayers = useCallback((next: LivePerceptionLayers) => {
const previous = livePerceptionLayersRef.current;
const revision = ++livePerceptionRevisionRef.current;
livePerceptionLayersRef.current = next;
setLivePerceptionLayers(next);
if (replayActiveRef.current) return;
void runtimeUpdateViewerSettingsRef.current(
toViewerSettings(sceneSettingsRef.current, next),
).then((applied) => {
if (applied || livePerceptionRevisionRef.current !== revision) return;
livePerceptionLayersRef.current = previous;
setLivePerceptionLayers(previous);
});
}, []);
const beginRecordedReplaySwitch = useCallback(async () => {
if (sourceSwitchBlockedRef.current) {
throw new Error(SPATIAL_SOURCE_SWITCH_BLOCKED_REASON);
@@ -706,6 +755,8 @@ export default function App() {
onAccumulationChange={(accumulationSeconds) =>
stageDisplayPatch({ accumulationSeconds })}
onAccumulationCommit={flushDisplaySettings}
livePerceptionLayers={livePerceptionLayers}
onLivePerceptionLayersChange={changeLivePerceptionLayers}
observationLayout={observationLayout}
spatialControls={selection?.SpatialControlsView
? {
@@ -72,6 +72,7 @@ export function FloatingObservationWindow({
rect,
maximized,
active,
hidden = false,
onRectChange,
onMaximizedChange,
onActivate,
@@ -89,6 +90,7 @@ export function FloatingObservationWindow({
rect?: ObservationWindowRect;
maximized: boolean;
active: boolean;
hidden?: boolean;
onRectChange: (rect: ObservationWindowRect) => void;
onMaximizedChange: (maximized: boolean) => void;
onActivate: () => void;
@@ -153,7 +155,7 @@ export function FloatingObservationWindow({
resizable={source.capabilities.resizable}
active={active}
zIndex={maximized ? 15 : active ? 9 : 7}
className="floating-observation-window"
className={`floating-observation-window${hidden ? " floating-observation-window--hidden" : ""}`}
onPointerDownCapture={(event) => {
if (!shouldCaptureWorkspacePointer(event.button, event.target as HTMLElement)) return;
try {
@@ -52,18 +52,17 @@ const preparationLabel: Record<ObservationPreparationPhase, string> = {
requesting: "Запрашиваем подготовку",
queued: "В очереди",
validating: "Проверяем запись",
exporting: "Готовим облако точек",
exporting: "Готовим операторскую сцену",
finalizing: "Завершаем подготовку",
failed: "Подготовка не выполнена",
cancelled: "Подготовка отменена",
};
function progressCopy(
phase: ObservationPreparationPhase,
progress: number | null,
): string {
const label = preparationLabel[phase];
return progress === null ? label : `${label} · ${Math.round(progress * 100)}%`;
function progressCopy(phase: ObservationPreparationPhase): string {
// Backend progress values are phase markers and heartbeat revisions, not a
// measured fraction of bytes or frames. Presenting 0.5 as "50%" made a
// healthy long export look stalled, especially after a browser reload.
return preparationLabel[phase];
}
function formatStartedAt(value: string): string {
@@ -126,7 +125,7 @@ export function ObservationSessionSelect({
onReplaySettled,
});
const triggerCopy = sessions.replayProgress
? progressCopy(sessions.replayProgress.phase, sessions.replayProgress.progress)
? progressCopy(sessions.replayProgress.phase)
: sessions.state === "loading"
? "Загружаем сессии…"
: "Сохранённые сессии";
@@ -1,5 +1,3 @@
import { sha256 } from "@noble/hashes/sha2.js";
import { bytesToHex } from "@noble/hashes/utils.js";
import { useEffect, useMemo, useRef, useState } from "react";
import {
@@ -11,7 +9,6 @@ import {
type ObservationSessionFetch,
} from "../core/observation/sessionArchive";
import {
MAX_RECORDED_MEDIA_SOURCE_BYTES,
type RecordedAdmissionPhase,
type RecordedCameraAdmissionState,
} from "../core/observation/recordedSessionAdmission";
@@ -22,32 +19,11 @@ export interface RecordedObservationPlayback {
playing: boolean;
}
export interface RecordedMediaLoadProgress {
loadedBytes: number;
totalBytes: number;
loadedParts: number;
totalParts: number;
}
interface VerifiedRecordedMediaEpoch {
descriptor: ObservationRecordedMediaEpoch;
readonly init: ArrayBuffer;
readonly segments: readonly ArrayBuffer[];
}
export interface VerifiedRecordedMediaArchive {
export interface RecordedMediaArchive {
manifest: ObservationRecordedMediaManifest;
epochs: readonly VerifiedRecordedMediaEpoch[];
byteLength: number;
}
interface RecordedMediaBinaryDescriptor {
url: string;
byteLength: number;
sha256: string;
accept: string;
}
export type RecordedMediaPresentationState = "loading" | "ready" | "waiting" | "error";
export const RECORDED_MEDIA_DURATION_TOLERANCE_SECONDS = 1;
@@ -134,205 +110,33 @@ function sourceContract(source: ObservationSourceDescriptor): ObservationRecorde
};
}
function expectedPayloadEtag(digest: string): string {
return `"sha256:${digest}"`;
}
export async function fetchVerifiedRecordedMediaBytes(
descriptor: RecordedMediaBinaryDescriptor,
{
signal,
fetcher = globalThis.fetch,
}: { signal?: AbortSignal; fetcher?: ObservationSessionFetch } = {},
): Promise<ArrayBuffer> {
let response: Response;
try {
response = await fetcher(descriptor.url, {
method: "GET",
headers: {
Accept: descriptor.accept,
"If-Match": expectedPayloadEtag(descriptor.sha256),
},
signal,
});
} catch (error) {
if (error instanceof DOMException && error.name === "AbortError") throw error;
throw new ObservationSessionContractError(
"Не удалось загрузить канонический фрагмент записанного видео.",
);
}
if (!response.ok || response.status !== 200) {
throw new ObservationSessionContractError(
`Фрагмент записанного видео вернул HTTP ${response.status}.`,
);
}
if (response.headers.get("ETag") !== expectedPayloadEtag(descriptor.sha256)) {
throw new ObservationSessionContractError(
"Фрагмент записанного видео не соответствует immutable manifest.",
);
}
const contentLength = response.headers.get("Content-Length");
if (
contentLength === null ||
!/^[1-9][0-9]*$/.test(contentLength) ||
Number(contentLength) !== descriptor.byteLength
) {
throw new ObservationSessionContractError(
"Длина фрагмента записанного видео не соответствует immutable manifest.",
);
}
const payload = await response.arrayBuffer();
if (payload.byteLength !== descriptor.byteLength) {
throw new ObservationSessionContractError(
"Фрагмент записанного видео был усечён во время передачи.",
);
}
const digest = bytesToHex(sha256(new Uint8Array(payload)));
if (digest !== descriptor.sha256) {
throw new ObservationSessionContractError(
"SHA-256 фрагмента записанного видео не совпадает с immutable manifest.",
);
}
return payload;
}
function manifestIdentity(manifest: ObservationRecordedMediaManifest): string {
return JSON.stringify(manifest);
}
export async function fetchVerifiedRecordedMediaArchive(
export async function fetchRecordedMediaArchive(
source: ObservationRecordedMediaSource,
{
signal,
fetcher = globalThis.fetch,
onProgress,
}: {
signal?: AbortSignal;
fetcher?: ObservationSessionFetch;
onProgress?: (progress: RecordedMediaLoadProgress) => void;
} = {},
): Promise<VerifiedRecordedMediaArchive> {
): Promise<RecordedMediaArchive> {
const manifest = await fetchObservationRecordedMediaManifest(source, {
signal,
fetcher,
expectedGenerationSha256: source.manifestGenerationSha256,
});
const totalBytes = manifest.epochs.reduce(
(archiveTotal, epoch) => archiveTotal + epoch.initByteLength + epoch.segments.reduce(
(epochTotal, segment) => epochTotal + segment.byteLength,
0,
),
0,
);
const totalParts = manifest.epochs.reduce(
(total, epoch) => total + 1 + epoch.segments.length,
0,
);
const totalBytes = manifest.epochs.reduce((total, epoch) => total + epoch.byteLength, 0);
if (
!Number.isSafeInteger(totalBytes) ||
totalBytes < 1 ||
totalBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES ||
totalBytes !== manifest.byteLength ||
totalBytes !== source.byteLength
) {
throw new ObservationSessionContractError(
"Размер записанного медиаканала выходит за безопасный лимит браузера.",
"Размеры потоков записанного медиаканала не совпадают с manifest.",
);
}
let loadedBytes = 0;
let loadedParts = 0;
const publishProgress = () => onProgress?.({
loadedBytes,
totalBytes,
loadedParts,
totalParts,
});
publishProgress();
const epochs: VerifiedRecordedMediaEpoch[] = [];
for (const epoch of manifest.epochs) {
const init = await fetchVerifiedRecordedMediaBytes({
url: epoch.initUrl,
byteLength: epoch.initByteLength,
sha256: epoch.initSha256,
accept: "video/mp4",
}, { signal, fetcher });
loadedBytes += init.byteLength;
loadedParts += 1;
publishProgress();
const segments: ArrayBuffer[] = [];
for (const segment of epoch.segments) {
const payload = await fetchVerifiedRecordedMediaBytes({
url: segment.url,
byteLength: segment.byteLength,
sha256: segment.sha256,
accept: "video/iso.segment",
}, { signal, fetcher });
segments.push(payload);
loadedBytes += payload.byteLength;
loadedParts += 1;
publishProgress();
}
epochs.push({ descriptor: epoch, init, segments });
}
// Bind the complete byte set to the same manifest generation at both ends
// of the transfer. A replacement during a long camera download fails closed.
const confirmed = await fetchObservationRecordedMediaManifest(source, {
signal,
fetcher,
expectedGenerationSha256: manifest.generationSha256,
});
if (manifestIdentity(confirmed) !== manifestIdentity(manifest)) {
throw new ObservationSessionContractError(
"Manifest записанного видео изменился во время полной загрузки.",
);
}
if (loadedBytes !== source.byteLength) {
throw new ObservationSessionContractError(
"Полная загрузка камеры не совпала с launch byte_length.",
);
}
return { manifest, epochs, byteLength: loadedBytes };
}
export function appendRecordedMediaBuffer(
sourceBuffer: SourceBuffer,
payload: ArrayBuffer,
signal: AbortSignal,
): Promise<void> {
return new Promise((resolve, reject) => {
const onUpdateEnd = () => {
cleanup();
resolve();
};
const onError = () => {
cleanup();
reject(new Error("SourceBuffer rejected archived fMP4 data"));
};
const onAbort = () => {
cleanup();
reject(new DOMException("Aborted", "AbortError"));
};
const cleanup = () => {
sourceBuffer.removeEventListener("updateend", onUpdateEnd);
sourceBuffer.removeEventListener("error", onError);
signal.removeEventListener("abort", onAbort);
};
if (signal.aborted) {
onAbort();
return;
}
sourceBuffer.addEventListener("updateend", onUpdateEnd, { once: true });
sourceBuffer.addEventListener("error", onError, { once: true });
signal.addEventListener("abort", onAbort, { once: true });
try {
sourceBuffer.appendBuffer(payload);
} catch (error) {
cleanup();
reject(error);
}
});
return { manifest, byteLength: totalBytes };
}
function videoHasSeekableArchive(
@@ -358,17 +162,22 @@ function waitForSeekableArchive(
if (videoHasSeekableArchive(video, declaredDurationSeconds)) return Promise.resolve();
return new Promise((resolve, reject) => {
const events = ["loadedmetadata", "durationchange", "progress", "canplay"] as const;
const timeout = globalThis.setTimeout(() => {
cleanup();
reject(new Error("Archived camera did not become seekable"));
}, 20_000);
let stallTimer: ReturnType<typeof globalThis.setTimeout> | undefined;
const armStallTimer = () => {
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
stallTimer = globalThis.setTimeout(() => {
cleanup();
reject(new Error("Archived camera stream stalled"));
}, 45_000);
};
const cleanup = () => {
globalThis.clearTimeout(timeout);
if (stallTimer !== undefined) globalThis.clearTimeout(stallTimer);
for (const event of events) video.removeEventListener(event, onProgress);
video.removeEventListener("error", onError);
signal.removeEventListener("abort", onAbort);
};
const onProgress = () => {
armStallTimer();
if (!videoHasSeekableArchive(video, declaredDurationSeconds)) return;
cleanup();
resolve();
@@ -384,43 +193,30 @@ function waitForSeekableArchive(
for (const event of events) video.addEventListener(event, onProgress);
video.addEventListener("error", onError, { once: true });
signal.addEventListener("abort", onAbort, { once: true });
armStallTimer();
});
}
async function mountVerifiedRecordedEpoch(
async function mountRecordedEpochStream(
video: HTMLVideoElement,
epoch: VerifiedRecordedMediaEpoch,
descriptor: ObservationRecordedMediaEpoch,
signal: AbortSignal,
): Promise<() => void> {
const descriptor = epoch.descriptor;
if (!descriptor.mediaType.startsWith("video/mp4;") || !video.canPlayType(descriptor.mediaType)) {
throw new Error("Archived camera codec is not supported");
}
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
// A sealed fragmented-MP4 generation is already complete and immutable.
// Present it as one Blob so Chromium can index and seek the whole archive
// without retaining the same 166+ MiB epoch in a quota-limited MSE
// SourceBuffer. Every constituent byte was fetched and SHA-256 verified
// before this boundary, and Blob preserves their canonical order.
const payload = new Blob([epoch.init, ...epoch.segments], {
type: descriptor.mediaType,
});
const expectedByteLength = epoch.init.byteLength + epoch.segments.reduce(
(total, segment) => total + segment.byteLength,
0,
);
if (payload.size !== expectedByteLength) {
throw new Error("Archived camera Blob is incomplete");
}
const objectUrl = URL.createObjectURL(payload);
// The generation token binds the native media request to the exact manifest.
// The browser range-streams the virtual init+fragment file from the server;
// no complete camera archive is copied into JavaScript memory.
const cleanup = () => {
video.pause();
video.removeAttribute("src");
video.load();
URL.revokeObjectURL(objectUrl);
};
video.src = objectUrl;
video.preload = "auto";
video.src = descriptor.streamUrl;
video.load();
try {
await waitForSeekableArchive(
@@ -483,22 +279,15 @@ export function RecordedFmp4Player({
recordedDelivery?.timelineEndSeconds,
],
);
const [archive, setArchive] = useState<VerifiedRecordedMediaArchive | null>(null);
const [archive, setArchive] = useState<RecordedMediaArchive | null>(null);
const [state, setState] = useState<"loading" | "ready" | "error">("loading");
const [readyGeneration, setReadyGeneration] = useState<string | null>(null);
const [progress, setProgress] = useState<RecordedMediaLoadProgress | null>(null);
const [bufferRevision, setBufferRevision] = useState(0);
const currentSeconds = playback?.currentSeconds ?? contract?.timelineStartSeconds ?? 0;
const epoch = useMemo(
() => selectRecordedMediaEpoch(archive?.manifest.epochs ?? [], currentSeconds),
[archive?.manifest.epochs, currentSeconds],
);
const verifiedEpoch = useMemo(
() => epoch
? archive?.epochs.find(({ descriptor }) => descriptor.ordinal === epoch.ordinal) ?? null
: null,
[archive?.epochs, epoch],
);
const waitingForEpoch = Boolean(archive && !epoch);
const selectedGeneration = contract && epoch
? `${contract.manifestGenerationSha256}:${epoch.ordinal}:${epoch.timelineStartSeconds}:${epoch.timelineEndSeconds}`
@@ -514,7 +303,6 @@ export function RecordedFmp4Player({
useEffect(() => {
if (!contract) {
setArchive(null);
setProgress(null);
setReadyGeneration(null);
setState("error");
reportAdmission({
@@ -527,7 +315,6 @@ export function RecordedFmp4Player({
if (!prepare) return;
const abort = new AbortController();
setArchive(null);
setProgress(null);
setReadyGeneration(null);
setState("loading");
reportAdmission({
@@ -535,12 +322,7 @@ export function RecordedFmp4Player({
byteLength: contract.byteLength,
message: null,
});
void fetchVerifiedRecordedMediaArchive(contract, {
signal: abort.signal,
onProgress: (next) => {
if (!abort.signal.aborted) setProgress(next);
},
})
void fetchRecordedMediaArchive(contract, { signal: abort.signal })
.then((loaded) => {
if (abort.signal.aborted) return;
setArchive(loaded);
@@ -566,11 +348,11 @@ export function RecordedFmp4Player({
const abort = new AbortController();
let disposed = false;
void (async () => {
for (const candidate of archive.epochs) {
for (const candidate of archive.manifest.epochs) {
const probe = document.createElement("video");
probe.muted = true;
probe.playsInline = true;
const cleanup = await mountVerifiedRecordedEpoch(probe, candidate, abort.signal);
const cleanup = await mountRecordedEpochStream(probe, candidate, abort.signal);
cleanup();
if (disposed || abort.signal.aborted) return;
}
@@ -602,8 +384,8 @@ export function RecordedFmp4Player({
useEffect(() => {
const video = videoRef.current;
if (!video || !verifiedEpoch) return;
const epochDescriptor = verifiedEpoch.descriptor;
if (!video || !epoch) return;
const epochDescriptor = epoch;
const generation = contract
? `${contract.manifestGenerationSha256}:${epochDescriptor.ordinal}:${epochDescriptor.timelineStartSeconds}:${epochDescriptor.timelineEndSeconds}`
: null;
@@ -615,7 +397,7 @@ export function RecordedFmp4Player({
const loadEpoch = async () => {
try {
cleanup = await mountVerifiedRecordedEpoch(video, verifiedEpoch, abort.signal);
cleanup = await mountRecordedEpochStream(video, epochDescriptor, abort.signal);
if (disposed || abort.signal.aborted) {
cleanup();
cleanup = null;
@@ -648,7 +430,7 @@ export function RecordedFmp4Player({
abort.abort();
cleanup?.();
};
}, [archive?.byteLength, contract, verifiedEpoch]);
}, [archive?.byteLength, contract, epoch]);
useEffect(() => {
const video = videoRef.current;
@@ -679,10 +461,6 @@ export function RecordedFmp4Player({
}
}, [archive?.byteLength, bufferRevision, currentSeconds, epoch, playback?.playing, visualState]);
const progressPercent = progress && progress.totalBytes > 0
? Math.min(100, Math.floor((progress.loadedBytes / progress.totalBytes) * 100))
: 0;
return (
<div
className="recorded-media-player"
@@ -707,8 +485,8 @@ export function RecordedFmp4Player({
: visualState === "error"
? "Записанное видео недоступно"
: archive
? "Проверяем полную готовность записанного видео…"
: `Загружаем и проверяем записанное видео · ${progressPercent}%`}
? "Проверяем seek и codec записанного видео…"
: "Читаем manifest записанного видео…"}
</div>
) : null}
</div>
@@ -4,7 +4,14 @@ import type { SceneSettings } from "../sceneSettings";
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
export type RecordedRerunView = "spatial" | "perception" | "metrics";
export type RecordedRerunView = "spatial" | "perception" | "perception3d" | "metrics";
export interface RecordedPerceptionLayers {
enabled: boolean;
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}
export interface RerunSelection {
entityPath: string;
@@ -57,6 +64,8 @@ export interface RerunViewportProps {
| "customColor"
>;
recordedView?: RecordedRerunView;
recordedViewResetGeneration?: 0 | 1;
recordedPerceptionLayers?: RecordedPerceptionLayers;
onPerceptionAvailabilityChange?: (available: boolean) => void;
}
@@ -438,11 +447,20 @@ export async function fetchRecordedBlueprintRrd(
origin,
signal,
activeView = "spatial",
viewResetGeneration = 0,
perceptionLayers = {
enabled: false,
detections2d: false,
segmentation: false,
cuboids3d: false,
},
fetcher = globalThis.fetch,
}: {
origin: string;
signal?: AbortSignal;
activeView?: RecordedRerunView;
viewResetGeneration?: 0 | 1;
perceptionLayers?: RecordedPerceptionLayers;
fetcher?: typeof globalThis.fetch;
},
): Promise<Uint8Array> {
@@ -461,7 +479,14 @@ export async function fetchRecordedBlueprintRrd(
settings.pointSize > 32 ||
!["turbo", "viridis", "plasma", "grayscale", "custom"].includes(settings.palette) ||
!/^#[0-9A-Fa-f]{6}$/.test(settings.customColor) ||
!["spatial", "perception", "metrics"].includes(activeView) ||
!["spatial", "perception", "perception3d", "metrics"].includes(activeView) ||
![0, 1].includes(viewResetGeneration) ||
[
perceptionLayers.enabled,
perceptionLayers.detections2d,
perceptionLayers.segmentation,
perceptionLayers.cuboids3d,
].some((value) => typeof value !== "boolean") ||
identity.applicationId !== "nodedc_mission_core_recorded" ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(identity.recordingId)
) {
@@ -485,6 +510,11 @@ export async function fetchRecordedBlueprintRrd(
palette: settings.palette,
custom_color: settings.customColor,
active_view: activeView,
view_reset_generation: viewResetGeneration,
unified_perception: perceptionLayers.enabled,
show_detections_2d: perceptionLayers.detections2d,
show_segmentation: perceptionLayers.segmentation,
show_cuboids_3d: perceptionLayers.cuboids3d,
}),
signal,
});
@@ -588,6 +618,13 @@ export function RerunViewport({
onPlaybackControllerChange,
sceneSettings,
recordedView = "spatial",
recordedViewResetGeneration = 0,
recordedPerceptionLayers = {
enabled: false,
detections2d: false,
segmentation: false,
cuboids3d: false,
},
onPerceptionAvailabilityChange,
}: RerunViewportProps) {
const hostRef = useRef<HTMLDivElement>(null);
@@ -596,6 +633,7 @@ export function RerunViewport({
const [retryNonce, setRetryNonce] = useState(0);
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const loadedPerceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const recordedIdentityRef = useRef<RecordedRerunIdentity | null>(null);
const presentationGateRef = useRef(presentationGate);
presentationGateRef.current = presentationGate;
@@ -1123,11 +1161,11 @@ export function RerunViewport({
]);
useEffect(() => {
onPerceptionAvailabilityChange?.(false);
}, [onPerceptionAvailabilityChange, recordedPerceptionUrl]);
loadedPerceptionChannelRef.current = null;
}, [recordedPerceptionUrl]);
useEffect(() => {
if (!recordedPerceptionUrl) return;
if (!recordedPerceptionUrl || !recordedPerceptionLayers.enabled) return;
const active = perceptionChannelRef.current;
const identity = recordedIdentityRef.current;
if (
@@ -1136,6 +1174,10 @@ export function RerunViewport({
active.endpointUrl !== recordedPerceptionUrl ||
!active.channel.ready
) return;
if (loadedPerceptionChannelRef.current === active) {
onPerceptionAvailabilityChange?.(true);
return;
}
const abort = new AbortController();
void fetchRecordedPerceptionRrd(recordedPerceptionUrl, identity, {
origin: window.location.origin,
@@ -1154,6 +1196,7 @@ export function RerunViewport({
return;
}
active.channel.send_rrd(payload);
loadedPerceptionChannelRef.current = active;
onPerceptionAvailabilityChange?.(true);
}).catch(() => {
onPerceptionAvailabilityChange?.(false);
@@ -1163,6 +1206,7 @@ export function RerunViewport({
}, [
onPerceptionAvailabilityChange,
perceptionChannelRevision,
recordedPerceptionLayers.enabled,
recordedPerceptionUrl,
]);
@@ -1181,6 +1225,8 @@ export function RerunViewport({
origin: window.location.origin,
signal: abort.signal,
activeView: recordedView,
viewResetGeneration: recordedViewResetGeneration,
perceptionLayers: recordedPerceptionLayers,
}).then((payload) => {
if (
abort.signal.aborted ||
@@ -1200,6 +1246,11 @@ export function RerunViewport({
blueprintChannelRevision,
recordedBlueprintUrl,
recordedView,
recordedViewResetGeneration,
recordedPerceptionLayers.enabled,
recordedPerceptionLayers.detections2d,
recordedPerceptionLayers.segmentation,
recordedPerceptionLayers.cuboids3d,
sceneSettings?.accumulationSeconds,
sceneSettings?.customColor,
sceneSettings?.palette,
@@ -38,47 +38,57 @@ export function recordedObservationSources(
spatialRegistration: "native",
},
};
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => ({
id: source.id,
sourceId: source.id,
semanticChannelId: "camera.video.recorded",
label: source.label,
description: "Сохранённый видеоканал на общей временной шкале сессии",
modality: "video",
role: "auxiliary",
availability: "available",
transport: "recording",
endpointLabel: "Сохранённая сессия",
previewUrl: null,
delivery: {
id: `${launch.sessionId}:${source.id}`,
kind: "recorded-fmp4-manifest",
url: source.manifestUrl,
mediaType: source.mediaType,
manifestGenerationSha256: source.manifestGenerationSha256,
byteLength: source.byteLength,
timelineStartSeconds: source.timelineStartSeconds,
timelineEndSeconds: source.timelineEndSeconds,
},
activation: null,
provider: {
pluginId: "missioncore.session-archive",
pluginVersion: "1",
modelId: "recorded-media",
compatibilityProfileId: null,
},
binding: {},
capabilities: {
overlay: true,
fullscreen: true,
resizable: true,
defaultVisible: index < 2,
timelineMode: "recorded",
seekable: true,
sessionRecording: true,
clockId: "session_time",
spatialRegistration: "unresolved",
},
}));
const media = launch.mediaSources.map((source, index): ObservationSourceDescriptor => {
const perception = source.id.startsWith("recorded.perception.");
return {
id: source.id,
sourceId: source.id,
semanticChannelId: perception
? "camera.perception.panoptic.recorded"
: "camera.video.recorded",
label: source.label,
description: perception
? "Покадровая instance + semantic сегментация на общей временной шкале"
: "Сохранённый видеоканал на общей временной шкале сессии",
modality: "video",
role: "auxiliary",
availability: "available",
transport: "recording",
endpointLabel: "Сохранённая сессия",
previewUrl: null,
delivery: {
id: `${launch.sessionId}:${source.id}`,
kind: "recorded-fmp4-manifest",
url: source.manifestUrl,
mediaType: source.mediaType,
manifestGenerationSha256: source.manifestGenerationSha256,
byteLength: source.byteLength,
timelineStartSeconds: source.timelineStartSeconds,
timelineEndSeconds: source.timelineEndSeconds,
},
activation: null,
provider: {
pluginId: "missioncore.session-archive",
pluginVersion: "1",
modelId: perception ? "recorded-panoptic-perception" : "recorded-media",
compatibilityProfileId: null,
},
binding: {},
capabilities: {
overlay: true,
fullscreen: true,
resizable: true,
// AI presentation belongs to the unified Rerun composition. Keep the
// pre-rendered perception video as an explicit fallback instead of
// opening it as a second, independently controlled stream.
defaultVisible: !perception && index < 2,
timelineMode: "recorded",
seekable: true,
sessionRecording: true,
clockId: "session_time",
spatialRegistration: perception ? "calibrated" : "unresolved",
},
};
});
return [spatial, ...media];
}
@@ -1,12 +1,8 @@
// A device-agnostic frontend safety policy. Sixteen channels covers multi-rig
// vehicles while the independent byte/concurrency limits keep admission
// bounded. One real accepted archive is ~163 MiB for a single camera, so the
// old 128 MiB laboratory ceiling rejected a valid sealed generation before the
// first manifest request. OPFS-backed sealed generations remain the scaling
// path beyond this in-memory policy.
// Camera count and preparation concurrency remain device-agnostic scheduling
// policy. Recorded duration and aggregate bytes are deliberately not admission
// criteria: sealed media is presented through a generation-bound HTTP stream,
// so a one-, three- or ten-hour recording never has to fit in browser memory.
export const MAX_RECORDED_CAMERA_SOURCES = 16;
export const MAX_RECORDED_MEDIA_SOURCE_BYTES = 256 * 1024 * 1024;
export const MAX_RECORDED_SESSION_CAMERA_BYTES = 512 * 1024 * 1024;
export const MAX_CONCURRENT_RECORDED_CAMERA_PREPARATIONS = 1;
export type RecordedAdmissionPhase = "loading" | "ready" | "error";
@@ -32,18 +28,12 @@ export function recordedCameraDescriptorPreflight(
): RecordedAdmissionPhase {
if (sources.length > MAX_RECORDED_CAMERA_SOURCES) return "error";
if (new Set(sources.map(({ id }) => id)).size !== sources.length) return "error";
let totalBytes = 0;
for (const source of sources) {
if (
!source.id ||
!Number.isSafeInteger(source.byteLength) ||
source.byteLength < 1 ||
source.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
source.byteLength < 1
) return "error";
totalBytes += source.byteLength;
if (!Number.isSafeInteger(totalBytes) || totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) {
return "error";
}
}
return "ready";
}
@@ -67,7 +57,6 @@ export function recordedSessionAdmissionPhase(
if (sourceIds.length > MAX_RECORDED_CAMERA_SOURCES || spatialPhase === "error") {
return "error";
}
let totalBytes = 0;
for (const sourceId of sourceIds) {
const camera = cameras[sourceId];
if (!camera || camera.phase === "error") return "error";
@@ -75,13 +64,10 @@ export function recordedSessionAdmissionPhase(
if (camera.byteLength !== null) {
if (
!Number.isSafeInteger(camera.byteLength) ||
camera.byteLength < 1 ||
camera.byteLength > MAX_RECORDED_MEDIA_SOURCE_BYTES
camera.byteLength < 1
) return "error";
totalBytes += camera.byteLength;
}
}
if (totalBytes > MAX_RECORDED_SESSION_CAMERA_BYTES) return "error";
if (spatialPhase !== "ready") return "loading";
return sourceIds.every((sourceId) => cameras[sourceId]?.phase === "ready")
? "ready"
@@ -1,7 +1,5 @@
import {
MAX_RECORDED_CAMERA_SOURCES,
MAX_RECORDED_MEDIA_SOURCE_BYTES,
MAX_RECORDED_SESSION_CAMERA_BYTES,
} from "./recordedSessionAdmission";
export type ObservationSessionStatus =
@@ -99,19 +97,8 @@ export interface ObservationRecordedMediaEpoch {
timelineStartSeconds: number;
timelineEndSeconds: number;
mediaType: string;
initUrl: string;
initByteLength: number;
initSha256: string;
segmentCount: number;
segmentUrlPrefix: string;
segments: readonly ObservationRecordedMediaSegment[];
}
export interface ObservationRecordedMediaSegment {
sequence: number;
url: string;
byteLength: number;
sha256: string;
streamUrl: string;
}
export interface ObservationRecordedMediaManifest {
@@ -215,18 +202,8 @@ const RECORDED_MEDIA_EPOCH_KEYS = new Set([
"timeline_start_seconds",
"timeline_end_seconds",
"media_type",
"init_url",
"init_byte_length",
"init_sha256",
"segment_count",
"segment_url_prefix",
"segments",
]);
const RECORDED_MEDIA_SEGMENT_KEYS = new Set([
"sequence",
"url",
"byte_length",
"sha256",
"stream_url",
]);
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
const SAFE_MODALITY = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/;
@@ -234,10 +211,9 @@ const ISO_WITH_TIMEZONE = /^\d{4}-\d{2}-\d{2}T.+(?:Z|[+-]\d{2}:\d{2})$/;
const SHA256 = /^[a-f0-9]{64}$/;
const SAFE_RECORDING_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording\.rrd$/;
const SAFE_PREPARATION_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/recording-preparation$/;
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/media\/[A-Za-z0-9._%-]+\/manifest$/;
const SAFE_MEDIA_MANIFEST_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+|perception-media\/result-[a-f0-9]{64})\/manifest$/;
const SAFE_MEDIA_STREAM_URL = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9._%:-]+\/(?:media\/[A-Za-z0-9._%-]+\/epochs\/[1-9][0-9]*|perception-media\/result-[a-f0-9]{64})\/recording\.mp4\?generation=[a-f0-9]{64}$/;
const SAFE_MP4_MEDIA_TYPE = /^video\/mp4(?:; codecs="[A-Za-z0-9.,_-]+")?$/;
const MAX_RECORDED_INIT_BYTES = 8 * 1024 * 1024;
const MAX_RECORDED_SEGMENT_BYTES = 64 * 1024 * 1024;
export class ObservationSessionContractError extends Error {
constructor(message: string) {
@@ -570,15 +546,6 @@ export function decodeObservationSessionReplay(
"Descriptor записи содержит повторяющиеся медиаканалы.",
);
}
const mediaByteLength = mediaSources.reduce((total, source) => total + source.byteLength, 0);
if (
!Number.isSafeInteger(mediaByteLength) ||
mediaByteLength > MAX_RECORDED_SESSION_CAMERA_BYTES
) {
throw new ObservationSessionContractError(
"Совокупный объём записанных медиаканалов превышает безопасный лимит браузера.",
);
}
return {
kind: "rerun-recording",
sessionId,
@@ -709,7 +676,10 @@ function decodeRecordedMediaSource(
}
assertExactKeys(value, RECORDED_MEDIA_SOURCE_KEYS, `Медиаканал ${index}`);
const id = requireString(value.id, `media_sources[${index}].id`, 128);
if (!SAFE_ID.test(id) || !id.startsWith("recorded.camera.")) {
if (
!SAFE_ID.test(id) ||
(!id.startsWith("recorded.camera.") && !id.startsWith("recorded.perception."))
) {
throw new ObservationSessionContractError("Медиаканал содержит небезопасный opaque id.");
}
if (value.modality !== "video" || value.media_type !== "video/mp4") {
@@ -728,10 +698,10 @@ function decodeRecordedMediaSource(
"Медиаканал не содержит immutable generation SHA-256.",
);
}
const sessionPrefix = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/media/`;
const sessionBase = `/api/v1/observation-sessions/${encodeURIComponent(sessionId)}/`;
if (
!SAFE_MEDIA_MANIFEST_URL.test(manifestUrl) ||
!manifestUrl.startsWith(sessionPrefix) ||
!manifestUrl.startsWith(sessionBase) ||
manifestUrl.includes("..")
) {
throw new ObservationSessionContractError(
@@ -765,7 +735,7 @@ function decodeRecordedMediaSource(
byteLength: requireFiniteNumber(
value.byte_length,
`media_sources[${index}].byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES, integer: true },
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
),
mediaType: "video/mp4",
timelineStartSeconds,
@@ -784,22 +754,21 @@ export function decodeObservationRecordedMediaManifest(
}
assertExactKeys(payload, RECORDED_MEDIA_MANIFEST_KEYS, "Manifest записанного видео");
if (
payload.schema_version !== "missioncore.observation-recorded-media/v2" ||
payload.schema_version !== "missioncore.observation-recorded-media/v3" ||
payload.source_id !== source.id ||
typeof payload.generation_sha256 !== "string" ||
!SHA256.test(payload.generation_sha256) ||
payload.generation_sha256 !== source.manifestGenerationSha256 ||
payload.synchronization !== "host-arrival-best-effort" ||
!Array.isArray(payload.epochs) ||
payload.epochs.length < 1 ||
payload.epochs.length > 1_000
payload.epochs.length < 1
) {
throw new ObservationSessionContractError("Manifest записанного видео несовместим.");
}
const manifestBase = source.manifestUrl.slice(0, -"/manifest".length);
const manifestByteLength = requireFiniteNumber(payload.byte_length, "manifest.byte_length", {
minimum: 1,
maximum: MAX_RECORDED_MEDIA_SOURCE_BYTES,
maximum: Number.MAX_SAFE_INTEGER,
integer: true,
});
if (manifestByteLength !== source.byteLength) {
@@ -833,7 +802,7 @@ export function decodeObservationRecordedMediaManifest(
assertExactKeys(entry, RECORDED_MEDIA_EPOCH_KEYS, `Codec epoch ${index}`);
const ordinal = requireFiniteNumber(entry.ordinal, `epochs[${index}].ordinal`, {
minimum: 1,
maximum: 1_000,
maximum: Number.MAX_SAFE_INTEGER,
integer: true,
});
if (ordinal !== index + 1) {
@@ -860,87 +829,28 @@ export function decodeObservationRecordedMediaManifest(
if (!SAFE_MP4_MEDIA_TYPE.test(mediaType)) {
throw new ObservationSessionContractError("Codec epoch содержит небезопасный media type.");
}
const initUrl = requireString(entry.init_url, `epochs[${index}].init_url`, 512);
const segmentUrlPrefix = requireString(
entry.segment_url_prefix,
`epochs[${index}].segment_url_prefix`,
512,
const byteLength = requireFiniteNumber(
entry.byte_length,
`epochs[${index}].byte_length`,
{ minimum: 1, maximum: Number.MAX_SAFE_INTEGER, integer: true },
);
const expectedEpochBase = `${manifestBase}/epochs/${ordinal}`;
const streamUrl = requireString(entry.stream_url, `epochs[${index}].stream_url`, 768);
const expectedEpochBase = source.id.startsWith("recorded.perception.")
? manifestBase
: `${manifestBase}/epochs/${ordinal}`;
const expectedStreamUrl =
`${expectedEpochBase}/recording.mp4?generation=${payload.generation_sha256}`;
if (
initUrl !== `${expectedEpochBase}/init.mp4` ||
segmentUrlPrefix !== `${expectedEpochBase}/segments/` ||
initUrl.includes("..") ||
segmentUrlPrefix.includes("..")
!SAFE_MEDIA_STREAM_URL.test(streamUrl) ||
streamUrl !== expectedStreamUrl ||
streamUrl.includes("..")
) {
throw new ObservationSessionContractError("Codec epoch содержит небезопасный API URL.");
}
const initByteLength = requireFiniteNumber(
entry.init_byte_length,
`epochs[${index}].init_byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_INIT_BYTES, integer: true },
);
if (typeof entry.init_sha256 !== "string" || !SHA256.test(entry.init_sha256)) {
throw new ObservationSessionContractError("Codec epoch не содержит SHA-256 init-сегмента.");
}
const segmentCount = requireFiniteNumber(
entry.segment_count,
`epochs[${index}].segment_count`,
{ minimum: 1, maximum: 500_000, integer: true },
);
if (!Array.isArray(entry.segments) || entry.segments.length !== segmentCount) {
declaredBytes += byteLength;
if (!Number.isSafeInteger(declaredBytes)) {
throw new ObservationSessionContractError(
"Codec epoch содержит неполный список канонических сегментов.",
);
}
const segments = entry.segments.map((segment, segmentIndex): ObservationRecordedMediaSegment => {
if (!isRecord(segment)) {
throw new ObservationSessionContractError(
`Сегмент epochs[${index}].segments[${segmentIndex}] должен быть объектом.`,
);
}
assertExactKeys(
segment,
RECORDED_MEDIA_SEGMENT_KEYS,
`Сегмент epochs[${index}].segments[${segmentIndex}]`,
);
const sequence = requireFiniteNumber(
segment.sequence,
`epochs[${index}].segments[${segmentIndex}].sequence`,
{ minimum: 1, maximum: 500_000, integer: true },
);
if (sequence !== segmentIndex + 1) {
throw new ObservationSessionContractError(
"Codec epoch содержит непоследовательный сегмент.",
);
}
const url = requireString(
segment.url,
`epochs[${index}].segments[${segmentIndex}].url`,
512,
);
if (url !== `${segmentUrlPrefix}${sequence}.m4s` || url.includes("..")) {
throw new ObservationSessionContractError(
"Codec epoch содержит небезопасный URL сегмента.",
);
}
const byteLength = requireFiniteNumber(
segment.byte_length,
`epochs[${index}].segments[${segmentIndex}].byte_length`,
{ minimum: 1, maximum: MAX_RECORDED_SEGMENT_BYTES, integer: true },
);
if (typeof segment.sha256 !== "string" || !SHA256.test(segment.sha256)) {
throw new ObservationSessionContractError(
"Codec epoch содержит сегмент без SHA-256.",
);
}
declaredBytes += byteLength;
return { sequence, url, byteLength, sha256: segment.sha256 };
});
declaredBytes += initByteLength;
if (declaredBytes > MAX_RECORDED_MEDIA_SOURCE_BYTES) {
throw new ObservationSessionContractError(
"Записанный медиаканал превышает безопасный лимит браузера.",
"Суммарный размер codec epoch выходит за точный числовой диапазон клиента.",
);
}
return {
@@ -948,17 +858,13 @@ export function decodeObservationRecordedMediaManifest(
timelineStartSeconds,
timelineEndSeconds,
mediaType,
initUrl,
initByteLength,
initSha256: entry.init_sha256,
segmentCount,
segmentUrlPrefix,
segments,
byteLength,
streamUrl,
};
});
if (declaredBytes !== manifestByteLength) {
throw new ObservationSessionContractError(
"Сумма init- и media-сегментов не совпадает с размером immutable manifest.",
"Сумма codec epoch не совпадает с размером immutable manifest.",
);
}
if (
@@ -285,11 +285,16 @@ export async function resolveObservationSessionReplay(
sessionId: string,
options: ObservationPreparationPollingOptions,
): Promise<ObservationSessionReplayLaunch> {
const response = await withRequestTimeout(
options.signal,
Math.max(100, options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS),
(signal) => replayObservationSession(sessionId, { signal, fetcher: options.fetcher }),
);
// The initial replay request may restore and integrity-check large, already
// published artifacts before it can return either a launch descriptor or a
// background-preparation handle. Its duration therefore scales with the
// recording package and must not be confused with a stalled status poll.
// Keep it cancellable by the owning UI attempt, but do not impose the short
// per-poll timeout used once the server has returned a preparation handle.
const response = await replayObservationSession(sessionId, {
signal: options.signal,
fetcher: options.fetcher,
});
if (response.kind === "ready") return response.launch;
if (!pendingPreparation(response.preparation)) {
options.onUpdate?.(response.preparation);
@@ -22,6 +22,9 @@ export interface ViewerSettings {
show_points: boolean;
show_trajectory: boolean;
show_grid: boolean;
show_detections_2d: boolean;
show_segmentation: boolean;
show_cuboids_3d: boolean;
}
export interface StreamMetrics {
@@ -29,6 +32,10 @@ export interface StreamMetrics {
frameRateHz?: number | null;
pointCount?: number | null;
droppedPreviewFrames?: number | null;
aiLatencyMs?: number | null;
aiFrameRateHz?: number | null;
aiDroppedFrames?: number | null;
aiStaleMs?: number | null;
elapsedSeconds?: number | null;
routeDistanceMeters?: number | null;
speedMetersPerSecond?: number | null;
@@ -32,6 +32,25 @@
backdrop-filter: blur(18px);
}
.scene-navigation-hint {
position: absolute;
z-index: 11;
right: 0.85rem;
bottom: 4.9rem;
display: flex;
align-items: center;
gap: 0.75rem;
border: 1px solid rgb(255 255 255 / 0.07);
border-radius: 999px;
background: rgb(9 10 13 / 0.7);
color: var(--nodedc-text-muted);
padding: 0.42rem 0.65rem;
font-size: 0.52rem;
font-weight: 680;
pointer-events: none;
backdrop-filter: blur(14px);
}
.scene-source-picker__trigger {
position: relative;
}
@@ -350,6 +369,11 @@ i[data-availability="error"] {
box-shadow: 0 1.5rem 4rem rgb(0 0 0 / 0.34);
}
.floating-observation-window--hidden {
visibility: hidden;
pointer-events: none;
}
.observation-timeline {
display: block;
min-width: 0;
@@ -25,6 +25,20 @@
gap: 0.45rem;
}
.spatial-toolbar__view-switch {
display: inline-flex;
align-items: center;
gap: 0.25rem;
border: 1px solid rgb(255 255 255 / 0.07);
border-radius: 999px;
background: rgb(255 255 255 / 0.025);
padding: 0.2rem;
}
.spatial-toolbar__view-switch .nodedc-button {
border-radius: 999px;
}
.spatial-viewport-shell {
position: relative;
min-width: 0;
@@ -45,7 +45,6 @@ import {
type RerunPlaybackState,
type RerunSelection,
type RerunViewportStatus,
type RecordedRerunView,
} from "../components/RerunViewport";
import {
capabilityStatusLabel,
@@ -124,6 +123,16 @@ export interface WorkspaceRendererProps {
accumulationSeconds: number;
onAccumulationChange: (value: number) => void;
onAccumulationCommit: () => void;
livePerceptionLayers: {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
};
onLivePerceptionLayersChange: (next: {
detections2d: boolean;
segmentation: boolean;
cuboids3d: boolean;
}) => void;
observationLayout: ObservationLayoutController;
navigation: WorkspaceNavigation;
spatialControls: {
@@ -286,6 +295,8 @@ function SpatialWorkspace({
accumulationSeconds,
onAccumulationChange,
onAccumulationCommit,
livePerceptionLayers,
onLivePerceptionLayersChange,
observationLayout,
navigation,
spatialControls,
@@ -295,8 +306,13 @@ function SpatialWorkspace({
const [selection, setSelection] = useState<RerunSelection | null>(null);
const [playbackState, setPlaybackState] = useState<RerunPlaybackState | null>(null);
const [playbackController, setPlaybackController] = useState<RerunPlaybackController | null>(null);
const [recordedRerunView, setRecordedRerunView] = useState<RecordedRerunView>("spatial");
const [perceptionAvailable, setPerceptionAvailable] = useState(false);
const [recordedViewResetGeneration, setRecordedViewResetGeneration] = useState<0 | 1>(0);
const [perceptionAvailability, setPerceptionAvailability] = useState<
"unknown" | "available" | "unavailable"
>("unknown");
const [showDetections2d, setShowDetections2d] = useState(false);
const [showSegmentation, setShowSegmentation] = useState(false);
const [showCuboids3d, setShowCuboids3d] = useState(false);
const recordedSource = state?.sourceMode === "replay" || /\.rrd(?:$|[?#])/i.test(sourceUrl);
const recordedSessionGate: RecordedAdmissionPhase = recordedSource
? recordedSessionAdmission?.phase ?? "loading"
@@ -309,6 +325,8 @@ function SpatialWorkspace({
const latency = pipelineLatency(metrics);
const frameRate = finiteMetric(metrics?.frameRateHz);
const points = finiteMetric(metrics?.pointCount);
const aiLatency = finiteMetric(metrics?.aiLatencyMs);
const aiFrameRate = finiteMetric(metrics?.aiFrameRateHz);
const observationSources = state?.observationSources ?? [];
const pointCloudSource = observationSources.find((source) => source.modality === "point-cloud");
const pointCloudVisible = pointCloudSource
@@ -317,13 +335,29 @@ function SpatialWorkspace({
const mediaSources = observationSources.filter(
(source) => source.capabilities.overlay && source.modality !== "point-cloud",
);
const recordedPerceptionSupported = recordedSource && perceptionAvailability !== "unavailable";
const recordedPerceptionEnabled = showDetections2d || showSegmentation || showCuboids3d;
const unifiedPerception = recordedPerceptionSupported && recordedPerceptionEnabled;
const livePerceptionAvailable = !recordedSource && streamActive;
const detections2dActive = recordedSource
? showDetections2d
: livePerceptionLayers.detections2d;
const segmentationActive = recordedSource
? showSegmentation
: livePerceptionLayers.segmentation;
const cuboids3dActive = recordedSource
? showCuboids3d
: livePerceptionLayers.cuboids3d;
const visibleMediaSources = mediaSources.filter((source) =>
observationLayout.visibleSourceIds.has(source.id),
observationLayout.visibleSourceIds.has(source.id) &&
!source.id.startsWith("recorded.perception."),
);
const presentedMediaSourceCount = unifiedPerception ? 0 : visibleMediaSources.length;
const pointCloudFocused = Boolean(
pointCloudSource && observationLayout.focusedSourceId === pointCloudSource.id,
);
const floatingSourceMaximized = Boolean(observationLayout.maximizedFloatingSourceId);
const floatingSourceMaximized = !unifiedPerception &&
Boolean(observationLayout.maximizedFloatingSourceId);
const timeline = state?.observationTimeline;
const viewportRef = useRef<HTMLDivElement>(null);
const intentionalSourceEnd = !recordedSource && [
@@ -373,13 +407,20 @@ function SpatialWorkspace({
[],
);
const onPerceptionAvailabilityChange = useCallback((available: boolean) => {
setPerceptionAvailable(available);
if (!available) setRecordedRerunView("spatial");
setPerceptionAvailability(available ? "available" : "unavailable");
if (!available) {
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
}
}, []);
useEffect(() => {
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
setPerceptionAvailability("unknown");
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
setRecordedViewResetGeneration(0);
}, [sourceUrl]);
useEffect(() => {
@@ -389,8 +430,10 @@ function SpatialWorkspace({
setSelection(null);
setPlaybackState(null);
setPlaybackController(null);
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
setPerceptionAvailability("unknown");
setShowDetections2d(false);
setShowSegmentation(false);
setShowCuboids3d(false);
}, [pointCloudVisible, sourceUrl]);
useEffect(() => {
@@ -428,20 +471,74 @@ function SpatialWorkspace({
className="spatial-workspace"
data-focused={pointCloudFocused || floatingSourceMaximized ? "true" : undefined}
>
<div className="spatial-toolbar">
<div className="spatial-toolbar" data-viewer-controls="v1">
<div className="spatial-toolbar__mode">
<span className="section-eyebrow">СЦЕНА 3D · RERUN</span>
</div>
<div className="spatial-toolbar__actions">
{recordedSource && perceptionAvailable ? (
{recordedPerceptionSupported || livePerceptionAvailable ? (
<div className="spatial-toolbar__view-switch" role="group" aria-label="Слои распознавания сцены">
<Button
size="compact"
variant="primary"
icon={<Icon name="video" />}
aria-pressed="true"
disabled
>
Оригинал
</Button>
<Button
size="compact"
variant={detections2dActive ? "primary" : "secondary"}
icon={<Icon name="target" />}
aria-pressed={detections2dActive}
onClick={() => recordedSource
? setShowDetections2d((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
detections2d: !livePerceptionLayers.detections2d,
})}
>
Объекты 2D
</Button>
<Button
size="compact"
variant={segmentationActive ? "primary" : "secondary"}
icon={<Icon name="image" />}
aria-pressed={segmentationActive}
onClick={() => recordedSource
? setShowSegmentation((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
segmentation: !livePerceptionLayers.segmentation,
})}
>
Сегментация
</Button>
<Button
size="compact"
variant={cuboids3dActive ? "primary" : "secondary"}
icon={<Icon name="apps" />}
aria-pressed={cuboids3dActive}
onClick={() => recordedSource
? setShowCuboids3d((current) => !current)
: onLivePerceptionLayersChange({
...livePerceptionLayers,
cuboids3d: !livePerceptionLayers.cuboids3d,
})}
>
Кубы 3D
</Button>
</div>
) : null}
{recordedSource && presentedViewerStatus === "ready" ? (
<Button
size="compact"
variant={recordedRerunView === "perception" ? "primary" : "secondary"}
icon={<Icon name={recordedRerunView === "perception" ? "globe" : "image"} />}
onClick={() => setRecordedRerunView((current) =>
current === "perception" ? "spatial" : "perception")}
variant="secondary"
icon={<Icon name="refresh" />}
onClick={() => setRecordedViewResetGeneration((current) => current === 0 ? 1 : 0)}
>
{recordedRerunView === "perception" ? "Облако точек" : "Распознавание"}
Сброс вида
</Button>
) : null}
<Button size="compact" variant="secondary" icon={<Icon name="network" />} onClick={navigation.openSource}>
@@ -476,7 +573,13 @@ function SpatialWorkspace({
? state?.observationTimeline?.range?.endSeconds
: undefined}
sceneSettings={sceneSettings}
recordedView={recordedRerunView}
recordedViewResetGeneration={recordedViewResetGeneration}
recordedPerceptionLayers={{
enabled: unifiedPerception,
detections2d: showDetections2d,
segmentation: showSegmentation,
cuboids3d: showCuboids3d,
}}
onPerceptionAvailabilityChange={onPerceptionAvailabilityChange}
onStatusChange={onStatusChange}
onSelectionChange={onSelectionChange}
@@ -511,7 +614,9 @@ function SpatialWorkspace({
) : !floatingSourceMaximized ? (
<div className="scene-source-controls">
<ObservationSourcePicker
sources={observationSources}
sources={unifiedPerception
? observationSources.filter((source) => source.modality === "point-cloud")
: observationSources}
visibleSourceIds={observationLayout.visibleSourceIds}
pendingSourceIds={observationLayout.pendingSourceIds}
onToggle={observationLayout.toggleSource}
@@ -555,6 +660,16 @@ function SpatialWorkspace({
<span>До публикации</span>
<strong>{formatNumber(latency)}<small> мс</small></strong>
</div>
{streamActive ? (
<div>
<span>AI</span>
<strong>
{aiLatency === null ? "—" : formatNumber(aiLatency)}
<small>{aiLatency === null ? "" : " мс"}</small>
{aiFrameRate === null ? null : <small> · {formatNumber(aiFrameRate)} Гц</small>}
</strong>
</div>
) : null}
</div>
{!pointCloudFocused && !floatingSourceMaximized && state?.sourceMode && state.sourceMode !== "idle" && !sourceUrl.trim() ? (
@@ -575,10 +690,17 @@ function SpatialWorkspace({
</div>
) : null}
{presentedViewerStatus === "ready" && !floatingSourceMaximized ? (
<div className="scene-navigation-hint" aria-label="Навигация по 3D-сцене">
<span>Колесо · зум к курсору</span>
<span>WASD · свободный проход</span>
</div>
) : null}
{!pointCloudFocused && !floatingSourceMaximized && recordedPlaybackReady ? (
<ObservationTimeline
active={presentedViewerStatus === "ready"}
sourceCount={Math.max(1, 1 + visibleMediaSources.length)}
sourceCount={Math.max(1, (unifiedPerception ? 2 : 1) + presentedMediaSourceCount)}
mode={recordedSource && playbackState?.rangeNs
? "recorded"
: timeline?.mode}
@@ -599,7 +721,7 @@ function SpatialWorkspace({
/>
) : null}
{!pointCloudFocused ? visibleMediaSources.map((source, index) => (
{visibleMediaSources.map((source, index) => (
<FloatingObservationWindow
key={source.id}
source={source}
@@ -609,6 +731,7 @@ function SpatialWorkspace({
rect={observationLayout.windowRects[source.id]}
maximized={observationLayout.maximizedFloatingSourceId === source.id}
active={observationLayout.activeFloatingSourceId === source.id}
hidden={pointCloudFocused || unifiedPerception}
onRectChange={(rect) => observationLayout.setWindowRect(source.id, rect)}
onMaximizedChange={(maximized) =>
observationLayout.setFloatingMaximized(source.id, maximized)}
@@ -627,12 +750,12 @@ function SpatialWorkspace({
void observationLayout.hideSource(source.id);
}}
/>
)) : null}
))}
{recordedSource ? (
<div className="recorded-session-preloaders" aria-hidden="true">
{mediaSources.filter((source) => (
source.delivery?.kind === "recorded-fmp4-manifest" &&
(pointCloudFocused || !observationLayout.visibleSourceIds.has(source.id)) &&
!observationLayout.visibleSourceIds.has(source.id) &&
shouldPrepareRecordedSource(source.id)
)).map((source) => (
<ObservationMedia
@@ -657,7 +780,9 @@ function SpatialWorkspace({
<span><i data-state="ready" />Траектория</span>
<span><i data-state="ready" />Преобразования</span>
<span><i data-state="contract" />Камеры в 3D</span>
<span><i data-state={perceptionAvailable ? "ready" : "contract"} />Объекты / рамки</span>
<span><i data-state={detections2dActive ? "ready" : "contract"} />Объекты 2D</span>
<span><i data-state={segmentationActive ? "ready" : "contract"} />Сегментация</span>
<span><i data-state={cuboids3dActive ? "ready" : "contract"} />Кубы 3D</span>
<span><i data-state="contract" />Компоновка</span>
</div>
</div>