SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): project recorded results into Rerun

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-07-19 17:39:13 +03:00
parent 31fc4f6567
commit 2b53168149
13 changed files with 1328 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/control-station/src/components/RerunViewport.tsx
+157 -1
View File
@@ -4,6 +4,7 @@ import type { SceneSettings } from "../sceneSettings";
import type { RecordedAdmissionPhase } from "../core/observation/recordedSessionAdmission";
export type RerunViewportStatus = "idle" | "loading" | "ready" | "error";
export type RecordedRerunView = "spatial" | "perception" | "metrics";
export interface RerunSelection {
entityPath: string;
@@ -55,6 +56,8 @@ export interface RerunViewportProps {
| "palette"
| "customColor"
>;
recordedView?: RecordedRerunView;
onPerceptionAvailabilityChange?: (available: boolean) => void;
}
export interface RecordedRrdArtifactDescriptor {
@@ -80,7 +83,9 @@ interface RecordedRerunIdentity {
const RECORDED_RRD_PATH = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/recording\.rrd$/;
const RECORDED_BLUEPRINT_PATH = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/blueprint\.rrd$/;
const RECORDED_PERCEPTION_PATH = /^\/api\/v1\/observation-sessions\/[A-Za-z0-9][A-Za-z0-9._:-]{0,127}\/perception\.rrd$/;
const MAX_BLUEPRINT_BYTES = 1_048_576;
const MAX_PERCEPTION_BYTES = 512 * 1024 * 1024;
const BUFFER_END_TOLERANCE_NS = 1_000_000;
const RECORDED_OPEN_MIN_TIMEOUT_MS = 120_000;
const RECORDED_OPEN_MAX_TIMEOUT_MS = 1_800_000;
@@ -405,6 +410,17 @@ export function resolveRecordedBlueprintUrl(sourceUrl: string, origin: string):
return endpoint.origin === base.origin ? endpoint.href : null;
}
export function resolveRecordedPerceptionUrl(sourceUrl: string, origin: string): string | null {
const normalized = sourceUrl.trim();
if (!RECORDED_RRD_PATH.test(normalized)) return null;
const base = new URL(origin);
const endpoint = new URL(
normalized.replace(/\/recording\.rrd$/, "/perception.rrd"),
`${base.origin}/`,
);
return endpoint.origin === base.origin ? endpoint.href : null;
}
export async function fetchRecordedBlueprintRrd(
endpointUrl: string,
settings: Pick<
@@ -421,10 +437,12 @@ export async function fetchRecordedBlueprintRrd(
{
origin,
signal,
activeView = "spatial",
fetcher = globalThis.fetch,
}: {
origin: string;
signal?: AbortSignal;
activeView?: RecordedRerunView;
fetcher?: typeof globalThis.fetch;
},
): Promise<Uint8Array> {
@@ -443,6 +461,7 @@ export async function fetchRecordedBlueprintRrd(
settings.pointSize > 32 ||
!["turbo", "viridis", "plasma", "grayscale", "custom"].includes(settings.palette) ||
!/^#[0-9A-Fa-f]{6}$/.test(settings.customColor) ||
!["spatial", "perception", "metrics"].includes(activeView) ||
identity.applicationId !== "nodedc_mission_core_recorded" ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(identity.recordingId)
) {
@@ -465,6 +484,7 @@ export async function fetchRecordedBlueprintRrd(
point_size: settings.pointSize,
palette: settings.palette,
custom_color: settings.customColor,
active_view: activeView,
}),
signal,
});
@@ -491,6 +511,69 @@ export async function fetchRecordedBlueprintRrd(
return payload;
}
export async function fetchRecordedPerceptionRrd(
endpointUrl: string,
identity: RecordedRerunIdentity,
{
origin,
signal,
fetcher = globalThis.fetch,
}: {
origin: string;
signal?: AbortSignal;
fetcher?: typeof globalThis.fetch;
},
): Promise<Uint8Array | null> {
const base = new URL(origin);
const endpoint = new URL(endpointUrl, base.origin);
if (
endpoint.origin !== base.origin ||
endpoint.search ||
endpoint.hash ||
!RECORDED_PERCEPTION_PATH.test(endpoint.pathname) ||
identity.applicationId !== "nodedc_mission_core_recorded" ||
!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(identity.recordingId)
) {
throw new Error("Unsafe recorded perception request");
}
const response = await fetcher(endpoint.href, {
method: "POST",
credentials: "same-origin",
headers: {
Accept: "application/vnd.rerun.rrd",
"Content-Type": "application/json",
},
body: JSON.stringify({
application_id: identity.applicationId,
recording_id: identity.recordingId,
}),
signal,
});
if (response.status === 204) return null;
const contentType = response.headers.get("Content-Type")?.split(";", 1)[0].trim();
const declaredLength = Number(response.headers.get("Content-Length"));
if (
!response.ok ||
contentType !== "application/vnd.rerun.rrd" ||
!Number.isSafeInteger(declaredLength) ||
declaredLength < 4 ||
declaredLength > MAX_PERCEPTION_BYTES
) {
throw new Error("Invalid recorded perception response");
}
const payload = new Uint8Array(await response.arrayBuffer());
if (
payload.byteLength !== declaredLength ||
payload[0] !== 0x52 ||
payload[1] !== 0x52 ||
payload[2] !== 0x46 ||
payload[3] !== 0x32
) {
throw new Error("Invalid recorded perception RRD");
}
return payload;
}
export function RerunViewport({
sourceUrl,
recordedArtifact = null,
@@ -504,19 +587,26 @@ export function RerunViewport({
onPlaybackChange,
onPlaybackControllerChange,
sceneSettings,
recordedView = "spatial",
onPerceptionAvailabilityChange,
}: RerunViewportProps) {
const hostRef = useRef<HTMLDivElement>(null);
const [status, setStatus] = useState<RerunViewportStatus>(sourceUrl ? "loading" : "idle");
const [recordingBufferProgress, setRecordingBufferProgress] = useState<number | null>(null);
const [retryNonce, setRetryNonce] = useState(0);
const blueprintChannelRef = useRef<RerunBlueprintChannel | null>(null);
const perceptionChannelRef = useRef<RerunBlueprintChannel | null>(null);
const recordedIdentityRef = useRef<RecordedRerunIdentity | null>(null);
const presentationGateRef = useRef(presentationGate);
presentationGateRef.current = presentationGate;
const [blueprintChannelRevision, setBlueprintChannelRevision] = useState(0);
const [perceptionChannelRevision, setPerceptionChannelRevision] = useState(0);
const recordedBlueprintUrl = sourceUrl
? resolveRecordedBlueprintUrl(sourceUrl, window.location.origin)
: null;
const recordedPerceptionUrl = sourceUrl
? resolveRecordedPerceptionUrl(sourceUrl, window.location.origin)
: null;
const presentationStatus = rerunPresentationStatus(
status,
presentationGate,
@@ -580,6 +670,7 @@ export function RerunViewport({
let playbackState: RerunPlaybackState | null = null;
const recordedAutoplay = createRecordedAutoplayGate();
let blueprintChannel: RerunBlueprintChannel | null = null;
let perceptionChannel: RerunBlueprintChannel | null = null;
const unsubscribers: Array<() => void> = [];
const unsubscribeAll = () => {
while (unsubscribers.length > 0) {
@@ -665,12 +756,20 @@ export function RerunViewport({
if (blueprintChannelRef.current === blueprintChannel) {
blueprintChannelRef.current = null;
}
if (perceptionChannelRef.current === perceptionChannel) {
perceptionChannelRef.current = null;
}
recordedIdentityRef.current = null;
try {
blueprintChannel?.channel.close();
} catch {
// The viewer may already have closed all auxiliary channels.
}
try {
perceptionChannel?.channel.close();
} catch {
// The viewer may already have closed all auxiliary channels.
}
try {
if (viewer.ready) viewer.close(resolvedSource);
} catch {
@@ -700,7 +799,11 @@ export function RerunViewport({
unsubscribers.push(
viewer.on("recording_open", (event) => {
if (disposed || recordingOpened) return;
if (
disposed ||
recordingOpened ||
(isRecordedSource && event.application_id !== "nodedc_mission_core_recorded")
) return;
recordingOpened = true;
if (
recordedBlueprintUrl &&
@@ -717,6 +820,7 @@ export function RerunViewport({
recordingId: event.recording_id,
};
setBlueprintChannelRevision((revision) => revision + 1);
setPerceptionChannelRevision((revision) => revision + 1);
}
}
if (!isRecordedSource) clearLiveRecordingOpenTimer();
@@ -961,6 +1065,12 @@ export function RerunViewport({
blueprintChannelRef.current = blueprintChannel;
setBlueprintChannelRevision((revision) => revision + 1);
}
if (recordedPerceptionUrl) {
const channel = viewer.open_channel("missioncore/recorded-perception");
perceptionChannel = { endpointUrl: recordedPerceptionUrl, channel };
perceptionChannelRef.current = perceptionChannel;
setPerceptionChannelRevision((revision) => revision + 1);
}
if (!recordingOpened && !isRecordedSource) {
recordingOpenTimer = window.setTimeout(() => {
@@ -1012,6 +1122,50 @@ export function RerunViewport({
sourceUrl,
]);
useEffect(() => {
onPerceptionAvailabilityChange?.(false);
}, [onPerceptionAvailabilityChange, recordedPerceptionUrl]);
useEffect(() => {
if (!recordedPerceptionUrl) return;
const active = perceptionChannelRef.current;
const identity = recordedIdentityRef.current;
if (
!active ||
!identity ||
active.endpointUrl !== recordedPerceptionUrl ||
!active.channel.ready
) return;
const abort = new AbortController();
void fetchRecordedPerceptionRrd(recordedPerceptionUrl, identity, {
origin: window.location.origin,
signal: abort.signal,
}).then((payload) => {
if (payload === null) {
onPerceptionAvailabilityChange?.(false);
return;
}
if (
abort.signal.aborted ||
perceptionChannelRef.current !== active ||
recordedIdentityRef.current !== identity ||
!active.channel.ready
) {
return;
}
active.channel.send_rrd(payload);
onPerceptionAvailabilityChange?.(true);
}).catch(() => {
onPerceptionAvailabilityChange?.(false);
// The base recording remains available when no admitted perception layer exists.
});
return () => abort.abort();
}, [
onPerceptionAvailabilityChange,
perceptionChannelRevision,
recordedPerceptionUrl,
]);
useEffect(() => {
if (!recordedBlueprintUrl || !sceneSettings) return;
const active = blueprintChannelRef.current;
@@ -1026,6 +1180,7 @@ export function RerunViewport({
void fetchRecordedBlueprintRrd(recordedBlueprintUrl, sceneSettings, identity, {
origin: window.location.origin,
signal: abort.signal,
activeView: recordedView,
}).then((payload) => {
if (
abort.signal.aborted ||
@@ -1044,6 +1199,7 @@ export function RerunViewport({
}, [
blueprintChannelRevision,
recordedBlueprintUrl,
recordedView,
sceneSettings?.accumulationSeconds,
sceneSettings?.customColor,
sceneSettings?.palette,
apps/control-station/src/workspaces/Workspaces.tsx
+28 -1
View File
@@ -45,6 +45,7 @@ import {
type RerunPlaybackState,
type RerunSelection,
type RerunViewportStatus,
type RecordedRerunView,
} from "../components/RerunViewport";
import {
capabilityStatusLabel,
@@ -294,6 +295,8 @@ function SpatialWorkspace({
const [selection, setSelection] = useState<RerunSelection | null>(null);
const [playbackState, setPlaybackState] = useState<RerunPlaybackState | null>(null);
const [playbackController, setPlaybackController] = useState<RerunPlaybackController | null>(null);
const [recordedRerunView, setRecordedRerunView] = useState<RecordedRerunView>("spatial");
const [perceptionAvailable, setPerceptionAvailable] = useState(false);
const recordedSource = state?.sourceMode === "replay" || /\.rrd(?:$|[?#])/i.test(sourceUrl);
const recordedSessionGate: RecordedAdmissionPhase = recordedSource
? recordedSessionAdmission?.phase ?? "loading"
@@ -369,6 +372,15 @@ function SpatialWorkspace({
(next: RerunPlaybackController | null) => setPlaybackController(next),
[],
);
const onPerceptionAvailabilityChange = useCallback((available: boolean) => {
setPerceptionAvailable(available);
if (!available) setRecordedRerunView("spatial");
}, []);
useEffect(() => {
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
}, [sourceUrl]);
useEffect(() => {
if (pointCloudVisible && sourceUrl.trim()) return;
@@ -377,6 +389,8 @@ function SpatialWorkspace({
setSelection(null);
setPlaybackState(null);
setPlaybackController(null);
setPerceptionAvailable(false);
setRecordedRerunView("spatial");
}, [pointCloudVisible, sourceUrl]);
useEffect(() => {
@@ -419,6 +433,17 @@ function SpatialWorkspace({
<span className="section-eyebrow">СЦЕНА 3D · RERUN</span>
</div>
<div className="spatial-toolbar__actions">
{recordedSource && perceptionAvailable ? (
<Button
size="compact"
variant={recordedRerunView === "perception" ? "primary" : "secondary"}
icon={<Icon name={recordedRerunView === "perception" ? "globe" : "image"} />}
onClick={() => setRecordedRerunView((current) =>
current === "perception" ? "spatial" : "perception")}
>
{recordedRerunView === "perception" ? "Облако точек" : "Распознавание"}
</Button>
) : null}
<Button size="compact" variant="secondary" icon={<Icon name="network" />} onClick={navigation.openSource}>
Движок
</Button>
@@ -451,6 +476,8 @@ function SpatialWorkspace({
? state?.observationTimeline?.range?.endSeconds
: undefined}
sceneSettings={sceneSettings}
recordedView={recordedRerunView}
onPerceptionAvailabilityChange={onPerceptionAvailabilityChange}
onStatusChange={onStatusChange}
onSelectionChange={onSelectionChange}
onPlaybackChange={onPlaybackChange}
@@ -630,7 +657,7 @@ function SpatialWorkspace({
<span><i data-state="ready" />Траектория</span>
<span><i data-state="ready" />Преобразования</span>
<span><i data-state="contract" />Камеры в 3D</span>
<span><i data-state="contract" />Объекты / маски</span>
<span><i data-state={perceptionAvailable ? "ready" : "contract"} />Объекты / рамки</span>
<span><i data-state="contract" />Компоновка</span>
</div>
</div>
apps/control-station/test/observationSources.test.mjs
+49
View File
@@ -23,6 +23,8 @@ let formatAccumulationDuration;
let resolveRerunSourceUrl;
let resolveRecordedBlueprintUrl;
let fetchRecordedBlueprintRrd;
let resolveRecordedPerceptionUrl;
let fetchRecordedPerceptionRrd;
let isRecordedPlaybackFullyBuffered;
let recordedObservationSources;
let selectRecordedMediaEpoch;
@@ -62,6 +64,8 @@ before(async () => {
resolveRerunSourceUrl,
resolveRecordedBlueprintUrl,
fetchRecordedBlueprintRrd,
resolveRecordedPerceptionUrl,
fetchRecordedPerceptionRrd,
isRecordedPlaybackFullyBuffered,
} = await server.ssrLoadModule(
"/src/components/RerunViewport.tsx",
@@ -265,6 +269,7 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
{ applicationId: "nodedc_mission_core_recorded", recordingId: "recording-001" },
{
origin: "http://127.0.0.1:5174",
activeView: "perception",
fetcher: async (input, init) => {
calls.push({ input: String(input), init, body: JSON.parse(String(init.body)) });
return new Response(payload, {
@@ -287,6 +292,7 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
point_size: 4.5,
palette: "custom",
custom_color: "#35d7c1",
active_view: "perception",
});
await assert.rejects(
@@ -308,6 +314,49 @@ test("recorded blueprint fetch is bounded, strict and sends only display setting
);
});
test("recorded perception fetch admits one complete same-origin RRD or no layer", async () => {
const endpoint = resolveRecordedPerceptionUrl(
"/api/v1/observation-sessions/session-1/recording.rrd",
"http://127.0.0.1:5174",
);
assert.equal(
endpoint,
"http://127.0.0.1:5174/api/v1/observation-sessions/session-1/perception.rrd",
);
const payload = Uint8Array.from([0x52, 0x52, 0x46, 0x32, 0x01]);
const result = await fetchRecordedPerceptionRrd(
endpoint,
{ applicationId: "nodedc_mission_core_recorded", recordingId: "recording-001" },
{
origin: "http://127.0.0.1:5174",
fetcher: async (_input, init) => {
assert.deepEqual(JSON.parse(String(init.body)), {
application_id: "nodedc_mission_core_recorded",
recording_id: "recording-001",
});
return new Response(payload, {
status: 200,
headers: {
"Content-Type": "application/vnd.rerun.rrd",
"Content-Length": String(payload.byteLength),
},
});
},
},
);
assert.deepEqual([...result], [...payload]);
const absent = await fetchRecordedPerceptionRrd(
endpoint,
{ applicationId: "nodedc_mission_core_recorded", recordingId: "recording-001" },
{
origin: "http://127.0.0.1:5174",
fetcher: async () => new Response(null, { status: 204 }),
},
);
assert.equal(absent, null);
});
test("recorded replay creates an isolated source catalog without live device bindings", () => {
const sources = recordedObservationSources({
kind: "rerun-recording",