From 9df9f58ab82376bad04d5db472568d19dec341bb Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 5 Aug 2026 23:17:48 +0300 Subject: [PATCH] feat(lab): add recorded realtime spatial playback --- .../src/components/ObservationTimeline.tsx | 20 +- .../src/components/RecordedFmp4Player.tsx | 16 +- .../laboratory/LaboratoryEvidenceViewer.tsx | 7 + .../LaboratoryMetricEvidenceScene.tsx | 56 ++- .../laboratory/RecordedEvidenceVideoScene.tsx | 6 +- .../laboratory/useRecordedEvidencePlayback.ts | 112 +++++ .../src/core/laboratory/m4ReplayThreat.ts | 301 ++++++++++--- .../styles/l3-pointpillars-visual-audit.css | 9 + .../control-station/src/styles/laboratory.css | 8 + .../src/styles/m4-replay-threat.css | 2 +- .../src/styles/observation.css | 5 + .../laboratory/M4ReplayThreatResult.tsx | 2 +- .../laboratory/M4ReplayThreatVisual.tsx | 413 +++++++----------- .../laboratory/useM4ThreatTimeline.ts | 130 ++++++ .../test/m4ReplayThreat.test.mjs | 125 +++++- ...E_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md | 34 +- src/k1link/perception/geometry.py | 19 + src/k1link/perception/spatial_evidence.py | 118 +++++ src/k1link/perception/threat_replay.py | 55 +-- src/k1link/perception/threat_timeline.py | 284 ++++++++++++ src/k1link/web/app.py | 1 + src/k1link/web/m4_threat_replay_api.py | 185 ++++---- tests/test_m4_threat_replay_result.py | 52 ++- 23 files changed, 1443 insertions(+), 517 deletions(-) create mode 100644 apps/control-station/src/components/laboratory/useRecordedEvidencePlayback.ts create mode 100644 apps/control-station/src/workspaces/laboratory/useM4ThreatTimeline.ts create mode 100644 src/k1link/perception/spatial_evidence.py create mode 100644 src/k1link/perception/threat_timeline.py diff --git a/apps/control-station/src/components/ObservationTimeline.tsx b/apps/control-station/src/components/ObservationTimeline.tsx index a030d87..d365337 100644 --- a/apps/control-station/src/components/ObservationTimeline.tsx +++ b/apps/control-station/src/components/ObservationTimeline.tsx @@ -1,4 +1,4 @@ -import { Button, Icon } from "@nodedc/ui-react"; +import { Button, Icon, Select } from "@nodedc/ui-react"; import type { ObservationTimelineMode } from "../core/runtime/contracts"; @@ -14,6 +14,8 @@ export function ObservationTimeline({ onSeek, onPlayingChange, onJumpToEnd, + playbackRate, + onPlaybackRateChange, accumulationSeconds, onAccumulationChange, onAccumulationCommit, @@ -30,6 +32,8 @@ export function ObservationTimeline({ onSeek?: (timeNs: number) => void; onPlayingChange?: (playing: boolean) => void; onJumpToEnd?: () => void; + playbackRate?: number; + onPlaybackRateChange?: (rate: number) => void; accumulationSeconds?: number; onAccumulationChange?: (value: number) => void; onAccumulationCommit?: () => void; @@ -102,6 +106,20 @@ export function ObservationTimeline({ > {buffered ? (playing ? "Пауза" : "Воспроизвести") : "Только эфир"} + {buffered && playbackRate !== undefined && onPlaybackRateChange ? ( + (null); const [bufferRevision, setBufferRevision] = useState(0); const currentSeconds = playback?.currentSeconds ?? contract?.timelineStartSeconds ?? 0; + const playbackRate = playback?.rate && Number.isFinite(playback.rate) + ? Math.min(4, Math.max(0.25, playback.rate)) + : 1; const epoch = useMemo( () => selectRecordedMediaEpoch(archive?.manifest.epochs ?? [], currentSeconds), [archive?.manifest.epochs, currentSeconds], @@ -460,12 +464,21 @@ export function RecordedFmp4Player({ return; } } + video.playbackRate = playbackRate; if (playback?.playing) { void video.play().catch(() => undefined); } else { video.pause(); } - }, [archive?.byteLength, bufferRevision, currentSeconds, epoch, playback?.playing, visualState]); + }, [ + archive?.byteLength, + bufferRevision, + currentSeconds, + epoch, + playback?.playing, + playbackRate, + visualState, + ]); useEffect(() => { const video = videoRef.current; @@ -475,6 +488,7 @@ export function RecordedFmp4Player({ onPlaybackChangeRef.current?.({ currentSeconds: epoch.timelineStartSeconds + video.currentTime, playing: !video.paused && !video.ended, + rate: video.playbackRate, }); }; const scheduleVideoFrame = () => { diff --git a/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx b/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx index 1d18e2e..402cb2c 100644 --- a/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx +++ b/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx @@ -29,6 +29,7 @@ export function LaboratoryEvidenceViewer< actions, secondaryMode, overlay, + transport, children, }: { label: string; @@ -46,6 +47,7 @@ export function LaboratoryEvidenceViewer< onChange: (mode: U) => void; }; overlay?: ReactNode; + transport?: ReactNode; children: ReactNode; }) { const expandButtonRef = useRef(null); @@ -75,6 +77,11 @@ export function LaboratoryEvidenceViewer< {children} {overlay} + {transport ? ( +
+ {transport} +
+ ) : null}
{actions} {secondaryMode ? ( diff --git a/apps/control-station/src/components/laboratory/LaboratoryMetricEvidenceScene.tsx b/apps/control-station/src/components/laboratory/LaboratoryMetricEvidenceScene.tsx index a9dd2cb..518349f 100644 --- a/apps/control-station/src/components/laboratory/LaboratoryMetricEvidenceScene.tsx +++ b/apps/control-station/src/components/laboratory/LaboratoryMetricEvidenceScene.tsx @@ -53,6 +53,15 @@ function disposeRenderable(object: THREE.Object3D): void { materials.forEach((material) => material.dispose()); } +function clearGroup(group: THREE.Group): void { + while (group.children.length) { + const child = group.children[0]; + if (!child) break; + group.remove(child); + child.traverse(disposeRenderable); + } +} + function scenePoint(point: LaboratoryMetricPoint3): LaboratoryMetricPoint3 { return [point[0], point[2], -point[1]]; } @@ -101,7 +110,8 @@ export function LaboratoryMetricEvidenceScene({ const sceneRef = useRef(null); const cameraRef = useRef(null); const controlsRef = useRef(null); - const contentRef = useRef(null); + const staticContentRef = useRef(null); + const dynamicContentRef = useRef(null); const [renderError, setRenderError] = useState(null); const [showCurrentIncrement, setShowCurrentIncrement] = useState(true); const [showRollingMap, setShowRollingMap] = useState(true); @@ -137,12 +147,14 @@ export function LaboratoryMetricEvidenceScene({ controls.screenSpacePanning = true; controls.minDistance = 0.4; controls.maxDistance = 80; - const content = new THREE.Group(); - scene.add(content); + const staticContent = new THREE.Group(); + const dynamicContent = new THREE.Group(); + scene.add(staticContent, dynamicContent); sceneRef.current = scene; cameraRef.current = camera; controlsRef.current = controls; - contentRef.current = content; + staticContentRef.current = staticContent; + dynamicContentRef.current = dynamicContent; const resize = () => { const width = Math.max(host.clientWidth, 1); @@ -172,20 +184,16 @@ export function LaboratoryMetricEvidenceScene({ sceneRef.current = null; cameraRef.current = null; controlsRef.current = null; - contentRef.current = null; + staticContentRef.current = null; + dynamicContentRef.current = null; }; }, [label]); useEffect(() => { const host = hostRef.current; - const content = contentRef.current; + const content = dynamicContentRef.current; if (!host || !content) return; - while (content.children.length) { - const child = content.children[0]; - if (!child) break; - content.remove(child); - child.traverse(disposeRenderable); - } + clearGroup(content); if (showCurrentIncrement) { const contextGeometry = new THREE.BufferGeometry(); @@ -244,6 +252,19 @@ export function LaboratoryMetricEvidenceScene({ content.add(centroid); } + }, [ + obstacles, + pointCloudBodyXyzM, + showCurrentIncrement, + showRollingMap, + ]); + + useEffect(() => { + const host = hostRef.current; + const content = staticContentRef.current; + if (!host || !content) return; + clearGroup(content); + const corridorLength = rig.lengthM / 2 + corridor.forwardLengthM + corridor.rearMarginM; const corridorCenterX = (rig.lengthM / 2 + corridor.forwardLengthM - corridor.rearMarginM) / 2; const corridorMesh = new THREE.Mesh( @@ -302,14 +323,7 @@ export function LaboratoryMetricEvidenceScene({ material.depthWrite = false; }); content.add(grid); - }, [ - corridor, - obstacles, - pointCloudBodyXyzM, - rig, - showCurrentIncrement, - showRollingMap, - ]); + }, [corridor, rig]); const resetView = () => { const camera = cameraRef.current; @@ -327,7 +341,7 @@ export function LaboratoryMetricEvidenceScene({ controls.update(); }; - useEffect(resetView, [corridor.forwardLengthM, mode, obstacles]); + useEffect(resetView, [corridor.forwardLengthM, mode]); return (
diff --git a/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx b/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx index e971df6..8f66b1c 100644 --- a/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx +++ b/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx @@ -18,6 +18,7 @@ export function RecordedEvidenceVideoScene({ imageHeight, boxes, ariaLabel, + interactive = true, onPlaybackChange, }: { source: ObservationSourceDescriptor; @@ -26,14 +27,15 @@ export function RecordedEvidenceVideoScene({ imageHeight: number; boxes: readonly RecordedEvidenceBox[]; ariaLabel: string; - onPlaybackChange: (playback: RecordedObservationPlayback) => void; + interactive?: boolean; + onPlaybackChange?: (playback: RecordedObservationPlayback) => void; }) { return (
diff --git a/apps/control-station/src/components/laboratory/useRecordedEvidencePlayback.ts b/apps/control-station/src/components/laboratory/useRecordedEvidencePlayback.ts new file mode 100644 index 0000000..9a38a79 --- /dev/null +++ b/apps/control-station/src/components/laboratory/useRecordedEvidencePlayback.ts @@ -0,0 +1,112 @@ +import { useCallback, useEffect, useMemo, useState } from "react"; + +import type { RecordedObservationPlayback } from "../RecordedFmp4Player"; + +export interface RecordedEvidencePlaybackRange { + startSeconds: number; + endSeconds: number; +} + +function validRange( + range: RecordedEvidencePlaybackRange | null, +): range is RecordedEvidencePlaybackRange { + return Boolean( + range + && Number.isFinite(range.startSeconds) + && Number.isFinite(range.endSeconds) + && range.endSeconds > range.startSeconds, + ); +} + +export function clampRecordedEvidenceSeconds( + seconds: number, + range: RecordedEvidencePlaybackRange, +): number { + return Math.min(range.endSeconds, Math.max(range.startSeconds, seconds)); +} + +export function advanceRecordedEvidencePlayback( + playback: RecordedObservationPlayback, + elapsedSeconds: number, + range: RecordedEvidencePlaybackRange, +): RecordedObservationPlayback { + if (!playback.playing || !Number.isFinite(elapsedSeconds) || elapsedSeconds <= 0) { + return playback; + } + const next = playback.currentSeconds + elapsedSeconds * (playback.rate ?? 1); + if (next >= range.endSeconds) { + return { ...playback, currentSeconds: range.endSeconds, playing: false }; + } + return { ...playback, currentSeconds: clampRecordedEvidenceSeconds(next, range) }; +} + +export function useRecordedEvidencePlayback( + range: RecordedEvidencePlaybackRange | null, +) { + const [playback, setPlayback] = useState({ + currentSeconds: 0, + playing: false, + rate: 1, + }); + + useEffect(() => { + if (!validRange(range)) return; + setPlayback((current) => ({ + ...current, + currentSeconds: current.currentSeconds === 0 + ? range.startSeconds + : clampRecordedEvidenceSeconds(current.currentSeconds, range), + })); + }, [range]); + + useEffect(() => { + if (!validRange(range) || !playback.playing) return; + let animationFrame = 0; + let previous = performance.now(); + const tick = (now: number) => { + const elapsed = Math.max(0, now - previous); + if (elapsed >= 32) { + previous = now; + setPlayback((current) => { + if (!current.playing) return current; + return advanceRecordedEvidencePlayback(current, elapsed / 1_000, range); + }); + } + animationFrame = window.requestAnimationFrame(tick); + }; + animationFrame = window.requestAnimationFrame(tick); + return () => window.cancelAnimationFrame(animationFrame); + }, [playback.playing, range]); + + const seek = useCallback((seconds: number, pause = true) => { + if (!validRange(range)) return; + setPlayback((current) => ({ + ...current, + currentSeconds: clampRecordedEvidenceSeconds(seconds, range), + playing: pause ? false : current.playing, + })); + }, [range]); + + const setPlaying = useCallback((playing: boolean) => { + if (!validRange(range)) return; + setPlayback((current) => ({ + ...current, + currentSeconds: playing && current.currentSeconds >= range.endSeconds + ? range.startSeconds + : current.currentSeconds, + playing, + })); + }, [range]); + + const setRate = useCallback((rate: number) => { + if (![0.5, 1, 2].includes(rate)) return; + setPlayback((current) => ({ ...current, rate })); + }, []); + + return useMemo(() => ({ + playback, + seek, + setPlaying, + setRate, + }), [playback, seek, setPlaying, setRate]); +} diff --git a/apps/control-station/src/core/laboratory/m4ReplayThreat.ts b/apps/control-station/src/core/laboratory/m4ReplayThreat.ts index 3ec6c85..dd2d764 100644 --- a/apps/control-station/src/core/laboratory/m4ReplayThreat.ts +++ b/apps/control-station/src/core/laboratory/m4ReplayThreat.ts @@ -125,22 +125,47 @@ export interface M4ThreatVisualIndexItem { pointCloudSampleCount: number; } -export interface M4ThreatVideoFrame { - frameIndex: number; +export interface M4ThreatTimelineFrame { + sequence: number; + frameId: string; + sourceTimeNs: number; sessionSeconds: number; sourceAvailable: boolean; + spatialAvailable: boolean; + pointCloudBodyXyzM: readonly M4Point3[]; + pointCloudSourceCount: number; + pointCloudSampleCount: number; + pointCloudLayer: "current-increment"; + rollingMapComponentCount: number; + metricObstacles: readonly M4ThreatMetricVisual[]; cameraProposals: readonly M4ThreatCameraProposal[]; decisionCounts: Record; + cameraUrl: string; } -export interface M4ThreatVideoOverlay { +export interface M4ThreatTimeline { resultId: string; recordedSourceSessionId: "20260720T065719Z_viewer_live"; imageWidth: 800; imageHeight: 600; + frameCount: 4489; + frameTimesNs: readonly number[]; timelineStartSeconds: number; timelineEndSeconds: number; - frames: readonly M4ThreatVideoFrame[]; + nominalFrameIntervalSeconds: number; + nominalRateHz: number; + maxChunkFrames: number; + pointSampleLimit: number; + rig: M4ThreatVisualFrame["rig"]; + corridor: M4ThreatVisualFrame["corridor"]; +} + +export interface M4ThreatTimelineChunk { + resultId: string; + startSequence: number; + frameCount: number; + nextSequence: number | null; + frames: readonly M4ThreatTimelineFrame[]; } type LaboratoryFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; @@ -245,6 +270,29 @@ function parseCameraProposal(value: unknown): M4ThreatCameraProposal { }; } +function parseMetricVisual(value: unknown): M4ThreatMetricVisual { + const item = object(value, "M4.6 metric visual"); + const state = text(item.state, "M4.6 temporal state"); + if ( + state !== "current" + && state !== "retained" + && state !== "held" + && state !== "expired" + ) { + throw new M4ThreatContractError("M4.6 temporal state: неизвестное состояние."); + } + return { + componentId: text(item.component_id, "M4.6 visual component"), + state, + motion: motion(item.motion), + centroidBodyXyzM: vector(item.centroid_body_xyz_m, 3, "M4.6 centroid") as [number, number, number], + cellCentersBodyXyzM: array(item.cell_centers_body_xyz_m, "M4.6 cells").map( + (point) => vector(point, 3, "M4.6 cell") as [number, number, number], + ), + assessment: parseAssessment(item.assessment), + }; +} + export async function fetchM4ThreatReplayResult({ fetcher = fetch, signal, @@ -425,28 +473,7 @@ export async function fetchM4ThreatVisual( rollingMapComponentCount: rollingMapV2 ? integer(item.rolling_map_component_count, "M4.6 rolling components") : 0, - metricObstacles: array(item.metric_obstacles, "M4.6 metric visuals").map((raw) => { - const value = object(raw, "M4.6 metric visual"); - const state = text(value.state, "M4.6 temporal state"); - if ( - state !== "current" - && state !== "retained" - && state !== "held" - && state !== "expired" - ) { - throw new M4ThreatContractError("M4.6 temporal state: неизвестное состояние."); - } - return { - componentId: text(value.component_id, "M4.6 visual component"), - state, - motion: motion(value.motion), - centroidBodyXyzM: vector(value.centroid_body_xyz_m, 3, "M4.6 centroid") as [number, number, number], - cellCentersBodyXyzM: array(value.cell_centers_body_xyz_m, "M4.6 cells").map( - (point) => vector(point, 3, "M4.6 cell") as [number, number, number], - ), - assessment: parseAssessment(value.assessment), - }; - }), + metricObstacles: array(item.metric_obstacles, "M4.6 metric visuals").map(parseMetricVisual), cameraProposals: array(item.camera_proposals, "M4.6 camera proposals").map(parseCameraProposal), rig: { lengthM: number(rig.length_m, "M4.6 rig length"), @@ -462,71 +489,211 @@ export async function fetchM4ThreatVisual( }; } -export async function fetchM4ThreatVideoOverlay( +export async function fetchM4ThreatTimeline( result: string, { fetcher = fetch, signal }: { fetcher?: LaboratoryFetch; signal?: AbortSignal } = {}, -): Promise { +): Promise { const response = await fetcher( - `/api/v1/laboratory/m4-threat/results/${result}/video-overlay`, + `/api/v1/laboratory/m4-threat/results/${result}/timeline`, { headers: { Accept: "application/json" }, signal }, ); - if (!response.ok) throw new M4ThreatContractError(`M4.6 video overlay: HTTP ${response.status}.`); - const payload = object(await response.json(), "M4.6 video overlay"); - exact(payload.schema_version, "missioncore.m4-threat-video-overlay/v1", "M4.6 video schema"); - exact(payload.result_id, result, "M4.6 video result"); - exact(payload.authority, "replay-simulated", "M4.6 video authority"); + if (!response.ok) throw new M4ThreatContractError(`M4.6 timeline: HTTP ${response.status}.`); + const payload = object(await response.json(), "M4.6 timeline"); + exact( + payload.schema_version, + "missioncore.recorded-spatial-evidence-timeline/v1", + "M4.6 timeline schema", + ); + exact(payload.result_id, result, "M4.6 timeline result"); + exact(payload.authority, "replay-simulated", "M4.6 timeline authority"); const recorded = object(payload.recorded_source, "M4.6 recorded source"); exact( recorded.session_id, "20260720T065719Z_viewer_live", "M4.6 recorded session", ); - const frames = array(payload.frames, "M4.6 video frames").map((raw, expectedIndex) => { - const item = object(raw, "M4.6 video frame"); - const frameIndex = integer(item.frame_index, "M4.6 video index"); - if (frameIndex !== expectedIndex) throw new M4ThreatContractError("M4.6 video order."); - const counts = object(item.decision_counts, "M4.6 video decisions"); - return { - frameIndex, - sessionSeconds: number(item.session_seconds, "M4.6 video time"), - sourceAvailable: typeof item.source_available === "boolean" ? item.source_available : false, - cameraProposals: array(item.camera_proposals, "M4.6 video proposals").map(parseCameraProposal), - decisionCounts: { - threat: integer(counts.threat, "M4.6 video threat"), - "not-threat": integer(counts["not-threat"], "M4.6 video clear"), - unknown: integer(counts.unknown, "M4.6 video unknown"), - }, - }; - }); - exact(payload.frame_count, 4489, "M4.6 video frame count"); + exact(recorded.source_id, "RAVNOVES00", "M4.6 recorded source id"); + exact( + recorded.synchronization, + "host-arrival-best-effort", + "M4.6 recorded synchronization", + ); + const frameCount = exact(payload.frame_count, 4489, "M4.6 timeline frame count"); + const frameTimesNs = array(payload.frame_times_ns, "M4.6 timeline index").map( + (value) => integer(value, "M4.6 timeline time"), + ); + if ( + frameTimesNs.length !== frameCount + || frameTimesNs.some((value, index) => index > 0 && value <= (frameTimesNs[index - 1] ?? value)) + ) { + throw new M4ThreatContractError("M4.6 timeline index: нарушен порядок."); + } + const rig = object(payload.rig, "M4.6 timeline rig"); + const corridor = object(payload.corridor, "M4.6 timeline corridor"); return { resultId: result, recordedSourceSessionId: "20260720T065719Z_viewer_live", imageWidth: exact(payload.image_width, 800, "M4.6 image width"), imageHeight: exact(payload.image_height, 600, "M4.6 image height"), - timelineStartSeconds: number(payload.timeline_start_seconds, "M4.6 video start"), - timelineEndSeconds: number(payload.timeline_end_seconds, "M4.6 video end"), + frameCount, + frameTimesNs, + timelineStartSeconds: number(payload.timeline_start_seconds, "M4.6 timeline start"), + timelineEndSeconds: number(payload.timeline_end_seconds, "M4.6 timeline end"), + nominalFrameIntervalSeconds: number( + payload.nominal_frame_interval_seconds, + "M4.6 timeline interval", + ), + nominalRateHz: number(payload.nominal_rate_hz, "M4.6 timeline rate"), + maxChunkFrames: integer(payload.max_chunk_frames, "M4.6 max chunk"), + pointSampleLimit: integer(payload.point_sample_limit, "M4.6 point limit"), + rig: { + lengthM: number(rig.length_m, "M4.6 rig length"), + widthM: number(rig.width_m, "M4.6 rig width"), + nominalSensorHeightM: number(rig.nominal_sensor_height_m, "M4.6 sensor height"), + }, + corridor: { + forwardLengthM: number(corridor.forward_length_m, "M4.6 forward corridor"), + rearMarginM: number(corridor.rear_margin_m, "M4.6 rear corridor"), + halfWidthM: number(corridor.half_width_m, "M4.6 half width"), + predictionHorizonSeconds: number( + corridor.prediction_horizon_seconds, + "M4.6 prediction horizon", + ), + }, + }; +} + +export async function fetchM4ThreatTimelineChunk( + result: string, + startSequence: number, + frameCount: number, + { fetcher = fetch, signal }: { fetcher?: LaboratoryFetch; signal?: AbortSignal } = {}, +): Promise { + const params = new URLSearchParams({ + start: String(startSequence), + count: String(frameCount), + }); + const response = await fetcher( + `/api/v1/laboratory/m4-threat/results/${result}/timeline/chunk?${params}`, + { headers: { Accept: "application/json" }, signal }, + ); + if (!response.ok) throw new M4ThreatContractError(`M4.6 timeline chunk: HTTP ${response.status}.`); + const payload = object(await response.json(), "M4.6 timeline chunk"); + exact( + payload.schema_version, + "missioncore.recorded-spatial-evidence-chunk/v1", + "M4.6 timeline chunk schema", + ); + exact(payload.result_id, result, "M4.6 timeline chunk result"); + exact(payload.authority, "replay-simulated", "M4.6 timeline chunk authority"); + const parsedStart = integer(payload.start_sequence, "M4.6 timeline chunk start"); + if (parsedStart !== startSequence) { + throw new M4ThreatContractError("M4.6 timeline chunk start: нарушен контракт."); + } + const frames = array(payload.frames, "M4.6 timeline frames").map((raw, offset) => + parseTimelineFrame(raw, result, parsedStart + offset)); + const parsedCount = integer(payload.frame_count, "M4.6 timeline chunk count"); + if (parsedCount !== frames.length || parsedCount > frameCount) { + throw new M4ThreatContractError("M4.6 timeline chunk count: нарушен контракт."); + } + return { + resultId: result, + startSequence: parsedStart, + frameCount: parsedCount, + nextSequence: payload.next_sequence === null + ? null + : integer(payload.next_sequence, "M4.6 timeline next sequence"), frames, }; } -export function selectM4ThreatVideoFrame( - frames: readonly M4ThreatVideoFrame[], +function parseTimelineFrame( + value: unknown, + result: string, + expectedSequence: number, +): M4ThreatTimelineFrame { + const item = object(value, "M4.6 timeline frame"); + exact( + item.schema_version, + "missioncore.recorded-spatial-evidence-frame/v1", + "M4.6 timeline frame schema", + ); + exact(item.authority, "replay-simulated", "M4.6 timeline frame authority"); + const sequence = integer(item.sequence, "M4.6 timeline sequence"); + if (sequence !== expectedSequence) { + throw new M4ThreatContractError("M4.6 timeline frame order: нарушен контракт."); + } + const counts = object(item.decision_counts, "M4.6 timeline decisions"); + const cameraUrl = text(item.camera_url, "M4.6 timeline camera URL"); + if (!cameraUrl.includes(`/results/${result}/timeline/frames/${sequence}/camera`)) { + throw new M4ThreatContractError("M4.6 timeline camera URL: нарушена идентичность."); + } + return { + sequence, + frameId: text(item.frame_id, "M4.6 timeline frame id"), + sourceTimeNs: integer(item.source_time_ns, "M4.6 timeline source time"), + sessionSeconds: number(item.session_seconds, "M4.6 timeline time"), + sourceAvailable: typeof item.source_available === "boolean" && item.source_available, + spatialAvailable: typeof item.spatial_available === "boolean" && item.spatial_available, + pointCloudBodyXyzM: array(item.point_cloud_body_xyz_m, "M4.6 timeline points").map( + (point) => vector(point, 3, "M4.6 timeline point") as [number, number, number], + ), + pointCloudSourceCount: integer(item.point_cloud_source_count, "M4.6 source points"), + pointCloudSampleCount: integer(item.point_cloud_sample_count, "M4.6 sampled points"), + pointCloudLayer: exact( + item.point_cloud_layer, + "current-increment", + "M4.6 timeline point layer", + ), + rollingMapComponentCount: integer( + item.rolling_map_component_count, + "M4.6 rolling components", + ), + metricObstacles: array(item.metric_obstacles, "M4.6 timeline obstacles").map( + parseMetricVisual, + ), + cameraProposals: array(item.camera_proposals, "M4.6 timeline proposals").map( + parseCameraProposal, + ), + decisionCounts: { + threat: integer(counts.threat, "M4.6 timeline threat"), + "not-threat": integer(counts["not-threat"], "M4.6 timeline clear"), + unknown: integer(counts.unknown, "M4.6 timeline unknown"), + }, + cameraUrl, + }; +} + +export function selectM4ThreatTimelineSequence( + frameTimesNs: readonly number[], seconds: number, -): M4ThreatVideoFrame | null { - if (!frames.length) return null; +): number | null { + if (!frameTimesNs.length || !Number.isFinite(seconds)) return null; + const targetNs = seconds * 1_000_000_000; let low = 0; - let high = frames.length - 1; + let high = frameTimesNs.length - 1; while (low < high) { const middle = Math.floor((low + high) / 2); - const current = frames[middle]; - if (!current || current.sessionSeconds < seconds) low = middle + 1; + const current = frameTimesNs[middle]; + if (current === undefined || current < targetNs) low = middle + 1; else high = middle; } - const current = frames[low] ?? frames[frames.length - 1] ?? null; - const previous = frames[Math.max(0, low - 1)] ?? null; - if (!current || !previous) return current; - return Math.abs(previous.sessionSeconds - seconds) <= Math.abs(current.sessionSeconds - seconds) - ? previous - : current; + const current = frameTimesNs[low]; + const previousIndex = Math.max(0, low - 1); + const previous = frameTimesNs[previousIndex]; + if (current === undefined) return frameTimesNs.length - 1; + if (previous === undefined) return low; + return Math.abs(previous - targetNs) <= Math.abs(current - targetNs) ? previousIndex : low; +} + +export function selectM4ThreatTimelineFrame( + frames: readonly M4ThreatTimelineFrame[], + seconds: number, +): M4ThreatTimelineFrame | null { + if (!frames.length) return null; + const local = selectM4ThreatTimelineSequence( + frames.map((frame) => frame.sourceTimeNs), + seconds, + ); + return local === null ? null : frames[local] ?? null; } diff --git a/apps/control-station/src/styles/l3-pointpillars-visual-audit.css b/apps/control-station/src/styles/l3-pointpillars-visual-audit.css index 223322f..f97fb61 100644 --- a/apps/control-station/src/styles/l3-pointpillars-visual-audit.css +++ b/apps/control-station/src/styles/l3-pointpillars-visual-audit.css @@ -94,6 +94,15 @@ flex-wrap: wrap; } +.m4-replay-threat-evidence-viewer .laboratory-metric-evidence-scene__legend, +.m4-replay-threat-evidence-viewer .m4-replay-threat-visual__overlay { + bottom: 5.3rem; +} + +.m4-replay-threat-visual__timeline { + width: 100%; +} + .e46e-ready-stack-video { width: 100%; height: 100%; diff --git a/apps/control-station/src/styles/laboratory.css b/apps/control-station/src/styles/laboratory.css index d59189d..990d7a3 100644 --- a/apps/control-station/src/styles/laboratory.css +++ b/apps/control-station/src/styles/laboratory.css @@ -460,6 +460,14 @@ gap: 0.45rem; } +.laboratory-evidence-viewer__transport { + position: absolute; + z-index: 4; + right: 0.6rem; + bottom: 0.6rem; + left: 0.6rem; +} + .laboratory-evidence-viewer[data-expanded="true"] { position: fixed; z-index: var(--nodedc-layer-overlay); diff --git a/apps/control-station/src/styles/m4-replay-threat.css b/apps/control-station/src/styles/m4-replay-threat.css index ca55fe9..f4a7360 100644 --- a/apps/control-station/src/styles/m4-replay-threat.css +++ b/apps/control-station/src/styles/m4-replay-threat.css @@ -49,7 +49,7 @@ gap: 0.5rem; } -.laboratory-metric-evidence-scene__toolbar .nodedc-button, +.laboratory-metric-evidence-scene__toolbar .nodedc-button:not([data-variant="primary"]), .laboratory-metric-evidence-scene__toolbar > span, .laboratory-metric-evidence-scene__legend { background: var(--nodedc-floating-surface); diff --git a/apps/control-station/src/styles/observation.css b/apps/control-station/src/styles/observation.css index 5fd1f05..6a1468c 100644 --- a/apps/control-station/src/styles/observation.css +++ b/apps/control-station/src/styles/observation.css @@ -399,6 +399,11 @@ i[data-availability="error"] { gap: 0.6rem; } +.observation-timeline__playback > .nodedc-select-anchor { + width: 7.5rem; + flex: none; +} + .observation-timeline__accumulation { display: grid; min-width: 0; diff --git a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx index bb9f7df..dfc9dba 100644 --- a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx +++ b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx @@ -47,7 +47,7 @@ export function M4ReplayThreatResultView({ }, { label: "Визуал", - value: "4489-frame VIDEO · 32 exact CAMERA/3D/PLAN samples", + value: "4489-frame VIDEO/CAMERA/3D/PLAN · единый recorded clock", }, ]} brief={{ diff --git a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx index f4119f2..bc14f2f 100644 --- a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx +++ b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState } from "react"; -import { Icon, IconButton, Select } from "@nodedc/ui-react"; +import { Icon, IconButton } from "@nodedc/ui-react"; -import type { RecordedObservationPlayback } from "../../components/RecordedFmp4Player"; +import { ObservationTimeline } from "../../components/ObservationTimeline"; import { LaboratoryMetricEvidenceScene, type LaboratoryMetricSceneMode, @@ -12,19 +12,15 @@ import { RecordedEvidenceVideoScene, type RecordedEvidenceBox, } from "../../components/laboratory/RecordedEvidenceVideoScene"; -import { - fetchM4ThreatVideoOverlay, - fetchM4ThreatVisual, - fetchM4ThreatVisualIndex, - selectM4ThreatVideoFrame, - type M4ThreatCameraProposal, - type M4ThreatVideoOverlay, - type M4ThreatVisualFrame, - type M4ThreatVisualIndexItem, -} from "../../core/laboratory/m4ReplayThreat"; +import { useRecordedEvidencePlayback } from "../../components/laboratory/useRecordedEvidencePlayback"; +import type { M4ThreatCameraProposal } from "../../core/laboratory/m4ReplayThreat"; import { recordedObservationSources } from "../../core/observation/recordedObservationSources"; import { replayObservationSession } from "../../core/observation/sessionArchive"; import type { ObservationSourceDescriptor } from "../../core/runtime/contracts"; +import { + useM4ThreatTimelineFrame, + useM4ThreatTimelineMetadata, +} from "./useM4ThreatTimeline"; type M4ThreatViewMode = "video" | "camera" | LaboratoryMetricSceneMode; @@ -55,98 +51,68 @@ function message(error: unknown, fallback: string): string { return error instanceof Error && error.message.trim() ? error.message : fallback; } +function SpatialState({ message: text }: { message: string }) { + return ( +
+ + {text} +
+ ); +} + export function M4ReplayThreatVisual({ resultId }: { resultId: string }) { const [mode, setMode] = useState("video"); const [expanded, setExpanded] = useState(false); - const [index, setIndex] = useState([]); - const [ordinal, setOrdinal] = useState(1); - const [frame, setFrame] = useState(null); - const [sampleLoading, setSampleLoading] = useState(true); - const [sampleError, setSampleError] = useState(null); - const [videoOverlay, setVideoOverlay] = useState(null); + const metadata = useM4ThreatTimelineMetadata(resultId); + const playbackRange = useMemo(() => metadata.timeline ? ({ + startSeconds: metadata.timeline.timelineStartSeconds, + endSeconds: metadata.timeline.timelineEndSeconds, + }) : null, [metadata.timeline]); + const playbackController = useRecordedEvidencePlayback(playbackRange); + const timelineFrame = useM4ThreatTimelineFrame({ + resultId, + timeline: metadata.timeline, + currentSeconds: playbackController.playback.currentSeconds, + }); const [videoSource, setVideoSource] = useState(null); const [videoLoading, setVideoLoading] = useState(false); const [videoError, setVideoError] = useState(null); - const [videoPlayback, setVideoPlayback] = useState({ - currentSeconds: 0, - playing: false, - }); useEffect(() => { - const controller = new AbortController(); - setSampleLoading(true); - setSampleError(null); - void fetchM4ThreatVisualIndex(resultId, { signal: controller.signal }) - .then((items) => { - if (!controller.signal.aborted) setIndex(items); - }) - .catch((caught: unknown) => { - if (!controller.signal.aborted) { - setSampleError(message(caught, "Индекс визуальных кадров M4.6 недоступен.")); - } - }); - return () => controller.abort(); + setVideoSource(null); + setVideoError(null); }, [resultId]); useEffect(() => { - const controller = new AbortController(); - setSampleLoading(true); - setSampleError(null); - setFrame(null); - void fetchM4ThreatVisual(resultId, ordinal, { signal: controller.signal }) - .then((next) => { - if (!controller.signal.aborted) setFrame(next); - }) - .catch((caught: unknown) => { - if (!controller.signal.aborted) { - setSampleError(message(caught, "Метрический visual M4.6 недоступен.")); - } - }) - .finally(() => { - if (!controller.signal.aborted) setSampleLoading(false); - }); - return () => controller.abort(); - }, [ordinal, resultId]); - - useEffect(() => { - if (mode !== "video" || (videoOverlay && videoSource)) return; + const timeline = metadata.timeline; + if (mode !== "video" || !timeline || videoSource) return; const controller = new AbortController(); setVideoLoading(true); setVideoError(null); - void (async () => { - const overlay = await fetchM4ThreatVideoOverlay(resultId, { - signal: controller.signal, - }); - const replay = await replayObservationSession(overlay.recordedSourceSessionId, { - signal: controller.signal, - }); - if (replay.kind !== "ready") { - throw new Error("RIGHT-видео RAVNOVES00 ещё готовится к воспроизведению."); - } - const source = recordedObservationSources(replay.launch).find( - (candidate) => - candidate.modality === "video" && - candidate.semanticChannelId === "camera.video.recorded", - ); - const delivery = source?.delivery?.kind === "recorded-fmp4-manifest" - ? source.delivery - : null; - if ( - !source || - !delivery || - delivery.timelineStartSeconds !== overlay.timelineStartSeconds || - delivery.timelineEndSeconds < overlay.timelineEndSeconds - ) { - throw new Error("RIGHT-видео не совпало с временным контрактом M4.6."); - } - if (controller.signal.aborted) return; - setVideoOverlay(overlay); - setVideoSource(source); - setVideoPlayback({ - currentSeconds: overlay.timelineStartSeconds, - playing: false, - }); - })() + void replayObservationSession(timeline.recordedSourceSessionId, { + signal: controller.signal, + }) + .then((replay) => { + if (replay.kind !== "ready") { + throw new Error("RIGHT-видео RAVNOVES00 ещё готовится к воспроизведению."); + } + const source = recordedObservationSources(replay.launch).find( + (candidate) => candidate.modality === "video" + && candidate.semanticChannelId === "camera.video.recorded", + ); + const delivery = source?.delivery?.kind === "recorded-fmp4-manifest" + ? source.delivery + : null; + if ( + !source + || !delivery + || delivery.timelineStartSeconds !== timeline.timelineStartSeconds + || delivery.timelineEndSeconds < timeline.timelineEndSeconds + ) { + throw new Error("RIGHT-видео не совпало с recorded-realtime timeline M4.6."); + } + if (!controller.signal.aborted) setVideoSource(source); + }) .catch((caught: unknown) => { if (!controller.signal.aborted) { setVideoError(message(caught, "Видео-доказательство M4.6 недоступно.")); @@ -156,22 +122,17 @@ export function M4ReplayThreatVisual({ resultId }: { resultId: string }) { if (!controller.signal.aborted) setVideoLoading(false); }); return () => controller.abort(); - }, [mode, resultId, videoOverlay, videoSource]); + }, [metadata.timeline, mode, videoSource]); - const activeVideoFrame = useMemo( - () => videoOverlay - ? selectM4ThreatVideoFrame(videoOverlay.frames, videoPlayback.currentSeconds) - : null, - [videoOverlay, videoPlayback.currentSeconds], - ); - const activeProposals = mode === "camera" - ? frame?.cameraProposals ?? [] - : activeVideoFrame?.cameraProposals ?? []; - const activeBoxes = useMemo(() => boxes(activeProposals), [activeProposals]); - const selectedItem = index.find((item) => item.ordinal === ordinal) ?? null; - const threatObstacles = frame?.metricObstacles.filter( - (item) => item.assessment.decision === "threat", - ) ?? []; + const frame = timelineFrame.activeFrame; + const activeBoxes = useMemo(() => boxes(frame?.cameraProposals ?? []), [frame]); + const sceneObstacles = useMemo(() => frame?.metricObstacles.map((obstacle) => ({ + id: obstacle.componentId, + decision: obstacle.assessment.decision, + state: obstacle.state, + centroidBodyXyzM: obstacle.centroidBodyXyzM, + cellCentersBodyXyzM: obstacle.cellCentersBodyXyzM, + })) ?? [], [frame]); const currentIncrementObstacles = frame?.metricObstacles.filter( (item) => item.state === "current", ) ?? []; @@ -183,208 +144,151 @@ export function M4ReplayThreatVisual({ resultId }: { resultId: string }) { .filter((value): value is number => value !== null) .sort((left, right) => left - right)[0] ?? null; - const seekVideo = (seconds: number) => { - if (!videoOverlay) return; - setVideoPlayback({ - currentSeconds: Math.min( - videoOverlay.timelineEndSeconds, - Math.max(videoOverlay.timelineStartSeconds, seconds), - ), - playing: false, - }); - }; - const navigate = (offset: -1 | 1) => { - const count = Math.max(index.length, 32); - setOrdinal((current) => ((current - 1 + offset + count) % count) + 1); + const seek = (seconds: number) => playbackController.seek(seconds); + const handleModeChange = (next: M4ThreatViewMode) => { + if (next === "camera") playbackController.setPlaying(false); + setMode(next); }; - const actions = mode === "video" ? ( + const actions = (
- seekVideo(videoPlayback.currentSeconds - 5)}> + seek(playbackController.playback.currentSeconds - 5)} + > - seekVideo(videoPlayback.currentSeconds + 5)}> + seek(playbackController.playback.currentSeconds + 5)} + >
- ({ - value: String(item.ordinal), - label: `${item.ordinal}/32 · frame ${item.sequence} · ${item.metricObstacleCount} metric · ${item.cameraProposalCount} camera`, - }))} - variant="split" - menuWidth="anchor" - searchable - searchPlaceholder="Найти sample" - onChange={(value) => setOrdinal(Number(value))} - />
); - const overlay = mode === "video" && videoOverlay ? ( -
+ const overlay = metadata.timeline && frame ? ( +
- RAVNOVES00 · recorded RIGHT - - +{(videoPlayback.currentSeconds - videoOverlay.timelineStartSeconds).toFixed(1)} с - {activeVideoFrame ? ` · frame ${activeVideoFrame.frameIndex}` : ""} - - {videoPlayback.playing ? "воспроизведение" : "пауза / seek"} + RAVNOVES00 · recorded realtime + frame {frame.sequence + 1}/{metadata.timeline.frameCount} + + +{(frame.sessionSeconds - metadata.timeline.timelineStartSeconds).toFixed(3)} с + · {playbackController.playback.playing ? "воспроизведение" : "пауза / seek"} +
- Camera evidence - {activeVideoFrame?.cameraProposals.length ?? 0} рамок · distance при LiDAR support - пунктир = camera-only · всегда unknown -
-
- Replay decision - - {activeVideoFrame?.decisionCounts.threat ?? 0} threat · {activeVideoFrame?.decisionCounts["not-threat"] ?? 0} clear · {activeVideoFrame?.decisionCounts.unknown ?? 0} unknown - - REPLAY-SIMULATED · не live и не safety authority -
-
- ) : frame ? ( -
-
- RAVNOVES00 · exact replay sample - frame {frame.sequence} · sample {frame.ordinal}/32 - {(frame.sourceTimeNs / 1_000_000_000).toFixed(3)} с · {selectedItem?.frameId} -
-
- Representation layers + Spatial evidence {currentIncrementObstacles.length} current · {rollingMapObstacles.length} rolling - CURRENT INCREMENT {frame.pointCloudSampleCount}/{frame.pointCloudSourceCount} points - · ROLLING MAP {frame.rollingMapComponentCount} components + {frame.spatialAvailable + ? `${frame.pointCloudSampleCount}/${frame.pointCloudSourceCount} LiDAR points` + : "body frame / current increment unavailable"}
Virtual corridor - {threatObstacles.length} threat · nearest {nearest === null ? "—" : `${nearest.toFixed(2)} м`} - {frame.corridor.forwardLengthM} м · body {frame.rig.lengthM}×{frame.rig.widthM} м · REPLAY-SIMULATED + + {frame.decisionCounts.threat} threat · nearest {nearest === null ? "—" : `${nearest.toFixed(2)} м`} + + + {metadata.timeline.corridor.forwardLengthM} м · body {metadata.timeline.rig.lengthM}×{metadata.timeline.rig.widthM} м · REPLAY-SIMULATED +
) : undefined; let content; - if (mode === "video") { - content = videoLoading ? ( + if (metadata.error) { + content = ; + } else if (mode === "video") { + content = videoLoading + || metadata.loading + || Boolean(metadata.timeline && !videoSource && !videoError) ? (
- ) : videoError || !videoOverlay || !videoSource ? ( -
- - {videoError ?? "Видео-доказательство M4.6 недоступно."} + Открываем синхронное RIGHT-видео RAVNOVES00
+ ) : videoError || !metadata.timeline || !videoSource ? ( + ) : ( ); + } else if (timelineFrame.error) { + content = ; + } else if (timelineFrame.loading || !metadata.timeline || !frame) { + content = ( +
+
+ ); } else if (mode === "camera") { - content = sampleLoading ? ( -
-
- ) : sampleError || !frame ? ( -
- - {sampleError ?? "CAMERA-кадр M4.6 недоступен."} -
- ) : ( + content = ( ); + } else if (!frame.spatialAvailable) { + content = ; } else { - content = sampleLoading ? ( -
-
- ) : sampleError || !frame ? ( -
- - {sampleError ?? "Метрический visual M4.6 недоступен."} -
- ) : ( + content = ( ({ - id: obstacle.componentId, - decision: obstacle.assessment.decision, - state: obstacle.state, - centroidBodyXyzM: obstacle.centroidBodyXyzM, - cellCentersBodyXyzM: obstacle.cellCentersBodyXyzM, - }))} - rig={frame.rig} - corridor={frame.corridor} + obstacles={sceneObstacles} + rig={metadata.timeline.rig} + corridor={metadata.timeline.corridor} mode={mode} - label={`M4.6 current increment and rolling map, frame ${frame.sequence}`} + label="M4.6 recorded-realtime current increment and rolling occupancy" /> ); } + const timeline = metadata.timeline; + const transport = timeline ? ( + playbackController.seek(timeNs / 1_000_000_000)} + onPlayingChange={playbackController.setPlaying} + onPlaybackRateChange={playbackController.setRate} + onJumpToEnd={() => playbackController.seek(timeline.timelineEndSeconds)} + /> + ) : undefined; + return (
{content} diff --git a/apps/control-station/src/workspaces/laboratory/useM4ThreatTimeline.ts b/apps/control-station/src/workspaces/laboratory/useM4ThreatTimeline.ts new file mode 100644 index 0000000..79c7a69 --- /dev/null +++ b/apps/control-station/src/workspaces/laboratory/useM4ThreatTimeline.ts @@ -0,0 +1,130 @@ +import { useEffect, useMemo, useRef, useState } from "react"; + +import { + fetchM4ThreatTimeline, + fetchM4ThreatTimelineChunk, + selectM4ThreatTimelineSequence, + type M4ThreatTimeline, + type M4ThreatTimelineChunk, + type M4ThreatTimelineFrame, +} from "../../core/laboratory/m4ReplayThreat"; + +const REQUESTED_CHUNK_FRAMES = 12; +const RETAINED_CHUNK_COUNT = 4; + +function errorMessage(error: unknown, fallback: string): string { + return error instanceof Error && error.message.trim() ? error.message : fallback; +} + +export function useM4ThreatTimelineMetadata(resultId: string) { + const [timeline, setTimeline] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + const controller = new AbortController(); + setTimeline(null); + setError(null); + void fetchM4ThreatTimeline(resultId, { signal: controller.signal }) + .then((next) => { + if (!controller.signal.aborted) setTimeline(next); + }) + .catch((caught: unknown) => { + if (!controller.signal.aborted) { + setError(errorMessage(caught, "Recorded-realtime timeline M4.6 недоступен.")); + } + }); + return () => controller.abort(); + }, [resultId]); + + return { timeline, loading: !timeline && !error, error }; +} + +export function useM4ThreatTimelineFrame({ + resultId, + timeline, + currentSeconds, +}: { + resultId: string; + timeline: M4ThreatTimeline | null; + currentSeconds: number; +}) { + const [chunks, setChunks] = useState>( + () => new Map(), + ); + const [error, setError] = useState(null); + const inFlight = useRef(new Set()); + const chunksRef = useRef(chunks); + chunksRef.current = chunks; + + useEffect(() => { + setChunks(new Map()); + setError(null); + inFlight.current.clear(); + }, [resultId, timeline]); + + const activeSequence = useMemo( + () => timeline + ? selectM4ThreatTimelineSequence(timeline.frameTimesNs, currentSeconds) + : null, + [currentSeconds, timeline], + ); + const chunkSize = Math.min( + REQUESTED_CHUNK_FRAMES, + timeline?.maxChunkFrames ?? REQUESTED_CHUNK_FRAMES, + ); + const activeChunkStart = activeSequence === null + ? null + : Math.floor(activeSequence / chunkSize) * chunkSize; + + useEffect(() => { + if (!timeline || activeChunkStart === null) return; + const starts = [activeChunkStart, activeChunkStart + chunkSize].filter( + (start) => start < timeline.frameCount, + ); + const controllers: AbortController[] = []; + for (const start of starts) { + if (chunksRef.current.has(start) || inFlight.current.has(start)) continue; + const controller = new AbortController(); + controllers.push(controller); + inFlight.current.add(start); + void fetchM4ThreatTimelineChunk(resultId, start, chunkSize, { + signal: controller.signal, + }) + .then((chunk) => { + if (controller.signal.aborted) return; + setChunks((current) => { + const next = new Map(current); + next.set(start, chunk); + const retained = [...next.keys()] + .sort((left, right) => ( + Math.abs(left - activeChunkStart) - Math.abs(right - activeChunkStart) + )) + .slice(0, RETAINED_CHUNK_COUNT); + return new Map(retained.map((key) => [key, next.get(key)!])); + }); + if (start === activeChunkStart) setError(null); + }) + .catch((caught: unknown) => { + if (!controller.signal.aborted && start === activeChunkStart) { + setError(errorMessage(caught, "3D chunk M4.6 недоступен.")); + } + }) + .finally(() => inFlight.current.delete(start)); + } + return () => controllers.forEach((controller) => controller.abort()); + }, [activeChunkStart, chunkSize, resultId, timeline]); + + const activeFrame: M4ThreatTimelineFrame | null = useMemo(() => { + if (activeSequence === null || activeChunkStart === null) return null; + return chunks.get(activeChunkStart)?.frames.find( + (frame) => frame.sequence === activeSequence, + ) ?? null; + }, [activeChunkStart, activeSequence, chunks]); + + return { + activeSequence, + activeFrame, + loading: error === null && Boolean(timeline) && !activeFrame, + error, + }; +} diff --git a/apps/control-station/test/m4ReplayThreat.test.mjs b/apps/control-station/test/m4ReplayThreat.test.mjs index 3c3782e..a1f4a40 100644 --- a/apps/control-station/test/m4ReplayThreat.test.mjs +++ b/apps/control-station/test/m4ReplayThreat.test.mjs @@ -7,8 +7,11 @@ import { createServer } from "vite"; let server; let fetchM4ThreatReplayResult; let fetchM4ThreatVisual; -let fetchM4ThreatVideoOverlay; -let selectM4ThreatVideoFrame; +let fetchM4ThreatTimeline; +let fetchM4ThreatTimelineChunk; +let selectM4ThreatTimelineFrame; +let selectM4ThreatTimelineSequence; +let advanceRecordedEvidencePlayback; const resultId = `m4-threat-replay-${"a".repeat(64)}`; @@ -21,9 +24,14 @@ before(async () => { ({ fetchM4ThreatReplayResult, fetchM4ThreatVisual, - fetchM4ThreatVideoOverlay, - selectM4ThreatVideoFrame, + fetchM4ThreatTimeline, + fetchM4ThreatTimelineChunk, + selectM4ThreatTimelineFrame, + selectM4ThreatTimelineSequence, } = await server.ssrLoadModule("/src/core/laboratory/m4ReplayThreat.ts")); + ({ advanceRecordedEvidencePlayback } = await server.ssrLoadModule( + "/src/components/laboratory/useRecordedEvidencePlayback.ts", + )); }); after(async () => { @@ -44,6 +52,29 @@ function proposal(overrides = {}) { }; } +function timelineFrame(sequence, sessionSeconds, overrides = {}) { + return { + schema_version: "missioncore.recorded-spatial-evidence-frame/v1", + sequence, + frame_id: `frame-${String(sequence).padStart(6, "0")}`, + source_time_ns: Math.round(sessionSeconds * 1_000_000_000), + session_seconds: sessionSeconds, + source_available: true, + spatial_available: true, + point_cloud_body_xyz_m: [[1, 0, 0.1]], + point_cloud_source_count: 1, + point_cloud_sample_count: 1, + point_cloud_layer: "current-increment", + rolling_map_component_count: 0, + metric_obstacles: [], + camera_proposals: [], + decision_counts: { threat: 0, "not-threat": 0, unknown: 0 }, + camera_url: `/api/v1/laboratory/m4-threat/results/${resultId}/timeline/frames/${sequence}/camera`, + authority: "replay-simulated", + ...overrides, + }; +} + test("M4.6 decodes accepted dual-evidence result without physical authority", async () => { const result = await fetchM4ThreatReplayResult({ fetcher: async () => new Response(JSON.stringify({ @@ -200,38 +231,82 @@ test("M4.6 v2 keeps CURRENT INCREMENT separate from ROLLING MAP", async () => { assert.equal(frame.metricObstacles[0].assessment.decision, "threat"); }); -test("M4.6 full video preserves camera-only unknown and nearest-frame selection", async () => { - const overlay = await fetchM4ThreatVideoOverlay(resultId, { +test("M4.6 timeline keeps only a compact index and decodes bounded spatial chunks", async () => { + const frameTimesNs = Array.from( + { length: 4489 }, + (_, index) => 35_421_857_292 + index * 100_000_000, + ); + const timeline = await fetchM4ThreatTimeline(resultId, { fetcher: async () => new Response(JSON.stringify({ - schema_version: "missioncore.m4-threat-video-overlay/v1", + schema_version: "missioncore.recorded-spatial-evidence-timeline/v1", result_id: resultId, - recorded_source: { session_id: "20260720T065719Z_viewer_live" }, + recorded_source: { + session_id: "20260720T065719Z_viewer_live", + source_id: "RAVNOVES00", + synchronization: "host-arrival-best-effort", + }, image_width: 800, image_height: 600, - timeline_start_seconds: 35.421857292, - timeline_end_seconds: 484.044857292, frame_count: 4489, + frame_times_ns: frameTimesNs, + timeline_start_seconds: 35.421857292, + timeline_end_seconds: 484.221857292, + nominal_frame_interval_seconds: 0.1, + nominal_rate_hz: 10, + max_chunk_frames: 24, + point_sample_limit: 2000, + rig: { length_m: 1, width_m: 0.6, nominal_sensor_height_m: 1.25 }, + corridor: { + forward_length_m: 8, + rear_margin_m: 0.5, + half_width_m: 0.5, + prediction_horizon_seconds: 5, + }, + authority: "replay-simulated", + }), { status: 200 }), + }); + assert.equal(timeline.frameTimesNs.length, 4489); + assert.equal(selectM4ThreatTimelineSequence(timeline.frameTimesNs, 35.50), 1); + + const chunk = await fetchM4ThreatTimelineChunk(resultId, 0, 2, { + fetcher: async () => new Response(JSON.stringify({ + schema_version: "missioncore.recorded-spatial-evidence-chunk/v1", + result_id: resultId, + start_sequence: 0, + frame_count: 2, + next_sequence: 2, frames: [ - { - frame_index: 0, - session_seconds: 35.421857292, - source_available: true, - camera_proposals: [], - decision_counts: { threat: 0, "not-threat": 0, unknown: 0 }, - }, - { - frame_index: 1, - session_seconds: 35.521857292, - source_available: true, + timelineFrame(0, 35.421857292), + timelineFrame(1, 35.521857292, { camera_proposals: [proposal()], decision_counts: { threat: 0, "not-threat": 0, unknown: 1 }, - }, + }), ], authority: "replay-simulated", }), { status: 200 }), }); - assert.equal(overlay.frames[1].cameraProposals[0].rangeM, null); - assert.equal(selectM4ThreatVideoFrame(overlay.frames, 35.50).frameIndex, 1); + assert.equal(chunk.frames[1].cameraProposals[0].rangeM, null); + assert.equal(selectM4ThreatTimelineFrame(chunk.frames, 35.50).sequence, 1); +}); + +test("recorded evidence clock advances by selected rate and stops at the sealed end", () => { + const range = { startSeconds: 10, endSeconds: 20 }; + assert.deepEqual( + advanceRecordedEvidencePlayback( + { currentSeconds: 12, playing: true, rate: 2 }, + 1.5, + range, + ), + { currentSeconds: 15, playing: true, rate: 2 }, + ); + assert.deepEqual( + advanceRecordedEvidencePlayback( + { currentSeconds: 19.5, playing: true, rate: 1 }, + 1, + range, + ), + { currentSeconds: 20, playing: false, rate: 1 }, + ); }); test("M4.6 viewer reuses shared camera, video and metric evidence renderers", async () => { @@ -244,6 +319,8 @@ test("M4.6 viewer reuses shared camera, video and metric evidence renderers", as assert.match(visual, / FloatArray | None: + """Expose one verified increment to a read-only recorded-evidence projector.""" + + if not isinstance(frame_index, int) or isinstance(frame_index, bool): + raise GeometryProviderError("replay evidence frame index is invalid") + if not 0 <= frame_index < self.profile.frame_count: + raise GeometryProviderError("replay evidence frame is outside the source profile") + if int(self._source["frame_indices"][frame_index]) != frame_index: + raise GeometryProviderError("source pack frame sequence changed") + if not bool(self._source["sample_available"][frame_index]) or not bool( + self._surface["frame_valid"][frame_index] + ): + return None + offsets = self._source["cloud_offsets"] + start, end = int(offsets[frame_index]), int(offsets[frame_index + 1]) + points = np.asarray(self._source["cloud_points_map"][start:end], dtype=np.float64) + points.setflags(write=False) + return points + def pose_values_for_frame( self, frame_id: str, diff --git a/src/k1link/perception/spatial_evidence.py b/src/k1link/perception/spatial_evidence.py new file mode 100644 index 0000000..8a4fbdf --- /dev/null +++ b/src/k1link/perception/spatial_evidence.py @@ -0,0 +1,118 @@ +"""Deterministic projections shared by recorded spatial evidence producers.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence + +import numpy as np +import numpy.typing as npt + +from .threat import ReplayBodyFrame + +FloatArray = npt.NDArray[np.float64] + + +class SpatialEvidenceProjectionError(RuntimeError): + """Recorded spatial evidence cannot be projected without changing meaning.""" + + +def sample_points_in_body_frame( + points_map: FloatArray, + body_frame: ReplayBodyFrame, + *, + point_limit: int, +) -> tuple[list[list[float]], int]: + """Project one immutable map-frame increment into the current body frame.""" + + if point_limit < 1: + raise SpatialEvidenceProjectionError("spatial evidence point limit must be positive") + points = np.asarray(points_map, dtype=np.float64) + if points.ndim != 2 or points.shape[1] != 3 or not np.isfinite(points).all(): + raise SpatialEvidenceProjectionError("spatial evidence point array is invalid") + basis = np.asarray(body_frame.basis_map_from_body, dtype=np.float64) + origin = np.asarray(body_frame.origin_map_xyz_m, dtype=np.float64) + if basis.shape != (3, 3) or origin.shape != (3,): + raise SpatialEvidenceProjectionError("spatial evidence body frame is invalid") + points_body = (points - origin) @ basis + stride = max(1, math.ceil(points_body.shape[0] / point_limit)) + sampled = points_body[::stride][:point_limit] + return np.round(sampled, 6).tolist(), int(points.shape[0]) + + +def project_metric_obstacles_to_body( + metric_rows: Sequence[Mapping[str, object]], + body_frame: ReplayBodyFrame, + *, + occupied_voxel_size_m: float, +) -> list[dict[str, object]]: + """Project ledger-owned metric components without recomputing their decision.""" + + if not math.isfinite(occupied_voxel_size_m) or occupied_voxel_size_m <= 0: + raise SpatialEvidenceProjectionError("occupied voxel size is invalid") + visuals: list[dict[str, object]] = [] + for row in metric_rows: + centroid = row.get("centroid_map_xyz_m") + cells = row.get("cells") + if not isinstance(centroid, list) or len(centroid) != 3 or not isinstance(cells, list): + continue + centroid_map = _finite_vector3(centroid, "metric centroid") + centroid_body = body_frame.map_point_to_body(centroid_map) + cell_centers: list[list[float]] = [] + for raw_cell in cells: + if not isinstance(raw_cell, dict): + raise SpatialEvidenceProjectionError("occupied cell is invalid") + indices = ( + _signed_integer(raw_cell.get("x"), "cell x"), + _signed_integer(raw_cell.get("y"), "cell y"), + _signed_integer(raw_cell.get("z"), "cell z"), + ) + point_map = ( + (indices[0] + 0.5) * occupied_voxel_size_m, + (indices[1] + 0.5) * occupied_voxel_size_m, + (indices[2] + 0.5) * occupied_voxel_size_m, + ) + cell_centers.append(list(body_frame.map_point_to_body(point_map))) + visuals.append( + { + "component_id": row.get("component_id"), + "state": row.get("state"), + "motion": row.get("motion"), + "centroid_body_xyz_m": list(centroid_body), + "cell_centers_body_xyz_m": cell_centers, + "assessment": row.get("assessment"), + } + ) + return visuals + + +def _finite_vector3(value: Sequence[object], label: str) -> tuple[float, float, float]: + if len(value) != 3: + raise SpatialEvidenceProjectionError(f"{label} is invalid") + return ( + _finite_float(value[0], label), + _finite_float(value[1], label), + _finite_float(value[2], label), + ) + + +def _finite_float(value: object, label: str) -> float: + if not isinstance(value, (int, float)) or isinstance(value, bool): + raise SpatialEvidenceProjectionError(f"{label} is invalid") + parsed = float(value) + if not math.isfinite(parsed): + raise SpatialEvidenceProjectionError(f"{label} is invalid") + return parsed + + +def _signed_integer(value: object, label: str) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise SpatialEvidenceProjectionError(f"{label} is invalid") + return value + + +__all__ = [ + "SpatialEvidenceProjectionError", + "project_metric_obstacles_to_body", + "sample_points_in_body_frame", +] diff --git a/src/k1link/perception/threat_replay.py b/src/k1link/perception/threat_replay.py index a762ece..71b225f 100644 --- a/src/k1link/perception/threat_replay.py +++ b/src/k1link/perception/threat_replay.py @@ -38,6 +38,10 @@ from .geometry import RecordedGeometryStore from .geometry_replay import GeometryReplayResult, read_geometry_replay_result from .providers import SourcePacket from .recorded_source import RecordedRavnoves00Source, ReplayPacing +from .spatial_evidence import ( + project_metric_obstacles_to_body, + sample_points_in_body_frame, +) from .temporal_replay import TemporalReplayResult, read_temporal_replay_result from .threat import ( DEFAULT_REPLAY_THREAT_PROFILE_PATH, @@ -654,49 +658,24 @@ def _visual_frame( points = store.current_points(packet) if points is None: raise ThreatReplayError("visual frame has no current point cloud") - basis = np.asarray(body_frame.basis_map_from_body, dtype=np.float64) - origin = np.asarray(body_frame.origin_map_xyz_m, dtype=np.float64) - points_body = (points - origin) @ basis - stride = max(1, math.ceil(points_body.shape[0] / VISUAL_POINT_LIMIT)) - sampled = points_body[::stride][:VISUAL_POINT_LIMIT] - metric_visuals = [] - for row in metric_rows: - centroid = row.get("centroid_map_xyz_m") - cells = row.get("cells") - if not isinstance(centroid, list) or not isinstance(cells, list): - continue - centroid_body = body_frame.map_point_to_body( - (float(centroid[0]), float(centroid[1]), float(centroid[2])) - ) - cell_centers = [] - for raw_cell in cells: - cell = _object(raw_cell, "visual occupied cell") - point_map = tuple( - (_signed_integer(cell.get(key), f"cell {key}") + 0.5) - * profile.corridor.occupied_voxel_size_m - for key in ("x", "y", "z") - ) - cell_centers.append( - list(body_frame.map_point_to_body((point_map[0], point_map[1], point_map[2]))) - ) - metric_visuals.append( - { - "component_id": row["component_id"], - "state": row["state"], - "motion": row["motion"], - "centroid_body_xyz_m": list(centroid_body), - "cell_centers_body_xyz_m": cell_centers, - "assessment": row["assessment"], - } - ) + sampled, source_count = sample_points_in_body_frame( + points, + body_frame, + point_limit=VISUAL_POINT_LIMIT, + ) + metric_visuals = project_metric_obstacles_to_body( + metric_rows, + body_frame, + occupied_voxel_size_m=profile.corridor.occupied_voxel_size_m, + ) return { "schema_version": THREAT_REPLAY_VISUAL_SCHEMA_V2, "sequence": packet.envelope.sequence, "frame_id": packet.envelope.frame_id, "source_time_ns": packet.envelope.timestamps.source_ns, - "point_cloud_body_xyz_m": np.round(sampled, 6).tolist(), - "point_cloud_source_count": int(points.shape[0]), - "point_cloud_sample_count": int(sampled.shape[0]), + "point_cloud_body_xyz_m": sampled, + "point_cloud_source_count": source_count, + "point_cloud_sample_count": len(sampled), "point_cloud_layer": "current-increment", "rolling_map_component_count": sum( row.get("state") == TemporalState.RETAINED.value for row in metric_rows diff --git a/src/k1link/perception/threat_timeline.py b/src/k1link/perception/threat_timeline.py new file mode 100644 index 0000000..89a88fc --- /dev/null +++ b/src/k1link/perception/threat_timeline.py @@ -0,0 +1,284 @@ +"""Bounded recorded-realtime projection of a sealed replay threat ledger.""" + +from __future__ import annotations + +import copy +import json +import math +import re +import statistics +from dataclasses import dataclass +from itertools import pairwise +from pathlib import Path +from threading import RLock +from typing import Final + +from .geometry import RecordedGeometryStore +from .spatial_evidence import ( + project_metric_obstacles_to_body, + sample_points_in_body_frame, +) +from .threat import ( + DEFAULT_REPLAY_THREAT_PROFILE_PATH, + RecordedReplayBodyFrameResolver, + load_replay_threat_profile, +) +from .threat_replay import ( + THREAT_REPLAY_FRAME_SCHEMA, + THREAT_REPLAY_FRAME_SCHEMA_V2, + ThreatReplayResult, +) + +RECORDED_SPATIAL_TIMELINE_SCHEMA: Final = "missioncore.recorded-spatial-evidence-timeline/v1" +RECORDED_SPATIAL_CHUNK_SCHEMA: Final = "missioncore.recorded-spatial-evidence-chunk/v1" +RECORDED_SPATIAL_FRAME_SCHEMA: Final = "missioncore.recorded-spatial-evidence-frame/v1" +RECORDED_SPATIAL_POINT_LIMIT: Final = 2_000 +RECORDED_SPATIAL_MAX_CHUNK_FRAMES: Final = 24 +_EXPECTED_FRAME_COUNT: Final = 4_489 +_SOURCE_TIME = re.compile(rb'"source_time_ns":([0-9]+)') + + +class RecordedThreatTimelineError(RuntimeError): + """A bounded timeline projection escaped its sealed result or source.""" + + +@dataclass(frozen=True, slots=True) +class RecordedThreatTimelineIndex: + offsets: tuple[int, ...] + source_times_ns: tuple[int, ...] + + +class RecordedThreatTimeline: + """Read bounded spatial chunks without materializing the full ledger in memory.""" + + def __init__(self, *, repository_root: Path, result: ThreatReplayResult) -> None: + self.repository_root = repository_root.resolve(strict=True) + self.result = result + self.frames_path = (result.result_root / "frames.jsonl").resolve(strict=True) + if self.frames_path.is_symlink() or self.frames_path.parent != result.result_root: + raise RecordedThreatTimelineError("recorded timeline frame ledger is invalid") + self.profile = load_replay_threat_profile( + self.repository_root / DEFAULT_REPLAY_THREAT_PROFILE_PATH + ) + identity = result.manifest.get("identity") + if not isinstance(identity, dict): + raise RecordedThreatTimelineError("recorded timeline identity is missing") + expected_identity = { + "profile_id": self.profile.profile_id, + "profile_sha256": self.profile.profile_sha256, + "source_id": self.profile.source_id, + "source_session_id": self.profile.session_id, + "source_pack_id": self.profile.source_pack_id, + "source_pack_sha256": self.profile.source_pack_sha256, + } + if any(identity.get(key) != value for key, value in expected_identity.items()): + raise RecordedThreatTimelineError("recorded timeline escaped the threat profile") + self.store = RecordedGeometryStore.from_repository(self.repository_root) + if ( + self.store.profile.source_pack_id != self.profile.source_pack_id + or self.store.profile.source_pack_sha256 != self.profile.source_pack_sha256 + or self.store.profile.frame_count != _EXPECTED_FRAME_COUNT + ): + raise RecordedThreatTimelineError("recorded timeline geometry identity changed") + self.body_frames = RecordedReplayBodyFrameResolver( + self.store, + profile=self.profile.body_frame, + ) + self.index = _index_frame_ledger(self.frames_path) + self._lock = RLock() + + def metadata(self) -> dict[str, object]: + times = self.index.source_times_ns + intervals = [(current - previous) / 1_000_000_000 for previous, current in pairwise(times)] + nominal_interval = statistics.median(intervals) + if not math.isfinite(nominal_interval) or nominal_interval <= 0: + raise RecordedThreatTimelineError("recorded timeline cadence is invalid") + return { + "schema_version": RECORDED_SPATIAL_TIMELINE_SCHEMA, + "result_id": self.result.result_id, + "recorded_source": { + "session_id": self.profile.session_id, + "source_id": self.profile.source_id, + "synchronization": "host-arrival-best-effort", + }, + "frame_count": len(times), + "frame_times_ns": list(times), + "timeline_start_seconds": times[0] / 1_000_000_000, + "timeline_end_seconds": times[-1] / 1_000_000_000, + "nominal_frame_interval_seconds": nominal_interval, + "nominal_rate_hz": 1 / nominal_interval, + "max_chunk_frames": RECORDED_SPATIAL_MAX_CHUNK_FRAMES, + "point_sample_limit": RECORDED_SPATIAL_POINT_LIMIT, + "image_width": 800, + "image_height": 600, + "rig": { + "length_m": self.profile.rig.body_length_m, + "width_m": self.profile.rig.body_width_m, + "nominal_sensor_height_m": self.profile.rig.nominal_sensor_height_m, + }, + "corridor": { + "forward_length_m": self.profile.corridor.forward_length_m, + "rear_margin_m": self.profile.corridor.rear_margin_m, + "half_width_m": ( + self.profile.rig.body_width_m / 2 + self.profile.corridor.lateral_clearance_m + ), + "prediction_horizon_seconds": (self.profile.corridor.prediction_horizon_seconds), + }, + "ground_truth": False, + "authority": "replay-simulated", + "access": "read-only-bounded-recorded-replay", + } + + def chunk(self, *, start_sequence: int, frame_count: int) -> dict[str, object]: + if not 0 <= start_sequence < len(self.index.offsets): + raise RecordedThreatTimelineError("recorded timeline chunk start is invalid") + if not 1 <= frame_count <= RECORDED_SPATIAL_MAX_CHUNK_FRAMES: + raise RecordedThreatTimelineError("recorded timeline chunk size is invalid") + stop = min(len(self.index.offsets), start_sequence + frame_count) + with self._lock: + frames = [self._project_frame(sequence) for sequence in range(start_sequence, stop)] + return { + "schema_version": RECORDED_SPATIAL_CHUNK_SCHEMA, + "result_id": self.result.result_id, + "start_sequence": start_sequence, + "frame_count": len(frames), + "next_sequence": stop if stop < len(self.index.offsets) else None, + "frames": frames, + "ground_truth": False, + "authority": "replay-simulated", + "access": "read-only-bounded-recorded-replay", + } + + def _project_frame(self, sequence: int) -> dict[str, object]: + row = _read_frame_at(self.frames_path, self.index, sequence) + frame_id = row.get("frame_id") + if not isinstance(frame_id, str) or not frame_id: + raise RecordedThreatTimelineError("recorded timeline frame identity is invalid") + body_frame = self.body_frames.body_frame_for_frame(frame_id) + body_frame_declared = row.get("body_frame_available") + if not isinstance(body_frame_declared, bool) or body_frame_declared is not ( + body_frame is not None + ): + raise RecordedThreatTimelineError("recorded timeline body-frame binding changed") + source_available = row.get("source_available") + if not isinstance(source_available, bool): + raise RecordedThreatTimelineError("recorded timeline source state is invalid") + point_cloud: list[list[float]] = [] + point_source_count = 0 + metric_visuals: list[dict[str, object]] = [] + if body_frame is not None: + points = self.store.current_points_for_frame(sequence) + if points is None or not source_available: + raise RecordedThreatTimelineError( + "recorded timeline current increment binding changed" + ) + point_cloud, point_source_count = sample_points_in_body_frame( + points, + body_frame, + point_limit=RECORDED_SPATIAL_POINT_LIMIT, + ) + metric_visuals = project_metric_obstacles_to_body( + _mapping_array(row.get("metric_obstacles"), "metric obstacles"), + body_frame, + occupied_voxel_size_m=self.profile.corridor.occupied_voxel_size_m, + ) + assessments = _mapping_array(row.get("assessments"), "threat assessments") + camera_proposals = row.get("camera_proposals") + if not isinstance(camera_proposals, list): + raise RecordedThreatTimelineError("recorded timeline camera proposals are invalid") + return { + "schema_version": RECORDED_SPATIAL_FRAME_SCHEMA, + "sequence": sequence, + "frame_id": frame_id, + "source_time_ns": self.index.source_times_ns[sequence], + "session_seconds": self.index.source_times_ns[sequence] / 1_000_000_000, + "source_available": source_available, + "spatial_available": body_frame is not None, + "point_cloud_body_xyz_m": point_cloud, + "point_cloud_source_count": point_source_count, + "point_cloud_sample_count": len(point_cloud), + "point_cloud_layer": "current-increment", + "rolling_map_component_count": sum( + item.get("state") == "retained" for item in metric_visuals + ), + "metric_obstacles": metric_visuals, + "camera_proposals": copy.deepcopy(camera_proposals), + "decision_counts": _decision_counts(assessments), + "camera_url": ( + f"/api/v1/laboratory/m4-threat/results/{self.result.result_id}" + f"/timeline/frames/{sequence}/camera" + ), + "ground_truth": False, + "authority": "replay-simulated", + } + + +def _index_frame_ledger(path: Path) -> RecordedThreatTimelineIndex: + offsets: list[int] = [] + source_times: list[int] = [] + with path.open("rb") as handle: + while True: + offset = handle.tell() + line = handle.readline() + if not line: + break + match = _SOURCE_TIME.search(line) + if match is None: + raise RecordedThreatTimelineError("recorded timeline source time is missing") + offsets.append(offset) + source_times.append(int(match.group(1))) + if len(offsets) != _EXPECTED_FRAME_COUNT: + raise RecordedThreatTimelineError("recorded timeline frame count changed") + if any(current <= previous for previous, current in pairwise(source_times)): + raise RecordedThreatTimelineError("recorded timeline source time is not monotonic") + return RecordedThreatTimelineIndex(tuple(offsets), tuple(source_times)) + + +def _read_frame_at( + path: Path, + index: RecordedThreatTimelineIndex, + sequence: int, +) -> dict[str, object]: + with path.open("rb") as handle: + handle.seek(index.offsets[sequence]) + line = handle.readline() + try: + row = json.loads(line) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise RecordedThreatTimelineError("recorded timeline frame JSON is invalid") from error + if ( + not isinstance(row, dict) + or row.get("schema_version") + not in {THREAT_REPLAY_FRAME_SCHEMA, THREAT_REPLAY_FRAME_SCHEMA_V2} + or row.get("sequence") != sequence + or row.get("source_time_ns") != index.source_times_ns[sequence] + ): + raise RecordedThreatTimelineError("recorded timeline frame binding changed") + return row + + +def _mapping_array(value: object, label: str) -> list[dict[str, object]]: + if not isinstance(value, list) or any(not isinstance(item, dict) for item in value): + raise RecordedThreatTimelineError(f"recorded timeline {label} are invalid") + return value + + +def _decision_counts(assessments: list[dict[str, object]]) -> dict[str, int]: + result = {"threat": 0, "not-threat": 0, "unknown": 0} + for item in assessments: + decision = item.get("decision") + if not isinstance(decision, str) or decision not in result: + raise RecordedThreatTimelineError("recorded timeline decision is invalid") + result[decision] += 1 + return result + + +__all__ = [ + "RECORDED_SPATIAL_CHUNK_SCHEMA", + "RECORDED_SPATIAL_FRAME_SCHEMA", + "RECORDED_SPATIAL_MAX_CHUNK_FRAMES", + "RECORDED_SPATIAL_POINT_LIMIT", + "RECORDED_SPATIAL_TIMELINE_SCHEMA", + "RecordedThreatTimeline", + "RecordedThreatTimelineError", +] diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index 5cc931c..7c85c12 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -767,6 +767,7 @@ app.include_router( root_provider=lambda: ( REPOSITORY_ROOT / ".runtime" / "compute-experiments" / "m4" / "replay-threat" ), + repository_root_provider=lambda: REPOSITORY_ROOT, camera_frame_provider=( session_recorded_camera_frame_service.extract if session_recorded_camera_frame_service is not None diff --git a/src/k1link/web/m4_threat_replay_api.py b/src/k1link/web/m4_threat_replay_api.py index d59032b..ef2af8a 100644 --- a/src/k1link/web/m4_threat_replay_api.py +++ b/src/k1link/web/m4_threat_replay_api.py @@ -22,11 +22,15 @@ from k1link.perception.threat_replay import ( ThreatReplayResult, read_threat_replay_result, ) +from k1link.perception.threat_timeline import ( + RECORDED_SPATIAL_MAX_CHUNK_FRAMES, + RecordedThreatTimeline, + RecordedThreatTimelineError, +) from k1link.sessions import RecordedCameraFrame, SessionIntegrityError M4_THREAT_CATALOG_SCHEMA: Final = "missioncore.m4-threat-replay-catalog/v1" M4_THREAT_VIEW_SCHEMA: Final = "missioncore.m4-threat-replay-view/v1" -M4_THREAT_VIDEO_SCHEMA: Final = "missioncore.m4-threat-video-overlay/v1" M4_THREAT_VISUAL_CATALOG_SCHEMA: Final = "missioncore.m4-threat-visual-catalog/v1" _RESULT_ID = re.compile(rf"^{THREAT_REPLAY_RESULT_PREFIX}[a-f0-9]{{64}}$") RootProvider = Callable[[], Path | None] @@ -36,6 +40,7 @@ CameraFrameProvider = Callable[[str, int], RecordedCameraFrame] def build_m4_threat_replay_router( *, root_provider: RootProvider = lambda: None, + repository_root_provider: RootProvider = lambda: None, camera_frame_provider: CameraFrameProvider | None = None, ) -> APIRouter: router = APIRouter(prefix="/api/v1/laboratory/m4-threat", tags=["laboratory"]) @@ -54,6 +59,23 @@ def build_m4_threat_replay_router( except (ThreatReplayError, OSError, ValueError): raise HTTPException(status_code=404, detail="M4.6 result не найден") from None + def timeline(result_id: str) -> RecordedThreatTimeline: + frozen = result(result_id) + repository_root = _configured_root(repository_root_provider) + if repository_root is None: + raise HTTPException(status_code=503, detail="M4.6 timeline source недоступен") + try: + return _read_threat_timeline_cached( + str(repository_root), + str(frozen.result_root), + _result_signature(frozen.result_root), + ) + except (OSError, ThreatReplayError, RecordedThreatTimelineError, ValueError): + raise HTTPException( + status_code=503, + detail="M4.6 bounded timeline не прошёл проверку", + ) from None + @router.get("/results") def list_results(limit: int = Query(default=1, ge=1, le=10)) -> dict[str, object]: candidates = _candidates(root_provider) @@ -134,41 +156,38 @@ def build_m4_threat_replay_router( sequence = frames[ordinal - 1].get("sequence") if not isinstance(session_id, str) or not isinstance(sequence, int): raise HTTPException(status_code=404, detail="M4.6 camera identity не найдена") - try: - camera = camera_frame_provider(session_id, sequence) - except (OSError, SessionIntegrityError, ValueError): - raise HTTPException( - status_code=503, - detail="M4.6 exact camera frame недоступен", - ) from None - if camera.width != 800 or camera.height != 600: - raise HTTPException( - status_code=503, - detail="M4.6 camera frame нарушил размерный контракт", - ) - return Response( - content=camera.payload, - media_type=camera.media_type, - headers={ - "Cache-Control": "private, max-age=31536000, immutable", - "ETag": f'"{camera.sha256}"', - "X-Content-Type-Options": "nosniff", - }, - ) + return _camera_response(camera_frame_provider, session_id, sequence) - @router.get("/results/{result_id}/video-overlay") - def get_video_overlay(result_id: str) -> dict[str, object]: - frozen = result(result_id) - identity = frozen.manifest["identity"] - assert isinstance(identity, dict) - return copy.deepcopy( - _cached_video_overlay( - result_id, - str(frozen.result_root), - str(identity["frames_sha256"]), - str(identity["source_session_id"]), - ) - ) + @router.get("/results/{result_id}/timeline") + def get_timeline(result_id: str) -> dict[str, object]: + return copy.deepcopy(timeline(result_id).metadata()) + + @router.get("/results/{result_id}/timeline/chunk") + def get_timeline_chunk( + result_id: str, + start: int = Query(default=0, ge=0), + count: int = Query( + default=12, + ge=1, + le=RECORDED_SPATIAL_MAX_CHUNK_FRAMES, + ), + ) -> dict[str, object]: + try: + return timeline(result_id).chunk(start_sequence=start, frame_count=count) + except RecordedThreatTimelineError: + raise HTTPException( + status_code=404, + detail="M4.6 timeline chunk не найден", + ) from None + + @router.get("/results/{result_id}/timeline/frames/{sequence}/camera") + def get_timeline_camera(result_id: str, sequence: int) -> Response: + if camera_frame_provider is None: + raise HTTPException(status_code=503, detail="M4.6 camera decoder недоступен") + projected = timeline(result_id) + if not 0 <= sequence < len(projected.index.source_times_ns): + raise HTTPException(status_code=404, detail="M4.6 timeline frame не найден") + return _camera_response(camera_frame_provider, projected.profile.session_id, sequence) return router @@ -183,53 +202,16 @@ def _read_threat_result_cached( @lru_cache(maxsize=4) -def _cached_video_overlay( - result_id: str, +def _read_threat_timeline_cached( + repository_root_value: str, root_value: str, - frames_sha256: str, - source_session_id: str, -) -> dict[str, object]: - root = Path(root_value).resolve(strict=True) - if root.is_symlink() or not root.is_dir() or len(frames_sha256) != 64: - raise ValueError("M4.6 video evidence identity changed") - frames = [] - for expected_sequence, row in enumerate(_iter_jsonl(root / "frames.jsonl")): - if ( - row.get("schema_version") - not in {THREAT_REPLAY_FRAME_SCHEMA, THREAT_REPLAY_FRAME_SCHEMA_V2} - or row.get("sequence") != expected_sequence - ): - raise ValueError("M4.6 video frame order changed") - frames.append( - { - "frame_index": expected_sequence, - "session_seconds": _nonnegative_int(row.get("source_time_ns"), "source time") - / 1_000_000_000, - "source_available": row["source_available"], - "camera_proposals": copy.deepcopy(row["camera_proposals"]), - "decision_counts": _decision_counts(_array(row.get("assessments"))), - } - ) - if len(frames) != 4489: - raise ValueError("M4.6 video frame coverage changed") - return { - "schema_version": M4_THREAT_VIDEO_SCHEMA, - "result_id": result_id, - "recorded_source": { - "session_id": source_session_id, - "source_id": "sensor.camera.right", - "synchronization": "host-arrival-best-effort", - }, - "image_width": 800, - "image_height": 600, - "timeline_start_seconds": frames[0]["session_seconds"], - "timeline_end_seconds": frames[-1]["session_seconds"], - "frame_count": len(frames), - "frames": frames, - "ground_truth": False, - "authority": "replay-simulated", - "access": "read-only-replay-simulated-video", - } + signature: tuple[int, ...], +) -> RecordedThreatTimeline: + result = _read_threat_result_cached(root_value, signature) + return RecordedThreatTimeline( + repository_root=Path(repository_root_value), + result=result, + ) def _project_result(result: ThreatReplayResult) -> dict[str, object]: @@ -261,14 +243,32 @@ def _project_result(result: ThreatReplayResult) -> dict[str, object]: } -def _decision_counts(raw: list[object]) -> dict[str, int]: - result = {"threat": 0, "not-threat": 0, "unknown": 0} - for item in raw: - assessment = item if isinstance(item, dict) else {} - decision = assessment.get("decision") - if isinstance(decision, str) and decision in result: - result[decision] += 1 - return result +def _camera_response( + provider: CameraFrameProvider, + session_id: str, + sequence: int, +) -> Response: + try: + camera = provider(session_id, sequence) + except (OSError, SessionIntegrityError, ValueError): + raise HTTPException( + status_code=503, + detail="M4.6 exact camera frame недоступен", + ) from None + if camera.width != 800 or camera.height != 600: + raise HTTPException( + status_code=503, + detail="M4.6 camera frame нарушил размерный контракт", + ) + return Response( + content=camera.payload, + media_type=camera.media_type, + headers={ + "Cache-Control": "private, max-age=31536000, immutable", + "ETag": f'"{camera.sha256}"', + "X-Content-Type-Options": "nosniff", + }, + ) def _configured_root(provider: RootProvider) -> Path | None: @@ -343,15 +343,8 @@ def _array(value: object) -> list[object]: return value -def _nonnegative_int(value: object, label: str) -> int: - if not isinstance(value, int) or isinstance(value, bool) or value < 0: - raise ValueError(f"M4.6 {label} is invalid") - return value - - __all__ = [ "M4_THREAT_CATALOG_SCHEMA", - "M4_THREAT_VIDEO_SCHEMA", "M4_THREAT_VIEW_SCHEMA", "M4_THREAT_VISUAL_CATALOG_SCHEMA", "build_m4_threat_replay_router", diff --git a/tests/test_m4_threat_replay_result.py b/tests/test_m4_threat_replay_result.py index 417cb16..034fa00 100644 --- a/tests/test_m4_threat_replay_result.py +++ b/tests/test_m4_threat_replay_result.py @@ -16,6 +16,7 @@ RESULTS_ROOT = REPOSITORY_ROOT / ".runtime/compute-experiments/m4/replay-threat" def _endpoint(path: str, *, camera_frame_provider=None): router = build_m4_threat_replay_router( root_provider=lambda: RESULTS_ROOT, + repository_root_provider=lambda: REPOSITORY_ROOT, camera_frame_provider=camera_frame_provider, ) return next( @@ -120,16 +121,28 @@ def test_m4_6_lab_api_projects_report_and_exact_visual_frame() -> None: } -def test_m4_6_video_overlay_covers_the_exact_recorded_camera_timeline() -> None: - get_overlay = _endpoint("/api/v1/laboratory/m4-threat/results/{result_id}/video-overlay") +def test_m4_6_timeline_is_indexed_and_spatial_evidence_is_chunked() -> None: + get_timeline = _endpoint("/api/v1/laboratory/m4-threat/results/{result_id}/timeline") + get_chunk = _endpoint("/api/v1/laboratory/m4-threat/results/{result_id}/timeline/chunk") - overlay = get_overlay(RESULT_ID) + timeline = get_timeline(RESULT_ID) + assert timeline["schema_version"] == "missioncore.recorded-spatial-evidence-timeline/v1" + assert timeline["frame_count"] == 4489 + assert len(timeline["frame_times_ns"]) == 4489 + assert timeline["recorded_source"]["session_id"] == "20260720T065719Z_viewer_live" + assert "frames" not in timeline - assert overlay["frame_count"] == 4489 - assert overlay["recorded_source"]["session_id"] == ("20260720T065719Z_viewer_live") - assert overlay["frames"][0]["frame_index"] == 0 - assert overlay["frames"][-1]["frame_index"] == 4488 - assert overlay["authority"] == "replay-simulated" + chunk = get_chunk(RESULT_ID, start=1880, count=12) + assert chunk["schema_version"] == "missioncore.recorded-spatial-evidence-chunk/v1" + assert chunk["start_sequence"] == 1880 + assert chunk["frame_count"] == 12 + assert [frame["sequence"] for frame in chunk["frames"]] == list(range(1880, 1892)) + first = chunk["frames"][0] + assert first["schema_version"] == "missioncore.recorded-spatial-evidence-frame/v1" + assert first["spatial_available"] is True + assert first["point_cloud_layer"] == "current-increment" + assert 0 < first["point_cloud_sample_count"] <= 2000 + assert first["camera_url"].endswith(f"/{RESULT_ID}/timeline/frames/1880/camera") def test_m4_6_exact_camera_endpoint_is_bound_to_selected_visual_sequence() -> None: @@ -155,3 +168,26 @@ def test_m4_6_exact_camera_endpoint_is_bound_to_selected_visual_sequence() -> No assert response.media_type == "image/jpeg" assert response.headers["etag"] == f'"{"a" * 64}"' assert calls == [("20260720T065719Z_viewer_live", 2584)] + + +def test_m4_6_timeline_camera_endpoint_is_bound_to_exact_sequence() -> None: + calls: list[tuple[str, int]] = [] + + def provide(session_id: str, frame_index: int) -> RecordedCameraFrame: + calls.append((session_id, frame_index)) + return RecordedCameraFrame( + payload=b"timeline-jpeg", + media_type="image/jpeg", + width=800, + height=600, + sha256="b" * 64, + ) + + endpoint = _endpoint( + "/api/v1/laboratory/m4-threat/results/{result_id}/timeline/frames/{sequence}/camera", + camera_frame_provider=provide, + ) + response = endpoint(RESULT_ID, 2584) + + assert response.body == b"timeline-jpeg" + assert calls == [("20260720T065719Z_viewer_live", 2584)]