From aacc6dc43ba16a7fe63ff2058ccc625ea57d449c Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Wed, 5 Aug 2026 22:16:33 +0300 Subject: [PATCH] fix(m4): retain compact obstacle evidence --- .../laboratory/LaboratoryEvidenceViewer.tsx | 7 +- .../laboratory/RecordedEvidenceBoxOverlay.tsx | 110 ++++++ .../laboratory/RecordedEvidenceImageScene.tsx | 58 ++++ .../laboratory/RecordedEvidenceVideoScene.tsx | 112 +------ .../src/core/laboratory/m4ReplayThreat.ts | 2 + .../styles/l3-pointpillars-visual-audit.css | 28 +- .../laboratory/M4ReplayThreatResult.tsx | 10 +- .../laboratory/M4ReplayThreatVisual.tsx | 40 ++- .../test/m4ReplayThreat.test.mjs | 14 +- config/laboratory-value-review.json | 2 +- .../m4-geometry-association-v1.json | 2 +- config/perception/m4-replay-threat-v3.json | 8 +- config/perception/m4-temporal-motion-v1.json | 4 +- ...E_4_OBJECT_CENTRIC_RECORDED_REALTIME_CV.md | 62 ++-- ...d-map-increment-rolling-local-occupancy.md | 32 +- src/k1link/perception/temporal_replay.py | 145 +++----- src/k1link/perception/threat_replay.py | 194 ++++++++--- src/k1link/sessions/__init__.py | 3 + src/k1link/sessions/camera_frame.py | 314 ++++++++++++++++++ src/k1link/web/app.py | 38 ++- src/k1link/web/m4_threat_replay_api.py | 63 +++- tests/test_geometry_replay_result.py | 19 +- tests/test_m4_threat_replay_result.py | 107 +++--- tests/test_temporal_replay_result.py | 17 +- 24 files changed, 981 insertions(+), 410 deletions(-) create mode 100644 apps/control-station/src/components/laboratory/RecordedEvidenceBoxOverlay.tsx create mode 100644 apps/control-station/src/components/laboratory/RecordedEvidenceImageScene.tsx create mode 100644 src/k1link/sessions/camera_frame.py diff --git a/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx b/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx index 5b5af5d..1d18e2e 100644 --- a/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx +++ b/apps/control-station/src/components/laboratory/LaboratoryEvidenceViewer.tsx @@ -20,6 +20,7 @@ export function LaboratoryEvidenceViewer< U extends string = string, >({ label, + className, mode, modes, expanded, @@ -31,6 +32,7 @@ export function LaboratoryEvidenceViewer< children, }: { label: string; + className?: string; mode: T; modes: readonly LaboratoryEvidenceViewerMode[]; expanded: boolean; @@ -62,7 +64,10 @@ export function LaboratoryEvidenceViewer< const viewer = (
diff --git a/apps/control-station/src/components/laboratory/RecordedEvidenceBoxOverlay.tsx b/apps/control-station/src/components/laboratory/RecordedEvidenceBoxOverlay.tsx new file mode 100644 index 0000000..c8a1065 --- /dev/null +++ b/apps/control-station/src/components/laboratory/RecordedEvidenceBoxOverlay.tsx @@ -0,0 +1,110 @@ +import { useEffect, useRef } from "react"; + +export type RecordedEvidenceBoxTone = + | "accent" + | "danger" + | "success" + | "warning" + | "neutral"; + +export interface RecordedEvidenceBox { + boxXyxy: readonly [number, number, number, number]; + label: string; + tone: RecordedEvidenceBoxTone; + dashed?: boolean; +} + +function rgba( + host: HTMLElement, + token: string, + fallback: readonly [number, number, number], + alpha = 1, +): string { + const channels = getComputedStyle(host) + .getPropertyValue(token) + .trim() + .match(/[\d.]+/g) + ?.slice(0, 3) + .map(Number); + const [red, green, blue] = channels?.length === 3 ? channels : fallback; + return `rgba(${red}, ${green}, ${blue}, ${alpha})`; +} + +function toneColor(host: HTMLElement, tone: RecordedEvidenceBoxTone): string { + if (tone === "danger") return rgba(host, "--nodedc-danger-rgb", [255, 104, 112]); + if (tone === "success") return rgba(host, "--nodedc-success-rgb", [181, 255, 90]); + if (tone === "warning") return rgba(host, "--nodedc-warning-rgb", [255, 197, 92]); + if (tone === "neutral") return rgba(host, "--nodedc-foreground-rgb", [245, 245, 245], 0.7); + return rgba(host, "--nodedc-accent-rgb", [232, 56, 126]); +} + +export function RecordedEvidenceBoxOverlay({ + imageWidth, + imageHeight, + boxes, + ariaLabel, +}: { + imageWidth: number; + imageHeight: number; + boxes: readonly RecordedEvidenceBox[]; + ariaLabel: string; +}) { + const canvasRef = useRef(null); + + useEffect(() => { + const canvas = canvasRef.current; + const host = canvas?.parentElement; + if (!host || !canvas) return; + const context = canvas.getContext("2d"); + if (!context) return; + + const render = () => { + const width = Math.max(host.clientWidth, 1); + const height = Math.max(host.clientHeight, 1); + const pixelRatio = Math.min(window.devicePixelRatio, 1.5); + canvas.width = Math.round(width * pixelRatio); + canvas.height = Math.round(height * pixelRatio); + canvas.style.width = `${width}px`; + canvas.style.height = `${height}px`; + context.setTransform(pixelRatio, 0, 0, pixelRatio, 0, 0); + context.clearRect(0, 0, width, height); + + const scale = Math.min(width / imageWidth, height / imageHeight); + const drawWidth = imageWidth * scale; + const drawHeight = imageHeight * scale; + const offsetX = (width - drawWidth) / 2; + const offsetY = (height - drawHeight) / 2; + for (const item of boxes) { + const [left, top, right, bottom] = item.boxXyxy; + const x = offsetX + left * scale; + const y = offsetY + top * scale; + const boxWidth = (right - left) * scale; + const boxHeight = (bottom - top) * scale; + const stroke = toneColor(host, item.tone); + context.strokeStyle = stroke; + context.lineWidth = Math.max(1.5, 2 * scale); + context.setLineDash(item.dashed ? [5, 4] : []); + context.strokeRect(x, y, boxWidth, boxHeight); + context.setLineDash([]); + + const fontSize = Math.max(9, 10 * scale); + context.font = `650 ${fontSize}px Inter, system-ui, sans-serif`; + const labelWidth = Math.min(drawWidth, context.measureText(item.label).width + 8); + const labelHeight = fontSize + 6; + const labelX = Math.min(offsetX + drawWidth - labelWidth, Math.max(offsetX, x)); + const labelY = Math.max(offsetY, y - labelHeight); + context.fillStyle = rgba(host, "--nodedc-canvas-rgb", [5, 5, 6], 0.9); + context.fillRect(labelX, labelY, labelWidth, labelHeight); + context.fillStyle = stroke; + context.fillText(item.label, labelX + 4, labelY + fontSize + 1, labelWidth - 8); + } + }; + + const observer = new ResizeObserver(render); + observer.observe(host); + render(); + return () => observer.disconnect(); + }, [boxes, imageHeight, imageWidth]); + + return ; +} diff --git a/apps/control-station/src/components/laboratory/RecordedEvidenceImageScene.tsx b/apps/control-station/src/components/laboratory/RecordedEvidenceImageScene.tsx new file mode 100644 index 0000000..cc30a5c --- /dev/null +++ b/apps/control-station/src/components/laboratory/RecordedEvidenceImageScene.tsx @@ -0,0 +1,58 @@ +import { useEffect, useState } from "react"; +import { Icon } from "@nodedc/ui-react"; + +import { + RecordedEvidenceBoxOverlay, + type RecordedEvidenceBox, +} from "./RecordedEvidenceBoxOverlay"; + +export function RecordedEvidenceImageScene({ + src, + imageWidth, + imageHeight, + boxes, + ariaLabel, +}: { + src: string; + imageWidth: number; + imageHeight: number; + boxes: readonly RecordedEvidenceBox[]; + ariaLabel: string; +}) { + const [state, setState] = useState<"loading" | "ready" | "error">("loading"); + + useEffect(() => setState("loading"), [src]); + + return ( +
+ setState("ready")} + onError={() => setState("error")} + /> + {state === "ready" ? ( + + ) : ( +
+ {state === "loading" ? ( +
+ )} +
+ ); +} diff --git a/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx b/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx index e0ba202..e971df6 100644 --- a/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx +++ b/apps/control-station/src/components/laboratory/RecordedEvidenceVideoScene.tsx @@ -1,48 +1,15 @@ -import { useEffect, useRef } from "react"; - import { RecordedFmp4Player, type RecordedObservationPlayback, } from "../RecordedFmp4Player"; import type { ObservationSourceDescriptor } from "../../core/runtime/contracts"; +import { + RecordedEvidenceBoxOverlay, + type RecordedEvidenceBox, + type RecordedEvidenceBoxTone, +} from "./RecordedEvidenceBoxOverlay"; -export type RecordedEvidenceBoxTone = - | "accent" - | "danger" - | "success" - | "warning" - | "neutral"; - -export interface RecordedEvidenceBox { - boxXyxy: readonly [number, number, number, number]; - label: string; - tone: RecordedEvidenceBoxTone; - dashed?: boolean; -} - -function rgba( - host: HTMLElement, - token: string, - fallback: readonly [number, number, number], - alpha = 1, -): string { - const channels = getComputedStyle(host) - .getPropertyValue(token) - .trim() - .match(/[\d.]+/g) - ?.slice(0, 3) - .map(Number); - const [red, green, blue] = channels?.length === 3 ? channels : fallback; - return `rgba(${red}, ${green}, ${blue}, ${alpha})`; -} - -function toneColor(host: HTMLElement, tone: RecordedEvidenceBoxTone): string { - if (tone === "danger") return rgba(host, "--nodedc-danger-rgb", [255, 104, 112]); - if (tone === "success") return rgba(host, "--nodedc-success-rgb", [181, 255, 90]); - if (tone === "warning") return rgba(host, "--nodedc-warning-rgb", [255, 197, 92]); - if (tone === "neutral") return rgba(host, "--nodedc-foreground-rgb", [245, 245, 245], 0.7); - return rgba(host, "--nodedc-accent-rgb", [232, 56, 126]); -} +export type { RecordedEvidenceBox, RecordedEvidenceBoxTone }; export function RecordedEvidenceVideoScene({ source, @@ -61,66 +28,8 @@ export function RecordedEvidenceVideoScene({ ariaLabel: string; onPlaybackChange: (playback: RecordedObservationPlayback) => void; }) { - const hostRef = useRef(null); - const canvasRef = useRef(null); - - useEffect(() => { - const host = hostRef.current; - const canvas = canvasRef.current; - if (!host || !canvas) return; - const context = canvas.getContext("2d"); - if (!context) return; - - const render = () => { - const width = Math.max(host.clientWidth, 1); - const height = Math.max(host.clientHeight, 1); - const pixelRatio = Math.min(window.devicePixelRatio, 1.5); - canvas.width = Math.round(width * pixelRatio); - canvas.height = Math.round(height * pixelRatio); - canvas.style.width = `${width}px`; - canvas.style.height = `${height}px`; - context.setTransform(pixelRatio, 0, 0, pixelRatio, 0, 0); - context.clearRect(0, 0, width, height); - - const scale = Math.min(width / imageWidth, height / imageHeight); - const drawWidth = imageWidth * scale; - const drawHeight = imageHeight * scale; - const offsetX = (width - drawWidth) / 2; - const offsetY = (height - drawHeight) / 2; - for (const item of boxes) { - const [left, top, right, bottom] = item.boxXyxy; - const x = offsetX + left * scale; - const y = offsetY + top * scale; - const boxWidth = (right - left) * scale; - const boxHeight = (bottom - top) * scale; - const stroke = toneColor(host, item.tone); - context.strokeStyle = stroke; - context.lineWidth = Math.max(1.5, 2 * scale); - context.setLineDash(item.dashed ? [5, 4] : []); - context.strokeRect(x, y, boxWidth, boxHeight); - context.setLineDash([]); - - const fontSize = Math.max(9, 10 * scale); - context.font = `650 ${fontSize}px Inter, system-ui, sans-serif`; - const labelWidth = Math.min(drawWidth, context.measureText(item.label).width + 8); - const labelHeight = fontSize + 6; - const labelX = Math.min(offsetX + drawWidth - labelWidth, Math.max(offsetX, x)); - const labelY = Math.max(offsetY, y - labelHeight); - context.fillStyle = rgba(host, "--nodedc-canvas-rgb", [5, 5, 6], 0.9); - context.fillRect(labelX, labelY, labelWidth, labelHeight); - context.fillStyle = stroke; - context.fillText(item.label, labelX + 4, labelY + fontSize + 1, labelWidth - 8); - } - }; - - const observer = new ResizeObserver(render); - observer.observe(host); - render(); - return () => observer.disconnect(); - }, [boxes, imageHeight, imageWidth]); - return ( -
+
- +
); } diff --git a/apps/control-station/src/core/laboratory/m4ReplayThreat.ts b/apps/control-station/src/core/laboratory/m4ReplayThreat.ts index 6c6039d..3ec6c85 100644 --- a/apps/control-station/src/core/laboratory/m4ReplayThreat.ts +++ b/apps/control-station/src/core/laboratory/m4ReplayThreat.ts @@ -90,6 +90,7 @@ export interface M4ThreatCameraProposal { export interface M4ThreatVisualFrame { resultId: string; + cameraUrl: string; ordinal: number; sequence: number; frameId: string; @@ -408,6 +409,7 @@ export async function fetchM4ThreatVisual( const corridor = object(item.corridor, "M4.6 visual corridor"); return { resultId: result, + cameraUrl: text(item.camera_url, "M4.6 camera URL"), ordinal: integer(item.ordinal, "M4.6 ordinal"), sequence: integer(item.sequence, "M4.6 sequence"), frameId: text(item.frame_id, "M4.6 frame id"), diff --git a/apps/control-station/src/styles/l3-pointpillars-visual-audit.css b/apps/control-station/src/styles/l3-pointpillars-visual-audit.css index e3bdccd..223322f 100644 --- a/apps/control-station/src/styles/l3-pointpillars-visual-audit.css +++ b/apps/control-station/src/styles/l3-pointpillars-visual-audit.css @@ -46,7 +46,8 @@ user-select: none; } -.recorded-evidence-video-scene { +.recorded-evidence-video-scene, +.recorded-evidence-image-scene { position: relative; width: 100%; height: 100%; @@ -55,12 +56,23 @@ background: var(--nodedc-canvas); } +.recorded-evidence-image-scene > img { + position: absolute; + inset: 0; + display: block; + width: 100%; + height: 100%; + object-fit: contain; + user-select: none; +} + .recorded-evidence-video-scene > .recorded-media-player { position: absolute; inset: 0; } -.recorded-evidence-video-scene > canvas { +.recorded-evidence-video-scene > canvas, +.recorded-evidence-image-scene > canvas { position: absolute; z-index: 2; inset: 0; @@ -70,6 +82,18 @@ pointer-events: none; } +.recorded-evidence-image-scene > .l3-visual-audit__state { + position: absolute; + z-index: 3; + inset: 0; +} + +.m4-replay-threat-evidence-viewer .laboratory-metric-evidence-scene__toolbar { + top: 5.4rem; + max-width: calc(100% - 1.2rem); + flex-wrap: wrap; +} + .e46e-ready-stack-video { width: 100%; height: 100%; diff --git a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx index 1ee2408..bb9f7df 100644 --- a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx +++ b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatResult.tsx @@ -26,7 +26,7 @@ export function M4ReplayThreatResultView({ { - if ((mode !== "video" && mode !== "camera") || (videoOverlay && videoSource)) return; + if (mode !== "video" || (videoOverlay && videoSource)) return; const controller = new AbortController(); setVideoLoading(true); setVideoError(null); @@ -157,14 +158,6 @@ export function M4ReplayThreatVisual({ resultId }: { resultId: string }) { return () => controller.abort(); }, [mode, resultId, videoOverlay, videoSource]); - useEffect(() => { - if (mode !== "camera" || !frame) return; - setVideoPlayback({ - currentSeconds: frame.sourceTimeNs / 1_000_000_000, - playing: false, - }); - }, [frame, mode]); - const activeVideoFrame = useMemo( () => videoOverlay ? selectM4ThreatVideoFrame(videoOverlay.frames, videoPlayback.currentSeconds) @@ -317,7 +310,7 @@ export function M4ReplayThreatVisual({ resultId }: { resultId: string }) { ) : undefined; let content; - if (mode === "video" || mode === "camera") { + if (mode === "video") { content = videoLoading ? (