diff --git a/apps/control-station/src/App.tsx b/apps/control-station/src/App.tsx index abdd0f7..014eeeb 100644 --- a/apps/control-station/src/App.tsx +++ b/apps/control-station/src/App.tsx @@ -72,8 +72,8 @@ import { } from "./sceneSettings"; import { DeviceWorkspace } from "./workspaces/DeviceWorkspace"; import { WorkspaceRenderer } from "./workspaces/Workspaces"; +import { useLaboratoryAnnotationHeader } from "./components/laboratory/useLaboratoryAnnotationHeader"; import "./styles/scene-windows.css"; - type SceneToolWindowId = "sources" | "display" | "layers"; const viewerSettingsQuietPeriodMs = 750; @@ -179,6 +179,7 @@ export default function App() { const [layerInspectorOpen, setLayerInspectorOpen] = useState(false); const [sceneWindowOrder, setSceneWindowOrder] = useState([]); const [layoutSaveNotice, setLayoutSaveNotice] = useState(null); + const laboratoryAnnotation = useLaboratoryAnnotationHeader(); const [sceneSettings, setSceneSettings] = useState(defaultSceneSettings); const [displayDraft, setDisplayDraft] = useState(defaultSceneSettings); const [livePerceptionLayers, setLivePerceptionLayers] = useState( @@ -811,7 +812,7 @@ export default function App() { ) : activeDefinition.kind === "datasets" ? ( Offline evaluation ) : activeDefinition.kind === "lab-archive" ? ( - null + laboratoryAnnotation.control ) : activeDefinition.root === "system" ? ( void; + onPlaybackChange?: (playback: RecordedObservationPlayback) => void; }) { const videoRef = useRef(null); const onAdmissionChangeRef = useRef(onAdmissionChange); onAdmissionChangeRef.current = onAdmissionChange; + const onPlaybackChangeRef = useRef(onPlaybackChange); + onPlaybackChangeRef.current = onPlaybackChange; const workerRef = useRef<{ admissionKey: string | null; generation: number } | null>(null); if (!workerRef.current || workerRef.current.admissionKey !== admissionKey) { recordedMediaWorkerGeneration += 1; @@ -461,15 +467,49 @@ export function RecordedFmp4Player({ } }, [archive?.byteLength, bufferRevision, currentSeconds, epoch, playback?.playing, visualState]); + useEffect(() => { + const video = videoRef.current; + if (!interactive || !video || !epoch || visualState !== "ready") return; + let videoFrameRequest: number | null = null; + const emitPlayback = () => { + onPlaybackChangeRef.current?.({ + currentSeconds: epoch.timelineStartSeconds + video.currentTime, + playing: !video.paused && !video.ended, + }); + }; + const scheduleVideoFrame = () => { + if (typeof video.requestVideoFrameCallback !== "function") return; + videoFrameRequest = video.requestVideoFrameCallback(() => { + emitPlayback(); + scheduleVideoFrame(); + }); + }; + const events = ["play", "pause", "seeking", "seeked", "timeupdate", "ended"] as const; + for (const event of events) video.addEventListener(event, emitPlayback); + scheduleVideoFrame(); + emitPlayback(); + return () => { + for (const event of events) video.removeEventListener(event, emitPlayback); + if ( + videoFrameRequest !== null && + typeof video.cancelVideoFrameCallback === "function" + ) { + video.cancelVideoFrameCallback(videoFrameRequest); + } + }; + }, [epoch, interactive, visualState]); + return (
+ {annotationWorkspace} ); } diff --git a/apps/control-station/src/workspaces/laboratory/annotation/L34AnnotationCanvas.tsx b/apps/control-station/src/workspaces/laboratory/annotation/L34AnnotationCanvas.tsx new file mode 100644 index 0000000..7078fc3 --- /dev/null +++ b/apps/control-station/src/workspaces/laboratory/annotation/L34AnnotationCanvas.tsx @@ -0,0 +1,488 @@ +import { + useEffect, + useRef, + useState, + type PointerEvent as ReactPointerEvent, +} from "react"; +import { Select } from "@nodedc/ui-react"; + +import { + L34_ANNOTATION_CLASS_OPTIONS, + L34_ANNOTATION_UNMAPPED_OPTION, + annotationOperationKey, + type L34AnnotationCategory, + type L34AnnotationObject, + type L34AnnotationSourceFrame, +} from "../../../core/laboratory/l34Annotation"; + +export interface L34AnnotationObjectDraft + extends Omit { + category: L34AnnotationCategory | null; + origin: string; +} + +export interface L34AnnotationComparisonObject { + objectId: string; + category: string; + boxXyxy: readonly [number, number, number, number]; +} + +export type L34AnnotationLayerMode = "overlay" | "candidate" | "review" | "source"; + +type Point = Readonly<{ x: number; y: number }>; +type PlaneSize = Readonly<{ width: number; height: number }>; +type ResizeHandle = "nw" | "ne" | "sw" | "se"; +type Interaction = + | Readonly<{ + kind: "draw"; + pointerId: number; + start: Point; + end: Point; + }> + | Readonly<{ + kind: "move" | "resize"; + pointerId: number; + objectId: string; + start: Point; + originalBox: readonly [number, number, number, number]; + handle?: ResizeHandle; + }>; + +function boundedPoint( + clientX: number, + clientY: number, + svg: SVGSVGElement, + frame: L34AnnotationSourceFrame, +): Point { + const bounds = svg.getBoundingClientRect(); + const x = (clientX - bounds.left) * frame.cameraWidth / bounds.width; + const y = (clientY - bounds.top) * frame.cameraHeight / bounds.height; + return { + x: Math.max(0, Math.min(frame.cameraWidth, x)), + y: Math.max(0, Math.min(frame.cameraHeight, y)), + }; +} + +function movedBox( + original: readonly [number, number, number, number], + start: Point, + end: Point, + frame: L34AnnotationSourceFrame, +): readonly [number, number, number, number] { + const width = original[2] - original[0]; + const height = original[3] - original[1]; + const left = Math.max( + 0, + Math.min(frame.cameraWidth - width, original[0] + end.x - start.x), + ); + const top = Math.max( + 0, + Math.min(frame.cameraHeight - height, original[1] + end.y - start.y), + ); + return [left, top, left + width, top + height]; +} + +function resizedBox( + original: readonly [number, number, number, number], + handle: ResizeHandle, + point: Point, +): readonly [number, number, number, number] { + let [left, top, right, bottom] = original; + if (handle.includes("n")) top = Math.min(point.y, bottom - 4); + if (handle.includes("s")) bottom = Math.max(point.y, top + 4); + if (handle.includes("w")) left = Math.min(point.x, right - 4); + if (handle.includes("e")) right = Math.max(point.x, left + 4); + return [left, top, right, bottom]; +} + +function boxesNearlyEqual( + left: readonly [number, number, number, number], + right: readonly [number, number, number, number], +): boolean { + return left.every((value, index) => Math.abs(value - right[index]) < 0.5); +} + +function customLabelValue(label: string): string { + return `custom:${encodeURIComponent(label)}`; +} + +function objectLabel(object: L34AnnotationObjectDraft): string { + if (object.category === null) return "Выберите класс"; + if (object.category === "unmapped") { + return object.proposedLabel ?? L34_ANNOTATION_UNMAPPED_OPTION.label; + } + return L34_ANNOTATION_CLASS_OPTIONS.find( + ({ value }) => value === object.category, + )?.label ?? object.category; +} + +function boxFromPoints( + start: Point, + end: Point, +): readonly [number, number, number, number] { + return [ + Math.min(start.x, end.x), + Math.min(start.y, end.y), + Math.max(start.x, end.x), + Math.max(start.y, end.y), + ]; +} + +export function L34AnnotationCanvas({ + frame, + objects, + drawingEnabled, + selectedObjectId, + unmappedLabels, + allowUnmapped = true, + comparisonObjects = [], + layerMode = "review", + newObjectOrigin = "manual", + onObjectsChange, + onSelectedObjectIdChange, + onRequestUnmappedLabel, +}: { + frame: L34AnnotationSourceFrame; + objects: readonly L34AnnotationObjectDraft[]; + drawingEnabled: boolean; + selectedObjectId: string | null; + unmappedLabels: readonly string[]; + allowUnmapped?: boolean; + comparisonObjects?: readonly L34AnnotationComparisonObject[]; + layerMode?: L34AnnotationLayerMode; + newObjectOrigin?: string; + onObjectsChange: (objects: readonly L34AnnotationObjectDraft[]) => void; + onSelectedObjectIdChange: (objectId: string | null) => void; + onRequestUnmappedLabel: (objectId: string) => void; +}) { + const stageRef = useRef(null); + const [planeSize, setPlaneSize] = useState({ width: 1, height: 1 }); + const [interaction, setInteraction] = useState(null); + + useEffect(() => { + const host = stageRef.current; + if (!host) return; + const measure = () => { + const width = Math.max(host.clientWidth, 1); + const height = Math.max(host.clientHeight, 1); + const scale = Math.min( + width / frame.cameraWidth, + height / frame.cameraHeight, + ); + setPlaneSize({ + width: Math.max(frame.cameraWidth * scale, 1), + height: Math.max(frame.cameraHeight * scale, 1), + }); + }; + const observer = new ResizeObserver(measure); + observer.observe(host); + measure(); + return () => observer.disconnect(); + }, [frame.cameraHeight, frame.cameraWidth]); + + useEffect(() => setInteraction(null), [frame.truthIslandSequence]); + + const updateObject = ( + objectId: string, + patch: Partial, + ) => { + let changed = false; + const next = objects.map((object) => { + if (object.objectId !== objectId) return object; + const keys = Object.keys(patch) as Array; + const scalarChanged = keys.some((key) => ( + key !== "boxXyxy" && object[key] !== patch[key] + )); + const boxChanged = patch.boxXyxy !== undefined + && !boxesNearlyEqual(object.boxXyxy, patch.boxXyxy); + if (!scalarChanged && !boxChanged) return object; + changed = true; + return { ...object, ...patch }; + }); + if (changed) onObjectsChange(next); + }; + + const finishInteraction = (event: ReactPointerEvent) => { + if (!interaction || interaction.pointerId !== event.pointerId) return; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + const end = boundedPoint( + event.clientX, + event.clientY, + event.currentTarget, + frame, + ); + if (interaction.kind !== "draw") { + const box = interaction.kind === "move" + ? movedBox(interaction.originalBox, interaction.start, end, frame) + : resizedBox( + interaction.originalBox, + interaction.handle ?? "se", + end, + ); + updateObject(interaction.objectId, { boxXyxy: box }); + setInteraction(null); + return; + } + const box = boxFromPoints(interaction.start, end); + setInteraction(null); + if (box[2] - box[0] < 4 || box[3] - box[1] < 4) return; + const objectId = annotationOperationKey("box").replace(":", "-"); + onObjectsChange([ + ...objects, + { + objectId, + category: null, + proposedLabel: null, + origin: newObjectOrigin, + boxXyxy: box, + occluded: false, + truncated: false, + }, + ]); + onSelectedObjectIdChange(objectId); + }; + + const humanVisible = layerMode === "overlay" || layerMode === "review"; + const candidateVisible = layerMode === "overlay" || layerMode === "candidate"; + const humanLabelsVisible = layerMode === "review"; + const candidateLabelsVisible = layerMode === "candidate"; + const editable = humanVisible; + const effectiveDrawingEnabled = drawingEnabled && editable; + const draftBox = interaction?.kind === "draw" + ? boxFromPoints(interaction.start, interaction.end) + : null; + const selectedObject = objects.find( + ({ objectId }) => objectId === selectedObjectId, + ) ?? null; + return ( +
+
+ {`Исходный + { + if (!effectiveDrawingEnabled || event.button !== 0) return; + const point = boundedPoint( + event.clientX, + event.clientY, + event.currentTarget, + frame, + ); + event.currentTarget.setPointerCapture(event.pointerId); + setInteraction({ + kind: "draw", + pointerId: event.pointerId, + start: point, + end: point, + }); + onSelectedObjectIdChange(null); + }} + onPointerMove={(event) => { + if (!interaction || interaction.pointerId !== event.pointerId) return; + const point = boundedPoint( + event.clientX, + event.clientY, + event.currentTarget, + frame, + ); + if (interaction.kind === "draw") { + setInteraction({ ...interaction, end: point }); + return; + } + const box = interaction.kind === "move" + ? movedBox(interaction.originalBox, interaction.start, point, frame) + : resizedBox( + interaction.originalBox, + interaction.handle ?? "se", + point, + ); + updateObject(interaction.objectId, { boxXyxy: box }); + }} + onPointerUp={finishInteraction} + onPointerCancel={() => setInteraction(null)} + > + {candidateVisible ? comparisonObjects.map((object) => { + const [left, top, right, bottom] = object.boxXyxy; + return ( + + ); + }) : null} + {humanVisible ? objects.map((object) => { + const [left, top, right, bottom] = object.boxXyxy; + return ( + { + if (effectiveDrawingEnabled || event.button !== 0) return; + event.stopPropagation(); + const svg = event.currentTarget.ownerSVGElement; + if (!svg) return; + const point = boundedPoint( + event.clientX, + event.clientY, + svg, + frame, + ); + svg.setPointerCapture(event.pointerId); + onSelectedObjectIdChange(object.objectId); + setInteraction({ + kind: "move", + pointerId: event.pointerId, + objectId: object.objectId, + start: point, + originalBox: object.boxXyxy, + }); + }} + /> + ); + }) : null} + {draftBox ? ( + + ) : null} + {selectedObject && editable && !effectiveDrawingEnabled ? ([ + ["nw", selectedObject.boxXyxy[0], selectedObject.boxXyxy[1]], + ["ne", selectedObject.boxXyxy[2], selectedObject.boxXyxy[1]], + ["sw", selectedObject.boxXyxy[0], selectedObject.boxXyxy[3]], + ["se", selectedObject.boxXyxy[2], selectedObject.boxXyxy[3]], + ] as const).map(([handle, x, y]) => ( + { + if (event.button !== 0) return; + event.stopPropagation(); + const svg = event.currentTarget.ownerSVGElement; + if (!svg) return; + svg.setPointerCapture(event.pointerId); + setInteraction({ + kind: "resize", + pointerId: event.pointerId, + objectId: selectedObject.objectId, + start: boundedPoint( + event.clientX, + event.clientY, + svg, + frame, + ), + originalBox: selectedObject.boxXyxy, + handle, + }); + }} + /> + )) : null} + + {candidateLabelsVisible ? comparisonObjects.map((object) => { + const [left, top] = object.boxXyxy; + return ( +
+ Candidate · {object.category} +
+ ); + }) : null} + {humanLabelsVisible ? objects.map((object) => { + const [left, top] = object.boxXyxy; + const below = top < 42; + const selected = object.objectId === selectedObjectId; + return ( +
onSelectedObjectIdChange(object.objectId)} + > + {selected ? ({ + value: item.sessionId, + label: sessionOptionLabel(item), + })) : [{ value: "", label: "Сессия не создана", disabled: true }]} + disabled={busy} + searchable + searchPlaceholder="Найти сессию" + menuWidth={360} + minMenuWidth={280} + onChange={(value) => void selectSession(value)} + /> + = 2)} + onClick={() => void createSession()} + > + + + + + {independentBlind ? ( + + ) : null} +
+
+ navigate(-1)} + > + + + navigate(1)} + > + + + ({ + value: item.sessionId, + label: sessionLabel(item), + })) : [{ value: "", label: "Сессия не создана", disabled: true }]} + disabled={busy} + searchable + menuWidth={380} + minMenuWidth={300} + onChange={(value) => void selectSession(value)} + /> + void createSession()} + > + + + + + +
+
+ navigate(-1)}> + + + navigate(1)}> + + +