feat(lab): visualize semantic SLAM shadow

2026-08-06 11:26:46 +03:00
parent 8eaa3ab497
commit 81ae4425cb
18 changed files with 1747 additions and 42 deletions
@@ -1,5 +1,6 @@
 import {
  forwardRef,
+  type CSSProperties,
  useEffect,
  useImperativeHandle,
  useRef,
@@ -8,6 +9,13 @@ import {
 import * as THREE from "three";
 import { OrbitControls } from "three/addons/controls/OrbitControls.js";

+import {
+  recordedEvidenceSemanticCssColor,
+  resolveRecordedEvidenceSemanticRgb,
+  type RecordedEvidenceSemanticClass,
+  type RecordedEvidenceSemanticPaletteEntry,
+} from "./RecordedEvidenceSemanticMaskOverlay";
+
 export type LaboratoryMetricPoint3 = readonly [number, number, number];
 export type LaboratoryMetricDecision = "threat" | "not-threat" | "unknown";
 export type LaboratoryMetricSceneMode = "3d" | "plan";
@@ -114,6 +122,9 @@ LaboratoryMetricEvidenceSceneHandle,
  showCurrentIncrement: boolean;
  showLocalSurface: boolean;
  showRollingMap: boolean;
+  pointSemanticClassIds?: readonly (number | null)[];
+  semanticClasses?: readonly RecordedEvidenceSemanticClass[];
+  semanticPalette?: readonly RecordedEvidenceSemanticPaletteEntry[];
 }
 >(function LaboratoryMetricEvidenceScene({
  pointCloudBodyXyzM,
@@ -127,6 +138,9 @@ LaboratoryMetricEvidenceSceneHandle,
  showCurrentIncrement,
  showLocalSurface,
  showRollingMap,
+  pointSemanticClassIds,
+  semanticClasses,
+  semanticPalette,
 }, ref) {
  const hostRef = useRef<HTMLDivElement | null>(null);
  const sceneRef = useRef<THREE.Scene | null>(null);
@@ -240,10 +254,40 @@ LaboratoryMetricEvidenceSceneHandle,
        "position",
        new THREE.BufferAttribute(positions(pointCloudBodyXyzM), 3),
      );
+      const hasAlignedSemanticClasses =
+        pointSemanticClassIds !== undefined
+        && pointSemanticClassIds.length === pointCloudBodyXyzM.length
+        && semanticClasses !== undefined
+        && semanticPalette !== undefined;
+      if (hasAlignedSemanticClasses) {
+        const declaredIds = new Set(semanticClasses.map((item) => item.id));
+        const colorsByClassId = new Map<number, readonly [number, number, number]>();
+        for (const entry of semanticPalette) {
+          if (!declaredIds.has(entry.classId)) continue;
+          const rgb = resolveRecordedEvidenceSemanticRgb(host, entry.color);
+          if (rgb) colorsByClassId.set(entry.classId, rgb);
+        }
+        const context = tokenColor(host, "--nodedc-text-muted", [147, 151, 159]);
+        const pointColors = new Float32Array(pointCloudBodyXyzM.length * 3);
+        pointSemanticClassIds.forEach((classId, index) => {
+          const rgb = classId === null ? undefined : colorsByClassId.get(classId);
+          const offset = index * 3;
+          pointColors[offset] = rgb ? rgb[0] / 255 : context.r;
+          pointColors[offset + 1] = rgb ? rgb[1] / 255 : context.g;
+          pointColors[offset + 2] = rgb ? rgb[2] / 255 : context.b;
+        });
+        contextGeometry.setAttribute(
+          "color",
+          new THREE.BufferAttribute(pointColors, 3),
+        );
+      }
      content.add(new THREE.Points(
        contextGeometry,
        new THREE.PointsMaterial({
-          color: tokenColor(host, "--nodedc-text-muted", [147, 151, 159]),
+          color: hasAlignedSemanticClasses
+            ? new THREE.Color(1, 1, 1)
+            : tokenColor(host, "--nodedc-text-muted", [147, 151, 159]),
+          vertexColors: hasAlignedSemanticClasses,
          size: 1.55,
          sizeAttenuation: false,
          transparent: true,
@@ -321,6 +365,9 @@ LaboratoryMetricEvidenceSceneHandle,
    occupiedVoxelSizeM,
    localSurfaceBodyXyzM,
    pointCloudBodyXyzM,
+    pointSemanticClassIds,
+    semanticClasses,
+    semanticPalette,
    showCurrentIncrement,
    showLocalSurface,
    showRollingMap,
@@ -411,6 +458,27 @@ LaboratoryMetricEvidenceSceneHandle,
  useEffect(resetView, [corridor.forwardLengthM, mode]);
  useImperativeHandle(ref, () => ({ resetView }));

+  const semanticLegendEntries = (() => {
+    if (
+      !showCurrentIncrement
+      || !pointSemanticClassIds
+      || pointSemanticClassIds.length !== pointCloudBodyXyzM.length
+      || !semanticClasses
+      || !semanticPalette
+    ) return [];
+    const presentIds = new Set(pointSemanticClassIds.filter((item): item is number => item !== null));
+    const classesById = new Map(semanticClasses.map((item) => [item.id, item]));
+    return semanticPalette.flatMap((entry) => {
+      const semanticClass = classesById.get(entry.classId);
+      if (!semanticClass || !presentIds.has(entry.classId) || entry.color.kind === "transparent") return [];
+      return [{
+        id: entry.classId,
+        label: semanticClass.label,
+        cssColor: recordedEvidenceSemanticCssColor(entry.color),
+      }];
+    });
+  })();
+
  return (
    <div className="laboratory-metric-evidence-scene">
      <div ref={hostRef} className="laboratory-metric-evidence-scene__viewport">
@@ -423,6 +491,15 @@ LaboratoryMetricEvidenceSceneHandle,
        <span data-decision="context">Current increment</span>
        <span data-decision="local-surface">Local SLAM surface</span>
        <span data-decision="rolling">Rolling-map occupied</span>
+        {semanticLegendEntries.map((entry) => (
+          <span
+            key={entry.id}
+            data-decision="semantic"
+            style={{ "--laboratory-metric-legend-color": entry.cssColor } as CSSProperties}
+          >
+            {entry.label}
+          </span>
+        ))}
      </div>
    </div>
  );
@@ -5,18 +5,24 @@ import {
  RecordedEvidenceBoxOverlay,
  type RecordedEvidenceBox,
 } from "./RecordedEvidenceBoxOverlay";
+import {
+  RecordedEvidenceSemanticMaskOverlay,
+  type RecordedEvidenceSemanticOverlay,
+} from "./RecordedEvidenceSemanticMaskOverlay";

 export function RecordedEvidenceImageScene({
  src,
  imageWidth,
  imageHeight,
  boxes,
+  semanticOverlay,
  ariaLabel,
 }: {
  src: string;
  imageWidth: number;
  imageHeight: number;
  boxes: readonly RecordedEvidenceBox[];
+  semanticOverlay?: RecordedEvidenceSemanticOverlay;
  ariaLabel: string;
 }) {
  const [state, setState] = useState<"loading" | "ready" | "error">("loading");
@@ -33,12 +39,21 @@ export function RecordedEvidenceImageScene({
        onError={() => setState("error")}
      />
      {state === "ready" ? (
-        <RecordedEvidenceBoxOverlay
-          imageWidth={imageWidth}
-          imageHeight={imageHeight}
-          boxes={boxes}
-          ariaLabel={ariaLabel}
-        />
+        <>
+          {semanticOverlay ? (
+            <RecordedEvidenceSemanticMaskOverlay
+              {...semanticOverlay}
+              imageWidth={imageWidth}
+              imageHeight={imageHeight}
+            />
+          ) : null}
+          <RecordedEvidenceBoxOverlay
+            imageWidth={imageWidth}
+            imageHeight={imageHeight}
+            boxes={boxes}
+            ariaLabel={ariaLabel}
+          />
+        </>
      ) : (
        <div className="l3-visual-audit__state" role="status">
          {state === "loading" ? (
@@ -0,0 +1,391 @@
+import { useEffect, useRef, useState } from "react";
+
+export interface RecordedEvidenceSemanticClass {
+  id: number;
+  label: string;
+}
+
+export type RecordedEvidenceSemanticToken =
+  | "--nodedc-accent-rgb"
+  | "--nodedc-danger-rgb"
+  | "--nodedc-foreground-rgb"
+  | "--nodedc-success-rgb"
+  | "--nodedc-text-muted"
+  | "--nodedc-warning-rgb";
+
+export type RecordedEvidenceSemanticColor =
+  | {
+      kind: "token";
+      token: RecordedEvidenceSemanticToken;
+    }
+  | {
+      kind: "diagnostic";
+      rgb: readonly [number, number, number];
+    }
+  | {
+      kind: "transparent";
+    };
+
+export interface RecordedEvidenceSemanticPaletteEntry {
+  classId: number;
+  color: RecordedEvidenceSemanticColor;
+  opacity?: number;
+}
+
+export interface RecordedEvidenceSemanticOverlay {
+  src: string;
+  classes: readonly RecordedEvidenceSemanticClass[];
+  palette: readonly RecordedEvidenceSemanticPaletteEntry[];
+  opacity?: number;
+  ariaLabel: string;
+}
+
+interface DecodedSemanticMask {
+  key: string;
+  width: number;
+  height: number;
+  classIds: Uint8Array;
+}
+
+interface PendingSemanticMask {
+  controller: AbortController;
+  subscribers: number;
+  promise: Promise<DecodedSemanticMask>;
+}
+
+const MASK_CACHE_LIMIT = 48;
+const decodedMaskCache = new Map<string, DecodedSemanticMask>();
+const pendingMaskCache = new Map<string, PendingSemanticMask>();
+
+const TOKEN_FALLBACKS: Record<RecordedEvidenceSemanticToken, readonly [number, number, number]> = {
+  "--nodedc-accent-rgb": [232, 56, 126],
+  "--nodedc-danger-rgb": [255, 104, 112],
+  "--nodedc-foreground-rgb": [245, 245, 245],
+  "--nodedc-success-rgb": [181, 255, 90],
+  "--nodedc-text-muted": [147, 151, 159],
+  "--nodedc-warning-rgb": [255, 197, 92],
+};
+
+function clampChannel(value: number): number {
+  return Math.max(0, Math.min(255, Math.round(value)));
+}
+
+function clampOpacity(value: number | undefined, fallback: number): number {
+  return Math.max(0, Math.min(1, Number.isFinite(value) ? Number(value) : fallback));
+}
+
+function semanticMaskKey(src: string, width: number, height: number): string {
+  return `${width}x${height}:${src}`;
+}
+
+function rememberMask(mask: DecodedSemanticMask): void {
+  decodedMaskCache.delete(mask.key);
+  decodedMaskCache.set(mask.key, mask);
+  while (decodedMaskCache.size > MASK_CACHE_LIMIT) {
+    const oldest = decodedMaskCache.keys().next().value;
+    if (typeof oldest !== "string") break;
+    decodedMaskCache.delete(oldest);
+  }
+}
+
+async function imageSourceFromBlob(
+  blob: Blob,
+  signal: AbortSignal,
+): Promise<{ source: CanvasImageSource; width: number; height: number; release: () => void }> {
+  if (typeof createImageBitmap === "function") {
+    const bitmap = await createImageBitmap(blob, {
+      colorSpaceConversion: "none",
+      premultiplyAlpha: "none",
+    });
+    if (signal.aborted) {
+      bitmap.close();
+      throw new DOMException("Aborted", "AbortError");
+    }
+    return {
+      source: bitmap,
+      width: bitmap.width,
+      height: bitmap.height,
+      release: () => bitmap.close(),
+    };
+  }
+
+  const objectUrl = URL.createObjectURL(blob);
+  const image = new Image();
+  image.decoding = "async";
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const cleanup = () => {
+        image.removeEventListener("load", onLoad);
+        image.removeEventListener("error", onError);
+        signal.removeEventListener("abort", onAbort);
+      };
+      const onLoad = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = () => {
+        cleanup();
+        reject(new Error("Semantic mask image decode failed"));
+      };
+      const onAbort = () => {
+        cleanup();
+        reject(new DOMException("Aborted", "AbortError"));
+      };
+      image.addEventListener("load", onLoad, { once: true });
+      image.addEventListener("error", onError, { once: true });
+      signal.addEventListener("abort", onAbort, { once: true });
+      image.src = objectUrl;
+    });
+    return {
+      source: image,
+      width: image.naturalWidth,
+      height: image.naturalHeight,
+      release: () => URL.revokeObjectURL(objectUrl),
+    };
+  } catch (error) {
+    URL.revokeObjectURL(objectUrl);
+    throw error;
+  }
+}
+
+async function decodeSemanticMask(
+  key: string,
+  src: string,
+  expectedWidth: number,
+  expectedHeight: number,
+  signal: AbortSignal,
+): Promise<DecodedSemanticMask> {
+  const response = await fetch(src, { cache: "force-cache", signal });
+  if (!response.ok) throw new Error(`Semantic mask request failed: ${response.status}`);
+  const blob = await response.blob();
+  if (signal.aborted) throw new DOMException("Aborted", "AbortError");
+  const decoded = await imageSourceFromBlob(blob, signal);
+  try {
+    if (decoded.width !== expectedWidth || decoded.height !== expectedHeight) {
+      throw new Error(
+        `Semantic mask dimensions ${decoded.width}x${decoded.height} do not match ${expectedWidth}x${expectedHeight}`,
+      );
+    }
+    const decodeCanvas = document.createElement("canvas");
+    decodeCanvas.width = decoded.width;
+    decodeCanvas.height = decoded.height;
+    const context = decodeCanvas.getContext("2d", { willReadFrequently: true });
+    if (!context) throw new Error("Semantic mask canvas is unavailable");
+    context.drawImage(decoded.source, 0, 0);
+    const rgba = context.getImageData(0, 0, decoded.width, decoded.height).data;
+    const classIds = new Uint8Array(decoded.width * decoded.height);
+    for (let sourceOffset = 0, targetOffset = 0; targetOffset < classIds.length; sourceOffset += 4, targetOffset += 1) {
+      const classId = rgba[sourceOffset] ?? 0;
+      if (rgba[sourceOffset + 1] !== classId || rgba[sourceOffset + 2] !== classId) {
+        throw new Error("Semantic mask must be an 8-bit grayscale class-id PNG");
+      }
+      classIds[targetOffset] = classId;
+    }
+    return { key, width: decoded.width, height: decoded.height, classIds };
+  } finally {
+    decoded.release();
+  }
+}
+
+function subscribeToSemanticMask(
+  src: string,
+  width: number,
+  height: number,
+): { promise: Promise<DecodedSemanticMask>; release: () => void } {
+  const key = semanticMaskKey(src, width, height);
+  const cached = decodedMaskCache.get(key);
+  if (cached) {
+    decodedMaskCache.delete(key);
+    decodedMaskCache.set(key, cached);
+    return { promise: Promise.resolve(cached), release: () => undefined };
+  }
+
+  let pending = pendingMaskCache.get(key);
+  if (!pending) {
+    const controller = new AbortController();
+    let next: PendingSemanticMask;
+    const promise = decodeSemanticMask(key, src, width, height, controller.signal)
+      .then((mask) => {
+        rememberMask(mask);
+        return mask;
+      })
+      .finally(() => {
+        if (pendingMaskCache.get(key) === next) pendingMaskCache.delete(key);
+      });
+    next = { controller, subscribers: 0, promise };
+    pendingMaskCache.set(key, next);
+    pending = next;
+  }
+  pending.subscribers += 1;
+  let released = false;
+  return {
+    promise: pending.promise,
+    release: () => {
+      if (released) return;
+      released = true;
+      pending!.subscribers -= 1;
+      if (pending!.subscribers > 0 || decodedMaskCache.has(key)) return;
+      pending!.controller.abort();
+      if (pendingMaskCache.get(key) === pending) pendingMaskCache.delete(key);
+    },
+  };
+}
+
+export function recordedEvidenceSemanticCssColor(
+  color: RecordedEvidenceSemanticColor,
+): string {
+  if (color.kind === "transparent") return "transparent";
+  if (color.kind === "token") return `rgb(var(${color.token}))`;
+  const [red, green, blue] = color.rgb.map(clampChannel);
+  return `rgb(${red} ${green} ${blue})`;
+}
+
+export function resolveRecordedEvidenceSemanticRgb(
+  host: HTMLElement,
+  color: RecordedEvidenceSemanticColor,
+): readonly [number, number, number] | null {
+  if (color.kind === "transparent") return null;
+  if (color.kind === "diagnostic") return color.rgb.map(clampChannel) as [number, number, number];
+  const channels = getComputedStyle(host)
+    .getPropertyValue(color.token)
+    .trim()
+    .match(/[\d.]+/g)
+    ?.slice(0, 3)
+    .map(Number);
+  return channels?.length === 3
+    ? channels.map(clampChannel) as [number, number, number]
+    : TOKEN_FALLBACKS[color.token];
+}
+
+export function RecordedEvidenceSemanticMaskOverlay({
+  src,
+  imageWidth,
+  imageHeight,
+  classes,
+  palette,
+  opacity = 0.46,
+  ariaLabel,
+}: RecordedEvidenceSemanticOverlay & {
+  imageWidth: number;
+  imageHeight: number;
+}) {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null);
+  const [mask, setMask] = useState<DecodedSemanticMask | null>(null);
+  const [failure, setFailure] = useState<string | null>(null);
+  const expectedKey = semanticMaskKey(src, imageWidth, imageHeight);
+  const renderMask = failure
+    ? null
+    : mask?.key === expectedKey
+      ? mask
+      : decodedMaskCache.get(expectedKey) ?? null;
+
+  useEffect(() => {
+    setFailure(null);
+    const subscription = subscribeToSemanticMask(src, imageWidth, imageHeight);
+    let current = true;
+    void subscription.promise.then((decoded) => {
+      if (!current || decoded.key !== expectedKey) return;
+      const declaredClassIds = new Set(classes.map((item) => item.id));
+      const undeclaredClassId = decoded.classIds.find(
+        (classId) => !declaredClassIds.has(classId),
+      );
+      if (undeclaredClassId !== undefined) {
+        setFailure(`Semantic mask содержит необъявленный class ID ${undeclaredClassId}.`);
+        return;
+      }
+      setMask(decoded);
+    }).catch((error: unknown) => {
+      if (!current || (error instanceof DOMException && error.name === "AbortError")) return;
+      setFailure("Semantic mask не прошла проверку или декодирование.");
+    });
+    return () => {
+      current = false;
+      subscription.release();
+    };
+  }, [classes, expectedKey, imageHeight, imageWidth, src]);
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    const host = canvas?.parentElement;
+    if (!canvas || !host) return;
+    const context = canvas.getContext("2d");
+    if (!context) return;
+
+    const render = () => {
+      const width = Math.max(host.clientWidth, 1);
+      const height = Math.max(host.clientHeight, 1);
+      const pixelRatio = Math.min(window.devicePixelRatio, 1.5);
+      canvas.width = Math.round(width * pixelRatio);
+      canvas.height = Math.round(height * pixelRatio);
+      canvas.style.width = `${width}px`;
+      canvas.style.height = `${height}px`;
+      context.setTransform(pixelRatio, 0, 0, pixelRatio, 0, 0);
+      context.clearRect(0, 0, width, height);
+      if (!renderMask) return;
+
+      const declaredClassIds = new Set(
+        classes
+          .map((item) => item.id)
+          .filter((classId) => Number.isInteger(classId) && classId >= 0 && classId <= 255),
+      );
+      const resolvedPalette = new Map<number, { rgb: readonly [number, number, number]; alpha: number }>();
+      for (const entry of palette) {
+        if (!declaredClassIds.has(entry.classId)) continue;
+        const rgb = resolveRecordedEvidenceSemanticRgb(host, entry.color);
+        if (!rgb) continue;
+        resolvedPalette.set(entry.classId, {
+          rgb,
+          alpha: clampOpacity(entry.opacity, 1) * clampOpacity(opacity, 0.46),
+        });
+      }
+
+      const colorCanvas = document.createElement("canvas");
+      colorCanvas.width = renderMask.width;
+      colorCanvas.height = renderMask.height;
+      const colorContext = colorCanvas.getContext("2d");
+      if (!colorContext) return;
+      const imageData = colorContext.createImageData(renderMask.width, renderMask.height);
+      for (let sourceOffset = 0, targetOffset = 0; sourceOffset < renderMask.classIds.length; sourceOffset += 1, targetOffset += 4) {
+        const color = resolvedPalette.get(renderMask.classIds[sourceOffset] ?? -1);
+        if (!color) continue;
+        imageData.data[targetOffset] = color.rgb[0];
+        imageData.data[targetOffset + 1] = color.rgb[1];
+        imageData.data[targetOffset + 2] = color.rgb[2];
+        imageData.data[targetOffset + 3] = Math.round(color.alpha * 255);
+      }
+      colorContext.putImageData(imageData, 0, 0);
+
+      const scale = Math.min(width / imageWidth, height / imageHeight);
+      const drawWidth = imageWidth * scale;
+      const drawHeight = imageHeight * scale;
+      const offsetX = (width - drawWidth) / 2;
+      const offsetY = (height - drawHeight) / 2;
+      context.imageSmoothingEnabled = false;
+      context.drawImage(colorCanvas, offsetX, offsetY, drawWidth, drawHeight);
+    };
+
+    const observer = new ResizeObserver(render);
+    observer.observe(host);
+    render();
+    return () => observer.disconnect();
+  }, [classes, expectedKey, imageHeight, imageWidth, opacity, palette, renderMask]);
+
+  return (
+    <>
+      <canvas
+        ref={canvasRef}
+        className="recorded-evidence-semantic-mask-overlay"
+        role="img"
+        aria-label={ariaLabel}
+        aria-busy={!failure && !renderMask}
+        data-state={failure ? "error" : renderMask ? "ready" : "loading"}
+        style={{ zIndex: 1 }}
+      />
+      {failure ? (
+        <div className="recorded-evidence-semantic-mask-overlay__error" role="alert">
+          {failure}
+        </div>
+      ) : null}
+    </>
+  );
+}
@@ -8,6 +8,10 @@ import {
  type RecordedEvidenceBox,
  type RecordedEvidenceBoxTone,
 } from "./RecordedEvidenceBoxOverlay";
+import {
+  RecordedEvidenceSemanticMaskOverlay,
+  type RecordedEvidenceSemanticOverlay,
+} from "./RecordedEvidenceSemanticMaskOverlay";

 export type { RecordedEvidenceBox, RecordedEvidenceBoxTone };

@@ -17,6 +21,7 @@ export function RecordedEvidenceVideoScene({
  imageWidth,
  imageHeight,
  boxes,
+  semanticOverlay,
  ariaLabel,
  interactive = true,
  onPlaybackChange,
@@ -26,6 +31,7 @@ export function RecordedEvidenceVideoScene({
  imageWidth: number;
  imageHeight: number;
  boxes: readonly RecordedEvidenceBox[];
+  semanticOverlay?: RecordedEvidenceSemanticOverlay;
  ariaLabel: string;
  interactive?: boolean;
  onPlaybackChange?: (playback: RecordedObservationPlayback) => void;
@@ -39,6 +45,13 @@ export function RecordedEvidenceVideoScene({
        prepare
        onPlaybackChange={onPlaybackChange}
      />
+      {semanticOverlay ? (
+        <RecordedEvidenceSemanticMaskOverlay
+          {...semanticOverlay}
+          imageWidth={imageWidth}
+          imageHeight={imageHeight}
+        />
+      ) : null}
      <RecordedEvidenceBoxOverlay
        imageWidth={imageWidth}
        imageHeight={imageHeight}