diff --git a/apps/control-station/src/core/laboratory/vegetationShadow.ts b/apps/control-station/src/core/laboratory/vegetationShadow.ts index dcd8539..f4d3ff5 100644 --- a/apps/control-station/src/core/laboratory/vegetationShadow.ts +++ b/apps/control-station/src/core/laboratory/vegetationShadow.ts @@ -51,6 +51,26 @@ export interface VegetationVisualCase { assets: Readonly>; } +export interface VegetationVideoSemanticClass { + classId: number; + label: string; + colorRgb: readonly [number, number, number]; + disposition: "prediction" | "undefined"; +} + +export interface VegetationRouteVideo { + workerResultId: string; + m47ReferenceGraphResultId: string; + baseM4ResultId: string; + frameCount: 4489; + width: 800; + height: 600; + centerCropXyxy: readonly [100, 0, 700, 600]; + outsideCropState: "undefined"; + taxonomy: readonly VegetationVideoSemanticClass[]; + aggregatePredictionPixels: readonly number[]; +} + export interface VegetationShadowResult { resultId: string; createdAtUtc: string; @@ -59,6 +79,7 @@ export interface VegetationShadowResult { candidates: readonly VegetationCandidateMetrics[]; routeCases: readonly VegetationVisualCase[]; validationCases: readonly VegetationVisualCase[]; + routeVideo: VegetationRouteVideo | null; limitations: readonly string[]; visualShadowReady: true; missionPolicyReadyForConfiguration: true; @@ -227,6 +248,98 @@ function visualCaseValue( }; } +function routeVideoValue(value: unknown): VegetationRouteVideo | null { + if (value === null || value === undefined) return null; + const row = objectValue(value, "vegetation.route_video"); + const workerResultId = textValue(row.worker_result_id, "vegetation.route_video.worker_result_id"); + const m47ReferenceGraphResultId = textValue( + row.m47_reference_graph_result_id, + "vegetation.route_video.m47_reference_graph_result_id", + ); + const baseM4ResultId = textValue(row.base_m4_result_id, "vegetation.route_video.base_m4_result_id"); + if ( + !/^lab-v1-ravnoves-video-ddrnet-[a-f0-9]{64}$/.test(workerResultId) + || !/^m47-reference-graph-lab-[a-f0-9]{64}$/.test(m47ReferenceGraphResultId) + || !/^m4-threat-replay-[a-f0-9]{64}$/.test(baseM4ResultId) + ) { + throw new VegetationShadowContractError("vegetation.route_video: identity invalid."); + } + exact(row.frame_count, 4489, "vegetation.route_video.frame_count"); + exact(row.width, 800, "vegetation.route_video.width"); + exact(row.height, 600, "vegetation.route_video.height"); + exact(row.outside_crop_state, "undefined", "vegetation.route_video.outside_crop_state"); + exact( + row.sequence_binding, + "sequence-0-to-masks/frame-000001.png", + "vegetation.route_video.sequence_binding", + ); + const crop = arrayValue(row.center_crop_xyxy, "vegetation.route_video.center_crop_xyxy") + .map((item, index) => integerValue(item, `vegetation.route_video.crop[${index}]`)); + if (crop.join(",") !== "100,0,700,600") { + throw new VegetationShadowContractError("vegetation.route_video: crop contract changed."); + } + const taxonomy = objectValue(row.taxonomy, "vegetation.route_video.taxonomy"); + exact( + taxonomy.schema_version, + "missioncore.lab-v1-vegetation-taxonomy/v1", + "vegetation.route_video.taxonomy.schema", + ); + const classes = arrayValue(taxonomy.classes, "vegetation.route_video.taxonomy.classes") + .map((value, expectedId): VegetationVideoSemanticClass => { + const item = objectValue(value, `vegetation.route_video.taxonomy[${expectedId}]`); + const classId = integerValue(item.class_id, `vegetation.route_video.class_id[${expectedId}]`); + if (classId !== expectedId) { + throw new VegetationShadowContractError("vegetation.route_video: taxonomy order changed."); + } + const color = arrayValue(item.color_rgb, `vegetation.route_video.color[${expectedId}]`) + .map((channel, index) => integerValue(channel, `vegetation.route_video.color[${expectedId}][${index}]`)); + if (color.length !== 3 || color.some((channel) => channel > 255)) { + throw new VegetationShadowContractError("vegetation.route_video: taxonomy color invalid."); + } + const disposition: VegetationVideoSemanticClass["disposition"] = expectedId === 0 + ? "undefined" + : "prediction"; + if (item.disposition !== disposition) { + throw new VegetationShadowContractError("vegetation.route_video: taxonomy disposition changed."); + } + return { + classId, + label: textValue(item.label, `vegetation.route_video.label[${expectedId}]`), + colorRgb: color as unknown as readonly [number, number, number], + disposition, + }; + }); + if (classes.length !== 64) { + throw new VegetationShadowContractError("vegetation.route_video: taxonomy must contain 64 classes."); + } + const aggregatePredictionPixels = arrayValue( + row.aggregate_prediction_pixels, + "vegetation.route_video.aggregate_prediction_pixels", + ).map((value, index) => integerValue(value, `vegetation.route_video.pixels[${index}]`)); + if (aggregatePredictionPixels.length !== 64) { + throw new VegetationShadowContractError("vegetation.route_video: class accounting changed."); + } + const maskArchive = objectValue(row.mask_archive, "vegetation.route_video.mask_archive"); + exact(maskArchive.path, "video/ddrnet-semantic-masks.zip", "vegetation.route_video.mask_archive.path"); + const archiveSha256 = textValue(maskArchive.sha256, "vegetation.route_video.mask_archive.sha256"); + if (!SHA256.test(archiveSha256)) { + throw new VegetationShadowContractError("vegetation.route_video: archive digest invalid."); + } + integerValue(maskArchive.byte_length, "vegetation.route_video.mask_archive.byte_length"); + return { + workerResultId, + m47ReferenceGraphResultId, + baseM4ResultId, + frameCount: 4489, + width: 800, + height: 600, + centerCropXyxy: [100, 0, 700, 600], + outsideCropState: "undefined", + taxonomy: classes, + aggregatePredictionPixels, + }; +} + function parseResult(value: unknown, resultId: string): VegetationShadowResult { const payload = objectValue(value, "Vegetation LAB"); exact(payload.schema_version, "missioncore.lab-v1-vegetation-shadow/v1", "vegetation.schema"); @@ -277,6 +390,7 @@ function parseResult(value: unknown, resultId: string): VegetationShadowResult { candidates: CANDIDATES.map((candidate) => candidateMetricsValue(candidates[candidate], candidate)), routeCases, validationCases, + routeVideo: routeVideoValue(payload.route_video), limitations: arrayValue(payload.limitations, "vegetation.limitations") .map((item, index) => textValue(item, `vegetation.limitations[${index}]`)), visualShadowReady: true, @@ -290,6 +404,13 @@ function parseResult(value: unknown, resultId: string): VegetationShadowResult { }; } +export function vegetationVideoMaskUrl(resultId: string, sequence: number): string { + if (!RESULT_ID.test(resultId) || !Number.isInteger(sequence) || sequence < 0 || sequence >= 4489) { + throw new VegetationShadowContractError("Vegetation video mask identity недопустима."); + } + return `/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}/masks/${sequence}`; +} + export async function fetchVegetationShadowResult( resultId: string, { diff --git a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx index 41af4c0..616253b 100644 --- a/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx +++ b/apps/control-station/src/workspaces/laboratory/M4ReplayThreatVisual.tsx @@ -97,7 +97,16 @@ function SpatialState({ message: text }: { message: string }) { export interface M4ReplayThreatSemanticLayer { resultId: string; - taxonomy: readonly E47SemanticClass[]; + spatialResultId?: string | null; + maskUrl?: (sequence: number) => string; + label?: string; + maskAriaLabel?: string; + taxonomy: readonly { + classId: number; + label: string; + disposition: "labeled" | "ambiguous" | "prediction" | "undefined"; + colorRgb: readonly [number, number, number]; + }[]; } export interface M4ReplayThreatReviewAnchor { @@ -153,6 +162,8 @@ export function M4ReplayThreatVisual({ evidenceLabel = "M4.6", initialSpatialMode = null, classifiedSpatialLayer, + showReferenceMediaLayers = true, + showSpatialOverlaySummary = true, onActiveSequenceChange, }: { resultId: string; @@ -164,6 +175,8 @@ export function M4ReplayThreatVisual({ evidenceLabel?: string; initialSpatialMode?: LaboratoryMetricSceneMode | null; classifiedSpatialLayer?: M4ReplayClassifiedSpatialLayer; + showReferenceMediaLayers?: boolean; + showSpatialOverlaySummary?: boolean; onActiveSequenceChange?: (sequence: number | null) => void; }) { const [mediaMode, setMediaMode] = useState("video"); @@ -280,16 +293,30 @@ export function M4ReplayThreatVisual({ ? lastSpatialFrameRef.current.frame : null; const cameraPointOverlay = useM4ThreatCameraPointOverlay({ - enabled: showMediaPoints, + enabled: showReferenceMediaLayers && showMediaPoints, resultId, sequence: frame?.sequence ?? null, endpointRoot: timelineEndpointRoot, }); + const semanticSpatialResultId = semantic + ? semantic.spatialResultId === undefined ? semantic.resultId : semantic.spatialResultId + : null; + const spatialSemanticTaxonomy = useMemo( + () => semanticSpatialResultId && semantic + ? semantic.taxonomy.map((item) => ({ + classId: item.classId, + label: item.label, + disposition: item.disposition === "ambiguous" ? "ambiguous" : "labeled", + colorRgb: item.colorRgb, + })) + : [], + [semantic, semanticSpatialResultId], + ); const semanticTimeline = useE47SemanticTimelineFrame({ - resultId: semantic?.resultId ?? null, + resultId: semanticSpatialResultId, activeSequence: frame?.sequence ?? timelineFrame.activeSequence, frameCount: metadata.timeline?.frameCount ?? 0, - taxonomy: semantic?.taxonomy ?? [], + taxonomy: spatialSemanticTaxonomy, }); const displayingBufferedFrame = Boolean( frame @@ -339,7 +366,8 @@ export function M4ReplayThreatVisual({ const staticObstacleBoxes = useMemo(() => { const timeline = metadata.timeline; if ( - !frame + !showReferenceMediaLayers + || !frame || !timeline?.cameraObstacleProjectionDelivery || !showStaticObstacles ) return []; @@ -348,14 +376,14 @@ export function M4ReplayThreatVisual({ timeline.imageWidth, timeline.imageHeight, ); - }, [frame, metadata.timeline, showStaticObstacles]); + }, [frame, metadata.timeline, showReferenceMediaLayers, showStaticObstacles]); const activeBoxes = useMemo( - () => classifiedSpatialLayer ? [] : [ + () => classifiedSpatialLayer || !showReferenceMediaLayers ? [] : [ ...boxes(frame?.cameraProposals ?? []), ...staticObstacleBoxes, ...reviewAnchorBoxes, ], - [classifiedSpatialLayer, frame, reviewAnchorBoxes, staticObstacleBoxes], + [classifiedSpatialLayer, frame, reviewAnchorBoxes, showReferenceMediaLayers, staticObstacleBoxes], ); const semanticClasses = useMemo( () => semantic?.taxonomy.map((item) => ({ @@ -367,10 +395,14 @@ export function M4ReplayThreatVisual({ const semanticPalette = useMemo( () => semantic?.taxonomy.map((item) => ({ classId: item.classId, - color: item.disposition === "ambiguous" + color: item.disposition === "undefined" + ? { kind: "transparent" as const } + : item.disposition === "ambiguous" ? { kind: "token" as const, token: "--nodedc-warning-rgb" as const } : { kind: "diagnostic" as const, rgb: item.colorRgb }, - opacity: item.disposition === "ambiguous" ? 0.52 : 0.92, + opacity: item.disposition === "undefined" + ? 0 + : item.disposition === "ambiguous" ? 0.52 : 0.92, })) ?? [], [semantic?.taxonomy], ); @@ -580,15 +612,17 @@ export function M4ReplayThreatVisual({ const semanticOverlay: RecordedEvidenceSemanticOverlay | undefined = semantic && showMediaSemantic && frame ? { - src: e47SemanticMaskUrl(semantic.resultId, frame.sequence), + src: semantic.maskUrl?.(frame.sequence) + ?? e47SemanticMaskUrl(semantic.resultId, frame.sequence), prefetchSrcs: Array.from({ length: 12 }, (_, index) => index + 1) .map((offset) => frame.sequence + offset) .filter((sequence) => sequence < (metadata.timeline?.frameCount ?? 0)) - .map((sequence) => e47SemanticMaskUrl(semantic.resultId, sequence)), + .map((sequence) => semantic.maskUrl?.(sequence) + ?? e47SemanticMaskUrl(semantic.resultId, sequence)), classes: semanticClasses, palette: semanticPalette, opacity: 0.9, - ariaLabel: `E47 semantic mask frame ${frame.sequence + 1}`, + ariaLabel: `${semantic.maskAriaLabel ?? "Semantic prediction"} frame ${frame.sequence + 1}`, } : undefined; const accumulatedCameraPoints = cameraPointOverlay.overlay?.sequence === frame?.sequence @@ -659,8 +693,8 @@ export function M4ReplayThreatVisual({ ); const mediaLayerControls = semantic - || metadata.timeline?.cameraPointDelivery - || metadata.timeline?.cameraObstacleProjectionDelivery ? ( + || (showReferenceMediaLayers && metadata.timeline?.cameraPointDelivery) + || (showReferenceMediaLayers && metadata.timeline?.cameraObstacleProjectionDelivery) ? (
) : null} - {metadata.timeline?.cameraPointDelivery ? ( + {showReferenceMediaLayers && metadata.timeline?.cameraPointDelivery ? ( - {semantic ? ( + {semanticSpatialResultId ? (
-
- Spatial evidence - {classifiedSpatialLayer + {showSpatialOverlaySummary ? ( + <> +
+ Spatial evidence + {classifiedSpatialLayer ? classifiedSpatialFrame ? replaceClassifiedPointCloud ? `${classifiedSpatialFrame.pointsMapGravityLocalXyzM.length.toLocaleString("ru-RU")} TGS points · ${classifiedCellCount.toLocaleString("ru-RU")} cells` @@ -939,15 +975,15 @@ export function M4ReplayThreatVisual({ : showMediaPoints && cameraPointOverlay.error ? " · накопленное camera cloud недоступно" : ""} - {semantic && spatialSemanticFrame + {semanticSpatialResultId && spatialSemanticFrame ? ` · semantic L ${spatialSemanticFrame.counts.labeled} · A ${spatialSemanticFrame.counts.ambiguous} · U ${spatialSemanticFrame.counts.unprojected} · Ø ${spatialSemanticFrame.counts.absent}` - : semantic ? " · semantic buffer" : ""} + : semanticSpatialResultId ? " · semantic buffer" : ""} )} -
-
- {classifiedSpatialLayer ? "TGS fail-closed" : "Virtual corridor"} - {classifiedSpatialLayer +
+
+ {classifiedSpatialLayer ? "TGS fail-closed" : "Virtual corridor"} + {classifiedSpatialLayer ? classifiedSpatialFrame ? `${classifiedCellCounts.occupied} occupied · ${classifiedCellCounts.rejected} rejected · ${classifiedCellCounts.unobserved} unobserved` : classifiedSpatialLayer.loading || displayingBufferedFrame ? "loading" : "unavailable" @@ -957,7 +993,9 @@ export function M4ReplayThreatVisual({ ? `${classifiedCellCounts.ground} ground-support · visual review only · navigation authority OFF` : "visual review only · navigation authority OFF" : `${metadata.timeline.corridor.forwardLengthM} м · body ${metadata.timeline.rig.lengthM}×${metadata.timeline.rig.widthM} м · REPLAY-SIMULATED`} -
+
+ + ) : null} ) : undefined; @@ -1153,13 +1191,13 @@ export function M4ReplayThreatVisual({ {timelineFrame.error} ) : null} - {semantic && semanticTimeline.loading ? ( + {semanticSpatialResultId && semanticTimeline.loading ? (
) : null} - {semanticTimeline.error ? ( + {semanticSpatialResultId && semanticTimeline.error ? (
{semanticTimeline.error} @@ -1201,7 +1239,7 @@ export function M4ReplayThreatVisual({
)} evidence={( - - - + <> + + + + {result.routeVideo ? ( + + vegetationVideoMaskUrl(result.resultId, sequence), + label: "DDRNet vegetation prediction · recorded video", + maskAriaLabel: "DDRNet vegetation prediction", + }} + /> + + ) : null} + )} result={( )} diff --git a/apps/control-station/test/vegetationShadow.test.mjs b/apps/control-station/test/vegetationShadow.test.mjs index 8066642..bd59e33 100644 --- a/apps/control-station/test/vegetationShadow.test.mjs +++ b/apps/control-station/test/vegetationShadow.test.mjs @@ -75,6 +75,35 @@ function visualCase(sourceKind, index) { }; } +function routeVideo() { + return { + worker_result_id: `lab-v1-ravnoves-video-ddrnet-${"e".repeat(64)}`, + m47_reference_graph_result_id: `m47-reference-graph-lab-${"f".repeat(64)}`, + base_m4_result_id: `m4-threat-replay-${"1".repeat(64)}`, + frame_count: 4489, + width: 800, + height: 600, + center_crop_xyxy: [100, 0, 700, 600], + outside_crop_state: "undefined", + sequence_binding: "sequence-0-to-masks/frame-000001.png", + taxonomy: { + schema_version: "missioncore.lab-v1-vegetation-taxonomy/v1", + classes: Array.from({ length: 64 }, (_, classId) => ({ + class_id: classId, + label: classId === 0 ? "undefined" : `class-${classId}`, + color_rgb: [classId, classId, classId], + disposition: classId === 0 ? "undefined" : "prediction", + })), + }, + aggregate_prediction_pixels: Array(64).fill(0), + mask_archive: { + path: "video/ddrnet-semantic-masks.zip", + sha256: "9".repeat(64), + byte_length: 1024, + }, + }; +} + test("vegetation LAB keeps autonomous assets and fail-closed authority", async () => { let requestedUrl = ""; const result = await fetchVegetationShadowResult(resultId, { @@ -111,6 +140,7 @@ test("vegetation LAB keeps autonomous assets and fail-closed authority", async ( goose: Array.from({ length: 12 }, (_, index) => visualCase("goose", index)), ravnoves: [], }, + route_video: routeVideo(), access: "read-only", }), { status: 200, headers: { "Content-Type": "application/json" } }); }, @@ -123,6 +153,8 @@ test("vegetation LAB keeps autonomous assets and fail-closed authority", async ( assert.equal(result.candidates[0].vegetationMeanIouPercent, 64); assert.equal(result.routeCases.length, 0); assert.equal(result.validationCases.length, 12); + assert.equal(result.routeVideo.frameCount, 4489); + assert.equal(result.routeVideo.taxonomy[0].disposition, "undefined"); assert.equal(result.validationCases[0].focus.className, "high_grass"); assert.match(result.validationCases[0].assets.ddrnet_error, /\/assets\/visual\/goose\//); assert.deepEqual(result.authority, { @@ -133,13 +165,15 @@ test("vegetation LAB keeps autonomous assets and fail-closed authority", async ( }); }); -test("vegetation LAB reuses the admitted M4.8 instrument", async () => { +test("vegetation LAB reuses the admitted M4.8 and M4.7 instruments", async () => { const resultSource = await readFile( new URL("../src/workspaces/laboratory/VegetationShadowResult.tsx", import.meta.url), "utf8", ); assert.match(resultSource, /M48MaskComparisonVisual/); - assert.equal(resultSource.match(/$null [ordered]@{ @@ -221,6 +236,16 @@ try { Export-RavnovesFrames -Destination $framesRoot Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot } + elseif ($Mode -eq "RavnovesVideo") { + if ($candidateKey -ne "ddrnet") { + throw "Full-video shadow is admitted only for the selected DDRNet candidate" + } + $runRoot = New-RunRoot -Kind "ravnoves-video" + $framesRoot = Join-Path $runRoot "input-frames" + Export-RavnovesVideoFrames -Destination $framesRoot + Invoke-IsolatedRun -RunMode "ravnoves-video" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot + Remove-Item -LiteralPath $framesRoot -Recurse -Force + } } finally { $canonicalAfter = Get-CanonicalTritonIdentity diff --git a/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py b/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py index 96ae45e..c5ae054 100644 --- a/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py +++ b/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py @@ -11,6 +11,7 @@ import os import platform import statistics import time +import zipfile from pathlib import Path from typing import Any @@ -43,7 +44,11 @@ class RunnerError(RuntimeError): def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() - parser.add_argument("--mode", choices=("goose", "ravnoves"), required=True) + parser.add_argument( + "--mode", + choices=("goose", "ravnoves", "ravnoves-video"), + required=True, + ) parser.add_argument("--candidate", choices=tuple(MODEL_NAMES), required=True) parser.add_argument("--config", type=Path, required=True) parser.add_argument("--policy", type=Path, required=True) @@ -338,6 +343,40 @@ def save_image(path: Path, value: Image.Image | np.ndarray, mode: str | None = N return sha256(path) +def write_mask_archive( + output: Path, + masks_root: Path, + frame_count: int, +) -> dict[str, Any]: + archive_path = output / "semantic-masks.zip" + expected = [f"frame-{sequence + 1:06d}.png" for sequence in range(frame_count)] + actual = sorted(path.name for path in masks_root.glob("frame-*.png")) + if actual != expected: + raise RunnerError("RAVNOVES video mask sequence is incomplete") + with zipfile.ZipFile( + archive_path, + mode="x", + compression=zipfile.ZIP_STORED, + allowZip64=True, + ) as archive: + for name in expected: + archive.write(masks_root / name, arcname=f"masks/{name}") + for path in masks_root.iterdir(): + path.unlink() + masks_root.rmdir() + return { + "path": archive_path.name, + "sha256": sha256(archive_path), + "byte_length": archive_path.stat().st_size, + "media_type": "application/zip", + "frame_count": frame_count, + "width": 800, + "height": 600, + "encoding": "uint8-class-id-png", + "sequence_binding": "sequence-0-to-masks/frame-000001.png", + } + + def expand_mask( mask: np.ndarray, original_size: tuple[int, int], @@ -517,7 +556,7 @@ def run() -> None: (image.stem.removesuffix("_windshield_vis"), image, label) for image, label in pairs ] - else: + elif args.mode == "ravnoves": if args.frames_root is None or not args.frames_root.is_dir(): raise RunnerError("frames-root is required for RAVNOVES mode") frames = sorted(args.frames_root.glob("frame-*.png")) @@ -525,6 +564,15 @@ def run() -> None: if {frame.stem for frame in frames} != expected: raise RunnerError("RAVNOVES frame island identity changed") items = [(frame.stem, frame, None) for frame in frames] + else: + if args.frames_root is None or not args.frames_root.is_dir(): + raise RunnerError("frames-root is required for RAVNOVES video mode") + frames = sorted(args.frames_root.glob("frame-*.png")) + expected_count = config["ravnoves"]["expected_frame_count"] + expected_names = [f"frame-{sequence + 1:06d}.png" for sequence in range(expected_count)] + if len(frames) != expected_count or [frame.name for frame in frames] != expected_names: + raise RunnerError("RAVNOVES full-video frame sequence changed") + items = [(frame.stem, frame, None) for frame in frames] if args.limit: items = items[: args.limit] @@ -539,8 +587,12 @@ def run() -> None: if args.visual_count != configured_visual_count: raise RunnerError("GOOSE visual count differs from the truth-focused contract") selected_visuals = truth_focused_visuals(items, names, visual_contract) - else: + elif args.mode == "ravnoves": selected_visuals = visual_indices(len(items), args.visual_count) + else: + if args.visual_count != 0 or args.limit: + raise RunnerError("RAVNOVES video mode requires the complete frame sequence") + selected_visuals = {} args.output.mkdir(parents=True, exist_ok=False) torch.cuda.empty_cache() @@ -552,12 +604,28 @@ def run() -> None: confusion = np.zeros((CLASS_COUNT, CLASS_COUNT), dtype=np.int64) latencies_ms: list[float] = [] visuals: list[dict[str, Any]] = [] + mask_root = args.output / "masks" if args.mode == "ravnoves-video" else None + if mask_root is not None: + mask_root.mkdir() + aggregate_prediction_pixels = np.zeros(CLASS_COUNT, dtype=np.int64) for index, (case_id, source_path, label_path) in enumerate(items): source = Image.open(source_path).convert("RGB") + if args.mode == "ravnoves-video" and source.size != ( + config["ravnoves"]["expected_width"], + config["ravnoves"]["expected_height"], + ): + raise RunnerError("RAVNOVES video frame dimensions changed") tensor, crop_box = preprocess(source) prediction, latency_ms = infer(model, tensor) latencies_ms.append(latency_ms) + if mask_root is not None: + expanded_prediction = expand_mask(prediction, source.size, crop_box) + save_image(mask_root / f"frame-{index + 1:06d}.png", expanded_prediction, "L") + aggregate_prediction_pixels += np.bincount( + expanded_prediction.reshape(-1), + minlength=CLASS_COUNT, + ) truth = preprocess_label(Image.open(label_path)) if label_path is not None else None if truth is not None: update_confusion(confusion, truth, prediction) @@ -579,6 +647,23 @@ def run() -> None: ) ) + mask_archive = ( + write_mask_archive(args.output, mask_root, len(items)) + if mask_root is not None + else None + ) + taxonomy = { + "schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1", + "classes": [ + { + "class_id": label_id, + "label": names[label_id], + "color_rgb": semantic_palette[label_id, :3].astype(int).tolist(), + "disposition": "undefined" if label_id == 0 else "prediction", + } + for label_id in range(CLASS_COUNT) + ], + } rows = class_metrics(confusion, names) if args.mode == "goose" else [] valid_ious = [row["iou"] for row in rows if row["iou"] is not None] vegetation_names = set(config["vegetation_class_names"]) @@ -615,6 +700,20 @@ def run() -> None: "ground_truth_available": args.mode == "goose", "mapping_sha256": dataset_config["mapping_sha256"], }, + "video_semantics": { + "base_m4_result_id": config["ravnoves"].get("base_m4_result_id"), + "mask_archive": mask_archive, + "taxonomy": taxonomy, + "aggregate_prediction_pixels": aggregate_prediction_pixels.tolist() + if mask_archive is not None + else None, + "center_crop_xyxy": [100, 0, 700, 600] + if mask_archive is not None + else None, + "outside_crop_state": "undefined" if mask_archive is not None else None, + } + if args.mode == "ravnoves-video" + else None, "preprocessing": dataset_config["preprocessing"], "metrics": { "mean_iou": round(statistics.fmean(valid_ious), 8) if valid_ious else None, @@ -650,6 +749,7 @@ def run() -> None: "schema_version": result["schema_version"], "candidate": result["candidate"], "source": result["source"], + "video_semantics": result["video_semantics"], "preprocessing": result["preprocessing"], "metrics": result["metrics"], "timing": result["timing"], diff --git a/src/k1link/laboratory/vegetation_shadow_lab.py b/src/k1link/laboratory/vegetation_shadow_lab.py index dbae493..bd1d734 100644 --- a/src/k1link/laboratory/vegetation_shadow_lab.py +++ b/src/k1link/laboratory/vegetation_shadow_lab.py @@ -5,17 +5,28 @@ from __future__ import annotations import argparse import hashlib import json +import re import shutil import tempfile +import zipfile from datetime import UTC, datetime from pathlib import Path, PurePosixPath from typing import Any, Final +from k1link.laboratory.m47_reference_graph import read_m47_reference_graph_lab + LAB_SCHEMA: Final = "missioncore.lab-v1-vegetation-shadow/v1" WORKER_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1" RESULT_PREFIX: Final = "lab-v1-vegetation-shadow-" _CANDIDATES: Final = ("ddrnet", "ppliteseg") _MODES: Final = ("goose", "ravnoves") +_VIDEO_MODE: Final = "ravnoves-video" +_VIDEO_FRAME_COUNT: Final = 4489 +_M4_RESULT_ID: Final = re.compile(r"^m4-threat-replay-[a-f0-9]{64}$") +_VIDEO_WORKER_RESULT_ID: Final = re.compile( + r"^lab-v1-ravnoves-video-ddrnet-[a-f0-9]{64}$", +) +_SHA256: Final = re.compile(r"^[a-f0-9]{64}$") _FOCUS_ORDER: Final = ( "high_grass", "low_grass", @@ -195,6 +206,106 @@ def _validation_metric_summary(result: dict[str, Any], candidate: str) -> dict[s } +def _validated_video_semantics( + root: Path, + result: dict[str, Any], +) -> tuple[dict[str, object], Path]: + source = _object(result.get("source"), "DDRNet video source") + video = _object(result.get("video_semantics"), "DDRNet video semantics") + archive = _object(video.get("mask_archive"), "DDRNet video mask archive") + taxonomy = _object(video.get("taxonomy"), "DDRNet video taxonomy") + classes = taxonomy.get("classes") + base_m4_result_id = video.get("base_m4_result_id") + worker_result_id = result.get("result_id") + aggregate_prediction_pixels = video.get("aggregate_prediction_pixels") + if ( + source.get("input_count") != _VIDEO_FRAME_COUNT + or source.get("ground_truth_available") is not False + or taxonomy.get("schema_version") + != "missioncore.lab-v1-vegetation-taxonomy/v1" + or not isinstance(classes, list) + or len(classes) != 64 + or not isinstance(base_m4_result_id, str) + or _M4_RESULT_ID.fullmatch(base_m4_result_id) is None + or not isinstance(worker_result_id, str) + or _VIDEO_WORKER_RESULT_ID.fullmatch(worker_result_id) is None + or not isinstance(aggregate_prediction_pixels, list) + or len(aggregate_prediction_pixels) != 64 + or any(type(count) is not int or count < 0 for count in aggregate_prediction_pixels) + or sum(aggregate_prediction_pixels) != _VIDEO_FRAME_COUNT * 800 * 600 + or video.get("center_crop_xyxy") != [100, 0, 700, 600] + or video.get("outside_crop_state") != "undefined" + or archive.get("path") != "semantic-masks.zip" + or archive.get("frame_count") != _VIDEO_FRAME_COUNT + or archive.get("width") != 800 + or archive.get("height") != 600 + or archive.get("encoding") != "uint8-class-id-png" + or archive.get("media_type") != "application/zip" + or archive.get("sequence_binding") + != "sequence-0-to-masks/frame-000001.png" + ): + raise VegetationShadowLabError("DDRNet full-video contract changed") + for expected_id, raw_class in enumerate(classes): + row = _object(raw_class, "DDRNet taxonomy class") + color = row.get("color_rgb") + if ( + row.get("class_id") != expected_id + or not isinstance(row.get("label"), str) + or not row["label"] + or row.get("disposition") + not in ({"undefined"} if expected_id == 0 else {"prediction"}) + or not isinstance(color, list) + or len(color) != 3 + or any(not isinstance(channel, int) or not 0 <= channel <= 255 for channel in color) + ): + raise VegetationShadowLabError("DDRNet video taxonomy changed") + archive_path = root / "semantic-masks.zip" + expected_sha256 = archive.get("sha256") + expected_bytes = archive.get("byte_length") + if ( + archive_path.is_symlink() + or not archive_path.is_file() + or type(expected_bytes) is not int + or expected_bytes <= 0 + or archive_path.stat().st_size != expected_bytes + or not isinstance(expected_sha256, str) + or _SHA256.fullmatch(expected_sha256) is None + or sha256_path(archive_path) != expected_sha256 + ): + raise VegetationShadowLabError("DDRNet video mask archive proof changed") + expected_members = [ + f"masks/frame-{sequence + 1:06d}.png" + for sequence in range(_VIDEO_FRAME_COUNT) + ] + try: + with zipfile.ZipFile(archive_path) as frozen: + members = frozen.infolist() + if ( + [member.filename for member in members] != expected_members + or any( + member.is_dir() + or member.file_size < 8 + or member.file_size > 1024 * 1024 + for member in members + ) + ): + raise VegetationShadowLabError("DDRNet video mask sequence changed") + except zipfile.BadZipFile as exc: + raise VegetationShadowLabError("DDRNet video mask archive is invalid") from exc + return { + "worker_result_id": worker_result_id, + "base_m4_result_id": base_m4_result_id, + "frame_count": _VIDEO_FRAME_COUNT, + "width": 800, + "height": 600, + "center_crop_xyxy": [100, 0, 700, 600], + "outside_crop_state": "undefined", + "sequence_binding": archive["sequence_binding"], + "taxonomy": taxonomy, + "aggregate_prediction_pixels": aggregate_prediction_pixels, + }, archive_path + + def seal_vegetation_shadow_lab( *, ddrnet_goose_root: Path, @@ -202,6 +313,8 @@ def seal_vegetation_shadow_lab( ddrnet_ravnoves_root: Path, ppliteseg_ravnoves_root: Path, output_root: Path, + ddrnet_ravnoves_video_root: Path | None = None, + m47_reference_graph_lab_root: Path | None = None, ) -> Path: roots = { ("ddrnet", "goose"): ddrnet_goose_root.resolve(), @@ -218,6 +331,29 @@ def seal_vegetation_shadow_lab( if cases[("ddrnet", mode)].keys() != cases[("ppliteseg", mode)].keys(): raise VegetationShadowLabError(f"{mode} candidate case islands differ") selected = _selected_candidate(results) + if (ddrnet_ravnoves_video_root is None) != (m47_reference_graph_lab_root is None): + raise VegetationShadowLabError("full-video Worker and M4.7 roots must be paired") + route_video: dict[str, object] | None = None + route_video_archive: Path | None = None + video_result: dict[str, Any] | None = None + if ddrnet_ravnoves_video_root is not None and m47_reference_graph_lab_root is not None: + video_root = ddrnet_ravnoves_video_root.resolve() + video_result = _read_worker_result( + video_root, + candidate="ddrnet", + mode=_VIDEO_MODE, + ) + route_video, route_video_archive = _validated_video_semantics(video_root, video_result) + m47 = read_m47_reference_graph_lab(m47_reference_graph_lab_root) + m47_source = _object(m47.report.get("source"), "M4.7 source") + m47_visual = _object(m47.report.get("visual_evidence"), "M4.7 visual evidence") + if ( + m47_source.get("source_id") != "RAVNOVES00" + or m47_visual.get("linked_result_id") != route_video["base_m4_result_id"] + or m47_visual.get("timeline_frames") != _VIDEO_FRAME_COUNT + ): + raise VegetationShadowLabError("M4.7 video binding differs from DDRNet source") + route_video["m47_reference_graph_result_id"] = m47.result_id output_root.mkdir(mode=0o700, parents=True, exist_ok=True) temporary = Path(tempfile.mkdtemp(prefix=".lab-v1-vegetation-", dir=output_root)) @@ -315,6 +451,34 @@ def seal_vegetation_shadow_lab( "path": relative, "sha256": descriptor["sha256"], } + if video_result is not None and route_video is not None and route_video_archive is not None: + video_root = ddrnet_ravnoves_video_root.resolve() # type: ignore[union-attr] + worker_descriptor = _copy_artifact( + video_root / "result.json", + temporary, + "worker/ddrnet-ravnoves-video.json", + artifacts, + role="worker-result", + media_type="application/json", + ) + worker_proofs["ddrnet_ravnoves_video"] = { + "result_id": video_result.get("result_id"), + "path": worker_descriptor["path"], + "sha256": worker_descriptor["sha256"], + } + archive_descriptor = _copy_artifact( + route_video_archive, + temporary, + "video/ddrnet-semantic-masks.zip", + artifacts, + role="route-semantic-mask-archive", + media_type="application/zip", + ) + route_video["mask_archive"] = { + "path": archive_descriptor["path"], + "sha256": archive_descriptor["sha256"], + "byte_length": archive_descriptor["byte_length"], + } candidate_metrics: dict[str, object] = {} for candidate in _CANDIDATES: @@ -348,11 +512,13 @@ def seal_vegetation_shadow_lab( "shadow_session": "RAVNOVES00", "shadow_camera": "sensor.camera.right", "shadow_frame_count": 12, + "video_shadow_frame_count": _VIDEO_FRAME_COUNT if route_video else 0, }, "selected_candidate": selected, "candidate_metrics": candidate_metrics, "worker_proofs": worker_proofs, "visual_catalog_sha256": hashlib.sha256(canonical_json(catalogs)).hexdigest(), + "route_video": route_video, "authority": authority, } identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() @@ -366,6 +532,7 @@ def seal_vegetation_shadow_lab( "status": "visual-shadow-ready-policy-not-authorized", "identity": identity, "source": identity["source"], + "route_video": route_video, "method": { "completeness": "complete", "execution_class": "ai-inference", @@ -375,13 +542,14 @@ def seal_vegetation_shadow_lab( "decision": { "selected_candidate": selected, "visual_shadow_ready": True, + "full_video_shadow_ready": route_video is not None, "mission_policy_ready_for_configuration": True, "navigation_accepted": False, "production_accepted": False, }, "limitations": [ "GOOSE validation is external-domain qualification, not RAVNOVES ground truth.", - "The RAVNOVES shadow remains in Worker proofs and is not catalogued as vegetation evidence because it has no independent labels.", + "The full RAVNOVES DDRNet playback is prediction-only and has no independent labels.", "Vegetation semantics never clears rigid LiDAR/TGS occupancy.", "Undefined pixels outside the 600x600 center crop remain fail-closed.", ], @@ -407,6 +575,8 @@ def _parse_args() -> argparse.Namespace: parser.add_argument("--ddrnet-ravnoves-root", type=Path, required=True) parser.add_argument("--ppliteseg-ravnoves-root", type=Path, required=True) parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--ddrnet-ravnoves-video-root", type=Path) + parser.add_argument("--m47-reference-graph-lab-root", type=Path) return parser.parse_args() @@ -418,6 +588,8 @@ def main() -> None: ddrnet_ravnoves_root=args.ddrnet_ravnoves_root, ppliteseg_ravnoves_root=args.ppliteseg_ravnoves_root, output_root=args.output_root, + ddrnet_ravnoves_video_root=args.ddrnet_ravnoves_video_root, + m47_reference_graph_lab_root=args.m47_reference_graph_lab_root, ) print(destination) diff --git a/src/k1link/web/vegetation_shadow_lab_api.py b/src/k1link/web/vegetation_shadow_lab_api.py index 1741d91..946891c 100644 --- a/src/k1link/web/vegetation_shadow_lab_api.py +++ b/src/k1link/web/vegetation_shadow_lab_api.py @@ -3,15 +3,17 @@ from __future__ import annotations import copy +import hashlib import json import re +import zipfile from collections.abc import Callable from functools import lru_cache from pathlib import Path, PurePosixPath from typing import Any, Final from fastapi import APIRouter, HTTPException -from fastapi.responses import FileResponse +from fastapi.responses import FileResponse, Response from k1link.laboratory.evidence_registry import LaboratoryEvidenceDefinition from k1link.laboratory.evidence_report import ( @@ -86,6 +88,48 @@ def build_vegetation_shadow_lab_router( }, ) + @router.get("/{result_id}/masks/{sequence}") + def get_video_mask(result_id: str, sequence: int) -> Response: + candidate = _resolve_candidate(root_provider, result_id) + manifest = _read_verified(candidate) + route_video = manifest.get("route_video") + if not isinstance(route_video, dict) or not 0 <= sequence < 4489: + raise HTTPException(status_code=404, detail="Vegetation video mask not found") + archive = route_video.get("mask_archive") + if not isinstance(archive, dict) or archive.get("path") != "video/ddrnet-semantic-masks.zip": + raise HTTPException(status_code=404, detail="Vegetation video mask not found") + archive_path = candidate / "video" / "ddrnet-semantic-masks.zip" + member = f"masks/frame-{sequence + 1:06d}.png" + try: + before = archive_path.stat() + with zipfile.ZipFile(archive_path) as frozen: + info = frozen.getinfo(member) + if info.is_dir() or info.file_size < 8 or info.file_size > 1024 * 1024: + raise ValueError("Vegetation video mask member is invalid") + payload = frozen.read(info) + after = archive_path.stat() + if ( + before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or len(payload) != info.file_size + ): + raise ValueError("Vegetation video mask archive changed during read") + except (KeyError, OSError, ValueError, zipfile.BadZipFile): + raise HTTPException( + status_code=503, + detail="Vegetation video mask failed verification", + ) from None + digest = hashlib.sha256(payload).hexdigest() + return Response( + content=payload, + media_type="image/png", + headers={ + "Cache-Control": "private, max-age=31536000, immutable", + "ETag": f'"{digest}"', + "X-Content-Type-Options": "nosniff", + }, + ) + return router diff --git a/tests/test_vegetation_shadow_lab.py b/tests/test_vegetation_shadow_lab.py index 6366186..3f005da 100644 --- a/tests/test_vegetation_shadow_lab.py +++ b/tests/test_vegetation_shadow_lab.py @@ -2,12 +2,15 @@ from __future__ import annotations import hashlib import json +import zipfile from pathlib import Path +from types import SimpleNamespace from fastapi import FastAPI from fastapi.testclient import TestClient from k1link.laboratory import LaboratoryEvidenceRegistry +import k1link.laboratory.vegetation_shadow_lab as vegetation_lab_module from k1link.laboratory.evidence_report import verify_laboratory_evidence_result from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router @@ -87,19 +90,98 @@ def _worker_result(root: Path, *, candidate: str, mode: str, vegetation_iou: flo (root / "result.json").write_text(json.dumps(payload), encoding="utf-8") -def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) -> None: +def _video_worker_result(root: Path) -> None: + root.mkdir(parents=True) + archive = root / "semantic-masks.zip" + mask = b"\x89PNG\r\n\x1a\n" + with zipfile.ZipFile(archive, "x", compression=zipfile.ZIP_STORED) as frozen: + for sequence in range(4489): + frozen.writestr(f"masks/frame-{sequence + 1:06d}.png", mask) + taxonomy = { + "schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1", + "classes": [ + { + "class_id": class_id, + "label": "undefined" if class_id == 0 else f"class-{class_id}", + "color_rgb": [class_id, class_id, class_id], + "disposition": "undefined" if class_id == 0 else "prediction", + } + for class_id in range(64) + ], + } + payload = { + "schema_version": "missioncore.lab-v1-goose-vegetation-run/v1", + "result_id": f"lab-v1-ravnoves-video-ddrnet-{'e' * 64}", + "mode": "ravnoves-video", + "candidate": {"candidate_key": "ddrnet"}, + "source": { + "input_count": 4489, + "ground_truth_available": False, + }, + "video_semantics": { + "base_m4_result_id": f"m4-threat-replay-{'f' * 64}", + "mask_archive": { + "path": "semantic-masks.zip", + "sha256": _sha256(archive), + "byte_length": archive.stat().st_size, + "frame_count": 4489, + "width": 800, + "height": 600, + "encoding": "uint8-class-id-png", + "media_type": "application/zip", + "sequence_binding": "sequence-0-to-masks/frame-000001.png", + }, + "taxonomy": taxonomy, + "aggregate_prediction_pixels": [4489 * 800 * 600, *([0] * 63)], + "center_crop_xyxy": [100, 0, 700, 600], + "outside_crop_state": "undefined", + }, + "authority": { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + }, + } + (root / "result.json").write_text(json.dumps(payload), encoding="utf-8") + + +def test_vegetation_shadow_lab_seals_autonomous_visual_evidence( + tmp_path: Path, + monkeypatch, +) -> None: roots = {} for candidate, vegetation_iou in (("ddrnet", 0.64), ("ppliteseg", 0.61)): for mode in ("goose", "ravnoves"): root = tmp_path / "worker" / f"{candidate}-{mode}" _worker_result(root, candidate=candidate, mode=mode, vegetation_iou=vegetation_iou) roots[(candidate, mode)] = root + video_root = tmp_path / "worker" / "ddrnet-ravnoves-video" + _video_worker_result(video_root) + m47_root = tmp_path / f"m47-reference-graph-lab-{'a' * 64}" + m47_root.mkdir() + monkeypatch.setattr( + vegetation_lab_module, + "read_m47_reference_graph_lab", + lambda _root: SimpleNamespace( + result_id=m47_root.name, + report={ + "source": {"source_id": "RAVNOVES00"}, + "visual_evidence": { + "linked_result_id": f"m4-threat-replay-{'f' * 64}", + "timeline_frames": 4489, + }, + }, + ), + ) result_root = seal_vegetation_shadow_lab( ddrnet_goose_root=roots[("ddrnet", "goose")], ppliteseg_goose_root=roots[("ppliteseg", "goose")], ddrnet_ravnoves_root=roots[("ddrnet", "ravnoves")], ppliteseg_ravnoves_root=roots[("ppliteseg", "ravnoves")], output_root=tmp_path / "results", + ddrnet_ravnoves_video_root=video_root, + m47_reference_graph_lab_root=m47_root, ) manifest = json.loads((result_root / "result.json").read_text("utf-8")) assert manifest["decision"]["selected_candidate"] == "ddrnet" @@ -108,7 +190,9 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) assert manifest["authority"]["navigation_or_safety_accepted"] is False assert len(manifest["catalogs"]["ravnoves"]) == 0 assert len(manifest["catalogs"]["goose"]) == 12 - assert len(manifest["artifacts"]) == 76 + assert len(manifest["artifacts"]) == 78 + assert manifest["route_video"]["frame_count"] == 4489 + assert manifest["route_video"]["outside_crop_state"] == "undefined" assert manifest["catalogs"]["goose"][0]["focus"]["class_name"] == "high_grass" assert "ddrnet_error" in manifest["catalogs"]["goose"][0]["assets"] assert "ppliteseg_error" in manifest["catalogs"]["goose"][0]["assets"] @@ -121,7 +205,7 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) ) proof = verify_laboratory_evidence_result(definition, result_root) assert proof["result_id"] == result_root.name - assert proof["artifact_count"] == 76 + assert proof["artifact_count"] == 78 app = FastAPI() app.include_router(build_vegetation_shadow_lab_router(root_provider=lambda: result_root.parent)) @@ -135,6 +219,10 @@ def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) ) assert asset.status_code == 200 assert asset.headers["cache-control"].endswith("immutable") + mask = client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}/masks/0") + assert mask.status_code == 200 + assert mask.content == b"\x89PNG\r\n\x1a\n" + assert mask.headers["cache-control"].endswith("immutable") (result_root / asset_path).write_bytes(b"tampered") assert (