SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(perception): add DDRNet full-video vegetation replay

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-28 01:13:18 +03:00
parent 67e6ae98e2
commit 30080c51aa
10 changed files with 729 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/worker/Invoke-LabV1VegetationGooseBenchmark.ps1
+29 -4
View File
@@ -1,6 +1,6 @@
[CmdletBinding()]
param(
[ValidateSet("Build", "Probe", "Validate", "Ravnoves", "Status")]
[ValidateSet("Build", "Probe", "Validate", "Ravnoves", "RavnovesVideo", "Status")]
[string]$Mode = "Status",
[ValidateSet("Ddrnet", "Ppliteseg")]
@@ -104,12 +104,13 @@ function New-RunRoot {
function Invoke-IsolatedRun {
param(
[ValidateSet("goose", "ravnoves")][string]$RunMode,
[ValidateSet("goose", "ravnoves", "ravnoves-video")][string]$RunMode,
[string]$RunRoot,
[int]$Limit,
[string]$FramesRoot = ""
)
$containerName = "ndc-lab-v1-goose-$candidateKey-$([Guid]::NewGuid().ToString('N').Substring(0, 10))"
$visualCount = if ($RunMode -eq "ravnoves-video") { 0 } else { 12 }
$arguments = @(
"run", "--rm", "--name", $containerName,
"--gpus", "all",
@@ -136,9 +137,9 @@ function Invoke-IsolatedRun {
"--dataset-root", "/data/goose",
"--output", "/output/result",
"--limit", $Limit.ToString(),
"--visual-count", "12"
"--visual-count", $visualCount.ToString()
)
if ($RunMode -eq "ravnoves") {
if ($RunMode -in @("ravnoves", "ravnoves-video")) {
$arguments = @($arguments[0..($arguments.Count - 1)])
$arguments += @("--frames-root", "/input")
$mountIndex = [Array]::IndexOf($arguments, $image)
@@ -172,6 +173,20 @@ function Export-RavnovesFrames {
}
}
function Export-RavnovesVideoFrames {
param([string]$Destination)
Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $ravnovesSha256
New-Item -ItemType Directory -Path $Destination | Out-Null
& ffmpeg -hide_banner -loglevel error -i $RavnovesVideo -map 0:v:0 -fps_mode passthrough (Join-Path $Destination "frame-%06d.png")
if ($LASTEXITCODE -ne 0) {
throw "RAVNOVES full-video frame extraction failed"
}
$frames = @(Get-ChildItem -LiteralPath $Destination -File -Filter "frame-*.png" | Sort-Object Name)
if ($frames.Count -ne 4489 -or $frames[0].Name -ne "frame-000001.png" -or $frames[-1].Name -ne "frame-004489.png") {
throw "RAVNOVES full-video frame sequence changed"
}
}
if ($Mode -eq "Status") {
$imageIdentity = & docker image inspect $image --format "{{.Id}}" 2>$null
[ordered]@{
@@ -221,6 +236,16 @@ try {
Export-RavnovesFrames -Destination $framesRoot
Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
}
elseif ($Mode -eq "RavnovesVideo") {
if ($candidateKey -ne "ddrnet") {
throw "Full-video shadow is admitted only for the selected DDRNet candidate"
}
$runRoot = New-RunRoot -Kind "ravnoves-video"
$framesRoot = Join-Path $runRoot "input-frames"
Export-RavnovesVideoFrames -Destination $framesRoot
Invoke-IsolatedRun -RunMode "ravnoves-video" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
Remove-Item -LiteralPath $framesRoot -Recurse -Force
}
}
finally {
$canonicalAfter = Get-CanonicalTritonIdentity
experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py
+103 -3
View File
@@ -11,6 +11,7 @@ import os
import platform
import statistics
import time
import zipfile
from pathlib import Path
from typing import Any
@@ -43,7 +44,11 @@ class RunnerError(RuntimeError):
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--mode", choices=("goose", "ravnoves"), required=True)
parser.add_argument(
"--mode",
choices=("goose", "ravnoves", "ravnoves-video"),
required=True,
)
parser.add_argument("--candidate", choices=tuple(MODEL_NAMES), required=True)
parser.add_argument("--config", type=Path, required=True)
parser.add_argument("--policy", type=Path, required=True)
@@ -338,6 +343,40 @@ def save_image(path: Path, value: Image.Image | np.ndarray, mode: str | None = N
return sha256(path)
def write_mask_archive(
output: Path,
masks_root: Path,
frame_count: int,
) -> dict[str, Any]:
archive_path = output / "semantic-masks.zip"
expected = [f"frame-{sequence + 1:06d}.png" for sequence in range(frame_count)]
actual = sorted(path.name for path in masks_root.glob("frame-*.png"))
if actual != expected:
raise RunnerError("RAVNOVES video mask sequence is incomplete")
with zipfile.ZipFile(
archive_path,
mode="x",
compression=zipfile.ZIP_STORED,
allowZip64=True,
) as archive:
for name in expected:
archive.write(masks_root / name, arcname=f"masks/{name}")
for path in masks_root.iterdir():
path.unlink()
masks_root.rmdir()
return {
"path": archive_path.name,
"sha256": sha256(archive_path),
"byte_length": archive_path.stat().st_size,
"media_type": "application/zip",
"frame_count": frame_count,
"width": 800,
"height": 600,
"encoding": "uint8-class-id-png",
"sequence_binding": "sequence-0-to-masks/frame-000001.png",
}
def expand_mask(
mask: np.ndarray,
original_size: tuple[int, int],
@@ -517,7 +556,7 @@ def run() -> None:
(image.stem.removesuffix("_windshield_vis"), image, label)
for image, label in pairs
]
else:
elif args.mode == "ravnoves":
if args.frames_root is None or not args.frames_root.is_dir():
raise RunnerError("frames-root is required for RAVNOVES mode")
frames = sorted(args.frames_root.glob("frame-*.png"))
@@ -525,6 +564,15 @@ def run() -> None:
if {frame.stem for frame in frames} != expected:
raise RunnerError("RAVNOVES frame island identity changed")
items = [(frame.stem, frame, None) for frame in frames]
else:
if args.frames_root is None or not args.frames_root.is_dir():
raise RunnerError("frames-root is required for RAVNOVES video mode")
frames = sorted(args.frames_root.glob("frame-*.png"))
expected_count = config["ravnoves"]["expected_frame_count"]
expected_names = [f"frame-{sequence + 1:06d}.png" for sequence in range(expected_count)]
if len(frames) != expected_count or [frame.name for frame in frames] != expected_names:
raise RunnerError("RAVNOVES full-video frame sequence changed")
items = [(frame.stem, frame, None) for frame in frames]
if args.limit:
items = items[: args.limit]
@@ -539,8 +587,12 @@ def run() -> None:
if args.visual_count != configured_visual_count:
raise RunnerError("GOOSE visual count differs from the truth-focused contract")
selected_visuals = truth_focused_visuals(items, names, visual_contract)
else:
elif args.mode == "ravnoves":
selected_visuals = visual_indices(len(items), args.visual_count)
else:
if args.visual_count != 0 or args.limit:
raise RunnerError("RAVNOVES video mode requires the complete frame sequence")
selected_visuals = {}
args.output.mkdir(parents=True, exist_ok=False)
torch.cuda.empty_cache()
@@ -552,12 +604,28 @@ def run() -> None:
confusion = np.zeros((CLASS_COUNT, CLASS_COUNT), dtype=np.int64)
latencies_ms: list[float] = []
visuals: list[dict[str, Any]] = []
mask_root = args.output / "masks" if args.mode == "ravnoves-video" else None
if mask_root is not None:
mask_root.mkdir()
aggregate_prediction_pixels = np.zeros(CLASS_COUNT, dtype=np.int64)
for index, (case_id, source_path, label_path) in enumerate(items):
source = Image.open(source_path).convert("RGB")
if args.mode == "ravnoves-video" and source.size != (
config["ravnoves"]["expected_width"],
config["ravnoves"]["expected_height"],
):
raise RunnerError("RAVNOVES video frame dimensions changed")
tensor, crop_box = preprocess(source)
prediction, latency_ms = infer(model, tensor)
latencies_ms.append(latency_ms)
if mask_root is not None:
expanded_prediction = expand_mask(prediction, source.size, crop_box)
save_image(mask_root / f"frame-{index + 1:06d}.png", expanded_prediction, "L")
aggregate_prediction_pixels += np.bincount(
expanded_prediction.reshape(-1),
minlength=CLASS_COUNT,
)
truth = preprocess_label(Image.open(label_path)) if label_path is not None else None
if truth is not None:
update_confusion(confusion, truth, prediction)
@@ -579,6 +647,23 @@ def run() -> None:
)
)
mask_archive = (
write_mask_archive(args.output, mask_root, len(items))
if mask_root is not None
else None
)
taxonomy = {
"schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1",
"classes": [
{
"class_id": label_id,
"label": names[label_id],
"color_rgb": semantic_palette[label_id, :3].astype(int).tolist(),
"disposition": "undefined" if label_id == 0 else "prediction",
}
for label_id in range(CLASS_COUNT)
],
}
rows = class_metrics(confusion, names) if args.mode == "goose" else []
valid_ious = [row["iou"] for row in rows if row["iou"] is not None]
vegetation_names = set(config["vegetation_class_names"])
@@ -615,6 +700,20 @@ def run() -> None:
"ground_truth_available": args.mode == "goose",
"mapping_sha256": dataset_config["mapping_sha256"],
},
"video_semantics": {
"base_m4_result_id": config["ravnoves"].get("base_m4_result_id"),
"mask_archive": mask_archive,
"taxonomy": taxonomy,
"aggregate_prediction_pixels": aggregate_prediction_pixels.tolist()
if mask_archive is not None
else None,
"center_crop_xyxy": [100, 0, 700, 600]
if mask_archive is not None
else None,
"outside_crop_state": "undefined" if mask_archive is not None else None,
}
if args.mode == "ravnoves-video"
else None,
"preprocessing": dataset_config["preprocessing"],
"metrics": {
"mean_iou": round(statistics.fmean(valid_ious), 8) if valid_ious else None,
@@ -650,6 +749,7 @@ def run() -> None:
"schema_version": result["schema_version"],
"candidate": result["candidate"],
"source": result["source"],
"video_semantics": result["video_semantics"],
"preprocessing": result["preprocessing"],
"metrics": result["metrics"],
"timing": result["timing"],