feat(perception): add autonomous vegetation shadow lab
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,230 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Build", "Probe", "Validate", "Ravnoves", "Status")]
|
||||
[string]$Mode = "Status",
|
||||
|
||||
[ValidateSet("Ddrnet", "Ppliteseg")]
|
||||
[string]$Candidate = "Ddrnet",
|
||||
|
||||
[string]$AssetRoot = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27",
|
||||
|
||||
[string]$ToolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\lab-v1-vegetation-goose",
|
||||
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\experiments\lab-v1-vegetation",
|
||||
|
||||
[string]$RavnovesVideo = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$datasetPrefix = "D:\NDC_MISSIONCORE\datasets\"
|
||||
$runtimePrefix = "D:\NDC_MISSIONCORE\runtime\experiments\"
|
||||
if (-not $AssetRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "AssetRoot must stay under $datasetPrefix"
|
||||
}
|
||||
if (-not $ToolRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "ToolRoot must stay under $datasetPrefix"
|
||||
}
|
||||
if (-not $OutputRoot.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "OutputRoot must stay under $runtimePrefix"
|
||||
}
|
||||
|
||||
$image = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1"
|
||||
$canonicalContainer = "ndc-mission-core-triton"
|
||||
$candidateKey = $Candidate.ToLowerInvariant()
|
||||
$contextRoot = Join-Path $ToolRoot "context"
|
||||
$configRoot = Join-Path $ToolRoot "config"
|
||||
$benchmarkConfig = Join-Path $configRoot "lab-v1-goose-vegetation-benchmark-v1.json"
|
||||
$policyConfig = Join-Path $configRoot "lab-v1-vegetation-mission-policy-v1.json"
|
||||
$providerMapConfig = Join-Path $configRoot "lab-v1-vegetation-provider-label-map-v1.json"
|
||||
$datasetRoot = Join-Path $AssetRoot "goose-2d\validation"
|
||||
$checkpointRelative = if ($candidateKey -eq "ddrnet") {
|
||||
"models\goose\ddrnet_class_512.pth"
|
||||
} else {
|
||||
"models\goose\ppliteseg_class_512.pth"
|
||||
}
|
||||
$checkpoint = Join-Path $AssetRoot $checkpointRelative
|
||||
$expectedCheckpointSha256 = if ($candidateKey -eq "ddrnet") {
|
||||
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
} else {
|
||||
"6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd"
|
||||
}
|
||||
$expectedCheckpointBytes = if ($candidateKey -eq "ddrnet") { 259419077 } else { 98208249 }
|
||||
$ravnovesSha256 = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8"
|
||||
$frameIndices = @(0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488)
|
||||
$dockerConfig = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation\docker-config"
|
||||
|
||||
function Get-CanonicalTritonIdentity {
|
||||
$identity = & docker inspect $canonicalContainer --format "{{.Id}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}"
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($identity)) {
|
||||
throw "Canonical Triton is unavailable"
|
||||
}
|
||||
$parts = $identity.Split("|")
|
||||
if ($parts.Count -ne 4 -or $parts[2] -ne "running" -or $parts[3] -ne "healthy") {
|
||||
throw "Canonical Triton is not running and healthy: $identity"
|
||||
}
|
||||
return $identity
|
||||
}
|
||||
|
||||
function Assert-FileIdentity {
|
||||
param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256)
|
||||
$file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue
|
||||
if ($null -eq $file -or $file.Length -ne $ExpectedBytes) {
|
||||
throw "File identity changed: $Path"
|
||||
}
|
||||
$actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actualSha256 -ne $ExpectedSha256) {
|
||||
throw "File digest changed: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RunnerInputs {
|
||||
foreach ($path in @($benchmarkConfig, $policyConfig, $providerMapConfig)) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Runner config is unavailable: $path"
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "Dockerfile") -PathType Leaf)) {
|
||||
throw "Runner Dockerfile is unavailable"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "run_goose_vegetation_benchmark.py") -PathType Leaf)) {
|
||||
throw "Runner source is unavailable"
|
||||
}
|
||||
Assert-FileIdentity -Path $checkpoint -ExpectedBytes $expectedCheckpointBytes -ExpectedSha256 $expectedCheckpointSha256
|
||||
}
|
||||
|
||||
function New-RunRoot {
|
||||
param([string]$Kind)
|
||||
$stamp = [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssfffZ")
|
||||
$path = Join-Path $OutputRoot ("{0}-{1}-{2}" -f $Kind, $candidateKey, $stamp)
|
||||
New-Item -ItemType Directory -Path $path | Out-Null
|
||||
return $path
|
||||
}
|
||||
|
||||
function Invoke-IsolatedRun {
|
||||
param(
|
||||
[ValidateSet("goose", "ravnoves")][string]$RunMode,
|
||||
[string]$RunRoot,
|
||||
[int]$Limit,
|
||||
[string]$FramesRoot = ""
|
||||
)
|
||||
$containerName = "ndc-lab-v1-goose-$candidateKey-$([Guid]::NewGuid().ToString('N').Substring(0, 10))"
|
||||
$arguments = @(
|
||||
"run", "--rm", "--name", $containerName,
|
||||
"--gpus", "all",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--cap-drop", "ALL",
|
||||
"--security-opt", "no-new-privileges",
|
||||
"--memory", "10g",
|
||||
"--cpus", "8",
|
||||
"--pids-limit", "512",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"--env", "HOME=/tmp",
|
||||
"--mount", "type=bind,src=$datasetRoot,dst=/data/goose,readonly",
|
||||
"--mount", "type=bind,src=$checkpoint,dst=/models/candidate.pth,readonly",
|
||||
"--mount", "type=bind,src=$configRoot,dst=/config,readonly",
|
||||
"--mount", "type=bind,src=$RunRoot,dst=/output",
|
||||
$image,
|
||||
"--mode", $RunMode,
|
||||
"--candidate", $candidateKey,
|
||||
"--config", "/config/lab-v1-goose-vegetation-benchmark-v1.json",
|
||||
"--policy", "/config/lab-v1-vegetation-mission-policy-v1.json",
|
||||
"--provider-map", "/config/lab-v1-vegetation-provider-label-map-v1.json",
|
||||
"--checkpoint", "/models/candidate.pth",
|
||||
"--dataset-root", "/data/goose",
|
||||
"--output", "/output/result",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--visual-count", "12"
|
||||
)
|
||||
if ($RunMode -eq "ravnoves") {
|
||||
$arguments = @($arguments[0..($arguments.Count - 1)])
|
||||
$arguments += @("--frames-root", "/input")
|
||||
$mountIndex = [Array]::IndexOf($arguments, $image)
|
||||
$head = @($arguments[0..($mountIndex - 1)])
|
||||
$tail = @($arguments[$mountIndex..($arguments.Count - 1)])
|
||||
$arguments = $head + @("--mount", "type=bind,src=$FramesRoot,dst=/input,readonly") + $tail
|
||||
}
|
||||
& docker @arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 container failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Export-RavnovesFrames {
|
||||
param([string]$Destination)
|
||||
Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $ravnovesSha256
|
||||
New-Item -ItemType Directory -Path $Destination | Out-Null
|
||||
$expression = ($frameIndices | ForEach-Object { "eq(n\,$_ )" }) -join "+"
|
||||
$temporaryPattern = Join-Path $Destination "selected-%03d.png"
|
||||
& ffmpeg -hide_banner -loglevel error -i $RavnovesVideo -vf "select='$expression'" -fps_mode vfr $temporaryPattern
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "RAVNOVES exact frame extraction failed"
|
||||
}
|
||||
$selected = @(Get-ChildItem -LiteralPath $Destination -Filter "selected-*.png" | Sort-Object Name)
|
||||
if ($selected.Count -ne $frameIndices.Count) {
|
||||
throw "RAVNOVES frame island changed: expected $($frameIndices.Count), got $($selected.Count)"
|
||||
}
|
||||
for ($index = 0; $index -lt $selected.Count; $index++) {
|
||||
$target = Join-Path $Destination ("frame-{0:D6}.png" -f $frameIndices[$index])
|
||||
Move-Item -LiteralPath $selected[$index].FullName -Destination $target
|
||||
}
|
||||
}
|
||||
|
||||
if ($Mode -eq "Status") {
|
||||
$imageIdentity = & docker image inspect $image --format "{{.Id}}" 2>$null
|
||||
[ordered]@{
|
||||
schema_version = "missioncore.lab-v1-goose-runner-status/v1"
|
||||
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
||||
worker_id = "worker-006"
|
||||
image = $image
|
||||
image_id = if ($LASTEXITCODE -eq 0) { $imageIdentity } else { $null }
|
||||
canonical_triton = Get-CanonicalTritonIdentity
|
||||
asset_root = $AssetRoot
|
||||
output_root = $OutputRoot
|
||||
candidate = $candidateKey
|
||||
} | ConvertTo-Json -Depth 6
|
||||
exit 0
|
||||
}
|
||||
|
||||
Assert-RunnerInputs
|
||||
$canonicalBefore = Get-CanonicalTritonIdentity
|
||||
try {
|
||||
if ($Mode -eq "Build") {
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $dockerConfig "config.json") -PathType Leaf)) {
|
||||
throw "Isolated Docker client configuration is unavailable"
|
||||
}
|
||||
$previousDockerConfig = $env:DOCKER_CONFIG
|
||||
try {
|
||||
$env:DOCKER_CONFIG = $dockerConfig
|
||||
& docker build --pull=false --label "com.nodedc.component=mission-core-lab-v1-goose" --label "com.nodedc.authority=shadow-only" --tag $image $contextRoot
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 image build failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$env:DOCKER_CONFIG = $previousDockerConfig
|
||||
}
|
||||
}
|
||||
elseif ($Mode -eq "Probe") {
|
||||
$runRoot = New-RunRoot -Kind "probe"
|
||||
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 8
|
||||
}
|
||||
elseif ($Mode -eq "Validate") {
|
||||
$runRoot = New-RunRoot -Kind "validation"
|
||||
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 0
|
||||
}
|
||||
elseif ($Mode -eq "Ravnoves") {
|
||||
$runRoot = New-RunRoot -Kind "ravnoves"
|
||||
$framesRoot = Join-Path $runRoot "input-frames"
|
||||
Export-RavnovesFrames -Destination $framesRoot
|
||||
Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$canonicalAfter = Get-CanonicalTritonIdentity
|
||||
if ($canonicalAfter -ne $canonicalBefore) {
|
||||
throw "Canonical Triton identity changed during LAB V1 work"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PATH=/opt/conda/bin:$PATH
|
||||
|
||||
ARG MINICONDA_INSTALLER=Miniconda3-py39_24.11.1-0-Linux-x86_64.sh
|
||||
ARG MINICONDA_SHA256=3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
libglib2.0-0 \
|
||||
libgl1 \
|
||||
&& curl --fail --location --retry 5 \
|
||||
--output /tmp/miniconda.sh \
|
||||
"https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}" \
|
||||
&& echo "${MINICONDA_SHA256} /tmp/miniconda.sh" | sha256sum --check --strict \
|
||||
&& bash /tmp/miniconda.sh -b -p /opt/conda \
|
||||
&& rm -f /tmp/miniconda.sh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN conda create --yes --name goose python=3.9 pip \
|
||||
&& conda install --yes --name goose --channel pytorch --channel nvidia \
|
||||
pytorch=1.13.1 torchvision=0.14.1 pytorch-cuda=11.7
|
||||
|
||||
RUN conda run --name goose python -m pip install --no-cache-dir \
|
||||
cmake==3.31.6 \
|
||||
numpy==1.23.0 \
|
||||
onnxsim==0.4.36 \
|
||||
opencv-python==4.8.1.78 \
|
||||
protobuf==3.20.3 \
|
||||
pyparsing==2.4.5
|
||||
|
||||
RUN conda run --name goose python -m pip install --no-cache-dir \
|
||||
super-gradients==3.2.0 \
|
||||
torchmetrics==0.8.0
|
||||
|
||||
RUN conda clean --all --yes
|
||||
|
||||
WORKDIR /opt/mission-core/lab-v1
|
||||
COPY run_goose_vegetation_benchmark.py /opt/mission-core/lab-v1/runner.py
|
||||
|
||||
ENTRYPOINT ["conda", "run", "--no-capture-output", "--name", "goose", "python", "/opt/mission-core/lab-v1/runner.py"]
|
||||
+556
@@ -0,0 +1,556 @@
|
||||
"""Run isolated GOOSE vegetation qualification and RAVNOVES shadow inference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from super_gradients.training import models
|
||||
|
||||
SCHEMA = "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||
VISUAL_SCHEMA = "missioncore.lab-v1-goose-vegetation-visual-case/v1"
|
||||
CLASS_COUNT = 64
|
||||
MAX_CONFIG_BYTES = 1024 * 1024
|
||||
MODEL_NAMES = {
|
||||
"ddrnet": ("ddrnet_39",),
|
||||
"ppliteseg": (
|
||||
"pp_lite_t_seg",
|
||||
"pp_lite_t_seg50",
|
||||
"pp_lite_t_seg75",
|
||||
"pp_lite_b_seg",
|
||||
"pp_lite_b_seg50",
|
||||
"pp_lite_b_seg75",
|
||||
),
|
||||
}
|
||||
RESAMPLE_NEAREST = getattr(Image, "Resampling", Image).NEAREST
|
||||
|
||||
|
||||
class RunnerError(RuntimeError):
|
||||
"""The bounded runner input or output contract is invalid."""
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=("goose", "ravnoves"), required=True)
|
||||
parser.add_argument("--candidate", choices=tuple(MODEL_NAMES), required=True)
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
parser.add_argument("--policy", type=Path, required=True)
|
||||
parser.add_argument("--provider-map", type=Path, required=True)
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--dataset-root", type=Path)
|
||||
parser.add_argument("--frames-root", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--visual-count", type=int, default=12)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_json(path: Path, label: str) -> dict[str, Any]:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
|
||||
raise RunnerError(f"{label} is unavailable")
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RunnerError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def stable_digest(value: object) -> str:
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def validate_contracts(
|
||||
config: dict[str, Any], policy: dict[str, Any], provider_map: dict[str, Any], candidate: str
|
||||
) -> dict[str, Any]:
|
||||
if config.get("schema_version") != "missioncore.lab-v1-goose-vegetation-benchmark/v1":
|
||||
raise RunnerError("benchmark configuration identity changed")
|
||||
if policy.get("schema_version") != "missioncore.vegetation-mission-policy/v1":
|
||||
raise RunnerError("mission policy identity changed")
|
||||
if provider_map.get("schema_version") != "missioncore.vegetation-provider-label-map/v1":
|
||||
raise RunnerError("provider map identity changed")
|
||||
invariants = config.get("invariants")
|
||||
true_invariants = {"one_heavy_candidate_at_a_time", "raw_fisheye_is_immutable"}
|
||||
if not isinstance(invariants, dict) or any(
|
||||
value is not False for key, value in invariants.items() if key not in true_invariants
|
||||
):
|
||||
raise RunnerError("benchmark fail-closed invariants changed")
|
||||
if (
|
||||
invariants.get("one_heavy_candidate_at_a_time") is not True
|
||||
or invariants.get("raw_fisheye_is_immutable") is not True
|
||||
):
|
||||
raise RunnerError("benchmark isolation invariants changed")
|
||||
candidates = config.get("candidates")
|
||||
if not isinstance(candidates, dict) or not isinstance(candidates.get(candidate), dict):
|
||||
raise RunnerError("candidate is not configured")
|
||||
expected_models = candidates[candidate].get("model_names")
|
||||
if expected_models != list(MODEL_NAMES[candidate]):
|
||||
raise RunnerError("candidate architecture probe order changed")
|
||||
return candidates[candidate]
|
||||
|
||||
|
||||
def load_mapping(path: Path, expected_sha256: str) -> tuple[dict[int, str], np.ndarray]:
|
||||
if sha256(path) != expected_sha256:
|
||||
raise RunnerError("GOOSE label mapping digest changed")
|
||||
names: dict[int, str] = {}
|
||||
palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8)
|
||||
with path.open(newline="", encoding="utf-8-sig") as stream:
|
||||
for row in csv.DictReader(stream):
|
||||
label_id = int(row["label_key"])
|
||||
if label_id < 0 or label_id >= CLASS_COUNT:
|
||||
raise RunnerError("GOOSE label id is outside the 64-class contract")
|
||||
color = row["hex"].lstrip("#")
|
||||
names[label_id] = row["class_name"]
|
||||
palette[label_id] = (*bytes.fromhex(color), 190)
|
||||
if set(names) != set(range(CLASS_COUNT)):
|
||||
raise RunnerError("GOOSE mapping does not cover exactly 64 classes")
|
||||
palette[0, 3] = 0
|
||||
return names, palette
|
||||
|
||||
|
||||
def center_crop(image: Image.Image) -> tuple[Image.Image, tuple[int, int, int, int]]:
|
||||
side = min(image.width, image.height)
|
||||
left = (image.width - side) // 2
|
||||
top = (image.height - side) // 2
|
||||
box = (left, top, left + side, top + side)
|
||||
return image.crop(box), box
|
||||
|
||||
|
||||
def preprocess(image: Image.Image) -> tuple[torch.Tensor, tuple[int, int, int, int]]:
|
||||
cropped, crop_box = center_crop(image.convert("RGB"))
|
||||
resized = cropped.resize((512, 512), resample=RESAMPLE_NEAREST)
|
||||
array = np.asarray(resized, dtype=np.float32) / 255.0
|
||||
tensor = torch.from_numpy(np.transpose(array, (2, 0, 1))).unsqueeze(0)
|
||||
return tensor, crop_box
|
||||
|
||||
|
||||
def preprocess_label(image: Image.Image) -> np.ndarray:
|
||||
cropped, _ = center_crop(image.convert("L"))
|
||||
return np.asarray(cropped.resize((512, 512), resample=RESAMPLE_NEAREST), dtype=np.uint8)
|
||||
|
||||
|
||||
def find_goose_pairs(root: Path) -> list[tuple[Path, Path]]:
|
||||
pairs: list[tuple[Path, Path]] = []
|
||||
image_root = root / "images" / "val"
|
||||
label_root = root / "labels" / "val"
|
||||
for image_path in sorted(image_root.rglob("*_windshield_vis.png")):
|
||||
stem = image_path.name.removesuffix("_windshield_vis.png")
|
||||
relative_parent = image_path.parent.relative_to(image_root)
|
||||
label_path = label_root / relative_parent / f"{stem}_labelids.png"
|
||||
if label_path.is_file() and not label_path.is_symlink():
|
||||
pairs.append((image_path, label_path))
|
||||
return pairs
|
||||
|
||||
|
||||
def visual_indices(count: int, visual_count: int) -> set[int]:
|
||||
if count <= 0 or visual_count <= 0:
|
||||
return set()
|
||||
selected_count = min(count, visual_count)
|
||||
if selected_count == 1:
|
||||
return {0}
|
||||
return {
|
||||
round(index * (count - 1) / (selected_count - 1))
|
||||
for index in range(selected_count)
|
||||
}
|
||||
|
||||
|
||||
def load_model(candidate: str, checkpoint: Path) -> tuple[torch.nn.Module, str, list[str]]:
|
||||
failures: list[str] = []
|
||||
for model_name in MODEL_NAMES[candidate]:
|
||||
try:
|
||||
model = models.get(
|
||||
model_name=model_name,
|
||||
num_classes=CLASS_COUNT,
|
||||
checkpoint_path=str(checkpoint),
|
||||
)
|
||||
model.eval()
|
||||
model.cuda()
|
||||
return model, model_name, failures
|
||||
except Exception as error: # noqa: BLE001 - each upstream architecture is a probe
|
||||
failures.append(f"{model_name}: {type(error).__name__}: {str(error)[:240]}")
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
raise RunnerError("checkpoint did not load: " + " | ".join(failures))
|
||||
|
||||
|
||||
def logits_from_output(value: object) -> torch.Tensor:
|
||||
if isinstance(value, torch.Tensor) and value.ndim == 4 and value.shape[1] == CLASS_COUNT:
|
||||
return value
|
||||
if isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
try:
|
||||
return logits_from_output(item)
|
||||
except RunnerError:
|
||||
continue
|
||||
raise RunnerError("model output does not contain a 64-class raster")
|
||||
|
||||
|
||||
def infer(model: torch.nn.Module, tensor: torch.Tensor) -> tuple[np.ndarray, float]:
|
||||
tensor = tensor.cuda(non_blocking=True)
|
||||
torch.cuda.synchronize()
|
||||
started = time.perf_counter_ns()
|
||||
with torch.inference_mode():
|
||||
logits = logits_from_output(model(tensor))
|
||||
prediction = torch.argmax(torch.sigmoid(logits), dim=1)
|
||||
torch.cuda.synchronize()
|
||||
elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000.0
|
||||
return prediction[0].to(device="cpu", dtype=torch.uint8).numpy(), elapsed_ms
|
||||
|
||||
|
||||
def update_confusion(confusion: np.ndarray, truth: np.ndarray, prediction: np.ndarray) -> None:
|
||||
valid = (truth >= 0) & (truth < CLASS_COUNT)
|
||||
indices = CLASS_COUNT * truth[valid].astype(np.int64) + prediction[valid].astype(np.int64)
|
||||
confusion += np.bincount(indices, minlength=CLASS_COUNT**2).reshape(CLASS_COUNT, CLASS_COUNT)
|
||||
|
||||
|
||||
def class_metrics(confusion: np.ndarray, names: dict[int, str]) -> list[dict[str, Any]]:
|
||||
truth = confusion.sum(axis=1)
|
||||
predicted = confusion.sum(axis=0)
|
||||
intersection = np.diag(confusion)
|
||||
union = truth + predicted - intersection
|
||||
rows: list[dict[str, Any]] = []
|
||||
for label_id in range(CLASS_COUNT):
|
||||
rows.append(
|
||||
{
|
||||
"label_id": label_id,
|
||||
"class_name": names[label_id],
|
||||
"support_pixels": int(truth[label_id]),
|
||||
"predicted_pixels": int(predicted[label_id]),
|
||||
"intersection_pixels": int(intersection[label_id]),
|
||||
"union_pixels": int(union[label_id]),
|
||||
"iou": round(float(intersection[label_id] / union[label_id]), 8)
|
||||
if union[label_id]
|
||||
else None,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def hex_rgb(value: str) -> tuple[int, int, int]:
|
||||
raw = bytes.fromhex(value.removeprefix("#"))
|
||||
if len(raw) != 3:
|
||||
raise RunnerError("policy action color must be RGB")
|
||||
return raw[0], raw[1], raw[2]
|
||||
|
||||
|
||||
def policy_palette(
|
||||
names: dict[int, str], policy: dict[str, Any], provider_map: dict[str, Any], preset: str,
|
||||
action_colors: dict[str, str]
|
||||
) -> np.ndarray:
|
||||
palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8)
|
||||
labels = provider_map["providers"]["goose-fine-64"]["labels"]
|
||||
rules = policy["presets"][preset]
|
||||
for label_id, class_name in names.items():
|
||||
material = labels.get(class_name)
|
||||
if material is None:
|
||||
continue
|
||||
action = rules[material]
|
||||
palette[label_id] = (*hex_rgb(action_colors[action]), 190)
|
||||
return palette
|
||||
|
||||
|
||||
def save_image(path: Path, value: Image.Image | np.ndarray, mode: str | None = None) -> str:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
image = value if isinstance(value, Image.Image) else Image.fromarray(value, mode=mode)
|
||||
image.save(path, format="PNG", optimize=True)
|
||||
return sha256(path)
|
||||
|
||||
|
||||
def expand_mask(
|
||||
mask: np.ndarray,
|
||||
original_size: tuple[int, int],
|
||||
crop_box: tuple[int, int, int, int],
|
||||
) -> np.ndarray:
|
||||
left, top, right, bottom = crop_box
|
||||
side = right - left
|
||||
resized = Image.fromarray(mask, mode="L").resize((side, side), resample=RESAMPLE_NEAREST)
|
||||
canvas = np.zeros((original_size[1], original_size[0]), dtype=np.uint8)
|
||||
canvas[top:bottom, left:right] = np.asarray(resized, dtype=np.uint8)
|
||||
return canvas
|
||||
|
||||
|
||||
def write_visual_case(
|
||||
output: Path,
|
||||
case_id: str,
|
||||
source: Image.Image,
|
||||
prediction: np.ndarray,
|
||||
semantic_palette: np.ndarray,
|
||||
policy_palettes: dict[str, np.ndarray],
|
||||
crop_box: tuple[int, int, int, int],
|
||||
truth: np.ndarray | None = None,
|
||||
preserve_source_size: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
case_root = output / "cases" / case_id
|
||||
if preserve_source_size:
|
||||
source_image = source.convert("RGB")
|
||||
prediction_image = expand_mask(prediction, source_image.size, crop_box)
|
||||
truth_image = expand_mask(truth, source_image.size, crop_box) if truth is not None else None
|
||||
else:
|
||||
cropped, _ = center_crop(source.convert("RGB"))
|
||||
source_image = cropped.resize((512, 512), resample=RESAMPLE_NEAREST)
|
||||
prediction_image = prediction
|
||||
truth_image = truth
|
||||
|
||||
files: dict[str, dict[str, str]] = {}
|
||||
source_path = case_root / "source.png"
|
||||
files["source"] = {
|
||||
"relative_path": source_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(source_path, source_image),
|
||||
}
|
||||
prediction_path = case_root / "prediction-labelids.png"
|
||||
files["prediction_labelids"] = {
|
||||
"relative_path": prediction_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(prediction_path, prediction_image, "L"),
|
||||
}
|
||||
semantic_path = case_root / "prediction-semantic.png"
|
||||
files["prediction_semantic"] = {
|
||||
"relative_path": semantic_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(semantic_path, semantic_palette[prediction_image], "RGBA"),
|
||||
}
|
||||
for preset, palette in policy_palettes.items():
|
||||
policy_path = case_root / f"policy-{preset}.png"
|
||||
files[f"policy_{preset}"] = {
|
||||
"relative_path": policy_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(policy_path, palette[prediction_image], "RGBA"),
|
||||
}
|
||||
if truth_image is not None:
|
||||
truth_path = case_root / "truth-labelids.png"
|
||||
files["truth_labelids"] = {
|
||||
"relative_path": truth_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(truth_path, truth_image, "L"),
|
||||
}
|
||||
truth_semantic_path = case_root / "truth-semantic.png"
|
||||
files["truth_semantic"] = {
|
||||
"relative_path": truth_semantic_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(truth_semantic_path, semantic_palette[truth_image], "RGBA"),
|
||||
}
|
||||
return {
|
||||
"schema_version": VISUAL_SCHEMA,
|
||||
"case_id": case_id,
|
||||
"source_width": source_image.width,
|
||||
"source_height": source_image.height,
|
||||
"center_crop_xyxy": list(crop_box),
|
||||
"outside_crop_state": "undefined" if preserve_source_size else "not-applicable",
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = (len(ordered) - 1) * fraction
|
||||
lower = math.floor(index)
|
||||
upper = math.ceil(index)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
return ordered[lower] * (upper - index) + ordered[upper] * (index - lower)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
args = parse_args()
|
||||
if not torch.cuda.is_available():
|
||||
raise RunnerError("CUDA is required for Worker 006 qualification")
|
||||
if args.limit < 0 or args.visual_count < 0:
|
||||
raise RunnerError("limit and visual-count must be non-negative")
|
||||
config = read_json(args.config, "benchmark config")
|
||||
policy = read_json(args.policy, "mission policy")
|
||||
provider_map = read_json(args.provider_map, "provider map")
|
||||
candidate_config = validate_contracts(config, policy, provider_map, args.candidate)
|
||||
if args.checkpoint.is_symlink() or not args.checkpoint.is_file():
|
||||
raise RunnerError("checkpoint is unavailable")
|
||||
if args.checkpoint.stat().st_size != candidate_config["checkpoint_size_bytes"]:
|
||||
raise RunnerError("checkpoint size changed")
|
||||
checkpoint_sha256 = sha256(args.checkpoint)
|
||||
if checkpoint_sha256 != candidate_config["checkpoint_sha256"]:
|
||||
raise RunnerError("checkpoint digest changed")
|
||||
|
||||
dataset_config = config["dataset"]
|
||||
mapping_root = args.dataset_root
|
||||
if mapping_root is None:
|
||||
raise RunnerError("dataset-root is required for the immutable mapping")
|
||||
mapping_path = mapping_root / dataset_config["mapping_relative_path"]
|
||||
names, semantic_palette = load_mapping(mapping_path, dataset_config["mapping_sha256"])
|
||||
policy_palettes = {
|
||||
preset: policy_palette(
|
||||
names,
|
||||
policy,
|
||||
provider_map,
|
||||
preset,
|
||||
config["policy_action_colors"],
|
||||
)
|
||||
for preset in ("urban", "rural", "offroad")
|
||||
}
|
||||
|
||||
if args.mode == "goose":
|
||||
pairs = find_goose_pairs(mapping_root)
|
||||
if len(pairs) != dataset_config["expected_pair_count"]:
|
||||
raise RunnerError(f"GOOSE pair count changed: {len(pairs)}")
|
||||
items: list[tuple[str, Path, Path | None]] = [
|
||||
(image.stem.removesuffix("_windshield_vis"), image, label)
|
||||
for image, label in pairs
|
||||
]
|
||||
else:
|
||||
if args.frames_root is None or not args.frames_root.is_dir():
|
||||
raise RunnerError("frames-root is required for RAVNOVES mode")
|
||||
frames = sorted(args.frames_root.glob("frame-*.png"))
|
||||
expected = {f"frame-{index:06d}" for index in config["ravnoves"]["frame_indices"]}
|
||||
if {frame.stem for frame in frames} != expected:
|
||||
raise RunnerError("RAVNOVES frame island identity changed")
|
||||
items = [(frame.stem, frame, None) for frame in frames]
|
||||
|
||||
if args.limit:
|
||||
items = items[: args.limit]
|
||||
if not items:
|
||||
raise RunnerError("no inputs were selected")
|
||||
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
torch.cuda.empty_cache()
|
||||
model, model_name, architecture_failures = load_model(args.candidate, args.checkpoint)
|
||||
warmup_source = Image.open(items[0][1]).convert("RGB")
|
||||
warmup_tensor, _ = preprocess(warmup_source)
|
||||
warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
selected_visuals = visual_indices(len(items), args.visual_count)
|
||||
confusion = np.zeros((CLASS_COUNT, CLASS_COUNT), dtype=np.int64)
|
||||
latencies_ms: list[float] = []
|
||||
visuals: list[dict[str, Any]] = []
|
||||
|
||||
for index, (case_id, source_path, label_path) in enumerate(items):
|
||||
source = Image.open(source_path).convert("RGB")
|
||||
tensor, crop_box = preprocess(source)
|
||||
prediction, latency_ms = infer(model, tensor)
|
||||
latencies_ms.append(latency_ms)
|
||||
truth = preprocess_label(Image.open(label_path)) if label_path is not None else None
|
||||
if truth is not None:
|
||||
update_confusion(confusion, truth, prediction)
|
||||
if index in selected_visuals:
|
||||
visuals.append(
|
||||
write_visual_case(
|
||||
args.output,
|
||||
case_id,
|
||||
source,
|
||||
prediction,
|
||||
semantic_palette,
|
||||
policy_palettes,
|
||||
crop_box,
|
||||
truth=truth,
|
||||
preserve_source_size=args.mode == "ravnoves",
|
||||
)
|
||||
)
|
||||
|
||||
rows = class_metrics(confusion, names) if args.mode == "goose" else []
|
||||
valid_ious = [row["iou"] for row in rows if row["iou"] is not None]
|
||||
vegetation_names = set(config["vegetation_class_names"])
|
||||
vegetation_rows = [row for row in rows if row["class_name"] in vegetation_names]
|
||||
vegetation_ious = [row["iou"] for row in vegetation_rows if row["iou"] is not None]
|
||||
timing = {
|
||||
"prewarm_inference_count": len(warmup_latencies_ms),
|
||||
"prewarm_latency_ms": round(warmup_latencies_ms[0], 4),
|
||||
"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4),
|
||||
"sample_count": len(latencies_ms),
|
||||
"latency_ms_p50": round(percentile(latencies_ms, 0.50), 4),
|
||||
"latency_ms_p95": round(percentile(latencies_ms, 0.95), 4),
|
||||
"latency_ms_mean": round(statistics.fmean(latencies_ms), 4),
|
||||
"throughput_fps_from_mean_inference": round(1000.0 / statistics.fmean(latencies_ms), 4),
|
||||
}
|
||||
result: dict[str, Any] = {
|
||||
"schema_version": SCHEMA,
|
||||
"lab_id": config["lab_id"],
|
||||
"worker_id": config["worker_id"],
|
||||
"mode": args.mode,
|
||||
"candidate": {
|
||||
"candidate_id": candidate_config["candidate_id"],
|
||||
"candidate_key": args.candidate,
|
||||
"loaded_model_name": model_name,
|
||||
"architecture_probe_failures": architecture_failures,
|
||||
"checkpoint_size_bytes": args.checkpoint.stat().st_size,
|
||||
"checkpoint_sha256": checkpoint_sha256,
|
||||
},
|
||||
"source": {
|
||||
"source_id": dataset_config["dataset_id"]
|
||||
if args.mode == "goose"
|
||||
else config["ravnoves"]["source_id"],
|
||||
"input_count": len(items),
|
||||
"ground_truth_available": args.mode == "goose",
|
||||
"mapping_sha256": dataset_config["mapping_sha256"],
|
||||
},
|
||||
"preprocessing": dataset_config["preprocessing"],
|
||||
"metrics": {
|
||||
"mean_iou": round(statistics.fmean(valid_ious), 8) if valid_ious else None,
|
||||
"mean_iou_percent": round(statistics.fmean(valid_ious) * 100.0, 4)
|
||||
if valid_ious
|
||||
else None,
|
||||
"published_mean_iou_percent": candidate_config["published_validation_miou_percent"],
|
||||
"vegetation_mean_iou": round(statistics.fmean(vegetation_ious), 8)
|
||||
if vegetation_ious
|
||||
else None,
|
||||
"vegetation_classes": vegetation_rows,
|
||||
"all_classes": rows,
|
||||
},
|
||||
"timing": timing,
|
||||
"resource": {
|
||||
"gpu_name": torch.cuda.get_device_name(0),
|
||||
"peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated()),
|
||||
"peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved()),
|
||||
"torch_version": torch.__version__,
|
||||
"cuda_runtime_version": torch.version.cuda,
|
||||
"python_version": platform.python_version(),
|
||||
"super_gradients_version": "3.2.0",
|
||||
},
|
||||
"visual_cases": visuals,
|
||||
"authority": {
|
||||
"navigation_accepted": False,
|
||||
"safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
},
|
||||
}
|
||||
identity_value = {
|
||||
"schema_version": result["schema_version"],
|
||||
"candidate": result["candidate"],
|
||||
"source": result["source"],
|
||||
"preprocessing": result["preprocessing"],
|
||||
"metrics": result["metrics"],
|
||||
"timing": result["timing"],
|
||||
"resource": result["resource"],
|
||||
"visual_cases": result["visual_cases"],
|
||||
"authority": result["authority"],
|
||||
"config_sha256": sha256(args.config),
|
||||
"policy_sha256": sha256(args.policy),
|
||||
"provider_map_sha256": sha256(args.provider_map),
|
||||
}
|
||||
result["result_id"] = f"lab-v1-{args.mode}-{args.candidate}-{stable_digest(identity_value)}"
|
||||
result["provenance"] = {
|
||||
"config_sha256": identity_value["config_sha256"],
|
||||
"policy_sha256": identity_value["policy_sha256"],
|
||||
"provider_map_sha256": identity_value["provider_map_sha256"],
|
||||
"hostname": platform.node(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
result_path = args.output / "result.json"
|
||||
result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"result_id": result["result_id"], "result_path": str(result_path)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user