feat(perception): add autonomous vegetation shadow lab
This commit is contained in:
DCCONSTRUCTIONS
@@ -48,8 +48,10 @@ import { fetchM48SFixedClassDetectorResult } from "./m48sFixedClassDetector";
|
||||
import { fetchM48TRiskQualityResult } from "./m48tRiskQuality";
|
||||
import { fetchM49TgsFailClosedResult } from "./m49TgsFailClosed";
|
||||
import { fetchM49TgsFullShadowResult } from "./m49TgsFullShadow";
|
||||
import { fetchVegetationShadowResult } from "./vegetationShadow";
|
||||
|
||||
export type AdvancedLaboratoryWorkId =
|
||||
| "lab-v1-vegetation-shadow"
|
||||
| "m48-object-centric-quality"
|
||||
| "m48-small-static-passage-regression"
|
||||
| "m48-static-occupancy-qualification"
|
||||
@@ -100,6 +102,7 @@ export interface AdvancedLaboratoryIndexItem {
|
||||
}
|
||||
|
||||
const WORK_IDS: readonly AdvancedLaboratoryWorkId[] = [
|
||||
"lab-v1-vegetation-shadow",
|
||||
"m48-object-centric-quality",
|
||||
"m48-small-static-passage-regression",
|
||||
"m48-static-occupancy-qualification",
|
||||
@@ -145,6 +148,7 @@ const WORK_IDS: readonly AdvancedLaboratoryWorkId[] = [
|
||||
];
|
||||
|
||||
const RESULT_PREFIX: Readonly<Record<AdvancedLaboratoryWorkId, string>> = {
|
||||
"lab-v1-vegetation-shadow": "lab-v1-vegetation-shadow",
|
||||
"m48-object-centric-quality": "m48-object-quality-(?:pack|result)",
|
||||
"m48-small-static-passage-regression": "m48-small-static-passage-regression",
|
||||
"m48-static-occupancy-qualification": "m48-static-occupancy-qualification",
|
||||
@@ -197,6 +201,7 @@ export function isAdvancedLaboratoryWorkId(
|
||||
|
||||
export function emptyAdvancedLaboratoryResults(): AdvancedLaboratoryResults {
|
||||
return {
|
||||
vegetationShadow: null,
|
||||
m47Graph: null,
|
||||
m48: null,
|
||||
m48SmallStatic: null,
|
||||
@@ -330,7 +335,8 @@ export function advancedLaboratoryResultAvailable(
|
||||
workId: AdvancedLaboratoryWorkId,
|
||||
results: AdvancedLaboratoryResults,
|
||||
): boolean {
|
||||
return workId === "m48-object-centric-quality" ? results.m48 !== null
|
||||
return workId === "lab-v1-vegetation-shadow" ? results.vegetationShadow !== null
|
||||
: workId === "m48-object-centric-quality" ? results.m48 !== null
|
||||
: workId === "m48-small-static-passage-regression" ? results.m48SmallStatic !== null
|
||||
: workId === "m48-static-occupancy-qualification" ? results.m48StaticOccupancy !== null
|
||||
: workId === "m48r3-static-occupancy-shadow" ? results.m48r3StaticOccupancy !== null
|
||||
@@ -387,7 +393,10 @@ export async function fetchAdvancedLaboratoryResult(
|
||||
} = {},
|
||||
): Promise<AdvancedLaboratoryResults> {
|
||||
const results = emptyAdvancedLaboratoryResults();
|
||||
if (workId === "m48-object-centric-quality") {
|
||||
if (workId === "lab-v1-vegetation-shadow") {
|
||||
if (!resultId) throw new AdvancedLaboratoryContractError("Vegetation LAB identity не выбрана.");
|
||||
results.vegetationShadow = await fetchVegetationShadowResult(resultId, { fetcher, signal });
|
||||
} else if (workId === "m48-object-centric-quality") {
|
||||
if (!resultId) throw new AdvancedLaboratoryContractError("M4.8 lifecycle evidence identity не выбрана.");
|
||||
results.m48 = await fetchM48LifecycleResult(resultId, { fetcher, signal });
|
||||
} else if (workId === "m48-small-static-passage-regression") {
|
||||
|
||||
@@ -42,8 +42,10 @@ import type { M48SFixedClassDetectorResult } from "./m48sFixedClassDetector";
|
||||
import type { M48TRiskQualityResult } from "./m48tRiskQuality";
|
||||
import type { M49TgsFailClosedResult } from "./m49TgsFailClosed";
|
||||
import type { M49TgsFullShadowResult } from "./m49TgsFullShadow";
|
||||
import type { VegetationShadowResult } from "./vegetationShadow";
|
||||
|
||||
export interface AdvancedLaboratoryResults {
|
||||
vegetationShadow: VegetationShadowResult | null;
|
||||
m47Graph: M47ReferenceGraphLabResult | null;
|
||||
m48: M48AdvancedResult | null;
|
||||
m48SmallStatic: M48SmallStaticRegressionResult | null;
|
||||
|
||||
@@ -967,6 +967,7 @@ export async function fetchAdvancedLaboratoryResults({
|
||||
const e39 = settledCatalogValue(settled[7]);
|
||||
const e40 = settledCatalogValue(settled[8]);
|
||||
return {
|
||||
vegetationShadow: null,
|
||||
m47Graph: null, m48: null, m48SmallStatic: null, m48StaticOccupancy: null,
|
||||
m48r3StaticOccupancy: null,
|
||||
m48s: null, m48t: null, m49Tgs: null, m49TgsFull: null, m4Threat: null,
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
import type { LaboratoryFetch } from "./advancedResults";
|
||||
|
||||
const RESULT_ID = /^lab-v1-vegetation-shadow-[a-f0-9]{64}$/;
|
||||
const SHA256 = /^[a-f0-9]{64}$/;
|
||||
const CANDIDATES = ["ddrnet", "ppliteseg"] as const;
|
||||
const ROUTE_MODES = ["source", "ddrnet", "ppliteseg", "urban", "rural", "offroad"] as const;
|
||||
const VALIDATION_MODES = ["source", "truth", "ddrnet", "ppliteseg"] as const;
|
||||
|
||||
export type VegetationCandidateKey = typeof CANDIDATES[number];
|
||||
export type VegetationRouteMode = typeof ROUTE_MODES[number];
|
||||
export type VegetationValidationMode = typeof VALIDATION_MODES[number];
|
||||
|
||||
export interface VegetationCandidateMetrics {
|
||||
candidate: VegetationCandidateKey;
|
||||
loadedModelName: string;
|
||||
checkpointSha256: string;
|
||||
meanIouPercent: number;
|
||||
publishedMeanIouPercent: number;
|
||||
vegetationMeanIouPercent: number;
|
||||
validationLatencyP95Ms: number;
|
||||
validationThroughputFps: number;
|
||||
shadowLatencyP95Ms: number;
|
||||
shadowThroughputFps: number;
|
||||
shadowPrewarmLatencyMs: number;
|
||||
peakReservedVramBytes: number;
|
||||
gpuName: string;
|
||||
}
|
||||
|
||||
export interface VegetationVisualCase {
|
||||
caseId: string;
|
||||
sourceKind: "goose" | "ravnoves";
|
||||
width: number;
|
||||
height: number;
|
||||
centerCropXyxy: readonly [number, number, number, number];
|
||||
outsideCropState: "undefined" | "not-applicable";
|
||||
assets: Readonly<Record<string, string>>;
|
||||
}
|
||||
|
||||
export interface VegetationShadowResult {
|
||||
resultId: string;
|
||||
createdAtUtc: string;
|
||||
status: "visual-shadow-ready-policy-not-authorized";
|
||||
selectedCandidate: VegetationCandidateKey;
|
||||
candidates: readonly VegetationCandidateMetrics[];
|
||||
routeCases: readonly VegetationVisualCase[];
|
||||
validationCases: readonly VegetationVisualCase[];
|
||||
limitations: readonly string[];
|
||||
visualShadowReady: true;
|
||||
missionPolicyReadyForConfiguration: true;
|
||||
authority: {
|
||||
commandsEnabled: false;
|
||||
navigationOrSafetyAccepted: false;
|
||||
actuationAccepted: false;
|
||||
cameraSemanticsCanClearRigidGeometry: false;
|
||||
};
|
||||
}
|
||||
|
||||
export class VegetationShadowContractError extends Error {}
|
||||
|
||||
function objectValue(value: unknown, label: string): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new VegetationShadowContractError(`${label}: ожидался объект.`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function arrayValue(value: unknown, label: string): readonly unknown[] {
|
||||
if (!Array.isArray(value)) throw new VegetationShadowContractError(`${label}: ожидался массив.`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function textValue(value: unknown, label: string): string {
|
||||
if (typeof value !== "string" || !value.trim()) {
|
||||
throw new VegetationShadowContractError(`${label}: ожидалась строка.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function numberValue(value: unknown, label: string): number {
|
||||
if (typeof value !== "number" || !Number.isFinite(value)) {
|
||||
throw new VegetationShadowContractError(`${label}: ожидалось число.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function integerValue(value: unknown, label: string): number {
|
||||
const parsed = numberValue(value, label);
|
||||
if (!Number.isInteger(parsed) || parsed < 0) {
|
||||
throw new VegetationShadowContractError(`${label}: ожидалось неотрицательное целое.`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function exact(value: unknown, expected: unknown, label: string): void {
|
||||
if (value !== expected) {
|
||||
throw new VegetationShadowContractError(`${label}: контракт изменён.`);
|
||||
}
|
||||
}
|
||||
|
||||
function candidateKey(value: unknown, label: string): VegetationCandidateKey {
|
||||
if (value !== "ddrnet" && value !== "ppliteseg") {
|
||||
throw new VegetationShadowContractError(`${label}: неизвестная модель.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function candidateMetricsValue(
|
||||
value: unknown,
|
||||
candidate: VegetationCandidateKey,
|
||||
): VegetationCandidateMetrics {
|
||||
const row = objectValue(value, `vegetation.metrics.${candidate}`);
|
||||
const validation = objectValue(row.validation_metrics, `${candidate}.validation_metrics`);
|
||||
const validationTiming = objectValue(row.validation_timing, `${candidate}.validation_timing`);
|
||||
const shadowTiming = objectValue(row.shadow_timing, `${candidate}.shadow_timing`);
|
||||
const resource = objectValue(row.resource, `${candidate}.resource`);
|
||||
const checkpointSha256 = textValue(row.checkpoint_sha256, `${candidate}.checkpoint_sha256`);
|
||||
if (!SHA256.test(checkpointSha256)) {
|
||||
throw new VegetationShadowContractError(`${candidate}.checkpoint_sha256: digest invalid.`);
|
||||
}
|
||||
return {
|
||||
candidate,
|
||||
loadedModelName: textValue(row.loaded_model_name, `${candidate}.loaded_model_name`),
|
||||
checkpointSha256,
|
||||
meanIouPercent: numberValue(validation.mean_iou_percent, `${candidate}.mean_iou_percent`),
|
||||
publishedMeanIouPercent: numberValue(
|
||||
validation.published_mean_iou_percent,
|
||||
`${candidate}.published_mean_iou_percent`,
|
||||
),
|
||||
vegetationMeanIouPercent: numberValue(validation.vegetation_mean_iou, `${candidate}.vegetation_mean_iou`) * 100,
|
||||
validationLatencyP95Ms: numberValue(validationTiming.latency_ms_p95, `${candidate}.validation_latency_p95`),
|
||||
validationThroughputFps: numberValue(
|
||||
validationTiming.throughput_fps_from_mean_inference,
|
||||
`${candidate}.validation_throughput`,
|
||||
),
|
||||
shadowLatencyP95Ms: numberValue(shadowTiming.latency_ms_p95, `${candidate}.shadow_latency_p95`),
|
||||
shadowThroughputFps: numberValue(
|
||||
shadowTiming.throughput_fps_from_mean_inference,
|
||||
`${candidate}.shadow_throughput`,
|
||||
),
|
||||
shadowPrewarmLatencyMs: numberValue(
|
||||
shadowTiming.prewarm_latency_ms,
|
||||
`${candidate}.shadow_prewarm_latency`,
|
||||
),
|
||||
peakReservedVramBytes: integerValue(resource.peak_reserved_vram_bytes, `${candidate}.vram`),
|
||||
gpuName: textValue(resource.gpu_name, `${candidate}.gpu_name`),
|
||||
};
|
||||
}
|
||||
|
||||
function visualCaseValue(
|
||||
value: unknown,
|
||||
resultId: string,
|
||||
expectedKind: "goose" | "ravnoves",
|
||||
): VegetationVisualCase {
|
||||
const row = objectValue(value, `vegetation.${expectedKind}.case`);
|
||||
exact(row.source_kind, expectedKind, "vegetation.case.source_kind");
|
||||
const caseId = textValue(row.case_id, "vegetation.case.case_id");
|
||||
const crop = arrayValue(row.center_crop_xyxy, "vegetation.case.center_crop_xyxy")
|
||||
.map((item, index) => integerValue(item, `vegetation.case.crop[${index}]`));
|
||||
if (crop.length !== 4) {
|
||||
throw new VegetationShadowContractError("vegetation.case.center_crop_xyxy: размер изменён.");
|
||||
}
|
||||
const assets = objectValue(row.assets, "vegetation.case.assets");
|
||||
const projected: Record<string, string> = {};
|
||||
for (const [key, raw] of Object.entries(assets)) {
|
||||
const descriptor = objectValue(raw, `vegetation.case.assets.${key}`);
|
||||
const path = textValue(descriptor.path, `vegetation.case.assets.${key}.path`);
|
||||
const sha256 = textValue(descriptor.sha256, `vegetation.case.assets.${key}.sha256`);
|
||||
if (!SHA256.test(sha256) || !path.startsWith(`visual/${expectedKind}/${caseId}/`)) {
|
||||
throw new VegetationShadowContractError(`vegetation.case.assets.${key}: proof invalid.`);
|
||||
}
|
||||
projected[key] = `/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}/assets/${path
|
||||
.split("/")
|
||||
.map(encodeURIComponent)
|
||||
.join("/")}`;
|
||||
}
|
||||
const expectedAssets = expectedKind === "goose"
|
||||
? VALIDATION_MODES
|
||||
: ROUTE_MODES;
|
||||
if (expectedAssets.some((key) => !projected[key])) {
|
||||
throw new VegetationShadowContractError(`vegetation.case.assets: ${expectedKind} набор неполон.`);
|
||||
}
|
||||
const outsideCropState = row.outside_crop_state;
|
||||
if (outsideCropState !== "undefined" && outsideCropState !== "not-applicable") {
|
||||
throw new VegetationShadowContractError("vegetation.case.outside_crop_state: контракт изменён.");
|
||||
}
|
||||
return {
|
||||
caseId,
|
||||
sourceKind: expectedKind,
|
||||
width: integerValue(row.width, "vegetation.case.width"),
|
||||
height: integerValue(row.height, "vegetation.case.height"),
|
||||
centerCropXyxy: crop as unknown as readonly [number, number, number, number],
|
||||
outsideCropState,
|
||||
assets: projected,
|
||||
};
|
||||
}
|
||||
|
||||
function parseResult(value: unknown, resultId: string): VegetationShadowResult {
|
||||
const payload = objectValue(value, "Vegetation LAB");
|
||||
exact(payload.schema_version, "missioncore.lab-v1-vegetation-shadow/v1", "vegetation.schema");
|
||||
exact(payload.result_id, resultId, "vegetation.result_id");
|
||||
exact(payload.status, "visual-shadow-ready-policy-not-authorized", "vegetation.status");
|
||||
exact(payload.ground_truth, false, "vegetation.ground_truth");
|
||||
exact(payload.access, "read-only", "vegetation.access");
|
||||
const identity = objectValue(payload.identity, "vegetation.identity");
|
||||
const metrics = objectValue(payload.metrics, "vegetation.metrics");
|
||||
const candidates = objectValue(metrics.candidates, "vegetation.metrics.candidates");
|
||||
const decision = objectValue(payload.decision, "vegetation.decision");
|
||||
const authority = objectValue(payload.authority, "vegetation.authority");
|
||||
const catalogs = objectValue(payload.catalogs, "vegetation.catalogs");
|
||||
const selectedCandidate = candidateKey(identity.selected_candidate, "vegetation.selected_candidate");
|
||||
exact(decision.selected_candidate, selectedCandidate, "vegetation.decision.selected_candidate");
|
||||
exact(decision.visual_shadow_ready, true, "vegetation.decision.visual_shadow_ready");
|
||||
exact(
|
||||
decision.mission_policy_ready_for_configuration,
|
||||
true,
|
||||
"vegetation.decision.mission_policy_ready_for_configuration",
|
||||
);
|
||||
exact(decision.navigation_accepted, false, "vegetation.decision.navigation_accepted");
|
||||
exact(decision.production_accepted, false, "vegetation.decision.production_accepted");
|
||||
exact(authority.commands_enabled, false, "vegetation.authority.commands_enabled");
|
||||
exact(
|
||||
authority.navigation_or_safety_accepted,
|
||||
false,
|
||||
"vegetation.authority.navigation_or_safety_accepted",
|
||||
);
|
||||
exact(authority.actuation_accepted, false, "vegetation.authority.actuation_accepted");
|
||||
exact(
|
||||
authority.camera_semantics_can_clear_rigid_geometry,
|
||||
false,
|
||||
"vegetation.authority.camera_semantics_can_clear_rigid_geometry",
|
||||
);
|
||||
const routeCases = arrayValue(catalogs.ravnoves, "vegetation.catalogs.ravnoves")
|
||||
.map((item) => visualCaseValue(item, resultId, "ravnoves"));
|
||||
const validationCases = arrayValue(catalogs.goose, "vegetation.catalogs.goose")
|
||||
.map((item) => visualCaseValue(item, resultId, "goose"));
|
||||
if (routeCases.length !== 12 || validationCases.length !== 12) {
|
||||
throw new VegetationShadowContractError("vegetation.catalogs: ожидалось 12 + 12 случаев.");
|
||||
}
|
||||
return {
|
||||
resultId,
|
||||
createdAtUtc: textValue(payload.created_at_utc, "vegetation.created_at_utc"),
|
||||
status: "visual-shadow-ready-policy-not-authorized",
|
||||
selectedCandidate,
|
||||
candidates: CANDIDATES.map((candidate) => candidateMetricsValue(candidates[candidate], candidate)),
|
||||
routeCases,
|
||||
validationCases,
|
||||
limitations: arrayValue(payload.limitations, "vegetation.limitations")
|
||||
.map((item, index) => textValue(item, `vegetation.limitations[${index}]`)),
|
||||
visualShadowReady: true,
|
||||
missionPolicyReadyForConfiguration: true,
|
||||
authority: {
|
||||
commandsEnabled: false,
|
||||
navigationOrSafetyAccepted: false,
|
||||
actuationAccepted: false,
|
||||
cameraSemanticsCanClearRigidGeometry: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function fetchVegetationShadowResult(
|
||||
resultId: string,
|
||||
{
|
||||
fetcher = fetch,
|
||||
signal,
|
||||
}: { fetcher?: LaboratoryFetch; signal?: AbortSignal } = {},
|
||||
): Promise<VegetationShadowResult> {
|
||||
if (!RESULT_ID.test(resultId)) {
|
||||
throw new VegetationShadowContractError("Vegetation LAB identity недопустима.");
|
||||
}
|
||||
const response = await fetcher(
|
||||
`/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}`,
|
||||
{ method: "GET", headers: { Accept: "application/json" }, signal },
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`);
|
||||
}
|
||||
return parseResult(await response.json(), resultId);
|
||||
}
|
||||
@@ -50,6 +50,7 @@ import { M48SFixedClassDetectorResultView } from "./M48SFixedClassDetectorResult
|
||||
import { M48TRiskQualityResultView } from "./M48TRiskQualityResult";
|
||||
import { M49TgsFailClosedResultView } from "./M49TgsFailClosedResult";
|
||||
import { M49TgsFullShadowResultView } from "./M49TgsFullShadowResult";
|
||||
import { VegetationShadowResultView } from "./VegetationShadowResult";
|
||||
|
||||
export { isAdvancedLaboratoryWorkId };
|
||||
export type { AdvancedLaboratoryWorkId };
|
||||
@@ -92,6 +93,9 @@ export function AdvancedLaboratoryResult({
|
||||
failedSessionId: string | null;
|
||||
replayError: string | null;
|
||||
}) {
|
||||
if (workId === "lab-v1-vegetation-shadow" && results.vegetationShadow) {
|
||||
return <VegetationShadowResultView rigLabel={rigLabel} result={results.vegetationShadow} />;
|
||||
}
|
||||
if (workId === "m48-object-centric-quality" && results.m48) {
|
||||
return <M48ObjectCentricQualityResultView rigLabel={rigLabel} result={results.m48} />;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import {
|
||||
LaboratoryEvidence,
|
||||
LaboratoryResultSummary,
|
||||
LaboratorySummary,
|
||||
LaboratoryWorkTemplate,
|
||||
} from "../../components/laboratory/LaboratoryPresentation";
|
||||
import type { VegetationShadowResult } from "../../core/laboratory/vegetationShadow";
|
||||
import {
|
||||
VegetationRouteVisual,
|
||||
VegetationValidationVisual,
|
||||
} from "./VegetationShadowVisual";
|
||||
|
||||
function decimal(value: number, digits = 1): string {
|
||||
return value.toLocaleString("ru-RU", { maximumFractionDigits: digits });
|
||||
}
|
||||
|
||||
export function VegetationShadowResultView({
|
||||
rigLabel,
|
||||
result,
|
||||
}: {
|
||||
rigLabel: string;
|
||||
result: VegetationShadowResult;
|
||||
}) {
|
||||
const selected = result.candidates.find(
|
||||
(candidate) => candidate.candidate === result.selectedCandidate,
|
||||
)!;
|
||||
const alternative = result.candidates.find(
|
||||
(candidate) => candidate.candidate !== result.selectedCandidate,
|
||||
)!;
|
||||
return (
|
||||
<LaboratoryWorkTemplate
|
||||
summary={(
|
||||
<LaboratorySummary
|
||||
title="LAB V1 · растительность и mission-policy пресеты"
|
||||
description="Две готовые fine-64 модели GOOSE проверены на полном validation split и перенесены в автономный визуальный shadow по RAVNOVES00. Интерфейс читает sealed-кадры локально и не зависит от доступности Worker 006."
|
||||
status="Визуальный shadow готов · navigation authority OFF"
|
||||
statusTone="warning"
|
||||
facts={[
|
||||
{ label: "Источник", value: `${rigLabel} RIGHT · 12 raw KB4 кадров + GOOSE validation 962` },
|
||||
{ label: "Сравнение", value: "DDRNet-39 vs PPLiteSeg · official fine-64 weights" },
|
||||
{ label: "Policy", value: "Urban / rural / off-road · mission-configurable" },
|
||||
{ label: "Authority", value: "SHADOW ONLY · commands OFF · geometry stays authoritative" },
|
||||
]}
|
||||
brief={{
|
||||
question: "Можно ли взять готовую сегментацию растительности, увидеть её на нашем маршруте и сразу проверить разные правила миссии?",
|
||||
approach: "Обе модели последовательно прогнаны в одном изолированном CUDA-runtime: сначала 962 размеченных GOOSE-кадра, затем 12 детерминированных кадров RAVNOVES00. Для каждого кадра запечатаны source, обе семантики и три policy-проекции.",
|
||||
principalResult: `${selected.loadedModelName} выбран по vegetation IoU ${decimal(selected.vegetationMeanIouPercent, 2)}% при shadow p95 ${decimal(selected.shadowLatencyP95Ms, 2)} ms. Визуальный результат доступен локально без Worker.`,
|
||||
limitation: "RAVNOVES00 не размечен по fine-64, поэтому это перенос и визуальная проверка, а не доказательство точности или безопасности. Камерная семантика не может очищать жёсткую LiDAR/TGS occupancy.",
|
||||
}}
|
||||
method={{
|
||||
completeness: "complete",
|
||||
executionClass: "ai-inference",
|
||||
pipelineId: "goose-fine64-ready-weights-to-ravnoves-policy-shadow/v1",
|
||||
components: result.candidates.map((candidate) => ({
|
||||
kind: "model" as const,
|
||||
name: candidate.loadedModelName,
|
||||
version: candidate.candidate,
|
||||
role: candidate.candidate === result.selectedCandidate ? "selected policy provider" : "comparison candidate",
|
||||
identitySha256: candidate.checkpointSha256,
|
||||
})),
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
evidence={(
|
||||
<>
|
||||
<LaboratoryEvidence
|
||||
eyebrow="RAVNOVES00 · AUTONOMOUS VISUAL SHADOW"
|
||||
title="Источник, обе модели и три правила миссии на одинаковых кадрах"
|
||||
kind="diagnostic-model"
|
||||
resizable
|
||||
>
|
||||
<VegetationRouteVisual result={result} />
|
||||
</LaboratoryEvidence>
|
||||
<LaboratoryEvidence
|
||||
eyebrow="GOOSE · EXTERNAL VALIDATION EVIDENCE"
|
||||
title="Независимая разметка: truth против DDRNet и PPLiteSeg"
|
||||
kind="diagnostic-model"
|
||||
resizable
|
||||
>
|
||||
<VegetationValidationVisual result={result} />
|
||||
</LaboratoryEvidence>
|
||||
</>
|
||||
)}
|
||||
result={(
|
||||
<LaboratoryResultSummary
|
||||
title="Готовые веса дают рабочую точку старта, но ещё не право ехать"
|
||||
status={`${selected.loadedModelName} выбран для shadow`}
|
||||
statusTone="warning"
|
||||
metrics={[
|
||||
{
|
||||
label: "GOOSE mIoU",
|
||||
value: `${decimal(selected.meanIouPercent, 2)}% / ${decimal(alternative.meanIouPercent, 2)}%`,
|
||||
hint: `${selected.candidate} / ${alternative.candidate} · полный validation split`,
|
||||
},
|
||||
{
|
||||
label: "Vegetation IoU",
|
||||
value: `${decimal(selected.vegetationMeanIouPercent, 2)}% / ${decimal(alternative.vegetationMeanIouPercent, 2)}%`,
|
||||
hint: "агрегация классов grass/vegetation/bush/tree и родственных fine-64 labels",
|
||||
},
|
||||
{
|
||||
label: "RAVNOVES p95",
|
||||
value: `${decimal(selected.shadowLatencyP95Ms, 2)} / ${decimal(alternative.shadowLatencyP95Ms, 2)} ms`,
|
||||
hint: "чистый inference · одна тяжёлая модель за раз",
|
||||
},
|
||||
{
|
||||
label: "Cold prewarm",
|
||||
value: `${decimal(selected.shadowPrewarmLatencyMs, 1)} / ${decimal(alternative.shadowPrewarmLatencyMs, 1)} ms`,
|
||||
hint: "один явный inference до допуска кадров; исключён из steady-state p95",
|
||||
},
|
||||
{
|
||||
label: "RAVNOVES throughput",
|
||||
value: `${decimal(selected.shadowThroughputFps, 1)} / ${decimal(alternative.shadowThroughputFps, 1)} FPS`,
|
||||
hint: "изолированный Worker 006 · не realtime graph целиком",
|
||||
},
|
||||
{
|
||||
label: "Peak VRAM",
|
||||
value: `${decimal(selected.peakReservedVramBytes / 1024 ** 3, 2)} / ${decimal(alternative.peakReservedVramBytes / 1024 ** 3, 2)} GiB`,
|
||||
hint: `${selected.candidate} / ${alternative.candidate} · RTX 4090`,
|
||||
},
|
||||
{
|
||||
label: "Autonomous evidence",
|
||||
value: "12 route + 12 validation",
|
||||
hint: "только текущий кадр загружается в viewer; Worker не требуется",
|
||||
},
|
||||
]}
|
||||
conclusion={{
|
||||
proved: "Обе официальные fine-64 модели запускаются на Worker 006, проходят полный GOOSE validation и дают воспроизводимые растительные маски на 12 фиксированных RAVNOVES00 кадрах. Urban/rural/off-road policy-проекции формируются без повторного inference.",
|
||||
notProved: "Не доказаны accuracy на нашем fisheye-домене, различение тонкой травы от толстого ствола во всех условиях, temporal stability, collision safety и physical-live поведение ровера.",
|
||||
decision: "Сохранить выбранную модель как shadow provider. Следующий критический блок — разметить небольшой hard-case island нашего офф-роуда: трава, папоротник, куст с толстыми стволами и дерево; затем калибровать policy без ослабления LiDAR/TGS fail-closed геометрии.",
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
import { useState } from "react";
|
||||
import { Icon, IconButton, StatusBadge } from "@nodedc/ui-react";
|
||||
|
||||
import { LaboratoryEvidenceViewer } from "../../components/laboratory/LaboratoryEvidenceViewer";
|
||||
import type {
|
||||
VegetationRouteMode,
|
||||
VegetationShadowResult,
|
||||
VegetationValidationMode,
|
||||
VegetationVisualCase,
|
||||
} from "../../core/laboratory/vegetationShadow";
|
||||
|
||||
const ROUTE_MODES = [
|
||||
{ value: "source", label: "SOURCE" },
|
||||
{ value: "ddrnet", label: "DDRNET" },
|
||||
{ value: "ppliteseg", label: "PPLITE" },
|
||||
{ value: "urban", label: "URBAN" },
|
||||
{ value: "rural", label: "RURAL" },
|
||||
{ value: "offroad", label: "OFF-ROAD" },
|
||||
] as const;
|
||||
|
||||
const VALIDATION_MODES = [
|
||||
{ value: "source", label: "SOURCE" },
|
||||
{ value: "truth", label: "TRUTH" },
|
||||
{ value: "ddrnet", label: "DDRNET" },
|
||||
{ value: "ppliteseg", label: "PPLITE" },
|
||||
] as const;
|
||||
|
||||
function VegetationScene({
|
||||
item,
|
||||
mode,
|
||||
}: {
|
||||
item: VegetationVisualCase;
|
||||
mode: string;
|
||||
}) {
|
||||
const overlay = mode === "source" ? null : item.assets[mode];
|
||||
return (
|
||||
<div className="recorded-evidence-image-scene">
|
||||
<img src={item.assets.source} alt="" draggable={false} />
|
||||
{overlay ? <img src={overlay} alt="" draggable={false} /> : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function VegetationRouteVisual({ result }: { result: VegetationShadowResult }) {
|
||||
const [index, setIndex] = useState(0);
|
||||
const [mode, setMode] = useState<VegetationRouteMode>("offroad");
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const item = result.routeCases[index] ?? null;
|
||||
const selected = result.candidates.find(
|
||||
(candidate) => candidate.candidate === result.selectedCandidate,
|
||||
);
|
||||
return (
|
||||
<LaboratoryEvidenceViewer
|
||||
label="RAVNOVES00 vegetation policy shadow"
|
||||
className="m48-atlas-visual"
|
||||
mode={mode}
|
||||
modes={ROUTE_MODES}
|
||||
expanded={expanded}
|
||||
onModeChange={setMode}
|
||||
onExpandedChange={setExpanded}
|
||||
actions={(
|
||||
<>
|
||||
<IconButton
|
||||
label="Предыдущий vegetation shadow кадр"
|
||||
disabled={!result.routeCases.length}
|
||||
onClick={() => setIndex((current) => (
|
||||
current - 1 + result.routeCases.length
|
||||
) % result.routeCases.length)}
|
||||
>
|
||||
<Icon name="chevron-left" size={16} />
|
||||
</IconButton>
|
||||
<IconButton
|
||||
label="Следующий vegetation shadow кадр"
|
||||
disabled={!result.routeCases.length}
|
||||
onClick={() => setIndex((current) => (current + 1) % result.routeCases.length)}
|
||||
>
|
||||
<Icon name="chevron-right" size={16} />
|
||||
</IconButton>
|
||||
</>
|
||||
)}
|
||||
overlay={item ? (
|
||||
<div className="m48-atlas-visual__case">
|
||||
<StatusBadge tone="warning">SHADOW ONLY</StatusBadge>
|
||||
<strong>RAVNOVES00 · {item.caseId} · {mode.toUpperCase()}</strong>
|
||||
<small>
|
||||
{selected?.loadedModelName ?? result.selectedCandidate} policy provider
|
||||
{" · "}center crop 600×600
|
||||
{" · "}outside crop UNKNOWN
|
||||
</small>
|
||||
</div>
|
||||
) : null}
|
||||
>
|
||||
{item ? (
|
||||
<VegetationScene item={item} mode={mode} />
|
||||
) : (
|
||||
<div className="m48-atlas-visual__state" role="alert">
|
||||
<Icon name="alert" size={18} />
|
||||
RAVNOVES vegetation shadow каталог пуст.
|
||||
</div>
|
||||
)}
|
||||
</LaboratoryEvidenceViewer>
|
||||
);
|
||||
}
|
||||
|
||||
export function VegetationValidationVisual({ result }: { result: VegetationShadowResult }) {
|
||||
const [index, setIndex] = useState(0);
|
||||
const [mode, setMode] = useState<VegetationValidationMode>("truth");
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const item = result.validationCases[index] ?? null;
|
||||
return (
|
||||
<LaboratoryEvidenceViewer
|
||||
label="GOOSE validation vegetation comparison"
|
||||
className="m48-atlas-visual"
|
||||
mode={mode}
|
||||
modes={VALIDATION_MODES}
|
||||
expanded={expanded}
|
||||
onModeChange={setMode}
|
||||
onExpandedChange={setExpanded}
|
||||
actions={(
|
||||
<>
|
||||
<IconButton
|
||||
label="Предыдущий GOOSE validation кадр"
|
||||
disabled={!result.validationCases.length}
|
||||
onClick={() => setIndex((current) => (
|
||||
current - 1 + result.validationCases.length
|
||||
) % result.validationCases.length)}
|
||||
>
|
||||
<Icon name="chevron-left" size={16} />
|
||||
</IconButton>
|
||||
<IconButton
|
||||
label="Следующий GOOSE validation кадр"
|
||||
disabled={!result.validationCases.length}
|
||||
onClick={() => setIndex((current) => (current + 1) % result.validationCases.length)}
|
||||
>
|
||||
<Icon name="chevron-right" size={16} />
|
||||
</IconButton>
|
||||
</>
|
||||
)}
|
||||
overlay={item ? (
|
||||
<div className="m48-atlas-visual__case">
|
||||
<StatusBadge tone="accent">GOOSE VALIDATION</StatusBadge>
|
||||
<strong>{item.caseId} · {mode.toUpperCase()}</strong>
|
||||
<small>official fine-64 labels · fixed 512×512 preprocessing · visual sample</small>
|
||||
</div>
|
||||
) : null}
|
||||
>
|
||||
{item ? (
|
||||
<VegetationScene item={item} mode={mode} />
|
||||
) : (
|
||||
<div className="m48-atlas-visual__state" role="alert">
|
||||
<Icon name="alert" size={18} />
|
||||
GOOSE validation каталог пуст.
|
||||
</div>
|
||||
)}
|
||||
</LaboratoryEvidenceViewer>
|
||||
);
|
||||
}
|
||||
@@ -63,6 +63,13 @@ interface KnownWorkDefinition {
|
||||
const rig = (rigLabel: string): string => rigLabel.trim() || "Сенсорный риг";
|
||||
|
||||
const KNOWN_WORKS: Readonly<Record<Exclude<LaboratoryWorkId, `session:${string}`>, KnownWorkDefinition>> = {
|
||||
"lab-v1-vegetation-shadow": {
|
||||
profileId: "rig-ravnoves-perception-gate-v1",
|
||||
profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · RAVNOVES00 vegetation shadow`,
|
||||
experimentId: "lab-v1-vegetation-mission-policy",
|
||||
experimentName: "GOOSE ready weights → RAVNOVES vegetation policy",
|
||||
variantName: "LAB V1 · DDRNet vs PPLiteSeg · urban/rural/off-road presets",
|
||||
},
|
||||
"m48-object-centric-quality": {
|
||||
profileId: "rig-dual-evidence-virtual-corridor-v1",
|
||||
profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · Camera + prediction-free spatial evidence`,
|
||||
|
||||
@@ -18,6 +18,7 @@ function mergeResults(
|
||||
next: AdvancedLaboratoryResults,
|
||||
): AdvancedLaboratoryResults {
|
||||
return {
|
||||
vegetationShadow: next.vegetationShadow ?? current.vegetationShadow,
|
||||
m47Graph: next.m47Graph ?? current.m47Graph,
|
||||
m48: next.m48 ?? current.m48,
|
||||
m48SmallStatic: next.m48SmallStatic ?? current.m48SmallStatic,
|
||||
@@ -121,6 +122,7 @@ export function useAdvancedLaboratoryCatalog({
|
||||
const indexedResultId = index.find((item) => item.workId === selectedWorkId)?.resultId;
|
||||
if (
|
||||
[
|
||||
"lab-v1-vegetation-shadow",
|
||||
"m47-reference-graph-shadow",
|
||||
"m48-object-centric-quality",
|
||||
"m48-small-static-passage-regression",
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, before, test } from "node:test";
|
||||
|
||||
import { createServer } from "vite";
|
||||
|
||||
let server;
|
||||
let fetchVegetationShadowResult;
|
||||
|
||||
before(async () => {
|
||||
server = await createServer({
|
||||
appType: "custom",
|
||||
logLevel: "silent",
|
||||
server: { middlewareMode: true },
|
||||
});
|
||||
({ fetchVegetationShadowResult } = await server.ssrLoadModule(
|
||||
"/src/core/laboratory/vegetationShadow.ts",
|
||||
));
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await server?.close();
|
||||
});
|
||||
|
||||
const resultId = `lab-v1-vegetation-shadow-${"a".repeat(64)}`;
|
||||
|
||||
function candidate(candidateKey, vegetationIou) {
|
||||
return {
|
||||
loaded_model_name: candidateKey === "ddrnet" ? "ddrnet_39" : "pp_lite_t_seg",
|
||||
checkpoint_sha256: (candidateKey === "ddrnet" ? "b" : "c").repeat(64),
|
||||
validation_metrics: {
|
||||
mean_iou_percent: 44.2,
|
||||
published_mean_iou_percent: 46.53,
|
||||
vegetation_mean_iou: vegetationIou,
|
||||
},
|
||||
validation_timing: {
|
||||
latency_ms_p95: 22.4,
|
||||
throughput_fps_from_mean_inference: 48.1,
|
||||
},
|
||||
shadow_timing: {
|
||||
prewarm_latency_ms: 612.4,
|
||||
latency_ms_p95: 21.8,
|
||||
throughput_fps_from_mean_inference: 49.2,
|
||||
},
|
||||
resource: {
|
||||
peak_reserved_vram_bytes: 2_000_000_000,
|
||||
gpu_name: "NVIDIA GeForce RTX 4090",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function visualCase(sourceKind, index) {
|
||||
const caseId = `case-${index}`;
|
||||
const keys = sourceKind === "goose"
|
||||
? ["source", "truth", "ddrnet", "ppliteseg"]
|
||||
: ["source", "ddrnet", "ppliteseg", "urban", "rural", "offroad"];
|
||||
return {
|
||||
case_id: caseId,
|
||||
source_kind: sourceKind,
|
||||
width: sourceKind === "goose" ? 512 : 800,
|
||||
height: sourceKind === "goose" ? 512 : 600,
|
||||
center_crop_xyxy: sourceKind === "goose" ? [0, 0, 512, 512] : [100, 0, 700, 600],
|
||||
outside_crop_state: sourceKind === "goose" ? "not-applicable" : "undefined",
|
||||
assets: Object.fromEntries(keys.map((key) => [key, {
|
||||
path: `visual/${sourceKind}/${caseId}/${key}.png`,
|
||||
sha256: "d".repeat(64),
|
||||
}])),
|
||||
};
|
||||
}
|
||||
|
||||
test("vegetation LAB keeps autonomous assets and fail-closed authority", async () => {
|
||||
let requestedUrl = "";
|
||||
const result = await fetchVegetationShadowResult(resultId, {
|
||||
fetcher: async (url) => {
|
||||
requestedUrl = String(url);
|
||||
return new Response(JSON.stringify({
|
||||
schema_version: "missioncore.lab-v1-vegetation-shadow/v1",
|
||||
result_id: resultId,
|
||||
created_at_utc: "2026-08-27T20:00:00Z",
|
||||
status: "visual-shadow-ready-policy-not-authorized",
|
||||
ground_truth: false,
|
||||
identity: { selected_candidate: "ddrnet" },
|
||||
metrics: {
|
||||
candidates: {
|
||||
ddrnet: candidate("ddrnet", 0.64),
|
||||
ppliteseg: candidate("ppliteseg", 0.61),
|
||||
},
|
||||
},
|
||||
decision: {
|
||||
selected_candidate: "ddrnet",
|
||||
visual_shadow_ready: true,
|
||||
mission_policy_ready_for_configuration: true,
|
||||
navigation_accepted: false,
|
||||
production_accepted: false,
|
||||
},
|
||||
limitations: ["shadow only"],
|
||||
authority: {
|
||||
commands_enabled: false,
|
||||
navigation_or_safety_accepted: false,
|
||||
actuation_accepted: false,
|
||||
camera_semantics_can_clear_rigid_geometry: false,
|
||||
},
|
||||
catalogs: {
|
||||
goose: Array.from({ length: 12 }, (_, index) => visualCase("goose", index)),
|
||||
ravnoves: Array.from({ length: 12 }, (_, index) => visualCase("ravnoves", index)),
|
||||
},
|
||||
access: "read-only",
|
||||
}), { status: 200, headers: { "Content-Type": "application/json" } });
|
||||
},
|
||||
});
|
||||
assert.equal(
|
||||
requestedUrl,
|
||||
`/api/v1/laboratory/vegetation-shadow/${resultId}`,
|
||||
);
|
||||
assert.equal(result.selectedCandidate, "ddrnet");
|
||||
assert.equal(result.candidates[0].vegetationMeanIouPercent, 64);
|
||||
assert.equal(result.routeCases.length, 12);
|
||||
assert.equal(result.validationCases.length, 12);
|
||||
assert.match(result.routeCases[0].assets.offroad, /\/assets\/visual\/ravnoves\//);
|
||||
assert.deepEqual(result.authority, {
|
||||
commandsEnabled: false,
|
||||
navigationOrSafetyAccepted: false,
|
||||
actuationAccepted: false,
|
||||
cameraSemanticsCanClearRigidGeometry: false,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"schema_version": "missioncore.laboratory-evidence-definition/v1",
|
||||
"work_id": "lab-v1-vegetation-shadow",
|
||||
"evidence": {
|
||||
"runtime_relative_root": "lab-v1-vegetation/results",
|
||||
"result_id_prefix": "lab-v1-vegetation-shadow",
|
||||
"document_name": "result.json",
|
||||
"schema_version": "missioncore.lab-v1-vegetation-shadow/v1"
|
||||
}
|
||||
}
|
||||
@@ -190,6 +190,25 @@
|
||||
"run": "missioncore.laboratory-run/v1",
|
||||
"evidence": "missioncore.m49-tgs-full-shadow-lab/v1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"work_id": "lab-v1-vegetation-shadow",
|
||||
"lifecycle": "experimental",
|
||||
"isolation": "bounded-adapter",
|
||||
"adapter_id": "experimental.lab-v1-vegetation-shadow/v1",
|
||||
"input_roles": [
|
||||
"ddrnet_goose_root",
|
||||
"ppliteseg_goose_root",
|
||||
"ddrnet_ravnoves_root",
|
||||
"ppliteseg_ravnoves_root"
|
||||
],
|
||||
"contracts": {
|
||||
"source": "missioncore.goose-ravnoves-vegetation-source-set/v1",
|
||||
"provider": "missioncore.goose-fine64-ready-weight-provider/v1",
|
||||
"graph": "missioncore.vegetation-mission-policy-shadow-graph/v1",
|
||||
"run": "missioncore.laboratory-run/v1",
|
||||
"evidence": "missioncore.lab-v1-vegetation-shadow/v1"
|
||||
}
|
||||
}
|
||||
],
|
||||
"legacy_work_ids": [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schema_version": "missioncore.laboratory-value-review-registry/v1",
|
||||
"reviewed_at_utc": "2026-08-26T08:34:34Z",
|
||||
"reviewed_at_utc": "2026-08-27T20:20:14Z",
|
||||
"entries": [
|
||||
{
|
||||
"catalog_id": "e28-local-surface",
|
||||
@@ -281,6 +281,13 @@
|
||||
"signal": "progress",
|
||||
"lifecycle": "current",
|
||||
"visual_evidence": "available"
|
||||
},
|
||||
{
|
||||
"catalog_id": "lab-v1-vegetation-shadow",
|
||||
"evidence_id": "lab-v1-vegetation-shadow-ad4d9fbbb21ff8a270b77f559b4e78dcdaf0455afd61afb5033009623984e554",
|
||||
"signal": "progress",
|
||||
"lifecycle": "current",
|
||||
"visual_evidence": "available"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"schema_version": "missioncore.lab-v1-goose-vegetation-benchmark/v1",
|
||||
"lab_id": "LAB-V1",
|
||||
"worker_id": "worker-006",
|
||||
"runtime": {
|
||||
"super_gradients_version": "3.2.0",
|
||||
"super_gradients_revision": "54d062ecb1081944a672ce447cf3e96a36708ff9",
|
||||
"python_version": "3.9",
|
||||
"numpy_version": "1.23.0",
|
||||
"cmake_version": "3.31.6",
|
||||
"onnxsim_version": "0.4.36",
|
||||
"opencv_python_version": "4.8.1.78",
|
||||
"pytorch_version": "1.13.1",
|
||||
"torchvision_version": "0.14.1",
|
||||
"pytorch_cuda_version": "11.7",
|
||||
"container_base": "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85",
|
||||
"miniconda_installer": "Miniconda3-py39_24.11.1-0-Linux-x86_64.sh",
|
||||
"miniconda_installer_sha256": "3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5"
|
||||
},
|
||||
"dataset": {
|
||||
"dataset_id": "goose-2d-validation-visible-rgb",
|
||||
"relative_root": "goose-2d/validation",
|
||||
"mapping_relative_path": "goose_label_mapping.csv",
|
||||
"mapping_sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f",
|
||||
"image_glob": "images/val/**/*_windshield_vis.png",
|
||||
"expected_pair_count": 962,
|
||||
"input_size": [512, 512],
|
||||
"preprocessing": [
|
||||
"center-square-crop",
|
||||
"nearest-neighbor-resize",
|
||||
"rgb-to-tensor-0-1"
|
||||
]
|
||||
},
|
||||
"candidates": {
|
||||
"ddrnet": {
|
||||
"candidate_id": "goose-ddrnet-class-512",
|
||||
"model_names": ["ddrnet_39"],
|
||||
"checkpoint_relative_path": "models/goose/ddrnet_class_512.pth",
|
||||
"checkpoint_size_bytes": 259419077,
|
||||
"checkpoint_sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
|
||||
"published_validation_miou_percent": 46.53
|
||||
},
|
||||
"ppliteseg": {
|
||||
"candidate_id": "goose-ppliteseg-class-512",
|
||||
"model_names": [
|
||||
"pp_lite_t_seg",
|
||||
"pp_lite_t_seg50",
|
||||
"pp_lite_t_seg75",
|
||||
"pp_lite_b_seg",
|
||||
"pp_lite_b_seg50",
|
||||
"pp_lite_b_seg75"
|
||||
],
|
||||
"checkpoint_relative_path": "models/goose/ppliteseg_class_512.pth",
|
||||
"checkpoint_size_bytes": 98208249,
|
||||
"checkpoint_sha256": "6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd",
|
||||
"published_validation_miou_percent": 45.09
|
||||
}
|
||||
},
|
||||
"ravnoves": {
|
||||
"source_id": "RAVNOVES00/right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"source_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8",
|
||||
"expected_width": 800,
|
||||
"expected_height": 600,
|
||||
"expected_frame_count": 4489,
|
||||
"frame_indices": [0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488],
|
||||
"crop_contract": "center-600-square-to-512; outside-crop-is-undefined"
|
||||
},
|
||||
"vegetation_class_names": [
|
||||
"leaves",
|
||||
"forest",
|
||||
"bush",
|
||||
"moss",
|
||||
"tree_crown",
|
||||
"tree_trunk",
|
||||
"crops",
|
||||
"low_grass",
|
||||
"high_grass",
|
||||
"scenery_vegetation",
|
||||
"hedge",
|
||||
"tree_root"
|
||||
],
|
||||
"policy_action_colors": {
|
||||
"ALLOW": "#22c55e",
|
||||
"HIGH_COST": "#f59e0b",
|
||||
"NO_GO": "#ef4444"
|
||||
},
|
||||
"invariants": {
|
||||
"one_heavy_candidate_at_a_time": true,
|
||||
"raw_fisheye_is_immutable": true,
|
||||
"outside_center_crop_is_free": false,
|
||||
"missing_or_unknown_is_free": false,
|
||||
"camera_semantics_can_clear_rigid_geometry": false,
|
||||
"navigation_authority": false,
|
||||
"actuation_authority": false,
|
||||
"canonical_triton_mutation_allowed": false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Build", "Probe", "Validate", "Ravnoves", "Status")]
|
||||
[string]$Mode = "Status",
|
||||
|
||||
[ValidateSet("Ddrnet", "Ppliteseg")]
|
||||
[string]$Candidate = "Ddrnet",
|
||||
|
||||
[string]$AssetRoot = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27",
|
||||
|
||||
[string]$ToolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\lab-v1-vegetation-goose",
|
||||
|
||||
[string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\experiments\lab-v1-vegetation",
|
||||
|
||||
[string]$RavnovesVideo = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$datasetPrefix = "D:\NDC_MISSIONCORE\datasets\"
|
||||
$runtimePrefix = "D:\NDC_MISSIONCORE\runtime\experiments\"
|
||||
if (-not $AssetRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "AssetRoot must stay under $datasetPrefix"
|
||||
}
|
||||
if (-not $ToolRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "ToolRoot must stay under $datasetPrefix"
|
||||
}
|
||||
if (-not $OutputRoot.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "OutputRoot must stay under $runtimePrefix"
|
||||
}
|
||||
|
||||
$image = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1"
|
||||
$canonicalContainer = "ndc-mission-core-triton"
|
||||
$candidateKey = $Candidate.ToLowerInvariant()
|
||||
$contextRoot = Join-Path $ToolRoot "context"
|
||||
$configRoot = Join-Path $ToolRoot "config"
|
||||
$benchmarkConfig = Join-Path $configRoot "lab-v1-goose-vegetation-benchmark-v1.json"
|
||||
$policyConfig = Join-Path $configRoot "lab-v1-vegetation-mission-policy-v1.json"
|
||||
$providerMapConfig = Join-Path $configRoot "lab-v1-vegetation-provider-label-map-v1.json"
|
||||
$datasetRoot = Join-Path $AssetRoot "goose-2d\validation"
|
||||
$checkpointRelative = if ($candidateKey -eq "ddrnet") {
|
||||
"models\goose\ddrnet_class_512.pth"
|
||||
} else {
|
||||
"models\goose\ppliteseg_class_512.pth"
|
||||
}
|
||||
$checkpoint = Join-Path $AssetRoot $checkpointRelative
|
||||
$expectedCheckpointSha256 = if ($candidateKey -eq "ddrnet") {
|
||||
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
} else {
|
||||
"6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd"
|
||||
}
|
||||
$expectedCheckpointBytes = if ($candidateKey -eq "ddrnet") { 259419077 } else { 98208249 }
|
||||
$ravnovesSha256 = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8"
|
||||
$frameIndices = @(0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488)
|
||||
$dockerConfig = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation\docker-config"
|
||||
|
||||
function Get-CanonicalTritonIdentity {
|
||||
$identity = & docker inspect $canonicalContainer --format "{{.Id}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}"
|
||||
if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($identity)) {
|
||||
throw "Canonical Triton is unavailable"
|
||||
}
|
||||
$parts = $identity.Split("|")
|
||||
if ($parts.Count -ne 4 -or $parts[2] -ne "running" -or $parts[3] -ne "healthy") {
|
||||
throw "Canonical Triton is not running and healthy: $identity"
|
||||
}
|
||||
return $identity
|
||||
}
|
||||
|
||||
function Assert-FileIdentity {
|
||||
param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256)
|
||||
$file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue
|
||||
if ($null -eq $file -or $file.Length -ne $ExpectedBytes) {
|
||||
throw "File identity changed: $Path"
|
||||
}
|
||||
$actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actualSha256 -ne $ExpectedSha256) {
|
||||
throw "File digest changed: $Path"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-RunnerInputs {
|
||||
foreach ($path in @($benchmarkConfig, $policyConfig, $providerMapConfig)) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
throw "Runner config is unavailable: $path"
|
||||
}
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "Dockerfile") -PathType Leaf)) {
|
||||
throw "Runner Dockerfile is unavailable"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "run_goose_vegetation_benchmark.py") -PathType Leaf)) {
|
||||
throw "Runner source is unavailable"
|
||||
}
|
||||
Assert-FileIdentity -Path $checkpoint -ExpectedBytes $expectedCheckpointBytes -ExpectedSha256 $expectedCheckpointSha256
|
||||
}
|
||||
|
||||
function New-RunRoot {
|
||||
param([string]$Kind)
|
||||
$stamp = [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssfffZ")
|
||||
$path = Join-Path $OutputRoot ("{0}-{1}-{2}" -f $Kind, $candidateKey, $stamp)
|
||||
New-Item -ItemType Directory -Path $path | Out-Null
|
||||
return $path
|
||||
}
|
||||
|
||||
function Invoke-IsolatedRun {
|
||||
param(
|
||||
[ValidateSet("goose", "ravnoves")][string]$RunMode,
|
||||
[string]$RunRoot,
|
||||
[int]$Limit,
|
||||
[string]$FramesRoot = ""
|
||||
)
|
||||
$containerName = "ndc-lab-v1-goose-$candidateKey-$([Guid]::NewGuid().ToString('N').Substring(0, 10))"
|
||||
$arguments = @(
|
||||
"run", "--rm", "--name", $containerName,
|
||||
"--gpus", "all",
|
||||
"--network", "none",
|
||||
"--read-only",
|
||||
"--cap-drop", "ALL",
|
||||
"--security-opt", "no-new-privileges",
|
||||
"--memory", "10g",
|
||||
"--cpus", "8",
|
||||
"--pids-limit", "512",
|
||||
"--tmpfs", "/tmp:rw,noexec,nosuid,size=2g",
|
||||
"--env", "HOME=/tmp",
|
||||
"--mount", "type=bind,src=$datasetRoot,dst=/data/goose,readonly",
|
||||
"--mount", "type=bind,src=$checkpoint,dst=/models/candidate.pth,readonly",
|
||||
"--mount", "type=bind,src=$configRoot,dst=/config,readonly",
|
||||
"--mount", "type=bind,src=$RunRoot,dst=/output",
|
||||
$image,
|
||||
"--mode", $RunMode,
|
||||
"--candidate", $candidateKey,
|
||||
"--config", "/config/lab-v1-goose-vegetation-benchmark-v1.json",
|
||||
"--policy", "/config/lab-v1-vegetation-mission-policy-v1.json",
|
||||
"--provider-map", "/config/lab-v1-vegetation-provider-label-map-v1.json",
|
||||
"--checkpoint", "/models/candidate.pth",
|
||||
"--dataset-root", "/data/goose",
|
||||
"--output", "/output/result",
|
||||
"--limit", $Limit.ToString(),
|
||||
"--visual-count", "12"
|
||||
)
|
||||
if ($RunMode -eq "ravnoves") {
|
||||
$arguments = @($arguments[0..($arguments.Count - 1)])
|
||||
$arguments += @("--frames-root", "/input")
|
||||
$mountIndex = [Array]::IndexOf($arguments, $image)
|
||||
$head = @($arguments[0..($mountIndex - 1)])
|
||||
$tail = @($arguments[$mountIndex..($arguments.Count - 1)])
|
||||
$arguments = $head + @("--mount", "type=bind,src=$FramesRoot,dst=/input,readonly") + $tail
|
||||
}
|
||||
& docker @arguments
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 container failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
function Export-RavnovesFrames {
|
||||
param([string]$Destination)
|
||||
Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $ravnovesSha256
|
||||
New-Item -ItemType Directory -Path $Destination | Out-Null
|
||||
$expression = ($frameIndices | ForEach-Object { "eq(n\,$_ )" }) -join "+"
|
||||
$temporaryPattern = Join-Path $Destination "selected-%03d.png"
|
||||
& ffmpeg -hide_banner -loglevel error -i $RavnovesVideo -vf "select='$expression'" -fps_mode vfr $temporaryPattern
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "RAVNOVES exact frame extraction failed"
|
||||
}
|
||||
$selected = @(Get-ChildItem -LiteralPath $Destination -Filter "selected-*.png" | Sort-Object Name)
|
||||
if ($selected.Count -ne $frameIndices.Count) {
|
||||
throw "RAVNOVES frame island changed: expected $($frameIndices.Count), got $($selected.Count)"
|
||||
}
|
||||
for ($index = 0; $index -lt $selected.Count; $index++) {
|
||||
$target = Join-Path $Destination ("frame-{0:D6}.png" -f $frameIndices[$index])
|
||||
Move-Item -LiteralPath $selected[$index].FullName -Destination $target
|
||||
}
|
||||
}
|
||||
|
||||
if ($Mode -eq "Status") {
|
||||
$imageIdentity = & docker image inspect $image --format "{{.Id}}" 2>$null
|
||||
[ordered]@{
|
||||
schema_version = "missioncore.lab-v1-goose-runner-status/v1"
|
||||
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
||||
worker_id = "worker-006"
|
||||
image = $image
|
||||
image_id = if ($LASTEXITCODE -eq 0) { $imageIdentity } else { $null }
|
||||
canonical_triton = Get-CanonicalTritonIdentity
|
||||
asset_root = $AssetRoot
|
||||
output_root = $OutputRoot
|
||||
candidate = $candidateKey
|
||||
} | ConvertTo-Json -Depth 6
|
||||
exit 0
|
||||
}
|
||||
|
||||
Assert-RunnerInputs
|
||||
$canonicalBefore = Get-CanonicalTritonIdentity
|
||||
try {
|
||||
if ($Mode -eq "Build") {
|
||||
if (-not (Test-Path -LiteralPath (Join-Path $dockerConfig "config.json") -PathType Leaf)) {
|
||||
throw "Isolated Docker client configuration is unavailable"
|
||||
}
|
||||
$previousDockerConfig = $env:DOCKER_CONFIG
|
||||
try {
|
||||
$env:DOCKER_CONFIG = $dockerConfig
|
||||
& docker build --pull=false --label "com.nodedc.component=mission-core-lab-v1-goose" --label "com.nodedc.authority=shadow-only" --tag $image $contextRoot
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 image build failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$env:DOCKER_CONFIG = $previousDockerConfig
|
||||
}
|
||||
}
|
||||
elseif ($Mode -eq "Probe") {
|
||||
$runRoot = New-RunRoot -Kind "probe"
|
||||
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 8
|
||||
}
|
||||
elseif ($Mode -eq "Validate") {
|
||||
$runRoot = New-RunRoot -Kind "validation"
|
||||
Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 0
|
||||
}
|
||||
elseif ($Mode -eq "Ravnoves") {
|
||||
$runRoot = New-RunRoot -Kind "ravnoves"
|
||||
$framesRoot = Join-Path $runRoot "input-frames"
|
||||
Export-RavnovesFrames -Destination $framesRoot
|
||||
Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot
|
||||
}
|
||||
}
|
||||
finally {
|
||||
$canonicalAfter = Get-CanonicalTritonIdentity
|
||||
if ($canonicalAfter -ne $canonicalBefore) {
|
||||
throw "Canonical Triton identity changed during LAB V1 work"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PATH=/opt/conda/bin:$PATH
|
||||
|
||||
ARG MINICONDA_INSTALLER=Miniconda3-py39_24.11.1-0-Linux-x86_64.sh
|
||||
ARG MINICONDA_SHA256=3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
git \
|
||||
libglib2.0-0 \
|
||||
libgl1 \
|
||||
&& curl --fail --location --retry 5 \
|
||||
--output /tmp/miniconda.sh \
|
||||
"https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}" \
|
||||
&& echo "${MINICONDA_SHA256} /tmp/miniconda.sh" | sha256sum --check --strict \
|
||||
&& bash /tmp/miniconda.sh -b -p /opt/conda \
|
||||
&& rm -f /tmp/miniconda.sh \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN conda create --yes --name goose python=3.9 pip \
|
||||
&& conda install --yes --name goose --channel pytorch --channel nvidia \
|
||||
pytorch=1.13.1 torchvision=0.14.1 pytorch-cuda=11.7
|
||||
|
||||
RUN conda run --name goose python -m pip install --no-cache-dir \
|
||||
cmake==3.31.6 \
|
||||
numpy==1.23.0 \
|
||||
onnxsim==0.4.36 \
|
||||
opencv-python==4.8.1.78 \
|
||||
protobuf==3.20.3 \
|
||||
pyparsing==2.4.5
|
||||
|
||||
RUN conda run --name goose python -m pip install --no-cache-dir \
|
||||
super-gradients==3.2.0 \
|
||||
torchmetrics==0.8.0
|
||||
|
||||
RUN conda clean --all --yes
|
||||
|
||||
WORKDIR /opt/mission-core/lab-v1
|
||||
COPY run_goose_vegetation_benchmark.py /opt/mission-core/lab-v1/runner.py
|
||||
|
||||
ENTRYPOINT ["conda", "run", "--no-capture-output", "--name", "goose", "python", "/opt/mission-core/lab-v1/runner.py"]
|
||||
+556
@@ -0,0 +1,556 @@
|
||||
"""Run isolated GOOSE vegetation qualification and RAVNOVES shadow inference."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from super_gradients.training import models
|
||||
|
||||
SCHEMA = "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||
VISUAL_SCHEMA = "missioncore.lab-v1-goose-vegetation-visual-case/v1"
|
||||
CLASS_COUNT = 64
|
||||
MAX_CONFIG_BYTES = 1024 * 1024
|
||||
MODEL_NAMES = {
|
||||
"ddrnet": ("ddrnet_39",),
|
||||
"ppliteseg": (
|
||||
"pp_lite_t_seg",
|
||||
"pp_lite_t_seg50",
|
||||
"pp_lite_t_seg75",
|
||||
"pp_lite_b_seg",
|
||||
"pp_lite_b_seg50",
|
||||
"pp_lite_b_seg75",
|
||||
),
|
||||
}
|
||||
RESAMPLE_NEAREST = getattr(Image, "Resampling", Image).NEAREST
|
||||
|
||||
|
||||
class RunnerError(RuntimeError):
|
||||
"""The bounded runner input or output contract is invalid."""
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--mode", choices=("goose", "ravnoves"), required=True)
|
||||
parser.add_argument("--candidate", choices=tuple(MODEL_NAMES), required=True)
|
||||
parser.add_argument("--config", type=Path, required=True)
|
||||
parser.add_argument("--policy", type=Path, required=True)
|
||||
parser.add_argument("--provider-map", type=Path, required=True)
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--dataset-root", type=Path)
|
||||
parser.add_argument("--frames-root", type=Path)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--limit", type=int, default=0)
|
||||
parser.add_argument("--visual-count", type=int, default=12)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_json(path: Path, label: str) -> dict[str, Any]:
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES:
|
||||
raise RunnerError(f"{label} is unavailable")
|
||||
value = json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
if not isinstance(value, dict):
|
||||
raise RunnerError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def stable_digest(value: object) -> str:
|
||||
encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def validate_contracts(
|
||||
config: dict[str, Any], policy: dict[str, Any], provider_map: dict[str, Any], candidate: str
|
||||
) -> dict[str, Any]:
|
||||
if config.get("schema_version") != "missioncore.lab-v1-goose-vegetation-benchmark/v1":
|
||||
raise RunnerError("benchmark configuration identity changed")
|
||||
if policy.get("schema_version") != "missioncore.vegetation-mission-policy/v1":
|
||||
raise RunnerError("mission policy identity changed")
|
||||
if provider_map.get("schema_version") != "missioncore.vegetation-provider-label-map/v1":
|
||||
raise RunnerError("provider map identity changed")
|
||||
invariants = config.get("invariants")
|
||||
true_invariants = {"one_heavy_candidate_at_a_time", "raw_fisheye_is_immutable"}
|
||||
if not isinstance(invariants, dict) or any(
|
||||
value is not False for key, value in invariants.items() if key not in true_invariants
|
||||
):
|
||||
raise RunnerError("benchmark fail-closed invariants changed")
|
||||
if (
|
||||
invariants.get("one_heavy_candidate_at_a_time") is not True
|
||||
or invariants.get("raw_fisheye_is_immutable") is not True
|
||||
):
|
||||
raise RunnerError("benchmark isolation invariants changed")
|
||||
candidates = config.get("candidates")
|
||||
if not isinstance(candidates, dict) or not isinstance(candidates.get(candidate), dict):
|
||||
raise RunnerError("candidate is not configured")
|
||||
expected_models = candidates[candidate].get("model_names")
|
||||
if expected_models != list(MODEL_NAMES[candidate]):
|
||||
raise RunnerError("candidate architecture probe order changed")
|
||||
return candidates[candidate]
|
||||
|
||||
|
||||
def load_mapping(path: Path, expected_sha256: str) -> tuple[dict[int, str], np.ndarray]:
|
||||
if sha256(path) != expected_sha256:
|
||||
raise RunnerError("GOOSE label mapping digest changed")
|
||||
names: dict[int, str] = {}
|
||||
palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8)
|
||||
with path.open(newline="", encoding="utf-8-sig") as stream:
|
||||
for row in csv.DictReader(stream):
|
||||
label_id = int(row["label_key"])
|
||||
if label_id < 0 or label_id >= CLASS_COUNT:
|
||||
raise RunnerError("GOOSE label id is outside the 64-class contract")
|
||||
color = row["hex"].lstrip("#")
|
||||
names[label_id] = row["class_name"]
|
||||
palette[label_id] = (*bytes.fromhex(color), 190)
|
||||
if set(names) != set(range(CLASS_COUNT)):
|
||||
raise RunnerError("GOOSE mapping does not cover exactly 64 classes")
|
||||
palette[0, 3] = 0
|
||||
return names, palette
|
||||
|
||||
|
||||
def center_crop(image: Image.Image) -> tuple[Image.Image, tuple[int, int, int, int]]:
|
||||
side = min(image.width, image.height)
|
||||
left = (image.width - side) // 2
|
||||
top = (image.height - side) // 2
|
||||
box = (left, top, left + side, top + side)
|
||||
return image.crop(box), box
|
||||
|
||||
|
||||
def preprocess(image: Image.Image) -> tuple[torch.Tensor, tuple[int, int, int, int]]:
|
||||
cropped, crop_box = center_crop(image.convert("RGB"))
|
||||
resized = cropped.resize((512, 512), resample=RESAMPLE_NEAREST)
|
||||
array = np.asarray(resized, dtype=np.float32) / 255.0
|
||||
tensor = torch.from_numpy(np.transpose(array, (2, 0, 1))).unsqueeze(0)
|
||||
return tensor, crop_box
|
||||
|
||||
|
||||
def preprocess_label(image: Image.Image) -> np.ndarray:
|
||||
cropped, _ = center_crop(image.convert("L"))
|
||||
return np.asarray(cropped.resize((512, 512), resample=RESAMPLE_NEAREST), dtype=np.uint8)
|
||||
|
||||
|
||||
def find_goose_pairs(root: Path) -> list[tuple[Path, Path]]:
|
||||
pairs: list[tuple[Path, Path]] = []
|
||||
image_root = root / "images" / "val"
|
||||
label_root = root / "labels" / "val"
|
||||
for image_path in sorted(image_root.rglob("*_windshield_vis.png")):
|
||||
stem = image_path.name.removesuffix("_windshield_vis.png")
|
||||
relative_parent = image_path.parent.relative_to(image_root)
|
||||
label_path = label_root / relative_parent / f"{stem}_labelids.png"
|
||||
if label_path.is_file() and not label_path.is_symlink():
|
||||
pairs.append((image_path, label_path))
|
||||
return pairs
|
||||
|
||||
|
||||
def visual_indices(count: int, visual_count: int) -> set[int]:
|
||||
if count <= 0 or visual_count <= 0:
|
||||
return set()
|
||||
selected_count = min(count, visual_count)
|
||||
if selected_count == 1:
|
||||
return {0}
|
||||
return {
|
||||
round(index * (count - 1) / (selected_count - 1))
|
||||
for index in range(selected_count)
|
||||
}
|
||||
|
||||
|
||||
def load_model(candidate: str, checkpoint: Path) -> tuple[torch.nn.Module, str, list[str]]:
|
||||
failures: list[str] = []
|
||||
for model_name in MODEL_NAMES[candidate]:
|
||||
try:
|
||||
model = models.get(
|
||||
model_name=model_name,
|
||||
num_classes=CLASS_COUNT,
|
||||
checkpoint_path=str(checkpoint),
|
||||
)
|
||||
model.eval()
|
||||
model.cuda()
|
||||
return model, model_name, failures
|
||||
except Exception as error: # noqa: BLE001 - each upstream architecture is a probe
|
||||
failures.append(f"{model_name}: {type(error).__name__}: {str(error)[:240]}")
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
raise RunnerError("checkpoint did not load: " + " | ".join(failures))
|
||||
|
||||
|
||||
def logits_from_output(value: object) -> torch.Tensor:
|
||||
if isinstance(value, torch.Tensor) and value.ndim == 4 and value.shape[1] == CLASS_COUNT:
|
||||
return value
|
||||
if isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
try:
|
||||
return logits_from_output(item)
|
||||
except RunnerError:
|
||||
continue
|
||||
raise RunnerError("model output does not contain a 64-class raster")
|
||||
|
||||
|
||||
def infer(model: torch.nn.Module, tensor: torch.Tensor) -> tuple[np.ndarray, float]:
|
||||
tensor = tensor.cuda(non_blocking=True)
|
||||
torch.cuda.synchronize()
|
||||
started = time.perf_counter_ns()
|
||||
with torch.inference_mode():
|
||||
logits = logits_from_output(model(tensor))
|
||||
prediction = torch.argmax(torch.sigmoid(logits), dim=1)
|
||||
torch.cuda.synchronize()
|
||||
elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000.0
|
||||
return prediction[0].to(device="cpu", dtype=torch.uint8).numpy(), elapsed_ms
|
||||
|
||||
|
||||
def update_confusion(confusion: np.ndarray, truth: np.ndarray, prediction: np.ndarray) -> None:
|
||||
valid = (truth >= 0) & (truth < CLASS_COUNT)
|
||||
indices = CLASS_COUNT * truth[valid].astype(np.int64) + prediction[valid].astype(np.int64)
|
||||
confusion += np.bincount(indices, minlength=CLASS_COUNT**2).reshape(CLASS_COUNT, CLASS_COUNT)
|
||||
|
||||
|
||||
def class_metrics(confusion: np.ndarray, names: dict[int, str]) -> list[dict[str, Any]]:
|
||||
truth = confusion.sum(axis=1)
|
||||
predicted = confusion.sum(axis=0)
|
||||
intersection = np.diag(confusion)
|
||||
union = truth + predicted - intersection
|
||||
rows: list[dict[str, Any]] = []
|
||||
for label_id in range(CLASS_COUNT):
|
||||
rows.append(
|
||||
{
|
||||
"label_id": label_id,
|
||||
"class_name": names[label_id],
|
||||
"support_pixels": int(truth[label_id]),
|
||||
"predicted_pixels": int(predicted[label_id]),
|
||||
"intersection_pixels": int(intersection[label_id]),
|
||||
"union_pixels": int(union[label_id]),
|
||||
"iou": round(float(intersection[label_id] / union[label_id]), 8)
|
||||
if union[label_id]
|
||||
else None,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def hex_rgb(value: str) -> tuple[int, int, int]:
|
||||
raw = bytes.fromhex(value.removeprefix("#"))
|
||||
if len(raw) != 3:
|
||||
raise RunnerError("policy action color must be RGB")
|
||||
return raw[0], raw[1], raw[2]
|
||||
|
||||
|
||||
def policy_palette(
|
||||
names: dict[int, str], policy: dict[str, Any], provider_map: dict[str, Any], preset: str,
|
||||
action_colors: dict[str, str]
|
||||
) -> np.ndarray:
|
||||
palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8)
|
||||
labels = provider_map["providers"]["goose-fine-64"]["labels"]
|
||||
rules = policy["presets"][preset]
|
||||
for label_id, class_name in names.items():
|
||||
material = labels.get(class_name)
|
||||
if material is None:
|
||||
continue
|
||||
action = rules[material]
|
||||
palette[label_id] = (*hex_rgb(action_colors[action]), 190)
|
||||
return palette
|
||||
|
||||
|
||||
def save_image(path: Path, value: Image.Image | np.ndarray, mode: str | None = None) -> str:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
image = value if isinstance(value, Image.Image) else Image.fromarray(value, mode=mode)
|
||||
image.save(path, format="PNG", optimize=True)
|
||||
return sha256(path)
|
||||
|
||||
|
||||
def expand_mask(
|
||||
mask: np.ndarray,
|
||||
original_size: tuple[int, int],
|
||||
crop_box: tuple[int, int, int, int],
|
||||
) -> np.ndarray:
|
||||
left, top, right, bottom = crop_box
|
||||
side = right - left
|
||||
resized = Image.fromarray(mask, mode="L").resize((side, side), resample=RESAMPLE_NEAREST)
|
||||
canvas = np.zeros((original_size[1], original_size[0]), dtype=np.uint8)
|
||||
canvas[top:bottom, left:right] = np.asarray(resized, dtype=np.uint8)
|
||||
return canvas
|
||||
|
||||
|
||||
def write_visual_case(
|
||||
output: Path,
|
||||
case_id: str,
|
||||
source: Image.Image,
|
||||
prediction: np.ndarray,
|
||||
semantic_palette: np.ndarray,
|
||||
policy_palettes: dict[str, np.ndarray],
|
||||
crop_box: tuple[int, int, int, int],
|
||||
truth: np.ndarray | None = None,
|
||||
preserve_source_size: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
case_root = output / "cases" / case_id
|
||||
if preserve_source_size:
|
||||
source_image = source.convert("RGB")
|
||||
prediction_image = expand_mask(prediction, source_image.size, crop_box)
|
||||
truth_image = expand_mask(truth, source_image.size, crop_box) if truth is not None else None
|
||||
else:
|
||||
cropped, _ = center_crop(source.convert("RGB"))
|
||||
source_image = cropped.resize((512, 512), resample=RESAMPLE_NEAREST)
|
||||
prediction_image = prediction
|
||||
truth_image = truth
|
||||
|
||||
files: dict[str, dict[str, str]] = {}
|
||||
source_path = case_root / "source.png"
|
||||
files["source"] = {
|
||||
"relative_path": source_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(source_path, source_image),
|
||||
}
|
||||
prediction_path = case_root / "prediction-labelids.png"
|
||||
files["prediction_labelids"] = {
|
||||
"relative_path": prediction_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(prediction_path, prediction_image, "L"),
|
||||
}
|
||||
semantic_path = case_root / "prediction-semantic.png"
|
||||
files["prediction_semantic"] = {
|
||||
"relative_path": semantic_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(semantic_path, semantic_palette[prediction_image], "RGBA"),
|
||||
}
|
||||
for preset, palette in policy_palettes.items():
|
||||
policy_path = case_root / f"policy-{preset}.png"
|
||||
files[f"policy_{preset}"] = {
|
||||
"relative_path": policy_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(policy_path, palette[prediction_image], "RGBA"),
|
||||
}
|
||||
if truth_image is not None:
|
||||
truth_path = case_root / "truth-labelids.png"
|
||||
files["truth_labelids"] = {
|
||||
"relative_path": truth_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(truth_path, truth_image, "L"),
|
||||
}
|
||||
truth_semantic_path = case_root / "truth-semantic.png"
|
||||
files["truth_semantic"] = {
|
||||
"relative_path": truth_semantic_path.relative_to(output).as_posix(),
|
||||
"sha256": save_image(truth_semantic_path, semantic_palette[truth_image], "RGBA"),
|
||||
}
|
||||
return {
|
||||
"schema_version": VISUAL_SCHEMA,
|
||||
"case_id": case_id,
|
||||
"source_width": source_image.width,
|
||||
"source_height": source_image.height,
|
||||
"center_crop_xyxy": list(crop_box),
|
||||
"outside_crop_state": "undefined" if preserve_source_size else "not-applicable",
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
ordered = sorted(values)
|
||||
index = (len(ordered) - 1) * fraction
|
||||
lower = math.floor(index)
|
||||
upper = math.ceil(index)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
return ordered[lower] * (upper - index) + ordered[upper] * (index - lower)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
args = parse_args()
|
||||
if not torch.cuda.is_available():
|
||||
raise RunnerError("CUDA is required for Worker 006 qualification")
|
||||
if args.limit < 0 or args.visual_count < 0:
|
||||
raise RunnerError("limit and visual-count must be non-negative")
|
||||
config = read_json(args.config, "benchmark config")
|
||||
policy = read_json(args.policy, "mission policy")
|
||||
provider_map = read_json(args.provider_map, "provider map")
|
||||
candidate_config = validate_contracts(config, policy, provider_map, args.candidate)
|
||||
if args.checkpoint.is_symlink() or not args.checkpoint.is_file():
|
||||
raise RunnerError("checkpoint is unavailable")
|
||||
if args.checkpoint.stat().st_size != candidate_config["checkpoint_size_bytes"]:
|
||||
raise RunnerError("checkpoint size changed")
|
||||
checkpoint_sha256 = sha256(args.checkpoint)
|
||||
if checkpoint_sha256 != candidate_config["checkpoint_sha256"]:
|
||||
raise RunnerError("checkpoint digest changed")
|
||||
|
||||
dataset_config = config["dataset"]
|
||||
mapping_root = args.dataset_root
|
||||
if mapping_root is None:
|
||||
raise RunnerError("dataset-root is required for the immutable mapping")
|
||||
mapping_path = mapping_root / dataset_config["mapping_relative_path"]
|
||||
names, semantic_palette = load_mapping(mapping_path, dataset_config["mapping_sha256"])
|
||||
policy_palettes = {
|
||||
preset: policy_palette(
|
||||
names,
|
||||
policy,
|
||||
provider_map,
|
||||
preset,
|
||||
config["policy_action_colors"],
|
||||
)
|
||||
for preset in ("urban", "rural", "offroad")
|
||||
}
|
||||
|
||||
if args.mode == "goose":
|
||||
pairs = find_goose_pairs(mapping_root)
|
||||
if len(pairs) != dataset_config["expected_pair_count"]:
|
||||
raise RunnerError(f"GOOSE pair count changed: {len(pairs)}")
|
||||
items: list[tuple[str, Path, Path | None]] = [
|
||||
(image.stem.removesuffix("_windshield_vis"), image, label)
|
||||
for image, label in pairs
|
||||
]
|
||||
else:
|
||||
if args.frames_root is None or not args.frames_root.is_dir():
|
||||
raise RunnerError("frames-root is required for RAVNOVES mode")
|
||||
frames = sorted(args.frames_root.glob("frame-*.png"))
|
||||
expected = {f"frame-{index:06d}" for index in config["ravnoves"]["frame_indices"]}
|
||||
if {frame.stem for frame in frames} != expected:
|
||||
raise RunnerError("RAVNOVES frame island identity changed")
|
||||
items = [(frame.stem, frame, None) for frame in frames]
|
||||
|
||||
if args.limit:
|
||||
items = items[: args.limit]
|
||||
if not items:
|
||||
raise RunnerError("no inputs were selected")
|
||||
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
torch.cuda.empty_cache()
|
||||
model, model_name, architecture_failures = load_model(args.candidate, args.checkpoint)
|
||||
warmup_source = Image.open(items[0][1]).convert("RGB")
|
||||
warmup_tensor, _ = preprocess(warmup_source)
|
||||
warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]
|
||||
torch.cuda.reset_peak_memory_stats()
|
||||
selected_visuals = visual_indices(len(items), args.visual_count)
|
||||
confusion = np.zeros((CLASS_COUNT, CLASS_COUNT), dtype=np.int64)
|
||||
latencies_ms: list[float] = []
|
||||
visuals: list[dict[str, Any]] = []
|
||||
|
||||
for index, (case_id, source_path, label_path) in enumerate(items):
|
||||
source = Image.open(source_path).convert("RGB")
|
||||
tensor, crop_box = preprocess(source)
|
||||
prediction, latency_ms = infer(model, tensor)
|
||||
latencies_ms.append(latency_ms)
|
||||
truth = preprocess_label(Image.open(label_path)) if label_path is not None else None
|
||||
if truth is not None:
|
||||
update_confusion(confusion, truth, prediction)
|
||||
if index in selected_visuals:
|
||||
visuals.append(
|
||||
write_visual_case(
|
||||
args.output,
|
||||
case_id,
|
||||
source,
|
||||
prediction,
|
||||
semantic_palette,
|
||||
policy_palettes,
|
||||
crop_box,
|
||||
truth=truth,
|
||||
preserve_source_size=args.mode == "ravnoves",
|
||||
)
|
||||
)
|
||||
|
||||
rows = class_metrics(confusion, names) if args.mode == "goose" else []
|
||||
valid_ious = [row["iou"] for row in rows if row["iou"] is not None]
|
||||
vegetation_names = set(config["vegetation_class_names"])
|
||||
vegetation_rows = [row for row in rows if row["class_name"] in vegetation_names]
|
||||
vegetation_ious = [row["iou"] for row in vegetation_rows if row["iou"] is not None]
|
||||
timing = {
|
||||
"prewarm_inference_count": len(warmup_latencies_ms),
|
||||
"prewarm_latency_ms": round(warmup_latencies_ms[0], 4),
|
||||
"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4),
|
||||
"sample_count": len(latencies_ms),
|
||||
"latency_ms_p50": round(percentile(latencies_ms, 0.50), 4),
|
||||
"latency_ms_p95": round(percentile(latencies_ms, 0.95), 4),
|
||||
"latency_ms_mean": round(statistics.fmean(latencies_ms), 4),
|
||||
"throughput_fps_from_mean_inference": round(1000.0 / statistics.fmean(latencies_ms), 4),
|
||||
}
|
||||
result: dict[str, Any] = {
|
||||
"schema_version": SCHEMA,
|
||||
"lab_id": config["lab_id"],
|
||||
"worker_id": config["worker_id"],
|
||||
"mode": args.mode,
|
||||
"candidate": {
|
||||
"candidate_id": candidate_config["candidate_id"],
|
||||
"candidate_key": args.candidate,
|
||||
"loaded_model_name": model_name,
|
||||
"architecture_probe_failures": architecture_failures,
|
||||
"checkpoint_size_bytes": args.checkpoint.stat().st_size,
|
||||
"checkpoint_sha256": checkpoint_sha256,
|
||||
},
|
||||
"source": {
|
||||
"source_id": dataset_config["dataset_id"]
|
||||
if args.mode == "goose"
|
||||
else config["ravnoves"]["source_id"],
|
||||
"input_count": len(items),
|
||||
"ground_truth_available": args.mode == "goose",
|
||||
"mapping_sha256": dataset_config["mapping_sha256"],
|
||||
},
|
||||
"preprocessing": dataset_config["preprocessing"],
|
||||
"metrics": {
|
||||
"mean_iou": round(statistics.fmean(valid_ious), 8) if valid_ious else None,
|
||||
"mean_iou_percent": round(statistics.fmean(valid_ious) * 100.0, 4)
|
||||
if valid_ious
|
||||
else None,
|
||||
"published_mean_iou_percent": candidate_config["published_validation_miou_percent"],
|
||||
"vegetation_mean_iou": round(statistics.fmean(vegetation_ious), 8)
|
||||
if vegetation_ious
|
||||
else None,
|
||||
"vegetation_classes": vegetation_rows,
|
||||
"all_classes": rows,
|
||||
},
|
||||
"timing": timing,
|
||||
"resource": {
|
||||
"gpu_name": torch.cuda.get_device_name(0),
|
||||
"peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated()),
|
||||
"peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved()),
|
||||
"torch_version": torch.__version__,
|
||||
"cuda_runtime_version": torch.version.cuda,
|
||||
"python_version": platform.python_version(),
|
||||
"super_gradients_version": "3.2.0",
|
||||
},
|
||||
"visual_cases": visuals,
|
||||
"authority": {
|
||||
"navigation_accepted": False,
|
||||
"safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
},
|
||||
}
|
||||
identity_value = {
|
||||
"schema_version": result["schema_version"],
|
||||
"candidate": result["candidate"],
|
||||
"source": result["source"],
|
||||
"preprocessing": result["preprocessing"],
|
||||
"metrics": result["metrics"],
|
||||
"timing": result["timing"],
|
||||
"resource": result["resource"],
|
||||
"visual_cases": result["visual_cases"],
|
||||
"authority": result["authority"],
|
||||
"config_sha256": sha256(args.config),
|
||||
"policy_sha256": sha256(args.policy),
|
||||
"provider_map_sha256": sha256(args.provider_map),
|
||||
}
|
||||
result["result_id"] = f"lab-v1-{args.mode}-{args.candidate}-{stable_digest(identity_value)}"
|
||||
result["provenance"] = {
|
||||
"config_sha256": identity_value["config_sha256"],
|
||||
"policy_sha256": identity_value["policy_sha256"],
|
||||
"provider_map_sha256": identity_value["provider_map_sha256"],
|
||||
"hostname": platform.node(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
result_path = args.output / "result.json"
|
||||
result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
print(json.dumps({"result_id": result["result_id"], "result_path": str(result_path)}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
@@ -326,6 +326,7 @@ def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]:
|
||||
"experimental.e47-semantic-slam-shadow/v1": _run_e47,
|
||||
"experimental.m48s-fixed-class-detector/v1": _run_m48s_fixed_class_detector,
|
||||
"experimental.m48t-risk-quality-temporal/v1": _run_m48t_risk_quality_temporal,
|
||||
"experimental.lab-v1-vegetation-shadow/v1": _run_lab_v1_vegetation_shadow,
|
||||
}
|
||||
|
||||
|
||||
@@ -382,6 +383,28 @@ def _run_m48t_risk_quality_temporal(
|
||||
)
|
||||
|
||||
|
||||
def _run_lab_v1_vegetation_shadow(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab
|
||||
|
||||
result_root = seal_vegetation_shadow_lab(
|
||||
ddrnet_goose_root=request.inputs["ddrnet_goose_root"],
|
||||
ppliteseg_goose_root=request.inputs["ppliteseg_goose_root"],
|
||||
ddrnet_ravnoves_root=request.inputs["ddrnet_ravnoves_root"],
|
||||
ppliteseg_ravnoves_root=request.inputs["ppliteseg_ravnoves_root"],
|
||||
output_root=request.output_root,
|
||||
)
|
||||
manifest = _object(
|
||||
json.loads((result_root / "result.json").read_text(encoding="utf-8")),
|
||||
"vegetation shadow result",
|
||||
)
|
||||
result_id = manifest.get("result_id")
|
||||
if not isinstance(result_id, str):
|
||||
raise LaboratoryExecutionError("vegetation shadow result_id is invalid")
|
||||
return LaboratoryAdapterResult(result_root=result_root, result_id=result_id)
|
||||
|
||||
|
||||
def _run_m48_small_static_passage_regression(
|
||||
request: LaboratoryRunRequest,
|
||||
) -> LaboratoryAdapterResult:
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
"""Seal GOOSE qualification and RAVNOVES vegetation shadow evidence for LAB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
LAB_SCHEMA: Final = "missioncore.lab-v1-vegetation-shadow/v1"
|
||||
WORKER_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||
RESULT_PREFIX: Final = "lab-v1-vegetation-shadow-"
|
||||
_CANDIDATES: Final = ("ddrnet", "ppliteseg")
|
||||
_MODES: Final = ("goose", "ravnoves")
|
||||
_IMAGE_KEYS: Final = (
|
||||
"source",
|
||||
"prediction_semantic",
|
||||
"policy_urban",
|
||||
"policy_rural",
|
||||
"policy_offroad",
|
||||
"truth_semantic",
|
||||
)
|
||||
|
||||
|
||||
class VegetationShadowLabError(ValueError):
|
||||
"""Raised when Worker evidence cannot be sealed without changing its meaning."""
|
||||
|
||||
|
||||
def canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def sha256_path(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _object(value: object, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or not all(isinstance(key, str) for key in value):
|
||||
raise VegetationShadowLabError(f"{label} must be an object")
|
||||
return value
|
||||
|
||||
|
||||
def _read_worker_result(root: Path, *, candidate: str, mode: str) -> dict[str, Any]:
|
||||
if root.is_symlink() or not root.is_dir():
|
||||
raise VegetationShadowLabError(f"{candidate}/{mode} result root is unavailable")
|
||||
path = root / "result.json"
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 1024 * 1024:
|
||||
raise VegetationShadowLabError(f"{candidate}/{mode} result document is unavailable")
|
||||
try:
|
||||
result = _object(json.loads(path.read_text("utf-8")), f"{candidate}/{mode} result")
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
raise VegetationShadowLabError(f"{candidate}/{mode} result is invalid") from exc
|
||||
candidate_value = _object(result.get("candidate"), f"{candidate}/{mode} candidate")
|
||||
authority = _object(result.get("authority"), f"{candidate}/{mode} authority")
|
||||
if (
|
||||
result.get("schema_version") != WORKER_SCHEMA
|
||||
or result.get("mode") != mode
|
||||
or candidate_value.get("candidate_key") != candidate
|
||||
or authority.get("navigation_accepted") is not False
|
||||
or authority.get("safety_accepted") is not False
|
||||
or authority.get("actuation_accepted") is not False
|
||||
or authority.get("camera_semantics_can_clear_rigid_geometry") is not False
|
||||
):
|
||||
raise VegetationShadowLabError(f"{candidate}/{mode} contract changed")
|
||||
return result
|
||||
|
||||
|
||||
def _case_map(result: dict[str, Any], label: str) -> dict[str, dict[str, Any]]:
|
||||
values = result.get("visual_cases")
|
||||
if not isinstance(values, list) or len(values) != 12:
|
||||
raise VegetationShadowLabError(f"{label} must contain exactly 12 visual cases")
|
||||
rows: dict[str, dict[str, Any]] = {}
|
||||
for raw in values:
|
||||
row = _object(raw, f"{label} visual case")
|
||||
case_id = row.get("case_id")
|
||||
if not isinstance(case_id, str) or not case_id or case_id in rows:
|
||||
raise VegetationShadowLabError(f"{label} case identity changed")
|
||||
rows[case_id] = row
|
||||
return rows
|
||||
|
||||
|
||||
def _file_from_case(
|
||||
root: Path,
|
||||
case: dict[str, Any],
|
||||
key: str,
|
||||
*,
|
||||
required: bool = True,
|
||||
) -> tuple[Path, str] | None:
|
||||
files = _object(case.get("files"), "visual case files")
|
||||
raw = files.get(key)
|
||||
if raw is None and not required:
|
||||
return None
|
||||
descriptor = _object(raw, f"visual case {key}")
|
||||
relative = descriptor.get("relative_path")
|
||||
expected_sha256 = descriptor.get("sha256")
|
||||
if not isinstance(relative, str) or not isinstance(expected_sha256, str):
|
||||
raise VegetationShadowLabError(f"visual case {key} proof is invalid")
|
||||
posix = PurePosixPath(relative)
|
||||
if posix.is_absolute() or str(posix) != relative or any(
|
||||
part in {"", ".", ".."} for part in posix.parts
|
||||
):
|
||||
raise VegetationShadowLabError(f"visual case {key} path is invalid")
|
||||
path = root.joinpath(*posix.parts)
|
||||
if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
|
||||
raise VegetationShadowLabError(f"visual case {key} file is unavailable")
|
||||
if sha256_path(path) != expected_sha256:
|
||||
raise VegetationShadowLabError(f"visual case {key} digest changed")
|
||||
return path, expected_sha256
|
||||
|
||||
|
||||
def _copy_artifact(
|
||||
source: Path,
|
||||
destination_root: Path,
|
||||
relative: str,
|
||||
artifacts: list[dict[str, object]],
|
||||
*,
|
||||
role: str,
|
||||
media_type: str,
|
||||
) -> dict[str, object]:
|
||||
destination = destination_root.joinpath(*PurePosixPath(relative).parts)
|
||||
destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
shutil.copyfile(source, destination)
|
||||
digest = sha256_path(destination)
|
||||
descriptor = {
|
||||
"role": role,
|
||||
"path": relative,
|
||||
"byte_length": destination.stat().st_size,
|
||||
"sha256": digest,
|
||||
"media_type": media_type,
|
||||
}
|
||||
artifacts.append(descriptor)
|
||||
return descriptor
|
||||
|
||||
|
||||
def _selected_candidate(results: dict[tuple[str, str], dict[str, Any]]) -> str:
|
||||
def rank(candidate: str) -> tuple[float, float]:
|
||||
validation = results[(candidate, "goose")]
|
||||
metrics = _object(validation.get("metrics"), f"{candidate} metrics")
|
||||
timing = _object(validation.get("timing"), f"{candidate} timing")
|
||||
vegetation_iou = metrics.get("vegetation_mean_iou")
|
||||
p95_ms = timing.get("latency_ms_p95")
|
||||
if not isinstance(vegetation_iou, (int, float)) or not isinstance(p95_ms, (int, float)):
|
||||
raise VegetationShadowLabError(f"{candidate} qualification metrics are incomplete")
|
||||
return float(vegetation_iou), -float(p95_ms)
|
||||
|
||||
return max(_CANDIDATES, key=rank)
|
||||
|
||||
|
||||
def _validation_metric_summary(result: dict[str, Any], candidate: str) -> dict[str, object]:
|
||||
metrics = _object(result.get("metrics"), f"{candidate} metrics")
|
||||
return {
|
||||
"mean_iou_percent": metrics.get("mean_iou_percent"),
|
||||
"published_mean_iou_percent": metrics.get("published_mean_iou_percent"),
|
||||
"vegetation_mean_iou": metrics.get("vegetation_mean_iou"),
|
||||
}
|
||||
|
||||
|
||||
def seal_vegetation_shadow_lab(
|
||||
*,
|
||||
ddrnet_goose_root: Path,
|
||||
ppliteseg_goose_root: Path,
|
||||
ddrnet_ravnoves_root: Path,
|
||||
ppliteseg_ravnoves_root: Path,
|
||||
output_root: Path,
|
||||
) -> Path:
|
||||
roots = {
|
||||
("ddrnet", "goose"): ddrnet_goose_root.resolve(),
|
||||
("ppliteseg", "goose"): ppliteseg_goose_root.resolve(),
|
||||
("ddrnet", "ravnoves"): ddrnet_ravnoves_root.resolve(),
|
||||
("ppliteseg", "ravnoves"): ppliteseg_ravnoves_root.resolve(),
|
||||
}
|
||||
results = {
|
||||
key: _read_worker_result(root, candidate=key[0], mode=key[1])
|
||||
for key, root in roots.items()
|
||||
}
|
||||
cases = {key: _case_map(result, f"{key[0]}/{key[1]}") for key, result in results.items()}
|
||||
for mode in _MODES:
|
||||
if cases[("ddrnet", mode)].keys() != cases[("ppliteseg", mode)].keys():
|
||||
raise VegetationShadowLabError(f"{mode} candidate case islands differ")
|
||||
selected = _selected_candidate(results)
|
||||
|
||||
output_root.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||
temporary = Path(tempfile.mkdtemp(prefix=".lab-v1-vegetation-", dir=output_root))
|
||||
artifacts: list[dict[str, object]] = []
|
||||
catalogs: dict[str, list[dict[str, object]]] = {"goose": [], "ravnoves": []}
|
||||
try:
|
||||
for mode in _MODES:
|
||||
for case_id in sorted(cases[("ddrnet", mode)]):
|
||||
ddr_case = cases[("ddrnet", mode)][case_id]
|
||||
pplite_case = cases[("ppliteseg", mode)][case_id]
|
||||
row: dict[str, object] = {
|
||||
"case_id": case_id,
|
||||
"source_kind": mode,
|
||||
"width": ddr_case.get("source_width"),
|
||||
"height": ddr_case.get("source_height"),
|
||||
"center_crop_xyxy": ddr_case.get("center_crop_xyxy"),
|
||||
"outside_crop_state": ddr_case.get("outside_crop_state"),
|
||||
"assets": {},
|
||||
}
|
||||
asset_map = _object(row["assets"], "sealed assets")
|
||||
sources: list[tuple[str, str, dict[str, Any], str]] = [
|
||||
("source", "ddrnet", ddr_case, "source"),
|
||||
("ddrnet", "ddrnet", ddr_case, "prediction_semantic"),
|
||||
("ppliteseg", "ppliteseg", pplite_case, "prediction_semantic"),
|
||||
]
|
||||
if mode == "goose":
|
||||
sources.append(("truth", "ddrnet", ddr_case, "truth_semantic"))
|
||||
else:
|
||||
selected_case = cases[(selected, mode)][case_id]
|
||||
sources.extend(
|
||||
(preset, selected, selected_case, f"policy_{preset}")
|
||||
for preset in ("urban", "rural", "offroad")
|
||||
)
|
||||
for asset_key, candidate, case, worker_key in sources:
|
||||
resolved = _file_from_case(roots[(candidate, mode)], case, worker_key)
|
||||
assert resolved is not None
|
||||
source_path, _ = resolved
|
||||
relative = f"visual/{mode}/{case_id}/{asset_key}.png"
|
||||
descriptor = _copy_artifact(
|
||||
source_path,
|
||||
temporary,
|
||||
relative,
|
||||
artifacts,
|
||||
role=f"visual-{mode}-{asset_key}",
|
||||
media_type="image/png",
|
||||
)
|
||||
asset_map[asset_key] = {
|
||||
"path": descriptor["path"],
|
||||
"sha256": descriptor["sha256"],
|
||||
}
|
||||
catalogs[mode].append(row)
|
||||
|
||||
worker_proofs: dict[str, dict[str, object]] = {}
|
||||
for candidate in _CANDIDATES:
|
||||
for mode in _MODES:
|
||||
source = roots[(candidate, mode)] / "result.json"
|
||||
relative = f"worker/{candidate}-{mode}.json"
|
||||
descriptor = _copy_artifact(
|
||||
source,
|
||||
temporary,
|
||||
relative,
|
||||
artifacts,
|
||||
role="worker-result",
|
||||
media_type="application/json",
|
||||
)
|
||||
worker_proofs[f"{candidate}_{mode}"] = {
|
||||
"result_id": results[(candidate, mode)].get("result_id"),
|
||||
"path": relative,
|
||||
"sha256": descriptor["sha256"],
|
||||
}
|
||||
|
||||
candidate_metrics: dict[str, object] = {}
|
||||
for candidate in _CANDIDATES:
|
||||
validation = results[(candidate, "goose")]
|
||||
shadow = results[(candidate, "ravnoves")]
|
||||
candidate_metrics[candidate] = {
|
||||
"loaded_model_name": _object(validation["candidate"], "candidate").get(
|
||||
"loaded_model_name"
|
||||
),
|
||||
"checkpoint_sha256": _object(validation["candidate"], "candidate").get(
|
||||
"checkpoint_sha256"
|
||||
),
|
||||
# Detailed per-class rows remain immutable in worker_proofs. The
|
||||
# top-level LAB manifest carries only the UI/index summary.
|
||||
"validation_metrics": _validation_metric_summary(validation, candidate),
|
||||
"validation_timing": validation.get("timing"),
|
||||
"shadow_timing": shadow.get("timing"),
|
||||
"resource": shadow.get("resource"),
|
||||
}
|
||||
|
||||
authority = {
|
||||
"commands_enabled": False,
|
||||
"navigation_or_safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
}
|
||||
identity = {
|
||||
"lab_id": "lab-v1-vegetation-mission-policy",
|
||||
"source": {
|
||||
"validation_dataset": "GOOSE-2D-validation-visible-962",
|
||||
"shadow_session": "RAVNOVES00",
|
||||
"shadow_camera": "sensor.camera.right",
|
||||
"shadow_frame_count": 12,
|
||||
},
|
||||
"selected_candidate": selected,
|
||||
"candidate_metrics": candidate_metrics,
|
||||
"worker_proofs": worker_proofs,
|
||||
"visual_catalog_sha256": hashlib.sha256(canonical_json(catalogs)).hexdigest(),
|
||||
"authority": authority,
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = f"{RESULT_PREFIX}{identity_sha256}"
|
||||
manifest = {
|
||||
"schema_version": LAB_SCHEMA,
|
||||
"result_id": result_id,
|
||||
"identity_sha256": identity_sha256,
|
||||
"created_at_utc": datetime.now(UTC).isoformat(),
|
||||
"ground_truth": False,
|
||||
"status": "visual-shadow-ready-policy-not-authorized",
|
||||
"identity": identity,
|
||||
"source": identity["source"],
|
||||
"method": {
|
||||
"completeness": "complete",
|
||||
"execution_class": "ai-inference",
|
||||
"pipeline_id": "goose-fine64-ready-weights-to-ravnoves-policy-shadow/v1",
|
||||
},
|
||||
"metrics": {"candidates": candidate_metrics},
|
||||
"decision": {
|
||||
"selected_candidate": selected,
|
||||
"visual_shadow_ready": True,
|
||||
"mission_policy_ready_for_configuration": True,
|
||||
"navigation_accepted": False,
|
||||
"production_accepted": False,
|
||||
},
|
||||
"limitations": [
|
||||
"GOOSE validation is external-domain qualification, not RAVNOVES ground truth.",
|
||||
"The RAVNOVES island is visual shadow evidence without independent labels.",
|
||||
"Vegetation semantics never clears rigid LiDAR/TGS occupancy.",
|
||||
"Undefined pixels outside the 600x600 center crop remain fail-closed.",
|
||||
],
|
||||
"authority": authority,
|
||||
"catalogs": catalogs,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
(temporary / "result.json").write_bytes(canonical_json(manifest) + b"\n")
|
||||
destination = output_root / result_id
|
||||
if destination.exists():
|
||||
raise VegetationShadowLabError("immutable vegetation LAB result already exists")
|
||||
temporary.replace(destination)
|
||||
return destination
|
||||
except Exception:
|
||||
shutil.rmtree(temporary, ignore_errors=True)
|
||||
raise
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--ddrnet-goose-root", type=Path, required=True)
|
||||
parser.add_argument("--ppliteseg-goose-root", type=Path, required=True)
|
||||
parser.add_argument("--ddrnet-ravnoves-root", type=Path, required=True)
|
||||
parser.add_argument("--ppliteseg-ravnoves-root", type=Path, required=True)
|
||||
parser.add_argument("--output-root", type=Path, required=True)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
destination = seal_vegetation_shadow_lab(
|
||||
ddrnet_goose_root=args.ddrnet_goose_root,
|
||||
ppliteseg_goose_root=args.ppliteseg_goose_root,
|
||||
ddrnet_ravnoves_root=args.ddrnet_ravnoves_root,
|
||||
ppliteseg_ravnoves_root=args.ppliteseg_ravnoves_root,
|
||||
output_root=args.output_root,
|
||||
)
|
||||
print(destination)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LAB_SCHEMA",
|
||||
"RESULT_PREFIX",
|
||||
"VegetationShadowLabError",
|
||||
"seal_vegetation_shadow_lab",
|
||||
]
|
||||
@@ -138,6 +138,7 @@ from k1link.web.m49_physical_safety_playback_api import (
|
||||
)
|
||||
from k1link.web.m49_tgs_fail_closed_api import build_m49_tgs_fail_closed_router
|
||||
from k1link.web.m49_tgs_full_shadow_api import build_m49_tgs_full_shadow_router
|
||||
from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router
|
||||
from k1link.web.map_api import (
|
||||
MapGatewayConfiguration,
|
||||
MapGatewayProxy,
|
||||
@@ -1019,6 +1020,17 @@ app.include_router(
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_vegetation_shadow_lab_router(
|
||||
root_provider=lambda: (
|
||||
REPOSITORY_ROOT
|
||||
/ ".runtime"
|
||||
/ "compute-experiments"
|
||||
/ "lab-v1-vegetation"
|
||||
/ "results"
|
||||
),
|
||||
)
|
||||
)
|
||||
app.include_router(
|
||||
build_m49_physical_safety_playback_router(
|
||||
root_provider=lambda: (
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Read-only API for the autonomous vegetation policy shadow LAB."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from functools import lru_cache
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Final
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from k1link.laboratory.evidence_registry import LaboratoryEvidenceDefinition
|
||||
from k1link.laboratory.evidence_report import (
|
||||
LaboratoryEvidenceReportError,
|
||||
verify_laboratory_evidence_result,
|
||||
)
|
||||
from k1link.laboratory.vegetation_shadow_lab import LAB_SCHEMA, RESULT_PREFIX
|
||||
|
||||
RootProvider = Callable[[], Path | None]
|
||||
RESULT_ID: Final = re.compile(rf"^{re.escape(RESULT_PREFIX)}[a-f0-9]{{64}}$")
|
||||
_MAX_DOCUMENT_BYTES: Final = 1024 * 1024
|
||||
_DEFINITION: Final = LaboratoryEvidenceDefinition(
|
||||
work_id="lab-v1-vegetation-shadow",
|
||||
runtime_relative_root=PurePosixPath("lab-v1-vegetation/results"),
|
||||
result_id_prefix="lab-v1-vegetation-shadow",
|
||||
document_name="result.json",
|
||||
result_schema_version=LAB_SCHEMA,
|
||||
)
|
||||
|
||||
|
||||
def build_vegetation_shadow_lab_router(
|
||||
*, root_provider: RootProvider = lambda: None,
|
||||
) -> APIRouter:
|
||||
router = APIRouter(
|
||||
prefix="/api/v1/laboratory/vegetation-shadow",
|
||||
tags=["laboratory"],
|
||||
)
|
||||
|
||||
@router.get("/{result_id}")
|
||||
def get_result(result_id: str) -> dict[str, object]:
|
||||
candidate = _resolve_candidate(root_provider, result_id)
|
||||
return {**copy.deepcopy(_read_verified(candidate)), "access": "read-only"}
|
||||
|
||||
@router.get("/{result_id}/assets/{asset_path:path}")
|
||||
def get_asset(result_id: str, asset_path: str) -> FileResponse:
|
||||
candidate = _resolve_candidate(root_provider, result_id)
|
||||
manifest = _read_verified(candidate)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list):
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB asset not found")
|
||||
descriptor = next(
|
||||
(
|
||||
item
|
||||
for item in artifacts
|
||||
if isinstance(item, dict) and item.get("path") == asset_path
|
||||
),
|
||||
None,
|
||||
)
|
||||
if descriptor is None:
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB asset not found")
|
||||
relative = PurePosixPath(asset_path)
|
||||
path = candidate.joinpath(*relative.parts)
|
||||
if (
|
||||
relative.is_absolute()
|
||||
or str(relative) != asset_path
|
||||
or any(part in {"", ".", ".."} for part in relative.parts)
|
||||
or path.is_symlink()
|
||||
or not path.is_file()
|
||||
or not path.resolve().is_relative_to(candidate)
|
||||
):
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB asset not found")
|
||||
media_type = descriptor.get("media_type")
|
||||
if not isinstance(media_type, str) or not media_type.startswith("image/"):
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB asset not found")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=media_type,
|
||||
headers={
|
||||
"Cache-Control": "private, max-age=31536000, immutable",
|
||||
"ETag": f'"{descriptor.get("sha256", "")}"',
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
},
|
||||
)
|
||||
|
||||
return router
|
||||
|
||||
|
||||
def _configured_root(provider: RootProvider) -> Path | None:
|
||||
candidate = provider()
|
||||
if candidate is None:
|
||||
return None
|
||||
absolute = candidate.expanduser().absolute()
|
||||
if absolute.is_symlink():
|
||||
return None
|
||||
try:
|
||||
root = absolute.resolve(strict=True)
|
||||
except OSError:
|
||||
return None
|
||||
return root if root.is_dir() else None
|
||||
|
||||
|
||||
def _resolve_candidate(provider: RootProvider, result_id: str) -> Path:
|
||||
root = _configured_root(provider)
|
||||
if root is None or RESULT_ID.fullmatch(result_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB result not found")
|
||||
candidate = root / result_id
|
||||
if candidate.is_symlink():
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB result not found")
|
||||
try:
|
||||
resolved = candidate.resolve(strict=True)
|
||||
except OSError:
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB result not found") from None
|
||||
if not resolved.is_dir() or not resolved.is_relative_to(root):
|
||||
raise HTTPException(status_code=404, detail="Vegetation LAB result not found")
|
||||
return resolved
|
||||
|
||||
|
||||
def _read_verified(candidate: Path) -> dict[str, Any]:
|
||||
try:
|
||||
rows: list[tuple[str, int, int, int, int]] = []
|
||||
for path in sorted(candidate.rglob("*"), key=lambda item: item.as_posix()):
|
||||
stat = path.lstat()
|
||||
rows.append((
|
||||
path.relative_to(candidate).as_posix(),
|
||||
stat.st_mode,
|
||||
stat.st_size,
|
||||
stat.st_mtime_ns,
|
||||
stat.st_ctime_ns,
|
||||
))
|
||||
signature = tuple(rows)
|
||||
except OSError:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Vegetation LAB evidence failed verification",
|
||||
) from None
|
||||
return _read_verified_cached(str(candidate), signature)
|
||||
|
||||
|
||||
@lru_cache(maxsize=16)
|
||||
def _read_verified_cached(
|
||||
candidate_text: str,
|
||||
signature: tuple[tuple[str, int, int, int, int], ...],
|
||||
) -> dict[str, Any]:
|
||||
del signature
|
||||
candidate = Path(candidate_text)
|
||||
try:
|
||||
verify_laboratory_evidence_result(_DEFINITION, candidate)
|
||||
path = candidate / "result.json"
|
||||
if path.stat().st_size > _MAX_DOCUMENT_BYTES:
|
||||
raise LaboratoryEvidenceReportError("Vegetation LAB document is too large")
|
||||
payload = json.loads(path.read_text("utf-8"))
|
||||
except (json.JSONDecodeError, OSError, LaboratoryEvidenceReportError):
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="Vegetation LAB evidence failed verification",
|
||||
) from None
|
||||
if not isinstance(payload, dict):
|
||||
raise HTTPException(status_code=503, detail="Vegetation LAB evidence is invalid")
|
||||
return payload
|
||||
|
||||
|
||||
__all__ = ["build_vegetation_shadow_lab_router"]
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIG_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "lab-v1-goose-vegetation-benchmark-v1.json"
|
||||
)
|
||||
RUNNER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "lab_v1_vegetation_goose"
|
||||
/ "run_goose_vegetation_benchmark.py"
|
||||
)
|
||||
POWERSHELL_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "Invoke-LabV1VegetationGooseBenchmark.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_benchmark_contract_is_bounded_and_fail_closed() -> None:
|
||||
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
assert config["schema_version"] == "missioncore.lab-v1-goose-vegetation-benchmark/v1"
|
||||
assert config["dataset"]["expected_pair_count"] == 962
|
||||
assert config["dataset"]["image_glob"].endswith("*_windshield_vis.png")
|
||||
assert config["dataset"]["preprocessing"] == [
|
||||
"center-square-crop",
|
||||
"nearest-neighbor-resize",
|
||||
"rgb-to-tensor-0-1",
|
||||
]
|
||||
assert set(config["candidates"]) == {"ddrnet", "ppliteseg"}
|
||||
assert all(
|
||||
len(candidate["checkpoint_sha256"]) == 64
|
||||
for candidate in config["candidates"].values()
|
||||
)
|
||||
assert config["ravnoves"]["expected_frame_count"] == 4489
|
||||
assert len(config["ravnoves"]["frame_indices"]) == 12
|
||||
assert config["invariants"] == {
|
||||
"one_heavy_candidate_at_a_time": True,
|
||||
"raw_fisheye_is_immutable": True,
|
||||
"outside_center_crop_is_free": False,
|
||||
"missing_or_unknown_is_free": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
"navigation_authority": False,
|
||||
"actuation_authority": False,
|
||||
"canonical_triton_mutation_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
def test_runner_uses_exact_visible_pairs_and_never_grants_authority() -> None:
|
||||
source = RUNNER_PATH.read_text(encoding="utf-8")
|
||||
assert 'rglob("*_windshield_vis.png")' in source
|
||||
assert "resample=RESAMPLE_NEAREST" in source
|
||||
assert '"navigation_accepted": False' in source
|
||||
assert '"actuation_accepted": False' in source
|
||||
assert '"camera_semantics_can_clear_rigid_geometry": False' in source
|
||||
assert "torch.cuda.reset_peak_memory_stats()" in source
|
||||
assert 'warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]' in source
|
||||
assert '"prewarm_inference_count": len(warmup_latencies_ms)' in source
|
||||
assert '"prewarm_latency_ms": round(warmup_latencies_ms[0], 4)' in source
|
||||
assert '"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4)' in source
|
||||
|
||||
|
||||
def test_worker_wrapper_is_isolated_from_canonical_triton() -> None:
|
||||
source = POWERSHELL_PATH.read_text(encoding="utf-8")
|
||||
assert '$canonicalContainer = "ndc-mission-core-triton"' in source
|
||||
assert '"--network", "none"' in source
|
||||
assert '"--read-only"' in source
|
||||
assert '"--cap-drop", "ALL"' in source
|
||||
assert '"--security-opt", "no-new-privileges"' in source
|
||||
assert "if ($canonicalAfter -ne $canonicalBefore)" in source
|
||||
@@ -127,8 +127,9 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
repository_root / "config" / "laboratories"
|
||||
)
|
||||
|
||||
assert len(registry.definitions) == 42
|
||||
assert len(registry.definitions) == 43
|
||||
assert {item.work_id for item in registry.definitions} >= {
|
||||
"lab-v1-vegetation-shadow",
|
||||
"e31-source-binding",
|
||||
"e46j-raw-fisheye-realtime",
|
||||
"e47-semantic-slam-shadow",
|
||||
|
||||
@@ -103,6 +103,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
by_work_id = {row.work_id: row for row in execution.definitions}
|
||||
assert by_work_id["m48-small-static-passage-regression"].evidence_contract == (
|
||||
@@ -124,6 +125,8 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
assert by_work_id["m49-tgs-fail-closed-evidence"].isolation == "bounded-adapter"
|
||||
assert by_work_id["m49-tgs-full-shadow"].lifecycle == "experimental"
|
||||
assert by_work_id["m49-tgs-full-shadow"].isolation == "bounded-adapter"
|
||||
assert by_work_id["lab-v1-vegetation-shadow"].lifecycle == "experimental"
|
||||
assert by_work_id["lab-v1-vegetation-shadow"].isolation == "bounded-adapter"
|
||||
assert all(
|
||||
row.lifecycle == "canonical"
|
||||
for row in execution.definitions
|
||||
@@ -134,6 +137,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
)
|
||||
assert len(execution.definitions) + len(execution.legacy_work_ids) == len(
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
root / "config" / "laboratory-value-review.json"
|
||||
)
|
||||
|
||||
assert len(registry.entries) == 40
|
||||
assert len(registry.entries) == 41
|
||||
assert {entry.catalog_id for entry in registry.entries} >= {
|
||||
"e28-local-surface",
|
||||
"e46d-temporal-failure-audit",
|
||||
@@ -93,4 +93,5 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.laboratory import LaboratoryEvidenceRegistry
|
||||
from k1link.laboratory.evidence_report import verify_laboratory_evidence_result
|
||||
from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab
|
||||
from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _worker_result(root: Path, *, candidate: str, mode: str, vegetation_iou: float) -> None:
|
||||
root.mkdir(parents=True)
|
||||
visual_cases = []
|
||||
for index in range(12):
|
||||
case_id = f"case-{index:02d}"
|
||||
case_root = root / "cases" / case_id
|
||||
case_root.mkdir(parents=True)
|
||||
keys = ["source", "prediction_semantic", "policy_urban", "policy_rural", "policy_offroad"]
|
||||
if mode == "goose":
|
||||
keys.append("truth_semantic")
|
||||
files = {}
|
||||
for key in keys:
|
||||
path = case_root / f"{key}.png"
|
||||
path.write_bytes(b"\x89PNG\r\n\x1a\n" + f"{candidate}:{mode}:{case_id}:{key}".encode())
|
||||
files[key] = {
|
||||
"relative_path": path.relative_to(root).as_posix(),
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
visual_cases.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"source_width": 800 if mode == "ravnoves" else 512,
|
||||
"source_height": 600 if mode == "ravnoves" else 512,
|
||||
"center_crop_xyxy": [100, 0, 700, 600] if mode == "ravnoves" else [0, 0, 512, 512],
|
||||
"outside_crop_state": "undefined" if mode == "ravnoves" else "not-applicable",
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"schema_version": "missioncore.lab-v1-goose-vegetation-run/v1",
|
||||
"result_id": f"lab-v1-{mode}-{candidate}-fixture",
|
||||
"mode": mode,
|
||||
"candidate": {
|
||||
"candidate_key": candidate,
|
||||
"loaded_model_name": "ddrnet_39" if candidate == "ddrnet" else "pp_lite_t_seg",
|
||||
"checkpoint_sha256": ("a" if candidate == "ddrnet" else "b") * 64,
|
||||
},
|
||||
"metrics": {
|
||||
"mean_iou_percent": 44.0 + vegetation_iou,
|
||||
"published_mean_iou_percent": 46.53 if candidate == "ddrnet" else 45.09,
|
||||
"vegetation_mean_iou": vegetation_iou,
|
||||
},
|
||||
"timing": {
|
||||
"latency_ms_p95": 20.0 if candidate == "ddrnet" else 15.0,
|
||||
"throughput_fps_from_mean_inference": 55.0,
|
||||
},
|
||||
"resource": {
|
||||
"peak_reserved_vram_bytes": 2_000_000_000,
|
||||
"gpu_name": "fixture RTX 4090",
|
||||
},
|
||||
"visual_cases": visual_cases,
|
||||
"authority": {
|
||||
"navigation_accepted": False,
|
||||
"safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
},
|
||||
}
|
||||
(root / "result.json").write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) -> None:
|
||||
roots = {}
|
||||
for candidate, vegetation_iou in (("ddrnet", 0.64), ("ppliteseg", 0.61)):
|
||||
for mode in ("goose", "ravnoves"):
|
||||
root = tmp_path / "worker" / f"{candidate}-{mode}"
|
||||
_worker_result(root, candidate=candidate, mode=mode, vegetation_iou=vegetation_iou)
|
||||
roots[(candidate, mode)] = root
|
||||
result_root = seal_vegetation_shadow_lab(
|
||||
ddrnet_goose_root=roots[("ddrnet", "goose")],
|
||||
ppliteseg_goose_root=roots[("ppliteseg", "goose")],
|
||||
ddrnet_ravnoves_root=roots[("ddrnet", "ravnoves")],
|
||||
ppliteseg_ravnoves_root=roots[("ppliteseg", "ravnoves")],
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
manifest = json.loads((result_root / "result.json").read_text("utf-8"))
|
||||
assert manifest["decision"]["selected_candidate"] == "ddrnet"
|
||||
assert manifest["ground_truth"] is False
|
||||
assert manifest["authority"]["commands_enabled"] is False
|
||||
assert manifest["authority"]["navigation_or_safety_accepted"] is False
|
||||
assert len(manifest["catalogs"]["ravnoves"]) == 12
|
||||
assert len(manifest["catalogs"]["goose"]) == 12
|
||||
assert len(manifest["artifacts"]) == 124
|
||||
assert "all_classes" not in manifest["metrics"]["candidates"]["ddrnet"]["validation_metrics"]
|
||||
assert (result_root / "result.json").stat().st_size <= 64 * 1024
|
||||
|
||||
registry = LaboratoryEvidenceRegistry.from_directory(REPOSITORY_ROOT / "config/laboratories")
|
||||
definition = next(
|
||||
row for row in registry.definitions if row.work_id == "lab-v1-vegetation-shadow"
|
||||
)
|
||||
proof = verify_laboratory_evidence_result(definition, result_root)
|
||||
assert proof["result_id"] == result_root.name
|
||||
assert proof["artifact_count"] == 124
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(build_vegetation_shadow_lab_router(root_provider=lambda: result_root.parent))
|
||||
client = TestClient(app)
|
||||
response = client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["access"] == "read-only"
|
||||
asset_path = manifest["catalogs"]["ravnoves"][0]["assets"]["offroad"]["path"]
|
||||
asset = client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{result_root.name}/assets/{asset_path}"
|
||||
)
|
||||
assert asset.status_code == 200
|
||||
assert asset.headers["cache-control"].endswith("immutable")
|
||||
|
||||
(result_root / asset_path).write_bytes(b"tampered")
|
||||
assert (
|
||||
client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}").status_code
|
||||
== 503
|
||||
)
|
||||
Reference in New Issue
Block a user