From 57204b3b0b7651d8e0bb0eacd5f712a4e11189f8 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Thu, 27 Aug 2026 23:26:35 +0300 Subject: [PATCH] feat(perception): add autonomous vegetation shadow lab --- .../src/core/laboratory/advancedIndex.ts | 13 +- .../laboratory/advancedLaboratoryResults.ts | 2 + .../src/core/laboratory/advancedResults.ts | 1 + .../src/core/laboratory/vegetationShadow.ts | 278 +++++++++ .../laboratory/AdvancedLaboratoryResult.tsx | 4 + .../laboratory/VegetationShadowResult.tsx | 135 +++++ .../laboratory/VegetationShadowVisual.tsx | 157 +++++ .../laboratory/laboratoryArchiveProfiles.ts | 7 + .../useAdvancedLaboratoryCatalog.ts | 2 + .../test/vegetationShadow.test.mjs | 125 ++++ .../lab-v1-vegetation-shadow.json | 10 + config/laboratory-execution.json | 19 + config/laboratory-value-review.json | 9 +- .../lab-v1-goose-vegetation-benchmark-v1.json | 97 +++ .../Invoke-LabV1VegetationGooseBenchmark.ps1 | 230 ++++++++ .../worker/lab_v1_vegetation_goose/Dockerfile | 49 ++ .../run_goose_vegetation_benchmark.py | 556 ++++++++++++++++++ src/k1link/laboratory/execution.py | 23 + .../laboratory/vegetation_shadow_lab.py | 381 ++++++++++++ src/k1link/web/app.py | 12 + src/k1link/web/vegetation_shadow_lab_api.py | 166 ++++++ .../test_lab_v1_goose_vegetation_benchmark.py | 80 +++ tests/test_laboratory_evidence_registry.py | 3 +- tests/test_laboratory_execution.py | 4 + .../test_laboratory_value_review_registry.py | 3 +- tests/test_vegetation_shadow_lab.py | 133 +++++ 26 files changed, 2494 insertions(+), 5 deletions(-) create mode 100644 apps/control-station/src/core/laboratory/vegetationShadow.ts create mode 100644 apps/control-station/src/workspaces/laboratory/VegetationShadowResult.tsx create mode 100644 apps/control-station/src/workspaces/laboratory/VegetationShadowVisual.tsx create mode 100644 apps/control-station/test/vegetationShadow.test.mjs create mode 100644 config/laboratories/lab-v1-vegetation-shadow.json create mode 100644 config/perception/lab-v1-goose-vegetation-benchmark-v1.json create mode 100644 experiments/perception/worker/Invoke-LabV1VegetationGooseBenchmark.ps1 create mode 100644 experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile create mode 100644 experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py create mode 100644 src/k1link/laboratory/vegetation_shadow_lab.py create mode 100644 src/k1link/web/vegetation_shadow_lab_api.py create mode 100644 tests/test_lab_v1_goose_vegetation_benchmark.py create mode 100644 tests/test_vegetation_shadow_lab.py diff --git a/apps/control-station/src/core/laboratory/advancedIndex.ts b/apps/control-station/src/core/laboratory/advancedIndex.ts index 9221b1e..48a601a 100644 --- a/apps/control-station/src/core/laboratory/advancedIndex.ts +++ b/apps/control-station/src/core/laboratory/advancedIndex.ts @@ -48,8 +48,10 @@ import { fetchM48SFixedClassDetectorResult } from "./m48sFixedClassDetector"; import { fetchM48TRiskQualityResult } from "./m48tRiskQuality"; import { fetchM49TgsFailClosedResult } from "./m49TgsFailClosed"; import { fetchM49TgsFullShadowResult } from "./m49TgsFullShadow"; +import { fetchVegetationShadowResult } from "./vegetationShadow"; export type AdvancedLaboratoryWorkId = + | "lab-v1-vegetation-shadow" | "m48-object-centric-quality" | "m48-small-static-passage-regression" | "m48-static-occupancy-qualification" @@ -100,6 +102,7 @@ export interface AdvancedLaboratoryIndexItem { } const WORK_IDS: readonly AdvancedLaboratoryWorkId[] = [ + "lab-v1-vegetation-shadow", "m48-object-centric-quality", "m48-small-static-passage-regression", "m48-static-occupancy-qualification", @@ -145,6 +148,7 @@ const WORK_IDS: readonly AdvancedLaboratoryWorkId[] = [ ]; const RESULT_PREFIX: Readonly> = { + "lab-v1-vegetation-shadow": "lab-v1-vegetation-shadow", "m48-object-centric-quality": "m48-object-quality-(?:pack|result)", "m48-small-static-passage-regression": "m48-small-static-passage-regression", "m48-static-occupancy-qualification": "m48-static-occupancy-qualification", @@ -197,6 +201,7 @@ export function isAdvancedLaboratoryWorkId( export function emptyAdvancedLaboratoryResults(): AdvancedLaboratoryResults { return { + vegetationShadow: null, m47Graph: null, m48: null, m48SmallStatic: null, @@ -330,7 +335,8 @@ export function advancedLaboratoryResultAvailable( workId: AdvancedLaboratoryWorkId, results: AdvancedLaboratoryResults, ): boolean { - return workId === "m48-object-centric-quality" ? results.m48 !== null + return workId === "lab-v1-vegetation-shadow" ? results.vegetationShadow !== null + : workId === "m48-object-centric-quality" ? results.m48 !== null : workId === "m48-small-static-passage-regression" ? results.m48SmallStatic !== null : workId === "m48-static-occupancy-qualification" ? results.m48StaticOccupancy !== null : workId === "m48r3-static-occupancy-shadow" ? results.m48r3StaticOccupancy !== null @@ -387,7 +393,10 @@ export async function fetchAdvancedLaboratoryResult( } = {}, ): Promise { const results = emptyAdvancedLaboratoryResults(); - if (workId === "m48-object-centric-quality") { + if (workId === "lab-v1-vegetation-shadow") { + if (!resultId) throw new AdvancedLaboratoryContractError("Vegetation LAB identity не выбрана."); + results.vegetationShadow = await fetchVegetationShadowResult(resultId, { fetcher, signal }); + } else if (workId === "m48-object-centric-quality") { if (!resultId) throw new AdvancedLaboratoryContractError("M4.8 lifecycle evidence identity не выбрана."); results.m48 = await fetchM48LifecycleResult(resultId, { fetcher, signal }); } else if (workId === "m48-small-static-passage-regression") { diff --git a/apps/control-station/src/core/laboratory/advancedLaboratoryResults.ts b/apps/control-station/src/core/laboratory/advancedLaboratoryResults.ts index ed6bed1..ca9ee36 100644 --- a/apps/control-station/src/core/laboratory/advancedLaboratoryResults.ts +++ b/apps/control-station/src/core/laboratory/advancedLaboratoryResults.ts @@ -42,8 +42,10 @@ import type { M48SFixedClassDetectorResult } from "./m48sFixedClassDetector"; import type { M48TRiskQualityResult } from "./m48tRiskQuality"; import type { M49TgsFailClosedResult } from "./m49TgsFailClosed"; import type { M49TgsFullShadowResult } from "./m49TgsFullShadow"; +import type { VegetationShadowResult } from "./vegetationShadow"; export interface AdvancedLaboratoryResults { + vegetationShadow: VegetationShadowResult | null; m47Graph: M47ReferenceGraphLabResult | null; m48: M48AdvancedResult | null; m48SmallStatic: M48SmallStaticRegressionResult | null; diff --git a/apps/control-station/src/core/laboratory/advancedResults.ts b/apps/control-station/src/core/laboratory/advancedResults.ts index 42ac6b6..7c9eb54 100644 --- a/apps/control-station/src/core/laboratory/advancedResults.ts +++ b/apps/control-station/src/core/laboratory/advancedResults.ts @@ -967,6 +967,7 @@ export async function fetchAdvancedLaboratoryResults({ const e39 = settledCatalogValue(settled[7]); const e40 = settledCatalogValue(settled[8]); return { + vegetationShadow: null, m47Graph: null, m48: null, m48SmallStatic: null, m48StaticOccupancy: null, m48r3StaticOccupancy: null, m48s: null, m48t: null, m49Tgs: null, m49TgsFull: null, m4Threat: null, diff --git a/apps/control-station/src/core/laboratory/vegetationShadow.ts b/apps/control-station/src/core/laboratory/vegetationShadow.ts new file mode 100644 index 0000000..23efd3c --- /dev/null +++ b/apps/control-station/src/core/laboratory/vegetationShadow.ts @@ -0,0 +1,278 @@ +import type { LaboratoryFetch } from "./advancedResults"; + +const RESULT_ID = /^lab-v1-vegetation-shadow-[a-f0-9]{64}$/; +const SHA256 = /^[a-f0-9]{64}$/; +const CANDIDATES = ["ddrnet", "ppliteseg"] as const; +const ROUTE_MODES = ["source", "ddrnet", "ppliteseg", "urban", "rural", "offroad"] as const; +const VALIDATION_MODES = ["source", "truth", "ddrnet", "ppliteseg"] as const; + +export type VegetationCandidateKey = typeof CANDIDATES[number]; +export type VegetationRouteMode = typeof ROUTE_MODES[number]; +export type VegetationValidationMode = typeof VALIDATION_MODES[number]; + +export interface VegetationCandidateMetrics { + candidate: VegetationCandidateKey; + loadedModelName: string; + checkpointSha256: string; + meanIouPercent: number; + publishedMeanIouPercent: number; + vegetationMeanIouPercent: number; + validationLatencyP95Ms: number; + validationThroughputFps: number; + shadowLatencyP95Ms: number; + shadowThroughputFps: number; + shadowPrewarmLatencyMs: number; + peakReservedVramBytes: number; + gpuName: string; +} + +export interface VegetationVisualCase { + caseId: string; + sourceKind: "goose" | "ravnoves"; + width: number; + height: number; + centerCropXyxy: readonly [number, number, number, number]; + outsideCropState: "undefined" | "not-applicable"; + assets: Readonly>; +} + +export interface VegetationShadowResult { + resultId: string; + createdAtUtc: string; + status: "visual-shadow-ready-policy-not-authorized"; + selectedCandidate: VegetationCandidateKey; + candidates: readonly VegetationCandidateMetrics[]; + routeCases: readonly VegetationVisualCase[]; + validationCases: readonly VegetationVisualCase[]; + limitations: readonly string[]; + visualShadowReady: true; + missionPolicyReadyForConfiguration: true; + authority: { + commandsEnabled: false; + navigationOrSafetyAccepted: false; + actuationAccepted: false; + cameraSemanticsCanClearRigidGeometry: false; + }; +} + +export class VegetationShadowContractError extends Error {} + +function objectValue(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new VegetationShadowContractError(`${label}: ожидался объект.`); + } + return value as Record; +} + +function arrayValue(value: unknown, label: string): readonly unknown[] { + if (!Array.isArray(value)) throw new VegetationShadowContractError(`${label}: ожидался массив.`); + return value; +} + +function textValue(value: unknown, label: string): string { + if (typeof value !== "string" || !value.trim()) { + throw new VegetationShadowContractError(`${label}: ожидалась строка.`); + } + return value; +} + +function numberValue(value: unknown, label: string): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + throw new VegetationShadowContractError(`${label}: ожидалось число.`); + } + return value; +} + +function integerValue(value: unknown, label: string): number { + const parsed = numberValue(value, label); + if (!Number.isInteger(parsed) || parsed < 0) { + throw new VegetationShadowContractError(`${label}: ожидалось неотрицательное целое.`); + } + return parsed; +} + +function exact(value: unknown, expected: unknown, label: string): void { + if (value !== expected) { + throw new VegetationShadowContractError(`${label}: контракт изменён.`); + } +} + +function candidateKey(value: unknown, label: string): VegetationCandidateKey { + if (value !== "ddrnet" && value !== "ppliteseg") { + throw new VegetationShadowContractError(`${label}: неизвестная модель.`); + } + return value; +} + +function candidateMetricsValue( + value: unknown, + candidate: VegetationCandidateKey, +): VegetationCandidateMetrics { + const row = objectValue(value, `vegetation.metrics.${candidate}`); + const validation = objectValue(row.validation_metrics, `${candidate}.validation_metrics`); + const validationTiming = objectValue(row.validation_timing, `${candidate}.validation_timing`); + const shadowTiming = objectValue(row.shadow_timing, `${candidate}.shadow_timing`); + const resource = objectValue(row.resource, `${candidate}.resource`); + const checkpointSha256 = textValue(row.checkpoint_sha256, `${candidate}.checkpoint_sha256`); + if (!SHA256.test(checkpointSha256)) { + throw new VegetationShadowContractError(`${candidate}.checkpoint_sha256: digest invalid.`); + } + return { + candidate, + loadedModelName: textValue(row.loaded_model_name, `${candidate}.loaded_model_name`), + checkpointSha256, + meanIouPercent: numberValue(validation.mean_iou_percent, `${candidate}.mean_iou_percent`), + publishedMeanIouPercent: numberValue( + validation.published_mean_iou_percent, + `${candidate}.published_mean_iou_percent`, + ), + vegetationMeanIouPercent: numberValue(validation.vegetation_mean_iou, `${candidate}.vegetation_mean_iou`) * 100, + validationLatencyP95Ms: numberValue(validationTiming.latency_ms_p95, `${candidate}.validation_latency_p95`), + validationThroughputFps: numberValue( + validationTiming.throughput_fps_from_mean_inference, + `${candidate}.validation_throughput`, + ), + shadowLatencyP95Ms: numberValue(shadowTiming.latency_ms_p95, `${candidate}.shadow_latency_p95`), + shadowThroughputFps: numberValue( + shadowTiming.throughput_fps_from_mean_inference, + `${candidate}.shadow_throughput`, + ), + shadowPrewarmLatencyMs: numberValue( + shadowTiming.prewarm_latency_ms, + `${candidate}.shadow_prewarm_latency`, + ), + peakReservedVramBytes: integerValue(resource.peak_reserved_vram_bytes, `${candidate}.vram`), + gpuName: textValue(resource.gpu_name, `${candidate}.gpu_name`), + }; +} + +function visualCaseValue( + value: unknown, + resultId: string, + expectedKind: "goose" | "ravnoves", +): VegetationVisualCase { + const row = objectValue(value, `vegetation.${expectedKind}.case`); + exact(row.source_kind, expectedKind, "vegetation.case.source_kind"); + const caseId = textValue(row.case_id, "vegetation.case.case_id"); + const crop = arrayValue(row.center_crop_xyxy, "vegetation.case.center_crop_xyxy") + .map((item, index) => integerValue(item, `vegetation.case.crop[${index}]`)); + if (crop.length !== 4) { + throw new VegetationShadowContractError("vegetation.case.center_crop_xyxy: размер изменён."); + } + const assets = objectValue(row.assets, "vegetation.case.assets"); + const projected: Record = {}; + for (const [key, raw] of Object.entries(assets)) { + const descriptor = objectValue(raw, `vegetation.case.assets.${key}`); + const path = textValue(descriptor.path, `vegetation.case.assets.${key}.path`); + const sha256 = textValue(descriptor.sha256, `vegetation.case.assets.${key}.sha256`); + if (!SHA256.test(sha256) || !path.startsWith(`visual/${expectedKind}/${caseId}/`)) { + throw new VegetationShadowContractError(`vegetation.case.assets.${key}: proof invalid.`); + } + projected[key] = `/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}/assets/${path + .split("/") + .map(encodeURIComponent) + .join("/")}`; + } + const expectedAssets = expectedKind === "goose" + ? VALIDATION_MODES + : ROUTE_MODES; + if (expectedAssets.some((key) => !projected[key])) { + throw new VegetationShadowContractError(`vegetation.case.assets: ${expectedKind} набор неполон.`); + } + const outsideCropState = row.outside_crop_state; + if (outsideCropState !== "undefined" && outsideCropState !== "not-applicable") { + throw new VegetationShadowContractError("vegetation.case.outside_crop_state: контракт изменён."); + } + return { + caseId, + sourceKind: expectedKind, + width: integerValue(row.width, "vegetation.case.width"), + height: integerValue(row.height, "vegetation.case.height"), + centerCropXyxy: crop as unknown as readonly [number, number, number, number], + outsideCropState, + assets: projected, + }; +} + +function parseResult(value: unknown, resultId: string): VegetationShadowResult { + const payload = objectValue(value, "Vegetation LAB"); + exact(payload.schema_version, "missioncore.lab-v1-vegetation-shadow/v1", "vegetation.schema"); + exact(payload.result_id, resultId, "vegetation.result_id"); + exact(payload.status, "visual-shadow-ready-policy-not-authorized", "vegetation.status"); + exact(payload.ground_truth, false, "vegetation.ground_truth"); + exact(payload.access, "read-only", "vegetation.access"); + const identity = objectValue(payload.identity, "vegetation.identity"); + const metrics = objectValue(payload.metrics, "vegetation.metrics"); + const candidates = objectValue(metrics.candidates, "vegetation.metrics.candidates"); + const decision = objectValue(payload.decision, "vegetation.decision"); + const authority = objectValue(payload.authority, "vegetation.authority"); + const catalogs = objectValue(payload.catalogs, "vegetation.catalogs"); + const selectedCandidate = candidateKey(identity.selected_candidate, "vegetation.selected_candidate"); + exact(decision.selected_candidate, selectedCandidate, "vegetation.decision.selected_candidate"); + exact(decision.visual_shadow_ready, true, "vegetation.decision.visual_shadow_ready"); + exact( + decision.mission_policy_ready_for_configuration, + true, + "vegetation.decision.mission_policy_ready_for_configuration", + ); + exact(decision.navigation_accepted, false, "vegetation.decision.navigation_accepted"); + exact(decision.production_accepted, false, "vegetation.decision.production_accepted"); + exact(authority.commands_enabled, false, "vegetation.authority.commands_enabled"); + exact( + authority.navigation_or_safety_accepted, + false, + "vegetation.authority.navigation_or_safety_accepted", + ); + exact(authority.actuation_accepted, false, "vegetation.authority.actuation_accepted"); + exact( + authority.camera_semantics_can_clear_rigid_geometry, + false, + "vegetation.authority.camera_semantics_can_clear_rigid_geometry", + ); + const routeCases = arrayValue(catalogs.ravnoves, "vegetation.catalogs.ravnoves") + .map((item) => visualCaseValue(item, resultId, "ravnoves")); + const validationCases = arrayValue(catalogs.goose, "vegetation.catalogs.goose") + .map((item) => visualCaseValue(item, resultId, "goose")); + if (routeCases.length !== 12 || validationCases.length !== 12) { + throw new VegetationShadowContractError("vegetation.catalogs: ожидалось 12 + 12 случаев."); + } + return { + resultId, + createdAtUtc: textValue(payload.created_at_utc, "vegetation.created_at_utc"), + status: "visual-shadow-ready-policy-not-authorized", + selectedCandidate, + candidates: CANDIDATES.map((candidate) => candidateMetricsValue(candidates[candidate], candidate)), + routeCases, + validationCases, + limitations: arrayValue(payload.limitations, "vegetation.limitations") + .map((item, index) => textValue(item, `vegetation.limitations[${index}]`)), + visualShadowReady: true, + missionPolicyReadyForConfiguration: true, + authority: { + commandsEnabled: false, + navigationOrSafetyAccepted: false, + actuationAccepted: false, + cameraSemanticsCanClearRigidGeometry: false, + }, + }; +} + +export async function fetchVegetationShadowResult( + resultId: string, + { + fetcher = fetch, + signal, + }: { fetcher?: LaboratoryFetch; signal?: AbortSignal } = {}, +): Promise { + if (!RESULT_ID.test(resultId)) { + throw new VegetationShadowContractError("Vegetation LAB identity недопустима."); + } + const response = await fetcher( + `/api/v1/laboratory/vegetation-shadow/${encodeURIComponent(resultId)}`, + { method: "GET", headers: { Accept: "application/json" }, signal }, + ); + if (!response.ok) { + throw new VegetationShadowContractError(`Vegetation LAB недоступна: HTTP ${response.status}.`); + } + return parseResult(await response.json(), resultId); +} diff --git a/apps/control-station/src/workspaces/laboratory/AdvancedLaboratoryResult.tsx b/apps/control-station/src/workspaces/laboratory/AdvancedLaboratoryResult.tsx index 7a284bf..58c1b6f 100644 --- a/apps/control-station/src/workspaces/laboratory/AdvancedLaboratoryResult.tsx +++ b/apps/control-station/src/workspaces/laboratory/AdvancedLaboratoryResult.tsx @@ -50,6 +50,7 @@ import { M48SFixedClassDetectorResultView } from "./M48SFixedClassDetectorResult import { M48TRiskQualityResultView } from "./M48TRiskQualityResult"; import { M49TgsFailClosedResultView } from "./M49TgsFailClosedResult"; import { M49TgsFullShadowResultView } from "./M49TgsFullShadowResult"; +import { VegetationShadowResultView } from "./VegetationShadowResult"; export { isAdvancedLaboratoryWorkId }; export type { AdvancedLaboratoryWorkId }; @@ -92,6 +93,9 @@ export function AdvancedLaboratoryResult({ failedSessionId: string | null; replayError: string | null; }) { + if (workId === "lab-v1-vegetation-shadow" && results.vegetationShadow) { + return ; + } if (workId === "m48-object-centric-quality" && results.m48) { return ; } diff --git a/apps/control-station/src/workspaces/laboratory/VegetationShadowResult.tsx b/apps/control-station/src/workspaces/laboratory/VegetationShadowResult.tsx new file mode 100644 index 0000000..63c64ce --- /dev/null +++ b/apps/control-station/src/workspaces/laboratory/VegetationShadowResult.tsx @@ -0,0 +1,135 @@ +import { + LaboratoryEvidence, + LaboratoryResultSummary, + LaboratorySummary, + LaboratoryWorkTemplate, +} from "../../components/laboratory/LaboratoryPresentation"; +import type { VegetationShadowResult } from "../../core/laboratory/vegetationShadow"; +import { + VegetationRouteVisual, + VegetationValidationVisual, +} from "./VegetationShadowVisual"; + +function decimal(value: number, digits = 1): string { + return value.toLocaleString("ru-RU", { maximumFractionDigits: digits }); +} + +export function VegetationShadowResultView({ + rigLabel, + result, +}: { + rigLabel: string; + result: VegetationShadowResult; +}) { + const selected = result.candidates.find( + (candidate) => candidate.candidate === result.selectedCandidate, + )!; + const alternative = result.candidates.find( + (candidate) => candidate.candidate !== result.selectedCandidate, + )!; + return ( + ({ + kind: "model" as const, + name: candidate.loadedModelName, + version: candidate.candidate, + role: candidate.candidate === result.selectedCandidate ? "selected policy provider" : "comparison candidate", + identitySha256: candidate.checkpointSha256, + })), + }} + /> + )} + evidence={( + <> + + + + + + + + )} + result={( + + )} + /> + ); +} diff --git a/apps/control-station/src/workspaces/laboratory/VegetationShadowVisual.tsx b/apps/control-station/src/workspaces/laboratory/VegetationShadowVisual.tsx new file mode 100644 index 0000000..e764bf9 --- /dev/null +++ b/apps/control-station/src/workspaces/laboratory/VegetationShadowVisual.tsx @@ -0,0 +1,157 @@ +import { useState } from "react"; +import { Icon, IconButton, StatusBadge } from "@nodedc/ui-react"; + +import { LaboratoryEvidenceViewer } from "../../components/laboratory/LaboratoryEvidenceViewer"; +import type { + VegetationRouteMode, + VegetationShadowResult, + VegetationValidationMode, + VegetationVisualCase, +} from "../../core/laboratory/vegetationShadow"; + +const ROUTE_MODES = [ + { value: "source", label: "SOURCE" }, + { value: "ddrnet", label: "DDRNET" }, + { value: "ppliteseg", label: "PPLITE" }, + { value: "urban", label: "URBAN" }, + { value: "rural", label: "RURAL" }, + { value: "offroad", label: "OFF-ROAD" }, +] as const; + +const VALIDATION_MODES = [ + { value: "source", label: "SOURCE" }, + { value: "truth", label: "TRUTH" }, + { value: "ddrnet", label: "DDRNET" }, + { value: "ppliteseg", label: "PPLITE" }, +] as const; + +function VegetationScene({ + item, + mode, +}: { + item: VegetationVisualCase; + mode: string; +}) { + const overlay = mode === "source" ? null : item.assets[mode]; + return ( +
+ + {overlay ? : null} +
+ ); +} + +export function VegetationRouteVisual({ result }: { result: VegetationShadowResult }) { + const [index, setIndex] = useState(0); + const [mode, setMode] = useState("offroad"); + const [expanded, setExpanded] = useState(false); + const item = result.routeCases[index] ?? null; + const selected = result.candidates.find( + (candidate) => candidate.candidate === result.selectedCandidate, + ); + return ( + + setIndex((current) => ( + current - 1 + result.routeCases.length + ) % result.routeCases.length)} + > + + + setIndex((current) => (current + 1) % result.routeCases.length)} + > + + + + )} + overlay={item ? ( +
+ SHADOW ONLY + RAVNOVES00 · {item.caseId} · {mode.toUpperCase()} + + {selected?.loadedModelName ?? result.selectedCandidate} policy provider + {" · "}center crop 600×600 + {" · "}outside crop UNKNOWN + +
+ ) : null} + > + {item ? ( + + ) : ( +
+ + RAVNOVES vegetation shadow каталог пуст. +
+ )} + + ); +} + +export function VegetationValidationVisual({ result }: { result: VegetationShadowResult }) { + const [index, setIndex] = useState(0); + const [mode, setMode] = useState("truth"); + const [expanded, setExpanded] = useState(false); + const item = result.validationCases[index] ?? null; + return ( + + setIndex((current) => ( + current - 1 + result.validationCases.length + ) % result.validationCases.length)} + > + + + setIndex((current) => (current + 1) % result.validationCases.length)} + > + + + + )} + overlay={item ? ( +
+ GOOSE VALIDATION + {item.caseId} · {mode.toUpperCase()} + official fine-64 labels · fixed 512×512 preprocessing · visual sample +
+ ) : null} + > + {item ? ( + + ) : ( +
+ + GOOSE validation каталог пуст. +
+ )} + + ); +} diff --git a/apps/control-station/src/workspaces/laboratory/laboratoryArchiveProfiles.ts b/apps/control-station/src/workspaces/laboratory/laboratoryArchiveProfiles.ts index 5993956..8af579f 100644 --- a/apps/control-station/src/workspaces/laboratory/laboratoryArchiveProfiles.ts +++ b/apps/control-station/src/workspaces/laboratory/laboratoryArchiveProfiles.ts @@ -63,6 +63,13 @@ interface KnownWorkDefinition { const rig = (rigLabel: string): string => rigLabel.trim() || "Сенсорный риг"; const KNOWN_WORKS: Readonly, KnownWorkDefinition>> = { + "lab-v1-vegetation-shadow": { + profileId: "rig-ravnoves-perception-gate-v1", + profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · RAVNOVES00 vegetation shadow`, + experimentId: "lab-v1-vegetation-mission-policy", + experimentName: "GOOSE ready weights → RAVNOVES vegetation policy", + variantName: "LAB V1 · DDRNet vs PPLiteSeg · urban/rural/off-road presets", + }, "m48-object-centric-quality": { profileId: "rig-dual-evidence-virtual-corridor-v1", profileName: (rigLabel) => `${rig(rigLabel)} RIGHT · Camera + prediction-free spatial evidence`, diff --git a/apps/control-station/src/workspaces/laboratory/useAdvancedLaboratoryCatalog.ts b/apps/control-station/src/workspaces/laboratory/useAdvancedLaboratoryCatalog.ts index 48983aa..4e84819 100644 --- a/apps/control-station/src/workspaces/laboratory/useAdvancedLaboratoryCatalog.ts +++ b/apps/control-station/src/workspaces/laboratory/useAdvancedLaboratoryCatalog.ts @@ -18,6 +18,7 @@ function mergeResults( next: AdvancedLaboratoryResults, ): AdvancedLaboratoryResults { return { + vegetationShadow: next.vegetationShadow ?? current.vegetationShadow, m47Graph: next.m47Graph ?? current.m47Graph, m48: next.m48 ?? current.m48, m48SmallStatic: next.m48SmallStatic ?? current.m48SmallStatic, @@ -121,6 +122,7 @@ export function useAdvancedLaboratoryCatalog({ const indexedResultId = index.find((item) => item.workId === selectedWorkId)?.resultId; if ( [ + "lab-v1-vegetation-shadow", "m47-reference-graph-shadow", "m48-object-centric-quality", "m48-small-static-passage-regression", diff --git a/apps/control-station/test/vegetationShadow.test.mjs b/apps/control-station/test/vegetationShadow.test.mjs new file mode 100644 index 0000000..345b661 --- /dev/null +++ b/apps/control-station/test/vegetationShadow.test.mjs @@ -0,0 +1,125 @@ +import assert from "node:assert/strict"; +import { after, before, test } from "node:test"; + +import { createServer } from "vite"; + +let server; +let fetchVegetationShadowResult; + +before(async () => { + server = await createServer({ + appType: "custom", + logLevel: "silent", + server: { middlewareMode: true }, + }); + ({ fetchVegetationShadowResult } = await server.ssrLoadModule( + "/src/core/laboratory/vegetationShadow.ts", + )); +}); + +after(async () => { + await server?.close(); +}); + +const resultId = `lab-v1-vegetation-shadow-${"a".repeat(64)}`; + +function candidate(candidateKey, vegetationIou) { + return { + loaded_model_name: candidateKey === "ddrnet" ? "ddrnet_39" : "pp_lite_t_seg", + checkpoint_sha256: (candidateKey === "ddrnet" ? "b" : "c").repeat(64), + validation_metrics: { + mean_iou_percent: 44.2, + published_mean_iou_percent: 46.53, + vegetation_mean_iou: vegetationIou, + }, + validation_timing: { + latency_ms_p95: 22.4, + throughput_fps_from_mean_inference: 48.1, + }, + shadow_timing: { + prewarm_latency_ms: 612.4, + latency_ms_p95: 21.8, + throughput_fps_from_mean_inference: 49.2, + }, + resource: { + peak_reserved_vram_bytes: 2_000_000_000, + gpu_name: "NVIDIA GeForce RTX 4090", + }, + }; +} + +function visualCase(sourceKind, index) { + const caseId = `case-${index}`; + const keys = sourceKind === "goose" + ? ["source", "truth", "ddrnet", "ppliteseg"] + : ["source", "ddrnet", "ppliteseg", "urban", "rural", "offroad"]; + return { + case_id: caseId, + source_kind: sourceKind, + width: sourceKind === "goose" ? 512 : 800, + height: sourceKind === "goose" ? 512 : 600, + center_crop_xyxy: sourceKind === "goose" ? [0, 0, 512, 512] : [100, 0, 700, 600], + outside_crop_state: sourceKind === "goose" ? "not-applicable" : "undefined", + assets: Object.fromEntries(keys.map((key) => [key, { + path: `visual/${sourceKind}/${caseId}/${key}.png`, + sha256: "d".repeat(64), + }])), + }; +} + +test("vegetation LAB keeps autonomous assets and fail-closed authority", async () => { + let requestedUrl = ""; + const result = await fetchVegetationShadowResult(resultId, { + fetcher: async (url) => { + requestedUrl = String(url); + return new Response(JSON.stringify({ + schema_version: "missioncore.lab-v1-vegetation-shadow/v1", + result_id: resultId, + created_at_utc: "2026-08-27T20:00:00Z", + status: "visual-shadow-ready-policy-not-authorized", + ground_truth: false, + identity: { selected_candidate: "ddrnet" }, + metrics: { + candidates: { + ddrnet: candidate("ddrnet", 0.64), + ppliteseg: candidate("ppliteseg", 0.61), + }, + }, + decision: { + selected_candidate: "ddrnet", + visual_shadow_ready: true, + mission_policy_ready_for_configuration: true, + navigation_accepted: false, + production_accepted: false, + }, + limitations: ["shadow only"], + authority: { + commands_enabled: false, + navigation_or_safety_accepted: false, + actuation_accepted: false, + camera_semantics_can_clear_rigid_geometry: false, + }, + catalogs: { + goose: Array.from({ length: 12 }, (_, index) => visualCase("goose", index)), + ravnoves: Array.from({ length: 12 }, (_, index) => visualCase("ravnoves", index)), + }, + access: "read-only", + }), { status: 200, headers: { "Content-Type": "application/json" } }); + }, + }); + assert.equal( + requestedUrl, + `/api/v1/laboratory/vegetation-shadow/${resultId}`, + ); + assert.equal(result.selectedCandidate, "ddrnet"); + assert.equal(result.candidates[0].vegetationMeanIouPercent, 64); + assert.equal(result.routeCases.length, 12); + assert.equal(result.validationCases.length, 12); + assert.match(result.routeCases[0].assets.offroad, /\/assets\/visual\/ravnoves\//); + assert.deepEqual(result.authority, { + commandsEnabled: false, + navigationOrSafetyAccepted: false, + actuationAccepted: false, + cameraSemanticsCanClearRigidGeometry: false, + }); +}); diff --git a/config/laboratories/lab-v1-vegetation-shadow.json b/config/laboratories/lab-v1-vegetation-shadow.json new file mode 100644 index 0000000..7ea9d32 --- /dev/null +++ b/config/laboratories/lab-v1-vegetation-shadow.json @@ -0,0 +1,10 @@ +{ + "schema_version": "missioncore.laboratory-evidence-definition/v1", + "work_id": "lab-v1-vegetation-shadow", + "evidence": { + "runtime_relative_root": "lab-v1-vegetation/results", + "result_id_prefix": "lab-v1-vegetation-shadow", + "document_name": "result.json", + "schema_version": "missioncore.lab-v1-vegetation-shadow/v1" + } +} diff --git a/config/laboratory-execution.json b/config/laboratory-execution.json index 0b3d4ba..da66dd9 100644 --- a/config/laboratory-execution.json +++ b/config/laboratory-execution.json @@ -190,6 +190,25 @@ "run": "missioncore.laboratory-run/v1", "evidence": "missioncore.m49-tgs-full-shadow-lab/v1" } + }, + { + "work_id": "lab-v1-vegetation-shadow", + "lifecycle": "experimental", + "isolation": "bounded-adapter", + "adapter_id": "experimental.lab-v1-vegetation-shadow/v1", + "input_roles": [ + "ddrnet_goose_root", + "ppliteseg_goose_root", + "ddrnet_ravnoves_root", + "ppliteseg_ravnoves_root" + ], + "contracts": { + "source": "missioncore.goose-ravnoves-vegetation-source-set/v1", + "provider": "missioncore.goose-fine64-ready-weight-provider/v1", + "graph": "missioncore.vegetation-mission-policy-shadow-graph/v1", + "run": "missioncore.laboratory-run/v1", + "evidence": "missioncore.lab-v1-vegetation-shadow/v1" + } } ], "legacy_work_ids": [ diff --git a/config/laboratory-value-review.json b/config/laboratory-value-review.json index dbbd083..9e04e6f 100644 --- a/config/laboratory-value-review.json +++ b/config/laboratory-value-review.json @@ -1,6 +1,6 @@ { "schema_version": "missioncore.laboratory-value-review-registry/v1", - "reviewed_at_utc": "2026-08-26T08:34:34Z", + "reviewed_at_utc": "2026-08-27T20:20:14Z", "entries": [ { "catalog_id": "e28-local-surface", @@ -281,6 +281,13 @@ "signal": "progress", "lifecycle": "current", "visual_evidence": "available" + }, + { + "catalog_id": "lab-v1-vegetation-shadow", + "evidence_id": "lab-v1-vegetation-shadow-ad4d9fbbb21ff8a270b77f559b4e78dcdaf0455afd61afb5033009623984e554", + "signal": "progress", + "lifecycle": "current", + "visual_evidence": "available" } ] } diff --git a/config/perception/lab-v1-goose-vegetation-benchmark-v1.json b/config/perception/lab-v1-goose-vegetation-benchmark-v1.json new file mode 100644 index 0000000..95cf895 --- /dev/null +++ b/config/perception/lab-v1-goose-vegetation-benchmark-v1.json @@ -0,0 +1,97 @@ +{ + "schema_version": "missioncore.lab-v1-goose-vegetation-benchmark/v1", + "lab_id": "LAB-V1", + "worker_id": "worker-006", + "runtime": { + "super_gradients_version": "3.2.0", + "super_gradients_revision": "54d062ecb1081944a672ce447cf3e96a36708ff9", + "python_version": "3.9", + "numpy_version": "1.23.0", + "cmake_version": "3.31.6", + "onnxsim_version": "0.4.36", + "opencv_python_version": "4.8.1.78", + "pytorch_version": "1.13.1", + "torchvision_version": "0.14.1", + "pytorch_cuda_version": "11.7", + "container_base": "nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85", + "miniconda_installer": "Miniconda3-py39_24.11.1-0-Linux-x86_64.sh", + "miniconda_installer_sha256": "3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5" + }, + "dataset": { + "dataset_id": "goose-2d-validation-visible-rgb", + "relative_root": "goose-2d/validation", + "mapping_relative_path": "goose_label_mapping.csv", + "mapping_sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f", + "image_glob": "images/val/**/*_windshield_vis.png", + "expected_pair_count": 962, + "input_size": [512, 512], + "preprocessing": [ + "center-square-crop", + "nearest-neighbor-resize", + "rgb-to-tensor-0-1" + ] + }, + "candidates": { + "ddrnet": { + "candidate_id": "goose-ddrnet-class-512", + "model_names": ["ddrnet_39"], + "checkpoint_relative_path": "models/goose/ddrnet_class_512.pth", + "checkpoint_size_bytes": 259419077, + "checkpoint_sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", + "published_validation_miou_percent": 46.53 + }, + "ppliteseg": { + "candidate_id": "goose-ppliteseg-class-512", + "model_names": [ + "pp_lite_t_seg", + "pp_lite_t_seg50", + "pp_lite_t_seg75", + "pp_lite_b_seg", + "pp_lite_b_seg50", + "pp_lite_b_seg75" + ], + "checkpoint_relative_path": "models/goose/ppliteseg_class_512.pth", + "checkpoint_size_bytes": 98208249, + "checkpoint_sha256": "6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd", + "published_validation_miou_percent": 45.09 + } + }, + "ravnoves": { + "source_id": "RAVNOVES00/right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8", + "source_sha256": "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8", + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": 4489, + "frame_indices": [0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488], + "crop_contract": "center-600-square-to-512; outside-crop-is-undefined" + }, + "vegetation_class_names": [ + "leaves", + "forest", + "bush", + "moss", + "tree_crown", + "tree_trunk", + "crops", + "low_grass", + "high_grass", + "scenery_vegetation", + "hedge", + "tree_root" + ], + "policy_action_colors": { + "ALLOW": "#22c55e", + "HIGH_COST": "#f59e0b", + "NO_GO": "#ef4444" + }, + "invariants": { + "one_heavy_candidate_at_a_time": true, + "raw_fisheye_is_immutable": true, + "outside_center_crop_is_free": false, + "missing_or_unknown_is_free": false, + "camera_semantics_can_clear_rigid_geometry": false, + "navigation_authority": false, + "actuation_authority": false, + "canonical_triton_mutation_allowed": false + } +} diff --git a/experiments/perception/worker/Invoke-LabV1VegetationGooseBenchmark.ps1 b/experiments/perception/worker/Invoke-LabV1VegetationGooseBenchmark.ps1 new file mode 100644 index 0000000..321b575 --- /dev/null +++ b/experiments/perception/worker/Invoke-LabV1VegetationGooseBenchmark.ps1 @@ -0,0 +1,230 @@ +[CmdletBinding()] +param( + [ValidateSet("Build", "Probe", "Validate", "Ravnoves", "Status")] + [string]$Mode = "Status", + + [ValidateSet("Ddrnet", "Ppliteseg")] + [string]$Candidate = "Ddrnet", + + [string]$AssetRoot = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27", + + [string]$ToolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\lab-v1-vegetation-goose", + + [string]$OutputRoot = "D:\NDC_MISSIONCORE\runtime\experiments\lab-v1-vegetation", + + [string]$RavnovesVideo = "D:\NDC_MISSIONCORE\runtime\experiments\e46e\inputs\right-cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8.mp4" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$datasetPrefix = "D:\NDC_MISSIONCORE\datasets\" +$runtimePrefix = "D:\NDC_MISSIONCORE\runtime\experiments\" +if (-not $AssetRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "AssetRoot must stay under $datasetPrefix" +} +if (-not $ToolRoot.StartsWith($datasetPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "ToolRoot must stay under $datasetPrefix" +} +if (-not $OutputRoot.StartsWith($runtimePrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "OutputRoot must stay under $runtimePrefix" +} + +$image = "ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1" +$canonicalContainer = "ndc-mission-core-triton" +$candidateKey = $Candidate.ToLowerInvariant() +$contextRoot = Join-Path $ToolRoot "context" +$configRoot = Join-Path $ToolRoot "config" +$benchmarkConfig = Join-Path $configRoot "lab-v1-goose-vegetation-benchmark-v1.json" +$policyConfig = Join-Path $configRoot "lab-v1-vegetation-mission-policy-v1.json" +$providerMapConfig = Join-Path $configRoot "lab-v1-vegetation-provider-label-map-v1.json" +$datasetRoot = Join-Path $AssetRoot "goose-2d\validation" +$checkpointRelative = if ($candidateKey -eq "ddrnet") { + "models\goose\ddrnet_class_512.pth" +} else { + "models\goose\ppliteseg_class_512.pth" +} +$checkpoint = Join-Path $AssetRoot $checkpointRelative +$expectedCheckpointSha256 = if ($candidateKey -eq "ddrnet") { + "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" +} else { + "6dd412c0c99115e359896c4cab43a8e6bce9e09b843e7fa885fe597b0a6121cd" +} +$expectedCheckpointBytes = if ($candidateKey -eq "ddrnet") { 259419077 } else { 98208249 } +$ravnovesSha256 = "cadd1696ff000904eb78633a0a8418104b8024f178b91f3421789021ccb160e8" +$frameIndices = @(0, 253, 512, 768, 1024, 1536, 2048, 2560, 3072, 3584, 4096, 4488) +$dockerConfig = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation\docker-config" + +function Get-CanonicalTritonIdentity { + $identity = & docker inspect $canonicalContainer --format "{{.Id}}|{{.Config.Image}}|{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{end}}" + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($identity)) { + throw "Canonical Triton is unavailable" + } + $parts = $identity.Split("|") + if ($parts.Count -ne 4 -or $parts[2] -ne "running" -or $parts[3] -ne "healthy") { + throw "Canonical Triton is not running and healthy: $identity" + } + return $identity +} + +function Assert-FileIdentity { + param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256) + $file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue + if ($null -eq $file -or $file.Length -ne $ExpectedBytes) { + throw "File identity changed: $Path" + } + $actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + if ($actualSha256 -ne $ExpectedSha256) { + throw "File digest changed: $Path" + } +} + +function Assert-RunnerInputs { + foreach ($path in @($benchmarkConfig, $policyConfig, $providerMapConfig)) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Runner config is unavailable: $path" + } + } + if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "Dockerfile") -PathType Leaf)) { + throw "Runner Dockerfile is unavailable" + } + if (-not (Test-Path -LiteralPath (Join-Path $contextRoot "run_goose_vegetation_benchmark.py") -PathType Leaf)) { + throw "Runner source is unavailable" + } + Assert-FileIdentity -Path $checkpoint -ExpectedBytes $expectedCheckpointBytes -ExpectedSha256 $expectedCheckpointSha256 +} + +function New-RunRoot { + param([string]$Kind) + $stamp = [DateTime]::UtcNow.ToString("yyyyMMddTHHmmssfffZ") + $path = Join-Path $OutputRoot ("{0}-{1}-{2}" -f $Kind, $candidateKey, $stamp) + New-Item -ItemType Directory -Path $path | Out-Null + return $path +} + +function Invoke-IsolatedRun { + param( + [ValidateSet("goose", "ravnoves")][string]$RunMode, + [string]$RunRoot, + [int]$Limit, + [string]$FramesRoot = "" + ) + $containerName = "ndc-lab-v1-goose-$candidateKey-$([Guid]::NewGuid().ToString('N').Substring(0, 10))" + $arguments = @( + "run", "--rm", "--name", $containerName, + "--gpus", "all", + "--network", "none", + "--read-only", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", + "--memory", "10g", + "--cpus", "8", + "--pids-limit", "512", + "--tmpfs", "/tmp:rw,noexec,nosuid,size=2g", + "--env", "HOME=/tmp", + "--mount", "type=bind,src=$datasetRoot,dst=/data/goose,readonly", + "--mount", "type=bind,src=$checkpoint,dst=/models/candidate.pth,readonly", + "--mount", "type=bind,src=$configRoot,dst=/config,readonly", + "--mount", "type=bind,src=$RunRoot,dst=/output", + $image, + "--mode", $RunMode, + "--candidate", $candidateKey, + "--config", "/config/lab-v1-goose-vegetation-benchmark-v1.json", + "--policy", "/config/lab-v1-vegetation-mission-policy-v1.json", + "--provider-map", "/config/lab-v1-vegetation-provider-label-map-v1.json", + "--checkpoint", "/models/candidate.pth", + "--dataset-root", "/data/goose", + "--output", "/output/result", + "--limit", $Limit.ToString(), + "--visual-count", "12" + ) + if ($RunMode -eq "ravnoves") { + $arguments = @($arguments[0..($arguments.Count - 1)]) + $arguments += @("--frames-root", "/input") + $mountIndex = [Array]::IndexOf($arguments, $image) + $head = @($arguments[0..($mountIndex - 1)]) + $tail = @($arguments[$mountIndex..($arguments.Count - 1)]) + $arguments = $head + @("--mount", "type=bind,src=$FramesRoot,dst=/input,readonly") + $tail + } + & docker @arguments + if ($LASTEXITCODE -ne 0) { + throw "LAB V1 container failed with exit code $LASTEXITCODE" + } +} + +function Export-RavnovesFrames { + param([string]$Destination) + Assert-FileIdentity -Path $RavnovesVideo -ExpectedBytes (Get-Item -LiteralPath $RavnovesVideo).Length -ExpectedSha256 $ravnovesSha256 + New-Item -ItemType Directory -Path $Destination | Out-Null + $expression = ($frameIndices | ForEach-Object { "eq(n\,$_ )" }) -join "+" + $temporaryPattern = Join-Path $Destination "selected-%03d.png" + & ffmpeg -hide_banner -loglevel error -i $RavnovesVideo -vf "select='$expression'" -fps_mode vfr $temporaryPattern + if ($LASTEXITCODE -ne 0) { + throw "RAVNOVES exact frame extraction failed" + } + $selected = @(Get-ChildItem -LiteralPath $Destination -Filter "selected-*.png" | Sort-Object Name) + if ($selected.Count -ne $frameIndices.Count) { + throw "RAVNOVES frame island changed: expected $($frameIndices.Count), got $($selected.Count)" + } + for ($index = 0; $index -lt $selected.Count; $index++) { + $target = Join-Path $Destination ("frame-{0:D6}.png" -f $frameIndices[$index]) + Move-Item -LiteralPath $selected[$index].FullName -Destination $target + } +} + +if ($Mode -eq "Status") { + $imageIdentity = & docker image inspect $image --format "{{.Id}}" 2>$null + [ordered]@{ + schema_version = "missioncore.lab-v1-goose-runner-status/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + worker_id = "worker-006" + image = $image + image_id = if ($LASTEXITCODE -eq 0) { $imageIdentity } else { $null } + canonical_triton = Get-CanonicalTritonIdentity + asset_root = $AssetRoot + output_root = $OutputRoot + candidate = $candidateKey + } | ConvertTo-Json -Depth 6 + exit 0 +} + +Assert-RunnerInputs +$canonicalBefore = Get-CanonicalTritonIdentity +try { + if ($Mode -eq "Build") { + if (-not (Test-Path -LiteralPath (Join-Path $dockerConfig "config.json") -PathType Leaf)) { + throw "Isolated Docker client configuration is unavailable" + } + $previousDockerConfig = $env:DOCKER_CONFIG + try { + $env:DOCKER_CONFIG = $dockerConfig + & docker build --pull=false --label "com.nodedc.component=mission-core-lab-v1-goose" --label "com.nodedc.authority=shadow-only" --tag $image $contextRoot + if ($LASTEXITCODE -ne 0) { + throw "LAB V1 image build failed with exit code $LASTEXITCODE" + } + } + finally { + $env:DOCKER_CONFIG = $previousDockerConfig + } + } + elseif ($Mode -eq "Probe") { + $runRoot = New-RunRoot -Kind "probe" + Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 8 + } + elseif ($Mode -eq "Validate") { + $runRoot = New-RunRoot -Kind "validation" + Invoke-IsolatedRun -RunMode "goose" -RunRoot $runRoot -Limit 0 + } + elseif ($Mode -eq "Ravnoves") { + $runRoot = New-RunRoot -Kind "ravnoves" + $framesRoot = Join-Path $runRoot "input-frames" + Export-RavnovesFrames -Destination $framesRoot + Invoke-IsolatedRun -RunMode "ravnoves" -RunRoot $runRoot -Limit 0 -FramesRoot $framesRoot + } +} +finally { + $canonicalAfter = Get-CanonicalTritonIdentity + if ($canonicalAfter -ne $canonicalBefore) { + throw "Canonical Triton identity changed during LAB V1 work" + } +} diff --git a/experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile b/experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile new file mode 100644 index 0000000..31d3f90 --- /dev/null +++ b/experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile @@ -0,0 +1,49 @@ +FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04@sha256:ad6d59a3bbf3e82c1c849c9ac09cfc2a3e0bbb8655042fd899be6681b3fe2a85 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/opt/conda/bin:$PATH + +ARG MINICONDA_INSTALLER=Miniconda3-py39_24.11.1-0-Linux-x86_64.sh +ARG MINICONDA_SHA256=3ea8373098d72140e08aac9217822b047ec094eb457e7f73945af7c6f68bf6f5 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + libglib2.0-0 \ + libgl1 \ + && curl --fail --location --retry 5 \ + --output /tmp/miniconda.sh \ + "https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}" \ + && echo "${MINICONDA_SHA256} /tmp/miniconda.sh" | sha256sum --check --strict \ + && bash /tmp/miniconda.sh -b -p /opt/conda \ + && rm -f /tmp/miniconda.sh \ + && rm -rf /var/lib/apt/lists/* + +RUN conda create --yes --name goose python=3.9 pip \ + && conda install --yes --name goose --channel pytorch --channel nvidia \ + pytorch=1.13.1 torchvision=0.14.1 pytorch-cuda=11.7 + +RUN conda run --name goose python -m pip install --no-cache-dir \ + cmake==3.31.6 \ + numpy==1.23.0 \ + onnxsim==0.4.36 \ + opencv-python==4.8.1.78 \ + protobuf==3.20.3 \ + pyparsing==2.4.5 + +RUN conda run --name goose python -m pip install --no-cache-dir \ + super-gradients==3.2.0 \ + torchmetrics==0.8.0 + +RUN conda clean --all --yes + +WORKDIR /opt/mission-core/lab-v1 +COPY run_goose_vegetation_benchmark.py /opt/mission-core/lab-v1/runner.py + +ENTRYPOINT ["conda", "run", "--no-capture-output", "--name", "goose", "python", "/opt/mission-core/lab-v1/runner.py"] diff --git a/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py b/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py new file mode 100644 index 0000000..2e84d59 --- /dev/null +++ b/experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py @@ -0,0 +1,556 @@ +"""Run isolated GOOSE vegetation qualification and RAVNOVES shadow inference.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import platform +import statistics +import time +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from PIL import Image +from super_gradients.training import models + +SCHEMA = "missioncore.lab-v1-goose-vegetation-run/v1" +VISUAL_SCHEMA = "missioncore.lab-v1-goose-vegetation-visual-case/v1" +CLASS_COUNT = 64 +MAX_CONFIG_BYTES = 1024 * 1024 +MODEL_NAMES = { + "ddrnet": ("ddrnet_39",), + "ppliteseg": ( + "pp_lite_t_seg", + "pp_lite_t_seg50", + "pp_lite_t_seg75", + "pp_lite_b_seg", + "pp_lite_b_seg50", + "pp_lite_b_seg75", + ), +} +RESAMPLE_NEAREST = getattr(Image, "Resampling", Image).NEAREST + + +class RunnerError(RuntimeError): + """The bounded runner input or output contract is invalid.""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--mode", choices=("goose", "ravnoves"), required=True) + parser.add_argument("--candidate", choices=tuple(MODEL_NAMES), required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--policy", type=Path, required=True) + parser.add_argument("--provider-map", type=Path, required=True) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--dataset-root", type=Path) + parser.add_argument("--frames-root", type=Path) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--limit", type=int, default=0) + parser.add_argument("--visual-count", type=int, default=12) + return parser.parse_args() + + +def read_json(path: Path, label: str) -> dict[str, Any]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_CONFIG_BYTES: + raise RunnerError(f"{label} is unavailable") + value = json.loads(path.read_text(encoding="utf-8-sig")) + if not isinstance(value, dict): + raise RunnerError(f"{label} must be an object") + return value + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def stable_digest(value: object) -> str: + encoded = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def validate_contracts( + config: dict[str, Any], policy: dict[str, Any], provider_map: dict[str, Any], candidate: str +) -> dict[str, Any]: + if config.get("schema_version") != "missioncore.lab-v1-goose-vegetation-benchmark/v1": + raise RunnerError("benchmark configuration identity changed") + if policy.get("schema_version") != "missioncore.vegetation-mission-policy/v1": + raise RunnerError("mission policy identity changed") + if provider_map.get("schema_version") != "missioncore.vegetation-provider-label-map/v1": + raise RunnerError("provider map identity changed") + invariants = config.get("invariants") + true_invariants = {"one_heavy_candidate_at_a_time", "raw_fisheye_is_immutable"} + if not isinstance(invariants, dict) or any( + value is not False for key, value in invariants.items() if key not in true_invariants + ): + raise RunnerError("benchmark fail-closed invariants changed") + if ( + invariants.get("one_heavy_candidate_at_a_time") is not True + or invariants.get("raw_fisheye_is_immutable") is not True + ): + raise RunnerError("benchmark isolation invariants changed") + candidates = config.get("candidates") + if not isinstance(candidates, dict) or not isinstance(candidates.get(candidate), dict): + raise RunnerError("candidate is not configured") + expected_models = candidates[candidate].get("model_names") + if expected_models != list(MODEL_NAMES[candidate]): + raise RunnerError("candidate architecture probe order changed") + return candidates[candidate] + + +def load_mapping(path: Path, expected_sha256: str) -> tuple[dict[int, str], np.ndarray]: + if sha256(path) != expected_sha256: + raise RunnerError("GOOSE label mapping digest changed") + names: dict[int, str] = {} + palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8) + with path.open(newline="", encoding="utf-8-sig") as stream: + for row in csv.DictReader(stream): + label_id = int(row["label_key"]) + if label_id < 0 or label_id >= CLASS_COUNT: + raise RunnerError("GOOSE label id is outside the 64-class contract") + color = row["hex"].lstrip("#") + names[label_id] = row["class_name"] + palette[label_id] = (*bytes.fromhex(color), 190) + if set(names) != set(range(CLASS_COUNT)): + raise RunnerError("GOOSE mapping does not cover exactly 64 classes") + palette[0, 3] = 0 + return names, palette + + +def center_crop(image: Image.Image) -> tuple[Image.Image, tuple[int, int, int, int]]: + side = min(image.width, image.height) + left = (image.width - side) // 2 + top = (image.height - side) // 2 + box = (left, top, left + side, top + side) + return image.crop(box), box + + +def preprocess(image: Image.Image) -> tuple[torch.Tensor, tuple[int, int, int, int]]: + cropped, crop_box = center_crop(image.convert("RGB")) + resized = cropped.resize((512, 512), resample=RESAMPLE_NEAREST) + array = np.asarray(resized, dtype=np.float32) / 255.0 + tensor = torch.from_numpy(np.transpose(array, (2, 0, 1))).unsqueeze(0) + return tensor, crop_box + + +def preprocess_label(image: Image.Image) -> np.ndarray: + cropped, _ = center_crop(image.convert("L")) + return np.asarray(cropped.resize((512, 512), resample=RESAMPLE_NEAREST), dtype=np.uint8) + + +def find_goose_pairs(root: Path) -> list[tuple[Path, Path]]: + pairs: list[tuple[Path, Path]] = [] + image_root = root / "images" / "val" + label_root = root / "labels" / "val" + for image_path in sorted(image_root.rglob("*_windshield_vis.png")): + stem = image_path.name.removesuffix("_windshield_vis.png") + relative_parent = image_path.parent.relative_to(image_root) + label_path = label_root / relative_parent / f"{stem}_labelids.png" + if label_path.is_file() and not label_path.is_symlink(): + pairs.append((image_path, label_path)) + return pairs + + +def visual_indices(count: int, visual_count: int) -> set[int]: + if count <= 0 or visual_count <= 0: + return set() + selected_count = min(count, visual_count) + if selected_count == 1: + return {0} + return { + round(index * (count - 1) / (selected_count - 1)) + for index in range(selected_count) + } + + +def load_model(candidate: str, checkpoint: Path) -> tuple[torch.nn.Module, str, list[str]]: + failures: list[str] = [] + for model_name in MODEL_NAMES[candidate]: + try: + model = models.get( + model_name=model_name, + num_classes=CLASS_COUNT, + checkpoint_path=str(checkpoint), + ) + model.eval() + model.cuda() + return model, model_name, failures + except Exception as error: # noqa: BLE001 - each upstream architecture is a probe + failures.append(f"{model_name}: {type(error).__name__}: {str(error)[:240]}") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + raise RunnerError("checkpoint did not load: " + " | ".join(failures)) + + +def logits_from_output(value: object) -> torch.Tensor: + if isinstance(value, torch.Tensor) and value.ndim == 4 and value.shape[1] == CLASS_COUNT: + return value + if isinstance(value, (list, tuple)): + for item in value: + try: + return logits_from_output(item) + except RunnerError: + continue + raise RunnerError("model output does not contain a 64-class raster") + + +def infer(model: torch.nn.Module, tensor: torch.Tensor) -> tuple[np.ndarray, float]: + tensor = tensor.cuda(non_blocking=True) + torch.cuda.synchronize() + started = time.perf_counter_ns() + with torch.inference_mode(): + logits = logits_from_output(model(tensor)) + prediction = torch.argmax(torch.sigmoid(logits), dim=1) + torch.cuda.synchronize() + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000.0 + return prediction[0].to(device="cpu", dtype=torch.uint8).numpy(), elapsed_ms + + +def update_confusion(confusion: np.ndarray, truth: np.ndarray, prediction: np.ndarray) -> None: + valid = (truth >= 0) & (truth < CLASS_COUNT) + indices = CLASS_COUNT * truth[valid].astype(np.int64) + prediction[valid].astype(np.int64) + confusion += np.bincount(indices, minlength=CLASS_COUNT**2).reshape(CLASS_COUNT, CLASS_COUNT) + + +def class_metrics(confusion: np.ndarray, names: dict[int, str]) -> list[dict[str, Any]]: + truth = confusion.sum(axis=1) + predicted = confusion.sum(axis=0) + intersection = np.diag(confusion) + union = truth + predicted - intersection + rows: list[dict[str, Any]] = [] + for label_id in range(CLASS_COUNT): + rows.append( + { + "label_id": label_id, + "class_name": names[label_id], + "support_pixels": int(truth[label_id]), + "predicted_pixels": int(predicted[label_id]), + "intersection_pixels": int(intersection[label_id]), + "union_pixels": int(union[label_id]), + "iou": round(float(intersection[label_id] / union[label_id]), 8) + if union[label_id] + else None, + } + ) + return rows + + +def hex_rgb(value: str) -> tuple[int, int, int]: + raw = bytes.fromhex(value.removeprefix("#")) + if len(raw) != 3: + raise RunnerError("policy action color must be RGB") + return raw[0], raw[1], raw[2] + + +def policy_palette( + names: dict[int, str], policy: dict[str, Any], provider_map: dict[str, Any], preset: str, + action_colors: dict[str, str] +) -> np.ndarray: + palette = np.zeros((CLASS_COUNT, 4), dtype=np.uint8) + labels = provider_map["providers"]["goose-fine-64"]["labels"] + rules = policy["presets"][preset] + for label_id, class_name in names.items(): + material = labels.get(class_name) + if material is None: + continue + action = rules[material] + palette[label_id] = (*hex_rgb(action_colors[action]), 190) + return palette + + +def save_image(path: Path, value: Image.Image | np.ndarray, mode: str | None = None) -> str: + path.parent.mkdir(parents=True, exist_ok=True) + image = value if isinstance(value, Image.Image) else Image.fromarray(value, mode=mode) + image.save(path, format="PNG", optimize=True) + return sha256(path) + + +def expand_mask( + mask: np.ndarray, + original_size: tuple[int, int], + crop_box: tuple[int, int, int, int], +) -> np.ndarray: + left, top, right, bottom = crop_box + side = right - left + resized = Image.fromarray(mask, mode="L").resize((side, side), resample=RESAMPLE_NEAREST) + canvas = np.zeros((original_size[1], original_size[0]), dtype=np.uint8) + canvas[top:bottom, left:right] = np.asarray(resized, dtype=np.uint8) + return canvas + + +def write_visual_case( + output: Path, + case_id: str, + source: Image.Image, + prediction: np.ndarray, + semantic_palette: np.ndarray, + policy_palettes: dict[str, np.ndarray], + crop_box: tuple[int, int, int, int], + truth: np.ndarray | None = None, + preserve_source_size: bool = False, +) -> dict[str, Any]: + case_root = output / "cases" / case_id + if preserve_source_size: + source_image = source.convert("RGB") + prediction_image = expand_mask(prediction, source_image.size, crop_box) + truth_image = expand_mask(truth, source_image.size, crop_box) if truth is not None else None + else: + cropped, _ = center_crop(source.convert("RGB")) + source_image = cropped.resize((512, 512), resample=RESAMPLE_NEAREST) + prediction_image = prediction + truth_image = truth + + files: dict[str, dict[str, str]] = {} + source_path = case_root / "source.png" + files["source"] = { + "relative_path": source_path.relative_to(output).as_posix(), + "sha256": save_image(source_path, source_image), + } + prediction_path = case_root / "prediction-labelids.png" + files["prediction_labelids"] = { + "relative_path": prediction_path.relative_to(output).as_posix(), + "sha256": save_image(prediction_path, prediction_image, "L"), + } + semantic_path = case_root / "prediction-semantic.png" + files["prediction_semantic"] = { + "relative_path": semantic_path.relative_to(output).as_posix(), + "sha256": save_image(semantic_path, semantic_palette[prediction_image], "RGBA"), + } + for preset, palette in policy_palettes.items(): + policy_path = case_root / f"policy-{preset}.png" + files[f"policy_{preset}"] = { + "relative_path": policy_path.relative_to(output).as_posix(), + "sha256": save_image(policy_path, palette[prediction_image], "RGBA"), + } + if truth_image is not None: + truth_path = case_root / "truth-labelids.png" + files["truth_labelids"] = { + "relative_path": truth_path.relative_to(output).as_posix(), + "sha256": save_image(truth_path, truth_image, "L"), + } + truth_semantic_path = case_root / "truth-semantic.png" + files["truth_semantic"] = { + "relative_path": truth_semantic_path.relative_to(output).as_posix(), + "sha256": save_image(truth_semantic_path, semantic_palette[truth_image], "RGBA"), + } + return { + "schema_version": VISUAL_SCHEMA, + "case_id": case_id, + "source_width": source_image.width, + "source_height": source_image.height, + "center_crop_xyxy": list(crop_box), + "outside_crop_state": "undefined" if preserve_source_size else "not-applicable", + "files": files, + } + + +def percentile(values: list[float], fraction: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = (len(ordered) - 1) * fraction + lower = math.floor(index) + upper = math.ceil(index) + if lower == upper: + return ordered[lower] + return ordered[lower] * (upper - index) + ordered[upper] * (index - lower) + + +def run() -> None: + args = parse_args() + if not torch.cuda.is_available(): + raise RunnerError("CUDA is required for Worker 006 qualification") + if args.limit < 0 or args.visual_count < 0: + raise RunnerError("limit and visual-count must be non-negative") + config = read_json(args.config, "benchmark config") + policy = read_json(args.policy, "mission policy") + provider_map = read_json(args.provider_map, "provider map") + candidate_config = validate_contracts(config, policy, provider_map, args.candidate) + if args.checkpoint.is_symlink() or not args.checkpoint.is_file(): + raise RunnerError("checkpoint is unavailable") + if args.checkpoint.stat().st_size != candidate_config["checkpoint_size_bytes"]: + raise RunnerError("checkpoint size changed") + checkpoint_sha256 = sha256(args.checkpoint) + if checkpoint_sha256 != candidate_config["checkpoint_sha256"]: + raise RunnerError("checkpoint digest changed") + + dataset_config = config["dataset"] + mapping_root = args.dataset_root + if mapping_root is None: + raise RunnerError("dataset-root is required for the immutable mapping") + mapping_path = mapping_root / dataset_config["mapping_relative_path"] + names, semantic_palette = load_mapping(mapping_path, dataset_config["mapping_sha256"]) + policy_palettes = { + preset: policy_palette( + names, + policy, + provider_map, + preset, + config["policy_action_colors"], + ) + for preset in ("urban", "rural", "offroad") + } + + if args.mode == "goose": + pairs = find_goose_pairs(mapping_root) + if len(pairs) != dataset_config["expected_pair_count"]: + raise RunnerError(f"GOOSE pair count changed: {len(pairs)}") + items: list[tuple[str, Path, Path | None]] = [ + (image.stem.removesuffix("_windshield_vis"), image, label) + for image, label in pairs + ] + else: + if args.frames_root is None or not args.frames_root.is_dir(): + raise RunnerError("frames-root is required for RAVNOVES mode") + frames = sorted(args.frames_root.glob("frame-*.png")) + expected = {f"frame-{index:06d}" for index in config["ravnoves"]["frame_indices"]} + if {frame.stem for frame in frames} != expected: + raise RunnerError("RAVNOVES frame island identity changed") + items = [(frame.stem, frame, None) for frame in frames] + + if args.limit: + items = items[: args.limit] + if not items: + raise RunnerError("no inputs were selected") + + args.output.mkdir(parents=True, exist_ok=False) + torch.cuda.empty_cache() + model, model_name, architecture_failures = load_model(args.candidate, args.checkpoint) + warmup_source = Image.open(items[0][1]).convert("RGB") + warmup_tensor, _ = preprocess(warmup_source) + warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)] + torch.cuda.reset_peak_memory_stats() + selected_visuals = visual_indices(len(items), args.visual_count) + confusion = np.zeros((CLASS_COUNT, CLASS_COUNT), dtype=np.int64) + latencies_ms: list[float] = [] + visuals: list[dict[str, Any]] = [] + + for index, (case_id, source_path, label_path) in enumerate(items): + source = Image.open(source_path).convert("RGB") + tensor, crop_box = preprocess(source) + prediction, latency_ms = infer(model, tensor) + latencies_ms.append(latency_ms) + truth = preprocess_label(Image.open(label_path)) if label_path is not None else None + if truth is not None: + update_confusion(confusion, truth, prediction) + if index in selected_visuals: + visuals.append( + write_visual_case( + args.output, + case_id, + source, + prediction, + semantic_palette, + policy_palettes, + crop_box, + truth=truth, + preserve_source_size=args.mode == "ravnoves", + ) + ) + + rows = class_metrics(confusion, names) if args.mode == "goose" else [] + valid_ious = [row["iou"] for row in rows if row["iou"] is not None] + vegetation_names = set(config["vegetation_class_names"]) + vegetation_rows = [row for row in rows if row["class_name"] in vegetation_names] + vegetation_ious = [row["iou"] for row in vegetation_rows if row["iou"] is not None] + timing = { + "prewarm_inference_count": len(warmup_latencies_ms), + "prewarm_latency_ms": round(warmup_latencies_ms[0], 4), + "prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4), + "sample_count": len(latencies_ms), + "latency_ms_p50": round(percentile(latencies_ms, 0.50), 4), + "latency_ms_p95": round(percentile(latencies_ms, 0.95), 4), + "latency_ms_mean": round(statistics.fmean(latencies_ms), 4), + "throughput_fps_from_mean_inference": round(1000.0 / statistics.fmean(latencies_ms), 4), + } + result: dict[str, Any] = { + "schema_version": SCHEMA, + "lab_id": config["lab_id"], + "worker_id": config["worker_id"], + "mode": args.mode, + "candidate": { + "candidate_id": candidate_config["candidate_id"], + "candidate_key": args.candidate, + "loaded_model_name": model_name, + "architecture_probe_failures": architecture_failures, + "checkpoint_size_bytes": args.checkpoint.stat().st_size, + "checkpoint_sha256": checkpoint_sha256, + }, + "source": { + "source_id": dataset_config["dataset_id"] + if args.mode == "goose" + else config["ravnoves"]["source_id"], + "input_count": len(items), + "ground_truth_available": args.mode == "goose", + "mapping_sha256": dataset_config["mapping_sha256"], + }, + "preprocessing": dataset_config["preprocessing"], + "metrics": { + "mean_iou": round(statistics.fmean(valid_ious), 8) if valid_ious else None, + "mean_iou_percent": round(statistics.fmean(valid_ious) * 100.0, 4) + if valid_ious + else None, + "published_mean_iou_percent": candidate_config["published_validation_miou_percent"], + "vegetation_mean_iou": round(statistics.fmean(vegetation_ious), 8) + if vegetation_ious + else None, + "vegetation_classes": vegetation_rows, + "all_classes": rows, + }, + "timing": timing, + "resource": { + "gpu_name": torch.cuda.get_device_name(0), + "peak_allocated_vram_bytes": int(torch.cuda.max_memory_allocated()), + "peak_reserved_vram_bytes": int(torch.cuda.max_memory_reserved()), + "torch_version": torch.__version__, + "cuda_runtime_version": torch.version.cuda, + "python_version": platform.python_version(), + "super_gradients_version": "3.2.0", + }, + "visual_cases": visuals, + "authority": { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + }, + } + identity_value = { + "schema_version": result["schema_version"], + "candidate": result["candidate"], + "source": result["source"], + "preprocessing": result["preprocessing"], + "metrics": result["metrics"], + "timing": result["timing"], + "resource": result["resource"], + "visual_cases": result["visual_cases"], + "authority": result["authority"], + "config_sha256": sha256(args.config), + "policy_sha256": sha256(args.policy), + "provider_map_sha256": sha256(args.provider_map), + } + result["result_id"] = f"lab-v1-{args.mode}-{args.candidate}-{stable_digest(identity_value)}" + result["provenance"] = { + "config_sha256": identity_value["config_sha256"], + "policy_sha256": identity_value["policy_sha256"], + "provider_map_sha256": identity_value["provider_map_sha256"], + "hostname": platform.node(), + "pid": os.getpid(), + } + result_path = args.output / "result.json" + result_path.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"result_id": result["result_id"], "result_path": str(result_path)})) + + +if __name__ == "__main__": + run() diff --git a/src/k1link/laboratory/execution.py b/src/k1link/laboratory/execution.py index 6ae8a24..f239e1e 100644 --- a/src/k1link/laboratory/execution.py +++ b/src/k1link/laboratory/execution.py @@ -326,6 +326,7 @@ def canonical_laboratory_adapters() -> dict[str, LaboratoryAdapter]: "experimental.e47-semantic-slam-shadow/v1": _run_e47, "experimental.m48s-fixed-class-detector/v1": _run_m48s_fixed_class_detector, "experimental.m48t-risk-quality-temporal/v1": _run_m48t_risk_quality_temporal, + "experimental.lab-v1-vegetation-shadow/v1": _run_lab_v1_vegetation_shadow, } @@ -382,6 +383,28 @@ def _run_m48t_risk_quality_temporal( ) +def _run_lab_v1_vegetation_shadow( + request: LaboratoryRunRequest, +) -> LaboratoryAdapterResult: + from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab + + result_root = seal_vegetation_shadow_lab( + ddrnet_goose_root=request.inputs["ddrnet_goose_root"], + ppliteseg_goose_root=request.inputs["ppliteseg_goose_root"], + ddrnet_ravnoves_root=request.inputs["ddrnet_ravnoves_root"], + ppliteseg_ravnoves_root=request.inputs["ppliteseg_ravnoves_root"], + output_root=request.output_root, + ) + manifest = _object( + json.loads((result_root / "result.json").read_text(encoding="utf-8")), + "vegetation shadow result", + ) + result_id = manifest.get("result_id") + if not isinstance(result_id, str): + raise LaboratoryExecutionError("vegetation shadow result_id is invalid") + return LaboratoryAdapterResult(result_root=result_root, result_id=result_id) + + def _run_m48_small_static_passage_regression( request: LaboratoryRunRequest, ) -> LaboratoryAdapterResult: diff --git a/src/k1link/laboratory/vegetation_shadow_lab.py b/src/k1link/laboratory/vegetation_shadow_lab.py new file mode 100644 index 0000000..a378a90 --- /dev/null +++ b/src/k1link/laboratory/vegetation_shadow_lab.py @@ -0,0 +1,381 @@ +"""Seal GOOSE qualification and RAVNOVES vegetation shadow evidence for LAB.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import tempfile +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from typing import Any, Final + +LAB_SCHEMA: Final = "missioncore.lab-v1-vegetation-shadow/v1" +WORKER_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1" +RESULT_PREFIX: Final = "lab-v1-vegetation-shadow-" +_CANDIDATES: Final = ("ddrnet", "ppliteseg") +_MODES: Final = ("goose", "ravnoves") +_IMAGE_KEYS: Final = ( + "source", + "prediction_semantic", + "policy_urban", + "policy_rural", + "policy_offroad", + "truth_semantic", +) + + +class VegetationShadowLabError(ValueError): + """Raised when Worker evidence cannot be sealed without changing its meaning.""" + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def sha256_path(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _object(value: object, label: str) -> dict[str, Any]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise VegetationShadowLabError(f"{label} must be an object") + return value + + +def _read_worker_result(root: Path, *, candidate: str, mode: str) -> dict[str, Any]: + if root.is_symlink() or not root.is_dir(): + raise VegetationShadowLabError(f"{candidate}/{mode} result root is unavailable") + path = root / "result.json" + if path.is_symlink() or not path.is_file() or path.stat().st_size > 1024 * 1024: + raise VegetationShadowLabError(f"{candidate}/{mode} result document is unavailable") + try: + result = _object(json.loads(path.read_text("utf-8")), f"{candidate}/{mode} result") + except (json.JSONDecodeError, OSError) as exc: + raise VegetationShadowLabError(f"{candidate}/{mode} result is invalid") from exc + candidate_value = _object(result.get("candidate"), f"{candidate}/{mode} candidate") + authority = _object(result.get("authority"), f"{candidate}/{mode} authority") + if ( + result.get("schema_version") != WORKER_SCHEMA + or result.get("mode") != mode + or candidate_value.get("candidate_key") != candidate + or authority.get("navigation_accepted") is not False + or authority.get("safety_accepted") is not False + or authority.get("actuation_accepted") is not False + or authority.get("camera_semantics_can_clear_rigid_geometry") is not False + ): + raise VegetationShadowLabError(f"{candidate}/{mode} contract changed") + return result + + +def _case_map(result: dict[str, Any], label: str) -> dict[str, dict[str, Any]]: + values = result.get("visual_cases") + if not isinstance(values, list) or len(values) != 12: + raise VegetationShadowLabError(f"{label} must contain exactly 12 visual cases") + rows: dict[str, dict[str, Any]] = {} + for raw in values: + row = _object(raw, f"{label} visual case") + case_id = row.get("case_id") + if not isinstance(case_id, str) or not case_id or case_id in rows: + raise VegetationShadowLabError(f"{label} case identity changed") + rows[case_id] = row + return rows + + +def _file_from_case( + root: Path, + case: dict[str, Any], + key: str, + *, + required: bool = True, +) -> tuple[Path, str] | None: + files = _object(case.get("files"), "visual case files") + raw = files.get(key) + if raw is None and not required: + return None + descriptor = _object(raw, f"visual case {key}") + relative = descriptor.get("relative_path") + expected_sha256 = descriptor.get("sha256") + if not isinstance(relative, str) or not isinstance(expected_sha256, str): + raise VegetationShadowLabError(f"visual case {key} proof is invalid") + posix = PurePosixPath(relative) + if posix.is_absolute() or str(posix) != relative or any( + part in {"", ".", ".."} for part in posix.parts + ): + raise VegetationShadowLabError(f"visual case {key} path is invalid") + path = root.joinpath(*posix.parts) + if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()): + raise VegetationShadowLabError(f"visual case {key} file is unavailable") + if sha256_path(path) != expected_sha256: + raise VegetationShadowLabError(f"visual case {key} digest changed") + return path, expected_sha256 + + +def _copy_artifact( + source: Path, + destination_root: Path, + relative: str, + artifacts: list[dict[str, object]], + *, + role: str, + media_type: str, +) -> dict[str, object]: + destination = destination_root.joinpath(*PurePosixPath(relative).parts) + destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + shutil.copyfile(source, destination) + digest = sha256_path(destination) + descriptor = { + "role": role, + "path": relative, + "byte_length": destination.stat().st_size, + "sha256": digest, + "media_type": media_type, + } + artifacts.append(descriptor) + return descriptor + + +def _selected_candidate(results: dict[tuple[str, str], dict[str, Any]]) -> str: + def rank(candidate: str) -> tuple[float, float]: + validation = results[(candidate, "goose")] + metrics = _object(validation.get("metrics"), f"{candidate} metrics") + timing = _object(validation.get("timing"), f"{candidate} timing") + vegetation_iou = metrics.get("vegetation_mean_iou") + p95_ms = timing.get("latency_ms_p95") + if not isinstance(vegetation_iou, (int, float)) or not isinstance(p95_ms, (int, float)): + raise VegetationShadowLabError(f"{candidate} qualification metrics are incomplete") + return float(vegetation_iou), -float(p95_ms) + + return max(_CANDIDATES, key=rank) + + +def _validation_metric_summary(result: dict[str, Any], candidate: str) -> dict[str, object]: + metrics = _object(result.get("metrics"), f"{candidate} metrics") + return { + "mean_iou_percent": metrics.get("mean_iou_percent"), + "published_mean_iou_percent": metrics.get("published_mean_iou_percent"), + "vegetation_mean_iou": metrics.get("vegetation_mean_iou"), + } + + +def seal_vegetation_shadow_lab( + *, + ddrnet_goose_root: Path, + ppliteseg_goose_root: Path, + ddrnet_ravnoves_root: Path, + ppliteseg_ravnoves_root: Path, + output_root: Path, +) -> Path: + roots = { + ("ddrnet", "goose"): ddrnet_goose_root.resolve(), + ("ppliteseg", "goose"): ppliteseg_goose_root.resolve(), + ("ddrnet", "ravnoves"): ddrnet_ravnoves_root.resolve(), + ("ppliteseg", "ravnoves"): ppliteseg_ravnoves_root.resolve(), + } + results = { + key: _read_worker_result(root, candidate=key[0], mode=key[1]) + for key, root in roots.items() + } + cases = {key: _case_map(result, f"{key[0]}/{key[1]}") for key, result in results.items()} + for mode in _MODES: + if cases[("ddrnet", mode)].keys() != cases[("ppliteseg", mode)].keys(): + raise VegetationShadowLabError(f"{mode} candidate case islands differ") + selected = _selected_candidate(results) + + output_root.mkdir(mode=0o700, parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=".lab-v1-vegetation-", dir=output_root)) + artifacts: list[dict[str, object]] = [] + catalogs: dict[str, list[dict[str, object]]] = {"goose": [], "ravnoves": []} + try: + for mode in _MODES: + for case_id in sorted(cases[("ddrnet", mode)]): + ddr_case = cases[("ddrnet", mode)][case_id] + pplite_case = cases[("ppliteseg", mode)][case_id] + row: dict[str, object] = { + "case_id": case_id, + "source_kind": mode, + "width": ddr_case.get("source_width"), + "height": ddr_case.get("source_height"), + "center_crop_xyxy": ddr_case.get("center_crop_xyxy"), + "outside_crop_state": ddr_case.get("outside_crop_state"), + "assets": {}, + } + asset_map = _object(row["assets"], "sealed assets") + sources: list[tuple[str, str, dict[str, Any], str]] = [ + ("source", "ddrnet", ddr_case, "source"), + ("ddrnet", "ddrnet", ddr_case, "prediction_semantic"), + ("ppliteseg", "ppliteseg", pplite_case, "prediction_semantic"), + ] + if mode == "goose": + sources.append(("truth", "ddrnet", ddr_case, "truth_semantic")) + else: + selected_case = cases[(selected, mode)][case_id] + sources.extend( + (preset, selected, selected_case, f"policy_{preset}") + for preset in ("urban", "rural", "offroad") + ) + for asset_key, candidate, case, worker_key in sources: + resolved = _file_from_case(roots[(candidate, mode)], case, worker_key) + assert resolved is not None + source_path, _ = resolved + relative = f"visual/{mode}/{case_id}/{asset_key}.png" + descriptor = _copy_artifact( + source_path, + temporary, + relative, + artifacts, + role=f"visual-{mode}-{asset_key}", + media_type="image/png", + ) + asset_map[asset_key] = { + "path": descriptor["path"], + "sha256": descriptor["sha256"], + } + catalogs[mode].append(row) + + worker_proofs: dict[str, dict[str, object]] = {} + for candidate in _CANDIDATES: + for mode in _MODES: + source = roots[(candidate, mode)] / "result.json" + relative = f"worker/{candidate}-{mode}.json" + descriptor = _copy_artifact( + source, + temporary, + relative, + artifacts, + role="worker-result", + media_type="application/json", + ) + worker_proofs[f"{candidate}_{mode}"] = { + "result_id": results[(candidate, mode)].get("result_id"), + "path": relative, + "sha256": descriptor["sha256"], + } + + candidate_metrics: dict[str, object] = {} + for candidate in _CANDIDATES: + validation = results[(candidate, "goose")] + shadow = results[(candidate, "ravnoves")] + candidate_metrics[candidate] = { + "loaded_model_name": _object(validation["candidate"], "candidate").get( + "loaded_model_name" + ), + "checkpoint_sha256": _object(validation["candidate"], "candidate").get( + "checkpoint_sha256" + ), + # Detailed per-class rows remain immutable in worker_proofs. The + # top-level LAB manifest carries only the UI/index summary. + "validation_metrics": _validation_metric_summary(validation, candidate), + "validation_timing": validation.get("timing"), + "shadow_timing": shadow.get("timing"), + "resource": shadow.get("resource"), + } + + authority = { + "commands_enabled": False, + "navigation_or_safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + } + identity = { + "lab_id": "lab-v1-vegetation-mission-policy", + "source": { + "validation_dataset": "GOOSE-2D-validation-visible-962", + "shadow_session": "RAVNOVES00", + "shadow_camera": "sensor.camera.right", + "shadow_frame_count": 12, + }, + "selected_candidate": selected, + "candidate_metrics": candidate_metrics, + "worker_proofs": worker_proofs, + "visual_catalog_sha256": hashlib.sha256(canonical_json(catalogs)).hexdigest(), + "authority": authority, + } + identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() + result_id = f"{RESULT_PREFIX}{identity_sha256}" + manifest = { + "schema_version": LAB_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "created_at_utc": datetime.now(UTC).isoformat(), + "ground_truth": False, + "status": "visual-shadow-ready-policy-not-authorized", + "identity": identity, + "source": identity["source"], + "method": { + "completeness": "complete", + "execution_class": "ai-inference", + "pipeline_id": "goose-fine64-ready-weights-to-ravnoves-policy-shadow/v1", + }, + "metrics": {"candidates": candidate_metrics}, + "decision": { + "selected_candidate": selected, + "visual_shadow_ready": True, + "mission_policy_ready_for_configuration": True, + "navigation_accepted": False, + "production_accepted": False, + }, + "limitations": [ + "GOOSE validation is external-domain qualification, not RAVNOVES ground truth.", + "The RAVNOVES island is visual shadow evidence without independent labels.", + "Vegetation semantics never clears rigid LiDAR/TGS occupancy.", + "Undefined pixels outside the 600x600 center crop remain fail-closed.", + ], + "authority": authority, + "catalogs": catalogs, + "artifacts": artifacts, + } + (temporary / "result.json").write_bytes(canonical_json(manifest) + b"\n") + destination = output_root / result_id + if destination.exists(): + raise VegetationShadowLabError("immutable vegetation LAB result already exists") + temporary.replace(destination) + return destination + except Exception: + shutil.rmtree(temporary, ignore_errors=True) + raise + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ddrnet-goose-root", type=Path, required=True) + parser.add_argument("--ppliteseg-goose-root", type=Path, required=True) + parser.add_argument("--ddrnet-ravnoves-root", type=Path, required=True) + parser.add_argument("--ppliteseg-ravnoves-root", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + destination = seal_vegetation_shadow_lab( + ddrnet_goose_root=args.ddrnet_goose_root, + ppliteseg_goose_root=args.ppliteseg_goose_root, + ddrnet_ravnoves_root=args.ddrnet_ravnoves_root, + ppliteseg_ravnoves_root=args.ppliteseg_ravnoves_root, + output_root=args.output_root, + ) + print(destination) + + +if __name__ == "__main__": + main() + + +__all__ = [ + "LAB_SCHEMA", + "RESULT_PREFIX", + "VegetationShadowLabError", + "seal_vegetation_shadow_lab", +] diff --git a/src/k1link/web/app.py b/src/k1link/web/app.py index a97155a..2ce851c 100644 --- a/src/k1link/web/app.py +++ b/src/k1link/web/app.py @@ -138,6 +138,7 @@ from k1link.web.m49_physical_safety_playback_api import ( ) from k1link.web.m49_tgs_fail_closed_api import build_m49_tgs_fail_closed_router from k1link.web.m49_tgs_full_shadow_api import build_m49_tgs_full_shadow_router +from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router from k1link.web.map_api import ( MapGatewayConfiguration, MapGatewayProxy, @@ -1019,6 +1020,17 @@ app.include_router( ), ) ) +app.include_router( + build_vegetation_shadow_lab_router( + root_provider=lambda: ( + REPOSITORY_ROOT + / ".runtime" + / "compute-experiments" + / "lab-v1-vegetation" + / "results" + ), + ) +) app.include_router( build_m49_physical_safety_playback_router( root_provider=lambda: ( diff --git a/src/k1link/web/vegetation_shadow_lab_api.py b/src/k1link/web/vegetation_shadow_lab_api.py new file mode 100644 index 0000000..1741d91 --- /dev/null +++ b/src/k1link/web/vegetation_shadow_lab_api.py @@ -0,0 +1,166 @@ +"""Read-only API for the autonomous vegetation policy shadow LAB.""" + +from __future__ import annotations + +import copy +import json +import re +from collections.abc import Callable +from functools import lru_cache +from pathlib import Path, PurePosixPath +from typing import Any, Final + +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse + +from k1link.laboratory.evidence_registry import LaboratoryEvidenceDefinition +from k1link.laboratory.evidence_report import ( + LaboratoryEvidenceReportError, + verify_laboratory_evidence_result, +) +from k1link.laboratory.vegetation_shadow_lab import LAB_SCHEMA, RESULT_PREFIX + +RootProvider = Callable[[], Path | None] +RESULT_ID: Final = re.compile(rf"^{re.escape(RESULT_PREFIX)}[a-f0-9]{{64}}$") +_MAX_DOCUMENT_BYTES: Final = 1024 * 1024 +_DEFINITION: Final = LaboratoryEvidenceDefinition( + work_id="lab-v1-vegetation-shadow", + runtime_relative_root=PurePosixPath("lab-v1-vegetation/results"), + result_id_prefix="lab-v1-vegetation-shadow", + document_name="result.json", + result_schema_version=LAB_SCHEMA, +) + + +def build_vegetation_shadow_lab_router( + *, root_provider: RootProvider = lambda: None, +) -> APIRouter: + router = APIRouter( + prefix="/api/v1/laboratory/vegetation-shadow", + tags=["laboratory"], + ) + + @router.get("/{result_id}") + def get_result(result_id: str) -> dict[str, object]: + candidate = _resolve_candidate(root_provider, result_id) + return {**copy.deepcopy(_read_verified(candidate)), "access": "read-only"} + + @router.get("/{result_id}/assets/{asset_path:path}") + def get_asset(result_id: str, asset_path: str) -> FileResponse: + candidate = _resolve_candidate(root_provider, result_id) + manifest = _read_verified(candidate) + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list): + raise HTTPException(status_code=404, detail="Vegetation LAB asset not found") + descriptor = next( + ( + item + for item in artifacts + if isinstance(item, dict) and item.get("path") == asset_path + ), + None, + ) + if descriptor is None: + raise HTTPException(status_code=404, detail="Vegetation LAB asset not found") + relative = PurePosixPath(asset_path) + path = candidate.joinpath(*relative.parts) + if ( + relative.is_absolute() + or str(relative) != asset_path + or any(part in {"", ".", ".."} for part in relative.parts) + or path.is_symlink() + or not path.is_file() + or not path.resolve().is_relative_to(candidate) + ): + raise HTTPException(status_code=404, detail="Vegetation LAB asset not found") + media_type = descriptor.get("media_type") + if not isinstance(media_type, str) or not media_type.startswith("image/"): + raise HTTPException(status_code=404, detail="Vegetation LAB asset not found") + return FileResponse( + path, + media_type=media_type, + headers={ + "Cache-Control": "private, max-age=31536000, immutable", + "ETag": f'"{descriptor.get("sha256", "")}"', + "X-Content-Type-Options": "nosniff", + }, + ) + + return router + + +def _configured_root(provider: RootProvider) -> Path | None: + candidate = provider() + if candidate is None: + return None + absolute = candidate.expanduser().absolute() + if absolute.is_symlink(): + return None + try: + root = absolute.resolve(strict=True) + except OSError: + return None + return root if root.is_dir() else None + + +def _resolve_candidate(provider: RootProvider, result_id: str) -> Path: + root = _configured_root(provider) + if root is None or RESULT_ID.fullmatch(result_id) is None: + raise HTTPException(status_code=404, detail="Vegetation LAB result not found") + candidate = root / result_id + if candidate.is_symlink(): + raise HTTPException(status_code=404, detail="Vegetation LAB result not found") + try: + resolved = candidate.resolve(strict=True) + except OSError: + raise HTTPException(status_code=404, detail="Vegetation LAB result not found") from None + if not resolved.is_dir() or not resolved.is_relative_to(root): + raise HTTPException(status_code=404, detail="Vegetation LAB result not found") + return resolved + + +def _read_verified(candidate: Path) -> dict[str, Any]: + try: + rows: list[tuple[str, int, int, int, int]] = [] + for path in sorted(candidate.rglob("*"), key=lambda item: item.as_posix()): + stat = path.lstat() + rows.append(( + path.relative_to(candidate).as_posix(), + stat.st_mode, + stat.st_size, + stat.st_mtime_ns, + stat.st_ctime_ns, + )) + signature = tuple(rows) + except OSError: + raise HTTPException( + status_code=503, + detail="Vegetation LAB evidence failed verification", + ) from None + return _read_verified_cached(str(candidate), signature) + + +@lru_cache(maxsize=16) +def _read_verified_cached( + candidate_text: str, + signature: tuple[tuple[str, int, int, int, int], ...], +) -> dict[str, Any]: + del signature + candidate = Path(candidate_text) + try: + verify_laboratory_evidence_result(_DEFINITION, candidate) + path = candidate / "result.json" + if path.stat().st_size > _MAX_DOCUMENT_BYTES: + raise LaboratoryEvidenceReportError("Vegetation LAB document is too large") + payload = json.loads(path.read_text("utf-8")) + except (json.JSONDecodeError, OSError, LaboratoryEvidenceReportError): + raise HTTPException( + status_code=503, + detail="Vegetation LAB evidence failed verification", + ) from None + if not isinstance(payload, dict): + raise HTTPException(status_code=503, detail="Vegetation LAB evidence is invalid") + return payload + + +__all__ = ["build_vegetation_shadow_lab_router"] diff --git a/tests/test_lab_v1_goose_vegetation_benchmark.py b/tests/test_lab_v1_goose_vegetation_benchmark.py new file mode 100644 index 0000000..d8a22ee --- /dev/null +++ b/tests/test_lab_v1_goose_vegetation_benchmark.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = ( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-goose-vegetation-benchmark-v1.json" +) +RUNNER_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "lab_v1_vegetation_goose" + / "run_goose_vegetation_benchmark.py" +) +POWERSHELL_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "Invoke-LabV1VegetationGooseBenchmark.ps1" +) + + +def test_benchmark_contract_is_bounded_and_fail_closed() -> None: + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + assert config["schema_version"] == "missioncore.lab-v1-goose-vegetation-benchmark/v1" + assert config["dataset"]["expected_pair_count"] == 962 + assert config["dataset"]["image_glob"].endswith("*_windshield_vis.png") + assert config["dataset"]["preprocessing"] == [ + "center-square-crop", + "nearest-neighbor-resize", + "rgb-to-tensor-0-1", + ] + assert set(config["candidates"]) == {"ddrnet", "ppliteseg"} + assert all( + len(candidate["checkpoint_sha256"]) == 64 + for candidate in config["candidates"].values() + ) + assert config["ravnoves"]["expected_frame_count"] == 4489 + assert len(config["ravnoves"]["frame_indices"]) == 12 + assert config["invariants"] == { + "one_heavy_candidate_at_a_time": True, + "raw_fisheye_is_immutable": True, + "outside_center_crop_is_free": False, + "missing_or_unknown_is_free": False, + "camera_semantics_can_clear_rigid_geometry": False, + "navigation_authority": False, + "actuation_authority": False, + "canonical_triton_mutation_allowed": False, + } + + +def test_runner_uses_exact_visible_pairs_and_never_grants_authority() -> None: + source = RUNNER_PATH.read_text(encoding="utf-8") + assert 'rglob("*_windshield_vis.png")' in source + assert "resample=RESAMPLE_NEAREST" in source + assert '"navigation_accepted": False' in source + assert '"actuation_accepted": False' in source + assert '"camera_semantics_can_clear_rigid_geometry": False' in source + assert "torch.cuda.reset_peak_memory_stats()" in source + assert 'warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]' in source + assert '"prewarm_inference_count": len(warmup_latencies_ms)' in source + assert '"prewarm_latency_ms": round(warmup_latencies_ms[0], 4)' in source + assert '"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4)' in source + + +def test_worker_wrapper_is_isolated_from_canonical_triton() -> None: + source = POWERSHELL_PATH.read_text(encoding="utf-8") + assert '$canonicalContainer = "ndc-mission-core-triton"' in source + assert '"--network", "none"' in source + assert '"--read-only"' in source + assert '"--cap-drop", "ALL"' in source + assert '"--security-opt", "no-new-privileges"' in source + assert "if ($canonicalAfter -ne $canonicalBefore)" in source diff --git a/tests/test_laboratory_evidence_registry.py b/tests/test_laboratory_evidence_registry.py index de5cc21..4017364 100644 --- a/tests/test_laboratory_evidence_registry.py +++ b/tests/test_laboratory_evidence_registry.py @@ -127,8 +127,9 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None: repository_root / "config" / "laboratories" ) - assert len(registry.definitions) == 42 + assert len(registry.definitions) == 43 assert {item.work_id for item in registry.definitions} >= { + "lab-v1-vegetation-shadow", "e31-source-binding", "e46j-raw-fisheye-realtime", "e47-semantic-slam-shadow", diff --git a/tests/test_laboratory_execution.py b/tests/test_laboratory_execution.py index 0bbbad2..9d5d830 100644 --- a/tests/test_laboratory_execution.py +++ b/tests/test_laboratory_execution.py @@ -103,6 +103,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None: "m48t-risk-quality-temporal", "m49-tgs-fail-closed-evidence", "m49-tgs-full-shadow", + "lab-v1-vegetation-shadow", } by_work_id = {row.work_id: row for row in execution.definitions} assert by_work_id["m48-small-static-passage-regression"].evidence_contract == ( @@ -124,6 +125,8 @@ def test_repository_registry_classifies_every_evidence_definition() -> None: assert by_work_id["m49-tgs-fail-closed-evidence"].isolation == "bounded-adapter" assert by_work_id["m49-tgs-full-shadow"].lifecycle == "experimental" assert by_work_id["m49-tgs-full-shadow"].isolation == "bounded-adapter" + assert by_work_id["lab-v1-vegetation-shadow"].lifecycle == "experimental" + assert by_work_id["lab-v1-vegetation-shadow"].isolation == "bounded-adapter" assert all( row.lifecycle == "canonical" for row in execution.definitions @@ -134,6 +137,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None: "m48t-risk-quality-temporal", "m49-tgs-fail-closed-evidence", "m49-tgs-full-shadow", + "lab-v1-vegetation-shadow", } ) assert len(execution.definitions) + len(execution.legacy_work_ids) == len( diff --git a/tests/test_laboratory_value_review_registry.py b/tests/test_laboratory_value_review_registry.py index 62d9202..230fcdf 100644 --- a/tests/test_laboratory_value_review_registry.py +++ b/tests/test_laboratory_value_review_registry.py @@ -80,7 +80,7 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() -> root / "config" / "laboratory-value-review.json" ) - assert len(registry.entries) == 40 + assert len(registry.entries) == 41 assert {entry.catalog_id for entry in registry.entries} >= { "e28-local-surface", "e46d-temporal-failure-audit", @@ -93,4 +93,5 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() -> "m48t-risk-quality-temporal", "m49-tgs-fail-closed-evidence", "m49-tgs-full-shadow", + "lab-v1-vegetation-shadow", } diff --git a/tests/test_vegetation_shadow_lab.py b/tests/test_vegetation_shadow_lab.py new file mode 100644 index 0000000..c672de2 --- /dev/null +++ b/tests/test_vegetation_shadow_lab.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from k1link.laboratory import LaboratoryEvidenceRegistry +from k1link.laboratory.evidence_report import verify_laboratory_evidence_result +from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab +from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _worker_result(root: Path, *, candidate: str, mode: str, vegetation_iou: float) -> None: + root.mkdir(parents=True) + visual_cases = [] + for index in range(12): + case_id = f"case-{index:02d}" + case_root = root / "cases" / case_id + case_root.mkdir(parents=True) + keys = ["source", "prediction_semantic", "policy_urban", "policy_rural", "policy_offroad"] + if mode == "goose": + keys.append("truth_semantic") + files = {} + for key in keys: + path = case_root / f"{key}.png" + path.write_bytes(b"\x89PNG\r\n\x1a\n" + f"{candidate}:{mode}:{case_id}:{key}".encode()) + files[key] = { + "relative_path": path.relative_to(root).as_posix(), + "sha256": _sha256(path), + } + visual_cases.append( + { + "case_id": case_id, + "source_width": 800 if mode == "ravnoves" else 512, + "source_height": 600 if mode == "ravnoves" else 512, + "center_crop_xyxy": [100, 0, 700, 600] if mode == "ravnoves" else [0, 0, 512, 512], + "outside_crop_state": "undefined" if mode == "ravnoves" else "not-applicable", + "files": files, + } + ) + payload = { + "schema_version": "missioncore.lab-v1-goose-vegetation-run/v1", + "result_id": f"lab-v1-{mode}-{candidate}-fixture", + "mode": mode, + "candidate": { + "candidate_key": candidate, + "loaded_model_name": "ddrnet_39" if candidate == "ddrnet" else "pp_lite_t_seg", + "checkpoint_sha256": ("a" if candidate == "ddrnet" else "b") * 64, + }, + "metrics": { + "mean_iou_percent": 44.0 + vegetation_iou, + "published_mean_iou_percent": 46.53 if candidate == "ddrnet" else 45.09, + "vegetation_mean_iou": vegetation_iou, + }, + "timing": { + "latency_ms_p95": 20.0 if candidate == "ddrnet" else 15.0, + "throughput_fps_from_mean_inference": 55.0, + }, + "resource": { + "peak_reserved_vram_bytes": 2_000_000_000, + "gpu_name": "fixture RTX 4090", + }, + "visual_cases": visual_cases, + "authority": { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, + }, + } + (root / "result.json").write_text(json.dumps(payload), encoding="utf-8") + + +def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) -> None: + roots = {} + for candidate, vegetation_iou in (("ddrnet", 0.64), ("ppliteseg", 0.61)): + for mode in ("goose", "ravnoves"): + root = tmp_path / "worker" / f"{candidate}-{mode}" + _worker_result(root, candidate=candidate, mode=mode, vegetation_iou=vegetation_iou) + roots[(candidate, mode)] = root + result_root = seal_vegetation_shadow_lab( + ddrnet_goose_root=roots[("ddrnet", "goose")], + ppliteseg_goose_root=roots[("ppliteseg", "goose")], + ddrnet_ravnoves_root=roots[("ddrnet", "ravnoves")], + ppliteseg_ravnoves_root=roots[("ppliteseg", "ravnoves")], + output_root=tmp_path / "results", + ) + manifest = json.loads((result_root / "result.json").read_text("utf-8")) + assert manifest["decision"]["selected_candidate"] == "ddrnet" + assert manifest["ground_truth"] is False + assert manifest["authority"]["commands_enabled"] is False + assert manifest["authority"]["navigation_or_safety_accepted"] is False + assert len(manifest["catalogs"]["ravnoves"]) == 12 + assert len(manifest["catalogs"]["goose"]) == 12 + assert len(manifest["artifacts"]) == 124 + assert "all_classes" not in manifest["metrics"]["candidates"]["ddrnet"]["validation_metrics"] + assert (result_root / "result.json").stat().st_size <= 64 * 1024 + + registry = LaboratoryEvidenceRegistry.from_directory(REPOSITORY_ROOT / "config/laboratories") + definition = next( + row for row in registry.definitions if row.work_id == "lab-v1-vegetation-shadow" + ) + proof = verify_laboratory_evidence_result(definition, result_root) + assert proof["result_id"] == result_root.name + assert proof["artifact_count"] == 124 + + app = FastAPI() + app.include_router(build_vegetation_shadow_lab_router(root_provider=lambda: result_root.parent)) + client = TestClient(app) + response = client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}") + assert response.status_code == 200 + assert response.json()["access"] == "read-only" + asset_path = manifest["catalogs"]["ravnoves"][0]["assets"]["offroad"]["path"] + asset = client.get( + f"/api/v1/laboratory/vegetation-shadow/{result_root.name}/assets/{asset_path}" + ) + assert asset.status_code == 200 + assert asset.headers["cache-control"].endswith("immutable") + + (result_root / asset_path).write_bytes(b"tampered") + assert ( + client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}").status_code + == 503 + )