feat(perception): add autonomous vegetation shadow lab
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIG_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "config"
|
||||
/ "perception"
|
||||
/ "lab-v1-goose-vegetation-benchmark-v1.json"
|
||||
)
|
||||
RUNNER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "lab_v1_vegetation_goose"
|
||||
/ "run_goose_vegetation_benchmark.py"
|
||||
)
|
||||
POWERSHELL_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "Invoke-LabV1VegetationGooseBenchmark.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_benchmark_contract_is_bounded_and_fail_closed() -> None:
|
||||
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
assert config["schema_version"] == "missioncore.lab-v1-goose-vegetation-benchmark/v1"
|
||||
assert config["dataset"]["expected_pair_count"] == 962
|
||||
assert config["dataset"]["image_glob"].endswith("*_windshield_vis.png")
|
||||
assert config["dataset"]["preprocessing"] == [
|
||||
"center-square-crop",
|
||||
"nearest-neighbor-resize",
|
||||
"rgb-to-tensor-0-1",
|
||||
]
|
||||
assert set(config["candidates"]) == {"ddrnet", "ppliteseg"}
|
||||
assert all(
|
||||
len(candidate["checkpoint_sha256"]) == 64
|
||||
for candidate in config["candidates"].values()
|
||||
)
|
||||
assert config["ravnoves"]["expected_frame_count"] == 4489
|
||||
assert len(config["ravnoves"]["frame_indices"]) == 12
|
||||
assert config["invariants"] == {
|
||||
"one_heavy_candidate_at_a_time": True,
|
||||
"raw_fisheye_is_immutable": True,
|
||||
"outside_center_crop_is_free": False,
|
||||
"missing_or_unknown_is_free": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
"navigation_authority": False,
|
||||
"actuation_authority": False,
|
||||
"canonical_triton_mutation_allowed": False,
|
||||
}
|
||||
|
||||
|
||||
def test_runner_uses_exact_visible_pairs_and_never_grants_authority() -> None:
|
||||
source = RUNNER_PATH.read_text(encoding="utf-8")
|
||||
assert 'rglob("*_windshield_vis.png")' in source
|
||||
assert "resample=RESAMPLE_NEAREST" in source
|
||||
assert '"navigation_accepted": False' in source
|
||||
assert '"actuation_accepted": False' in source
|
||||
assert '"camera_semantics_can_clear_rigid_geometry": False' in source
|
||||
assert "torch.cuda.reset_peak_memory_stats()" in source
|
||||
assert 'warmup_latencies_ms = [infer(model, warmup_tensor)[1] for _ in range(3)]' in source
|
||||
assert '"prewarm_inference_count": len(warmup_latencies_ms)' in source
|
||||
assert '"prewarm_latency_ms": round(warmup_latencies_ms[0], 4)' in source
|
||||
assert '"prewarm_latency_ms_last": round(warmup_latencies_ms[-1], 4)' in source
|
||||
|
||||
|
||||
def test_worker_wrapper_is_isolated_from_canonical_triton() -> None:
|
||||
source = POWERSHELL_PATH.read_text(encoding="utf-8")
|
||||
assert '$canonicalContainer = "ndc-mission-core-triton"' in source
|
||||
assert '"--network", "none"' in source
|
||||
assert '"--read-only"' in source
|
||||
assert '"--cap-drop", "ALL"' in source
|
||||
assert '"--security-opt", "no-new-privileges"' in source
|
||||
assert "if ($canonicalAfter -ne $canonicalBefore)" in source
|
||||
@@ -127,8 +127,9 @@ def test_product_registry_declares_every_advanced_evidence_source() -> None:
|
||||
repository_root / "config" / "laboratories"
|
||||
)
|
||||
|
||||
assert len(registry.definitions) == 42
|
||||
assert len(registry.definitions) == 43
|
||||
assert {item.work_id for item in registry.definitions} >= {
|
||||
"lab-v1-vegetation-shadow",
|
||||
"e31-source-binding",
|
||||
"e46j-raw-fisheye-realtime",
|
||||
"e47-semantic-slam-shadow",
|
||||
|
||||
@@ -103,6 +103,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
by_work_id = {row.work_id: row for row in execution.definitions}
|
||||
assert by_work_id["m48-small-static-passage-regression"].evidence_contract == (
|
||||
@@ -124,6 +125,8 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
assert by_work_id["m49-tgs-fail-closed-evidence"].isolation == "bounded-adapter"
|
||||
assert by_work_id["m49-tgs-full-shadow"].lifecycle == "experimental"
|
||||
assert by_work_id["m49-tgs-full-shadow"].isolation == "bounded-adapter"
|
||||
assert by_work_id["lab-v1-vegetation-shadow"].lifecycle == "experimental"
|
||||
assert by_work_id["lab-v1-vegetation-shadow"].isolation == "bounded-adapter"
|
||||
assert all(
|
||||
row.lifecycle == "canonical"
|
||||
for row in execution.definitions
|
||||
@@ -134,6 +137,7 @@ def test_repository_registry_classifies_every_evidence_definition() -> None:
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
)
|
||||
assert len(execution.definitions) + len(execution.legacy_work_ids) == len(
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
root / "config" / "laboratory-value-review.json"
|
||||
)
|
||||
|
||||
assert len(registry.entries) == 40
|
||||
assert len(registry.entries) == 41
|
||||
assert {entry.catalog_id for entry in registry.entries} >= {
|
||||
"e28-local-surface",
|
||||
"e46d-temporal-failure-audit",
|
||||
@@ -93,4 +93,5 @@ def test_product_value_review_registry_covers_reviewed_laboratory_families() ->
|
||||
"m48t-risk-quality-temporal",
|
||||
"m49-tgs-fail-closed-evidence",
|
||||
"m49-tgs-full-shadow",
|
||||
"lab-v1-vegetation-shadow",
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from k1link.laboratory import LaboratoryEvidenceRegistry
|
||||
from k1link.laboratory.evidence_report import verify_laboratory_evidence_result
|
||||
from k1link.laboratory.vegetation_shadow_lab import seal_vegetation_shadow_lab
|
||||
from k1link.web.vegetation_shadow_lab_api import build_vegetation_shadow_lab_router
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def _worker_result(root: Path, *, candidate: str, mode: str, vegetation_iou: float) -> None:
|
||||
root.mkdir(parents=True)
|
||||
visual_cases = []
|
||||
for index in range(12):
|
||||
case_id = f"case-{index:02d}"
|
||||
case_root = root / "cases" / case_id
|
||||
case_root.mkdir(parents=True)
|
||||
keys = ["source", "prediction_semantic", "policy_urban", "policy_rural", "policy_offroad"]
|
||||
if mode == "goose":
|
||||
keys.append("truth_semantic")
|
||||
files = {}
|
||||
for key in keys:
|
||||
path = case_root / f"{key}.png"
|
||||
path.write_bytes(b"\x89PNG\r\n\x1a\n" + f"{candidate}:{mode}:{case_id}:{key}".encode())
|
||||
files[key] = {
|
||||
"relative_path": path.relative_to(root).as_posix(),
|
||||
"sha256": _sha256(path),
|
||||
}
|
||||
visual_cases.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"source_width": 800 if mode == "ravnoves" else 512,
|
||||
"source_height": 600 if mode == "ravnoves" else 512,
|
||||
"center_crop_xyxy": [100, 0, 700, 600] if mode == "ravnoves" else [0, 0, 512, 512],
|
||||
"outside_crop_state": "undefined" if mode == "ravnoves" else "not-applicable",
|
||||
"files": files,
|
||||
}
|
||||
)
|
||||
payload = {
|
||||
"schema_version": "missioncore.lab-v1-goose-vegetation-run/v1",
|
||||
"result_id": f"lab-v1-{mode}-{candidate}-fixture",
|
||||
"mode": mode,
|
||||
"candidate": {
|
||||
"candidate_key": candidate,
|
||||
"loaded_model_name": "ddrnet_39" if candidate == "ddrnet" else "pp_lite_t_seg",
|
||||
"checkpoint_sha256": ("a" if candidate == "ddrnet" else "b") * 64,
|
||||
},
|
||||
"metrics": {
|
||||
"mean_iou_percent": 44.0 + vegetation_iou,
|
||||
"published_mean_iou_percent": 46.53 if candidate == "ddrnet" else 45.09,
|
||||
"vegetation_mean_iou": vegetation_iou,
|
||||
},
|
||||
"timing": {
|
||||
"latency_ms_p95": 20.0 if candidate == "ddrnet" else 15.0,
|
||||
"throughput_fps_from_mean_inference": 55.0,
|
||||
},
|
||||
"resource": {
|
||||
"peak_reserved_vram_bytes": 2_000_000_000,
|
||||
"gpu_name": "fixture RTX 4090",
|
||||
},
|
||||
"visual_cases": visual_cases,
|
||||
"authority": {
|
||||
"navigation_accepted": False,
|
||||
"safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
},
|
||||
}
|
||||
(root / "result.json").write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
|
||||
def test_vegetation_shadow_lab_seals_autonomous_visual_evidence(tmp_path: Path) -> None:
|
||||
roots = {}
|
||||
for candidate, vegetation_iou in (("ddrnet", 0.64), ("ppliteseg", 0.61)):
|
||||
for mode in ("goose", "ravnoves"):
|
||||
root = tmp_path / "worker" / f"{candidate}-{mode}"
|
||||
_worker_result(root, candidate=candidate, mode=mode, vegetation_iou=vegetation_iou)
|
||||
roots[(candidate, mode)] = root
|
||||
result_root = seal_vegetation_shadow_lab(
|
||||
ddrnet_goose_root=roots[("ddrnet", "goose")],
|
||||
ppliteseg_goose_root=roots[("ppliteseg", "goose")],
|
||||
ddrnet_ravnoves_root=roots[("ddrnet", "ravnoves")],
|
||||
ppliteseg_ravnoves_root=roots[("ppliteseg", "ravnoves")],
|
||||
output_root=tmp_path / "results",
|
||||
)
|
||||
manifest = json.loads((result_root / "result.json").read_text("utf-8"))
|
||||
assert manifest["decision"]["selected_candidate"] == "ddrnet"
|
||||
assert manifest["ground_truth"] is False
|
||||
assert manifest["authority"]["commands_enabled"] is False
|
||||
assert manifest["authority"]["navigation_or_safety_accepted"] is False
|
||||
assert len(manifest["catalogs"]["ravnoves"]) == 12
|
||||
assert len(manifest["catalogs"]["goose"]) == 12
|
||||
assert len(manifest["artifacts"]) == 124
|
||||
assert "all_classes" not in manifest["metrics"]["candidates"]["ddrnet"]["validation_metrics"]
|
||||
assert (result_root / "result.json").stat().st_size <= 64 * 1024
|
||||
|
||||
registry = LaboratoryEvidenceRegistry.from_directory(REPOSITORY_ROOT / "config/laboratories")
|
||||
definition = next(
|
||||
row for row in registry.definitions if row.work_id == "lab-v1-vegetation-shadow"
|
||||
)
|
||||
proof = verify_laboratory_evidence_result(definition, result_root)
|
||||
assert proof["result_id"] == result_root.name
|
||||
assert proof["artifact_count"] == 124
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(build_vegetation_shadow_lab_router(root_provider=lambda: result_root.parent))
|
||||
client = TestClient(app)
|
||||
response = client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["access"] == "read-only"
|
||||
asset_path = manifest["catalogs"]["ravnoves"][0]["assets"]["offroad"]["path"]
|
||||
asset = client.get(
|
||||
f"/api/v1/laboratory/vegetation-shadow/{result_root.name}/assets/{asset_path}"
|
||||
)
|
||||
assert asset.status_code == 200
|
||||
assert asset.headers["cache-control"].endswith("immutable")
|
||||
|
||||
(result_root / asset_path).write_bytes(b"tampered")
|
||||
assert (
|
||||
client.get(f"/api/v1/laboratory/vegetation-shadow/{result_root.name}").status_code
|
||||
== 503
|
||||
)
|
||||
Reference in New Issue
Block a user