diff --git a/config/perception/lab-v1-vegetation-mission-policy-v1.json b/config/perception/lab-v1-vegetation-mission-policy-v1.json new file mode 100644 index 0000000..76beb76 --- /dev/null +++ b/config/perception/lab-v1-vegetation-mission-policy-v1.json @@ -0,0 +1,108 @@ +{ + "schema_version": "missioncore.vegetation-mission-policy/v1", + "profile_id": "lab-v1-vegetation-mission-policy/v1", + "status": "planning-shadow-only-unqualified", + "vehicle_profile": { + "path": "config/perception/m49-physical-safety-shadow-v0.json", + "profile_id": "m49-mission-core-rover-physical-safety-shadow/v0" + }, + "actions": [ + "ALLOW", + "HIGH_COST", + "NO_GO" + ], + "precedence": [ + "hard_safety_interlock", + "explicit_mission_rule", + "selected_preset_default" + ], + "material_classes": [ + "hard_surface", + "bare_soil", + "grass", + "fern", + "herbaceous_vegetation", + "cultivated_vegetation", + "woody_shrub", + "tree_or_trunk", + "vegetation_unknown" + ], + "mission_overridable_materials": [ + "grass", + "fern", + "herbaceous_vegetation", + "cultivated_vegetation" + ], + "presets": { + "urban": { + "hard_surface": "ALLOW", + "bare_soil": "HIGH_COST", + "grass": "NO_GO", + "fern": "NO_GO", + "herbaceous_vegetation": "NO_GO", + "cultivated_vegetation": "NO_GO", + "woody_shrub": "NO_GO", + "tree_or_trunk": "NO_GO", + "vegetation_unknown": "NO_GO" + }, + "rural": { + "hard_surface": "ALLOW", + "bare_soil": "ALLOW", + "grass": "HIGH_COST", + "fern": "HIGH_COST", + "herbaceous_vegetation": "HIGH_COST", + "cultivated_vegetation": "NO_GO", + "woody_shrub": "NO_GO", + "tree_or_trunk": "NO_GO", + "vegetation_unknown": "NO_GO" + }, + "offroad": { + "hard_surface": "ALLOW", + "bare_soil": "ALLOW", + "grass": "HIGH_COST", + "fern": "HIGH_COST", + "herbaceous_vegetation": "HIGH_COST", + "cultivated_vegetation": "NO_GO", + "woody_shrub": "NO_GO", + "tree_or_trunk": "NO_GO", + "vegetation_unknown": "NO_GO" + } + }, + "evidence_policy": { + "policy_eligible_states": [ + "SUPPORTED_GROUND", + "VEGETATION_POTENTIALLY_TRAVERSABLE" + ], + "force_no_go_states": [ + "UNOBSERVED", + "NEGATIVE_OR_UNSUPPORTED", + "RIGID_OR_UNKNOWN_OBSTACLE", + "VEGETATION_UNKNOWN", + "VEGETATION_WITH_RIGID_GEOMETRY" + ], + "force_no_go_materials": [ + "woody_shrub", + "tree_or_trunk", + "vegetation_unknown" + ], + "missing_material_is": "vegetation_unknown", + "missing_evidence_is": "UNOBSERVED", + "camera_semantics_can_clear_rigid_geometry": false, + "mission_rule_can_override_hard_safety_interlock": false + }, + "mission_configuration_contract": { + "preset_is_required": true, + "explicit_rules_are_sparse_overrides": true, + "preset_may_be_suggested_automatically": true, + "automatic_preset_switch_requires_operator_opt_in": true, + "automatic_switch_may_weaken_explicit_mission_rule": false, + "operator_must_review_effective_rules_before_start": true + }, + "authority": { + "perception_selects_mission_action": false, + "preset_grants_actuation": false, + "navigation_accepted": false, + "safety_accepted": false, + "actuation_accepted": false + } +} diff --git a/experiments/perception/worker/Invoke-LabV1VegetationAssetAdmission.ps1 b/experiments/perception/worker/Invoke-LabV1VegetationAssetAdmission.ps1 new file mode 100644 index 0000000..8af1c6d --- /dev/null +++ b/experiments/perception/worker/Invoke-LabV1VegetationAssetAdmission.ps1 @@ -0,0 +1,492 @@ +[CmdletBinding()] +param( + [ValidateSet("Start", "Run", "Status", "Manifest", "VerifyModels", "Verify")] + [string]$Mode = "Status", + + [string]$Root = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27", + + [int]$DownloadLimitMiB = 30 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$canonicalPrefix = "D:\NDC_MISSIONCORE\datasets\" +if (-not $Root.StartsWith($canonicalPrefix, [System.StringComparison]::OrdinalIgnoreCase)) { + throw "LAB V1 assets must stay under $canonicalPrefix" +} +if ($DownloadLimitMiB -lt 1 -or $DownloadLimitMiB -gt 100) { + throw "DownloadLimitMiB must be between 1 and 100" +} + +$stateRoot = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation" +$toolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\aria2-1.37.0" +$ariaArchive = Join-Path $toolRoot "aria2-1.37.0-win-64bit-build1.zip" +$ariaExecutable = Join-Path $toolRoot "aria2-1.37.0-win-64bit-build1\aria2c.exe" +$ariaUrl = "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0-win-64bit-build1.zip" +$taskName = "MissionCore-LabV1-VegetationAssets" +$stagePath = Join-Path $stateRoot "download-stage.json" +$assetManifestPath = Join-Path $stateRoot "asset-manifest.json" +$verificationPath = Join-Path $stateRoot "verification.jsonl" +$bulkVerificationPath = Join-Path $stateRoot "bulk-verification.json" +$gooseInputPath = Join-Path $stateRoot "goose-and-models.input.txt" +$wildScenesInputPath = Join-Path $stateRoot "wildscenes-61541v003.input.txt" +$orfdTorrentPath = Join-Path $stateRoot "orfd-ec5ccf4b8e49271ee3b63660383facf43063f2f2.torrent" +$wildScenesDataFileCount = 76525 +$wildScenesMetadataFileCount = 4 +$wildScenesInputEntryCount = $wildScenesDataFileCount + $wildScenesMetadataFileCount +$orfdTorrentExpectedBytes = 4918171 +$orfdTorrentSha256 = "b886dc76d169c924d619ff07baf8a0cd2364ca6f9c4b36d8777f2971fd78bc92" +$orfdPublishedFileCount = 70983 +$orfdPublishedSizeBytes = 45941056780 +$wildScenesPublishedSizeBytes = 196358841020 + +$assets = @( + [pscustomobject]@{ group = "models"; id = "goose-ppliteseg-category-512"; url = "https://goose-dataset.de/models/ppliteseg_category_512.pth"; relative_path = "models\goose\ppliteseg_category_512.pth"; expected_bytes = $null }, + [pscustomobject]@{ group = "models"; id = "goose-ppliteseg-class-512"; url = "https://goose-dataset.de/models/ppliteseg_class_512.pth"; relative_path = "models\goose\ppliteseg_class_512.pth"; expected_bytes = $null }, + [pscustomobject]@{ group = "models"; id = "goose-ddrnet-category-512"; url = "https://goose-dataset.de/models/ddrnet_category_512.pth"; relative_path = "models\goose\ddrnet_category_512.pth"; expected_bytes = $null }, + [pscustomobject]@{ group = "models"; id = "goose-ddrnet-class-512"; url = "https://goose-dataset.de/models/ddrnet_class_512.pth"; relative_path = "models\goose\ddrnet_class_512.pth"; expected_bytes = $null }, + [pscustomobject]@{ group = "models"; id = "rellis-hrnet-ocr"; url = "https://drive.usercontent.google.com/download?id=137Lfw6HcDmdEReu_R7Q_I-zmRvvqFys3&export=download&confirm=t"; relative_path = "models\rellis-3d\hrnet_best.zip"; expected_bytes = 788183119 }, + [pscustomobject]@{ group = "models"; id = "rellis-gscnn"; url = "https://drive.usercontent.google.com/download?id=1Z8OlstkdzDrY9k-yxMQmVB192ac8j4MD&export=download&confirm=t"; relative_path = "models\rellis-3d\gscnn_best.pth"; expected_bytes = 1098515419 }, + [pscustomobject]@{ group = "models"; id = "rellis-salsanext"; url = "https://drive.usercontent.google.com/download?id=1DxuzlnFKnU8EpSuODRywJUrJlieUUheg&export=download&confirm=t"; relative_path = "models\rellis-3d\salsanext_best.zip"; expected_bytes = 164312336 }, + [pscustomobject]@{ group = "models"; id = "rellis-kpconv"; url = "https://drive.usercontent.google.com/download?id=1Exrt4yWDhgucx_vr08hAuXTcaLUcpXXm&export=download&confirm=t"; relative_path = "models\rellis-3d\kpconv_best.zip"; expected_bytes = 1522250650 }, + [pscustomobject]@{ group = "models"; id = "wildscenes-models"; url = "https://www.dropbox.com/scl/fo/1za8yjfj0dl8fsddbl9ns/AAZ4xc2PkwcPc14sjY_GhVA?rlkey=3yqq5lpiicjtqqau4cljl66y5&st=fwhd1l86&dl=1"; relative_path = "models\wildscenes\WildScenes-Models.zip"; expected_bytes = 8637507567 }, + [pscustomobject]@{ group = "goose"; id = "goose-2d-train"; url = "https://goose-dataset.de/storage/goose_2d_train.zip"; relative_path = "goose-2d\archives\goose_2d_train.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-2d-val"; url = "https://goose-dataset.de/storage/goose_2d_val.zip"; relative_path = "goose-2d\archives\goose_2d_val.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-2d-test"; url = "https://goose-dataset.de/storage/goose_2d_test.zip"; relative_path = "goose-2d\archives\goose_2d_test.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-3d-train"; url = "https://goose-dataset.de/storage/goose_3d_train.zip"; relative_path = "goose-3d\archives\goose_3d_train.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-3d-test"; url = "https://goose-dataset.de/storage/goose_3d_test.zip"; relative_path = "goose-3d\archives\goose_3d_test.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-2d-train"; url = "https://goose-dataset.de/storage/gooseEx_2d_train.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_train.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-2d-val"; url = "https://goose-dataset.de/storage/gooseEx_2d_val.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_val.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-2d-test"; url = "https://goose-dataset.de/storage/gooseEx_2d_test.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_test.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-3d-train"; url = "https://goose-dataset.de/storage/gooseEx_3d_train.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_train.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-3d-val"; url = "https://goose-dataset.de/storage/gooseEx_3d_val.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_val.zip"; expected_bytes = $null }, + [pscustomobject]@{ group = "goose"; id = "goose-ex-3d-test"; url = "https://goose-dataset.de/storage/gooseEx_3d_test.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_test.zip"; expected_bytes = $null } +) + +function Write-JsonFile { + param([string]$Path, [object]$Value) + $parent = Split-Path -Parent $Path + New-Item -ItemType Directory -Force -Path $parent | Out-Null + $json = $Value | ConvertTo-Json -Depth 12 + [System.IO.File]::WriteAllText($Path, $json + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false)) +} + +function Set-Stage { + param([string]$Stage, [string]$Status, [string]$Detail) + Write-JsonFile -Path $stagePath -Value ([ordered]@{ + schema_version = "missioncore.lab-v1-asset-download-stage/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + worker_id = "worker-006" + stage = $Stage + status = $Status + detail = $Detail + }) +} + +function Invoke-CurlDownload { + param([string]$Url, [string]$Destination) + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Destination) | Out-Null + & curl.exe --location --fail --retry 8 --retry-all-errors --continue-at - --output $Destination $Url + if ($LASTEXITCODE -ne 0) { + throw "curl failed for $Url with exit code $LASTEXITCODE" + } +} + +function Test-AriaInputList { + param([string]$Path, [int]$ExpectedEntries) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $false + } + + $urlCount = 0 + $dirCount = 0 + foreach ($line in [System.IO.File]::ReadLines($Path)) { + if ($line.StartsWith("https://", [System.StringComparison]::OrdinalIgnoreCase)) { + $urlCount++ + } + elseif ($line.StartsWith(" dir=", [System.StringComparison]::Ordinal)) { + $dirCount++ + } + } + return $urlCount -eq $ExpectedEntries -and $dirCount -eq $ExpectedEntries +} + +function Test-FileIntegrity { + param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256) + $file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue + if ($null -eq $file -or $file.Length -ne $ExpectedBytes) { + return $false + } + $actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() + return $actualSha256 -eq $ExpectedSha256 +} + +function Initialize-Aria2 { + New-Item -ItemType Directory -Force -Path $toolRoot | Out-Null + if (-not (Test-Path -LiteralPath $ariaExecutable -PathType Leaf)) { + Invoke-CurlDownload -Url $ariaUrl -Destination $ariaArchive + Expand-Archive -LiteralPath $ariaArchive -DestinationPath $toolRoot -Force + } + if (-not (Test-Path -LiteralPath $ariaExecutable -PathType Leaf)) { + throw "aria2 executable was not installed at $ariaExecutable" + } +} + +function Write-DirectInput { + $lines = [System.Collections.Generic.List[string]]::new() + foreach ($asset in $assets) { + $destination = Join-Path $Root $asset.relative_path + New-Item -ItemType Directory -Force -Path (Split-Path -Parent $destination) | Out-Null + $lines.Add([string]$asset.url) + $lines.Add(" dir=" + (Split-Path -Parent $destination)) + $lines.Add(" out=" + (Split-Path -Leaf $destination)) + } + [System.IO.File]::WriteAllLines($gooseInputPath, $lines, [System.Text.UTF8Encoding]::new($false)) +} + +function Invoke-AriaInput { + param([string]$InputPath, [int]$ConcurrentDownloads, [int]$ConnectionsPerServer) + $downloadLimit = "{0}M" -f $DownloadLimitMiB + & $ariaExecutable ` + --input-file=$InputPath ` + --continue=true ` + --max-concurrent-downloads=$ConcurrentDownloads ` + --max-connection-per-server=$ConnectionsPerServer ` + --split=$ConnectionsPerServer ` + --min-split-size=16M ` + "--max-overall-download-limit=$downloadLimit" ` + --file-allocation=none ` + --disk-cache=64M ` + --auto-file-renaming=false ` + --allow-overwrite=false ` + --retry-wait=5 ` + --max-tries=0 ` + --summary-interval=60 ` + --console-log-level=notice + if ($LASTEXITCODE -ne 0) { + throw "aria2 input run failed for $InputPath with exit code $LASTEXITCODE" + } +} + +function Get-DownloadTask { + return Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue +} + +function Get-AssetSnapshot { + $rows = foreach ($asset in $assets) { + $path = Join-Path $Root $asset.relative_path + $file = Get-Item -LiteralPath $path -ErrorAction SilentlyContinue + $partial = Get-Item -LiteralPath ($path + ".aria2") -ErrorAction SilentlyContinue + [ordered]@{ + id = $asset.id + group = $asset.group + relative_path = $asset.relative_path + status = if ($null -ne $file -and $null -eq $partial) { "downloaded" } elseif ($null -ne $file) { "partial" } else { "pending" } + size_bytes = if ($null -ne $file) { $file.Length } else { 0 } + expected_bytes = $asset.expected_bytes + } + } + return @($rows) +} + +function Invoke-Verify { + param([ValidateSet("All", "Models")][string]$Scope = "All") + New-Item -ItemType Directory -Force -Path $stateRoot | Out-Null + $writer = [System.IO.StreamWriter]::new($verificationPath, $false, [System.Text.UTF8Encoding]::new($false)) + try { + foreach ($asset in $assets) { + if ($Scope -eq "Models" -and $asset.group -ne "models") { + continue + } + $path = Join-Path $Root $asset.relative_path + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + continue + } + if (Test-Path -LiteralPath ($path + ".aria2") -PathType Leaf) { + throw "Cannot verify partial asset $($asset.id)" + } + $file = Get-Item -LiteralPath $path + $hash = Get-FileHash -LiteralPath $path -Algorithm SHA256 + $sizeMatches = $null + if ($null -ne $asset.expected_bytes) { + $sizeMatches = $file.Length -eq [long]$asset.expected_bytes + if (-not $sizeMatches) { + throw "Size mismatch for $($asset.id): expected $($asset.expected_bytes), got $($file.Length)" + } + } + $row = [ordered]@{ + schema_version = "missioncore.lab-v1-asset-verification/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + id = $asset.id + relative_path = $asset.relative_path + size_bytes = $file.Length + expected_bytes = $asset.expected_bytes + size_matches = $sizeMatches + sha256 = $hash.Hash.ToLowerInvariant() + source_url = $asset.url + } + $writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 8)) + $writer.Flush() + } + $offnetRoot = Join-Path $Root "models\orfd-offnet" + if (Test-Path -LiteralPath $offnetRoot -PathType Container) { + Get-ChildItem -LiteralPath $offnetRoot -File -Recurse | Sort-Object FullName | ForEach-Object { + $row = [ordered]@{ + schema_version = "missioncore.lab-v1-asset-verification/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + id = "orfd-offnet/" + $_.FullName.Substring($offnetRoot.Length + 1).Replace("\", "/") + relative_path = $_.FullName.Substring($Root.Length + 1) + size_bytes = $_.Length + expected_bytes = $null + size_matches = $null + sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + source_url = "https://drive.google.com/drive/folders/1lnm2M1HEkVs9W3-FSEX3ddE9GYz4rqCU" + } + $writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 8)) + $writer.Flush() + } + } + } + finally { + $writer.Dispose() + } +} + +function Measure-PayloadTree { + param([string]$Path) + [long]$bytes = 0 + [long]$fileCount = 0 + [long]$partialCount = 0 + Get-ChildItem -LiteralPath $Path -File -Recurse | ForEach-Object { + if ($_.Name.EndsWith(".aria2", [System.StringComparison]::OrdinalIgnoreCase)) { + $partialCount++ + } + else { + $fileCount++ + $bytes += $_.Length + } + } + return [ordered]@{ file_count = $fileCount; size_bytes = $bytes; partial_count = $partialCount } +} + +function Invoke-BulkVerify { + $wildScenesRoot = Join-Path $Root "wildscenes" + $orfdRoot = Join-Path $Root "orfd" + $wildScenes = Measure-PayloadTree -Path $wildScenesRoot + $orfd = Measure-PayloadTree -Path $orfdRoot + if ( + $wildScenes.file_count -ne $wildScenesInputEntryCount -or + $wildScenes.size_bytes -ne $wildScenesPublishedSizeBytes -or + $wildScenes.partial_count -ne 0 + ) { + throw "WildScenes tree failed count/size/partial verification" + } + if ( + $orfd.file_count -ne $orfdPublishedFileCount -or + $orfd.size_bytes -ne $orfdPublishedSizeBytes -or + $orfd.partial_count -ne 0 + ) { + throw "ORFD tree failed count/size/partial verification" + } + Write-JsonFile -Path $bulkVerificationPath -Value ([ordered]@{ + schema_version = "missioncore.lab-v1-bulk-verification/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + worker_id = "worker-006" + wildscenes = $wildScenes + orfd = $orfd + canonical_triton_action = "none" + }) +} + +function Write-AssetManifest { + if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) { + throw "WildScenes input list is not admitted" + } + if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) { + throw "ORFD torrent source is not admitted" + } + Write-JsonFile -Path $assetManifestPath -Value ([ordered]@{ + schema_version = "missioncore.lab-v1-asset-manifest/v1" + observed_at_utc = [DateTime]::UtcNow.ToString("o") + worker_id = "worker-006" + storage_policy = "worker-d-only" + root = $Root + runtime_models_mutated = $false + canonical_triton_action = "none" + existing_admitted_assets = @( + [ordered]@{ id = "goose-3d-val"; path = "D:\NDC_MISSIONCORE\datasets\goose-3d\v2025-08-22\archives\goose_3d_val.zip"; sha256 = "0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f" }, + [ordered]@{ id = "rellis-3d-v1.1"; path = "D:\NDC_MISSIONCORE\datasets\rellis-3d\v1.1"; admission_identity = "a3f3f161a5a7edccdf66cea75ecf9004b8f6a895282faabe661c075c15a82774" } + ) + direct_assets = $assets + wildscenes = [ordered]@{ collection_id = 63154; version = 3; published_size_bytes = $wildScenesPublishedSizeBytes; data_file_count = $wildScenesDataFileCount; metadata_file_count = $wildScenesMetadataFileCount; input_entry_count = $wildScenesInputEntryCount; url_list = $wildScenesInputPath; url_list_sha256 = (Get-FileHash -LiteralPath $wildScenesInputPath -Algorithm SHA256).Hash.ToLowerInvariant() } + orfd = [ordered]@{ info_hash = "ec5ccf4b8e49271ee3b63660383facf43063f2f2"; published_size_bytes = $orfdPublishedSizeBytes; published_file_count = $orfdPublishedFileCount; torrent = $orfdTorrentPath; torrent_size_bytes = $orfdTorrentExpectedBytes; torrent_sha256 = $orfdTorrentSha256 } + offnet_weights = [ordered]@{ source_folder = "https://drive.google.com/drive/folders/1lnm2M1HEkVs9W3-FSEX3ddE9GYz4rqCU"; relative_path = "models\orfd-offnet"; downloader = "gdown==5.2.0"; isolated_image = "nvcr.io/nvidia/tritonserver@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"; runtime_models_mutated = $false } + }) +} + +New-Item -ItemType Directory -Force -Path $Root, $stateRoot | Out-Null + +switch ($Mode) { + "Start" { + $existingTask = Get-DownloadTask + if ($null -ne $existingTask -and $existingTask.State -eq "Running") { + throw "LAB V1 downloader task is already running" + } + + $drive = Get-PSDrive -Name D + $currentUsage = Measure-PayloadTree -Path $Root + [long]$remainingPayloadBytes = 360GB - [long]$currentUsage.size_bytes + if ($remainingPayloadBytes -lt 0) { + $remainingPayloadBytes = 0 + } + [long]$requiredFreeBytes = 10GB + $remainingPayloadBytes + if ($drive.Free -lt $requiredFreeBytes) { + throw "Worker D: has only $([math]::Round($drive.Free / 1GB, 2)) GiB free; $([math]::Round($requiredFreeBytes / 1GB, 2)) GiB is required for remaining payload plus reserve" + } + + Set-Stage -Stage "preflight" -Status "running" -Detail "Installing bounded downloader and generating immutable source lists" + Initialize-Aria2 + Write-DirectInput + + if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) { + Invoke-CurlDownload -Url "https://data.csiro.au/dap/ws/v2/collections/63154/files/list" -Destination $wildScenesInputPath + } + if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) { + throw "WildScenes input list is incomplete: expected $wildScenesInputEntryCount URL/dir entries" + } + if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) { + Remove-Item -LiteralPath $orfdTorrentPath -Force -ErrorAction SilentlyContinue + Invoke-CurlDownload -Url "https://academictorrents.com/download/ec5ccf4b8e49271ee3b63660383facf43063f2f2.torrent" -Destination $orfdTorrentPath + } + if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) { + throw "ORFD torrent source failed size/SHA-256 verification" + } + + Write-AssetManifest + + $powershell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe" + $arguments = @( + "-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", + "-File", "`"$PSCommandPath`"", + "-Mode", "Run", + "-Root", "`"$Root`"", + "-DownloadLimitMiB", $DownloadLimitMiB + ) -join " " + $action = New-ScheduledTaskAction -Execute $powershell -Argument $arguments -WorkingDirectory (Split-Path -Parent $PSCommandPath) + $principal = New-ScheduledTaskPrincipal -UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) -LogonType Interactive -RunLevel Limited + $trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30)) + $settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -ExecutionTimeLimit ([TimeSpan]::FromHours(72)) + Register-ScheduledTask -TaskName $taskName -Action $action -Principal $principal -Trigger $trigger -Settings $settings -Description "Mission Core LAB V1 isolated vegetation dataset and model admission." -Force | Out-Null + Start-ScheduledTask -TaskName $taskName + Set-Stage -Stage "queued" -Status "running" -Detail "Bounded downloader started as scheduled task $taskName" + [ordered]@{ + ok = $true + mode = $Mode + task_name = $taskName + task_state = (Get-ScheduledTask -TaskName $taskName).State.ToString() + root = $Root + d_free_gib_before = [math]::Round($drive.Free / 1GB, 2) + download_limit_mib = $DownloadLimitMiB + canonical_triton_action = "none" + } | ConvertTo-Json -Depth 6 + } + "Run" { + try { + Initialize-Aria2 + Set-Stage -Stage "models-and-goose" -Status "running" -Detail "Downloading official pretrained weights and prepared GOOSE/GOOSE-Ex splits" + Invoke-AriaInput -InputPath $gooseInputPath -ConcurrentDownloads 2 -ConnectionsPerServer 4 + + Set-Stage -Stage "orfd" -Status "running" -Detail "Downloading the 45.94 GB Academic Torrents ORFD release" + $orfdRoot = Join-Path $Root "orfd" + $downloadLimit = "{0}M" -f $DownloadLimitMiB + New-Item -ItemType Directory -Force -Path $orfdRoot | Out-Null + & $ariaExecutable ` + --dir=$orfdRoot ` + --continue=true ` + --seed-time=0 ` + "--max-overall-download-limit=$downloadLimit" ` + --file-allocation=none ` + --disk-cache=64M ` + --summary-interval=60 ` + --console-log-level=notice ` + $orfdTorrentPath + if ($LASTEXITCODE -ne 0) { + throw "ORFD torrent run failed with exit code $LASTEXITCODE" + } + + Set-Stage -Stage "wildscenes" -Status "running" -Detail "Downloading the complete public WildScenes v3 file list" + $wildScenesRoot = Join-Path $Root "wildscenes" + New-Item -ItemType Directory -Force -Path $wildScenesRoot | Out-Null + Push-Location $wildScenesRoot + try { + Invoke-AriaInput -InputPath $wildScenesInputPath -ConcurrentDownloads 8 -ConnectionsPerServer 2 + } + finally { + Pop-Location + } + + Set-Stage -Stage "verification" -Status "running" -Detail "Computing SHA-256 for direct archive and model assets" + Invoke-Verify + Invoke-BulkVerify + Set-Stage -Stage "complete" -Status "complete" -Detail "All scheduled assets downloaded and verified by direct hashes or published tree totals" + } + catch { + Set-Stage -Stage "failed" -Status "failed" -Detail $_.Exception.Message + throw + } + } + "Status" { + $task = Get-DownloadTask + $stage = if (Test-Path -LiteralPath $stagePath -PathType Leaf) { Get-Content -LiteralPath $stagePath -Raw | ConvertFrom-Json } else { $null } + $snapshot = Get-AssetSnapshot + $drive = Get-PSDrive -Name D + [ordered]@{ + ok = $true + mode = $Mode + running = $null -ne $task -and $task.State -eq "Running" + task_name = $taskName + task_state = if ($null -ne $task) { $task.State.ToString() } else { $null } + stage = $stage + direct_assets = $snapshot + downloaded_direct_asset_count = @($snapshot | Where-Object { $_.status -eq "downloaded" }).Count + partial_direct_asset_count = @($snapshot | Where-Object { $_.status -eq "partial" }).Count + d_free_gib = [math]::Round($drive.Free / 1GB, 2) + canonical_triton_action = "none" + } | ConvertTo-Json -Depth 10 + } + "Manifest" { + Write-AssetManifest + [ordered]@{ + ok = $true + mode = $Mode + manifest_path = $assetManifestPath + manifest_sha256 = (Get-FileHash -LiteralPath $assetManifestPath -Algorithm SHA256).Hash.ToLowerInvariant() + canonical_triton_action = "none" + } | ConvertTo-Json -Depth 6 + } + "VerifyModels" { + Invoke-Verify -Scope "Models" + [ordered]@{ + ok = $true + mode = $Mode + verification_path = $verificationPath + verification_sha256 = (Get-FileHash -LiteralPath $verificationPath -Algorithm SHA256).Hash.ToLowerInvariant() + verified_asset_count = (Get-Content -LiteralPath $verificationPath).Count + canonical_triton_action = "none" + } | ConvertTo-Json -Depth 6 + } + "Verify" { + Set-Stage -Stage "verification" -Status "running" -Detail "Manual direct-asset and published-tree verification requested" + Invoke-Verify + Invoke-BulkVerify + Set-Stage -Stage "verification" -Status "complete" -Detail "Direct assets hashed and bulk trees match published totals" + [ordered]@{ + ok = $true + mode = $Mode + verification_path = $verificationPath + bulk_verification_path = $bulkVerificationPath + verified_asset_count = (Get-Content -LiteralPath $verificationPath).Count + } | ConvertTo-Json -Depth 6 + } +} diff --git a/src/k1link/laboratory/vegetation_mission_policy.py b/src/k1link/laboratory/vegetation_mission_policy.py new file mode 100644 index 0000000..a56366b --- /dev/null +++ b/src/k1link/laboratory/vegetation_mission_policy.py @@ -0,0 +1,219 @@ +"""Validate and resolve the planning-only vegetation mission-policy contract.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +SCHEMA: Final = "missioncore.vegetation-mission-policy/v1" +VEHICLE_SCHEMA: Final = "missioncore.m49-physical-safety-shadow-profile/v1" +_MAX_JSON_BYTES: Final = 1024 * 1024 + + +class VegetationMissionPolicyError(RuntimeError): + """The vegetation mission policy is absent, inconsistent, or unsafe.""" + + +@dataclass(frozen=True, slots=True) +class TerrainPolicyDecision: + preset_id: str + material_class: str + evidence_state: str + requested_action: str + effective_action: str + policy_source: str + safety_reason: str | None + actuation_authority: bool = False + + +def _json(path: Path, label: str) -> dict[str, Any]: + if path.is_symlink() or not path.is_file() or path.stat().st_size > _MAX_JSON_BYTES: + raise VegetationMissionPolicyError(f"{label} is unavailable") + try: + value = json.loads(path.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise VegetationMissionPolicyError(f"{label} is invalid") from error + if not isinstance(value, dict): + raise VegetationMissionPolicyError(f"{label} is invalid") + return value + + +def _string_set(value: object, label: str) -> set[str]: + if ( + not isinstance(value, list) + or not value + or any(not isinstance(item, str) or not item for item in value) + or len(set(value)) != len(value) + ): + raise VegetationMissionPolicyError(f"{label} is invalid") + return set(value) + + +def _validate_vehicle_profile(policy: dict[str, Any], repository_root: Path) -> None: + reference = policy.get("vehicle_profile") + if not isinstance(reference, dict) or not isinstance(reference.get("path"), str): + raise VegetationMissionPolicyError("vehicle profile reference is invalid") + relative = Path(reference["path"]) + if relative.is_absolute() or ".." in relative.parts: + raise VegetationMissionPolicyError("vehicle profile path escapes the repository") + path = (repository_root / relative).resolve() + try: + path.relative_to(repository_root.resolve()) + except ValueError as error: + raise VegetationMissionPolicyError("vehicle profile path escapes the repository") from error + vehicle_profile = _json(path, "vehicle profile") + vehicle = vehicle_profile.get("vehicle") + authority = vehicle_profile.get("authority") + if ( + vehicle_profile.get("schema_version") != VEHICLE_SCHEMA + or vehicle_profile.get("profile_id") != reference.get("profile_id") + or not isinstance(vehicle, dict) + or not isinstance(authority, dict) + or authority.get("commands_enabled") is not False + or authority.get("actuation_accepted") is not False + ): + raise VegetationMissionPolicyError("vehicle profile authority changed") + expected_vehicle = { + "body_length_m": 1.0, + "body_width_m": 0.8, + "body_height_m": 0.4, + "ground_clearance_m": 0.15, + "nominal_speed_mps": 0.2777777778, + "maximum_operating_speed_mps": 0.5555555556, + } + if any(vehicle.get(key) != expected for key, expected in expected_vehicle.items()): + raise VegetationMissionPolicyError("vehicle planning facts changed") + + +def load_vegetation_mission_policy( + path: Path, + *, + repository_root: Path | None = None, +) -> dict[str, Any]: + """Load the profile and fail closed if precedence or safety locks drift.""" + + policy = _json(path, "vegetation mission policy") + if ( + policy.get("schema_version") != SCHEMA + or policy.get("status") != "planning-shadow-only-unqualified" + or policy.get("precedence") + != [ + "hard_safety_interlock", + "explicit_mission_rule", + "selected_preset_default", + ] + ): + raise VegetationMissionPolicyError("policy identity or precedence changed") + + actions = _string_set(policy.get("actions"), "actions") + if actions != {"ALLOW", "HIGH_COST", "NO_GO"}: + raise VegetationMissionPolicyError("actions changed") + materials = _string_set(policy.get("material_classes"), "material classes") + overridable = _string_set( + policy.get("mission_overridable_materials"), "mission-overridable materials" + ) + if not overridable < materials: + raise VegetationMissionPolicyError("mission-overridable materials are invalid") + + presets = policy.get("presets") + if not isinstance(presets, dict) or set(presets) != {"urban", "rural", "offroad"}: + raise VegetationMissionPolicyError("presets changed") + for preset_id, rules in presets.items(): + if ( + not isinstance(rules, dict) + or set(rules) != materials + or any(action not in actions for action in rules.values()) + ): + raise VegetationMissionPolicyError(f"preset {preset_id} is invalid") + + evidence = policy.get("evidence_policy") + authority = policy.get("authority") + mission_contract = policy.get("mission_configuration_contract") + if ( + not isinstance(evidence, dict) + or not isinstance(authority, dict) + or not isinstance(mission_contract, dict) + ): + raise VegetationMissionPolicyError("policy boundaries are invalid") + eligible = _string_set(evidence.get("policy_eligible_states"), "eligible evidence states") + force_states = _string_set(evidence.get("force_no_go_states"), "force-no-go states") + force_materials = _string_set( + evidence.get("force_no_go_materials"), "force-no-go materials" + ) + if ( + eligible & force_states + or not force_materials <= materials + or overridable & force_materials + or evidence.get("missing_material_is") not in force_materials + or evidence.get("missing_evidence_is") not in force_states + or evidence.get("camera_semantics_can_clear_rigid_geometry") is not False + or evidence.get("mission_rule_can_override_hard_safety_interlock") is not False + or mission_contract.get("explicit_rules_are_sparse_overrides") is not True + or mission_contract.get("automatic_switch_may_weaken_explicit_mission_rule") is not False + or mission_contract.get("operator_must_review_effective_rules_before_start") is not True + or any(authority.get(key) is not False for key in authority) + ): + raise VegetationMissionPolicyError("fail-closed policy boundary changed") + + root = repository_root if repository_root is not None else path.resolve().parents[2] + _validate_vehicle_profile(policy, root) + return policy + + +def resolve_terrain_policy( + policy: dict[str, Any], + *, + preset_id: str, + material_class: str | None, + evidence_state: str | None, + mission_rules: Mapping[str, str] | None = None, +) -> TerrainPolicyDecision: + """Resolve preset and mission intent while retaining hard safety precedence.""" + + presets = policy["presets"] + if preset_id not in presets: + raise VegetationMissionPolicyError(f"unknown preset: {preset_id}") + evidence = policy["evidence_policy"] + material = material_class or evidence["missing_material_is"] + state = evidence_state or evidence["missing_evidence_is"] + if material not in policy["material_classes"]: + material = evidence["missing_material_is"] + + explicit_rules = dict(mission_rules or {}) + actions = set(policy["actions"]) + overridable = set(policy["mission_overridable_materials"]) + for rule_material, action in explicit_rules.items(): + if rule_material not in overridable: + raise VegetationMissionPolicyError( + f"material cannot be overridden by a mission: {rule_material}" + ) + if action not in actions: + raise VegetationMissionPolicyError(f"unknown mission action: {action}") + + if material in explicit_rules: + requested_action = explicit_rules[material] + source = "explicit_mission_rule" + else: + requested_action = presets[preset_id][material] + source = "selected_preset_default" + + reason: str | None = None + if state in evidence["force_no_go_states"]: + reason = f"hard_safety_interlock:{state}" + elif material in evidence["force_no_go_materials"]: + reason = f"hard_safety_interlock:{material}" + elif state not in evidence["policy_eligible_states"]: + reason = "hard_safety_interlock:unrecognized_evidence_state" + + return TerrainPolicyDecision( + preset_id=preset_id, + material_class=material, + evidence_state=state, + requested_action=requested_action, + effective_action="NO_GO" if reason is not None else requested_action, + policy_source=source, + safety_reason=reason, + ) diff --git a/tests/test_vegetation_mission_policy.py b/tests/test_vegetation_mission_policy.py new file mode 100644 index 0000000..a23283e --- /dev/null +++ b/tests/test_vegetation_mission_policy.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from k1link.laboratory.vegetation_mission_policy import ( + VegetationMissionPolicyError, + load_vegetation_mission_policy, + resolve_terrain_policy, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +POLICY_PATH = ( + REPOSITORY_ROOT / "config" / "perception" / "lab-v1-vegetation-mission-policy-v1.json" +) + + +def _policy() -> dict[str, object]: + return load_vegetation_mission_policy(POLICY_PATH, repository_root=REPOSITORY_ROOT) + + +def test_presets_are_defaults_and_explicit_mission_rule_wins() -> None: + policy = _policy() + + urban_default = resolve_terrain_policy( + policy, + preset_id="urban", + material_class="grass", + evidence_state="VEGETATION_POTENTIALLY_TRAVERSABLE", + ) + assert urban_default.requested_action == "NO_GO" + assert urban_default.effective_action == "NO_GO" + assert urban_default.policy_source == "selected_preset_default" + + explicit_allow = resolve_terrain_policy( + policy, + preset_id="urban", + material_class="grass", + evidence_state="VEGETATION_POTENTIALLY_TRAVERSABLE", + mission_rules={"grass": "ALLOW"}, + ) + assert explicit_allow.requested_action == "ALLOW" + assert explicit_allow.effective_action == "ALLOW" + assert explicit_allow.policy_source == "explicit_mission_rule" + assert explicit_allow.actuation_authority is False + + +def test_offroad_grass_is_high_cost_until_mission_changes_it() -> None: + decision = resolve_terrain_policy( + _policy(), + preset_id="offroad", + material_class="grass", + evidence_state="VEGETATION_POTENTIALLY_TRAVERSABLE", + ) + assert decision.requested_action == "HIGH_COST" + assert decision.effective_action == "HIGH_COST" + assert decision.safety_reason is None + + +@pytest.mark.parametrize( + "state", + [ + "UNOBSERVED", + "NEGATIVE_OR_UNSUPPORTED", + "RIGID_OR_UNKNOWN_OBSTACLE", + "VEGETATION_UNKNOWN", + "VEGETATION_WITH_RIGID_GEOMETRY", + ], +) +def test_mission_allow_never_clears_hard_safety_interlock(state: str) -> None: + decision = resolve_terrain_policy( + _policy(), + preset_id="offroad", + material_class="grass", + evidence_state=state, + mission_rules={"grass": "ALLOW"}, + ) + assert decision.requested_action == "ALLOW" + assert decision.effective_action == "NO_GO" + assert decision.safety_reason == f"hard_safety_interlock:{state}" + + +def test_missing_semantics_and_support_fail_closed() -> None: + decision = resolve_terrain_policy( + _policy(), + preset_id="rural", + material_class=None, + evidence_state=None, + ) + assert decision.material_class == "vegetation_unknown" + assert decision.evidence_state == "UNOBSERVED" + assert decision.effective_action == "NO_GO" + + +def test_woody_material_cannot_be_enabled_by_mission() -> None: + with pytest.raises( + VegetationMissionPolicyError, + match="material cannot be overridden by a mission: tree_or_trunk", + ): + resolve_terrain_policy( + _policy(), + preset_id="offroad", + material_class="tree_or_trunk", + evidence_state="SUPPORTED_GROUND", + mission_rules={"tree_or_trunk": "ALLOW"}, + ) + + +def test_unknown_material_is_treated_as_unknown_vegetation() -> None: + decision = resolve_terrain_policy( + _policy(), + preset_id="offroad", + material_class="provider_specific_leaf", + evidence_state="SUPPORTED_GROUND", + ) + assert decision.material_class == "vegetation_unknown" + assert decision.effective_action == "NO_GO" + assert decision.safety_reason == "hard_safety_interlock:vegetation_unknown"