493 lines
26 KiB
PowerShell
493 lines
26 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Start", "Run", "Status", "Manifest", "VerifyModels", "Verify")]
|
|
[string]$Mode = "Status",
|
|
|
|
[string]$Root = "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27",
|
|
|
|
[int]$DownloadLimitMiB = 30
|
|
)
|
|
|
|
Set-StrictMode -Version Latest
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$canonicalPrefix = "D:\NDC_MISSIONCORE\datasets\"
|
|
if (-not $Root.StartsWith($canonicalPrefix, [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
throw "LAB V1 assets must stay under $canonicalPrefix"
|
|
}
|
|
if ($DownloadLimitMiB -lt 1 -or $DownloadLimitMiB -gt 100) {
|
|
throw "DownloadLimitMiB must be between 1 and 100"
|
|
}
|
|
|
|
$stateRoot = "D:\NDC_MISSIONCORE\datasets\state\lab-v1-vegetation"
|
|
$toolRoot = "D:\NDC_MISSIONCORE\datasets\tooling\aria2-1.37.0"
|
|
$ariaArchive = Join-Path $toolRoot "aria2-1.37.0-win-64bit-build1.zip"
|
|
$ariaExecutable = Join-Path $toolRoot "aria2-1.37.0-win-64bit-build1\aria2c.exe"
|
|
$ariaUrl = "https://github.com/aria2/aria2/releases/download/release-1.37.0/aria2-1.37.0-win-64bit-build1.zip"
|
|
$taskName = "MissionCore-LabV1-VegetationAssets"
|
|
$stagePath = Join-Path $stateRoot "download-stage.json"
|
|
$assetManifestPath = Join-Path $stateRoot "asset-manifest.json"
|
|
$verificationPath = Join-Path $stateRoot "verification.jsonl"
|
|
$bulkVerificationPath = Join-Path $stateRoot "bulk-verification.json"
|
|
$gooseInputPath = Join-Path $stateRoot "goose-and-models.input.txt"
|
|
$wildScenesInputPath = Join-Path $stateRoot "wildscenes-61541v003.input.txt"
|
|
$orfdTorrentPath = Join-Path $stateRoot "orfd-ec5ccf4b8e49271ee3b63660383facf43063f2f2.torrent"
|
|
$wildScenesDataFileCount = 76525
|
|
$wildScenesMetadataFileCount = 4
|
|
$wildScenesInputEntryCount = $wildScenesDataFileCount + $wildScenesMetadataFileCount
|
|
$orfdTorrentExpectedBytes = 4918171
|
|
$orfdTorrentSha256 = "b886dc76d169c924d619ff07baf8a0cd2364ca6f9c4b36d8777f2971fd78bc92"
|
|
$orfdPublishedFileCount = 70983
|
|
$orfdPublishedSizeBytes = 45941056780
|
|
$wildScenesPublishedSizeBytes = 196358841020
|
|
|
|
$assets = @(
|
|
[pscustomobject]@{ group = "models"; id = "goose-ppliteseg-category-512"; url = "https://goose-dataset.de/models/ppliteseg_category_512.pth"; relative_path = "models\goose\ppliteseg_category_512.pth"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "models"; id = "goose-ppliteseg-class-512"; url = "https://goose-dataset.de/models/ppliteseg_class_512.pth"; relative_path = "models\goose\ppliteseg_class_512.pth"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "models"; id = "goose-ddrnet-category-512"; url = "https://goose-dataset.de/models/ddrnet_category_512.pth"; relative_path = "models\goose\ddrnet_category_512.pth"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "models"; id = "goose-ddrnet-class-512"; url = "https://goose-dataset.de/models/ddrnet_class_512.pth"; relative_path = "models\goose\ddrnet_class_512.pth"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "models"; id = "rellis-hrnet-ocr"; url = "https://drive.usercontent.google.com/download?id=137Lfw6HcDmdEReu_R7Q_I-zmRvvqFys3&export=download&confirm=t"; relative_path = "models\rellis-3d\hrnet_best.zip"; expected_bytes = 788183119 },
|
|
[pscustomobject]@{ group = "models"; id = "rellis-gscnn"; url = "https://drive.usercontent.google.com/download?id=1Z8OlstkdzDrY9k-yxMQmVB192ac8j4MD&export=download&confirm=t"; relative_path = "models\rellis-3d\gscnn_best.pth"; expected_bytes = 1098515419 },
|
|
[pscustomobject]@{ group = "models"; id = "rellis-salsanext"; url = "https://drive.usercontent.google.com/download?id=1DxuzlnFKnU8EpSuODRywJUrJlieUUheg&export=download&confirm=t"; relative_path = "models\rellis-3d\salsanext_best.zip"; expected_bytes = 164312336 },
|
|
[pscustomobject]@{ group = "models"; id = "rellis-kpconv"; url = "https://drive.usercontent.google.com/download?id=1Exrt4yWDhgucx_vr08hAuXTcaLUcpXXm&export=download&confirm=t"; relative_path = "models\rellis-3d\kpconv_best.zip"; expected_bytes = 1522250650 },
|
|
[pscustomobject]@{ group = "models"; id = "wildscenes-models"; url = "https://www.dropbox.com/scl/fo/1za8yjfj0dl8fsddbl9ns/AAZ4xc2PkwcPc14sjY_GhVA?rlkey=3yqq5lpiicjtqqau4cljl66y5&st=fwhd1l86&dl=1"; relative_path = "models\wildscenes\WildScenes-Models.zip"; expected_bytes = 8637507567 },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-2d-train"; url = "https://goose-dataset.de/storage/goose_2d_train.zip"; relative_path = "goose-2d\archives\goose_2d_train.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-2d-val"; url = "https://goose-dataset.de/storage/goose_2d_val.zip"; relative_path = "goose-2d\archives\goose_2d_val.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-2d-test"; url = "https://goose-dataset.de/storage/goose_2d_test.zip"; relative_path = "goose-2d\archives\goose_2d_test.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-3d-train"; url = "https://goose-dataset.de/storage/goose_3d_train.zip"; relative_path = "goose-3d\archives\goose_3d_train.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-3d-test"; url = "https://goose-dataset.de/storage/goose_3d_test.zip"; relative_path = "goose-3d\archives\goose_3d_test.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-2d-train"; url = "https://goose-dataset.de/storage/gooseEx_2d_train.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_train.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-2d-val"; url = "https://goose-dataset.de/storage/gooseEx_2d_val.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_val.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-2d-test"; url = "https://goose-dataset.de/storage/gooseEx_2d_test.zip"; relative_path = "goose-ex-2d\archives\gooseEx_2d_test.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-3d-train"; url = "https://goose-dataset.de/storage/gooseEx_3d_train.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_train.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-3d-val"; url = "https://goose-dataset.de/storage/gooseEx_3d_val.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_val.zip"; expected_bytes = $null },
|
|
[pscustomobject]@{ group = "goose"; id = "goose-ex-3d-test"; url = "https://goose-dataset.de/storage/gooseEx_3d_test.zip"; relative_path = "goose-ex-3d\archives\gooseEx_3d_test.zip"; expected_bytes = $null }
|
|
)
|
|
|
|
function Write-JsonFile {
|
|
param([string]$Path, [object]$Value)
|
|
$parent = Split-Path -Parent $Path
|
|
New-Item -ItemType Directory -Force -Path $parent | Out-Null
|
|
$json = $Value | ConvertTo-Json -Depth 12
|
|
[System.IO.File]::WriteAllText($Path, $json + [Environment]::NewLine, [System.Text.UTF8Encoding]::new($false))
|
|
}
|
|
|
|
function Set-Stage {
|
|
param([string]$Stage, [string]$Status, [string]$Detail)
|
|
Write-JsonFile -Path $stagePath -Value ([ordered]@{
|
|
schema_version = "missioncore.lab-v1-asset-download-stage/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
worker_id = "worker-006"
|
|
stage = $Stage
|
|
status = $Status
|
|
detail = $Detail
|
|
})
|
|
}
|
|
|
|
function Invoke-CurlDownload {
|
|
param([string]$Url, [string]$Destination)
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $Destination) | Out-Null
|
|
& curl.exe --location --fail --retry 8 --retry-all-errors --continue-at - --output $Destination $Url
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "curl failed for $Url with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Test-AriaInputList {
|
|
param([string]$Path, [int]$ExpectedEntries)
|
|
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
return $false
|
|
}
|
|
|
|
$urlCount = 0
|
|
$dirCount = 0
|
|
foreach ($line in [System.IO.File]::ReadLines($Path)) {
|
|
if ($line.StartsWith("https://", [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
$urlCount++
|
|
}
|
|
elseif ($line.StartsWith(" dir=", [System.StringComparison]::Ordinal)) {
|
|
$dirCount++
|
|
}
|
|
}
|
|
return $urlCount -eq $ExpectedEntries -and $dirCount -eq $ExpectedEntries
|
|
}
|
|
|
|
function Test-FileIntegrity {
|
|
param([string]$Path, [long]$ExpectedBytes, [string]$ExpectedSha256)
|
|
$file = Get-Item -LiteralPath $Path -ErrorAction SilentlyContinue
|
|
if ($null -eq $file -or $file.Length -ne $ExpectedBytes) {
|
|
return $false
|
|
}
|
|
$actualSha256 = (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
return $actualSha256 -eq $ExpectedSha256
|
|
}
|
|
|
|
function Initialize-Aria2 {
|
|
New-Item -ItemType Directory -Force -Path $toolRoot | Out-Null
|
|
if (-not (Test-Path -LiteralPath $ariaExecutable -PathType Leaf)) {
|
|
Invoke-CurlDownload -Url $ariaUrl -Destination $ariaArchive
|
|
Expand-Archive -LiteralPath $ariaArchive -DestinationPath $toolRoot -Force
|
|
}
|
|
if (-not (Test-Path -LiteralPath $ariaExecutable -PathType Leaf)) {
|
|
throw "aria2 executable was not installed at $ariaExecutable"
|
|
}
|
|
}
|
|
|
|
function Write-DirectInput {
|
|
$lines = [System.Collections.Generic.List[string]]::new()
|
|
foreach ($asset in $assets) {
|
|
$destination = Join-Path $Root $asset.relative_path
|
|
New-Item -ItemType Directory -Force -Path (Split-Path -Parent $destination) | Out-Null
|
|
$lines.Add([string]$asset.url)
|
|
$lines.Add(" dir=" + (Split-Path -Parent $destination))
|
|
$lines.Add(" out=" + (Split-Path -Leaf $destination))
|
|
}
|
|
[System.IO.File]::WriteAllLines($gooseInputPath, $lines, [System.Text.UTF8Encoding]::new($false))
|
|
}
|
|
|
|
function Invoke-AriaInput {
|
|
param([string]$InputPath, [int]$ConcurrentDownloads, [int]$ConnectionsPerServer)
|
|
$downloadLimit = "{0}M" -f $DownloadLimitMiB
|
|
& $ariaExecutable `
|
|
--input-file=$InputPath `
|
|
--continue=true `
|
|
--max-concurrent-downloads=$ConcurrentDownloads `
|
|
--max-connection-per-server=$ConnectionsPerServer `
|
|
--split=$ConnectionsPerServer `
|
|
--min-split-size=16M `
|
|
"--max-overall-download-limit=$downloadLimit" `
|
|
--file-allocation=none `
|
|
--disk-cache=64M `
|
|
--auto-file-renaming=false `
|
|
--allow-overwrite=false `
|
|
--retry-wait=5 `
|
|
--max-tries=0 `
|
|
--summary-interval=60 `
|
|
--console-log-level=notice
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "aria2 input run failed for $InputPath with exit code $LASTEXITCODE"
|
|
}
|
|
}
|
|
|
|
function Get-DownloadTask {
|
|
return Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
|
|
}
|
|
|
|
function Get-AssetSnapshot {
|
|
$rows = foreach ($asset in $assets) {
|
|
$path = Join-Path $Root $asset.relative_path
|
|
$file = Get-Item -LiteralPath $path -ErrorAction SilentlyContinue
|
|
$partial = Get-Item -LiteralPath ($path + ".aria2") -ErrorAction SilentlyContinue
|
|
[ordered]@{
|
|
id = $asset.id
|
|
group = $asset.group
|
|
relative_path = $asset.relative_path
|
|
status = if ($null -ne $file -and $null -eq $partial) { "downloaded" } elseif ($null -ne $file) { "partial" } else { "pending" }
|
|
size_bytes = if ($null -ne $file) { $file.Length } else { 0 }
|
|
expected_bytes = $asset.expected_bytes
|
|
}
|
|
}
|
|
return @($rows)
|
|
}
|
|
|
|
function Invoke-Verify {
|
|
param([ValidateSet("All", "Models")][string]$Scope = "All")
|
|
New-Item -ItemType Directory -Force -Path $stateRoot | Out-Null
|
|
$writer = [System.IO.StreamWriter]::new($verificationPath, $false, [System.Text.UTF8Encoding]::new($false))
|
|
try {
|
|
foreach ($asset in $assets) {
|
|
if ($Scope -eq "Models" -and $asset.group -ne "models") {
|
|
continue
|
|
}
|
|
$path = Join-Path $Root $asset.relative_path
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
|
continue
|
|
}
|
|
if (Test-Path -LiteralPath ($path + ".aria2") -PathType Leaf) {
|
|
throw "Cannot verify partial asset $($asset.id)"
|
|
}
|
|
$file = Get-Item -LiteralPath $path
|
|
$hash = Get-FileHash -LiteralPath $path -Algorithm SHA256
|
|
$sizeMatches = $null
|
|
if ($null -ne $asset.expected_bytes) {
|
|
$sizeMatches = $file.Length -eq [long]$asset.expected_bytes
|
|
if (-not $sizeMatches) {
|
|
throw "Size mismatch for $($asset.id): expected $($asset.expected_bytes), got $($file.Length)"
|
|
}
|
|
}
|
|
$row = [ordered]@{
|
|
schema_version = "missioncore.lab-v1-asset-verification/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
id = $asset.id
|
|
relative_path = $asset.relative_path
|
|
size_bytes = $file.Length
|
|
expected_bytes = $asset.expected_bytes
|
|
size_matches = $sizeMatches
|
|
sha256 = $hash.Hash.ToLowerInvariant()
|
|
source_url = $asset.url
|
|
}
|
|
$writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 8))
|
|
$writer.Flush()
|
|
}
|
|
$offnetRoot = Join-Path $Root "models\orfd-offnet"
|
|
if (Test-Path -LiteralPath $offnetRoot -PathType Container) {
|
|
Get-ChildItem -LiteralPath $offnetRoot -File -Recurse | Sort-Object FullName | ForEach-Object {
|
|
$row = [ordered]@{
|
|
schema_version = "missioncore.lab-v1-asset-verification/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
id = "orfd-offnet/" + $_.FullName.Substring($offnetRoot.Length + 1).Replace("\", "/")
|
|
relative_path = $_.FullName.Substring($Root.Length + 1)
|
|
size_bytes = $_.Length
|
|
expected_bytes = $null
|
|
size_matches = $null
|
|
sha256 = (Get-FileHash -LiteralPath $_.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
source_url = "https://drive.google.com/drive/folders/1lnm2M1HEkVs9W3-FSEX3ddE9GYz4rqCU"
|
|
}
|
|
$writer.WriteLine(($row | ConvertTo-Json -Compress -Depth 8))
|
|
$writer.Flush()
|
|
}
|
|
}
|
|
}
|
|
finally {
|
|
$writer.Dispose()
|
|
}
|
|
}
|
|
|
|
function Measure-PayloadTree {
|
|
param([string]$Path)
|
|
[long]$bytes = 0
|
|
[long]$fileCount = 0
|
|
[long]$partialCount = 0
|
|
Get-ChildItem -LiteralPath $Path -File -Recurse | ForEach-Object {
|
|
if ($_.Name.EndsWith(".aria2", [System.StringComparison]::OrdinalIgnoreCase)) {
|
|
$partialCount++
|
|
}
|
|
else {
|
|
$fileCount++
|
|
$bytes += $_.Length
|
|
}
|
|
}
|
|
return [ordered]@{ file_count = $fileCount; size_bytes = $bytes; partial_count = $partialCount }
|
|
}
|
|
|
|
function Invoke-BulkVerify {
|
|
$wildScenesRoot = Join-Path $Root "wildscenes"
|
|
$orfdRoot = Join-Path $Root "orfd"
|
|
$wildScenes = Measure-PayloadTree -Path $wildScenesRoot
|
|
$orfd = Measure-PayloadTree -Path $orfdRoot
|
|
if (
|
|
$wildScenes.file_count -ne $wildScenesInputEntryCount -or
|
|
$wildScenes.size_bytes -ne $wildScenesPublishedSizeBytes -or
|
|
$wildScenes.partial_count -ne 0
|
|
) {
|
|
throw "WildScenes tree failed count/size/partial verification"
|
|
}
|
|
if (
|
|
$orfd.file_count -ne $orfdPublishedFileCount -or
|
|
$orfd.size_bytes -ne $orfdPublishedSizeBytes -or
|
|
$orfd.partial_count -ne 0
|
|
) {
|
|
throw "ORFD tree failed count/size/partial verification"
|
|
}
|
|
Write-JsonFile -Path $bulkVerificationPath -Value ([ordered]@{
|
|
schema_version = "missioncore.lab-v1-bulk-verification/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
worker_id = "worker-006"
|
|
wildscenes = $wildScenes
|
|
orfd = $orfd
|
|
canonical_triton_action = "none"
|
|
})
|
|
}
|
|
|
|
function Write-AssetManifest {
|
|
if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) {
|
|
throw "WildScenes input list is not admitted"
|
|
}
|
|
if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) {
|
|
throw "ORFD torrent source is not admitted"
|
|
}
|
|
Write-JsonFile -Path $assetManifestPath -Value ([ordered]@{
|
|
schema_version = "missioncore.lab-v1-asset-manifest/v1"
|
|
observed_at_utc = [DateTime]::UtcNow.ToString("o")
|
|
worker_id = "worker-006"
|
|
storage_policy = "worker-d-only"
|
|
root = $Root
|
|
runtime_models_mutated = $false
|
|
canonical_triton_action = "none"
|
|
existing_admitted_assets = @(
|
|
[ordered]@{ id = "goose-3d-val"; path = "D:\NDC_MISSIONCORE\datasets\goose-3d\v2025-08-22\archives\goose_3d_val.zip"; sha256 = "0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f" },
|
|
[ordered]@{ id = "rellis-3d-v1.1"; path = "D:\NDC_MISSIONCORE\datasets\rellis-3d\v1.1"; admission_identity = "a3f3f161a5a7edccdf66cea75ecf9004b8f6a895282faabe661c075c15a82774" }
|
|
)
|
|
direct_assets = $assets
|
|
wildscenes = [ordered]@{ collection_id = 63154; version = 3; published_size_bytes = $wildScenesPublishedSizeBytes; data_file_count = $wildScenesDataFileCount; metadata_file_count = $wildScenesMetadataFileCount; input_entry_count = $wildScenesInputEntryCount; url_list = $wildScenesInputPath; url_list_sha256 = (Get-FileHash -LiteralPath $wildScenesInputPath -Algorithm SHA256).Hash.ToLowerInvariant() }
|
|
orfd = [ordered]@{ info_hash = "ec5ccf4b8e49271ee3b63660383facf43063f2f2"; published_size_bytes = $orfdPublishedSizeBytes; published_file_count = $orfdPublishedFileCount; torrent = $orfdTorrentPath; torrent_size_bytes = $orfdTorrentExpectedBytes; torrent_sha256 = $orfdTorrentSha256 }
|
|
offnet_weights = [ordered]@{ source_folder = "https://drive.google.com/drive/folders/1lnm2M1HEkVs9W3-FSEX3ddE9GYz4rqCU"; relative_path = "models\orfd-offnet"; downloader = "gdown==5.2.0"; isolated_image = "nvcr.io/nvidia/tritonserver@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"; runtime_models_mutated = $false }
|
|
})
|
|
}
|
|
|
|
New-Item -ItemType Directory -Force -Path $Root, $stateRoot | Out-Null
|
|
|
|
switch ($Mode) {
|
|
"Start" {
|
|
$existingTask = Get-DownloadTask
|
|
if ($null -ne $existingTask -and $existingTask.State -eq "Running") {
|
|
throw "LAB V1 downloader task is already running"
|
|
}
|
|
|
|
$drive = Get-PSDrive -Name D
|
|
$currentUsage = Measure-PayloadTree -Path $Root
|
|
[long]$remainingPayloadBytes = 360GB - [long]$currentUsage.size_bytes
|
|
if ($remainingPayloadBytes -lt 0) {
|
|
$remainingPayloadBytes = 0
|
|
}
|
|
[long]$requiredFreeBytes = 10GB + $remainingPayloadBytes
|
|
if ($drive.Free -lt $requiredFreeBytes) {
|
|
throw "Worker D: has only $([math]::Round($drive.Free / 1GB, 2)) GiB free; $([math]::Round($requiredFreeBytes / 1GB, 2)) GiB is required for remaining payload plus reserve"
|
|
}
|
|
|
|
Set-Stage -Stage "preflight" -Status "running" -Detail "Installing bounded downloader and generating immutable source lists"
|
|
Initialize-Aria2
|
|
Write-DirectInput
|
|
|
|
if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) {
|
|
Invoke-CurlDownload -Url "https://data.csiro.au/dap/ws/v2/collections/63154/files/list" -Destination $wildScenesInputPath
|
|
}
|
|
if (-not (Test-AriaInputList -Path $wildScenesInputPath -ExpectedEntries $wildScenesInputEntryCount)) {
|
|
throw "WildScenes input list is incomplete: expected $wildScenesInputEntryCount URL/dir entries"
|
|
}
|
|
if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) {
|
|
Remove-Item -LiteralPath $orfdTorrentPath -Force -ErrorAction SilentlyContinue
|
|
Invoke-CurlDownload -Url "https://academictorrents.com/download/ec5ccf4b8e49271ee3b63660383facf43063f2f2.torrent" -Destination $orfdTorrentPath
|
|
}
|
|
if (-not (Test-FileIntegrity -Path $orfdTorrentPath -ExpectedBytes $orfdTorrentExpectedBytes -ExpectedSha256 $orfdTorrentSha256)) {
|
|
throw "ORFD torrent source failed size/SHA-256 verification"
|
|
}
|
|
|
|
Write-AssetManifest
|
|
|
|
$powershell = "$env:SystemRoot\System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
$arguments = @(
|
|
"-NoLogo", "-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass",
|
|
"-File", "`"$PSCommandPath`"",
|
|
"-Mode", "Run",
|
|
"-Root", "`"$Root`"",
|
|
"-DownloadLimitMiB", $DownloadLimitMiB
|
|
) -join " "
|
|
$action = New-ScheduledTaskAction -Execute $powershell -Argument $arguments -WorkingDirectory (Split-Path -Parent $PSCommandPath)
|
|
$principal = New-ScheduledTaskPrincipal -UserId ([System.Security.Principal.WindowsIdentity]::GetCurrent().Name) -LogonType Interactive -RunLevel Limited
|
|
$trigger = New-ScheduledTaskTrigger -Once -At ((Get-Date).AddMinutes(30))
|
|
$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DontStopIfGoingOnBatteries -StartWhenAvailable -ExecutionTimeLimit ([TimeSpan]::FromHours(72))
|
|
Register-ScheduledTask -TaskName $taskName -Action $action -Principal $principal -Trigger $trigger -Settings $settings -Description "Mission Core LAB V1 isolated vegetation dataset and model admission." -Force | Out-Null
|
|
Start-ScheduledTask -TaskName $taskName
|
|
Set-Stage -Stage "queued" -Status "running" -Detail "Bounded downloader started as scheduled task $taskName"
|
|
[ordered]@{
|
|
ok = $true
|
|
mode = $Mode
|
|
task_name = $taskName
|
|
task_state = (Get-ScheduledTask -TaskName $taskName).State.ToString()
|
|
root = $Root
|
|
d_free_gib_before = [math]::Round($drive.Free / 1GB, 2)
|
|
download_limit_mib = $DownloadLimitMiB
|
|
canonical_triton_action = "none"
|
|
} | ConvertTo-Json -Depth 6
|
|
}
|
|
"Run" {
|
|
try {
|
|
Initialize-Aria2
|
|
Set-Stage -Stage "models-and-goose" -Status "running" -Detail "Downloading official pretrained weights and prepared GOOSE/GOOSE-Ex splits"
|
|
Invoke-AriaInput -InputPath $gooseInputPath -ConcurrentDownloads 2 -ConnectionsPerServer 4
|
|
|
|
Set-Stage -Stage "orfd" -Status "running" -Detail "Downloading the 45.94 GB Academic Torrents ORFD release"
|
|
$orfdRoot = Join-Path $Root "orfd"
|
|
$downloadLimit = "{0}M" -f $DownloadLimitMiB
|
|
New-Item -ItemType Directory -Force -Path $orfdRoot | Out-Null
|
|
& $ariaExecutable `
|
|
--dir=$orfdRoot `
|
|
--continue=true `
|
|
--seed-time=0 `
|
|
"--max-overall-download-limit=$downloadLimit" `
|
|
--file-allocation=none `
|
|
--disk-cache=64M `
|
|
--summary-interval=60 `
|
|
--console-log-level=notice `
|
|
$orfdTorrentPath
|
|
if ($LASTEXITCODE -ne 0) {
|
|
throw "ORFD torrent run failed with exit code $LASTEXITCODE"
|
|
}
|
|
|
|
Set-Stage -Stage "wildscenes" -Status "running" -Detail "Downloading the complete public WildScenes v3 file list"
|
|
$wildScenesRoot = Join-Path $Root "wildscenes"
|
|
New-Item -ItemType Directory -Force -Path $wildScenesRoot | Out-Null
|
|
Push-Location $wildScenesRoot
|
|
try {
|
|
Invoke-AriaInput -InputPath $wildScenesInputPath -ConcurrentDownloads 8 -ConnectionsPerServer 2
|
|
}
|
|
finally {
|
|
Pop-Location
|
|
}
|
|
|
|
Set-Stage -Stage "verification" -Status "running" -Detail "Computing SHA-256 for direct archive and model assets"
|
|
Invoke-Verify
|
|
Invoke-BulkVerify
|
|
Set-Stage -Stage "complete" -Status "complete" -Detail "All scheduled assets downloaded and verified by direct hashes or published tree totals"
|
|
}
|
|
catch {
|
|
Set-Stage -Stage "failed" -Status "failed" -Detail $_.Exception.Message
|
|
throw
|
|
}
|
|
}
|
|
"Status" {
|
|
$task = Get-DownloadTask
|
|
$stage = if (Test-Path -LiteralPath $stagePath -PathType Leaf) { Get-Content -LiteralPath $stagePath -Raw | ConvertFrom-Json } else { $null }
|
|
$snapshot = Get-AssetSnapshot
|
|
$drive = Get-PSDrive -Name D
|
|
[ordered]@{
|
|
ok = $true
|
|
mode = $Mode
|
|
running = $null -ne $task -and $task.State -eq "Running"
|
|
task_name = $taskName
|
|
task_state = if ($null -ne $task) { $task.State.ToString() } else { $null }
|
|
stage = $stage
|
|
direct_assets = $snapshot
|
|
downloaded_direct_asset_count = @($snapshot | Where-Object { $_.status -eq "downloaded" }).Count
|
|
partial_direct_asset_count = @($snapshot | Where-Object { $_.status -eq "partial" }).Count
|
|
d_free_gib = [math]::Round($drive.Free / 1GB, 2)
|
|
canonical_triton_action = "none"
|
|
} | ConvertTo-Json -Depth 10
|
|
}
|
|
"Manifest" {
|
|
Write-AssetManifest
|
|
[ordered]@{
|
|
ok = $true
|
|
mode = $Mode
|
|
manifest_path = $assetManifestPath
|
|
manifest_sha256 = (Get-FileHash -LiteralPath $assetManifestPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
canonical_triton_action = "none"
|
|
} | ConvertTo-Json -Depth 6
|
|
}
|
|
"VerifyModels" {
|
|
Invoke-Verify -Scope "Models"
|
|
[ordered]@{
|
|
ok = $true
|
|
mode = $Mode
|
|
verification_path = $verificationPath
|
|
verification_sha256 = (Get-FileHash -LiteralPath $verificationPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
verified_asset_count = (Get-Content -LiteralPath $verificationPath).Count
|
|
canonical_triton_action = "none"
|
|
} | ConvertTo-Json -Depth 6
|
|
}
|
|
"Verify" {
|
|
Set-Stage -Stage "verification" -Status "running" -Detail "Manual direct-asset and published-tree verification requested"
|
|
Invoke-Verify
|
|
Invoke-BulkVerify
|
|
Set-Stage -Stage "verification" -Status "complete" -Detail "Direct assets hashed and bulk trees match published totals"
|
|
[ordered]@{
|
|
ok = $true
|
|
mode = $Mode
|
|
verification_path = $verificationPath
|
|
bulk_verification_path = $bulkVerificationPath
|
|
verified_asset_count = (Get-Content -LiteralPath $verificationPath).Count
|
|
} | ConvertTo-Json -Depth 6
|
|
}
|
|
}
|