From 759830b05403a8d90434bf10f2a43dd5a69ef181 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 31 Aug 2026 19:24:42 +0300 Subject: [PATCH] feat(lab): seal reusable EoMT runtime trees --- .../Seal-LabV1EomtTreeAssets.ps1 | 589 ++++++++++++++++++ tests/test_lab_v1_eomt_tree_asset_sealer.py | 59 ++ 2 files changed, 648 insertions(+) create mode 100644 experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 create mode 100644 tests/test_lab_v1_eomt_tree_asset_sealer.py diff --git a/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 new file mode 100644 index 0000000..78e38ce --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 @@ -0,0 +1,589 @@ +[CmdletBinding()] +param( + [ValidateSet( + "eomt-environment", + "eomt-model-cache", + "eomt-python-environment", + "eomt-runner-bundle", + "eomt-transformers-environment", + "k1-valid-fov-root" + )] + [string[]]$AssetId = @( + "eomt-environment", + "eomt-model-cache", + "eomt-python-environment", + "eomt-transformers-environment" + ) +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\") +$ManifestName = "tree-manifest.tsv" +$ReceiptName = "tree-receipt.json" +$SchemaVersion = "missioncore.sealed-tree-runtime/v1" +$IdentityAlgorithm = "relative-path-tab-size-tab-file-sha256-lf/v1" +$MaximumManifestBytes = 32MB +$Utf8 = New-Object Text.UTF8Encoding($false, $true) +$Invariant = [Globalization.CultureInfo]::InvariantCulture + +# The first four roots already exist on Worker 006. The runner bundle and +# valid-FOV root deliberately point at separate, immutable asset directories; +# their payloads must be materialized there before this sealer is invoked. +$AssetRelativeRoots = [ordered]@{ + "eomt-environment" = "derived\perception-e3-opencv413092-v1" + "eomt-model-cache" = "cache\perception-e3-models-v1" + "eomt-python-environment" = "derived\perception-p0-env-v1" + "eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1" + "eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1" + "k1-valid-fov-root" = ( + "assets\observatory-portable\" + + "k1-valid-fov-root-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1" + ) +} + +function Get-Sha256Hex { + param([byte[]]$Payload) + + $algorithm = [Security.Cryptography.SHA256]::Create() + try { + return -join @($algorithm.ComputeHash($Payload) | ForEach-Object { + $_.ToString("x2", $Invariant) + }) + } + finally { + $algorithm.Dispose() + } +} + +function Assert-SafeRelativePath { + param( + [string]$RelativePath, + [string]$Label + ) + + if ( + [string]::IsNullOrWhiteSpace($RelativePath) -or + $RelativePath.StartsWith("/", [StringComparison]::Ordinal) -or + $RelativePath.Contains("\") -or + $Utf8.GetByteCount($RelativePath) -gt 4096 + ) { + throw "$Label path is invalid" + } + [string[]]$components = $RelativePath.Split([char]"/") + if ($components.Count -gt 256) { + throw "$Label path has too many components" + } + foreach ($component in $components) { + if ( + [string]::IsNullOrEmpty($component) -or + $component -ceq "." -or + $component -ceq ".." -or + $Utf8.GetByteCount($component) -gt 255 -or + $component.EndsWith(" ", [StringComparison]::Ordinal) -or + $component.EndsWith(".", [StringComparison]::Ordinal) -or + [Text.RegularExpressions.Regex]::IsMatch( + $component, + '[\p{Cc}<>:"|?*]' + ) + ) { + throw "$Label path is incompatible with the component adapter" + } + } +} + +function Get-ConfinedRelativePath { + param( + [string]$Root, + [string]$Candidate, + [string]$Label + ) + + $prefix = $Root.TrimEnd("\") + "\" + $full = [IO.Path]::GetFullPath($Candidate) + if (-not $full.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "$Label escaped its fixed tree root" + } + $relative = $full.Substring($prefix.Length).Replace("\", "/") + Assert-SafeRelativePath $relative $Label + return $relative +} + +function Get-FileDigest { + param([string]$Path) + + $before = Get-Item -LiteralPath $Path -Force + if ( + -not ($before -is [IO.FileInfo]) -or + ($before.Attributes -band [IO.FileAttributes]::ReparsePoint) + ) { + throw "Tree payload contains an unsafe file" + } + $stream = [IO.File]::Open( + $before.FullName, + [IO.FileMode]::Open, + [IO.FileAccess]::Read, + [IO.FileShare]::Read + ) + $algorithm = [Security.Cryptography.SHA256]::Create() + try { + $length = [int64]$stream.Length + $digest = -join @($algorithm.ComputeHash($stream) | ForEach-Object { + $_.ToString("x2", $Invariant) + }) + } + finally { + $algorithm.Dispose() + $stream.Dispose() + } + $after = Get-Item -LiteralPath $before.FullName -Force + if ( + [int64]$after.Length -ne $length -or + $after.LastWriteTimeUtc.Ticks -ne $before.LastWriteTimeUtc.Ticks + ) { + throw "Tree payload changed while it was being hashed" + } + return [pscustomobject]@{ + byte_length = $length + sha256 = $digest + last_write_ticks = [int64]$after.LastWriteTimeUtc.Ticks + } +} + +function Get-PayloadInventory { + param( + [string]$Root, + [bool]$HashPayload + ) + + $records = New-Object "Collections.Generic.List[object]" + $directories = New-Object "Collections.Generic.Stack[string]" + $directories.Push($Root) + while ($directories.Count -gt 0) { + $directory = $directories.Pop() + foreach ($entry in [IO.Directory]::EnumerateFileSystemEntries($directory)) { + $attributes = [IO.File]::GetAttributes($entry) + if ($attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Tree payload contains a reparse point" + } + $relative = Get-ConfinedRelativePath $Root $entry "Tree payload member" + if ($attributes -band [IO.FileAttributes]::Directory) { + $directories.Push([IO.Path]::GetFullPath($entry)) + continue + } + if ($relative -eq $ManifestName -or $relative -eq $ReceiptName) { + continue + } + $item = Get-Item -LiteralPath $entry -Force + if (-not ($item -is [IO.FileInfo])) { + throw "Tree payload contains a non-regular member" + } + if ($HashPayload) { + $identity = Get-FileDigest $item.FullName + $records.Add([pscustomobject]@{ + relative_path = $relative + byte_length = [int64]$identity.byte_length + sha256 = [string]$identity.sha256 + last_write_ticks = [int64]$identity.last_write_ticks + }) + } + else { + $records.Add([pscustomobject]@{ + relative_path = $relative + byte_length = [int64]$item.Length + sha256 = $null + last_write_ticks = [int64]$item.LastWriteTimeUtc.Ticks + }) + } + } + } + # A PowerShell hashtable is case-insensitive, which is the stricter and + # correct choice for payloads hosted by Windows. The emitted key array is + # sorted independently with the adapter's required ordinal comparer. + $recordByPath = @{} + foreach ($record in $records) { + $relativePath = [string]$record.relative_path + if ($recordByPath.ContainsKey($relativePath)) { + throw "Tree payload contains an ambiguous case-insensitive path" + } + $recordByPath[$relativePath] = $record + } + [string[]]$paths = @($recordByPath.Keys) + [Array]::Sort($paths, [StringComparer]::Ordinal) + $ordered = New-Object "Collections.Generic.List[object]" + foreach ($path in $paths) { + $ordered.Add($recordByPath[$path]) + } + return @($ordered.ToArray()) +} + +function ConvertTo-ManifestBytes { + param([object[]]$Records) + + if ($Records.Count -eq 0) { + throw "Tree payload is empty" + } + $builder = New-Object Text.StringBuilder + [int64]$totalBytes = 0 + foreach ($record in $Records) { + if ([string]$record.sha256 -notmatch "^[a-f0-9]{64}$") { + throw "Tree payload digest is invalid" + } + $null = $builder.Append([string]$record.relative_path) + $null = $builder.Append([char]9) + $null = $builder.Append(([int64]$record.byte_length).ToString($Invariant)) + $null = $builder.Append([char]9) + $null = $builder.Append([string]$record.sha256) + $null = $builder.Append([char]10) + $totalBytes += [int64]$record.byte_length + } + if ($totalBytes -le 0) { + throw "Tree payload byte length is invalid" + } + $payload = $Utf8.GetBytes($builder.ToString()) + if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) { + throw "Tree manifest is outside component-adapter bounds" + } + return [pscustomobject]@{ + payload = $payload + byte_length = $totalBytes + } +} + +function Read-SealedManifest { + param( + [string]$Path, + [string]$Asset + ) + + $payload = [IO.File]::ReadAllBytes($Path) + if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) { + throw "Existing $Asset manifest is outside component-adapter bounds" + } + try { + $text = $Utf8.GetString($payload) + } + catch { + throw "Existing $Asset manifest is not canonical UTF-8" + } + if (-not $text.EndsWith([char]10) -or $text.Contains([char]13)) { + throw "Existing $Asset manifest is not canonical LF text" + } + $body = $text.Substring(0, $text.Length - 1) + $lines = $body.Split([char[]]@([char]10), [StringSplitOptions]::None) + $records = New-Object "Collections.Generic.List[object]" + $previous = $null + foreach ($line in $lines) { + $fields = $line.Split([char[]]@([char]9), [StringSplitOptions]::None) + if ($fields.Count -ne 3) { + throw "Existing $Asset manifest row is invalid" + } + $relative = [string]$fields[0] + Assert-SafeRelativePath $relative "Existing $Asset manifest member" + if ( + $null -ne $previous -and + [StringComparer]::Ordinal.Compare([string]$previous, $relative) -ge 0 + ) { + throw "Existing $Asset manifest order is not canonical" + } + [int64]$size = 0 + if (-not [int64]::TryParse( + [string]$fields[1], + [Globalization.NumberStyles]::None, + $Invariant, + [ref]$size + ) -or $size -lt 0) { + throw "Existing $Asset manifest size is invalid" + } + $digest = [string]$fields[2] + if ($digest -notmatch "^[a-f0-9]{64}$") { + throw "Existing $Asset manifest digest is invalid" + } + $records.Add([pscustomobject]@{ + relative_path = $relative + byte_length = $size + sha256 = $digest + }) + $previous = $relative + } + return [pscustomobject]@{ + payload = $payload + records = @($records.ToArray()) + identity_sha256 = Get-Sha256Hex $payload + } +} + +function Read-SealedReceipt { + param( + [string]$Path, + [string]$Asset + ) + + $payload = [IO.File]::ReadAllBytes($Path) + if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) { + throw "Existing $Asset receipt is outside component-adapter bounds" + } + try { + $text = $Utf8.GetString($payload) + } + catch { + throw "Existing $Asset receipt is not canonical UTF-8" + } + if (-not $text.EndsWith([char]10) -or $text.Contains([char]13)) { + throw "Existing $Asset receipt is not canonical LF text" + } + try { + $receipt = $text | ConvertFrom-Json + } + catch { + throw "Existing $Asset receipt is not valid JSON" + } + $expectedKeys = @( + "schema_version", + "asset_id", + "identity_algorithm", + "identity_sha256", + "file_count", + "byte_length", + "manifest_relative_path" + ) + $observedKeys = @($receipt.PSObject.Properties.Name) + if ( + $observedKeys.Count -ne $expectedKeys.Count -or + [string]::Join([char]10, $observedKeys) -cne [string]::Join([char]10, $expectedKeys) + ) { + throw "Existing $Asset receipt keys are not canonical" + } + if ( + -not ($receipt.schema_version -is [string]) -or + -not ($receipt.asset_id -is [string]) -or + -not ($receipt.identity_algorithm -is [string]) -or + -not ($receipt.identity_sha256 -is [string]) -or + -not ($receipt.manifest_relative_path -is [string]) -or + -not ( + ($receipt.file_count -is [int]) -or + ($receipt.file_count -is [long]) + ) -or + -not ( + ($receipt.byte_length -is [int]) -or + ($receipt.byte_length -is [long]) + ) -or + [int64]$receipt.file_count -le 0 -or + [int64]$receipt.byte_length -le 0 -or + $receipt.schema_version -cne $SchemaVersion -or + $receipt.asset_id -cne $Asset -or + $receipt.identity_algorithm -cne $IdentityAlgorithm -or + [string]$receipt.identity_sha256 -notmatch "^[a-f0-9]{64}$" -or + $receipt.manifest_relative_path -cne $ManifestName + ) { + throw "Existing $Asset receipt contract changed" + } + return $receipt +} + +function Assert-InventoryMatches { + param( + [object[]]$Expected, + [object[]]$Observed, + [string]$Asset, + [bool]$CheckWriteTime + ) + + if ($Expected.Count -ne $Observed.Count) { + throw "$Asset payload member set changed" + } + for ($index = 0; $index -lt $Expected.Count; $index += 1) { + $left = $Expected[$index] + $right = $Observed[$index] + if ( + [string]$left.relative_path -cne [string]$right.relative_path -or + [int64]$left.byte_length -ne [int64]$right.byte_length -or + ($CheckWriteTime -and ( + [int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks + )) + ) { + throw "$Asset payload changed while it was being sealed" + } + } +} + +function Test-ExistingSeal { + param( + [string]$Root, + [string]$Asset + ) + + $manifestPath = Join-Path $Root $ManifestName + $receiptPath = Join-Path $Root $ReceiptName + if (-not (Test-Path -LiteralPath $receiptPath -PathType Leaf)) { + return $null + } + if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) { + throw "Existing $Asset receipt has no manifest" + } + $receipt = Read-SealedReceipt $receiptPath $Asset + $manifest = Read-SealedManifest $manifestPath $Asset + [int64]$totalBytes = 0 + foreach ($record in $manifest.records) { + $totalBytes += [int64]$record.byte_length + } + if ( + $receipt.identity_sha256 -cne $manifest.identity_sha256 -or + [int64]$receipt.file_count -ne $manifest.records.Count -or + [int64]$receipt.byte_length -ne $totalBytes -or + $totalBytes -le 0 + ) { + throw "Existing $Asset receipt does not bind its manifest" + } + + # Fast path: verify names and sizes, but deliberately do not re-hash the + # already sealed multi-gigabyte payload. + $observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false) + Assert-InventoryMatches $manifest.records $observed $Asset $false + return $receipt +} + +function New-SealedReceipt { + param( + [string]$Asset, + [string]$IdentitySha256, + [int]$FileCount, + [int64]$ByteLength + ) + + return [ordered]@{ + schema_version = $SchemaVersion + asset_id = $Asset + identity_algorithm = $IdentityAlgorithm + identity_sha256 = $IdentitySha256 + file_count = $FileCount + byte_length = $ByteLength + manifest_relative_path = $ManifestName + } +} + +function Seal-TreeAsset { + param( + [string]$Root, + [string]$Asset + ) + + if (-not (Test-Path -LiteralPath $Root -PathType Container)) { + throw "$Asset payload must be materialized at its fixed local root before sealing" + } + $rootItem = Get-Item -LiteralPath $Root -Force + if ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "$Asset root is a reparse point" + } + $lockPath = "$Root.tree-seal.lock" + $lock = $null + $ownsLock = $false + $staging = "$Root.tree-seal-staging-$([Guid]::NewGuid().ToString('N'))" + try { + try { + $lock = [IO.File]::Open( + $lockPath, + [IO.FileMode]::OpenOrCreate, + [IO.FileAccess]::ReadWrite, + [IO.FileShare]::None + ) + $ownsLock = $true + } + catch [IO.IOException] { + throw "$Asset is already being sealed" + } + + $existing = Test-ExistingSeal $Root $Asset + if ($null -ne $existing) { + return [pscustomobject]@{ + status = "already-sealed" + path = $Root + receipt = $existing + } + } + + $records = @(Get-PayloadInventory -Root $Root -HashPayload:$true) + $manifest = ConvertTo-ManifestBytes $records + $identitySha256 = Get-Sha256Hex $manifest.payload + $receipt = New-SealedReceipt ` + $Asset ` + $identitySha256 ` + $records.Count ` + ([int64]$manifest.byte_length) + $receiptPayload = $Utf8.GetBytes( + (($receipt | ConvertTo-Json -Compress -Depth 4) + [char]10) + ) + + # Re-enumerate without hashing before publication. This catches file, + # size, and timestamp changes that occurred during the one-time hash. + $observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false) + Assert-InventoryMatches $records $observed $Asset $true + + $null = New-Item -ItemType Directory -Path $staging + $stagedManifest = Join-Path $staging $ManifestName + $stagedReceipt = Join-Path $staging $ReceiptName + [IO.File]::WriteAllBytes($stagedManifest, $manifest.payload) + [IO.File]::WriteAllBytes($stagedReceipt, $receiptPayload) + if ( + (Get-Sha256Hex ([IO.File]::ReadAllBytes($stagedManifest))) -cne $identitySha256 + ) { + throw "$Asset staged manifest identity changed" + } + $stagedReceiptDocument = Read-SealedReceipt $stagedReceipt $Asset + if ($stagedReceiptDocument.identity_sha256 -cne $identitySha256) { + throw "$Asset staged receipt identity changed" + } + + $manifestPath = Join-Path $Root $ManifestName + $receiptPath = Join-Path $Root $ReceiptName + if (Test-Path -LiteralPath $receiptPath) { + throw "$Asset receipt appeared concurrently" + } + if (Test-Path -LiteralPath $manifestPath) { + # A manifest without its commit-marker receipt is incomplete + # metadata from an interrupted prior seal, never payload. + Remove-Item -LiteralPath $manifestPath -Force + } + [IO.File]::Move($stagedManifest, $manifestPath) + # The receipt is the commit marker and is always published last. + [IO.File]::Move($stagedReceipt, $receiptPath) + + return [pscustomobject]@{ + status = "sealed" + path = $Root + receipt = $receipt + } + } + finally { + if ($null -ne $lock) { + $lock.Dispose() + } + if ($ownsLock -and (Test-Path -LiteralPath $lockPath -PathType Leaf)) { + Remove-Item -LiteralPath $lockPath -Force + } + if (Test-Path -LiteralPath $staging -PathType Container) { + Remove-Item -LiteralPath $staging -Recurse -Force + } + } +} + +$requested = @($AssetId) +if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) { + throw "LAB V1 EoMT asset selection is empty or contains duplicates" +} +$outputs = New-Object "Collections.Generic.List[object]" +foreach ($asset in $requested) { + $relativeRoot = [string]$AssetRelativeRoots[$asset] + $root = [IO.Path]::GetFullPath((Join-Path $RuntimeRoot $relativeRoot)) + if (-not $root.StartsWith( + ($RuntimeRoot + "\"), + [StringComparison]::OrdinalIgnoreCase + )) { + throw "$asset resolved outside the fixed local runtime root" + } + $outputs.Add((Seal-TreeAsset $root $asset)) +} + +@($outputs.ToArray()) | ConvertTo-Json -Compress -Depth 8 diff --git a/tests/test_lab_v1_eomt_tree_asset_sealer.py b/tests/test_lab_v1_eomt_tree_asset_sealer.py new file mode 100644 index 0000000..6c6cdb1 --- /dev/null +++ b/tests/test_lab_v1_eomt_tree_asset_sealer.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SEALER_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "Seal-LabV1EomtTreeAssets.ps1" +) + + +def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None: + script = SEALER_PATH.read_text(encoding="utf-8") + + assert '"missioncore.sealed-tree-runtime/v1"' in script + assert '"relative-path-tab-size-tab-file-sha256-lf/v1"' in script + assert '"tree-manifest.tsv"' in script + assert '"tree-receipt.json"' in script + assert '"D:\\NDC_MISSIONCORE\\runtime"' in script + assert '"derived\\perception-e3-opencv413092-v1"' in script + assert '"cache\\perception-e3-models-v1"' in script + assert '"derived\\perception-p0-env-v1"' in script + assert '"derived\\perception-p0-transformers4576-v1"' in script + assert '$Utf8.GetByteCount($component) -gt 255' in script + assert "$Utf8.GetByteCount($RelativePath) -gt 4096" in script + assert "$components.Count -gt 256" in script + assert "[\\p{Cc}<>:" in script + assert "$SafeComponentPattern" not in script + + lowered = script.lower() + assert "docker " not in lowered + assert "ssh " not in lowered + assert "invoke-webrequest" not in lowered + assert "start-bitstransfer" not in lowered + assert "smb" not in lowered + + +def test_eomt_tree_sealer_has_atomic_commit_marker_and_no_rehash_fast_path() -> None: + script = SEALER_PATH.read_text(encoding="utf-8") + + fast_path = script.split("function Test-ExistingSeal", maxsplit=1)[1].split( + "function New-SealedReceipt", maxsplit=1 + )[0] + assert "-HashPayload:$false" in fast_path + assert "-HashPayload:$true" not in fast_path + assert "Get-FileDigest" not in fast_path + + publish = script.split("$manifestPath = Join-Path $Root $ManifestName", maxsplit=1)[1] + manifest_move = publish.index("[IO.File]::Move($stagedManifest, $manifestPath)") + receipt_move = publish.index("[IO.File]::Move($stagedReceipt, $receiptPath)") + assert manifest_move < receipt_move + assert "The receipt is the commit marker and is always published last" in publish + assert "$relative -eq $ManifestName -or $relative -eq $ReceiptName" in script + assert "New-Object Text.UTF8Encoding($false, $true)" in script + assert "[StringComparer]::Ordinal.Compare" in script