feat(lab): seal reusable EoMT runtime trees

This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 19:24:42 +03:00
parent 19c0bd70d0
commit 759830b054
2 changed files with 648 additions and 0 deletions
@@ -0,0 +1,589 @@
[CmdletBinding()]
param(
[ValidateSet(
"eomt-environment",
"eomt-model-cache",
"eomt-python-environment",
"eomt-runner-bundle",
"eomt-transformers-environment",
"k1-valid-fov-root"
)]
[string[]]$AssetId = @(
"eomt-environment",
"eomt-model-cache",
"eomt-python-environment",
"eomt-transformers-environment"
)
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
$ManifestName = "tree-manifest.tsv"
$ReceiptName = "tree-receipt.json"
$SchemaVersion = "missioncore.sealed-tree-runtime/v1"
$IdentityAlgorithm = "relative-path-tab-size-tab-file-sha256-lf/v1"
$MaximumManifestBytes = 32MB
$Utf8 = New-Object Text.UTF8Encoding($false, $true)
$Invariant = [Globalization.CultureInfo]::InvariantCulture
# The first four roots already exist on Worker 006. The runner bundle and
# valid-FOV root deliberately point at separate, immutable asset directories;
# their payloads must be materialized there before this sealer is invoked.
$AssetRelativeRoots = [ordered]@{
"eomt-environment" = "derived\perception-e3-opencv413092-v1"
"eomt-model-cache" = "cache\perception-e3-models-v1"
"eomt-python-environment" = "derived\perception-p0-env-v1"
"eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1"
"eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1"
"k1-valid-fov-root" = (
"assets\observatory-portable\" +
"k1-valid-fov-root-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1"
)
}
function Get-Sha256Hex {
param([byte[]]$Payload)
$algorithm = [Security.Cryptography.SHA256]::Create()
try {
return -join @($algorithm.ComputeHash($Payload) | ForEach-Object {
$_.ToString("x2", $Invariant)
})
}
finally {
$algorithm.Dispose()
}
}
function Assert-SafeRelativePath {
param(
[string]$RelativePath,
[string]$Label
)
if (
[string]::IsNullOrWhiteSpace($RelativePath) -or
$RelativePath.StartsWith("/", [StringComparison]::Ordinal) -or
$RelativePath.Contains("\") -or
$Utf8.GetByteCount($RelativePath) -gt 4096
) {
throw "$Label path is invalid"
}
[string[]]$components = $RelativePath.Split([char]"/")
if ($components.Count -gt 256) {
throw "$Label path has too many components"
}
foreach ($component in $components) {
if (
[string]::IsNullOrEmpty($component) -or
$component -ceq "." -or
$component -ceq ".." -or
$Utf8.GetByteCount($component) -gt 255 -or
$component.EndsWith(" ", [StringComparison]::Ordinal) -or
$component.EndsWith(".", [StringComparison]::Ordinal) -or
[Text.RegularExpressions.Regex]::IsMatch(
$component,
'[\p{Cc}<>:"|?*]'
)
) {
throw "$Label path is incompatible with the component adapter"
}
}
}
function Get-ConfinedRelativePath {
param(
[string]$Root,
[string]$Candidate,
[string]$Label
)
$prefix = $Root.TrimEnd("\") + "\"
$full = [IO.Path]::GetFullPath($Candidate)
if (-not $full.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) {
throw "$Label escaped its fixed tree root"
}
$relative = $full.Substring($prefix.Length).Replace("\", "/")
Assert-SafeRelativePath $relative $Label
return $relative
}
function Get-FileDigest {
param([string]$Path)
$before = Get-Item -LiteralPath $Path -Force
if (
-not ($before -is [IO.FileInfo]) -or
($before.Attributes -band [IO.FileAttributes]::ReparsePoint)
) {
throw "Tree payload contains an unsafe file"
}
$stream = [IO.File]::Open(
$before.FullName,
[IO.FileMode]::Open,
[IO.FileAccess]::Read,
[IO.FileShare]::Read
)
$algorithm = [Security.Cryptography.SHA256]::Create()
try {
$length = [int64]$stream.Length
$digest = -join @($algorithm.ComputeHash($stream) | ForEach-Object {
$_.ToString("x2", $Invariant)
})
}
finally {
$algorithm.Dispose()
$stream.Dispose()
}
$after = Get-Item -LiteralPath $before.FullName -Force
if (
[int64]$after.Length -ne $length -or
$after.LastWriteTimeUtc.Ticks -ne $before.LastWriteTimeUtc.Ticks
) {
throw "Tree payload changed while it was being hashed"
}
return [pscustomobject]@{
byte_length = $length
sha256 = $digest
last_write_ticks = [int64]$after.LastWriteTimeUtc.Ticks
}
}
function Get-PayloadInventory {
param(
[string]$Root,
[bool]$HashPayload
)
$records = New-Object "Collections.Generic.List[object]"
$directories = New-Object "Collections.Generic.Stack[string]"
$directories.Push($Root)
while ($directories.Count -gt 0) {
$directory = $directories.Pop()
foreach ($entry in [IO.Directory]::EnumerateFileSystemEntries($directory)) {
$attributes = [IO.File]::GetAttributes($entry)
if ($attributes -band [IO.FileAttributes]::ReparsePoint) {
throw "Tree payload contains a reparse point"
}
$relative = Get-ConfinedRelativePath $Root $entry "Tree payload member"
if ($attributes -band [IO.FileAttributes]::Directory) {
$directories.Push([IO.Path]::GetFullPath($entry))
continue
}
if ($relative -eq $ManifestName -or $relative -eq $ReceiptName) {
continue
}
$item = Get-Item -LiteralPath $entry -Force
if (-not ($item -is [IO.FileInfo])) {
throw "Tree payload contains a non-regular member"
}
if ($HashPayload) {
$identity = Get-FileDigest $item.FullName
$records.Add([pscustomobject]@{
relative_path = $relative
byte_length = [int64]$identity.byte_length
sha256 = [string]$identity.sha256
last_write_ticks = [int64]$identity.last_write_ticks
})
}
else {
$records.Add([pscustomobject]@{
relative_path = $relative
byte_length = [int64]$item.Length
sha256 = $null
last_write_ticks = [int64]$item.LastWriteTimeUtc.Ticks
})
}
}
}
# A PowerShell hashtable is case-insensitive, which is the stricter and
# correct choice for payloads hosted by Windows. The emitted key array is
# sorted independently with the adapter's required ordinal comparer.
$recordByPath = @{}
foreach ($record in $records) {
$relativePath = [string]$record.relative_path
if ($recordByPath.ContainsKey($relativePath)) {
throw "Tree payload contains an ambiguous case-insensitive path"
}
$recordByPath[$relativePath] = $record
}
[string[]]$paths = @($recordByPath.Keys)
[Array]::Sort($paths, [StringComparer]::Ordinal)
$ordered = New-Object "Collections.Generic.List[object]"
foreach ($path in $paths) {
$ordered.Add($recordByPath[$path])
}
return @($ordered.ToArray())
}
function ConvertTo-ManifestBytes {
param([object[]]$Records)
if ($Records.Count -eq 0) {
throw "Tree payload is empty"
}
$builder = New-Object Text.StringBuilder
[int64]$totalBytes = 0
foreach ($record in $Records) {
if ([string]$record.sha256 -notmatch "^[a-f0-9]{64}$") {
throw "Tree payload digest is invalid"
}
$null = $builder.Append([string]$record.relative_path)
$null = $builder.Append([char]9)
$null = $builder.Append(([int64]$record.byte_length).ToString($Invariant))
$null = $builder.Append([char]9)
$null = $builder.Append([string]$record.sha256)
$null = $builder.Append([char]10)
$totalBytes += [int64]$record.byte_length
}
if ($totalBytes -le 0) {
throw "Tree payload byte length is invalid"
}
$payload = $Utf8.GetBytes($builder.ToString())
if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) {
throw "Tree manifest is outside component-adapter bounds"
}
return [pscustomobject]@{
payload = $payload
byte_length = $totalBytes
}
}
function Read-SealedManifest {
param(
[string]$Path,
[string]$Asset
)
$payload = [IO.File]::ReadAllBytes($Path)
if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) {
throw "Existing $Asset manifest is outside component-adapter bounds"
}
try {
$text = $Utf8.GetString($payload)
}
catch {
throw "Existing $Asset manifest is not canonical UTF-8"
}
if (-not $text.EndsWith([char]10) -or $text.Contains([char]13)) {
throw "Existing $Asset manifest is not canonical LF text"
}
$body = $text.Substring(0, $text.Length - 1)
$lines = $body.Split([char[]]@([char]10), [StringSplitOptions]::None)
$records = New-Object "Collections.Generic.List[object]"
$previous = $null
foreach ($line in $lines) {
$fields = $line.Split([char[]]@([char]9), [StringSplitOptions]::None)
if ($fields.Count -ne 3) {
throw "Existing $Asset manifest row is invalid"
}
$relative = [string]$fields[0]
Assert-SafeRelativePath $relative "Existing $Asset manifest member"
if (
$null -ne $previous -and
[StringComparer]::Ordinal.Compare([string]$previous, $relative) -ge 0
) {
throw "Existing $Asset manifest order is not canonical"
}
[int64]$size = 0
if (-not [int64]::TryParse(
[string]$fields[1],
[Globalization.NumberStyles]::None,
$Invariant,
[ref]$size
) -or $size -lt 0) {
throw "Existing $Asset manifest size is invalid"
}
$digest = [string]$fields[2]
if ($digest -notmatch "^[a-f0-9]{64}$") {
throw "Existing $Asset manifest digest is invalid"
}
$records.Add([pscustomobject]@{
relative_path = $relative
byte_length = $size
sha256 = $digest
})
$previous = $relative
}
return [pscustomobject]@{
payload = $payload
records = @($records.ToArray())
identity_sha256 = Get-Sha256Hex $payload
}
}
function Read-SealedReceipt {
param(
[string]$Path,
[string]$Asset
)
$payload = [IO.File]::ReadAllBytes($Path)
if ($payload.Length -le 0 -or $payload.Length -gt $MaximumManifestBytes) {
throw "Existing $Asset receipt is outside component-adapter bounds"
}
try {
$text = $Utf8.GetString($payload)
}
catch {
throw "Existing $Asset receipt is not canonical UTF-8"
}
if (-not $text.EndsWith([char]10) -or $text.Contains([char]13)) {
throw "Existing $Asset receipt is not canonical LF text"
}
try {
$receipt = $text | ConvertFrom-Json
}
catch {
throw "Existing $Asset receipt is not valid JSON"
}
$expectedKeys = @(
"schema_version",
"asset_id",
"identity_algorithm",
"identity_sha256",
"file_count",
"byte_length",
"manifest_relative_path"
)
$observedKeys = @($receipt.PSObject.Properties.Name)
if (
$observedKeys.Count -ne $expectedKeys.Count -or
[string]::Join([char]10, $observedKeys) -cne [string]::Join([char]10, $expectedKeys)
) {
throw "Existing $Asset receipt keys are not canonical"
}
if (
-not ($receipt.schema_version -is [string]) -or
-not ($receipt.asset_id -is [string]) -or
-not ($receipt.identity_algorithm -is [string]) -or
-not ($receipt.identity_sha256 -is [string]) -or
-not ($receipt.manifest_relative_path -is [string]) -or
-not (
($receipt.file_count -is [int]) -or
($receipt.file_count -is [long])
) -or
-not (
($receipt.byte_length -is [int]) -or
($receipt.byte_length -is [long])
) -or
[int64]$receipt.file_count -le 0 -or
[int64]$receipt.byte_length -le 0 -or
$receipt.schema_version -cne $SchemaVersion -or
$receipt.asset_id -cne $Asset -or
$receipt.identity_algorithm -cne $IdentityAlgorithm -or
[string]$receipt.identity_sha256 -notmatch "^[a-f0-9]{64}$" -or
$receipt.manifest_relative_path -cne $ManifestName
) {
throw "Existing $Asset receipt contract changed"
}
return $receipt
}
function Assert-InventoryMatches {
param(
[object[]]$Expected,
[object[]]$Observed,
[string]$Asset,
[bool]$CheckWriteTime
)
if ($Expected.Count -ne $Observed.Count) {
throw "$Asset payload member set changed"
}
for ($index = 0; $index -lt $Expected.Count; $index += 1) {
$left = $Expected[$index]
$right = $Observed[$index]
if (
[string]$left.relative_path -cne [string]$right.relative_path -or
[int64]$left.byte_length -ne [int64]$right.byte_length -or
($CheckWriteTime -and (
[int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks
))
) {
throw "$Asset payload changed while it was being sealed"
}
}
}
function Test-ExistingSeal {
param(
[string]$Root,
[string]$Asset
)
$manifestPath = Join-Path $Root $ManifestName
$receiptPath = Join-Path $Root $ReceiptName
if (-not (Test-Path -LiteralPath $receiptPath -PathType Leaf)) {
return $null
}
if (-not (Test-Path -LiteralPath $manifestPath -PathType Leaf)) {
throw "Existing $Asset receipt has no manifest"
}
$receipt = Read-SealedReceipt $receiptPath $Asset
$manifest = Read-SealedManifest $manifestPath $Asset
[int64]$totalBytes = 0
foreach ($record in $manifest.records) {
$totalBytes += [int64]$record.byte_length
}
if (
$receipt.identity_sha256 -cne $manifest.identity_sha256 -or
[int64]$receipt.file_count -ne $manifest.records.Count -or
[int64]$receipt.byte_length -ne $totalBytes -or
$totalBytes -le 0
) {
throw "Existing $Asset receipt does not bind its manifest"
}
# Fast path: verify names and sizes, but deliberately do not re-hash the
# already sealed multi-gigabyte payload.
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
Assert-InventoryMatches $manifest.records $observed $Asset $false
return $receipt
}
function New-SealedReceipt {
param(
[string]$Asset,
[string]$IdentitySha256,
[int]$FileCount,
[int64]$ByteLength
)
return [ordered]@{
schema_version = $SchemaVersion
asset_id = $Asset
identity_algorithm = $IdentityAlgorithm
identity_sha256 = $IdentitySha256
file_count = $FileCount
byte_length = $ByteLength
manifest_relative_path = $ManifestName
}
}
function Seal-TreeAsset {
param(
[string]$Root,
[string]$Asset
)
if (-not (Test-Path -LiteralPath $Root -PathType Container)) {
throw "$Asset payload must be materialized at its fixed local root before sealing"
}
$rootItem = Get-Item -LiteralPath $Root -Force
if ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint) {
throw "$Asset root is a reparse point"
}
$lockPath = "$Root.tree-seal.lock"
$lock = $null
$ownsLock = $false
$staging = "$Root.tree-seal-staging-$([Guid]::NewGuid().ToString('N'))"
try {
try {
$lock = [IO.File]::Open(
$lockPath,
[IO.FileMode]::OpenOrCreate,
[IO.FileAccess]::ReadWrite,
[IO.FileShare]::None
)
$ownsLock = $true
}
catch [IO.IOException] {
throw "$Asset is already being sealed"
}
$existing = Test-ExistingSeal $Root $Asset
if ($null -ne $existing) {
return [pscustomobject]@{
status = "already-sealed"
path = $Root
receipt = $existing
}
}
$records = @(Get-PayloadInventory -Root $Root -HashPayload:$true)
$manifest = ConvertTo-ManifestBytes $records
$identitySha256 = Get-Sha256Hex $manifest.payload
$receipt = New-SealedReceipt `
$Asset `
$identitySha256 `
$records.Count `
([int64]$manifest.byte_length)
$receiptPayload = $Utf8.GetBytes(
(($receipt | ConvertTo-Json -Compress -Depth 4) + [char]10)
)
# Re-enumerate without hashing before publication. This catches file,
# size, and timestamp changes that occurred during the one-time hash.
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
Assert-InventoryMatches $records $observed $Asset $true
$null = New-Item -ItemType Directory -Path $staging
$stagedManifest = Join-Path $staging $ManifestName
$stagedReceipt = Join-Path $staging $ReceiptName
[IO.File]::WriteAllBytes($stagedManifest, $manifest.payload)
[IO.File]::WriteAllBytes($stagedReceipt, $receiptPayload)
if (
(Get-Sha256Hex ([IO.File]::ReadAllBytes($stagedManifest))) -cne $identitySha256
) {
throw "$Asset staged manifest identity changed"
}
$stagedReceiptDocument = Read-SealedReceipt $stagedReceipt $Asset
if ($stagedReceiptDocument.identity_sha256 -cne $identitySha256) {
throw "$Asset staged receipt identity changed"
}
$manifestPath = Join-Path $Root $ManifestName
$receiptPath = Join-Path $Root $ReceiptName
if (Test-Path -LiteralPath $receiptPath) {
throw "$Asset receipt appeared concurrently"
}
if (Test-Path -LiteralPath $manifestPath) {
# A manifest without its commit-marker receipt is incomplete
# metadata from an interrupted prior seal, never payload.
Remove-Item -LiteralPath $manifestPath -Force
}
[IO.File]::Move($stagedManifest, $manifestPath)
# The receipt is the commit marker and is always published last.
[IO.File]::Move($stagedReceipt, $receiptPath)
return [pscustomobject]@{
status = "sealed"
path = $Root
receipt = $receipt
}
}
finally {
if ($null -ne $lock) {
$lock.Dispose()
}
if ($ownsLock -and (Test-Path -LiteralPath $lockPath -PathType Leaf)) {
Remove-Item -LiteralPath $lockPath -Force
}
if (Test-Path -LiteralPath $staging -PathType Container) {
Remove-Item -LiteralPath $staging -Recurse -Force
}
}
}
$requested = @($AssetId)
if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) {
throw "LAB V1 EoMT asset selection is empty or contains duplicates"
}
$outputs = New-Object "Collections.Generic.List[object]"
foreach ($asset in $requested) {
$relativeRoot = [string]$AssetRelativeRoots[$asset]
$root = [IO.Path]::GetFullPath((Join-Path $RuntimeRoot $relativeRoot))
if (-not $root.StartsWith(
($RuntimeRoot + "\"),
[StringComparison]::OrdinalIgnoreCase
)) {
throw "$asset resolved outside the fixed local runtime root"
}
$outputs.Add((Seal-TreeAsset $root $asset))
}
@($outputs.ToArray()) | ConvertTo-Json -Compress -Depth 8