feat(lab): install sealed component adapters
This commit is contained in:
DCCONSTRUCTIONS
@@ -1,4 +1,4 @@
|
|||||||
FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1
|
FROM sha256:591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd
|
||||||
|
|
||||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||||
PYTHONUNBUFFERED=1
|
PYTHONUNBUFFERED=1
|
||||||
|
|||||||
+237
@@ -0,0 +1,237 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param()
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$ProgressPreference = "SilentlyContinue"
|
||||||
|
|
||||||
|
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
|
||||||
|
$SourceRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-static-v1"
|
||||||
|
$AssetRoot = Join-Path $RuntimeRoot "assets\observatory-portable"
|
||||||
|
$RunnerRoot = Join-Path $AssetRoot "eomt-runner-bundle-v1"
|
||||||
|
$ValidFovRoot = Join-Path $AssetRoot (
|
||||||
|
"k1-valid-fov-root-" +
|
||||||
|
"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1"
|
||||||
|
)
|
||||||
|
$StaticRoot = Join-Path $AssetRoot "lab-v1-static-files-v1"
|
||||||
|
$ValidFovSource = Join-Path $RuntimeRoot (
|
||||||
|
"inputs\e2\valid-fov-mask-" +
|
||||||
|
"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"
|
||||||
|
)
|
||||||
|
$CheckpointSource = (
|
||||||
|
"D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27\" +
|
||||||
|
"models\goose\ddrnet_class_512.pth"
|
||||||
|
)
|
||||||
|
$MappingSource = (
|
||||||
|
"D:\NDC_MISSIONCORE\datasets\goose-3d\v2025-08-22\installs\" +
|
||||||
|
"0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f\" +
|
||||||
|
"goose_label_mapping.csv"
|
||||||
|
)
|
||||||
|
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||||
|
|
||||||
|
$RunnerFiles = [ordered]@{
|
||||||
|
"e3_k1_camera1_profile.json" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "e3_k1_camera1_profile.json")
|
||||||
|
byte_length = [int64]3805
|
||||||
|
sha256 = "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||||
|
}
|
||||||
|
"run_e3_rectified_segmentation.py" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "run_e3_rectified_segmentation.py")
|
||||||
|
byte_length = [int64]45789
|
||||||
|
sha256 = "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6"
|
||||||
|
}
|
||||||
|
"run_e4_full_session_segmentation.py" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "run_e4_full_session_segmentation.py")
|
||||||
|
byte_length = [int64]30720
|
||||||
|
sha256 = "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||||
|
}
|
||||||
|
"run_evaluation_prelabels.py" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "run_evaluation_prelabels.py")
|
||||||
|
byte_length = [int64]28915
|
||||||
|
sha256 = "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4"
|
||||||
|
}
|
||||||
|
"run_recorded_perception_epoch.py" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "run_recorded_perception_epoch.py")
|
||||||
|
byte_length = [int64]34899
|
||||||
|
sha256 = "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$ValidFovFiles = [ordered]@{
|
||||||
|
"manifest.json" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $ValidFovSource "manifest.json")
|
||||||
|
byte_length = [int64]2351
|
||||||
|
sha256 = "3dea056db506f47b8a7672e847d0ffe4c66a380a5032555a2e0dde958293742e"
|
||||||
|
}
|
||||||
|
"mask.png" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $ValidFovSource "mask.png")
|
||||||
|
byte_length = [int64]3668
|
||||||
|
sha256 = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$StaticFiles = [ordered]@{
|
||||||
|
"ddrnet-checkpoint.pth" = [pscustomobject]@{
|
||||||
|
source = $CheckpointSource
|
||||||
|
byte_length = [int64]259419077
|
||||||
|
sha256 = "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||||
|
}
|
||||||
|
"ddrnet-goose-mapping.csv" = [pscustomobject]@{
|
||||||
|
source = $MappingSource
|
||||||
|
byte_length = [int64]1427
|
||||||
|
sha256 = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
|
||||||
|
}
|
||||||
|
"ddrnet-goose-runner.py" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "run_goose_vegetation_benchmark.py")
|
||||||
|
byte_length = [int64]32877
|
||||||
|
sha256 = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"
|
||||||
|
}
|
||||||
|
"vegetation-policy.json" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "lab-v1-vegetation-mission-policy-v1.json")
|
||||||
|
byte_length = [int64]3022
|
||||||
|
sha256 = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||||
|
}
|
||||||
|
"vegetation-provider-map.json" = [pscustomobject]@{
|
||||||
|
source = (Join-Path $SourceRoot "lab-v1-vegetation-provider-label-map-v1.json")
|
||||||
|
byte_length = [int64]2756
|
||||||
|
sha256 = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-FileIdentity {
|
||||||
|
param([string]$Path)
|
||||||
|
|
||||||
|
$item = Get-Item -LiteralPath $Path -Force
|
||||||
|
if (
|
||||||
|
-not ($item -is [IO.FileInfo]) -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||||
|
) {
|
||||||
|
throw "LAB V1 static source is not a regular file"
|
||||||
|
}
|
||||||
|
$stream = [IO.File]::Open(
|
||||||
|
$item.FullName,
|
||||||
|
[IO.FileMode]::Open,
|
||||||
|
[IO.FileAccess]::Read,
|
||||||
|
[IO.FileShare]::Read
|
||||||
|
)
|
||||||
|
$algorithm = [Security.Cryptography.SHA256]::Create()
|
||||||
|
try {
|
||||||
|
$length = [int64]$stream.Length
|
||||||
|
$sha256 = -join @($algorithm.ComputeHash($stream) | ForEach-Object {
|
||||||
|
$_.ToString("x2", $Invariant)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
$algorithm.Dispose()
|
||||||
|
$stream.Dispose()
|
||||||
|
}
|
||||||
|
return [pscustomobject]@{
|
||||||
|
byte_length = $length
|
||||||
|
sha256 = $sha256
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-ExactFile {
|
||||||
|
param(
|
||||||
|
[string]$Path,
|
||||||
|
[object]$Descriptor,
|
||||||
|
[string]$Label
|
||||||
|
)
|
||||||
|
|
||||||
|
$identity = Get-FileIdentity $Path
|
||||||
|
if (
|
||||||
|
[int64]$identity.byte_length -ne [int64]$Descriptor.byte_length -or
|
||||||
|
[string]$identity.sha256 -cne [string]$Descriptor.sha256
|
||||||
|
) {
|
||||||
|
throw "$Label identity changed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Install-HardlinkedDirectory {
|
||||||
|
param(
|
||||||
|
[string]$Target,
|
||||||
|
[Collections.IDictionary]$Files,
|
||||||
|
[bool]$AllowTreeSealMetadata
|
||||||
|
)
|
||||||
|
|
||||||
|
foreach ($name in $Files.Keys) {
|
||||||
|
Assert-ExactFile $Files[$name].source $Files[$name] "LAB V1 source $name"
|
||||||
|
}
|
||||||
|
if (Test-Path -LiteralPath $Target -PathType Container) {
|
||||||
|
$allowed = @($Files.Keys)
|
||||||
|
if ($AllowTreeSealMetadata) {
|
||||||
|
$allowed += @("tree-manifest.tsv", "tree-receipt.json")
|
||||||
|
}
|
||||||
|
$observed = @(
|
||||||
|
Get-ChildItem -LiteralPath $Target -Force | ForEach-Object {
|
||||||
|
if (-not ($_ -is [IO.FileInfo])) {
|
||||||
|
throw "Installed LAB V1 asset contains a non-file member"
|
||||||
|
}
|
||||||
|
$_.Name
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
@($observed | Sort-Object).Count -ne @($allowed | Sort-Object).Count -or
|
||||||
|
[string]::Join([char]10, @($observed | Sort-Object)) -cne
|
||||||
|
[string]::Join([char]10, @($allowed | Sort-Object))
|
||||||
|
) {
|
||||||
|
throw "Installed LAB V1 asset inventory changed"
|
||||||
|
}
|
||||||
|
foreach ($name in $Files.Keys) {
|
||||||
|
Assert-ExactFile (Join-Path $Target $name) $Files[$name] "Installed $name"
|
||||||
|
}
|
||||||
|
return "already-installed"
|
||||||
|
}
|
||||||
|
if (Test-Path -LiteralPath $Target) {
|
||||||
|
throw "LAB V1 asset target is not a directory"
|
||||||
|
}
|
||||||
|
$parent = Split-Path -Parent $Target
|
||||||
|
if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
|
||||||
|
throw "LAB V1 asset parent is unavailable"
|
||||||
|
}
|
||||||
|
$staging = "$Target.staging-$([Guid]::NewGuid().ToString('N'))"
|
||||||
|
try {
|
||||||
|
$null = New-Item -ItemType Directory -Path $staging
|
||||||
|
foreach ($name in $Files.Keys) {
|
||||||
|
$null = New-Item `
|
||||||
|
-ItemType HardLink `
|
||||||
|
-Path (Join-Path $staging $name) `
|
||||||
|
-Target $Files[$name].source
|
||||||
|
Assert-ExactFile (Join-Path $staging $name) $Files[$name] "Staged $name"
|
||||||
|
}
|
||||||
|
[IO.Directory]::Move($staging, $Target)
|
||||||
|
return "installed"
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
if (Test-Path -LiteralPath $staging -PathType Container) {
|
||||||
|
Remove-Item -LiteralPath $staging -Recurse -Force
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $SourceRoot -PathType Container)) {
|
||||||
|
throw "LAB V1 static source staging root is unavailable"
|
||||||
|
}
|
||||||
|
if (-not (Test-Path -LiteralPath $AssetRoot -PathType Container)) {
|
||||||
|
throw "LAB V1 immutable asset root is unavailable"
|
||||||
|
}
|
||||||
|
|
||||||
|
$outputs = @(
|
||||||
|
[ordered]@{
|
||||||
|
asset_id = "eomt-runner-bundle"
|
||||||
|
status = Install-HardlinkedDirectory $RunnerRoot $RunnerFiles $true
|
||||||
|
path = $RunnerRoot
|
||||||
|
}
|
||||||
|
[ordered]@{
|
||||||
|
asset_id = "k1-valid-fov-root"
|
||||||
|
status = Install-HardlinkedDirectory $ValidFovRoot $ValidFovFiles $true
|
||||||
|
path = $ValidFovRoot
|
||||||
|
}
|
||||||
|
[ordered]@{
|
||||||
|
asset_id = "lab-v1-static-files"
|
||||||
|
status = Install-HardlinkedDirectory $StaticRoot $StaticFiles $false
|
||||||
|
path = $StaticRoot
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
$outputs | ConvertTo-Json -Compress -Depth 4
|
||||||
@@ -28,13 +28,41 @@ $IdentityAlgorithm = "relative-path-tab-size-tab-file-sha256-lf/v1"
|
|||||||
$MaximumManifestBytes = 32MB
|
$MaximumManifestBytes = 32MB
|
||||||
$Utf8 = New-Object Text.UTF8Encoding($false, $true)
|
$Utf8 = New-Object Text.UTF8Encoding($false, $true)
|
||||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||||
|
$OriginalModelCache = (
|
||||||
|
"D:\NDC_MISSIONCORE\runtime\cache\perception-e3-models-v1\huggingface\" +
|
||||||
|
"models--tue-mps--cityscapes_semantic_eomt_large_1024"
|
||||||
|
)
|
||||||
|
$SafeModelCache = (
|
||||||
|
"D:\NDC_MISSIONCORE\runtime\assets\observatory-portable\" +
|
||||||
|
"eomt-model-cache-cityscapes-large-1024-v1"
|
||||||
|
)
|
||||||
|
$ModelRevision = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||||
|
$ModelFiles = [ordered]@{
|
||||||
|
"config.json" = [pscustomobject]@{
|
||||||
|
blob = "66b2d540c2a4d9e9fbc958c1a43464cb13d933c5"
|
||||||
|
byte_length = [int64]1575
|
||||||
|
sha256 = "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"
|
||||||
|
}
|
||||||
|
"model.safetensors" = [pscustomobject]@{
|
||||||
|
blob = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||||
|
byte_length = [int64]1276175488
|
||||||
|
sha256 = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||||
|
}
|
||||||
|
"preprocessor_config.json" = [pscustomobject]@{
|
||||||
|
blob = "cf30bcc29300598a6b581ed043dd65abc893e948"
|
||||||
|
byte_length = [int64]666
|
||||||
|
sha256 = "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# The first four roots already exist on Worker 006. The runner bundle and
|
# The first four roots already exist on Worker 006. The runner bundle and
|
||||||
# valid-FOV root deliberately point at separate, immutable asset directories;
|
# valid-FOV root deliberately point at separate, immutable asset directories;
|
||||||
# their payloads must be materialized there before this sealer is invoked.
|
# their payloads must be materialized there before this sealer is invoked.
|
||||||
$AssetRelativeRoots = [ordered]@{
|
$AssetRelativeRoots = [ordered]@{
|
||||||
"eomt-environment" = "derived\perception-e3-opencv413092-v1"
|
"eomt-environment" = "derived\perception-e3-opencv413092-v1"
|
||||||
"eomt-model-cache" = "cache\perception-e3-models-v1"
|
"eomt-model-cache" = (
|
||||||
|
"assets\observatory-portable\eomt-model-cache-cityscapes-large-1024-v1"
|
||||||
|
)
|
||||||
"eomt-python-environment" = "derived\perception-p0-env-v1"
|
"eomt-python-environment" = "derived\perception-p0-env-v1"
|
||||||
"eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1"
|
"eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1"
|
||||||
"eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1"
|
"eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1"
|
||||||
@@ -152,6 +180,58 @@ function Get-FileDigest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function Install-SafeEomtModelCache {
|
||||||
|
if (Test-Path -LiteralPath $SafeModelCache -PathType Container) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (Test-Path -LiteralPath $SafeModelCache) {
|
||||||
|
throw "Safe EoMT model cache target is not a directory"
|
||||||
|
}
|
||||||
|
$assetParent = Split-Path -Parent $SafeModelCache
|
||||||
|
if (-not (Test-Path -LiteralPath $assetParent -PathType Container)) {
|
||||||
|
throw "Safe EoMT model cache parent is unavailable"
|
||||||
|
}
|
||||||
|
$staging = "$SafeModelCache.staging-$([Guid]::NewGuid().ToString('N'))"
|
||||||
|
try {
|
||||||
|
$blobs = Join-Path $staging (
|
||||||
|
"huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\blobs"
|
||||||
|
)
|
||||||
|
$snapshot = Join-Path $staging (
|
||||||
|
"huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\" +
|
||||||
|
"snapshots\$ModelRevision"
|
||||||
|
)
|
||||||
|
$null = New-Item -ItemType Directory -Path $blobs
|
||||||
|
$null = New-Item -ItemType Directory -Path $snapshot
|
||||||
|
foreach ($name in $ModelFiles.Keys) {
|
||||||
|
$descriptor = $ModelFiles[$name]
|
||||||
|
$source = Join-Path (Join-Path $OriginalModelCache "blobs") $descriptor.blob
|
||||||
|
$identity = Get-FileDigest $source
|
||||||
|
if (
|
||||||
|
[int64]$identity.byte_length -ne [int64]$descriptor.byte_length -or
|
||||||
|
[string]$identity.sha256 -cne [string]$descriptor.sha256
|
||||||
|
) {
|
||||||
|
throw "Original EoMT model cache identity changed"
|
||||||
|
}
|
||||||
|
$blobTarget = Join-Path $blobs $descriptor.blob
|
||||||
|
$snapshotTarget = Join-Path $snapshot $name
|
||||||
|
$null = New-Item -ItemType HardLink -Path $blobTarget -Target $source
|
||||||
|
$null = New-Item -ItemType HardLink -Path $snapshotTarget -Target $source
|
||||||
|
}
|
||||||
|
foreach ($entry in [IO.Directory]::EnumerateFiles($staging, "*", "AllDirectories")) {
|
||||||
|
$attributes = [IO.File]::GetAttributes($entry)
|
||||||
|
if ($attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||||
|
throw "Safe EoMT model cache contains a reparse point"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
[IO.Directory]::Move($staging, $SafeModelCache)
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
if (Test-Path -LiteralPath $staging -PathType Container) {
|
||||||
|
Remove-Item -LiteralPath $staging -Recurse -Force
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function Get-PayloadInventory {
|
function Get-PayloadInventory {
|
||||||
param(
|
param(
|
||||||
[string]$Root,
|
[string]$Root,
|
||||||
@@ -388,7 +468,8 @@ function Assert-InventoryMatches {
|
|||||||
[object[]]$Expected,
|
[object[]]$Expected,
|
||||||
[object[]]$Observed,
|
[object[]]$Observed,
|
||||||
[string]$Asset,
|
[string]$Asset,
|
||||||
[bool]$CheckWriteTime
|
[bool]$CheckWriteTime,
|
||||||
|
[bool]$CheckDigest
|
||||||
)
|
)
|
||||||
|
|
||||||
if ($Expected.Count -ne $Observed.Count) {
|
if ($Expected.Count -ne $Observed.Count) {
|
||||||
@@ -400,6 +481,9 @@ function Assert-InventoryMatches {
|
|||||||
if (
|
if (
|
||||||
[string]$left.relative_path -cne [string]$right.relative_path -or
|
[string]$left.relative_path -cne [string]$right.relative_path -or
|
||||||
[int64]$left.byte_length -ne [int64]$right.byte_length -or
|
[int64]$left.byte_length -ne [int64]$right.byte_length -or
|
||||||
|
($CheckDigest -and (
|
||||||
|
[string]$left.sha256 -cne [string]$right.sha256
|
||||||
|
)) -or
|
||||||
($CheckWriteTime -and (
|
($CheckWriteTime -and (
|
||||||
[int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks
|
[int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks
|
||||||
))
|
))
|
||||||
@@ -438,10 +522,11 @@ function Test-ExistingSeal {
|
|||||||
throw "Existing $Asset receipt does not bind its manifest"
|
throw "Existing $Asset receipt does not bind its manifest"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Fast path: verify names and sizes, but deliberately do not re-hash the
|
# A seal is reusable only after the payload has been re-verified. Tree
|
||||||
# already sealed multi-gigabyte payload.
|
# roots contain executable Python dependencies, so names and sizes alone
|
||||||
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
|
# must never authenticate an existing seal.
|
||||||
Assert-InventoryMatches $manifest.records $observed $Asset $false
|
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$true)
|
||||||
|
Assert-InventoryMatches $manifest.records $observed $Asset $false $true
|
||||||
return $receipt
|
return $receipt
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -519,7 +604,7 @@ function Seal-TreeAsset {
|
|||||||
# Re-enumerate without hashing before publication. This catches file,
|
# Re-enumerate without hashing before publication. This catches file,
|
||||||
# size, and timestamp changes that occurred during the one-time hash.
|
# size, and timestamp changes that occurred during the one-time hash.
|
||||||
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
|
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
|
||||||
Assert-InventoryMatches $records $observed $Asset $true
|
Assert-InventoryMatches $records $observed $Asset $true $false
|
||||||
|
|
||||||
$null = New-Item -ItemType Directory -Path $staging
|
$null = New-Item -ItemType Directory -Path $staging
|
||||||
$stagedManifest = Join-Path $staging $ManifestName
|
$stagedManifest = Join-Path $staging $ManifestName
|
||||||
@@ -573,6 +658,9 @@ $requested = @($AssetId)
|
|||||||
if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) {
|
if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) {
|
||||||
throw "LAB V1 EoMT asset selection is empty or contains duplicates"
|
throw "LAB V1 EoMT asset selection is empty or contains duplicates"
|
||||||
}
|
}
|
||||||
|
if ($requested -contains "eomt-model-cache") {
|
||||||
|
Install-SafeEomtModelCache
|
||||||
|
}
|
||||||
$outputs = New-Object "Collections.Generic.List[object]"
|
$outputs = New-Object "Collections.Generic.List[object]"
|
||||||
foreach ($asset in $requested) {
|
foreach ($asset in $requested) {
|
||||||
$relativeRoot = [string]$AssetRelativeRoots[$asset]
|
$relativeRoot = [string]$AssetRelativeRoots[$asset]
|
||||||
|
|||||||
+1116
File diff suppressed because it is too large
Load Diff
+828
@@ -0,0 +1,828 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run the sealed LAB V1 DDRNet component over EoMT-decoded K1 frames."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final, cast
|
||||||
|
|
||||||
|
from PIL import Image
|
||||||
|
from portable_lab_v1_component_adapter import (
|
||||||
|
DDRNET_CHECKPOINT_SHA256,
|
||||||
|
EOMT_PIPELINE,
|
||||||
|
GOOSE_MAPPING_SHA256,
|
||||||
|
AssetExpectation,
|
||||||
|
CommandRunner,
|
||||||
|
ComponentAdapterError,
|
||||||
|
ComponentRequest,
|
||||||
|
RuntimeLayout,
|
||||||
|
canonical_json,
|
||||||
|
load_canonical_json,
|
||||||
|
load_component_request,
|
||||||
|
load_json,
|
||||||
|
parse_fixed_request_argument,
|
||||||
|
run_command,
|
||||||
|
sha256_file,
|
||||||
|
validate_camera_compute_job,
|
||||||
|
validate_file_asset,
|
||||||
|
validate_fixed_result,
|
||||||
|
validate_grayscale_png_payload,
|
||||||
|
)
|
||||||
|
|
||||||
|
DDRNET_RESULT_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||||
|
EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2"
|
||||||
|
EFFECTIVE_CONFIG_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-benchmark/v1"
|
||||||
|
DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1"
|
||||||
|
DDRNET_CANDIDATE_ID: Final = "goose-ddrnet-class-512"
|
||||||
|
DDRNET_LOADED_MODEL: Final = "ddrnet_39"
|
||||||
|
DDRNET_CHECKPOINT_BYTES: Final = 259_419_077
|
||||||
|
DDRNET_STATIC_CONFIG_SHA256: Final = (
|
||||||
|
"3c2a60def178d824e6ce612cad9a108da496de769edf0de66099d2ecca6b51bb"
|
||||||
|
)
|
||||||
|
RUNNER_SHA256: Final = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"
|
||||||
|
POLICY_SHA256: Final = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||||
|
PROVIDER_MAP_SHA256: Final = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||||
|
DDRNET_AUTHORITY: Final = {
|
||||||
|
"navigation_accepted": False,
|
||||||
|
"safety_accepted": False,
|
||||||
|
"actuation_accepted": False,
|
||||||
|
"camera_semantics_can_clear_rigid_geometry": False,
|
||||||
|
}
|
||||||
|
EOMT_MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||||
|
EOMT_MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||||
|
EOMT_MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation"
|
||||||
|
SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SourceFrameRow:
|
||||||
|
sequence: int
|
||||||
|
path: str
|
||||||
|
byte_length: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
DDRNET_ASSETS: Final = (
|
||||||
|
AssetExpectation(
|
||||||
|
"ddrnet-checkpoint",
|
||||||
|
"/opt/nodedc/assets/ddrnet-checkpoint",
|
||||||
|
"file",
|
||||||
|
"sha256",
|
||||||
|
DDRNET_CHECKPOINT_SHA256,
|
||||||
|
DDRNET_CHECKPOINT_BYTES,
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"ddrnet-goose-mapping",
|
||||||
|
"/opt/nodedc/assets/ddrnet-goose-mapping",
|
||||||
|
"file",
|
||||||
|
"sha256",
|
||||||
|
GOOSE_MAPPING_SHA256,
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"ddrnet-goose-runner",
|
||||||
|
"/opt/nodedc/assets/ddrnet-goose-runner",
|
||||||
|
"file",
|
||||||
|
"sha256",
|
||||||
|
RUNNER_SHA256,
|
||||||
|
32_877,
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"vegetation-policy",
|
||||||
|
"/opt/nodedc/assets/vegetation-policy",
|
||||||
|
"file",
|
||||||
|
"sha256",
|
||||||
|
POLICY_SHA256,
|
||||||
|
3_022,
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"vegetation-provider-map",
|
||||||
|
"/opt/nodedc/assets/vegetation-provider-map",
|
||||||
|
"file",
|
||||||
|
"sha256",
|
||||||
|
PROVIDER_MAP_SHA256,
|
||||||
|
2_756,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def execute_ddrnet_component(
|
||||||
|
*,
|
||||||
|
request_path: Path,
|
||||||
|
layout: RuntimeLayout,
|
||||||
|
command_runner: CommandRunner = run_command,
|
||||||
|
work_root: Path = Path("/tmp/nodedc-lab-v1-ddrnet"),
|
||||||
|
) -> None:
|
||||||
|
request = load_component_request(
|
||||||
|
request_path,
|
||||||
|
component="ddrnet",
|
||||||
|
expectations=DDRNET_ASSETS,
|
||||||
|
)
|
||||||
|
validate_camera_compute_job(layout.camera_job_root, request.source)
|
||||||
|
assets = {
|
||||||
|
item.asset_id: validate_file_asset(request, layout, item.asset_id) for item in DDRNET_ASSETS
|
||||||
|
}
|
||||||
|
effective = _validate_effective_config(
|
||||||
|
layout.effective_ddrnet_config,
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
eomt_root, source_frames, decode_repair = _validate_eomt_input(
|
||||||
|
layout.eomt_result_root,
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
output = _empty_output_root(layout.output_root)
|
||||||
|
workspace = _prepare_workspace(work_root)
|
||||||
|
try:
|
||||||
|
frames_root = _snapshot_source_frames(
|
||||||
|
eomt_root,
|
||||||
|
source_frames,
|
||||||
|
workspace / "source-frames",
|
||||||
|
)
|
||||||
|
mapping_copy = workspace / "goose_label_mapping.csv"
|
||||||
|
shutil.copyfile(assets["ddrnet-goose-mapping"], mapping_copy)
|
||||||
|
os.chmod(mapping_copy, 0o400)
|
||||||
|
if sha256_file(mapping_copy) != GOOSE_MAPPING_SHA256:
|
||||||
|
raise ComponentAdapterError("DDRNet private mapping copy changed")
|
||||||
|
staging = output / ".ddrnet-component"
|
||||||
|
argv = (
|
||||||
|
sys.executable,
|
||||||
|
str(assets["ddrnet-goose-runner"]),
|
||||||
|
"--mode",
|
||||||
|
"ravnoves-video",
|
||||||
|
"--candidate",
|
||||||
|
"ddrnet",
|
||||||
|
"--config",
|
||||||
|
str(layout.effective_ddrnet_config),
|
||||||
|
"--policy",
|
||||||
|
str(assets["vegetation-policy"]),
|
||||||
|
"--provider-map",
|
||||||
|
str(assets["vegetation-provider-map"]),
|
||||||
|
"--checkpoint",
|
||||||
|
str(assets["ddrnet-checkpoint"]),
|
||||||
|
"--dataset-root",
|
||||||
|
str(workspace),
|
||||||
|
"--frames-root",
|
||||||
|
str(frames_root),
|
||||||
|
"--output",
|
||||||
|
str(staging),
|
||||||
|
"--limit",
|
||||||
|
"0",
|
||||||
|
"--visual-count",
|
||||||
|
"0",
|
||||||
|
)
|
||||||
|
command_runner(argv, _ddrnet_environment())
|
||||||
|
shutil.copyfile(eomt_root / "decode-repair.json", staging / "decode-repair.json")
|
||||||
|
if (
|
||||||
|
load_json(
|
||||||
|
staging / "decode-repair.json",
|
||||||
|
label="DDRNet decode repair",
|
||||||
|
maximum=1024 * 1024,
|
||||||
|
confinement_root=staging,
|
||||||
|
)
|
||||||
|
!= decode_repair
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet decode repair copy changed")
|
||||||
|
_validate_ddrnet_result(
|
||||||
|
staging,
|
||||||
|
request,
|
||||||
|
effective,
|
||||||
|
mapping_path=assets["ddrnet-goose-mapping"],
|
||||||
|
)
|
||||||
|
_publish_component_children(
|
||||||
|
staging,
|
||||||
|
output,
|
||||||
|
expected_names={
|
||||||
|
"decode-repair.json",
|
||||||
|
"result.json",
|
||||||
|
"semantic-masks.zip",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(workspace, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_effective_config(
|
||||||
|
path: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
config = load_canonical_json(
|
||||||
|
path,
|
||||||
|
label="effective DDRNet config",
|
||||||
|
maximum=1024 * 1024,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
request.effective_ddrnet_config_sha256 is None
|
||||||
|
or sha256_file(path) != request.effective_ddrnet_config_sha256
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("effective DDRNet config identity changed")
|
||||||
|
expected_keys = {
|
||||||
|
"schema_version",
|
||||||
|
"lab_id",
|
||||||
|
"worker_id",
|
||||||
|
"runtime",
|
||||||
|
"dataset",
|
||||||
|
"candidates",
|
||||||
|
"vegetation_class_names",
|
||||||
|
"visual_case_contract",
|
||||||
|
"policy_action_colors",
|
||||||
|
"invariants",
|
||||||
|
"ravnoves",
|
||||||
|
}
|
||||||
|
if set(config) != expected_keys:
|
||||||
|
raise ComponentAdapterError("effective DDRNet config fields changed")
|
||||||
|
static = dict(config)
|
||||||
|
ravnoves = static.pop("ravnoves")
|
||||||
|
if (
|
||||||
|
config.get("schema_version") != EFFECTIVE_CONFIG_SCHEMA
|
||||||
|
or config.get("lab_id") != "LAB-V1"
|
||||||
|
or config.get("worker_id") != "worker-006"
|
||||||
|
or hashlib.sha256(canonical_json(static)).hexdigest() != DDRNET_STATIC_CONFIG_SHA256
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("effective DDRNet static configuration changed")
|
||||||
|
source = request.source
|
||||||
|
expected_ravnoves = {
|
||||||
|
"source_id": (
|
||||||
|
f"portable-k1/{source.source_session_id}/"
|
||||||
|
f"{source.camera_source_id}@{source.camera_input_sha256}"
|
||||||
|
),
|
||||||
|
"source_sha256": source.camera_input_sha256,
|
||||||
|
"base_m4_result_id": None,
|
||||||
|
"expected_width": 800,
|
||||||
|
"expected_height": 600,
|
||||||
|
"expected_frame_count": source.frame_count,
|
||||||
|
"frame_indices": _representative_frame_indices(source.frame_count),
|
||||||
|
"crop_contract": "center-600-square-to-512; outside-crop-is-undefined",
|
||||||
|
}
|
||||||
|
if ravnoves != expected_ravnoves:
|
||||||
|
raise ComponentAdapterError("effective DDRNet source configuration changed")
|
||||||
|
return config
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_eomt_input(
|
||||||
|
root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> tuple[Path, tuple[SourceFrameRow, ...], dict[str, object]]:
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ComponentAdapterError("EoMT result root is a symbolic link")
|
||||||
|
resolved = root.resolve(strict=True)
|
||||||
|
result = validate_fixed_result(
|
||||||
|
resolved,
|
||||||
|
schema_version=EOMT_RESULT_SCHEMA,
|
||||||
|
label="EoMT",
|
||||||
|
)
|
||||||
|
identity = result.get("identity")
|
||||||
|
if not isinstance(identity, dict):
|
||||||
|
raise ComponentAdapterError("EoMT result identity is unavailable")
|
||||||
|
configuration = identity.get("configuration")
|
||||||
|
models = identity.get("models")
|
||||||
|
semantic = models.get("semantic") if isinstance(models, dict) else None
|
||||||
|
observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||||
|
source = request.source
|
||||||
|
if (
|
||||||
|
not isinstance(configuration, dict)
|
||||||
|
or result.get("identity_sha256") != observed_identity
|
||||||
|
or result.get("result_id") != f"result-{observed_identity}"
|
||||||
|
or result.get("job_id") != source.camera_job_id
|
||||||
|
or result.get("input_sha256") != source.camera_input_sha256
|
||||||
|
or result.get("session_id") != source.source_session_id
|
||||||
|
or result.get("source_id") != source.camera_source_id
|
||||||
|
or result.get("codec_epoch") != source.codec_epoch
|
||||||
|
or result.get("timestamp_basis") != "session-time-seconds"
|
||||||
|
or result.get("timeline_start_seconds") != source.timeline_start_seconds
|
||||||
|
or result.get("timeline_end_seconds") != source.timeline_end_seconds
|
||||||
|
or result.get("frames_processed") != source.frame_count
|
||||||
|
or result.get("ground_truth") is not False
|
||||||
|
or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2"
|
||||||
|
or configuration.get("pipeline") != EOMT_PIPELINE
|
||||||
|
or not isinstance(semantic, dict)
|
||||||
|
or semantic.get("id") != EOMT_MODEL_ID
|
||||||
|
or semantic.get("revision") != EOMT_MODEL_REVISION
|
||||||
|
or semantic.get("architecture") != EOMT_MODEL_ARCHITECTURE
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source result changed")
|
||||||
|
artifacts = result.get("artifacts")
|
||||||
|
if not isinstance(artifacts, list) or len(artifacts) != 6:
|
||||||
|
raise ComponentAdapterError("EoMT source artifact set changed")
|
||||||
|
expected_artifacts = {
|
||||||
|
"panoptic-overlay-video": ("perception.mp4", "video/mp4"),
|
||||||
|
"panoptic-mask-archive": ("masks.tar.gz", "application/gzip"),
|
||||||
|
"panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"),
|
||||||
|
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"),
|
||||||
|
"perception-run-report": ("run-report.json", "application/json"),
|
||||||
|
"decoded-source-frame-manifest": ("source-frames.json", "application/json"),
|
||||||
|
}
|
||||||
|
observed_kinds: set[str] = set()
|
||||||
|
for value in artifacts:
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise ComponentAdapterError("EoMT source artifact is invalid")
|
||||||
|
kind = value.get("kind")
|
||||||
|
encoded = value.get("path")
|
||||||
|
if (
|
||||||
|
not isinstance(kind, str)
|
||||||
|
or kind in observed_kinds
|
||||||
|
or expected_artifacts.get(kind) != (encoded, value.get("media_type"))
|
||||||
|
or not isinstance(encoded, str)
|
||||||
|
or not encoded
|
||||||
|
or "/" in encoded
|
||||||
|
or "\\" in encoded
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source artifact path changed")
|
||||||
|
observed_kinds.add(kind)
|
||||||
|
artifact = _confined_regular_file(resolved / encoded, resolved, "EoMT artifact")
|
||||||
|
if (
|
||||||
|
artifact.stat().st_size < 1
|
||||||
|
or value.get("byte_length") != artifact.stat().st_size
|
||||||
|
or value.get("sha256") != sha256_file(artifact)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source artifact identity changed")
|
||||||
|
if observed_kinds != set(expected_artifacts):
|
||||||
|
raise ComponentAdapterError("EoMT source artifact roles changed")
|
||||||
|
manifest_path = _confined_regular_file(
|
||||||
|
resolved / "source-frames.json",
|
||||||
|
resolved,
|
||||||
|
"EoMT source frame manifest",
|
||||||
|
)
|
||||||
|
source_frame_binding = identity.get("source_frames")
|
||||||
|
expected_binding = {
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"path": "source-frames.json",
|
||||||
|
"sha256": sha256_file(manifest_path),
|
||||||
|
"byte_length": manifest_path.stat().st_size,
|
||||||
|
"frame_count": source.frame_count,
|
||||||
|
}
|
||||||
|
if source_frame_binding != expected_binding:
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest binding changed")
|
||||||
|
source_frames = _source_frame_manifest(manifest_path, resolved, request)
|
||||||
|
repair = load_json(
|
||||||
|
resolved / "decode-repair.json",
|
||||||
|
label="EoMT decode repair",
|
||||||
|
maximum=1024 * 1024,
|
||||||
|
confinement_root=resolved,
|
||||||
|
)
|
||||||
|
_validate_decode_repair(repair, source.frame_count)
|
||||||
|
return resolved, source_frames, repair
|
||||||
|
|
||||||
|
|
||||||
|
def _source_frame_manifest(
|
||||||
|
path: Path,
|
||||||
|
eomt_root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> tuple[SourceFrameRow, ...]:
|
||||||
|
document = load_canonical_json(
|
||||||
|
path,
|
||||||
|
label="EoMT source frame manifest",
|
||||||
|
maximum=32 * 1024 * 1024,
|
||||||
|
confinement_root=eomt_root,
|
||||||
|
)
|
||||||
|
source = request.source
|
||||||
|
if set(document) != {
|
||||||
|
"schema_version",
|
||||||
|
"job_id",
|
||||||
|
"input_sha256",
|
||||||
|
"source_id",
|
||||||
|
"codec_epoch",
|
||||||
|
"frame_count",
|
||||||
|
"frames",
|
||||||
|
} or document != {
|
||||||
|
**document,
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"job_id": source.camera_job_id,
|
||||||
|
"input_sha256": source.camera_input_sha256,
|
||||||
|
"source_id": source.camera_source_id,
|
||||||
|
"codec_epoch": source.codec_epoch,
|
||||||
|
"frame_count": source.frame_count,
|
||||||
|
}:
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest identity changed")
|
||||||
|
raw_rows = document["frames"]
|
||||||
|
if not isinstance(raw_rows, list) or len(raw_rows) != source.frame_count:
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest row count changed")
|
||||||
|
rows: list[SourceFrameRow] = []
|
||||||
|
for sequence, value in enumerate(raw_rows, start=1):
|
||||||
|
if not isinstance(value, dict) or set(value) != {
|
||||||
|
"sequence",
|
||||||
|
"path",
|
||||||
|
"byte_length",
|
||||||
|
"sha256",
|
||||||
|
}:
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest row changed")
|
||||||
|
encoded = f"source-frames/frame-{sequence:06d}.png"
|
||||||
|
byte_length = value.get("byte_length")
|
||||||
|
digest = value.get("sha256")
|
||||||
|
if (
|
||||||
|
value.get("sequence") != sequence
|
||||||
|
or value.get("path") != encoded
|
||||||
|
or isinstance(byte_length, bool)
|
||||||
|
or not isinstance(byte_length, int)
|
||||||
|
or not 0 < byte_length <= 16 * 1024 * 1024
|
||||||
|
or not isinstance(digest, str)
|
||||||
|
or len(digest) != 64
|
||||||
|
or any(character not in "0123456789abcdef" for character in digest)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest row identity changed")
|
||||||
|
frame = _confined_regular_file(
|
||||||
|
eomt_root / encoded,
|
||||||
|
eomt_root,
|
||||||
|
"EoMT source frame",
|
||||||
|
)
|
||||||
|
if frame.stat().st_size != byte_length:
|
||||||
|
raise ComponentAdapterError("EoMT source frame length changed")
|
||||||
|
rows.append(SourceFrameRow(sequence, encoded, byte_length, digest))
|
||||||
|
frames_root = eomt_root / "source-frames"
|
||||||
|
expected_names = tuple(
|
||||||
|
f"frame-{sequence:06d}.png" for sequence in range(1, source.frame_count + 1)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
observed_names = tuple(sorted(candidate.name for candidate in frames_root.iterdir()))
|
||||||
|
except OSError as exc:
|
||||||
|
raise ComponentAdapterError("EoMT source frame set is unavailable") from exc
|
||||||
|
if observed_names != expected_names:
|
||||||
|
raise ComponentAdapterError("EoMT source frame set changed")
|
||||||
|
return tuple(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_source_frames(
|
||||||
|
eomt_root: Path,
|
||||||
|
rows: Sequence[SourceFrameRow],
|
||||||
|
snapshot_root: Path,
|
||||||
|
) -> Path:
|
||||||
|
snapshot_root.mkdir(mode=0o700)
|
||||||
|
for row in rows:
|
||||||
|
source = _confined_regular_file(
|
||||||
|
eomt_root / row.path,
|
||||||
|
eomt_root,
|
||||||
|
"EoMT source frame",
|
||||||
|
)
|
||||||
|
destination = snapshot_root / f"frame-{row.sequence:06d}.png"
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
byte_length = 0
|
||||||
|
signature = b""
|
||||||
|
descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW)
|
||||||
|
try:
|
||||||
|
opened = os.fstat(descriptor)
|
||||||
|
admitted = source.stat()
|
||||||
|
if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != (
|
||||||
|
admitted.st_dev,
|
||||||
|
admitted.st_ino,
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source frame changed during admission")
|
||||||
|
with (
|
||||||
|
os.fdopen(descriptor, "rb", closefd=False) as source_stream,
|
||||||
|
destination.open("xb") as destination_stream,
|
||||||
|
):
|
||||||
|
while block := source_stream.read(1024 * 1024):
|
||||||
|
if not signature:
|
||||||
|
signature = block[:8]
|
||||||
|
byte_length += len(block)
|
||||||
|
if byte_length > row.byte_length:
|
||||||
|
raise ComponentAdapterError("EoMT source frame grew during admission")
|
||||||
|
digest.update(block)
|
||||||
|
destination_stream.write(block)
|
||||||
|
destination_stream.flush()
|
||||||
|
os.fsync(destination_stream.fileno())
|
||||||
|
finally:
|
||||||
|
os.close(descriptor)
|
||||||
|
if (
|
||||||
|
signature != b"\x89PNG\r\n\x1a\n"
|
||||||
|
or byte_length != row.byte_length
|
||||||
|
or digest.hexdigest() != row.sha256
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT source frame identity changed")
|
||||||
|
destination.chmod(0o400)
|
||||||
|
return snapshot_root.resolve(strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_decode_repair(document: Mapping[str, object], frame_count: int) -> None:
|
||||||
|
if set(document) != {
|
||||||
|
"schema_version",
|
||||||
|
"decoder",
|
||||||
|
"packets_requested",
|
||||||
|
"frames_decoded",
|
||||||
|
"repaired_frame_count",
|
||||||
|
"repairs",
|
||||||
|
}:
|
||||||
|
raise ComponentAdapterError("decode repair fields changed")
|
||||||
|
decoded = document.get("frames_decoded")
|
||||||
|
repaired = document.get("repaired_frame_count")
|
||||||
|
repairs = document.get("repairs")
|
||||||
|
if (
|
||||||
|
document.get("schema_version") != DECODE_REPAIR_SCHEMA
|
||||||
|
or document.get("decoder") != "ffmpeg-h264_cuvid-output-corrupt"
|
||||||
|
or document.get("packets_requested") != frame_count
|
||||||
|
or isinstance(decoded, bool)
|
||||||
|
or not isinstance(decoded, int)
|
||||||
|
or isinstance(repaired, bool)
|
||||||
|
or not isinstance(repaired, int)
|
||||||
|
or repaired not in (0, 1)
|
||||||
|
or decoded + repaired != frame_count
|
||||||
|
or not isinstance(repairs, list)
|
||||||
|
or len(repairs) != repaired
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("decode repair contract changed")
|
||||||
|
for value in repairs:
|
||||||
|
if (
|
||||||
|
not isinstance(value, dict)
|
||||||
|
or set(value) != {"sequence", "packet_pts", "method"}
|
||||||
|
or isinstance(value.get("sequence"), bool)
|
||||||
|
or not isinstance(value.get("sequence"), int)
|
||||||
|
or not 2 <= cast(int, value["sequence"]) <= frame_count
|
||||||
|
or isinstance(value.get("packet_pts"), bool)
|
||||||
|
or not isinstance(value.get("packet_pts"), int)
|
||||||
|
or value.get("method") != "duplicate-previous-decoded-frame"
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("decode repair entry changed")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_ddrnet_result(
|
||||||
|
root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
effective: Mapping[str, object],
|
||||||
|
*,
|
||||||
|
mapping_path: Path,
|
||||||
|
) -> None:
|
||||||
|
result = validate_fixed_result(
|
||||||
|
root,
|
||||||
|
schema_version=DDRNET_RESULT_SCHEMA,
|
||||||
|
label="DDRNet",
|
||||||
|
)
|
||||||
|
candidate = result.get("candidate")
|
||||||
|
source = result.get("source")
|
||||||
|
semantics = result.get("video_semantics")
|
||||||
|
provenance = result.get("provenance")
|
||||||
|
authority = result.get("authority")
|
||||||
|
ravnoves = effective.get("ravnoves")
|
||||||
|
if not all(
|
||||||
|
isinstance(value, dict) for value in (candidate, source, semantics, provenance, ravnoves)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet result contract is incomplete")
|
||||||
|
candidate = cast(dict[str, object], candidate)
|
||||||
|
source = cast(dict[str, object], source)
|
||||||
|
semantics = cast(dict[str, object], semantics)
|
||||||
|
provenance = cast(dict[str, object], provenance)
|
||||||
|
ravnoves = cast(dict[str, object], ravnoves)
|
||||||
|
archive = semantics.get("mask_archive")
|
||||||
|
if not isinstance(archive, dict):
|
||||||
|
raise ComponentAdapterError("DDRNet mask archive contract is unavailable")
|
||||||
|
if (
|
||||||
|
result.get("lab_id") != "LAB-V1"
|
||||||
|
or result.get("worker_id") != "worker-006"
|
||||||
|
or result.get("mode") != "ravnoves-video"
|
||||||
|
or candidate.get("candidate_id") != DDRNET_CANDIDATE_ID
|
||||||
|
or candidate.get("candidate_key") != "ddrnet"
|
||||||
|
or candidate.get("loaded_model_name") != DDRNET_LOADED_MODEL
|
||||||
|
or candidate.get("checkpoint_size_bytes") != DDRNET_CHECKPOINT_BYTES
|
||||||
|
or candidate.get("checkpoint_sha256") != DDRNET_CHECKPOINT_SHA256
|
||||||
|
or source.get("source_id") != ravnoves.get("source_id")
|
||||||
|
or source.get("input_count") != request.source.frame_count
|
||||||
|
or source.get("ground_truth_available") is not False
|
||||||
|
or source.get("mapping_sha256") != GOOSE_MAPPING_SHA256
|
||||||
|
or archive.get("path") != "semantic-masks.zip"
|
||||||
|
or archive.get("media_type") != "application/zip"
|
||||||
|
or archive.get("frame_count") != request.source.frame_count
|
||||||
|
or archive.get("width") != 800
|
||||||
|
or archive.get("height") != 600
|
||||||
|
or archive.get("encoding") != "uint8-class-id-png"
|
||||||
|
or archive.get("sequence_binding") != "sequence-0-to-masks/frame-000001.png"
|
||||||
|
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
|
||||||
|
or semantics.get("outside_crop_state") != "undefined"
|
||||||
|
or semantics.get("base_m4_result_id") is not None
|
||||||
|
or provenance.get("config_sha256") != request.effective_ddrnet_config_sha256
|
||||||
|
or provenance.get("policy_sha256") != POLICY_SHA256
|
||||||
|
or provenance.get("provider_map_sha256") != PROVIDER_MAP_SHA256
|
||||||
|
or authority != DDRNET_AUTHORITY
|
||||||
|
or result.get("visual_cases") != []
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet result identity changed")
|
||||||
|
archive_path = _confined_regular_file(
|
||||||
|
root / "semantic-masks.zip",
|
||||||
|
root,
|
||||||
|
"DDRNet semantic mask archive",
|
||||||
|
)
|
||||||
|
if archive.get("byte_length") != archive_path.stat().st_size or archive.get(
|
||||||
|
"sha256"
|
||||||
|
) != sha256_file(archive_path):
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask archive changed")
|
||||||
|
observed_histogram = _validate_mask_archive_inventory(
|
||||||
|
archive_path,
|
||||||
|
request.source.frame_count,
|
||||||
|
)
|
||||||
|
aggregate = semantics.get("aggregate_prediction_pixels")
|
||||||
|
if (
|
||||||
|
not isinstance(aggregate, list)
|
||||||
|
or len(aggregate) != 64
|
||||||
|
or any(isinstance(value, bool) or not isinstance(value, int) for value in aggregate)
|
||||||
|
or tuple(aggregate) != observed_histogram
|
||||||
|
or semantics.get("taxonomy") != _sealed_taxonomy(mapping_path)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet semantic statistics changed")
|
||||||
|
identity_value = {
|
||||||
|
"schema_version": result["schema_version"],
|
||||||
|
"candidate": result["candidate"],
|
||||||
|
"source": result["source"],
|
||||||
|
"video_semantics": result["video_semantics"],
|
||||||
|
"preprocessing": result["preprocessing"],
|
||||||
|
"metrics": result["metrics"],
|
||||||
|
"timing": result["timing"],
|
||||||
|
"resource": result["resource"],
|
||||||
|
"visual_cases": result["visual_cases"],
|
||||||
|
"authority": result["authority"],
|
||||||
|
"config_sha256": provenance["config_sha256"],
|
||||||
|
"policy_sha256": provenance["policy_sha256"],
|
||||||
|
"provider_map_sha256": provenance["provider_map_sha256"],
|
||||||
|
}
|
||||||
|
identity_sha256 = hashlib.sha256(
|
||||||
|
json.dumps(identity_value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
if result.get("result_id") != f"lab-v1-ravnoves-video-ddrnet-{identity_sha256}":
|
||||||
|
raise ComponentAdapterError("DDRNet result id changed")
|
||||||
|
repair = load_json(
|
||||||
|
root / "decode-repair.json",
|
||||||
|
label="DDRNet decode repair",
|
||||||
|
maximum=1024 * 1024,
|
||||||
|
confinement_root=root,
|
||||||
|
)
|
||||||
|
_validate_decode_repair(repair, request.source.frame_count)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_mask_archive_inventory(path: Path, frame_count: int) -> tuple[int, ...]:
|
||||||
|
expected = tuple(f"masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1))
|
||||||
|
histogram = [0] * 64
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(path) as archive:
|
||||||
|
infos = archive.infolist()
|
||||||
|
names = tuple(info.filename for info in infos)
|
||||||
|
if (
|
||||||
|
names != expected
|
||||||
|
or len(names) != len(set(names))
|
||||||
|
or any(
|
||||||
|
info.is_dir()
|
||||||
|
or info.compress_type != zipfile.ZIP_STORED
|
||||||
|
or info.flag_bits & 0x1
|
||||||
|
or stat.S_ISLNK(info.external_attr >> 16)
|
||||||
|
or not 0 < info.file_size <= 2 * 1024 * 1024
|
||||||
|
or info.compress_size != info.file_size
|
||||||
|
for info in infos
|
||||||
|
)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask archive inventory changed")
|
||||||
|
for info in infos:
|
||||||
|
with archive.open(info, mode="r") as stream:
|
||||||
|
payload = stream.read(info.file_size + 1)
|
||||||
|
if len(payload) != info.file_size:
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask archive member changed")
|
||||||
|
validate_grayscale_png_payload(
|
||||||
|
payload,
|
||||||
|
width=800,
|
||||||
|
height=600,
|
||||||
|
label=f"DDRNet mask {info.filename}",
|
||||||
|
)
|
||||||
|
with Image.open(io.BytesIO(payload)) as image:
|
||||||
|
if image.format != "PNG" or image.mode != "L" or image.size != (800, 600):
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask raster changed")
|
||||||
|
image.load()
|
||||||
|
counts = image.histogram()
|
||||||
|
if len(counts) != 256 or any(counts[64:]):
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask class id escaped 0..63")
|
||||||
|
for class_id, count in enumerate(counts[:64]):
|
||||||
|
histogram[class_id] += count
|
||||||
|
except (OSError, zipfile.BadZipFile) as exc:
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask archive is invalid") from exc
|
||||||
|
if sum(histogram) != frame_count * 800 * 600:
|
||||||
|
raise ComponentAdapterError("DDRNet semantic mask pixel count changed")
|
||||||
|
return tuple(histogram)
|
||||||
|
|
||||||
|
|
||||||
|
def _sealed_taxonomy(mapping_path: Path) -> dict[str, object]:
|
||||||
|
classes: dict[int, dict[str, object]] = {}
|
||||||
|
try:
|
||||||
|
with mapping_path.open(newline="", encoding="utf-8-sig") as stream:
|
||||||
|
reader = csv.DictReader(stream)
|
||||||
|
if reader.fieldnames is None or not {"label_key", "class_name", "hex"}.issubset(
|
||||||
|
reader.fieldnames
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("sealed GOOSE mapping columns changed")
|
||||||
|
for row in reader:
|
||||||
|
try:
|
||||||
|
class_id = int(row["label_key"])
|
||||||
|
color = bytes.fromhex(row["hex"].removeprefix("#"))
|
||||||
|
except (KeyError, TypeError, ValueError) as exc:
|
||||||
|
raise ComponentAdapterError("sealed GOOSE mapping row changed") from exc
|
||||||
|
label = row.get("class_name")
|
||||||
|
if (
|
||||||
|
not 0 <= class_id < 64
|
||||||
|
or class_id in classes
|
||||||
|
or not isinstance(label, str)
|
||||||
|
or not label
|
||||||
|
or len(color) != 3
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("sealed GOOSE taxonomy changed")
|
||||||
|
classes[class_id] = {
|
||||||
|
"class_id": class_id,
|
||||||
|
"label": label,
|
||||||
|
"color_rgb": list(color),
|
||||||
|
"disposition": "undefined" if class_id == 0 else "prediction",
|
||||||
|
}
|
||||||
|
except OSError as exc:
|
||||||
|
raise ComponentAdapterError("sealed GOOSE mapping is unavailable") from exc
|
||||||
|
if set(classes) != set(range(64)):
|
||||||
|
raise ComponentAdapterError("sealed GOOSE mapping does not cover 64 classes")
|
||||||
|
return {
|
||||||
|
"schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1",
|
||||||
|
"classes": [classes[class_id] for class_id in range(64)],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ddrnet_environment() -> dict[str, str]:
|
||||||
|
retained = {
|
||||||
|
key: value
|
||||||
|
for key, value in os.environ.items()
|
||||||
|
if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
**retained,
|
||||||
|
"HOME": "/tmp",
|
||||||
|
"PYTHONDONTWRITEBYTECODE": "1",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_output_root(path: Path) -> Path:
|
||||||
|
if path.is_symlink():
|
||||||
|
raise ComponentAdapterError("DDRNet output root is a symbolic link")
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
if not resolved.is_dir() or any(resolved.iterdir()):
|
||||||
|
raise ComponentAdapterError("DDRNet output root is not an empty real directory")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_workspace(path: Path) -> Path:
|
||||||
|
if path.exists() or path.is_symlink():
|
||||||
|
raise ComponentAdapterError("DDRNet workspace already exists")
|
||||||
|
path.mkdir(mode=0o700, parents=False)
|
||||||
|
return path.resolve(strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _confined_regular_file(path: Path, root: Path, label: str) -> Path:
|
||||||
|
try:
|
||||||
|
metadata = path.lstat()
|
||||||
|
resolved_root = root.resolve(strict=True)
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ComponentAdapterError(f"{label} is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not resolved.is_relative_to(resolved_root)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError(f"{label} is not a confined regular file")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _publish_component_children(
|
||||||
|
staging: Path,
|
||||||
|
output: Path,
|
||||||
|
*,
|
||||||
|
expected_names: set[str],
|
||||||
|
) -> None:
|
||||||
|
children = tuple(staging.iterdir())
|
||||||
|
if {child.name for child in children} != expected_names or any(
|
||||||
|
child.is_dir() or child.is_symlink() for child in children
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("DDRNet staging contains unexpected directories")
|
||||||
|
for child in children:
|
||||||
|
destination = output / child.name
|
||||||
|
if destination.exists() or destination.is_symlink():
|
||||||
|
raise ComponentAdapterError("DDRNet publication target already exists")
|
||||||
|
os.replace(child, destination)
|
||||||
|
staging.rmdir()
|
||||||
|
|
||||||
|
|
||||||
|
def _representative_frame_indices(frame_count: int) -> list[int]:
|
||||||
|
if frame_count <= 12:
|
||||||
|
return list(range(frame_count))
|
||||||
|
return sorted({round(index * (frame_count - 1) / 11) for index in range(12)})
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv)
|
||||||
|
execute_ddrnet_component(
|
||||||
|
request_path=request_path,
|
||||||
|
layout=RuntimeLayout.fixed(DDRNET_ASSETS),
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
raise SystemExit(main())
|
||||||
|
except ComponentAdapterError as exc:
|
||||||
|
print(f"portable LAB V1 DDRNet rejected: {exc}", file=sys.stderr)
|
||||||
|
raise SystemExit(2) from exc
|
||||||
+987
@@ -0,0 +1,987 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Run the sealed LAB V1 EoMT component over one CameraComputeJob."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import time
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Final, cast
|
||||||
|
|
||||||
|
from portable_lab_v1_component_adapter import (
|
||||||
|
EOMT_DEPENDENCY_IDENTITY_SHA256,
|
||||||
|
EOMT_PIPELINE,
|
||||||
|
VALID_FOV_IDENTITY_SHA256,
|
||||||
|
VALID_FOV_MASK_SHA256,
|
||||||
|
AssetExpectation,
|
||||||
|
CommandRunner,
|
||||||
|
ComponentAdapterError,
|
||||||
|
ComponentRequest,
|
||||||
|
RuntimeLayout,
|
||||||
|
available_bytes,
|
||||||
|
canonical_json,
|
||||||
|
load_component_request,
|
||||||
|
load_json,
|
||||||
|
parse_fixed_request_argument,
|
||||||
|
run_command,
|
||||||
|
sha256_file,
|
||||||
|
validate_camera_compute_job,
|
||||||
|
validate_critical_tree_file,
|
||||||
|
validate_fixed_result,
|
||||||
|
validate_grayscale_png_payload,
|
||||||
|
validate_identity_manifest,
|
||||||
|
validate_tree_asset,
|
||||||
|
validate_tree_receipt,
|
||||||
|
)
|
||||||
|
|
||||||
|
EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2"
|
||||||
|
DEPENDENCY_SCHEMA: Final = "missioncore.k1-e3-dependency-manifest/v1"
|
||||||
|
VALID_FOV_SCHEMA: Final = "missioncore.k1-valid-fov-mask/v1"
|
||||||
|
DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1"
|
||||||
|
SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1"
|
||||||
|
MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||||
|
MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||||
|
MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation"
|
||||||
|
DISK_FLOOR_BYTES: Final = 360 * 1024**3
|
||||||
|
FFMPEG_TREE_SOURCE_IMAGE_SHA256: Final = (
|
||||||
|
"8a364092b03561b9c08ac00730206e363a53d07ea0304f7d543b403b65432b5e"
|
||||||
|
)
|
||||||
|
FFMPEG_TREE_IDENTITY_SHA256: Final = (
|
||||||
|
"03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69"
|
||||||
|
)
|
||||||
|
FFMPEG_TREE_BYTE_LENGTH: Final = 256_208_352
|
||||||
|
FFMPEG_SHA256: Final = "7c6e870c57d16a755d7255cb93b65379218f47d3208a3f9b107f3a80646af6af"
|
||||||
|
FFPROBE_SHA256: Final = "39e7b15b94ef2d37e9fcf052ad1cb1e6550fdb6c6ac6765c5f1aba53cc367929"
|
||||||
|
|
||||||
|
EOMT_ASSETS: Final = (
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-environment",
|
||||||
|
"/environment",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-ffmpeg-runtime",
|
||||||
|
"/opt/ffmpeg",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
FFMPEG_TREE_IDENTITY_SHA256,
|
||||||
|
FFMPEG_TREE_BYTE_LENGTH,
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-model-cache",
|
||||||
|
"/cache",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-python-environment",
|
||||||
|
"/opt/env",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-runner-bundle",
|
||||||
|
"/runner",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"eomt-transformers-environment",
|
||||||
|
"/opt/transformers",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
AssetExpectation(
|
||||||
|
"k1-valid-fov-root",
|
||||||
|
"/valid-fov",
|
||||||
|
"tree",
|
||||||
|
"identity-sha256",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
_RUNNER_FILES: Final = {
|
||||||
|
"run_e4_full_session_segmentation.py": (
|
||||||
|
30_720,
|
||||||
|
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4",
|
||||||
|
),
|
||||||
|
"run_e3_rectified_segmentation.py": (
|
||||||
|
45_789,
|
||||||
|
"01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6",
|
||||||
|
),
|
||||||
|
"run_recorded_perception_epoch.py": (
|
||||||
|
34_899,
|
||||||
|
"4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675",
|
||||||
|
),
|
||||||
|
"run_evaluation_prelabels.py": (
|
||||||
|
28_915,
|
||||||
|
"25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4",
|
||||||
|
),
|
||||||
|
"e3_k1_camera1_profile.json": (
|
||||||
|
3_805,
|
||||||
|
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_MODEL_FILES: Final = {
|
||||||
|
"config.json": (
|
||||||
|
1_575,
|
||||||
|
"7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650",
|
||||||
|
),
|
||||||
|
"preprocessor_config.json": (
|
||||||
|
666,
|
||||||
|
"97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7",
|
||||||
|
),
|
||||||
|
"model.safetensors": (
|
||||||
|
1_276_175_488,
|
||||||
|
"c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def execute_eomt_component(
|
||||||
|
*,
|
||||||
|
request_path: Path,
|
||||||
|
layout: RuntimeLayout,
|
||||||
|
command_runner: CommandRunner = run_command,
|
||||||
|
work_root: Path | None = None,
|
||||||
|
disk_floor_bytes: int = DISK_FLOOR_BYTES,
|
||||||
|
) -> None:
|
||||||
|
request = load_component_request(
|
||||||
|
request_path,
|
||||||
|
component="eomt",
|
||||||
|
expectations=EOMT_ASSETS,
|
||||||
|
)
|
||||||
|
input_document = validate_camera_compute_job(layout.camera_job_root, request.source)
|
||||||
|
roots = _validate_release_assets(request, layout)
|
||||||
|
output = _empty_output_root(layout.output_root)
|
||||||
|
reserve = request.source.frame_count * 800 * 600 * 7 + request.source.input_byte_length
|
||||||
|
free_before = available_bytes(output)
|
||||||
|
if disk_floor_bytes < 0 or free_before < disk_floor_bytes + reserve:
|
||||||
|
raise ComponentAdapterError("EoMT output does not satisfy its disk reserve")
|
||||||
|
workspace = _prepare_workspace(output / ".eomt-work" if work_root is None else work_root)
|
||||||
|
try:
|
||||||
|
total_started = time.perf_counter()
|
||||||
|
extract_started = time.perf_counter()
|
||||||
|
frames_root, timeline_path, decode_repair = _decode_camera_epoch(
|
||||||
|
request=request,
|
||||||
|
input_document=input_document,
|
||||||
|
camera_job_root=layout.camera_job_root,
|
||||||
|
output_root=output,
|
||||||
|
work_root=workspace,
|
||||||
|
ffmpeg_root=roots["eomt-ffmpeg-runtime"],
|
||||||
|
command_runner=command_runner,
|
||||||
|
)
|
||||||
|
extract_seconds = _elapsed(extract_started)
|
||||||
|
free_post_extract = available_bytes(output)
|
||||||
|
if free_post_extract < disk_floor_bytes:
|
||||||
|
raise ComponentAdapterError("EoMT frame extraction crossed its disk floor")
|
||||||
|
|
||||||
|
inference_root = output / ".eomt-component"
|
||||||
|
runner = roots["eomt-runner-bundle"] / "run_e4_full_session_segmentation.py"
|
||||||
|
profile = roots["eomt-runner-bundle"] / "e3_k1_camera1_profile.json"
|
||||||
|
environment = _eomt_environment(roots)
|
||||||
|
run_argv = (
|
||||||
|
sys.executable,
|
||||||
|
str(runner),
|
||||||
|
"run",
|
||||||
|
"--job",
|
||||||
|
str(layout.camera_job_root),
|
||||||
|
"--profile",
|
||||||
|
str(profile),
|
||||||
|
"--valid-fov-root",
|
||||||
|
str(roots["k1-valid-fov-root"]),
|
||||||
|
"--frames",
|
||||||
|
str(frames_root),
|
||||||
|
"--timeline",
|
||||||
|
str(timeline_path),
|
||||||
|
"--cache",
|
||||||
|
str(roots["eomt-model-cache"]),
|
||||||
|
"--environment",
|
||||||
|
str(roots["eomt-environment"]),
|
||||||
|
"--output",
|
||||||
|
str(inference_root),
|
||||||
|
"--frame-limit",
|
||||||
|
str(request.source.frame_count),
|
||||||
|
"--free-bytes-floor",
|
||||||
|
str(disk_floor_bytes),
|
||||||
|
"--orchestrator-sha256",
|
||||||
|
sha256_file(Path(__file__).resolve(strict=True)),
|
||||||
|
"--container-image",
|
||||||
|
f"sha256:{request.component_image_sha256}",
|
||||||
|
"--telemetry-interval-seconds",
|
||||||
|
"1",
|
||||||
|
)
|
||||||
|
command_runner(run_argv, environment)
|
||||||
|
_write_json(inference_root / "decode-repair.json", decode_repair)
|
||||||
|
free_post_inference = available_bytes(output)
|
||||||
|
if free_post_inference < disk_floor_bytes:
|
||||||
|
raise ComponentAdapterError("EoMT inference crossed its disk floor")
|
||||||
|
|
||||||
|
video_path = inference_root / "perception.mp4"
|
||||||
|
encode_started = time.perf_counter()
|
||||||
|
duration = request.source.timeline_end_seconds - request.source.timeline_start_seconds
|
||||||
|
fps = request.source.frame_count / duration
|
||||||
|
encode_argv = (
|
||||||
|
str(roots["eomt-ffmpeg-runtime"] / "bin" / "ffmpeg"),
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-framerate",
|
||||||
|
_number(fps),
|
||||||
|
"-i",
|
||||||
|
str(inference_root / "overlay-frames" / "frame-%06d.png"),
|
||||||
|
"-t",
|
||||||
|
_number(duration),
|
||||||
|
"-c:v",
|
||||||
|
"h264_nvenc",
|
||||||
|
"-preset",
|
||||||
|
"p4",
|
||||||
|
"-tune",
|
||||||
|
"hq",
|
||||||
|
"-rc",
|
||||||
|
"vbr",
|
||||||
|
"-cq",
|
||||||
|
"21",
|
||||||
|
"-b:v",
|
||||||
|
"0",
|
||||||
|
"-pix_fmt",
|
||||||
|
"yuv420p",
|
||||||
|
"-movflags",
|
||||||
|
"+faststart",
|
||||||
|
str(video_path),
|
||||||
|
)
|
||||||
|
command_runner(encode_argv, None)
|
||||||
|
encode_seconds = _elapsed(encode_started)
|
||||||
|
_required_nonempty_file(video_path, inference_root, "EoMT overlay video")
|
||||||
|
|
||||||
|
masks_path = inference_root / "masks.tar.gz"
|
||||||
|
archive_started = time.perf_counter()
|
||||||
|
archive_argv = (
|
||||||
|
"/usr/bin/tar",
|
||||||
|
"-czf",
|
||||||
|
str(masks_path),
|
||||||
|
"-C",
|
||||||
|
str(inference_root),
|
||||||
|
"semantic-masks",
|
||||||
|
)
|
||||||
|
command_runner(archive_argv, None)
|
||||||
|
archive_seconds = _elapsed(archive_started)
|
||||||
|
_required_nonempty_file(masks_path, inference_root, "EoMT mask archive")
|
||||||
|
free_post_artifacts = available_bytes(output)
|
||||||
|
if free_post_artifacts < disk_floor_bytes:
|
||||||
|
raise ComponentAdapterError("EoMT publication crossed its disk floor")
|
||||||
|
|
||||||
|
wall_seconds = _elapsed(total_started)
|
||||||
|
finalize_argv = (
|
||||||
|
sys.executable,
|
||||||
|
str(runner),
|
||||||
|
"finalize",
|
||||||
|
"--output",
|
||||||
|
str(inference_root),
|
||||||
|
"--video",
|
||||||
|
str(video_path),
|
||||||
|
"--masks",
|
||||||
|
str(masks_path),
|
||||||
|
"--extract-seconds",
|
||||||
|
_number(extract_seconds),
|
||||||
|
"--encode-seconds",
|
||||||
|
_number(encode_seconds),
|
||||||
|
"--archive-seconds",
|
||||||
|
_number(archive_seconds),
|
||||||
|
"--wall-seconds",
|
||||||
|
_number(wall_seconds),
|
||||||
|
"--encoder",
|
||||||
|
"ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
|
||||||
|
"--disk-free-before-bytes",
|
||||||
|
str(free_before),
|
||||||
|
"--disk-free-post-extract-bytes",
|
||||||
|
str(free_post_extract),
|
||||||
|
"--disk-free-post-inference-bytes",
|
||||||
|
str(free_post_inference),
|
||||||
|
"--disk-free-post-artifacts-bytes",
|
||||||
|
str(free_post_artifacts),
|
||||||
|
"--disk-floor-bytes",
|
||||||
|
str(disk_floor_bytes),
|
||||||
|
"--working-set-reserve-bytes",
|
||||||
|
str(reserve),
|
||||||
|
)
|
||||||
|
command_runner(finalize_argv, environment)
|
||||||
|
_bind_source_frame_manifest(inference_root, frames_root, request)
|
||||||
|
_validate_eomt_result(inference_root, frames_root, request)
|
||||||
|
for temporary in ("overlay-frames", "semantic-masks"):
|
||||||
|
path = inference_root / temporary
|
||||||
|
if path.exists():
|
||||||
|
shutil.rmtree(path)
|
||||||
|
_publish_component_children(
|
||||||
|
inference_root,
|
||||||
|
output,
|
||||||
|
expected_names={
|
||||||
|
"decode-repair.json",
|
||||||
|
"frames.jsonl",
|
||||||
|
"gpu-telemetry.jsonl",
|
||||||
|
"masks.tar.gz",
|
||||||
|
"perception.mp4",
|
||||||
|
"result.json",
|
||||||
|
"run-report.json",
|
||||||
|
"source-frames.json",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(workspace, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_release_assets(
|
||||||
|
request: ComponentRequest,
|
||||||
|
layout: RuntimeLayout,
|
||||||
|
) -> dict[str, Path]:
|
||||||
|
roots = {
|
||||||
|
item.asset_id: validate_tree_asset(request, layout, item.asset_id) for item in EOMT_ASSETS
|
||||||
|
}
|
||||||
|
for asset_id in (
|
||||||
|
"eomt-environment",
|
||||||
|
"eomt-model-cache",
|
||||||
|
"eomt-python-environment",
|
||||||
|
"eomt-transformers-environment",
|
||||||
|
):
|
||||||
|
validate_tree_receipt(
|
||||||
|
request,
|
||||||
|
roots[asset_id],
|
||||||
|
asset_id,
|
||||||
|
verify_payload=True,
|
||||||
|
)
|
||||||
|
for asset_id in ("eomt-runner-bundle", "k1-valid-fov-root"):
|
||||||
|
validate_tree_receipt(
|
||||||
|
request,
|
||||||
|
roots[asset_id],
|
||||||
|
asset_id,
|
||||||
|
verify_payload=True,
|
||||||
|
)
|
||||||
|
ffmpeg_receipt = validate_tree_receipt(
|
||||||
|
request,
|
||||||
|
roots["eomt-ffmpeg-runtime"],
|
||||||
|
"eomt-ffmpeg-runtime",
|
||||||
|
expected_metadata={
|
||||||
|
"source_image_sha256": FFMPEG_TREE_SOURCE_IMAGE_SHA256,
|
||||||
|
"source_path": "/usr/lib/ffmpeg/7.0",
|
||||||
|
},
|
||||||
|
additional_metadata_keys=frozenset({"binaries"}),
|
||||||
|
verify_payload=True,
|
||||||
|
)
|
||||||
|
environment_manifest = validate_identity_manifest(
|
||||||
|
roots["eomt-environment"] / "manifest.json",
|
||||||
|
expected_identity_sha256=EOMT_DEPENDENCY_IDENTITY_SHA256,
|
||||||
|
expected_schema=DEPENDENCY_SCHEMA,
|
||||||
|
label="EoMT dependency manifest",
|
||||||
|
)
|
||||||
|
identity = cast(dict[str, object], environment_manifest["identity"])
|
||||||
|
if identity.get("profile_sha256") != _RUNNER_FILES["e3_k1_camera1_profile.json"][1]:
|
||||||
|
raise ComponentAdapterError("EoMT dependency profile changed")
|
||||||
|
valid_manifest = validate_identity_manifest(
|
||||||
|
roots["k1-valid-fov-root"] / "manifest.json",
|
||||||
|
expected_identity_sha256=VALID_FOV_IDENTITY_SHA256,
|
||||||
|
expected_schema=VALID_FOV_SCHEMA,
|
||||||
|
label="valid-FOV manifest",
|
||||||
|
)
|
||||||
|
artifact = valid_manifest.get("artifact")
|
||||||
|
if not isinstance(artifact, dict) or artifact.get("sha256") != VALID_FOV_MASK_SHA256:
|
||||||
|
raise ComponentAdapterError("valid-FOV mask binding changed")
|
||||||
|
mask_byte_length = artifact.get("byte_length")
|
||||||
|
if (
|
||||||
|
isinstance(mask_byte_length, bool)
|
||||||
|
or not isinstance(mask_byte_length, int)
|
||||||
|
or mask_byte_length < 1
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("valid-FOV mask byte length changed")
|
||||||
|
validate_critical_tree_file(
|
||||||
|
roots["k1-valid-fov-root"],
|
||||||
|
"mask.png",
|
||||||
|
byte_length=mask_byte_length,
|
||||||
|
identity_sha256=VALID_FOV_MASK_SHA256,
|
||||||
|
label="valid-FOV mask",
|
||||||
|
)
|
||||||
|
for name, (byte_length, digest) in _RUNNER_FILES.items():
|
||||||
|
validate_critical_tree_file(
|
||||||
|
roots["eomt-runner-bundle"],
|
||||||
|
name,
|
||||||
|
byte_length=byte_length,
|
||||||
|
identity_sha256=digest,
|
||||||
|
label=f"EoMT runner file {name}",
|
||||||
|
)
|
||||||
|
snapshot = (
|
||||||
|
roots["eomt-model-cache"]
|
||||||
|
/ "huggingface"
|
||||||
|
/ "models--tue-mps--cityscapes_semantic_eomt_large_1024"
|
||||||
|
/ "snapshots"
|
||||||
|
/ MODEL_REVISION
|
||||||
|
)
|
||||||
|
for name, (byte_length, digest) in _MODEL_FILES.items():
|
||||||
|
validate_critical_tree_file(
|
||||||
|
roots["eomt-model-cache"],
|
||||||
|
(snapshot / name).relative_to(roots["eomt-model-cache"]).as_posix(),
|
||||||
|
byte_length=byte_length,
|
||||||
|
identity_sha256=digest,
|
||||||
|
label=f"EoMT model file {name}",
|
||||||
|
)
|
||||||
|
binaries = ffmpeg_receipt.get("binaries")
|
||||||
|
if not isinstance(binaries, dict) or set(binaries) != {"ffmpeg", "ffprobe"}:
|
||||||
|
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||||
|
for name, digest in (("ffmpeg", FFMPEG_SHA256), ("ffprobe", FFPROBE_SHA256)):
|
||||||
|
row = binaries.get(name)
|
||||||
|
if not isinstance(row, dict) or set(row) != {
|
||||||
|
"relative_path",
|
||||||
|
"byte_length",
|
||||||
|
"sha256",
|
||||||
|
}:
|
||||||
|
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||||
|
path = validate_critical_tree_file(
|
||||||
|
roots["eomt-ffmpeg-runtime"],
|
||||||
|
f"bin/{name}",
|
||||||
|
byte_length=cast(int, row["byte_length"]),
|
||||||
|
identity_sha256=digest,
|
||||||
|
label=f"sealed {name}",
|
||||||
|
)
|
||||||
|
if row != {
|
||||||
|
"relative_path": f"bin/{name}",
|
||||||
|
"byte_length": path.stat().st_size,
|
||||||
|
"sha256": digest,
|
||||||
|
} or not os.access(path, os.X_OK):
|
||||||
|
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||||
|
return roots
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_camera_epoch(
|
||||||
|
*,
|
||||||
|
request: ComponentRequest,
|
||||||
|
input_document: Mapping[str, object],
|
||||||
|
camera_job_root: Path,
|
||||||
|
output_root: Path,
|
||||||
|
work_root: Path,
|
||||||
|
ffmpeg_root: Path,
|
||||||
|
command_runner: CommandRunner,
|
||||||
|
) -> tuple[Path, Path, dict[str, object]]:
|
||||||
|
source = request.source
|
||||||
|
epoch = (
|
||||||
|
camera_job_root
|
||||||
|
/ "input"
|
||||||
|
/ "camera"
|
||||||
|
/ source.camera_source_id
|
||||||
|
/ f"epoch-{source.codec_epoch}"
|
||||||
|
)
|
||||||
|
stream_path = work_root / "camera.mp4"
|
||||||
|
with stream_path.open("xb") as stream:
|
||||||
|
for path in (
|
||||||
|
epoch / "init.mp4",
|
||||||
|
*(epoch / "segments" / f"{index}.m4s" for index in range(1, source.frame_count + 1)),
|
||||||
|
):
|
||||||
|
with path.open("rb") as segment:
|
||||||
|
shutil.copyfileobj(segment, stream, length=1024 * 1024)
|
||||||
|
stream.flush()
|
||||||
|
os.fsync(stream.fileno())
|
||||||
|
packets_path = work_root / "packets.csv"
|
||||||
|
ffprobe_argv = (
|
||||||
|
str(ffmpeg_root / "bin" / "ffprobe"),
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-select_streams",
|
||||||
|
"v:0",
|
||||||
|
"-show_packets",
|
||||||
|
"-show_entries",
|
||||||
|
"packet=pts,flags",
|
||||||
|
"-of",
|
||||||
|
"csv=p=0",
|
||||||
|
"-o",
|
||||||
|
str(packets_path),
|
||||||
|
str(stream_path),
|
||||||
|
)
|
||||||
|
command_runner(ffprobe_argv, None)
|
||||||
|
packet_rows = packets_path.read_text(encoding="utf-8").splitlines()
|
||||||
|
if len(packet_rows) != source.frame_count:
|
||||||
|
raise ComponentAdapterError("camera packet count changed")
|
||||||
|
packet_pts: list[int] = []
|
||||||
|
for row in packet_rows:
|
||||||
|
fields = row.split(",")
|
||||||
|
try:
|
||||||
|
packet_pts.append(int(fields[0].strip()))
|
||||||
|
except (IndexError, ValueError) as exc:
|
||||||
|
raise ComponentAdapterError("camera packet timestamp is invalid") from exc
|
||||||
|
decoded_root = work_root / "decoded-by-pts"
|
||||||
|
decoded_root.mkdir(mode=0o700)
|
||||||
|
decode_argv = (
|
||||||
|
str(ffmpeg_root / "bin" / "ffmpeg"),
|
||||||
|
"-hide_banner",
|
||||||
|
"-loglevel",
|
||||||
|
"error",
|
||||||
|
"-hwaccel",
|
||||||
|
"cuda",
|
||||||
|
"-hwaccel_output_format",
|
||||||
|
"cuda",
|
||||||
|
"-c:v",
|
||||||
|
"h264_cuvid",
|
||||||
|
"-err_detect",
|
||||||
|
"ignore_err",
|
||||||
|
"-flags",
|
||||||
|
"+output_corrupt",
|
||||||
|
"-copyts",
|
||||||
|
"-i",
|
||||||
|
str(stream_path),
|
||||||
|
"-map",
|
||||||
|
"0:v:0",
|
||||||
|
"-vf",
|
||||||
|
"hwdownload,format=nv12",
|
||||||
|
"-fps_mode",
|
||||||
|
"passthrough",
|
||||||
|
"-enc_time_base",
|
||||||
|
"demux",
|
||||||
|
"-frames:v",
|
||||||
|
str(source.frame_count),
|
||||||
|
"-frame_pts",
|
||||||
|
"1",
|
||||||
|
str(decoded_root / "frame-%d.png"),
|
||||||
|
)
|
||||||
|
command_runner(decode_argv, None)
|
||||||
|
frames_root = output_root / "source-frames"
|
||||||
|
frames_root.mkdir(mode=0o700)
|
||||||
|
repairs: list[dict[str, object]] = []
|
||||||
|
decoded_count = len(tuple(decoded_root.glob("frame-*.png")))
|
||||||
|
for index, pts in enumerate(packet_pts):
|
||||||
|
decoded = decoded_root / f"frame-{pts}.png"
|
||||||
|
canonical = frames_root / f"frame-{index + 1:06d}.png"
|
||||||
|
if decoded.is_file() and not decoded.is_symlink():
|
||||||
|
os.replace(decoded, canonical)
|
||||||
|
continue
|
||||||
|
if index == 0 or repairs:
|
||||||
|
raise ComponentAdapterError("camera decode has more than one recoverable gap")
|
||||||
|
previous = frames_root / f"frame-{index:06d}.png"
|
||||||
|
shutil.copyfile(previous, canonical)
|
||||||
|
repairs.append(
|
||||||
|
{
|
||||||
|
"sequence": index + 1,
|
||||||
|
"packet_pts": pts,
|
||||||
|
"method": "duplicate-previous-decoded-frame",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if any(decoded_root.iterdir()):
|
||||||
|
raise ComponentAdapterError("camera decoder emitted unexpected frames")
|
||||||
|
timeline = cast(Mapping[str, object], input_document["timeline"])
|
||||||
|
timeline_start = timeline["start_seconds"]
|
||||||
|
if isinstance(timeline_start, bool) or not isinstance(timeline_start, (int, float)):
|
||||||
|
raise ComponentAdapterError("camera timeline start changed")
|
||||||
|
duration = source.timeline_end_seconds - source.timeline_start_seconds
|
||||||
|
first_pts = packet_pts[0]
|
||||||
|
previous_seconds = -1.0
|
||||||
|
timeline_path = work_root / "timeline.jsonl"
|
||||||
|
with timeline_path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||||
|
for index, pts in enumerate(packet_pts):
|
||||||
|
epoch_seconds = (pts - first_pts) / 90_000.0
|
||||||
|
if epoch_seconds <= previous_seconds or epoch_seconds > duration + 0.001:
|
||||||
|
raise ComponentAdapterError("camera packet timeline changed")
|
||||||
|
timeline_row = {
|
||||||
|
"frame_index": index,
|
||||||
|
"epoch_seconds": epoch_seconds,
|
||||||
|
"session_seconds": float(timeline_start) + epoch_seconds,
|
||||||
|
}
|
||||||
|
stream.write(canonical_json(timeline_row).decode("utf-8") + "\n")
|
||||||
|
previous_seconds = epoch_seconds
|
||||||
|
return (
|
||||||
|
frames_root,
|
||||||
|
timeline_path,
|
||||||
|
{
|
||||||
|
"schema_version": DECODE_REPAIR_SCHEMA,
|
||||||
|
"decoder": "ffmpeg-h264_cuvid-output-corrupt",
|
||||||
|
"packets_requested": source.frame_count,
|
||||||
|
"frames_decoded": decoded_count,
|
||||||
|
"repaired_frame_count": len(repairs),
|
||||||
|
"repairs": repairs,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_eomt_result(
|
||||||
|
root: Path,
|
||||||
|
frames_root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> None:
|
||||||
|
document = validate_fixed_result(root, schema_version=EOMT_RESULT_SCHEMA, label="EoMT")
|
||||||
|
identity = document.get("identity")
|
||||||
|
if not isinstance(identity, dict):
|
||||||
|
raise ComponentAdapterError("EoMT result identity is unavailable")
|
||||||
|
configuration = identity.get("configuration")
|
||||||
|
models = identity.get("models")
|
||||||
|
semantic = models.get("semantic") if isinstance(models, dict) else None
|
||||||
|
if not isinstance(configuration, dict):
|
||||||
|
raise ComponentAdapterError("EoMT result configuration is unavailable")
|
||||||
|
observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||||
|
source = request.source
|
||||||
|
if (
|
||||||
|
document.get("identity_sha256") != observed_identity
|
||||||
|
or document.get("result_id") != f"result-{observed_identity}"
|
||||||
|
or document.get("job_id") != source.camera_job_id
|
||||||
|
or document.get("input_sha256") != source.camera_input_sha256
|
||||||
|
or document.get("session_id") != source.source_session_id
|
||||||
|
or document.get("source_id") != source.camera_source_id
|
||||||
|
or document.get("codec_epoch") != source.codec_epoch
|
||||||
|
or document.get("timestamp_basis") != "session-time-seconds"
|
||||||
|
or document.get("timeline_start_seconds") != source.timeline_start_seconds
|
||||||
|
or document.get("timeline_end_seconds") != source.timeline_end_seconds
|
||||||
|
or document.get("frames_processed") != source.frame_count
|
||||||
|
or document.get("ground_truth") is not False
|
||||||
|
or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2"
|
||||||
|
or configuration.get("pipeline") != EOMT_PIPELINE
|
||||||
|
or configuration.get("dependency_identity_sha256") != EOMT_DEPENDENCY_IDENTITY_SHA256
|
||||||
|
or configuration.get("container_image") != f"sha256:{request.component_image_sha256}"
|
||||||
|
or not isinstance(semantic, dict)
|
||||||
|
or semantic.get("id") != MODEL_ID
|
||||||
|
or semantic.get("revision") != MODEL_REVISION
|
||||||
|
or semantic.get("architecture") != MODEL_ARCHITECTURE
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT result source identity changed")
|
||||||
|
artifacts = document.get("artifacts")
|
||||||
|
if not isinstance(artifacts, list) or len(artifacts) != 6:
|
||||||
|
raise ComponentAdapterError("EoMT result artifact set changed")
|
||||||
|
expected_artifacts = {
|
||||||
|
"panoptic-overlay-video": ("perception.mp4", "video/mp4"),
|
||||||
|
"panoptic-mask-archive": ("masks.tar.gz", "application/gzip"),
|
||||||
|
"panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"),
|
||||||
|
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"),
|
||||||
|
"perception-run-report": ("run-report.json", "application/json"),
|
||||||
|
"decoded-source-frame-manifest": ("source-frames.json", "application/json"),
|
||||||
|
}
|
||||||
|
observed_kinds: set[str] = set()
|
||||||
|
for value in artifacts:
|
||||||
|
if not isinstance(value, dict):
|
||||||
|
raise ComponentAdapterError("EoMT result artifact is invalid")
|
||||||
|
kind = value.get("kind")
|
||||||
|
encoded = value.get("path")
|
||||||
|
if (
|
||||||
|
not isinstance(kind, str)
|
||||||
|
or kind in observed_kinds
|
||||||
|
or expected_artifacts.get(kind) != (encoded, value.get("media_type"))
|
||||||
|
or not isinstance(encoded, str)
|
||||||
|
or "/" in encoded
|
||||||
|
or "\\" in encoded
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT result artifact path changed")
|
||||||
|
observed_kinds.add(kind)
|
||||||
|
path = _required_nonempty_file(root / encoded, root, "EoMT result artifact")
|
||||||
|
if value.get("byte_length") != path.stat().st_size or value.get("sha256") != sha256_file(
|
||||||
|
path
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT result artifact identity changed")
|
||||||
|
if observed_kinds != set(expected_artifacts):
|
||||||
|
raise ComponentAdapterError("EoMT result artifact roles changed")
|
||||||
|
manifest_path = _required_nonempty_file(
|
||||||
|
root / "source-frames.json",
|
||||||
|
root,
|
||||||
|
"EoMT source frame manifest",
|
||||||
|
)
|
||||||
|
expected_binding = {
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"path": "source-frames.json",
|
||||||
|
"sha256": sha256_file(manifest_path),
|
||||||
|
"byte_length": manifest_path.stat().st_size,
|
||||||
|
"frame_count": source.frame_count,
|
||||||
|
}
|
||||||
|
if identity.get("source_frames") != expected_binding:
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest binding changed")
|
||||||
|
_validate_source_frame_manifest(manifest_path, frames_root, request)
|
||||||
|
_validate_mask_archive_inventory(root / "masks.tar.gz", source.frame_count)
|
||||||
|
|
||||||
|
|
||||||
|
def _bind_source_frame_manifest(
|
||||||
|
result_root: Path,
|
||||||
|
frames_root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> None:
|
||||||
|
document = _source_frame_manifest_document(frames_root, request)
|
||||||
|
manifest_path = result_root / "source-frames.json"
|
||||||
|
_write_canonical_json(manifest_path, document)
|
||||||
|
binding = {
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"path": manifest_path.name,
|
||||||
|
"sha256": sha256_file(manifest_path),
|
||||||
|
"byte_length": manifest_path.stat().st_size,
|
||||||
|
"frame_count": request.source.frame_count,
|
||||||
|
}
|
||||||
|
result = load_json(
|
||||||
|
result_root / "result.json",
|
||||||
|
label="legacy EoMT result",
|
||||||
|
maximum=8 * 1024 * 1024,
|
||||||
|
confinement_root=result_root,
|
||||||
|
)
|
||||||
|
identity = result.get("identity")
|
||||||
|
artifacts = result.get("artifacts")
|
||||||
|
old_result_id = result.get("result_id")
|
||||||
|
if (
|
||||||
|
result.get("schema_version") != EOMT_RESULT_SCHEMA
|
||||||
|
or not isinstance(identity, dict)
|
||||||
|
or "source_frames" in identity
|
||||||
|
or not isinstance(artifacts, list)
|
||||||
|
or len(artifacts) != 5
|
||||||
|
or not isinstance(old_result_id, str)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("legacy EoMT result cannot bind source frames")
|
||||||
|
identity["source_frames"] = binding
|
||||||
|
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||||
|
result_id = f"result-{identity_sha256}"
|
||||||
|
result["identity_sha256"] = identity_sha256
|
||||||
|
result["result_id"] = result_id
|
||||||
|
|
||||||
|
report_path = _required_nonempty_file(
|
||||||
|
result_root / "run-report.json",
|
||||||
|
result_root,
|
||||||
|
"legacy EoMT run report",
|
||||||
|
)
|
||||||
|
report = load_json(
|
||||||
|
report_path,
|
||||||
|
label="legacy EoMT run report",
|
||||||
|
maximum=8 * 1024 * 1024,
|
||||||
|
confinement_root=result_root,
|
||||||
|
)
|
||||||
|
if report.get("result_id") != old_result_id or "source_frames" in report:
|
||||||
|
raise ComponentAdapterError("legacy EoMT run report cannot bind source frames")
|
||||||
|
report["result_id"] = result_id
|
||||||
|
report["source_frames"] = binding
|
||||||
|
_write_json(report_path, report)
|
||||||
|
|
||||||
|
observed_kinds: set[str] = set()
|
||||||
|
for value in artifacts:
|
||||||
|
if not isinstance(value, dict) or not isinstance(value.get("kind"), str):
|
||||||
|
raise ComponentAdapterError("legacy EoMT artifact set changed")
|
||||||
|
kind = cast(str, value["kind"])
|
||||||
|
if kind in observed_kinds:
|
||||||
|
raise ComponentAdapterError("legacy EoMT artifact set changed")
|
||||||
|
observed_kinds.add(kind)
|
||||||
|
if kind == "perception-run-report":
|
||||||
|
value["byte_length"] = report_path.stat().st_size
|
||||||
|
value["sha256"] = sha256_file(report_path)
|
||||||
|
if observed_kinds != {
|
||||||
|
"panoptic-overlay-video",
|
||||||
|
"panoptic-mask-archive",
|
||||||
|
"panoptic-frame-metadata",
|
||||||
|
"worker-gpu-telemetry",
|
||||||
|
"perception-run-report",
|
||||||
|
}:
|
||||||
|
raise ComponentAdapterError("legacy EoMT artifact roles changed")
|
||||||
|
artifacts.append(
|
||||||
|
{
|
||||||
|
"kind": "decoded-source-frame-manifest",
|
||||||
|
"path": manifest_path.name,
|
||||||
|
"media_type": "application/json",
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"byte_length": manifest_path.stat().st_size,
|
||||||
|
"sha256": sha256_file(manifest_path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_write_json(result_root / "result.json", result)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_frame_manifest_document(
|
||||||
|
frames_root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
if frames_root.is_symlink():
|
||||||
|
raise ComponentAdapterError("EoMT source frame root is a symbolic link")
|
||||||
|
resolved = frames_root.resolve(strict=True)
|
||||||
|
expected_names = tuple(
|
||||||
|
f"frame-{sequence:06d}.png" for sequence in range(1, request.source.frame_count + 1)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
observed_names = tuple(sorted(path.name for path in resolved.iterdir()))
|
||||||
|
except OSError as exc:
|
||||||
|
raise ComponentAdapterError("EoMT source frame set is unavailable") from exc
|
||||||
|
if observed_names != expected_names:
|
||||||
|
raise ComponentAdapterError("EoMT source frame set changed")
|
||||||
|
rows: list[dict[str, object]] = []
|
||||||
|
for sequence, name in enumerate(expected_names, start=1):
|
||||||
|
path = _required_nonempty_file(resolved / name, resolved, "EoMT source frame")
|
||||||
|
if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024:
|
||||||
|
raise ComponentAdapterError("EoMT source frame is unsafe")
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"sequence": sequence,
|
||||||
|
"path": f"source-frames/{name}",
|
||||||
|
"byte_length": path.stat().st_size,
|
||||||
|
"sha256": sha256_file(path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
source = request.source
|
||||||
|
return {
|
||||||
|
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||||
|
"job_id": source.camera_job_id,
|
||||||
|
"input_sha256": source.camera_input_sha256,
|
||||||
|
"source_id": source.camera_source_id,
|
||||||
|
"codec_epoch": source.codec_epoch,
|
||||||
|
"frame_count": source.frame_count,
|
||||||
|
"frames": rows,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_source_frame_manifest(
|
||||||
|
path: Path,
|
||||||
|
frames_root: Path,
|
||||||
|
request: ComponentRequest,
|
||||||
|
) -> None:
|
||||||
|
document = load_json(
|
||||||
|
path,
|
||||||
|
label="EoMT source frame manifest",
|
||||||
|
maximum=32 * 1024 * 1024,
|
||||||
|
confinement_root=path.parent,
|
||||||
|
)
|
||||||
|
if document != _source_frame_manifest_document(frames_root, request):
|
||||||
|
raise ComponentAdapterError("EoMT source frame manifest changed")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_mask_archive_inventory(path: Path, frame_count: int) -> None:
|
||||||
|
expected_files = {
|
||||||
|
f"semantic-masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1)
|
||||||
|
}
|
||||||
|
expected_names = {"semantic-masks", *expected_files}
|
||||||
|
try:
|
||||||
|
with tarfile.open(path, mode="r:gz") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
names = [member.name for member in members]
|
||||||
|
if len(names) != len(expected_names) or set(names) != expected_names:
|
||||||
|
raise ComponentAdapterError("EoMT mask archive inventory changed")
|
||||||
|
if len(names) != len(set(names)):
|
||||||
|
raise ComponentAdapterError("EoMT mask archive contains duplicate members")
|
||||||
|
for member in members:
|
||||||
|
if member.name == "semantic-masks":
|
||||||
|
if member.type != tarfile.DIRTYPE or member.size != 0:
|
||||||
|
raise ComponentAdapterError("EoMT mask archive directory type changed")
|
||||||
|
continue
|
||||||
|
if (
|
||||||
|
member.name not in expected_files
|
||||||
|
or member.type not in {tarfile.REGTYPE, tarfile.AREGTYPE}
|
||||||
|
or not 0 < member.size <= 2 * 1024 * 1024
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT mask archive member type changed")
|
||||||
|
stream = archive.extractfile(member)
|
||||||
|
if stream is None:
|
||||||
|
raise ComponentAdapterError("EoMT mask archive member is unavailable")
|
||||||
|
payload = stream.read(member.size + 1)
|
||||||
|
if len(payload) != member.size:
|
||||||
|
raise ComponentAdapterError("EoMT mask archive member length changed")
|
||||||
|
validate_grayscale_png_payload(
|
||||||
|
payload,
|
||||||
|
width=800,
|
||||||
|
height=600,
|
||||||
|
label=f"EoMT mask {member.name}",
|
||||||
|
)
|
||||||
|
except (OSError, EOFError, tarfile.TarError) as exc:
|
||||||
|
raise ComponentAdapterError("EoMT mask archive is invalid") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def _eomt_environment(roots: Mapping[str, Path]) -> dict[str, str]:
|
||||||
|
retained = {
|
||||||
|
key: value
|
||||||
|
for key, value in os.environ.items()
|
||||||
|
if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"}
|
||||||
|
}
|
||||||
|
python_path = f"{roots['eomt-transformers-environment']}:{roots['eomt-python-environment']}"
|
||||||
|
return {
|
||||||
|
**retained,
|
||||||
|
"PYTHONPATH": python_path,
|
||||||
|
"TORCH_HOME": str(roots["eomt-model-cache"] / "torch"),
|
||||||
|
"HF_HOME": str(roots["eomt-model-cache"] / "huggingface"),
|
||||||
|
"HF_HUB_OFFLINE": "1",
|
||||||
|
"TRANSFORMERS_OFFLINE": "1",
|
||||||
|
"PYTHONDONTWRITEBYTECODE": "1",
|
||||||
|
"HOME": "/tmp",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_output_root(path: Path) -> Path:
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
if path.is_symlink() or not resolved.is_dir() or any(resolved.iterdir()):
|
||||||
|
raise ComponentAdapterError("EoMT output root is not an empty real directory")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_workspace(path: Path) -> Path:
|
||||||
|
if path.exists() or path.is_symlink():
|
||||||
|
raise ComponentAdapterError("EoMT workspace already exists")
|
||||||
|
path.mkdir(mode=0o700, parents=False)
|
||||||
|
return path.resolve(strict=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _required_nonempty_file(path: Path, root: Path, label: str) -> Path:
|
||||||
|
resolved = _required_confined_file(path, root, label)
|
||||||
|
if resolved.stat().st_size < 1:
|
||||||
|
raise ComponentAdapterError(f"{label} is empty")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _required_confined_file(path: Path, root: Path, label: str) -> Path:
|
||||||
|
try:
|
||||||
|
metadata = path.lstat()
|
||||||
|
resolved_root = root.resolve(strict=True)
|
||||||
|
resolved = path.resolve(strict=True)
|
||||||
|
except OSError as exc:
|
||||||
|
raise ComponentAdapterError(f"{label} is unavailable") from exc
|
||||||
|
if (
|
||||||
|
stat.S_ISLNK(metadata.st_mode)
|
||||||
|
or not stat.S_ISREG(metadata.st_mode)
|
||||||
|
or not resolved.is_relative_to(resolved_root)
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError(f"{label} is not confined")
|
||||||
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, value: object) -> None:
|
||||||
|
payload = json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n"
|
||||||
|
path.write_text(payload, encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _write_canonical_json(path: Path, value: object) -> None:
|
||||||
|
path.write_bytes(canonical_json(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _publish_component_children(
|
||||||
|
staging: Path,
|
||||||
|
output: Path,
|
||||||
|
*,
|
||||||
|
expected_names: set[str],
|
||||||
|
) -> None:
|
||||||
|
children = tuple(staging.iterdir())
|
||||||
|
if {child.name for child in children} != expected_names or any(
|
||||||
|
child.is_dir() or child.is_symlink() for child in children
|
||||||
|
):
|
||||||
|
raise ComponentAdapterError("EoMT staging contains unexpected directories")
|
||||||
|
for child in children:
|
||||||
|
destination = output / child.name
|
||||||
|
if destination.exists() or destination.is_symlink():
|
||||||
|
raise ComponentAdapterError("EoMT publication target already exists")
|
||||||
|
os.replace(child, destination)
|
||||||
|
staging.rmdir()
|
||||||
|
|
||||||
|
|
||||||
|
def _elapsed(started: float) -> float:
|
||||||
|
return max(time.perf_counter() - started, 0.000001)
|
||||||
|
|
||||||
|
|
||||||
|
def _number(value: float) -> str:
|
||||||
|
return format(value, ".9f").rstrip("0").rstrip(".")
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv)
|
||||||
|
execute_eomt_component(
|
||||||
|
request_path=request_path,
|
||||||
|
layout=RuntimeLayout.fixed(EOMT_ASSETS),
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
raise SystemExit(main())
|
||||||
|
except ComponentAdapterError as exc:
|
||||||
|
print(f"portable LAB V1 EoMT rejected: {exc}", file=sys.stderr)
|
||||||
|
raise SystemExit(2) from exc
|
||||||
@@ -793,6 +793,7 @@ class InstalledPortableLabV1EomtRunner:
|
|||||||
try:
|
try:
|
||||||
request = _component_request(
|
request = _component_request(
|
||||||
component="eomt",
|
component="eomt",
|
||||||
|
component_image_sha256=component.image_sha256,
|
||||||
source=source,
|
source=source,
|
||||||
plan=plan,
|
plan=plan,
|
||||||
assets=component.assets,
|
assets=component.assets,
|
||||||
@@ -854,6 +855,7 @@ class InstalledPortableLabV1DdrnetRunner:
|
|||||||
try:
|
try:
|
||||||
request = _component_request(
|
request = _component_request(
|
||||||
component="ddrnet",
|
component="ddrnet",
|
||||||
|
component_image_sha256=component.image_sha256,
|
||||||
source=source,
|
source=source,
|
||||||
plan=plan,
|
plan=plan,
|
||||||
assets=component.assets,
|
assets=component.assets,
|
||||||
@@ -1022,14 +1024,17 @@ def _prepare_invocation(
|
|||||||
def _component_request(
|
def _component_request(
|
||||||
*,
|
*,
|
||||||
component: PortableLabV1Component,
|
component: PortableLabV1Component,
|
||||||
|
component_image_sha256: str,
|
||||||
source: PortableLabV1MaterializedSource,
|
source: PortableLabV1MaterializedSource,
|
||||||
plan: PortableLabV1OrchestrationPlan,
|
plan: PortableLabV1OrchestrationPlan,
|
||||||
assets: tuple[PortableLabV1HostAsset, ...],
|
assets: tuple[PortableLabV1HostAsset, ...],
|
||||||
effective_config_sha256: str | None,
|
effective_config_sha256: str | None,
|
||||||
) -> dict[str, object]:
|
) -> dict[str, object]:
|
||||||
|
_digest(component_image_sha256, "component request image sha256")
|
||||||
return {
|
return {
|
||||||
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||||
"component": component,
|
"component": component,
|
||||||
|
"component_image_sha256": component_image_sha256,
|
||||||
"plan_sha256": plan.plan_sha256,
|
"plan_sha256": plan.plan_sha256,
|
||||||
"definition_sha256": plan.definition_sha256,
|
"definition_sha256": plan.definition_sha256,
|
||||||
"release_candidate_sha256": plan.release_candidate_sha256,
|
"release_candidate_sha256": plan.release_candidate_sha256,
|
||||||
|
|||||||
@@ -22,7 +22,10 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None:
|
|||||||
assert '"tree-receipt.json"' in script
|
assert '"tree-receipt.json"' in script
|
||||||
assert '"D:\\NDC_MISSIONCORE\\runtime"' in script
|
assert '"D:\\NDC_MISSIONCORE\\runtime"' in script
|
||||||
assert '"derived\\perception-e3-opencv413092-v1"' in script
|
assert '"derived\\perception-e3-opencv413092-v1"' in script
|
||||||
assert '"cache\\perception-e3-models-v1"' in script
|
assert '"eomt-model-cache-cityscapes-large-1024-v1"' in script
|
||||||
|
assert "Install-SafeEomtModelCache" in script
|
||||||
|
assert 'New-Item -ItemType HardLink -Path $blobTarget -Target $source' in script
|
||||||
|
assert 'New-Item -ItemType HardLink -Path $snapshotTarget -Target $source' in script
|
||||||
assert '"derived\\perception-p0-env-v1"' in script
|
assert '"derived\\perception-p0-env-v1"' in script
|
||||||
assert '"derived\\perception-p0-transformers4576-v1"' in script
|
assert '"derived\\perception-p0-transformers4576-v1"' in script
|
||||||
assert '$Utf8.GetByteCount($component) -gt 255' in script
|
assert '$Utf8.GetByteCount($component) -gt 255' in script
|
||||||
@@ -39,15 +42,14 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None:
|
|||||||
assert "smb" not in lowered
|
assert "smb" not in lowered
|
||||||
|
|
||||||
|
|
||||||
def test_eomt_tree_sealer_has_atomic_commit_marker_and_no_rehash_fast_path() -> None:
|
def test_eomt_tree_sealer_has_atomic_commit_marker_and_rehashes_existing_seal() -> None:
|
||||||
script = SEALER_PATH.read_text(encoding="utf-8")
|
script = SEALER_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
fast_path = script.split("function Test-ExistingSeal", maxsplit=1)[1].split(
|
fast_path = script.split("function Test-ExistingSeal", maxsplit=1)[1].split(
|
||||||
"function New-SealedReceipt", maxsplit=1
|
"function New-SealedReceipt", maxsplit=1
|
||||||
)[0]
|
)[0]
|
||||||
assert "-HashPayload:$false" in fast_path
|
assert "-HashPayload:$true" in fast_path
|
||||||
assert "-HashPayload:$true" not in fast_path
|
assert "Assert-InventoryMatches $manifest.records $observed $Asset $false $true" in fast_path
|
||||||
assert "Get-FileDigest" not in fast_path
|
|
||||||
|
|
||||||
publish = script.split("$manifestPath = Join-Path $Root $ManifestName", maxsplit=1)[1]
|
publish = script.split("$manifestPath = Join-Path $Root $ManifestName", maxsplit=1)[1]
|
||||||
manifest_move = publish.index("[IO.File]::Move($stagedManifest, $manifestPath)")
|
manifest_move = publish.index("[IO.File]::Move($stagedManifest, $manifestPath)")
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
INSTALLER_PATH = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments"
|
||||||
|
/ "perception"
|
||||||
|
/ "worker"
|
||||||
|
/ "observatory_portable"
|
||||||
|
/ "Install-LabV1PortableStaticAssets.ps1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_lab_v1_static_installer_is_fixed_offline_and_content_addressed() -> None:
|
||||||
|
script = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert '"D:\\NDC_MISSIONCORE\\runtime"' in script
|
||||||
|
assert '"staging\\observatory-lab-v1-static-v1"' in script
|
||||||
|
assert '"eomt-runner-bundle-v1"' in script
|
||||||
|
assert '"lab-v1-static-files-v1"' in script
|
||||||
|
assert "New-Item `\n -ItemType HardLink" in script
|
||||||
|
assert "Get-FileIdentity" in script
|
||||||
|
assert "tree-manifest.tsv" in script
|
||||||
|
assert "tree-receipt.json" in script
|
||||||
|
|
||||||
|
lowered = script.lower()
|
||||||
|
assert "invoke-webrequest" not in lowered
|
||||||
|
assert "start-bitstransfer" not in lowered
|
||||||
|
assert "docker " not in lowered
|
||||||
|
assert "ssh " not in lowered
|
||||||
|
assert "smb" not in lowered
|
||||||
|
|
||||||
|
|
||||||
|
def test_lab_v1_static_installer_seals_expected_exact_payloads() -> None:
|
||||||
|
script = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4",
|
||||||
|
"01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6",
|
||||||
|
"4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675",
|
||||||
|
"25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4",
|
||||||
|
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875",
|
||||||
|
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
|
||||||
|
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f",
|
||||||
|
"b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1",
|
||||||
|
"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35",
|
||||||
|
"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352",
|
||||||
|
"a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63",
|
||||||
|
}
|
||||||
|
assert all(digest in script for digest in expected)
|
||||||
|
assert "259419077" in script
|
||||||
|
assert "32877" in script
|
||||||
|
assert "1427" in script
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -552,7 +552,11 @@ def test_component_adapter_images_are_thin_offline_wrappers() -> None:
|
|||||||
).read_text(encoding="utf-8")
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
assert "FROM nvcr.io/nvidia/tritonserver:26.06-py3@sha256:" in eomt
|
assert "FROM nvcr.io/nvidia/tritonserver:26.06-py3@sha256:" in eomt
|
||||||
assert "FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1" in ddrnet
|
assert (
|
||||||
|
"FROM sha256:"
|
||||||
|
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||||
|
in ddrnet
|
||||||
|
)
|
||||||
assert "COPY portable_lab_v1_component_adapter.py" in eomt
|
assert "COPY portable_lab_v1_component_adapter.py" in eomt
|
||||||
assert "COPY portable_lab_v1_component_adapter.py" in ddrnet
|
assert "COPY portable_lab_v1_component_adapter.py" in ddrnet
|
||||||
assert 'ENTRYPOINT ["python3"]' in eomt
|
assert 'ENTRYPOINT ["python3"]' in eomt
|
||||||
|
|||||||
Reference in New Issue
Block a user