feat(lab): install sealed component adapters
This commit is contained in:
DCCONSTRUCTIONS
@@ -1,4 +1,4 @@
|
||||
FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1
|
||||
FROM sha256:591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
|
||||
$SourceRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-static-v1"
|
||||
$AssetRoot = Join-Path $RuntimeRoot "assets\observatory-portable"
|
||||
$RunnerRoot = Join-Path $AssetRoot "eomt-runner-bundle-v1"
|
||||
$ValidFovRoot = Join-Path $AssetRoot (
|
||||
"k1-valid-fov-root-" +
|
||||
"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1"
|
||||
)
|
||||
$StaticRoot = Join-Path $AssetRoot "lab-v1-static-files-v1"
|
||||
$ValidFovSource = Join-Path $RuntimeRoot (
|
||||
"inputs\e2\valid-fov-mask-" +
|
||||
"b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2"
|
||||
)
|
||||
$CheckpointSource = (
|
||||
"D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27\" +
|
||||
"models\goose\ddrnet_class_512.pth"
|
||||
)
|
||||
$MappingSource = (
|
||||
"D:\NDC_MISSIONCORE\datasets\goose-3d\v2025-08-22\installs\" +
|
||||
"0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f\" +
|
||||
"goose_label_mapping.csv"
|
||||
)
|
||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
|
||||
$RunnerFiles = [ordered]@{
|
||||
"e3_k1_camera1_profile.json" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "e3_k1_camera1_profile.json")
|
||||
byte_length = [int64]3805
|
||||
sha256 = "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875"
|
||||
}
|
||||
"run_e3_rectified_segmentation.py" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "run_e3_rectified_segmentation.py")
|
||||
byte_length = [int64]45789
|
||||
sha256 = "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6"
|
||||
}
|
||||
"run_e4_full_session_segmentation.py" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "run_e4_full_session_segmentation.py")
|
||||
byte_length = [int64]30720
|
||||
sha256 = "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4"
|
||||
}
|
||||
"run_evaluation_prelabels.py" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "run_evaluation_prelabels.py")
|
||||
byte_length = [int64]28915
|
||||
sha256 = "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4"
|
||||
}
|
||||
"run_recorded_perception_epoch.py" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "run_recorded_perception_epoch.py")
|
||||
byte_length = [int64]34899
|
||||
sha256 = "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675"
|
||||
}
|
||||
}
|
||||
|
||||
$ValidFovFiles = [ordered]@{
|
||||
"manifest.json" = [pscustomobject]@{
|
||||
source = (Join-Path $ValidFovSource "manifest.json")
|
||||
byte_length = [int64]2351
|
||||
sha256 = "3dea056db506f47b8a7672e847d0ffe4c66a380a5032555a2e0dde958293742e"
|
||||
}
|
||||
"mask.png" = [pscustomobject]@{
|
||||
source = (Join-Path $ValidFovSource "mask.png")
|
||||
byte_length = [int64]3668
|
||||
sha256 = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63"
|
||||
}
|
||||
}
|
||||
|
||||
$StaticFiles = [ordered]@{
|
||||
"ddrnet-checkpoint.pth" = [pscustomobject]@{
|
||||
source = $CheckpointSource
|
||||
byte_length = [int64]259419077
|
||||
sha256 = "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"
|
||||
}
|
||||
"ddrnet-goose-mapping.csv" = [pscustomobject]@{
|
||||
source = $MappingSource
|
||||
byte_length = [int64]1427
|
||||
sha256 = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"
|
||||
}
|
||||
"ddrnet-goose-runner.py" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "run_goose_vegetation_benchmark.py")
|
||||
byte_length = [int64]32877
|
||||
sha256 = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"
|
||||
}
|
||||
"vegetation-policy.json" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "lab-v1-vegetation-mission-policy-v1.json")
|
||||
byte_length = [int64]3022
|
||||
sha256 = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||
}
|
||||
"vegetation-provider-map.json" = [pscustomobject]@{
|
||||
source = (Join-Path $SourceRoot "lab-v1-vegetation-provider-label-map-v1.json")
|
||||
byte_length = [int64]2756
|
||||
sha256 = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-FileIdentity {
|
||||
param([string]$Path)
|
||||
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (
|
||||
-not ($item -is [IO.FileInfo]) -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||
) {
|
||||
throw "LAB V1 static source is not a regular file"
|
||||
}
|
||||
$stream = [IO.File]::Open(
|
||||
$item.FullName,
|
||||
[IO.FileMode]::Open,
|
||||
[IO.FileAccess]::Read,
|
||||
[IO.FileShare]::Read
|
||||
)
|
||||
$algorithm = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$length = [int64]$stream.Length
|
||||
$sha256 = -join @($algorithm.ComputeHash($stream) | ForEach-Object {
|
||||
$_.ToString("x2", $Invariant)
|
||||
})
|
||||
}
|
||||
finally {
|
||||
$algorithm.Dispose()
|
||||
$stream.Dispose()
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
byte_length = $length
|
||||
sha256 = $sha256
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-ExactFile {
|
||||
param(
|
||||
[string]$Path,
|
||||
[object]$Descriptor,
|
||||
[string]$Label
|
||||
)
|
||||
|
||||
$identity = Get-FileIdentity $Path
|
||||
if (
|
||||
[int64]$identity.byte_length -ne [int64]$Descriptor.byte_length -or
|
||||
[string]$identity.sha256 -cne [string]$Descriptor.sha256
|
||||
) {
|
||||
throw "$Label identity changed"
|
||||
}
|
||||
}
|
||||
|
||||
function Install-HardlinkedDirectory {
|
||||
param(
|
||||
[string]$Target,
|
||||
[Collections.IDictionary]$Files,
|
||||
[bool]$AllowTreeSealMetadata
|
||||
)
|
||||
|
||||
foreach ($name in $Files.Keys) {
|
||||
Assert-ExactFile $Files[$name].source $Files[$name] "LAB V1 source $name"
|
||||
}
|
||||
if (Test-Path -LiteralPath $Target -PathType Container) {
|
||||
$allowed = @($Files.Keys)
|
||||
if ($AllowTreeSealMetadata) {
|
||||
$allowed += @("tree-manifest.tsv", "tree-receipt.json")
|
||||
}
|
||||
$observed = @(
|
||||
Get-ChildItem -LiteralPath $Target -Force | ForEach-Object {
|
||||
if (-not ($_ -is [IO.FileInfo])) {
|
||||
throw "Installed LAB V1 asset contains a non-file member"
|
||||
}
|
||||
$_.Name
|
||||
}
|
||||
)
|
||||
if (
|
||||
@($observed | Sort-Object).Count -ne @($allowed | Sort-Object).Count -or
|
||||
[string]::Join([char]10, @($observed | Sort-Object)) -cne
|
||||
[string]::Join([char]10, @($allowed | Sort-Object))
|
||||
) {
|
||||
throw "Installed LAB V1 asset inventory changed"
|
||||
}
|
||||
foreach ($name in $Files.Keys) {
|
||||
Assert-ExactFile (Join-Path $Target $name) $Files[$name] "Installed $name"
|
||||
}
|
||||
return "already-installed"
|
||||
}
|
||||
if (Test-Path -LiteralPath $Target) {
|
||||
throw "LAB V1 asset target is not a directory"
|
||||
}
|
||||
$parent = Split-Path -Parent $Target
|
||||
if (-not (Test-Path -LiteralPath $parent -PathType Container)) {
|
||||
throw "LAB V1 asset parent is unavailable"
|
||||
}
|
||||
$staging = "$Target.staging-$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$null = New-Item -ItemType Directory -Path $staging
|
||||
foreach ($name in $Files.Keys) {
|
||||
$null = New-Item `
|
||||
-ItemType HardLink `
|
||||
-Path (Join-Path $staging $name) `
|
||||
-Target $Files[$name].source
|
||||
Assert-ExactFile (Join-Path $staging $name) $Files[$name] "Staged $name"
|
||||
}
|
||||
[IO.Directory]::Move($staging, $Target)
|
||||
return "installed"
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $staging -PathType Container) {
|
||||
Remove-Item -LiteralPath $staging -Recurse -Force
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $SourceRoot -PathType Container)) {
|
||||
throw "LAB V1 static source staging root is unavailable"
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $AssetRoot -PathType Container)) {
|
||||
throw "LAB V1 immutable asset root is unavailable"
|
||||
}
|
||||
|
||||
$outputs = @(
|
||||
[ordered]@{
|
||||
asset_id = "eomt-runner-bundle"
|
||||
status = Install-HardlinkedDirectory $RunnerRoot $RunnerFiles $true
|
||||
path = $RunnerRoot
|
||||
}
|
||||
[ordered]@{
|
||||
asset_id = "k1-valid-fov-root"
|
||||
status = Install-HardlinkedDirectory $ValidFovRoot $ValidFovFiles $true
|
||||
path = $ValidFovRoot
|
||||
}
|
||||
[ordered]@{
|
||||
asset_id = "lab-v1-static-files"
|
||||
status = Install-HardlinkedDirectory $StaticRoot $StaticFiles $false
|
||||
path = $StaticRoot
|
||||
}
|
||||
)
|
||||
|
||||
$outputs | ConvertTo-Json -Compress -Depth 4
|
||||
@@ -28,13 +28,41 @@ $IdentityAlgorithm = "relative-path-tab-size-tab-file-sha256-lf/v1"
|
||||
$MaximumManifestBytes = 32MB
|
||||
$Utf8 = New-Object Text.UTF8Encoding($false, $true)
|
||||
$Invariant = [Globalization.CultureInfo]::InvariantCulture
|
||||
$OriginalModelCache = (
|
||||
"D:\NDC_MISSIONCORE\runtime\cache\perception-e3-models-v1\huggingface\" +
|
||||
"models--tue-mps--cityscapes_semantic_eomt_large_1024"
|
||||
)
|
||||
$SafeModelCache = (
|
||||
"D:\NDC_MISSIONCORE\runtime\assets\observatory-portable\" +
|
||||
"eomt-model-cache-cityscapes-large-1024-v1"
|
||||
)
|
||||
$ModelRevision = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||
$ModelFiles = [ordered]@{
|
||||
"config.json" = [pscustomobject]@{
|
||||
blob = "66b2d540c2a4d9e9fbc958c1a43464cb13d933c5"
|
||||
byte_length = [int64]1575
|
||||
sha256 = "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650"
|
||||
}
|
||||
"model.safetensors" = [pscustomobject]@{
|
||||
blob = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||
byte_length = [int64]1276175488
|
||||
sha256 = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782"
|
||||
}
|
||||
"preprocessor_config.json" = [pscustomobject]@{
|
||||
blob = "cf30bcc29300598a6b581ed043dd65abc893e948"
|
||||
byte_length = [int64]666
|
||||
sha256 = "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7"
|
||||
}
|
||||
}
|
||||
|
||||
# The first four roots already exist on Worker 006. The runner bundle and
|
||||
# valid-FOV root deliberately point at separate, immutable asset directories;
|
||||
# their payloads must be materialized there before this sealer is invoked.
|
||||
$AssetRelativeRoots = [ordered]@{
|
||||
"eomt-environment" = "derived\perception-e3-opencv413092-v1"
|
||||
"eomt-model-cache" = "cache\perception-e3-models-v1"
|
||||
"eomt-model-cache" = (
|
||||
"assets\observatory-portable\eomt-model-cache-cityscapes-large-1024-v1"
|
||||
)
|
||||
"eomt-python-environment" = "derived\perception-p0-env-v1"
|
||||
"eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1"
|
||||
"eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1"
|
||||
@@ -152,6 +180,58 @@ function Get-FileDigest {
|
||||
}
|
||||
}
|
||||
|
||||
function Install-SafeEomtModelCache {
|
||||
if (Test-Path -LiteralPath $SafeModelCache -PathType Container) {
|
||||
return
|
||||
}
|
||||
if (Test-Path -LiteralPath $SafeModelCache) {
|
||||
throw "Safe EoMT model cache target is not a directory"
|
||||
}
|
||||
$assetParent = Split-Path -Parent $SafeModelCache
|
||||
if (-not (Test-Path -LiteralPath $assetParent -PathType Container)) {
|
||||
throw "Safe EoMT model cache parent is unavailable"
|
||||
}
|
||||
$staging = "$SafeModelCache.staging-$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$blobs = Join-Path $staging (
|
||||
"huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\blobs"
|
||||
)
|
||||
$snapshot = Join-Path $staging (
|
||||
"huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\" +
|
||||
"snapshots\$ModelRevision"
|
||||
)
|
||||
$null = New-Item -ItemType Directory -Path $blobs
|
||||
$null = New-Item -ItemType Directory -Path $snapshot
|
||||
foreach ($name in $ModelFiles.Keys) {
|
||||
$descriptor = $ModelFiles[$name]
|
||||
$source = Join-Path (Join-Path $OriginalModelCache "blobs") $descriptor.blob
|
||||
$identity = Get-FileDigest $source
|
||||
if (
|
||||
[int64]$identity.byte_length -ne [int64]$descriptor.byte_length -or
|
||||
[string]$identity.sha256 -cne [string]$descriptor.sha256
|
||||
) {
|
||||
throw "Original EoMT model cache identity changed"
|
||||
}
|
||||
$blobTarget = Join-Path $blobs $descriptor.blob
|
||||
$snapshotTarget = Join-Path $snapshot $name
|
||||
$null = New-Item -ItemType HardLink -Path $blobTarget -Target $source
|
||||
$null = New-Item -ItemType HardLink -Path $snapshotTarget -Target $source
|
||||
}
|
||||
foreach ($entry in [IO.Directory]::EnumerateFiles($staging, "*", "AllDirectories")) {
|
||||
$attributes = [IO.File]::GetAttributes($entry)
|
||||
if ($attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Safe EoMT model cache contains a reparse point"
|
||||
}
|
||||
}
|
||||
[IO.Directory]::Move($staging, $SafeModelCache)
|
||||
}
|
||||
finally {
|
||||
if (Test-Path -LiteralPath $staging -PathType Container) {
|
||||
Remove-Item -LiteralPath $staging -Recurse -Force
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-PayloadInventory {
|
||||
param(
|
||||
[string]$Root,
|
||||
@@ -388,7 +468,8 @@ function Assert-InventoryMatches {
|
||||
[object[]]$Expected,
|
||||
[object[]]$Observed,
|
||||
[string]$Asset,
|
||||
[bool]$CheckWriteTime
|
||||
[bool]$CheckWriteTime,
|
||||
[bool]$CheckDigest
|
||||
)
|
||||
|
||||
if ($Expected.Count -ne $Observed.Count) {
|
||||
@@ -400,6 +481,9 @@ function Assert-InventoryMatches {
|
||||
if (
|
||||
[string]$left.relative_path -cne [string]$right.relative_path -or
|
||||
[int64]$left.byte_length -ne [int64]$right.byte_length -or
|
||||
($CheckDigest -and (
|
||||
[string]$left.sha256 -cne [string]$right.sha256
|
||||
)) -or
|
||||
($CheckWriteTime -and (
|
||||
[int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks
|
||||
))
|
||||
@@ -438,10 +522,11 @@ function Test-ExistingSeal {
|
||||
throw "Existing $Asset receipt does not bind its manifest"
|
||||
}
|
||||
|
||||
# Fast path: verify names and sizes, but deliberately do not re-hash the
|
||||
# already sealed multi-gigabyte payload.
|
||||
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
|
||||
Assert-InventoryMatches $manifest.records $observed $Asset $false
|
||||
# A seal is reusable only after the payload has been re-verified. Tree
|
||||
# roots contain executable Python dependencies, so names and sizes alone
|
||||
# must never authenticate an existing seal.
|
||||
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$true)
|
||||
Assert-InventoryMatches $manifest.records $observed $Asset $false $true
|
||||
return $receipt
|
||||
}
|
||||
|
||||
@@ -519,7 +604,7 @@ function Seal-TreeAsset {
|
||||
# Re-enumerate without hashing before publication. This catches file,
|
||||
# size, and timestamp changes that occurred during the one-time hash.
|
||||
$observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false)
|
||||
Assert-InventoryMatches $records $observed $Asset $true
|
||||
Assert-InventoryMatches $records $observed $Asset $true $false
|
||||
|
||||
$null = New-Item -ItemType Directory -Path $staging
|
||||
$stagedManifest = Join-Path $staging $ManifestName
|
||||
@@ -573,6 +658,9 @@ $requested = @($AssetId)
|
||||
if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) {
|
||||
throw "LAB V1 EoMT asset selection is empty or contains duplicates"
|
||||
}
|
||||
if ($requested -contains "eomt-model-cache") {
|
||||
Install-SafeEomtModelCache
|
||||
}
|
||||
$outputs = New-Object "Collections.Generic.List[object]"
|
||||
foreach ($asset in $requested) {
|
||||
$relativeRoot = [string]$AssetRelativeRoots[$asset]
|
||||
|
||||
+1116
File diff suppressed because it is too large
Load Diff
+828
@@ -0,0 +1,828 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the sealed LAB V1 DDRNet component over EoMT-decoded K1 frames."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import zipfile
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from PIL import Image
|
||||
from portable_lab_v1_component_adapter import (
|
||||
DDRNET_CHECKPOINT_SHA256,
|
||||
EOMT_PIPELINE,
|
||||
GOOSE_MAPPING_SHA256,
|
||||
AssetExpectation,
|
||||
CommandRunner,
|
||||
ComponentAdapterError,
|
||||
ComponentRequest,
|
||||
RuntimeLayout,
|
||||
canonical_json,
|
||||
load_canonical_json,
|
||||
load_component_request,
|
||||
load_json,
|
||||
parse_fixed_request_argument,
|
||||
run_command,
|
||||
sha256_file,
|
||||
validate_camera_compute_job,
|
||||
validate_file_asset,
|
||||
validate_fixed_result,
|
||||
validate_grayscale_png_payload,
|
||||
)
|
||||
|
||||
DDRNET_RESULT_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1"
|
||||
EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2"
|
||||
EFFECTIVE_CONFIG_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-benchmark/v1"
|
||||
DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1"
|
||||
DDRNET_CANDIDATE_ID: Final = "goose-ddrnet-class-512"
|
||||
DDRNET_LOADED_MODEL: Final = "ddrnet_39"
|
||||
DDRNET_CHECKPOINT_BYTES: Final = 259_419_077
|
||||
DDRNET_STATIC_CONFIG_SHA256: Final = (
|
||||
"3c2a60def178d824e6ce612cad9a108da496de769edf0de66099d2ecca6b51bb"
|
||||
)
|
||||
RUNNER_SHA256: Final = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"
|
||||
POLICY_SHA256: Final = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"
|
||||
PROVIDER_MAP_SHA256: Final = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"
|
||||
DDRNET_AUTHORITY: Final = {
|
||||
"navigation_accepted": False,
|
||||
"safety_accepted": False,
|
||||
"actuation_accepted": False,
|
||||
"camera_semantics_can_clear_rigid_geometry": False,
|
||||
}
|
||||
EOMT_MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||
EOMT_MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||
EOMT_MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation"
|
||||
SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SourceFrameRow:
|
||||
sequence: int
|
||||
path: str
|
||||
byte_length: int
|
||||
sha256: str
|
||||
|
||||
|
||||
DDRNET_ASSETS: Final = (
|
||||
AssetExpectation(
|
||||
"ddrnet-checkpoint",
|
||||
"/opt/nodedc/assets/ddrnet-checkpoint",
|
||||
"file",
|
||||
"sha256",
|
||||
DDRNET_CHECKPOINT_SHA256,
|
||||
DDRNET_CHECKPOINT_BYTES,
|
||||
),
|
||||
AssetExpectation(
|
||||
"ddrnet-goose-mapping",
|
||||
"/opt/nodedc/assets/ddrnet-goose-mapping",
|
||||
"file",
|
||||
"sha256",
|
||||
GOOSE_MAPPING_SHA256,
|
||||
),
|
||||
AssetExpectation(
|
||||
"ddrnet-goose-runner",
|
||||
"/opt/nodedc/assets/ddrnet-goose-runner",
|
||||
"file",
|
||||
"sha256",
|
||||
RUNNER_SHA256,
|
||||
32_877,
|
||||
),
|
||||
AssetExpectation(
|
||||
"vegetation-policy",
|
||||
"/opt/nodedc/assets/vegetation-policy",
|
||||
"file",
|
||||
"sha256",
|
||||
POLICY_SHA256,
|
||||
3_022,
|
||||
),
|
||||
AssetExpectation(
|
||||
"vegetation-provider-map",
|
||||
"/opt/nodedc/assets/vegetation-provider-map",
|
||||
"file",
|
||||
"sha256",
|
||||
PROVIDER_MAP_SHA256,
|
||||
2_756,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def execute_ddrnet_component(
|
||||
*,
|
||||
request_path: Path,
|
||||
layout: RuntimeLayout,
|
||||
command_runner: CommandRunner = run_command,
|
||||
work_root: Path = Path("/tmp/nodedc-lab-v1-ddrnet"),
|
||||
) -> None:
|
||||
request = load_component_request(
|
||||
request_path,
|
||||
component="ddrnet",
|
||||
expectations=DDRNET_ASSETS,
|
||||
)
|
||||
validate_camera_compute_job(layout.camera_job_root, request.source)
|
||||
assets = {
|
||||
item.asset_id: validate_file_asset(request, layout, item.asset_id) for item in DDRNET_ASSETS
|
||||
}
|
||||
effective = _validate_effective_config(
|
||||
layout.effective_ddrnet_config,
|
||||
request,
|
||||
)
|
||||
eomt_root, source_frames, decode_repair = _validate_eomt_input(
|
||||
layout.eomt_result_root,
|
||||
request,
|
||||
)
|
||||
output = _empty_output_root(layout.output_root)
|
||||
workspace = _prepare_workspace(work_root)
|
||||
try:
|
||||
frames_root = _snapshot_source_frames(
|
||||
eomt_root,
|
||||
source_frames,
|
||||
workspace / "source-frames",
|
||||
)
|
||||
mapping_copy = workspace / "goose_label_mapping.csv"
|
||||
shutil.copyfile(assets["ddrnet-goose-mapping"], mapping_copy)
|
||||
os.chmod(mapping_copy, 0o400)
|
||||
if sha256_file(mapping_copy) != GOOSE_MAPPING_SHA256:
|
||||
raise ComponentAdapterError("DDRNet private mapping copy changed")
|
||||
staging = output / ".ddrnet-component"
|
||||
argv = (
|
||||
sys.executable,
|
||||
str(assets["ddrnet-goose-runner"]),
|
||||
"--mode",
|
||||
"ravnoves-video",
|
||||
"--candidate",
|
||||
"ddrnet",
|
||||
"--config",
|
||||
str(layout.effective_ddrnet_config),
|
||||
"--policy",
|
||||
str(assets["vegetation-policy"]),
|
||||
"--provider-map",
|
||||
str(assets["vegetation-provider-map"]),
|
||||
"--checkpoint",
|
||||
str(assets["ddrnet-checkpoint"]),
|
||||
"--dataset-root",
|
||||
str(workspace),
|
||||
"--frames-root",
|
||||
str(frames_root),
|
||||
"--output",
|
||||
str(staging),
|
||||
"--limit",
|
||||
"0",
|
||||
"--visual-count",
|
||||
"0",
|
||||
)
|
||||
command_runner(argv, _ddrnet_environment())
|
||||
shutil.copyfile(eomt_root / "decode-repair.json", staging / "decode-repair.json")
|
||||
if (
|
||||
load_json(
|
||||
staging / "decode-repair.json",
|
||||
label="DDRNet decode repair",
|
||||
maximum=1024 * 1024,
|
||||
confinement_root=staging,
|
||||
)
|
||||
!= decode_repair
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet decode repair copy changed")
|
||||
_validate_ddrnet_result(
|
||||
staging,
|
||||
request,
|
||||
effective,
|
||||
mapping_path=assets["ddrnet-goose-mapping"],
|
||||
)
|
||||
_publish_component_children(
|
||||
staging,
|
||||
output,
|
||||
expected_names={
|
||||
"decode-repair.json",
|
||||
"result.json",
|
||||
"semantic-masks.zip",
|
||||
},
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(workspace, ignore_errors=True)
|
||||
|
||||
|
||||
def _validate_effective_config(
|
||||
path: Path,
|
||||
request: ComponentRequest,
|
||||
) -> dict[str, object]:
|
||||
config = load_canonical_json(
|
||||
path,
|
||||
label="effective DDRNet config",
|
||||
maximum=1024 * 1024,
|
||||
)
|
||||
if (
|
||||
request.effective_ddrnet_config_sha256 is None
|
||||
or sha256_file(path) != request.effective_ddrnet_config_sha256
|
||||
):
|
||||
raise ComponentAdapterError("effective DDRNet config identity changed")
|
||||
expected_keys = {
|
||||
"schema_version",
|
||||
"lab_id",
|
||||
"worker_id",
|
||||
"runtime",
|
||||
"dataset",
|
||||
"candidates",
|
||||
"vegetation_class_names",
|
||||
"visual_case_contract",
|
||||
"policy_action_colors",
|
||||
"invariants",
|
||||
"ravnoves",
|
||||
}
|
||||
if set(config) != expected_keys:
|
||||
raise ComponentAdapterError("effective DDRNet config fields changed")
|
||||
static = dict(config)
|
||||
ravnoves = static.pop("ravnoves")
|
||||
if (
|
||||
config.get("schema_version") != EFFECTIVE_CONFIG_SCHEMA
|
||||
or config.get("lab_id") != "LAB-V1"
|
||||
or config.get("worker_id") != "worker-006"
|
||||
or hashlib.sha256(canonical_json(static)).hexdigest() != DDRNET_STATIC_CONFIG_SHA256
|
||||
):
|
||||
raise ComponentAdapterError("effective DDRNet static configuration changed")
|
||||
source = request.source
|
||||
expected_ravnoves = {
|
||||
"source_id": (
|
||||
f"portable-k1/{source.source_session_id}/"
|
||||
f"{source.camera_source_id}@{source.camera_input_sha256}"
|
||||
),
|
||||
"source_sha256": source.camera_input_sha256,
|
||||
"base_m4_result_id": None,
|
||||
"expected_width": 800,
|
||||
"expected_height": 600,
|
||||
"expected_frame_count": source.frame_count,
|
||||
"frame_indices": _representative_frame_indices(source.frame_count),
|
||||
"crop_contract": "center-600-square-to-512; outside-crop-is-undefined",
|
||||
}
|
||||
if ravnoves != expected_ravnoves:
|
||||
raise ComponentAdapterError("effective DDRNet source configuration changed")
|
||||
return config
|
||||
|
||||
|
||||
def _validate_eomt_input(
|
||||
root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> tuple[Path, tuple[SourceFrameRow, ...], dict[str, object]]:
|
||||
if root.is_symlink():
|
||||
raise ComponentAdapterError("EoMT result root is a symbolic link")
|
||||
resolved = root.resolve(strict=True)
|
||||
result = validate_fixed_result(
|
||||
resolved,
|
||||
schema_version=EOMT_RESULT_SCHEMA,
|
||||
label="EoMT",
|
||||
)
|
||||
identity = result.get("identity")
|
||||
if not isinstance(identity, dict):
|
||||
raise ComponentAdapterError("EoMT result identity is unavailable")
|
||||
configuration = identity.get("configuration")
|
||||
models = identity.get("models")
|
||||
semantic = models.get("semantic") if isinstance(models, dict) else None
|
||||
observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
source = request.source
|
||||
if (
|
||||
not isinstance(configuration, dict)
|
||||
or result.get("identity_sha256") != observed_identity
|
||||
or result.get("result_id") != f"result-{observed_identity}"
|
||||
or result.get("job_id") != source.camera_job_id
|
||||
or result.get("input_sha256") != source.camera_input_sha256
|
||||
or result.get("session_id") != source.source_session_id
|
||||
or result.get("source_id") != source.camera_source_id
|
||||
or result.get("codec_epoch") != source.codec_epoch
|
||||
or result.get("timestamp_basis") != "session-time-seconds"
|
||||
or result.get("timeline_start_seconds") != source.timeline_start_seconds
|
||||
or result.get("timeline_end_seconds") != source.timeline_end_seconds
|
||||
or result.get("frames_processed") != source.frame_count
|
||||
or result.get("ground_truth") is not False
|
||||
or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2"
|
||||
or configuration.get("pipeline") != EOMT_PIPELINE
|
||||
or not isinstance(semantic, dict)
|
||||
or semantic.get("id") != EOMT_MODEL_ID
|
||||
or semantic.get("revision") != EOMT_MODEL_REVISION
|
||||
or semantic.get("architecture") != EOMT_MODEL_ARCHITECTURE
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source result changed")
|
||||
artifacts = result.get("artifacts")
|
||||
if not isinstance(artifacts, list) or len(artifacts) != 6:
|
||||
raise ComponentAdapterError("EoMT source artifact set changed")
|
||||
expected_artifacts = {
|
||||
"panoptic-overlay-video": ("perception.mp4", "video/mp4"),
|
||||
"panoptic-mask-archive": ("masks.tar.gz", "application/gzip"),
|
||||
"panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"),
|
||||
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"),
|
||||
"perception-run-report": ("run-report.json", "application/json"),
|
||||
"decoded-source-frame-manifest": ("source-frames.json", "application/json"),
|
||||
}
|
||||
observed_kinds: set[str] = set()
|
||||
for value in artifacts:
|
||||
if not isinstance(value, dict):
|
||||
raise ComponentAdapterError("EoMT source artifact is invalid")
|
||||
kind = value.get("kind")
|
||||
encoded = value.get("path")
|
||||
if (
|
||||
not isinstance(kind, str)
|
||||
or kind in observed_kinds
|
||||
or expected_artifacts.get(kind) != (encoded, value.get("media_type"))
|
||||
or not isinstance(encoded, str)
|
||||
or not encoded
|
||||
or "/" in encoded
|
||||
or "\\" in encoded
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source artifact path changed")
|
||||
observed_kinds.add(kind)
|
||||
artifact = _confined_regular_file(resolved / encoded, resolved, "EoMT artifact")
|
||||
if (
|
||||
artifact.stat().st_size < 1
|
||||
or value.get("byte_length") != artifact.stat().st_size
|
||||
or value.get("sha256") != sha256_file(artifact)
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source artifact identity changed")
|
||||
if observed_kinds != set(expected_artifacts):
|
||||
raise ComponentAdapterError("EoMT source artifact roles changed")
|
||||
manifest_path = _confined_regular_file(
|
||||
resolved / "source-frames.json",
|
||||
resolved,
|
||||
"EoMT source frame manifest",
|
||||
)
|
||||
source_frame_binding = identity.get("source_frames")
|
||||
expected_binding = {
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"path": "source-frames.json",
|
||||
"sha256": sha256_file(manifest_path),
|
||||
"byte_length": manifest_path.stat().st_size,
|
||||
"frame_count": source.frame_count,
|
||||
}
|
||||
if source_frame_binding != expected_binding:
|
||||
raise ComponentAdapterError("EoMT source frame manifest binding changed")
|
||||
source_frames = _source_frame_manifest(manifest_path, resolved, request)
|
||||
repair = load_json(
|
||||
resolved / "decode-repair.json",
|
||||
label="EoMT decode repair",
|
||||
maximum=1024 * 1024,
|
||||
confinement_root=resolved,
|
||||
)
|
||||
_validate_decode_repair(repair, source.frame_count)
|
||||
return resolved, source_frames, repair
|
||||
|
||||
|
||||
def _source_frame_manifest(
|
||||
path: Path,
|
||||
eomt_root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> tuple[SourceFrameRow, ...]:
|
||||
document = load_canonical_json(
|
||||
path,
|
||||
label="EoMT source frame manifest",
|
||||
maximum=32 * 1024 * 1024,
|
||||
confinement_root=eomt_root,
|
||||
)
|
||||
source = request.source
|
||||
if set(document) != {
|
||||
"schema_version",
|
||||
"job_id",
|
||||
"input_sha256",
|
||||
"source_id",
|
||||
"codec_epoch",
|
||||
"frame_count",
|
||||
"frames",
|
||||
} or document != {
|
||||
**document,
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"job_id": source.camera_job_id,
|
||||
"input_sha256": source.camera_input_sha256,
|
||||
"source_id": source.camera_source_id,
|
||||
"codec_epoch": source.codec_epoch,
|
||||
"frame_count": source.frame_count,
|
||||
}:
|
||||
raise ComponentAdapterError("EoMT source frame manifest identity changed")
|
||||
raw_rows = document["frames"]
|
||||
if not isinstance(raw_rows, list) or len(raw_rows) != source.frame_count:
|
||||
raise ComponentAdapterError("EoMT source frame manifest row count changed")
|
||||
rows: list[SourceFrameRow] = []
|
||||
for sequence, value in enumerate(raw_rows, start=1):
|
||||
if not isinstance(value, dict) or set(value) != {
|
||||
"sequence",
|
||||
"path",
|
||||
"byte_length",
|
||||
"sha256",
|
||||
}:
|
||||
raise ComponentAdapterError("EoMT source frame manifest row changed")
|
||||
encoded = f"source-frames/frame-{sequence:06d}.png"
|
||||
byte_length = value.get("byte_length")
|
||||
digest = value.get("sha256")
|
||||
if (
|
||||
value.get("sequence") != sequence
|
||||
or value.get("path") != encoded
|
||||
or isinstance(byte_length, bool)
|
||||
or not isinstance(byte_length, int)
|
||||
or not 0 < byte_length <= 16 * 1024 * 1024
|
||||
or not isinstance(digest, str)
|
||||
or len(digest) != 64
|
||||
or any(character not in "0123456789abcdef" for character in digest)
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source frame manifest row identity changed")
|
||||
frame = _confined_regular_file(
|
||||
eomt_root / encoded,
|
||||
eomt_root,
|
||||
"EoMT source frame",
|
||||
)
|
||||
if frame.stat().st_size != byte_length:
|
||||
raise ComponentAdapterError("EoMT source frame length changed")
|
||||
rows.append(SourceFrameRow(sequence, encoded, byte_length, digest))
|
||||
frames_root = eomt_root / "source-frames"
|
||||
expected_names = tuple(
|
||||
f"frame-{sequence:06d}.png" for sequence in range(1, source.frame_count + 1)
|
||||
)
|
||||
try:
|
||||
observed_names = tuple(sorted(candidate.name for candidate in frames_root.iterdir()))
|
||||
except OSError as exc:
|
||||
raise ComponentAdapterError("EoMT source frame set is unavailable") from exc
|
||||
if observed_names != expected_names:
|
||||
raise ComponentAdapterError("EoMT source frame set changed")
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def _snapshot_source_frames(
|
||||
eomt_root: Path,
|
||||
rows: Sequence[SourceFrameRow],
|
||||
snapshot_root: Path,
|
||||
) -> Path:
|
||||
snapshot_root.mkdir(mode=0o700)
|
||||
for row in rows:
|
||||
source = _confined_regular_file(
|
||||
eomt_root / row.path,
|
||||
eomt_root,
|
||||
"EoMT source frame",
|
||||
)
|
||||
destination = snapshot_root / f"frame-{row.sequence:06d}.png"
|
||||
digest = hashlib.sha256()
|
||||
byte_length = 0
|
||||
signature = b""
|
||||
descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
try:
|
||||
opened = os.fstat(descriptor)
|
||||
admitted = source.stat()
|
||||
if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != (
|
||||
admitted.st_dev,
|
||||
admitted.st_ino,
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source frame changed during admission")
|
||||
with (
|
||||
os.fdopen(descriptor, "rb", closefd=False) as source_stream,
|
||||
destination.open("xb") as destination_stream,
|
||||
):
|
||||
while block := source_stream.read(1024 * 1024):
|
||||
if not signature:
|
||||
signature = block[:8]
|
||||
byte_length += len(block)
|
||||
if byte_length > row.byte_length:
|
||||
raise ComponentAdapterError("EoMT source frame grew during admission")
|
||||
digest.update(block)
|
||||
destination_stream.write(block)
|
||||
destination_stream.flush()
|
||||
os.fsync(destination_stream.fileno())
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
if (
|
||||
signature != b"\x89PNG\r\n\x1a\n"
|
||||
or byte_length != row.byte_length
|
||||
or digest.hexdigest() != row.sha256
|
||||
):
|
||||
raise ComponentAdapterError("EoMT source frame identity changed")
|
||||
destination.chmod(0o400)
|
||||
return snapshot_root.resolve(strict=True)
|
||||
|
||||
|
||||
def _validate_decode_repair(document: Mapping[str, object], frame_count: int) -> None:
|
||||
if set(document) != {
|
||||
"schema_version",
|
||||
"decoder",
|
||||
"packets_requested",
|
||||
"frames_decoded",
|
||||
"repaired_frame_count",
|
||||
"repairs",
|
||||
}:
|
||||
raise ComponentAdapterError("decode repair fields changed")
|
||||
decoded = document.get("frames_decoded")
|
||||
repaired = document.get("repaired_frame_count")
|
||||
repairs = document.get("repairs")
|
||||
if (
|
||||
document.get("schema_version") != DECODE_REPAIR_SCHEMA
|
||||
or document.get("decoder") != "ffmpeg-h264_cuvid-output-corrupt"
|
||||
or document.get("packets_requested") != frame_count
|
||||
or isinstance(decoded, bool)
|
||||
or not isinstance(decoded, int)
|
||||
or isinstance(repaired, bool)
|
||||
or not isinstance(repaired, int)
|
||||
or repaired not in (0, 1)
|
||||
or decoded + repaired != frame_count
|
||||
or not isinstance(repairs, list)
|
||||
or len(repairs) != repaired
|
||||
):
|
||||
raise ComponentAdapterError("decode repair contract changed")
|
||||
for value in repairs:
|
||||
if (
|
||||
not isinstance(value, dict)
|
||||
or set(value) != {"sequence", "packet_pts", "method"}
|
||||
or isinstance(value.get("sequence"), bool)
|
||||
or not isinstance(value.get("sequence"), int)
|
||||
or not 2 <= cast(int, value["sequence"]) <= frame_count
|
||||
or isinstance(value.get("packet_pts"), bool)
|
||||
or not isinstance(value.get("packet_pts"), int)
|
||||
or value.get("method") != "duplicate-previous-decoded-frame"
|
||||
):
|
||||
raise ComponentAdapterError("decode repair entry changed")
|
||||
|
||||
|
||||
def _validate_ddrnet_result(
|
||||
root: Path,
|
||||
request: ComponentRequest,
|
||||
effective: Mapping[str, object],
|
||||
*,
|
||||
mapping_path: Path,
|
||||
) -> None:
|
||||
result = validate_fixed_result(
|
||||
root,
|
||||
schema_version=DDRNET_RESULT_SCHEMA,
|
||||
label="DDRNet",
|
||||
)
|
||||
candidate = result.get("candidate")
|
||||
source = result.get("source")
|
||||
semantics = result.get("video_semantics")
|
||||
provenance = result.get("provenance")
|
||||
authority = result.get("authority")
|
||||
ravnoves = effective.get("ravnoves")
|
||||
if not all(
|
||||
isinstance(value, dict) for value in (candidate, source, semantics, provenance, ravnoves)
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet result contract is incomplete")
|
||||
candidate = cast(dict[str, object], candidate)
|
||||
source = cast(dict[str, object], source)
|
||||
semantics = cast(dict[str, object], semantics)
|
||||
provenance = cast(dict[str, object], provenance)
|
||||
ravnoves = cast(dict[str, object], ravnoves)
|
||||
archive = semantics.get("mask_archive")
|
||||
if not isinstance(archive, dict):
|
||||
raise ComponentAdapterError("DDRNet mask archive contract is unavailable")
|
||||
if (
|
||||
result.get("lab_id") != "LAB-V1"
|
||||
or result.get("worker_id") != "worker-006"
|
||||
or result.get("mode") != "ravnoves-video"
|
||||
or candidate.get("candidate_id") != DDRNET_CANDIDATE_ID
|
||||
or candidate.get("candidate_key") != "ddrnet"
|
||||
or candidate.get("loaded_model_name") != DDRNET_LOADED_MODEL
|
||||
or candidate.get("checkpoint_size_bytes") != DDRNET_CHECKPOINT_BYTES
|
||||
or candidate.get("checkpoint_sha256") != DDRNET_CHECKPOINT_SHA256
|
||||
or source.get("source_id") != ravnoves.get("source_id")
|
||||
or source.get("input_count") != request.source.frame_count
|
||||
or source.get("ground_truth_available") is not False
|
||||
or source.get("mapping_sha256") != GOOSE_MAPPING_SHA256
|
||||
or archive.get("path") != "semantic-masks.zip"
|
||||
or archive.get("media_type") != "application/zip"
|
||||
or archive.get("frame_count") != request.source.frame_count
|
||||
or archive.get("width") != 800
|
||||
or archive.get("height") != 600
|
||||
or archive.get("encoding") != "uint8-class-id-png"
|
||||
or archive.get("sequence_binding") != "sequence-0-to-masks/frame-000001.png"
|
||||
or semantics.get("center_crop_xyxy") != [100, 0, 700, 600]
|
||||
or semantics.get("outside_crop_state") != "undefined"
|
||||
or semantics.get("base_m4_result_id") is not None
|
||||
or provenance.get("config_sha256") != request.effective_ddrnet_config_sha256
|
||||
or provenance.get("policy_sha256") != POLICY_SHA256
|
||||
or provenance.get("provider_map_sha256") != PROVIDER_MAP_SHA256
|
||||
or authority != DDRNET_AUTHORITY
|
||||
or result.get("visual_cases") != []
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet result identity changed")
|
||||
archive_path = _confined_regular_file(
|
||||
root / "semantic-masks.zip",
|
||||
root,
|
||||
"DDRNet semantic mask archive",
|
||||
)
|
||||
if archive.get("byte_length") != archive_path.stat().st_size or archive.get(
|
||||
"sha256"
|
||||
) != sha256_file(archive_path):
|
||||
raise ComponentAdapterError("DDRNet semantic mask archive changed")
|
||||
observed_histogram = _validate_mask_archive_inventory(
|
||||
archive_path,
|
||||
request.source.frame_count,
|
||||
)
|
||||
aggregate = semantics.get("aggregate_prediction_pixels")
|
||||
if (
|
||||
not isinstance(aggregate, list)
|
||||
or len(aggregate) != 64
|
||||
or any(isinstance(value, bool) or not isinstance(value, int) for value in aggregate)
|
||||
or tuple(aggregate) != observed_histogram
|
||||
or semantics.get("taxonomy") != _sealed_taxonomy(mapping_path)
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet semantic statistics changed")
|
||||
identity_value = {
|
||||
"schema_version": result["schema_version"],
|
||||
"candidate": result["candidate"],
|
||||
"source": result["source"],
|
||||
"video_semantics": result["video_semantics"],
|
||||
"preprocessing": result["preprocessing"],
|
||||
"metrics": result["metrics"],
|
||||
"timing": result["timing"],
|
||||
"resource": result["resource"],
|
||||
"visual_cases": result["visual_cases"],
|
||||
"authority": result["authority"],
|
||||
"config_sha256": provenance["config_sha256"],
|
||||
"policy_sha256": provenance["policy_sha256"],
|
||||
"provider_map_sha256": provenance["provider_map_sha256"],
|
||||
}
|
||||
identity_sha256 = hashlib.sha256(
|
||||
json.dumps(identity_value, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
if result.get("result_id") != f"lab-v1-ravnoves-video-ddrnet-{identity_sha256}":
|
||||
raise ComponentAdapterError("DDRNet result id changed")
|
||||
repair = load_json(
|
||||
root / "decode-repair.json",
|
||||
label="DDRNet decode repair",
|
||||
maximum=1024 * 1024,
|
||||
confinement_root=root,
|
||||
)
|
||||
_validate_decode_repair(repair, request.source.frame_count)
|
||||
|
||||
|
||||
def _validate_mask_archive_inventory(path: Path, frame_count: int) -> tuple[int, ...]:
|
||||
expected = tuple(f"masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1))
|
||||
histogram = [0] * 64
|
||||
try:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
infos = archive.infolist()
|
||||
names = tuple(info.filename for info in infos)
|
||||
if (
|
||||
names != expected
|
||||
or len(names) != len(set(names))
|
||||
or any(
|
||||
info.is_dir()
|
||||
or info.compress_type != zipfile.ZIP_STORED
|
||||
or info.flag_bits & 0x1
|
||||
or stat.S_ISLNK(info.external_attr >> 16)
|
||||
or not 0 < info.file_size <= 2 * 1024 * 1024
|
||||
or info.compress_size != info.file_size
|
||||
for info in infos
|
||||
)
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet semantic mask archive inventory changed")
|
||||
for info in infos:
|
||||
with archive.open(info, mode="r") as stream:
|
||||
payload = stream.read(info.file_size + 1)
|
||||
if len(payload) != info.file_size:
|
||||
raise ComponentAdapterError("DDRNet semantic mask archive member changed")
|
||||
validate_grayscale_png_payload(
|
||||
payload,
|
||||
width=800,
|
||||
height=600,
|
||||
label=f"DDRNet mask {info.filename}",
|
||||
)
|
||||
with Image.open(io.BytesIO(payload)) as image:
|
||||
if image.format != "PNG" or image.mode != "L" or image.size != (800, 600):
|
||||
raise ComponentAdapterError("DDRNet semantic mask raster changed")
|
||||
image.load()
|
||||
counts = image.histogram()
|
||||
if len(counts) != 256 or any(counts[64:]):
|
||||
raise ComponentAdapterError("DDRNet semantic mask class id escaped 0..63")
|
||||
for class_id, count in enumerate(counts[:64]):
|
||||
histogram[class_id] += count
|
||||
except (OSError, zipfile.BadZipFile) as exc:
|
||||
raise ComponentAdapterError("DDRNet semantic mask archive is invalid") from exc
|
||||
if sum(histogram) != frame_count * 800 * 600:
|
||||
raise ComponentAdapterError("DDRNet semantic mask pixel count changed")
|
||||
return tuple(histogram)
|
||||
|
||||
|
||||
def _sealed_taxonomy(mapping_path: Path) -> dict[str, object]:
|
||||
classes: dict[int, dict[str, object]] = {}
|
||||
try:
|
||||
with mapping_path.open(newline="", encoding="utf-8-sig") as stream:
|
||||
reader = csv.DictReader(stream)
|
||||
if reader.fieldnames is None or not {"label_key", "class_name", "hex"}.issubset(
|
||||
reader.fieldnames
|
||||
):
|
||||
raise ComponentAdapterError("sealed GOOSE mapping columns changed")
|
||||
for row in reader:
|
||||
try:
|
||||
class_id = int(row["label_key"])
|
||||
color = bytes.fromhex(row["hex"].removeprefix("#"))
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise ComponentAdapterError("sealed GOOSE mapping row changed") from exc
|
||||
label = row.get("class_name")
|
||||
if (
|
||||
not 0 <= class_id < 64
|
||||
or class_id in classes
|
||||
or not isinstance(label, str)
|
||||
or not label
|
||||
or len(color) != 3
|
||||
):
|
||||
raise ComponentAdapterError("sealed GOOSE taxonomy changed")
|
||||
classes[class_id] = {
|
||||
"class_id": class_id,
|
||||
"label": label,
|
||||
"color_rgb": list(color),
|
||||
"disposition": "undefined" if class_id == 0 else "prediction",
|
||||
}
|
||||
except OSError as exc:
|
||||
raise ComponentAdapterError("sealed GOOSE mapping is unavailable") from exc
|
||||
if set(classes) != set(range(64)):
|
||||
raise ComponentAdapterError("sealed GOOSE mapping does not cover 64 classes")
|
||||
return {
|
||||
"schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1",
|
||||
"classes": [classes[class_id] for class_id in range(64)],
|
||||
}
|
||||
|
||||
|
||||
def _ddrnet_environment() -> dict[str, str]:
|
||||
retained = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"}
|
||||
}
|
||||
return {
|
||||
**retained,
|
||||
"HOME": "/tmp",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
}
|
||||
|
||||
|
||||
def _empty_output_root(path: Path) -> Path:
|
||||
if path.is_symlink():
|
||||
raise ComponentAdapterError("DDRNet output root is a symbolic link")
|
||||
resolved = path.resolve(strict=True)
|
||||
if not resolved.is_dir() or any(resolved.iterdir()):
|
||||
raise ComponentAdapterError("DDRNet output root is not an empty real directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _prepare_workspace(path: Path) -> Path:
|
||||
if path.exists() or path.is_symlink():
|
||||
raise ComponentAdapterError("DDRNet workspace already exists")
|
||||
path.mkdir(mode=0o700, parents=False)
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _confined_regular_file(path: Path, root: Path, label: str) -> Path:
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
resolved_root = root.resolve(strict=True)
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ComponentAdapterError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not resolved.is_relative_to(resolved_root)
|
||||
):
|
||||
raise ComponentAdapterError(f"{label} is not a confined regular file")
|
||||
return resolved
|
||||
|
||||
|
||||
def _publish_component_children(
|
||||
staging: Path,
|
||||
output: Path,
|
||||
*,
|
||||
expected_names: set[str],
|
||||
) -> None:
|
||||
children = tuple(staging.iterdir())
|
||||
if {child.name for child in children} != expected_names or any(
|
||||
child.is_dir() or child.is_symlink() for child in children
|
||||
):
|
||||
raise ComponentAdapterError("DDRNet staging contains unexpected directories")
|
||||
for child in children:
|
||||
destination = output / child.name
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ComponentAdapterError("DDRNet publication target already exists")
|
||||
os.replace(child, destination)
|
||||
staging.rmdir()
|
||||
|
||||
|
||||
def _representative_frame_indices(frame_count: int) -> list[int]:
|
||||
if frame_count <= 12:
|
||||
return list(range(frame_count))
|
||||
return sorted({round(index * (frame_count - 1) / 11) for index in range(12)})
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv)
|
||||
execute_ddrnet_component(
|
||||
request_path=request_path,
|
||||
layout=RuntimeLayout.fixed(DDRNET_ASSETS),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except ComponentAdapterError as exc:
|
||||
print(f"portable LAB V1 DDRNet rejected: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2) from exc
|
||||
+987
@@ -0,0 +1,987 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the sealed LAB V1 EoMT component over one CameraComputeJob."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import stat
|
||||
import sys
|
||||
import tarfile
|
||||
import time
|
||||
from collections.abc import Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
from portable_lab_v1_component_adapter import (
|
||||
EOMT_DEPENDENCY_IDENTITY_SHA256,
|
||||
EOMT_PIPELINE,
|
||||
VALID_FOV_IDENTITY_SHA256,
|
||||
VALID_FOV_MASK_SHA256,
|
||||
AssetExpectation,
|
||||
CommandRunner,
|
||||
ComponentAdapterError,
|
||||
ComponentRequest,
|
||||
RuntimeLayout,
|
||||
available_bytes,
|
||||
canonical_json,
|
||||
load_component_request,
|
||||
load_json,
|
||||
parse_fixed_request_argument,
|
||||
run_command,
|
||||
sha256_file,
|
||||
validate_camera_compute_job,
|
||||
validate_critical_tree_file,
|
||||
validate_fixed_result,
|
||||
validate_grayscale_png_payload,
|
||||
validate_identity_manifest,
|
||||
validate_tree_asset,
|
||||
validate_tree_receipt,
|
||||
)
|
||||
|
||||
EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2"
|
||||
DEPENDENCY_SCHEMA: Final = "missioncore.k1-e3-dependency-manifest/v1"
|
||||
VALID_FOV_SCHEMA: Final = "missioncore.k1-valid-fov-mask/v1"
|
||||
DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1"
|
||||
SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1"
|
||||
MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f"
|
||||
MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024"
|
||||
MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation"
|
||||
DISK_FLOOR_BYTES: Final = 360 * 1024**3
|
||||
FFMPEG_TREE_SOURCE_IMAGE_SHA256: Final = (
|
||||
"8a364092b03561b9c08ac00730206e363a53d07ea0304f7d543b403b65432b5e"
|
||||
)
|
||||
FFMPEG_TREE_IDENTITY_SHA256: Final = (
|
||||
"03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69"
|
||||
)
|
||||
FFMPEG_TREE_BYTE_LENGTH: Final = 256_208_352
|
||||
FFMPEG_SHA256: Final = "7c6e870c57d16a755d7255cb93b65379218f47d3208a3f9b107f3a80646af6af"
|
||||
FFPROBE_SHA256: Final = "39e7b15b94ef2d37e9fcf052ad1cb1e6550fdb6c6ac6765c5f1aba53cc367929"
|
||||
|
||||
EOMT_ASSETS: Final = (
|
||||
AssetExpectation(
|
||||
"eomt-environment",
|
||||
"/environment",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
AssetExpectation(
|
||||
"eomt-ffmpeg-runtime",
|
||||
"/opt/ffmpeg",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
FFMPEG_TREE_IDENTITY_SHA256,
|
||||
FFMPEG_TREE_BYTE_LENGTH,
|
||||
),
|
||||
AssetExpectation(
|
||||
"eomt-model-cache",
|
||||
"/cache",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
AssetExpectation(
|
||||
"eomt-python-environment",
|
||||
"/opt/env",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
AssetExpectation(
|
||||
"eomt-runner-bundle",
|
||||
"/runner",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
AssetExpectation(
|
||||
"eomt-transformers-environment",
|
||||
"/opt/transformers",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
AssetExpectation(
|
||||
"k1-valid-fov-root",
|
||||
"/valid-fov",
|
||||
"tree",
|
||||
"identity-sha256",
|
||||
),
|
||||
)
|
||||
|
||||
_RUNNER_FILES: Final = {
|
||||
"run_e4_full_session_segmentation.py": (
|
||||
30_720,
|
||||
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4",
|
||||
),
|
||||
"run_e3_rectified_segmentation.py": (
|
||||
45_789,
|
||||
"01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6",
|
||||
),
|
||||
"run_recorded_perception_epoch.py": (
|
||||
34_899,
|
||||
"4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675",
|
||||
),
|
||||
"run_evaluation_prelabels.py": (
|
||||
28_915,
|
||||
"25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4",
|
||||
),
|
||||
"e3_k1_camera1_profile.json": (
|
||||
3_805,
|
||||
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875",
|
||||
),
|
||||
}
|
||||
_MODEL_FILES: Final = {
|
||||
"config.json": (
|
||||
1_575,
|
||||
"7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650",
|
||||
),
|
||||
"preprocessor_config.json": (
|
||||
666,
|
||||
"97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7",
|
||||
),
|
||||
"model.safetensors": (
|
||||
1_276_175_488,
|
||||
"c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def execute_eomt_component(
|
||||
*,
|
||||
request_path: Path,
|
||||
layout: RuntimeLayout,
|
||||
command_runner: CommandRunner = run_command,
|
||||
work_root: Path | None = None,
|
||||
disk_floor_bytes: int = DISK_FLOOR_BYTES,
|
||||
) -> None:
|
||||
request = load_component_request(
|
||||
request_path,
|
||||
component="eomt",
|
||||
expectations=EOMT_ASSETS,
|
||||
)
|
||||
input_document = validate_camera_compute_job(layout.camera_job_root, request.source)
|
||||
roots = _validate_release_assets(request, layout)
|
||||
output = _empty_output_root(layout.output_root)
|
||||
reserve = request.source.frame_count * 800 * 600 * 7 + request.source.input_byte_length
|
||||
free_before = available_bytes(output)
|
||||
if disk_floor_bytes < 0 or free_before < disk_floor_bytes + reserve:
|
||||
raise ComponentAdapterError("EoMT output does not satisfy its disk reserve")
|
||||
workspace = _prepare_workspace(output / ".eomt-work" if work_root is None else work_root)
|
||||
try:
|
||||
total_started = time.perf_counter()
|
||||
extract_started = time.perf_counter()
|
||||
frames_root, timeline_path, decode_repair = _decode_camera_epoch(
|
||||
request=request,
|
||||
input_document=input_document,
|
||||
camera_job_root=layout.camera_job_root,
|
||||
output_root=output,
|
||||
work_root=workspace,
|
||||
ffmpeg_root=roots["eomt-ffmpeg-runtime"],
|
||||
command_runner=command_runner,
|
||||
)
|
||||
extract_seconds = _elapsed(extract_started)
|
||||
free_post_extract = available_bytes(output)
|
||||
if free_post_extract < disk_floor_bytes:
|
||||
raise ComponentAdapterError("EoMT frame extraction crossed its disk floor")
|
||||
|
||||
inference_root = output / ".eomt-component"
|
||||
runner = roots["eomt-runner-bundle"] / "run_e4_full_session_segmentation.py"
|
||||
profile = roots["eomt-runner-bundle"] / "e3_k1_camera1_profile.json"
|
||||
environment = _eomt_environment(roots)
|
||||
run_argv = (
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"run",
|
||||
"--job",
|
||||
str(layout.camera_job_root),
|
||||
"--profile",
|
||||
str(profile),
|
||||
"--valid-fov-root",
|
||||
str(roots["k1-valid-fov-root"]),
|
||||
"--frames",
|
||||
str(frames_root),
|
||||
"--timeline",
|
||||
str(timeline_path),
|
||||
"--cache",
|
||||
str(roots["eomt-model-cache"]),
|
||||
"--environment",
|
||||
str(roots["eomt-environment"]),
|
||||
"--output",
|
||||
str(inference_root),
|
||||
"--frame-limit",
|
||||
str(request.source.frame_count),
|
||||
"--free-bytes-floor",
|
||||
str(disk_floor_bytes),
|
||||
"--orchestrator-sha256",
|
||||
sha256_file(Path(__file__).resolve(strict=True)),
|
||||
"--container-image",
|
||||
f"sha256:{request.component_image_sha256}",
|
||||
"--telemetry-interval-seconds",
|
||||
"1",
|
||||
)
|
||||
command_runner(run_argv, environment)
|
||||
_write_json(inference_root / "decode-repair.json", decode_repair)
|
||||
free_post_inference = available_bytes(output)
|
||||
if free_post_inference < disk_floor_bytes:
|
||||
raise ComponentAdapterError("EoMT inference crossed its disk floor")
|
||||
|
||||
video_path = inference_root / "perception.mp4"
|
||||
encode_started = time.perf_counter()
|
||||
duration = request.source.timeline_end_seconds - request.source.timeline_start_seconds
|
||||
fps = request.source.frame_count / duration
|
||||
encode_argv = (
|
||||
str(roots["eomt-ffmpeg-runtime"] / "bin" / "ffmpeg"),
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-framerate",
|
||||
_number(fps),
|
||||
"-i",
|
||||
str(inference_root / "overlay-frames" / "frame-%06d.png"),
|
||||
"-t",
|
||||
_number(duration),
|
||||
"-c:v",
|
||||
"h264_nvenc",
|
||||
"-preset",
|
||||
"p4",
|
||||
"-tune",
|
||||
"hq",
|
||||
"-rc",
|
||||
"vbr",
|
||||
"-cq",
|
||||
"21",
|
||||
"-b:v",
|
||||
"0",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(video_path),
|
||||
)
|
||||
command_runner(encode_argv, None)
|
||||
encode_seconds = _elapsed(encode_started)
|
||||
_required_nonempty_file(video_path, inference_root, "EoMT overlay video")
|
||||
|
||||
masks_path = inference_root / "masks.tar.gz"
|
||||
archive_started = time.perf_counter()
|
||||
archive_argv = (
|
||||
"/usr/bin/tar",
|
||||
"-czf",
|
||||
str(masks_path),
|
||||
"-C",
|
||||
str(inference_root),
|
||||
"semantic-masks",
|
||||
)
|
||||
command_runner(archive_argv, None)
|
||||
archive_seconds = _elapsed(archive_started)
|
||||
_required_nonempty_file(masks_path, inference_root, "EoMT mask archive")
|
||||
free_post_artifacts = available_bytes(output)
|
||||
if free_post_artifacts < disk_floor_bytes:
|
||||
raise ComponentAdapterError("EoMT publication crossed its disk floor")
|
||||
|
||||
wall_seconds = _elapsed(total_started)
|
||||
finalize_argv = (
|
||||
sys.executable,
|
||||
str(runner),
|
||||
"finalize",
|
||||
"--output",
|
||||
str(inference_root),
|
||||
"--video",
|
||||
str(video_path),
|
||||
"--masks",
|
||||
str(masks_path),
|
||||
"--extract-seconds",
|
||||
_number(extract_seconds),
|
||||
"--encode-seconds",
|
||||
_number(encode_seconds),
|
||||
"--archive-seconds",
|
||||
_number(archive_seconds),
|
||||
"--wall-seconds",
|
||||
_number(wall_seconds),
|
||||
"--encoder",
|
||||
"ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart",
|
||||
"--disk-free-before-bytes",
|
||||
str(free_before),
|
||||
"--disk-free-post-extract-bytes",
|
||||
str(free_post_extract),
|
||||
"--disk-free-post-inference-bytes",
|
||||
str(free_post_inference),
|
||||
"--disk-free-post-artifacts-bytes",
|
||||
str(free_post_artifacts),
|
||||
"--disk-floor-bytes",
|
||||
str(disk_floor_bytes),
|
||||
"--working-set-reserve-bytes",
|
||||
str(reserve),
|
||||
)
|
||||
command_runner(finalize_argv, environment)
|
||||
_bind_source_frame_manifest(inference_root, frames_root, request)
|
||||
_validate_eomt_result(inference_root, frames_root, request)
|
||||
for temporary in ("overlay-frames", "semantic-masks"):
|
||||
path = inference_root / temporary
|
||||
if path.exists():
|
||||
shutil.rmtree(path)
|
||||
_publish_component_children(
|
||||
inference_root,
|
||||
output,
|
||||
expected_names={
|
||||
"decode-repair.json",
|
||||
"frames.jsonl",
|
||||
"gpu-telemetry.jsonl",
|
||||
"masks.tar.gz",
|
||||
"perception.mp4",
|
||||
"result.json",
|
||||
"run-report.json",
|
||||
"source-frames.json",
|
||||
},
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(workspace, ignore_errors=True)
|
||||
|
||||
|
||||
def _validate_release_assets(
|
||||
request: ComponentRequest,
|
||||
layout: RuntimeLayout,
|
||||
) -> dict[str, Path]:
|
||||
roots = {
|
||||
item.asset_id: validate_tree_asset(request, layout, item.asset_id) for item in EOMT_ASSETS
|
||||
}
|
||||
for asset_id in (
|
||||
"eomt-environment",
|
||||
"eomt-model-cache",
|
||||
"eomt-python-environment",
|
||||
"eomt-transformers-environment",
|
||||
):
|
||||
validate_tree_receipt(
|
||||
request,
|
||||
roots[asset_id],
|
||||
asset_id,
|
||||
verify_payload=True,
|
||||
)
|
||||
for asset_id in ("eomt-runner-bundle", "k1-valid-fov-root"):
|
||||
validate_tree_receipt(
|
||||
request,
|
||||
roots[asset_id],
|
||||
asset_id,
|
||||
verify_payload=True,
|
||||
)
|
||||
ffmpeg_receipt = validate_tree_receipt(
|
||||
request,
|
||||
roots["eomt-ffmpeg-runtime"],
|
||||
"eomt-ffmpeg-runtime",
|
||||
expected_metadata={
|
||||
"source_image_sha256": FFMPEG_TREE_SOURCE_IMAGE_SHA256,
|
||||
"source_path": "/usr/lib/ffmpeg/7.0",
|
||||
},
|
||||
additional_metadata_keys=frozenset({"binaries"}),
|
||||
verify_payload=True,
|
||||
)
|
||||
environment_manifest = validate_identity_manifest(
|
||||
roots["eomt-environment"] / "manifest.json",
|
||||
expected_identity_sha256=EOMT_DEPENDENCY_IDENTITY_SHA256,
|
||||
expected_schema=DEPENDENCY_SCHEMA,
|
||||
label="EoMT dependency manifest",
|
||||
)
|
||||
identity = cast(dict[str, object], environment_manifest["identity"])
|
||||
if identity.get("profile_sha256") != _RUNNER_FILES["e3_k1_camera1_profile.json"][1]:
|
||||
raise ComponentAdapterError("EoMT dependency profile changed")
|
||||
valid_manifest = validate_identity_manifest(
|
||||
roots["k1-valid-fov-root"] / "manifest.json",
|
||||
expected_identity_sha256=VALID_FOV_IDENTITY_SHA256,
|
||||
expected_schema=VALID_FOV_SCHEMA,
|
||||
label="valid-FOV manifest",
|
||||
)
|
||||
artifact = valid_manifest.get("artifact")
|
||||
if not isinstance(artifact, dict) or artifact.get("sha256") != VALID_FOV_MASK_SHA256:
|
||||
raise ComponentAdapterError("valid-FOV mask binding changed")
|
||||
mask_byte_length = artifact.get("byte_length")
|
||||
if (
|
||||
isinstance(mask_byte_length, bool)
|
||||
or not isinstance(mask_byte_length, int)
|
||||
or mask_byte_length < 1
|
||||
):
|
||||
raise ComponentAdapterError("valid-FOV mask byte length changed")
|
||||
validate_critical_tree_file(
|
||||
roots["k1-valid-fov-root"],
|
||||
"mask.png",
|
||||
byte_length=mask_byte_length,
|
||||
identity_sha256=VALID_FOV_MASK_SHA256,
|
||||
label="valid-FOV mask",
|
||||
)
|
||||
for name, (byte_length, digest) in _RUNNER_FILES.items():
|
||||
validate_critical_tree_file(
|
||||
roots["eomt-runner-bundle"],
|
||||
name,
|
||||
byte_length=byte_length,
|
||||
identity_sha256=digest,
|
||||
label=f"EoMT runner file {name}",
|
||||
)
|
||||
snapshot = (
|
||||
roots["eomt-model-cache"]
|
||||
/ "huggingface"
|
||||
/ "models--tue-mps--cityscapes_semantic_eomt_large_1024"
|
||||
/ "snapshots"
|
||||
/ MODEL_REVISION
|
||||
)
|
||||
for name, (byte_length, digest) in _MODEL_FILES.items():
|
||||
validate_critical_tree_file(
|
||||
roots["eomt-model-cache"],
|
||||
(snapshot / name).relative_to(roots["eomt-model-cache"]).as_posix(),
|
||||
byte_length=byte_length,
|
||||
identity_sha256=digest,
|
||||
label=f"EoMT model file {name}",
|
||||
)
|
||||
binaries = ffmpeg_receipt.get("binaries")
|
||||
if not isinstance(binaries, dict) or set(binaries) != {"ffmpeg", "ffprobe"}:
|
||||
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||
for name, digest in (("ffmpeg", FFMPEG_SHA256), ("ffprobe", FFPROBE_SHA256)):
|
||||
row = binaries.get(name)
|
||||
if not isinstance(row, dict) or set(row) != {
|
||||
"relative_path",
|
||||
"byte_length",
|
||||
"sha256",
|
||||
}:
|
||||
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||
path = validate_critical_tree_file(
|
||||
roots["eomt-ffmpeg-runtime"],
|
||||
f"bin/{name}",
|
||||
byte_length=cast(int, row["byte_length"]),
|
||||
identity_sha256=digest,
|
||||
label=f"sealed {name}",
|
||||
)
|
||||
if row != {
|
||||
"relative_path": f"bin/{name}",
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": digest,
|
||||
} or not os.access(path, os.X_OK):
|
||||
raise ComponentAdapterError("sealed ffmpeg binary receipt changed")
|
||||
return roots
|
||||
|
||||
|
||||
def _decode_camera_epoch(
|
||||
*,
|
||||
request: ComponentRequest,
|
||||
input_document: Mapping[str, object],
|
||||
camera_job_root: Path,
|
||||
output_root: Path,
|
||||
work_root: Path,
|
||||
ffmpeg_root: Path,
|
||||
command_runner: CommandRunner,
|
||||
) -> tuple[Path, Path, dict[str, object]]:
|
||||
source = request.source
|
||||
epoch = (
|
||||
camera_job_root
|
||||
/ "input"
|
||||
/ "camera"
|
||||
/ source.camera_source_id
|
||||
/ f"epoch-{source.codec_epoch}"
|
||||
)
|
||||
stream_path = work_root / "camera.mp4"
|
||||
with stream_path.open("xb") as stream:
|
||||
for path in (
|
||||
epoch / "init.mp4",
|
||||
*(epoch / "segments" / f"{index}.m4s" for index in range(1, source.frame_count + 1)),
|
||||
):
|
||||
with path.open("rb") as segment:
|
||||
shutil.copyfileobj(segment, stream, length=1024 * 1024)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
packets_path = work_root / "packets.csv"
|
||||
ffprobe_argv = (
|
||||
str(ffmpeg_root / "bin" / "ffprobe"),
|
||||
"-v",
|
||||
"error",
|
||||
"-select_streams",
|
||||
"v:0",
|
||||
"-show_packets",
|
||||
"-show_entries",
|
||||
"packet=pts,flags",
|
||||
"-of",
|
||||
"csv=p=0",
|
||||
"-o",
|
||||
str(packets_path),
|
||||
str(stream_path),
|
||||
)
|
||||
command_runner(ffprobe_argv, None)
|
||||
packet_rows = packets_path.read_text(encoding="utf-8").splitlines()
|
||||
if len(packet_rows) != source.frame_count:
|
||||
raise ComponentAdapterError("camera packet count changed")
|
||||
packet_pts: list[int] = []
|
||||
for row in packet_rows:
|
||||
fields = row.split(",")
|
||||
try:
|
||||
packet_pts.append(int(fields[0].strip()))
|
||||
except (IndexError, ValueError) as exc:
|
||||
raise ComponentAdapterError("camera packet timestamp is invalid") from exc
|
||||
decoded_root = work_root / "decoded-by-pts"
|
||||
decoded_root.mkdir(mode=0o700)
|
||||
decode_argv = (
|
||||
str(ffmpeg_root / "bin" / "ffmpeg"),
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-hwaccel",
|
||||
"cuda",
|
||||
"-hwaccel_output_format",
|
||||
"cuda",
|
||||
"-c:v",
|
||||
"h264_cuvid",
|
||||
"-err_detect",
|
||||
"ignore_err",
|
||||
"-flags",
|
||||
"+output_corrupt",
|
||||
"-copyts",
|
||||
"-i",
|
||||
str(stream_path),
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-vf",
|
||||
"hwdownload,format=nv12",
|
||||
"-fps_mode",
|
||||
"passthrough",
|
||||
"-enc_time_base",
|
||||
"demux",
|
||||
"-frames:v",
|
||||
str(source.frame_count),
|
||||
"-frame_pts",
|
||||
"1",
|
||||
str(decoded_root / "frame-%d.png"),
|
||||
)
|
||||
command_runner(decode_argv, None)
|
||||
frames_root = output_root / "source-frames"
|
||||
frames_root.mkdir(mode=0o700)
|
||||
repairs: list[dict[str, object]] = []
|
||||
decoded_count = len(tuple(decoded_root.glob("frame-*.png")))
|
||||
for index, pts in enumerate(packet_pts):
|
||||
decoded = decoded_root / f"frame-{pts}.png"
|
||||
canonical = frames_root / f"frame-{index + 1:06d}.png"
|
||||
if decoded.is_file() and not decoded.is_symlink():
|
||||
os.replace(decoded, canonical)
|
||||
continue
|
||||
if index == 0 or repairs:
|
||||
raise ComponentAdapterError("camera decode has more than one recoverable gap")
|
||||
previous = frames_root / f"frame-{index:06d}.png"
|
||||
shutil.copyfile(previous, canonical)
|
||||
repairs.append(
|
||||
{
|
||||
"sequence": index + 1,
|
||||
"packet_pts": pts,
|
||||
"method": "duplicate-previous-decoded-frame",
|
||||
}
|
||||
)
|
||||
if any(decoded_root.iterdir()):
|
||||
raise ComponentAdapterError("camera decoder emitted unexpected frames")
|
||||
timeline = cast(Mapping[str, object], input_document["timeline"])
|
||||
timeline_start = timeline["start_seconds"]
|
||||
if isinstance(timeline_start, bool) or not isinstance(timeline_start, (int, float)):
|
||||
raise ComponentAdapterError("camera timeline start changed")
|
||||
duration = source.timeline_end_seconds - source.timeline_start_seconds
|
||||
first_pts = packet_pts[0]
|
||||
previous_seconds = -1.0
|
||||
timeline_path = work_root / "timeline.jsonl"
|
||||
with timeline_path.open("x", encoding="utf-8", newline="\n") as stream:
|
||||
for index, pts in enumerate(packet_pts):
|
||||
epoch_seconds = (pts - first_pts) / 90_000.0
|
||||
if epoch_seconds <= previous_seconds or epoch_seconds > duration + 0.001:
|
||||
raise ComponentAdapterError("camera packet timeline changed")
|
||||
timeline_row = {
|
||||
"frame_index": index,
|
||||
"epoch_seconds": epoch_seconds,
|
||||
"session_seconds": float(timeline_start) + epoch_seconds,
|
||||
}
|
||||
stream.write(canonical_json(timeline_row).decode("utf-8") + "\n")
|
||||
previous_seconds = epoch_seconds
|
||||
return (
|
||||
frames_root,
|
||||
timeline_path,
|
||||
{
|
||||
"schema_version": DECODE_REPAIR_SCHEMA,
|
||||
"decoder": "ffmpeg-h264_cuvid-output-corrupt",
|
||||
"packets_requested": source.frame_count,
|
||||
"frames_decoded": decoded_count,
|
||||
"repaired_frame_count": len(repairs),
|
||||
"repairs": repairs,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _validate_eomt_result(
|
||||
root: Path,
|
||||
frames_root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> None:
|
||||
document = validate_fixed_result(root, schema_version=EOMT_RESULT_SCHEMA, label="EoMT")
|
||||
identity = document.get("identity")
|
||||
if not isinstance(identity, dict):
|
||||
raise ComponentAdapterError("EoMT result identity is unavailable")
|
||||
configuration = identity.get("configuration")
|
||||
models = identity.get("models")
|
||||
semantic = models.get("semantic") if isinstance(models, dict) else None
|
||||
if not isinstance(configuration, dict):
|
||||
raise ComponentAdapterError("EoMT result configuration is unavailable")
|
||||
observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
source = request.source
|
||||
if (
|
||||
document.get("identity_sha256") != observed_identity
|
||||
or document.get("result_id") != f"result-{observed_identity}"
|
||||
or document.get("job_id") != source.camera_job_id
|
||||
or document.get("input_sha256") != source.camera_input_sha256
|
||||
or document.get("session_id") != source.source_session_id
|
||||
or document.get("source_id") != source.camera_source_id
|
||||
or document.get("codec_epoch") != source.codec_epoch
|
||||
or document.get("timestamp_basis") != "session-time-seconds"
|
||||
or document.get("timeline_start_seconds") != source.timeline_start_seconds
|
||||
or document.get("timeline_end_seconds") != source.timeline_end_seconds
|
||||
or document.get("frames_processed") != source.frame_count
|
||||
or document.get("ground_truth") is not False
|
||||
or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2"
|
||||
or configuration.get("pipeline") != EOMT_PIPELINE
|
||||
or configuration.get("dependency_identity_sha256") != EOMT_DEPENDENCY_IDENTITY_SHA256
|
||||
or configuration.get("container_image") != f"sha256:{request.component_image_sha256}"
|
||||
or not isinstance(semantic, dict)
|
||||
or semantic.get("id") != MODEL_ID
|
||||
or semantic.get("revision") != MODEL_REVISION
|
||||
or semantic.get("architecture") != MODEL_ARCHITECTURE
|
||||
):
|
||||
raise ComponentAdapterError("EoMT result source identity changed")
|
||||
artifacts = document.get("artifacts")
|
||||
if not isinstance(artifacts, list) or len(artifacts) != 6:
|
||||
raise ComponentAdapterError("EoMT result artifact set changed")
|
||||
expected_artifacts = {
|
||||
"panoptic-overlay-video": ("perception.mp4", "video/mp4"),
|
||||
"panoptic-mask-archive": ("masks.tar.gz", "application/gzip"),
|
||||
"panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"),
|
||||
"worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"),
|
||||
"perception-run-report": ("run-report.json", "application/json"),
|
||||
"decoded-source-frame-manifest": ("source-frames.json", "application/json"),
|
||||
}
|
||||
observed_kinds: set[str] = set()
|
||||
for value in artifacts:
|
||||
if not isinstance(value, dict):
|
||||
raise ComponentAdapterError("EoMT result artifact is invalid")
|
||||
kind = value.get("kind")
|
||||
encoded = value.get("path")
|
||||
if (
|
||||
not isinstance(kind, str)
|
||||
or kind in observed_kinds
|
||||
or expected_artifacts.get(kind) != (encoded, value.get("media_type"))
|
||||
or not isinstance(encoded, str)
|
||||
or "/" in encoded
|
||||
or "\\" in encoded
|
||||
):
|
||||
raise ComponentAdapterError("EoMT result artifact path changed")
|
||||
observed_kinds.add(kind)
|
||||
path = _required_nonempty_file(root / encoded, root, "EoMT result artifact")
|
||||
if value.get("byte_length") != path.stat().st_size or value.get("sha256") != sha256_file(
|
||||
path
|
||||
):
|
||||
raise ComponentAdapterError("EoMT result artifact identity changed")
|
||||
if observed_kinds != set(expected_artifacts):
|
||||
raise ComponentAdapterError("EoMT result artifact roles changed")
|
||||
manifest_path = _required_nonempty_file(
|
||||
root / "source-frames.json",
|
||||
root,
|
||||
"EoMT source frame manifest",
|
||||
)
|
||||
expected_binding = {
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"path": "source-frames.json",
|
||||
"sha256": sha256_file(manifest_path),
|
||||
"byte_length": manifest_path.stat().st_size,
|
||||
"frame_count": source.frame_count,
|
||||
}
|
||||
if identity.get("source_frames") != expected_binding:
|
||||
raise ComponentAdapterError("EoMT source frame manifest binding changed")
|
||||
_validate_source_frame_manifest(manifest_path, frames_root, request)
|
||||
_validate_mask_archive_inventory(root / "masks.tar.gz", source.frame_count)
|
||||
|
||||
|
||||
def _bind_source_frame_manifest(
|
||||
result_root: Path,
|
||||
frames_root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> None:
|
||||
document = _source_frame_manifest_document(frames_root, request)
|
||||
manifest_path = result_root / "source-frames.json"
|
||||
_write_canonical_json(manifest_path, document)
|
||||
binding = {
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"path": manifest_path.name,
|
||||
"sha256": sha256_file(manifest_path),
|
||||
"byte_length": manifest_path.stat().st_size,
|
||||
"frame_count": request.source.frame_count,
|
||||
}
|
||||
result = load_json(
|
||||
result_root / "result.json",
|
||||
label="legacy EoMT result",
|
||||
maximum=8 * 1024 * 1024,
|
||||
confinement_root=result_root,
|
||||
)
|
||||
identity = result.get("identity")
|
||||
artifacts = result.get("artifacts")
|
||||
old_result_id = result.get("result_id")
|
||||
if (
|
||||
result.get("schema_version") != EOMT_RESULT_SCHEMA
|
||||
or not isinstance(identity, dict)
|
||||
or "source_frames" in identity
|
||||
or not isinstance(artifacts, list)
|
||||
or len(artifacts) != 5
|
||||
or not isinstance(old_result_id, str)
|
||||
):
|
||||
raise ComponentAdapterError("legacy EoMT result cannot bind source frames")
|
||||
identity["source_frames"] = binding
|
||||
identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest()
|
||||
result_id = f"result-{identity_sha256}"
|
||||
result["identity_sha256"] = identity_sha256
|
||||
result["result_id"] = result_id
|
||||
|
||||
report_path = _required_nonempty_file(
|
||||
result_root / "run-report.json",
|
||||
result_root,
|
||||
"legacy EoMT run report",
|
||||
)
|
||||
report = load_json(
|
||||
report_path,
|
||||
label="legacy EoMT run report",
|
||||
maximum=8 * 1024 * 1024,
|
||||
confinement_root=result_root,
|
||||
)
|
||||
if report.get("result_id") != old_result_id or "source_frames" in report:
|
||||
raise ComponentAdapterError("legacy EoMT run report cannot bind source frames")
|
||||
report["result_id"] = result_id
|
||||
report["source_frames"] = binding
|
||||
_write_json(report_path, report)
|
||||
|
||||
observed_kinds: set[str] = set()
|
||||
for value in artifacts:
|
||||
if not isinstance(value, dict) or not isinstance(value.get("kind"), str):
|
||||
raise ComponentAdapterError("legacy EoMT artifact set changed")
|
||||
kind = cast(str, value["kind"])
|
||||
if kind in observed_kinds:
|
||||
raise ComponentAdapterError("legacy EoMT artifact set changed")
|
||||
observed_kinds.add(kind)
|
||||
if kind == "perception-run-report":
|
||||
value["byte_length"] = report_path.stat().st_size
|
||||
value["sha256"] = sha256_file(report_path)
|
||||
if observed_kinds != {
|
||||
"panoptic-overlay-video",
|
||||
"panoptic-mask-archive",
|
||||
"panoptic-frame-metadata",
|
||||
"worker-gpu-telemetry",
|
||||
"perception-run-report",
|
||||
}:
|
||||
raise ComponentAdapterError("legacy EoMT artifact roles changed")
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "decoded-source-frame-manifest",
|
||||
"path": manifest_path.name,
|
||||
"media_type": "application/json",
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"byte_length": manifest_path.stat().st_size,
|
||||
"sha256": sha256_file(manifest_path),
|
||||
}
|
||||
)
|
||||
_write_json(result_root / "result.json", result)
|
||||
|
||||
|
||||
def _source_frame_manifest_document(
|
||||
frames_root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> dict[str, object]:
|
||||
if frames_root.is_symlink():
|
||||
raise ComponentAdapterError("EoMT source frame root is a symbolic link")
|
||||
resolved = frames_root.resolve(strict=True)
|
||||
expected_names = tuple(
|
||||
f"frame-{sequence:06d}.png" for sequence in range(1, request.source.frame_count + 1)
|
||||
)
|
||||
try:
|
||||
observed_names = tuple(sorted(path.name for path in resolved.iterdir()))
|
||||
except OSError as exc:
|
||||
raise ComponentAdapterError("EoMT source frame set is unavailable") from exc
|
||||
if observed_names != expected_names:
|
||||
raise ComponentAdapterError("EoMT source frame set changed")
|
||||
rows: list[dict[str, object]] = []
|
||||
for sequence, name in enumerate(expected_names, start=1):
|
||||
path = _required_nonempty_file(resolved / name, resolved, "EoMT source frame")
|
||||
if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024:
|
||||
raise ComponentAdapterError("EoMT source frame is unsafe")
|
||||
rows.append(
|
||||
{
|
||||
"sequence": sequence,
|
||||
"path": f"source-frames/{name}",
|
||||
"byte_length": path.stat().st_size,
|
||||
"sha256": sha256_file(path),
|
||||
}
|
||||
)
|
||||
source = request.source
|
||||
return {
|
||||
"schema_version": SOURCE_FRAME_MANIFEST_SCHEMA,
|
||||
"job_id": source.camera_job_id,
|
||||
"input_sha256": source.camera_input_sha256,
|
||||
"source_id": source.camera_source_id,
|
||||
"codec_epoch": source.codec_epoch,
|
||||
"frame_count": source.frame_count,
|
||||
"frames": rows,
|
||||
}
|
||||
|
||||
|
||||
def _validate_source_frame_manifest(
|
||||
path: Path,
|
||||
frames_root: Path,
|
||||
request: ComponentRequest,
|
||||
) -> None:
|
||||
document = load_json(
|
||||
path,
|
||||
label="EoMT source frame manifest",
|
||||
maximum=32 * 1024 * 1024,
|
||||
confinement_root=path.parent,
|
||||
)
|
||||
if document != _source_frame_manifest_document(frames_root, request):
|
||||
raise ComponentAdapterError("EoMT source frame manifest changed")
|
||||
|
||||
|
||||
def _validate_mask_archive_inventory(path: Path, frame_count: int) -> None:
|
||||
expected_files = {
|
||||
f"semantic-masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1)
|
||||
}
|
||||
expected_names = {"semantic-masks", *expected_files}
|
||||
try:
|
||||
with tarfile.open(path, mode="r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
names = [member.name for member in members]
|
||||
if len(names) != len(expected_names) or set(names) != expected_names:
|
||||
raise ComponentAdapterError("EoMT mask archive inventory changed")
|
||||
if len(names) != len(set(names)):
|
||||
raise ComponentAdapterError("EoMT mask archive contains duplicate members")
|
||||
for member in members:
|
||||
if member.name == "semantic-masks":
|
||||
if member.type != tarfile.DIRTYPE or member.size != 0:
|
||||
raise ComponentAdapterError("EoMT mask archive directory type changed")
|
||||
continue
|
||||
if (
|
||||
member.name not in expected_files
|
||||
or member.type not in {tarfile.REGTYPE, tarfile.AREGTYPE}
|
||||
or not 0 < member.size <= 2 * 1024 * 1024
|
||||
):
|
||||
raise ComponentAdapterError("EoMT mask archive member type changed")
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None:
|
||||
raise ComponentAdapterError("EoMT mask archive member is unavailable")
|
||||
payload = stream.read(member.size + 1)
|
||||
if len(payload) != member.size:
|
||||
raise ComponentAdapterError("EoMT mask archive member length changed")
|
||||
validate_grayscale_png_payload(
|
||||
payload,
|
||||
width=800,
|
||||
height=600,
|
||||
label=f"EoMT mask {member.name}",
|
||||
)
|
||||
except (OSError, EOFError, tarfile.TarError) as exc:
|
||||
raise ComponentAdapterError("EoMT mask archive is invalid") from exc
|
||||
|
||||
|
||||
def _eomt_environment(roots: Mapping[str, Path]) -> dict[str, str]:
|
||||
retained = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"}
|
||||
}
|
||||
python_path = f"{roots['eomt-transformers-environment']}:{roots['eomt-python-environment']}"
|
||||
return {
|
||||
**retained,
|
||||
"PYTHONPATH": python_path,
|
||||
"TORCH_HOME": str(roots["eomt-model-cache"] / "torch"),
|
||||
"HF_HOME": str(roots["eomt-model-cache"] / "huggingface"),
|
||||
"HF_HUB_OFFLINE": "1",
|
||||
"TRANSFORMERS_OFFLINE": "1",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"HOME": "/tmp",
|
||||
}
|
||||
|
||||
|
||||
def _empty_output_root(path: Path) -> Path:
|
||||
resolved = path.resolve(strict=True)
|
||||
if path.is_symlink() or not resolved.is_dir() or any(resolved.iterdir()):
|
||||
raise ComponentAdapterError("EoMT output root is not an empty real directory")
|
||||
return resolved
|
||||
|
||||
|
||||
def _prepare_workspace(path: Path) -> Path:
|
||||
if path.exists() or path.is_symlink():
|
||||
raise ComponentAdapterError("EoMT workspace already exists")
|
||||
path.mkdir(mode=0o700, parents=False)
|
||||
return path.resolve(strict=True)
|
||||
|
||||
|
||||
def _required_nonempty_file(path: Path, root: Path, label: str) -> Path:
|
||||
resolved = _required_confined_file(path, root, label)
|
||||
if resolved.stat().st_size < 1:
|
||||
raise ComponentAdapterError(f"{label} is empty")
|
||||
return resolved
|
||||
|
||||
|
||||
def _required_confined_file(path: Path, root: Path, label: str) -> Path:
|
||||
try:
|
||||
metadata = path.lstat()
|
||||
resolved_root = root.resolve(strict=True)
|
||||
resolved = path.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise ComponentAdapterError(f"{label} is unavailable") from exc
|
||||
if (
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or not stat.S_ISREG(metadata.st_mode)
|
||||
or not resolved.is_relative_to(resolved_root)
|
||||
):
|
||||
raise ComponentAdapterError(f"{label} is not confined")
|
||||
return resolved
|
||||
|
||||
|
||||
def _write_json(path: Path, value: object) -> None:
|
||||
payload = json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n"
|
||||
path.write_text(payload, encoding="utf-8")
|
||||
|
||||
|
||||
def _write_canonical_json(path: Path, value: object) -> None:
|
||||
path.write_bytes(canonical_json(value))
|
||||
|
||||
|
||||
def _publish_component_children(
|
||||
staging: Path,
|
||||
output: Path,
|
||||
*,
|
||||
expected_names: set[str],
|
||||
) -> None:
|
||||
children = tuple(staging.iterdir())
|
||||
if {child.name for child in children} != expected_names or any(
|
||||
child.is_dir() or child.is_symlink() for child in children
|
||||
):
|
||||
raise ComponentAdapterError("EoMT staging contains unexpected directories")
|
||||
for child in children:
|
||||
destination = output / child.name
|
||||
if destination.exists() or destination.is_symlink():
|
||||
raise ComponentAdapterError("EoMT publication target already exists")
|
||||
os.replace(child, destination)
|
||||
staging.rmdir()
|
||||
|
||||
|
||||
def _elapsed(started: float) -> float:
|
||||
return max(time.perf_counter() - started, 0.000001)
|
||||
|
||||
|
||||
def _number(value: float) -> str:
|
||||
return format(value, ".9f").rstrip("0").rstrip(".")
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv)
|
||||
execute_eomt_component(
|
||||
request_path=request_path,
|
||||
layout=RuntimeLayout.fixed(EOMT_ASSETS),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except ComponentAdapterError as exc:
|
||||
print(f"portable LAB V1 EoMT rejected: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2) from exc
|
||||
@@ -793,6 +793,7 @@ class InstalledPortableLabV1EomtRunner:
|
||||
try:
|
||||
request = _component_request(
|
||||
component="eomt",
|
||||
component_image_sha256=component.image_sha256,
|
||||
source=source,
|
||||
plan=plan,
|
||||
assets=component.assets,
|
||||
@@ -854,6 +855,7 @@ class InstalledPortableLabV1DdrnetRunner:
|
||||
try:
|
||||
request = _component_request(
|
||||
component="ddrnet",
|
||||
component_image_sha256=component.image_sha256,
|
||||
source=source,
|
||||
plan=plan,
|
||||
assets=component.assets,
|
||||
@@ -1022,14 +1024,17 @@ def _prepare_invocation(
|
||||
def _component_request(
|
||||
*,
|
||||
component: PortableLabV1Component,
|
||||
component_image_sha256: str,
|
||||
source: PortableLabV1MaterializedSource,
|
||||
plan: PortableLabV1OrchestrationPlan,
|
||||
assets: tuple[PortableLabV1HostAsset, ...],
|
||||
effective_config_sha256: str | None,
|
||||
) -> dict[str, object]:
|
||||
_digest(component_image_sha256, "component request image sha256")
|
||||
return {
|
||||
"schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA,
|
||||
"component": component,
|
||||
"component_image_sha256": component_image_sha256,
|
||||
"plan_sha256": plan.plan_sha256,
|
||||
"definition_sha256": plan.definition_sha256,
|
||||
"release_candidate_sha256": plan.release_candidate_sha256,
|
||||
|
||||
@@ -22,7 +22,10 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None:
|
||||
assert '"tree-receipt.json"' in script
|
||||
assert '"D:\\NDC_MISSIONCORE\\runtime"' in script
|
||||
assert '"derived\\perception-e3-opencv413092-v1"' in script
|
||||
assert '"cache\\perception-e3-models-v1"' in script
|
||||
assert '"eomt-model-cache-cityscapes-large-1024-v1"' in script
|
||||
assert "Install-SafeEomtModelCache" in script
|
||||
assert 'New-Item -ItemType HardLink -Path $blobTarget -Target $source' in script
|
||||
assert 'New-Item -ItemType HardLink -Path $snapshotTarget -Target $source' in script
|
||||
assert '"derived\\perception-p0-env-v1"' in script
|
||||
assert '"derived\\perception-p0-transformers4576-v1"' in script
|
||||
assert '$Utf8.GetByteCount($component) -gt 255' in script
|
||||
@@ -39,15 +42,14 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None:
|
||||
assert "smb" not in lowered
|
||||
|
||||
|
||||
def test_eomt_tree_sealer_has_atomic_commit_marker_and_no_rehash_fast_path() -> None:
|
||||
def test_eomt_tree_sealer_has_atomic_commit_marker_and_rehashes_existing_seal() -> None:
|
||||
script = SEALER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
fast_path = script.split("function Test-ExistingSeal", maxsplit=1)[1].split(
|
||||
"function New-SealedReceipt", maxsplit=1
|
||||
)[0]
|
||||
assert "-HashPayload:$false" in fast_path
|
||||
assert "-HashPayload:$true" not in fast_path
|
||||
assert "Get-FileDigest" not in fast_path
|
||||
assert "-HashPayload:$true" in fast_path
|
||||
assert "Assert-InventoryMatches $manifest.records $observed $Asset $false $true" in fast_path
|
||||
|
||||
publish = script.split("$manifestPath = Join-Path $Root $ManifestName", maxsplit=1)[1]
|
||||
manifest_move = publish.index("[IO.File]::Move($stagedManifest, $manifestPath)")
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
INSTALLER_PATH = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments"
|
||||
/ "perception"
|
||||
/ "worker"
|
||||
/ "observatory_portable"
|
||||
/ "Install-LabV1PortableStaticAssets.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_lab_v1_static_installer_is_fixed_offline_and_content_addressed() -> None:
|
||||
script = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
assert '"D:\\NDC_MISSIONCORE\\runtime"' in script
|
||||
assert '"staging\\observatory-lab-v1-static-v1"' in script
|
||||
assert '"eomt-runner-bundle-v1"' in script
|
||||
assert '"lab-v1-static-files-v1"' in script
|
||||
assert "New-Item `\n -ItemType HardLink" in script
|
||||
assert "Get-FileIdentity" in script
|
||||
assert "tree-manifest.tsv" in script
|
||||
assert "tree-receipt.json" in script
|
||||
|
||||
lowered = script.lower()
|
||||
assert "invoke-webrequest" not in lowered
|
||||
assert "start-bitstransfer" not in lowered
|
||||
assert "docker " not in lowered
|
||||
assert "ssh " not in lowered
|
||||
assert "smb" not in lowered
|
||||
|
||||
|
||||
def test_lab_v1_static_installer_seals_expected_exact_payloads() -> None:
|
||||
script = INSTALLER_PATH.read_text(encoding="utf-8")
|
||||
|
||||
expected = {
|
||||
"651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4",
|
||||
"01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6",
|
||||
"4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675",
|
||||
"25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4",
|
||||
"ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875",
|
||||
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
|
||||
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f",
|
||||
"b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1",
|
||||
"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35",
|
||||
"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352",
|
||||
"a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63",
|
||||
}
|
||||
assert all(digest in script for digest in expected)
|
||||
assert "259419077" in script
|
||||
assert "32877" in script
|
||||
assert "1427" in script
|
||||
File diff suppressed because it is too large
Load Diff
@@ -552,7 +552,11 @@ def test_component_adapter_images_are_thin_offline_wrappers() -> None:
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "FROM nvcr.io/nvidia/tritonserver:26.06-py3@sha256:" in eomt
|
||||
assert "FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1" in ddrnet
|
||||
assert (
|
||||
"FROM sha256:"
|
||||
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||
in ddrnet
|
||||
)
|
||||
assert "COPY portable_lab_v1_component_adapter.py" in eomt
|
||||
assert "COPY portable_lab_v1_component_adapter.py" in ddrnet
|
||||
assert 'ENTRYPOINT ["python3"]' in eomt
|
||||
|
||||
Reference in New Issue
Block a user