From 7b021782d3c4ec0f001c82e313a033d1df3df950 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 31 Aug 2026 19:46:47 +0300 Subject: [PATCH] feat(lab): install sealed component adapters --- .../Dockerfile.lab-v1-ddrnet-adapter | 2 +- .../Install-LabV1PortableStaticAssets.ps1 | 237 ++++ .../Seal-LabV1EomtTreeAssets.ps1 | 102 +- .../portable_lab_v1_component_adapter.py | 1116 +++++++++++++++ .../run_portable_lab_v1_ddrnet_component.py | 828 +++++++++++ .../run_portable_lab_v1_eomt_component.py | 987 ++++++++++++++ .../portable_lab_v1_local_runners.py | 5 + tests/test_lab_v1_eomt_tree_asset_sealer.py | 12 +- ..._lab_v1_portable_static_asset_installer.py | 55 + ...tory_portable_lab_v1_component_adapters.py | 1209 +++++++++++++++++ ...servatory_portable_lab_v1_local_runners.py | 6 +- 11 files changed, 4545 insertions(+), 14 deletions(-) create mode 100644 experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 create mode 100644 experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py create mode 100644 experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py create mode 100644 experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py create mode 100644 tests/test_lab_v1_portable_static_asset_installer.py create mode 100644 tests/test_observatory_portable_lab_v1_component_adapters.py diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.lab-v1-ddrnet-adapter b/experiments/perception/worker/observatory_portable/Dockerfile.lab-v1-ddrnet-adapter index 0659689..89ecf4c 100644 --- a/experiments/perception/worker/observatory_portable/Dockerfile.lab-v1-ddrnet-adapter +++ b/experiments/perception/worker/observatory_portable/Dockerfile.lab-v1-ddrnet-adapter @@ -1,4 +1,4 @@ -FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1 +FROM sha256:591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 diff --git a/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 b/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 new file mode 100644 index 0000000..25fc6f6 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 @@ -0,0 +1,237 @@ +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\") +$SourceRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-static-v1" +$AssetRoot = Join-Path $RuntimeRoot "assets\observatory-portable" +$RunnerRoot = Join-Path $AssetRoot "eomt-runner-bundle-v1" +$ValidFovRoot = Join-Path $AssetRoot ( + "k1-valid-fov-root-" + + "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1" +) +$StaticRoot = Join-Path $AssetRoot "lab-v1-static-files-v1" +$ValidFovSource = Join-Path $RuntimeRoot ( + "inputs\e2\valid-fov-mask-" + + "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +) +$CheckpointSource = ( + "D:\NDC_MISSIONCORE\datasets\vegetation-v1\observed-2026-08-27\" + + "models\goose\ddrnet_class_512.pth" +) +$MappingSource = ( + "D:\NDC_MISSIONCORE\datasets\goose-3d\v2025-08-22\installs\" + + "0be9e0f8459bafcbc92ff7c3cc366e9b4e2f6e1e9bdf50e6439e1557e864c26f\" + + "goose_label_mapping.csv" +) +$Invariant = [Globalization.CultureInfo]::InvariantCulture + +$RunnerFiles = [ordered]@{ + "e3_k1_camera1_profile.json" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "e3_k1_camera1_profile.json") + byte_length = [int64]3805 + sha256 = "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875" + } + "run_e3_rectified_segmentation.py" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "run_e3_rectified_segmentation.py") + byte_length = [int64]45789 + sha256 = "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6" + } + "run_e4_full_session_segmentation.py" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "run_e4_full_session_segmentation.py") + byte_length = [int64]30720 + sha256 = "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" + } + "run_evaluation_prelabels.py" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "run_evaluation_prelabels.py") + byte_length = [int64]28915 + sha256 = "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4" + } + "run_recorded_perception_epoch.py" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "run_recorded_perception_epoch.py") + byte_length = [int64]34899 + sha256 = "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675" + } +} + +$ValidFovFiles = [ordered]@{ + "manifest.json" = [pscustomobject]@{ + source = (Join-Path $ValidFovSource "manifest.json") + byte_length = [int64]2351 + sha256 = "3dea056db506f47b8a7672e847d0ffe4c66a380a5032555a2e0dde958293742e" + } + "mask.png" = [pscustomobject]@{ + source = (Join-Path $ValidFovSource "mask.png") + byte_length = [int64]3668 + sha256 = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" + } +} + +$StaticFiles = [ordered]@{ + "ddrnet-checkpoint.pth" = [pscustomobject]@{ + source = $CheckpointSource + byte_length = [int64]259419077 + sha256 = "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" + } + "ddrnet-goose-mapping.csv" = [pscustomobject]@{ + source = $MappingSource + byte_length = [int64]1427 + sha256 = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f" + } + "ddrnet-goose-runner.py" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "run_goose_vegetation_benchmark.py") + byte_length = [int64]32877 + sha256 = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1" + } + "vegetation-policy.json" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "lab-v1-vegetation-mission-policy-v1.json") + byte_length = [int64]3022 + sha256 = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35" + } + "vegetation-provider-map.json" = [pscustomobject]@{ + source = (Join-Path $SourceRoot "lab-v1-vegetation-provider-label-map-v1.json") + byte_length = [int64]2756 + sha256 = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352" + } +} + +function Get-FileIdentity { + param([string]$Path) + + $item = Get-Item -LiteralPath $Path -Force + if ( + -not ($item -is [IO.FileInfo]) -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) + ) { + throw "LAB V1 static source is not a regular file" + } + $stream = [IO.File]::Open( + $item.FullName, + [IO.FileMode]::Open, + [IO.FileAccess]::Read, + [IO.FileShare]::Read + ) + $algorithm = [Security.Cryptography.SHA256]::Create() + try { + $length = [int64]$stream.Length + $sha256 = -join @($algorithm.ComputeHash($stream) | ForEach-Object { + $_.ToString("x2", $Invariant) + }) + } + finally { + $algorithm.Dispose() + $stream.Dispose() + } + return [pscustomobject]@{ + byte_length = $length + sha256 = $sha256 + } +} + +function Assert-ExactFile { + param( + [string]$Path, + [object]$Descriptor, + [string]$Label + ) + + $identity = Get-FileIdentity $Path + if ( + [int64]$identity.byte_length -ne [int64]$Descriptor.byte_length -or + [string]$identity.sha256 -cne [string]$Descriptor.sha256 + ) { + throw "$Label identity changed" + } +} + +function Install-HardlinkedDirectory { + param( + [string]$Target, + [Collections.IDictionary]$Files, + [bool]$AllowTreeSealMetadata + ) + + foreach ($name in $Files.Keys) { + Assert-ExactFile $Files[$name].source $Files[$name] "LAB V1 source $name" + } + if (Test-Path -LiteralPath $Target -PathType Container) { + $allowed = @($Files.Keys) + if ($AllowTreeSealMetadata) { + $allowed += @("tree-manifest.tsv", "tree-receipt.json") + } + $observed = @( + Get-ChildItem -LiteralPath $Target -Force | ForEach-Object { + if (-not ($_ -is [IO.FileInfo])) { + throw "Installed LAB V1 asset contains a non-file member" + } + $_.Name + } + ) + if ( + @($observed | Sort-Object).Count -ne @($allowed | Sort-Object).Count -or + [string]::Join([char]10, @($observed | Sort-Object)) -cne + [string]::Join([char]10, @($allowed | Sort-Object)) + ) { + throw "Installed LAB V1 asset inventory changed" + } + foreach ($name in $Files.Keys) { + Assert-ExactFile (Join-Path $Target $name) $Files[$name] "Installed $name" + } + return "already-installed" + } + if (Test-Path -LiteralPath $Target) { + throw "LAB V1 asset target is not a directory" + } + $parent = Split-Path -Parent $Target + if (-not (Test-Path -LiteralPath $parent -PathType Container)) { + throw "LAB V1 asset parent is unavailable" + } + $staging = "$Target.staging-$([Guid]::NewGuid().ToString('N'))" + try { + $null = New-Item -ItemType Directory -Path $staging + foreach ($name in $Files.Keys) { + $null = New-Item ` + -ItemType HardLink ` + -Path (Join-Path $staging $name) ` + -Target $Files[$name].source + Assert-ExactFile (Join-Path $staging $name) $Files[$name] "Staged $name" + } + [IO.Directory]::Move($staging, $Target) + return "installed" + } + finally { + if (Test-Path -LiteralPath $staging -PathType Container) { + Remove-Item -LiteralPath $staging -Recurse -Force + } + } +} + +if (-not (Test-Path -LiteralPath $SourceRoot -PathType Container)) { + throw "LAB V1 static source staging root is unavailable" +} +if (-not (Test-Path -LiteralPath $AssetRoot -PathType Container)) { + throw "LAB V1 immutable asset root is unavailable" +} + +$outputs = @( + [ordered]@{ + asset_id = "eomt-runner-bundle" + status = Install-HardlinkedDirectory $RunnerRoot $RunnerFiles $true + path = $RunnerRoot + } + [ordered]@{ + asset_id = "k1-valid-fov-root" + status = Install-HardlinkedDirectory $ValidFovRoot $ValidFovFiles $true + path = $ValidFovRoot + } + [ordered]@{ + asset_id = "lab-v1-static-files" + status = Install-HardlinkedDirectory $StaticRoot $StaticFiles $false + path = $StaticRoot + } +) + +$outputs | ConvertTo-Json -Compress -Depth 4 diff --git a/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 index 78e38ce..05cbbc0 100644 --- a/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 +++ b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 @@ -28,13 +28,41 @@ $IdentityAlgorithm = "relative-path-tab-size-tab-file-sha256-lf/v1" $MaximumManifestBytes = 32MB $Utf8 = New-Object Text.UTF8Encoding($false, $true) $Invariant = [Globalization.CultureInfo]::InvariantCulture +$OriginalModelCache = ( + "D:\NDC_MISSIONCORE\runtime\cache\perception-e3-models-v1\huggingface\" + + "models--tue-mps--cityscapes_semantic_eomt_large_1024" +) +$SafeModelCache = ( + "D:\NDC_MISSIONCORE\runtime\assets\observatory-portable\" + + "eomt-model-cache-cityscapes-large-1024-v1" +) +$ModelRevision = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f" +$ModelFiles = [ordered]@{ + "config.json" = [pscustomobject]@{ + blob = "66b2d540c2a4d9e9fbc958c1a43464cb13d933c5" + byte_length = [int64]1575 + sha256 = "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650" + } + "model.safetensors" = [pscustomobject]@{ + blob = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782" + byte_length = [int64]1276175488 + sha256 = "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782" + } + "preprocessor_config.json" = [pscustomobject]@{ + blob = "cf30bcc29300598a6b581ed043dd65abc893e948" + byte_length = [int64]666 + sha256 = "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7" + } +} # The first four roots already exist on Worker 006. The runner bundle and # valid-FOV root deliberately point at separate, immutable asset directories; # their payloads must be materialized there before this sealer is invoked. $AssetRelativeRoots = [ordered]@{ "eomt-environment" = "derived\perception-e3-opencv413092-v1" - "eomt-model-cache" = "cache\perception-e3-models-v1" + "eomt-model-cache" = ( + "assets\observatory-portable\eomt-model-cache-cityscapes-large-1024-v1" + ) "eomt-python-environment" = "derived\perception-p0-env-v1" "eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1" "eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1" @@ -152,6 +180,58 @@ function Get-FileDigest { } } +function Install-SafeEomtModelCache { + if (Test-Path -LiteralPath $SafeModelCache -PathType Container) { + return + } + if (Test-Path -LiteralPath $SafeModelCache) { + throw "Safe EoMT model cache target is not a directory" + } + $assetParent = Split-Path -Parent $SafeModelCache + if (-not (Test-Path -LiteralPath $assetParent -PathType Container)) { + throw "Safe EoMT model cache parent is unavailable" + } + $staging = "$SafeModelCache.staging-$([Guid]::NewGuid().ToString('N'))" + try { + $blobs = Join-Path $staging ( + "huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\blobs" + ) + $snapshot = Join-Path $staging ( + "huggingface\models--tue-mps--cityscapes_semantic_eomt_large_1024\" + + "snapshots\$ModelRevision" + ) + $null = New-Item -ItemType Directory -Path $blobs + $null = New-Item -ItemType Directory -Path $snapshot + foreach ($name in $ModelFiles.Keys) { + $descriptor = $ModelFiles[$name] + $source = Join-Path (Join-Path $OriginalModelCache "blobs") $descriptor.blob + $identity = Get-FileDigest $source + if ( + [int64]$identity.byte_length -ne [int64]$descriptor.byte_length -or + [string]$identity.sha256 -cne [string]$descriptor.sha256 + ) { + throw "Original EoMT model cache identity changed" + } + $blobTarget = Join-Path $blobs $descriptor.blob + $snapshotTarget = Join-Path $snapshot $name + $null = New-Item -ItemType HardLink -Path $blobTarget -Target $source + $null = New-Item -ItemType HardLink -Path $snapshotTarget -Target $source + } + foreach ($entry in [IO.Directory]::EnumerateFiles($staging, "*", "AllDirectories")) { + $attributes = [IO.File]::GetAttributes($entry) + if ($attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Safe EoMT model cache contains a reparse point" + } + } + [IO.Directory]::Move($staging, $SafeModelCache) + } + finally { + if (Test-Path -LiteralPath $staging -PathType Container) { + Remove-Item -LiteralPath $staging -Recurse -Force + } + } +} + function Get-PayloadInventory { param( [string]$Root, @@ -388,7 +468,8 @@ function Assert-InventoryMatches { [object[]]$Expected, [object[]]$Observed, [string]$Asset, - [bool]$CheckWriteTime + [bool]$CheckWriteTime, + [bool]$CheckDigest ) if ($Expected.Count -ne $Observed.Count) { @@ -400,6 +481,9 @@ function Assert-InventoryMatches { if ( [string]$left.relative_path -cne [string]$right.relative_path -or [int64]$left.byte_length -ne [int64]$right.byte_length -or + ($CheckDigest -and ( + [string]$left.sha256 -cne [string]$right.sha256 + )) -or ($CheckWriteTime -and ( [int64]$left.last_write_ticks -ne [int64]$right.last_write_ticks )) @@ -438,10 +522,11 @@ function Test-ExistingSeal { throw "Existing $Asset receipt does not bind its manifest" } - # Fast path: verify names and sizes, but deliberately do not re-hash the - # already sealed multi-gigabyte payload. - $observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false) - Assert-InventoryMatches $manifest.records $observed $Asset $false + # A seal is reusable only after the payload has been re-verified. Tree + # roots contain executable Python dependencies, so names and sizes alone + # must never authenticate an existing seal. + $observed = @(Get-PayloadInventory -Root $Root -HashPayload:$true) + Assert-InventoryMatches $manifest.records $observed $Asset $false $true return $receipt } @@ -519,7 +604,7 @@ function Seal-TreeAsset { # Re-enumerate without hashing before publication. This catches file, # size, and timestamp changes that occurred during the one-time hash. $observed = @(Get-PayloadInventory -Root $Root -HashPayload:$false) - Assert-InventoryMatches $records $observed $Asset $true + Assert-InventoryMatches $records $observed $Asset $true $false $null = New-Item -ItemType Directory -Path $staging $stagedManifest = Join-Path $staging $ManifestName @@ -573,6 +658,9 @@ $requested = @($AssetId) if ($requested.Count -eq 0 -or @($requested | Select-Object -Unique).Count -ne $requested.Count) { throw "LAB V1 EoMT asset selection is empty or contains duplicates" } +if ($requested -contains "eomt-model-cache") { + Install-SafeEomtModelCache +} $outputs = New-Object "Collections.Generic.List[object]" foreach ($asset in $requested) { $relativeRoot = [string]$AssetRelativeRoots[$asset] diff --git a/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py b/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py new file mode 100644 index 0000000..6d6e1ca --- /dev/null +++ b/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py @@ -0,0 +1,1116 @@ +#!/usr/bin/env python3 +"""Strict request boundary shared by the two portable LAB V1 components. + +The persistent Worker owns every mount and writes the canonical request. A +queued Observatory job therefore supplies no executable, image, argument, or +filesystem path to either component. This module intentionally uses only the +Python standard library so the same validator runs in both historical sibling +images before their model dependencies are imported. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import stat +import struct +import subprocess +import unicodedata +import zlib +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, Literal, cast + +REQUEST_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-component-request/v1" +SOURCE_SCHEMA: Final = "missioncore.observatory-portable-lab-v1-source/v1" +COMPUTE_JOB_SCHEMA: Final = "missioncore.compute-job/v1" +COMPUTE_PROFILE: Final = "recorded-camera-perception/v1" +COMPUTE_RESULT_SCHEMA: Final = "missioncore.compute-result/v1" + +AUTHORITY: Final = { + "commands_enabled": False, + "actuation_allowed": False, + "navigation_or_safety_accepted": False, + "production_accepted": False, +} +CAMERA_SOURCE_ID: Final = "sensor.camera.right" +CALIBRATION_SHA256: Final = "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" +EOMT_PIPELINE: Final = "recorded-semantic-eomt-fisheye-mask/v1" +DDRNET_CHECKPOINT_SHA256: Final = "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" +GOOSE_MAPPING_SHA256: Final = "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f" +EOMT_DEPENDENCY_IDENTITY_SHA256: Final = ( + "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e" +) +VALID_FOV_IDENTITY_SHA256: Final = ( + "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" +) +VALID_FOV_MASK_SHA256: Final = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" + +FIXED_REQUEST_PATH: Final = "/run/nodedc/request.json" +MAX_REQUEST_BYTES: Final = 2 * 1024 * 1024 +MAX_JOB_BYTES: Final = 32 * 1024 * 1024 +MAX_JOB_FILES: Final = 100_003 +MAX_SOURCE_BYTES: Final = 2 * 1024 * 1024 * 1024 * 1024 +MAX_FRAME_COUNT: Final = 100_000 +MAX_TREE_PATH_BYTES: Final = 4_096 +MAX_TREE_COMPONENT_BYTES: Final = 255 +MAX_TREE_COMPONENTS: Final = 256 +MAX_SEMANTIC_PNG_BYTES: Final = 2 * 1024 * 1024 +SEALED_TREE_SCHEMA: Final = "missioncore.sealed-tree-runtime/v1" +SEALED_TREE_IDENTITY_ALGORITHM: Final = "relative-path-tab-size-tab-file-sha256-lf/v1" +SEALED_TREE_MANIFEST: Final = "tree-manifest.tsv" +SEALED_TREE_RECEIPT: Final = "tree-receipt.json" + +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_SAFE_COMPONENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +_OBSERVATORY_JOB_ID = re.compile(r"^observatory-run-[a-f0-9]{32}$") +_CAMERA_JOB_ID = re.compile(r"^recorded-camera-[a-f0-9]{24}$") + +type Component = Literal["eomt", "ddrnet"] +type AssetKind = Literal["file", "tree"] +type AssetVerification = Literal["sha256", "identity-sha256"] +type CommandRunner = Callable[[Sequence[str], Mapping[str, str] | None], None] + + +class ComponentAdapterError(RuntimeError): + """A sealed request, mounted input, or component output changed.""" + + +@dataclass(frozen=True, slots=True) +class AssetExpectation: + asset_id: str + path: str + kind: AssetKind + verification: AssetVerification + identity_sha256: str | None = None + byte_length: int | None = None + + def __post_init__(self) -> None: + _safe_component(self.asset_id, "asset id") + _fixed_absolute_path(self.path, "asset path") + if self.kind == "file" and self.verification != "sha256": + raise ValueError("file asset expectation must use sha256") + if self.kind == "tree" and self.verification != "identity-sha256": + raise ValueError("tree asset expectation must use identity-sha256") + if self.identity_sha256 is not None: + _digest(self.identity_sha256, "asset expectation sha256") + if self.byte_length is not None and self.byte_length < 1: + raise ValueError("asset expectation byte length is invalid") + + +@dataclass(frozen=True, slots=True) +class AssetBinding: + asset_id: str + path: str + kind: AssetKind + verification: AssetVerification + identity_sha256: str + byte_length: int | None + + +@dataclass(frozen=True, slots=True) +class SourceBinding: + observatory_job_id: str + source_session_id: str + camera_job_id: str + camera_input_sha256: str + camera_source_id: str + codec_epoch: int + input_byte_length: int + frame_count: int + timeline_start_seconds: float + timeline_end_seconds: float + + +@dataclass(frozen=True, slots=True) +class ComponentRequest: + component: Component + component_image_sha256: str + plan_sha256: str + definition_sha256: str + release_candidate_sha256: str + source: SourceBinding + paths: Mapping[str, str | None] + effective_ddrnet_config_sha256: str | None + assets: Mapping[str, AssetBinding] + + +@dataclass(frozen=True, slots=True) +class RuntimeLayout: + """Runtime-only test seam; production entrypoints use ``fixed()`` only.""" + + request: Path + camera_job_root: Path + output_root: Path + effective_ddrnet_config: Path + eomt_result_root: Path + asset_paths: Mapping[str, Path] + + @classmethod + def fixed(cls, expectations: Sequence[AssetExpectation]) -> RuntimeLayout: + return cls( + request=Path(FIXED_REQUEST_PATH), + camera_job_root=Path("/run/nodedc/source"), + output_root=Path("/run/nodedc/output"), + effective_ddrnet_config=Path("/run/nodedc/effective-ddrnet-config.json"), + eomt_result_root=Path("/run/nodedc/eomt-result"), + asset_paths={item.asset_id: Path(item.path) for item in expectations}, + ) + + +def canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def load_canonical_json( + path: Path, + *, + label: str, + maximum: int, + confinement_root: Path | None = None, +) -> dict[str, object]: + _payload, document = _canonical_document( + path, + label=label, + maximum=maximum, + confinement_root=confinement_root, + ) + return document + + +def load_json( + path: Path, + *, + label: str, + maximum: int, + confinement_root: Path | None = None, +) -> dict[str, object]: + _payload, document = _canonical_or_pretty_document( + path, + label=label, + maximum=maximum, + confinement_root=confinement_root, + ) + return document + + +def parse_fixed_request_argument(argv: Sequence[str]) -> Path: + if tuple(argv) != ("--request", FIXED_REQUEST_PATH): + raise ComponentAdapterError("component accepts only --request /run/nodedc/request.json") + return Path(FIXED_REQUEST_PATH) + + +def load_component_request( + request_path: Path, + *, + component: Component, + expectations: Sequence[AssetExpectation], +) -> ComponentRequest: + payload, document = _canonical_document( + request_path, + label="component request", + maximum=MAX_REQUEST_BYTES, + ) + del payload + _exact_keys( + document, + { + "schema_version", + "component", + "component_image_sha256", + "plan_sha256", + "definition_sha256", + "release_candidate_sha256", + "source", + "paths", + "effective_ddrnet_config_sha256", + "assets", + "authority", + }, + "component request", + ) + if ( + document["schema_version"] != REQUEST_SCHEMA + or document["component"] != component + or document["authority"] != AUTHORITY + ): + raise ComponentAdapterError("component request contract changed") + component_image_sha256 = _digest( + document["component_image_sha256"], + "component image sha256", + ) + plan_sha256 = _digest(document["plan_sha256"], "plan sha256") + definition_sha256 = _digest(document["definition_sha256"], "definition sha256") + release_sha256 = _digest( + document["release_candidate_sha256"], + "release candidate sha256", + ) + source = _source_binding(document["source"]) + paths = _paths(document["paths"], component) + effective_value = document["effective_ddrnet_config_sha256"] + if component == "eomt": + if effective_value is not None: + raise ComponentAdapterError("EoMT request contains a DDRNet config") + effective_sha256: str | None = None + else: + effective_sha256 = _digest(effective_value, "effective DDRNet config sha256") + assets = _asset_bindings(document["assets"], expectations) + return ComponentRequest( + component=component, + component_image_sha256=component_image_sha256, + plan_sha256=plan_sha256, + definition_sha256=definition_sha256, + release_candidate_sha256=release_sha256, + source=source, + paths=paths, + effective_ddrnet_config_sha256=effective_sha256, + assets=assets, + ) + + +def validate_camera_compute_job(root: Path, source: SourceBinding) -> dict[str, object]: + job_root = _real_directory(root, "camera compute job") + _payload, document = _canonical_document( + job_root / "job.json", + label="camera compute job manifest", + maximum=MAX_JOB_BYTES, + confinement_root=job_root, + ) + _exact_keys( + document, + { + "schema_version", + "job_id", + "profile", + "input_sha256", + "input", + "result_contract", + }, + "camera compute job manifest", + ) + input_document = _object(document["input"], "camera compute input") + input_sha256 = hashlib.sha256(canonical_json(input_document)).hexdigest() + if ( + document["schema_version"] != COMPUTE_JOB_SCHEMA + or document["profile"] != COMPUTE_PROFILE + or document["result_contract"] + != { + "schema_version": COMPUTE_RESULT_SCHEMA, + "timestamp_basis": "session-time-seconds", + } + or document["job_id"] != source.camera_job_id + or document["input_sha256"] != source.camera_input_sha256 + or input_sha256 != source.camera_input_sha256 + or source.camera_job_id != f"recorded-camera-{input_sha256[:24]}" + ): + raise ComponentAdapterError("camera compute job identity changed") + _exact_keys( + input_document, + { + "kind", + "session_id", + "source_id", + "codec_epoch", + "synchronization", + "media_type", + "timeline", + "segment_count", + "byte_length", + "archive_summary_sha256", + "archive_index_sha256", + "files", + }, + "camera compute input", + ) + timeline = _object(input_document["timeline"], "camera timeline") + _exact_keys(timeline, {"basis", "start_seconds", "end_seconds"}, "camera timeline") + if ( + input_document["kind"] != "canonical-camera-epoch" + or input_document["session_id"] != source.source_session_id + or input_document["source_id"] != source.camera_source_id + or input_document["codec_epoch"] != source.codec_epoch + or input_document["segment_count"] != source.frame_count + or input_document["byte_length"] != source.input_byte_length + or timeline["basis"] != "session-time-seconds" + or timeline["start_seconds"] != source.timeline_start_seconds + or timeline["end_seconds"] != source.timeline_end_seconds + ): + raise ComponentAdapterError("camera compute source binding changed") + rows = input_document["files"] + if not isinstance(rows, list) or not 4 <= len(rows) <= MAX_JOB_FILES: + raise ComponentAdapterError("camera compute file inventory is invalid") + prefix = ( + PurePosixPath("input") / "camera" / source.camera_source_id / f"epoch-{source.codec_epoch}" + ) + expected = { + (prefix / "summary.json").as_posix(), + (prefix / "index.jsonl").as_posix(), + (prefix / "init.mp4").as_posix(), + *( + (prefix / "segments" / f"{index}.m4s").as_posix() + for index in range(1, source.frame_count + 1) + ), + } + observed: set[str] = set() + total = 0 + for value in rows: + row = _object(value, "camera compute file") + _exact_keys(row, {"path", "byte_length", "sha256"}, "camera compute file") + relative = _relative_path(row["path"], "camera compute file path") + encoded = relative.as_posix() + byte_length = _positive_int(row["byte_length"], "camera compute file bytes") + digest = _digest(row["sha256"], "camera compute file sha256") + if encoded in observed: + raise ComponentAdapterError("camera compute file is duplicated") + path = _confined_file(job_root / Path(*relative.parts), job_root, "camera compute file") + if path.stat().st_size != byte_length or sha256_file(path) != digest: + raise ComponentAdapterError("camera compute file identity changed") + observed.add(encoded) + total += byte_length + if total > MAX_SOURCE_BYTES: + raise ComponentAdapterError("camera compute input is too large") + if observed != expected or total != source.input_byte_length: + raise ComponentAdapterError("camera compute file set changed") + return input_document + + +def validate_file_asset( + request: ComponentRequest, + layout: RuntimeLayout, + asset_id: str, +) -> Path: + binding = request.assets[asset_id] + path = _real_file(layout.asset_paths[asset_id], f"asset {asset_id}") + if ( + binding.byte_length is not None and path.stat().st_size != binding.byte_length + ) or sha256_file(path) != binding.identity_sha256: + raise ComponentAdapterError(f"asset {asset_id} identity changed") + return path + + +def validate_tree_asset( + request: ComponentRequest, + layout: RuntimeLayout, + asset_id: str, +) -> Path: + binding = request.assets[asset_id] + if binding.kind != "tree" or binding.verification != "identity-sha256": + raise ComponentAdapterError(f"asset {asset_id} tree contract changed") + return _real_directory(layout.asset_paths[asset_id], f"asset {asset_id}") + + +def validate_tree_receipt( + request: ComponentRequest, + root: Path, + asset_id: str, + *, + expected_metadata: Mapping[str, object] | None = None, + additional_metadata_keys: frozenset[str] = frozenset(), + verify_payload: bool, +) -> dict[str, object]: + """Bind one mounted tree to its install-time canonical inventory. + + The manifest and receipt are installation metadata and are deliberately + excluded from the payload identity. Large Python/model trees may use the + receipt plus component-specific critical-file checks at runtime; small + release trees are re-hashed completely before any executable is invoked. + """ + + resolved = _real_directory(root, f"asset {asset_id}") + _payload, receipt = _canonical_or_pretty_document( + resolved / SEALED_TREE_RECEIPT, + label=f"asset {asset_id} receipt", + maximum=MAX_JOB_BYTES, + confinement_root=resolved, + ) + base_keys = { + "schema_version", + "asset_id", + "identity_algorithm", + "identity_sha256", + "file_count", + "byte_length", + "manifest_relative_path", + } + metadata = {} if expected_metadata is None else dict(expected_metadata) + _exact_keys( + receipt, + base_keys | set(metadata) | set(additional_metadata_keys), + f"asset {asset_id} receipt", + ) + binding = request.assets[asset_id] + file_count = _positive_int(receipt["file_count"], f"asset {asset_id} file count") + byte_length = _positive_int( + receipt["byte_length"], + f"asset {asset_id} byte length", + ) + if ( + receipt["schema_version"] != SEALED_TREE_SCHEMA + or receipt["asset_id"] != asset_id + or receipt["identity_algorithm"] != SEALED_TREE_IDENTITY_ALGORITHM + or receipt["identity_sha256"] != binding.identity_sha256 + or receipt["manifest_relative_path"] != SEALED_TREE_MANIFEST + or binding.byte_length != byte_length + or any(receipt.get(key) != value for key, value in metadata.items()) + ): + raise ComponentAdapterError(f"asset {asset_id} receipt changed") + manifest_path = _confined_file( + resolved / SEALED_TREE_MANIFEST, + resolved, + f"asset {asset_id} manifest", + ) + try: + manifest = manifest_path.read_bytes() + except OSError as exc: + raise ComponentAdapterError(f"asset {asset_id} manifest is unavailable") from exc + if ( + not manifest + or len(manifest) > MAX_JOB_BYTES + or hashlib.sha256(manifest).hexdigest() != binding.identity_sha256 + ): + raise ComponentAdapterError(f"asset {asset_id} manifest identity changed") + rows = _tree_manifest_rows(manifest, asset_id) + if len(rows) != file_count or sum(row[1] for row in rows) != byte_length: + raise ComponentAdapterError(f"asset {asset_id} inventory totals changed") + if verify_payload: + for relative, expected_bytes, expected_sha256 in rows: + path = _confined_file( + resolved / Path(*relative.parts), + resolved, + f"asset {asset_id} payload", + ) + if path.stat().st_size != expected_bytes or sha256_file(path) != expected_sha256: + raise ComponentAdapterError(f"asset {asset_id} payload changed") + expected_members = {relative.as_posix() for relative, _size, _sha256 in rows} | { + SEALED_TREE_MANIFEST, + SEALED_TREE_RECEIPT, + } + observed_members: set[str] = set() + try: + for candidate in resolved.rglob("*"): + member = candidate.relative_to(resolved).as_posix() + if candidate.is_symlink() or not (candidate.is_file() or candidate.is_dir()): + raise ComponentAdapterError(f"asset {asset_id} has an unsafe member") + if candidate.is_file(): + observed_members.add(member) + except OSError as exc: + raise ComponentAdapterError(f"asset {asset_id} inventory is unavailable") from exc + if observed_members != expected_members: + raise ComponentAdapterError(f"asset {asset_id} member set changed") + return receipt + + +def validate_critical_tree_file( + root: Path, + relative_path: str, + *, + byte_length: int, + identity_sha256: str, + label: str, +) -> Path: + relative = _relative_path(relative_path, label) + path = _confined_file( + root / Path(*relative.parts), + root, + label, + ) + if path.stat().st_size != byte_length or sha256_file(path) != identity_sha256: + raise ComponentAdapterError(f"{label} identity changed") + return path + + +def validate_identity_manifest( + path: Path, + *, + expected_identity_sha256: str, + expected_schema: str, + label: str, +) -> dict[str, object]: + _payload, document = _canonical_or_pretty_document( + path, + label=label, + maximum=MAX_JOB_BYTES, + ) + identity = _object(document.get("identity"), f"{label} identity") + identity_sha256 = _digest(document.get("identity_sha256"), f"{label} identity sha256") + if ( + document.get("schema_version") != expected_schema + or identity_sha256 != expected_identity_sha256 + or hashlib.sha256(canonical_json(identity)).hexdigest() != identity_sha256 + ): + raise ComponentAdapterError(f"{label} identity changed") + return document + + +def validate_fixed_result( + root: Path, + *, + schema_version: str, + label: str, +) -> dict[str, object]: + result_root = _real_directory(root, f"{label} result root") + _payload, document = _canonical_or_pretty_document( + result_root / "result.json", + label=f"{label} result", + maximum=8 * 1024 * 1024, + confinement_root=result_root, + ) + if document.get("schema_version") != schema_version: + raise ComponentAdapterError(f"{label} result schema changed") + return document + + +def run_command(argv: Sequence[str], environment: Mapping[str, str] | None) -> None: + try: + subprocess.run( + list(argv), + check=True, + env=None if environment is None else dict(environment), + ) + except (OSError, subprocess.SubprocessError) as exc: + raise ComponentAdapterError("sealed component command failed") from exc + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + try: + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + except OSError as exc: + raise ComponentAdapterError("sealed file is unavailable") from exc + return digest.hexdigest() + + +def validate_grayscale_png_payload( + payload: bytes, + *, + width: int, + height: int, + label: str, +) -> None: + """Validate one bounded, non-interlaced, 8-bit grayscale PNG completely.""" + + if not 0 < len(payload) <= MAX_SEMANTIC_PNG_BYTES or payload[:8] != b"\x89PNG\r\n\x1a\n": + raise ComponentAdapterError(f"{label} encoding changed") + offset = 8 + chunk_index = 0 + idat_parts: list[bytes] = [] + idat_closed = False + saw_iend = False + while offset < len(payload): + if len(payload) - offset < 12: + raise ComponentAdapterError(f"{label} chunk framing changed") + byte_length = struct.unpack_from(">I", payload, offset)[0] + chunk_type = payload[offset + 4 : offset + 8] + end = offset + 12 + byte_length + if end > len(payload) or not all( + 65 <= value <= 90 or 97 <= value <= 122 for value in chunk_type + ): + raise ComponentAdapterError(f"{label} chunk framing changed") + chunk = payload[offset + 8 : offset + 8 + byte_length] + expected_crc = struct.unpack_from(">I", payload, offset + 8 + byte_length)[0] + observed_crc = zlib.crc32(chunk, zlib.crc32(chunk_type)) & 0xFFFFFFFF + if observed_crc != expected_crc: + raise ComponentAdapterError(f"{label} CRC changed") + if chunk_index == 0: + if chunk_type != b"IHDR" or byte_length != 13: + raise ComponentAdapterError(f"{label} IHDR changed") + png_width, png_height, depth, color, compression, filtering, interlace = struct.unpack( + ">IIBBBBB", chunk + ) + if ( + png_width != width + or png_height != height + or depth != 8 + or color != 0 + or compression != 0 + or filtering != 0 + or interlace != 0 + ): + raise ComponentAdapterError(f"{label} image contract changed") + elif chunk_type == b"IHDR": + raise ComponentAdapterError(f"{label} contains a duplicate IHDR") + if chunk_type == b"IDAT": + if idat_closed or not chunk: + raise ComponentAdapterError(f"{label} IDAT sequence changed") + idat_parts.append(chunk) + elif idat_parts: + idat_closed = True + if chunk_type == b"IEND": + if byte_length != 0 or end != len(payload): + raise ComponentAdapterError(f"{label} IEND changed") + saw_iend = True + offset = end + break + if chunk_type not in {b"IHDR", b"IDAT"} and chunk_type[0] & 0x20 == 0: + raise ComponentAdapterError(f"{label} has an unknown critical chunk") + offset = end + chunk_index += 1 + if not saw_iend or offset != len(payload) or not idat_parts: + raise ComponentAdapterError(f"{label} structure changed") + expected_raw_bytes = height * (width + 1) + inflater = zlib.decompressobj() + try: + decoded = inflater.decompress(b"".join(idat_parts), expected_raw_bytes + 1) + except zlib.error as exc: + raise ComponentAdapterError(f"{label} image data is invalid") from exc + if ( + len(decoded) != expected_raw_bytes + or not inflater.eof + or inflater.unused_data + or inflater.unconsumed_tail + or any(decoded[row * (width + 1)] > 4 for row in range(height)) + ): + raise ComponentAdapterError(f"{label} image data changed") + + +def available_bytes(path: Path) -> int: + try: + status = os.statvfs(path) + except OSError as exc: + raise ComponentAdapterError("component filesystem capacity is unavailable") from exc + return status.f_bavail * status.f_frsize + + +def _source_binding(value: object) -> SourceBinding: + document = _object(value, "portable source") + _exact_keys( + document, + {"schema_version", "observatory_job", "source", "camera_compute_job", "authority"}, + "portable source", + ) + if document["schema_version"] != SOURCE_SCHEMA or document["authority"] != AUTHORITY: + raise ComponentAdapterError("portable source contract changed") + observatory = _object(document["observatory_job"], "portable Observatory job") + source = _object(document["source"], "portable source identity") + camera = _object(document["camera_compute_job"], "portable camera identity") + _exact_keys( + observatory, + {"job_id", "request_sha256", "identity_sha256"}, + "portable Observatory job", + ) + _exact_keys( + source, + { + "session_id", + "catalog_sha256", + "bundle_sha256", + "capability_manifest_sha256", + "adapter_sha256", + }, + "portable source identity", + ) + _exact_keys( + camera, + { + "job_id", + "input_sha256", + "source_id", + "codec_epoch", + "input_byte_length", + "frame_count", + "timeline_start_seconds", + "timeline_end_seconds", + "generation_sha256", + "calibration_sha256", + }, + "portable camera identity", + ) + job_id = _string(observatory["job_id"], "Observatory job id") + if _OBSERVATORY_JOB_ID.fullmatch(job_id) is None: + raise ComponentAdapterError("Observatory job id is invalid") + for key in ("request_sha256", "identity_sha256"): + _digest(observatory[key], f"Observatory {key}") + for key in ( + "catalog_sha256", + "bundle_sha256", + "capability_manifest_sha256", + "adapter_sha256", + ): + _digest(source[key], f"portable source {key}") + session_id = _safe_component(source["session_id"], "portable source session id") + camera_job_id = _string(camera["job_id"], "camera job id") + if _CAMERA_JOB_ID.fullmatch(camera_job_id) is None: + raise ComponentAdapterError("camera job id is invalid") + camera_input_sha256 = _digest(camera["input_sha256"], "camera input sha256") + camera_source_id = _safe_component(camera["source_id"], "camera source id") + codec_epoch = _positive_int(camera["codec_epoch"], "camera codec epoch") + input_bytes = _positive_int(camera["input_byte_length"], "camera input bytes") + frame_count = _positive_int(camera["frame_count"], "camera frame count") + start = _finite_float(camera["timeline_start_seconds"], "camera timeline start") + end = _finite_float(camera["timeline_end_seconds"], "camera timeline end") + if ( + camera_source_id != CAMERA_SOURCE_ID + or camera.get("calibration_sha256") != CALIBRATION_SHA256 + or frame_count > MAX_FRAME_COUNT + or input_bytes > MAX_SOURCE_BYTES + or end <= start + ): + raise ComponentAdapterError("portable camera source is incompatible") + _digest(camera["generation_sha256"], "camera generation sha256") + return SourceBinding( + observatory_job_id=job_id, + source_session_id=session_id, + camera_job_id=camera_job_id, + camera_input_sha256=camera_input_sha256, + camera_source_id=camera_source_id, + codec_epoch=codec_epoch, + input_byte_length=input_bytes, + frame_count=frame_count, + timeline_start_seconds=start, + timeline_end_seconds=end, + ) + + +def _paths(value: object, component: Component) -> dict[str, str | None]: + document = _object(value, "component paths") + expected = { + "camera_job_root": "/run/nodedc/source", + "request": FIXED_REQUEST_PATH, + "output_root": "/run/nodedc/output", + "effective_ddrnet_config": ( + "/run/nodedc/effective-ddrnet-config.json" if component == "ddrnet" else None + ), + "eomt_result_root": "/run/nodedc/eomt-result" if component == "ddrnet" else None, + "decoded_frames_root": ( + "/run/nodedc/output/source-frames" + if component == "eomt" + else "/run/nodedc/eomt-result/source-frames" + ), + } + _exact_keys(document, set(expected), "component paths") + if document != expected: + raise ComponentAdapterError("component path contract changed") + return cast(dict[str, str | None], document) + + +def _asset_bindings( + value: object, + expectations: Sequence[AssetExpectation], +) -> dict[str, AssetBinding]: + if not isinstance(value, list): + raise ComponentAdapterError("component assets are not an array") + expected = {item.asset_id: item for item in expectations} + expected_ids = tuple(sorted(expected)) + rows: list[AssetBinding] = [] + for raw in value: + row = _object(raw, "component asset") + _exact_keys( + row, + {"asset_id", "path", "kind", "verification", "identity_sha256", "byte_length"}, + "component asset", + ) + asset_id = _safe_component(row["asset_id"], "component asset id") + expectation = expected.get(asset_id) + if expectation is None: + raise ComponentAdapterError("component asset set changed") + byte_length_value = row["byte_length"] + byte_length = ( + None + if byte_length_value is None + else _positive_int(byte_length_value, "component asset byte length") + ) + identity_sha256 = _digest(row["identity_sha256"], "component asset identity") + if ( + row["path"] != expectation.path + or row["kind"] != expectation.kind + or row["verification"] != expectation.verification + or (expectation.byte_length is not None and byte_length != expectation.byte_length) + or (expectation.kind == "tree" and byte_length is None) + or ( + expectation.identity_sha256 is not None + and identity_sha256 != expectation.identity_sha256 + ) + ): + raise ComponentAdapterError(f"component asset {asset_id} changed") + rows.append( + AssetBinding( + asset_id=asset_id, + path=expectation.path, + kind=expectation.kind, + verification=expectation.verification, + identity_sha256=identity_sha256, + byte_length=byte_length, + ) + ) + ids = tuple(row.asset_id for row in rows) + if ids != expected_ids or len(ids) != len(set(ids)): + raise ComponentAdapterError("component asset inventory is not canonical") + return {row.asset_id: row for row in rows} + + +def _tree_manifest_rows( + payload: bytes, + asset_id: str, +) -> tuple[tuple[PurePosixPath, int, str], ...]: + try: + text = payload.decode("utf-8") + except UnicodeDecodeError as exc: + raise ComponentAdapterError(f"asset {asset_id} manifest is not UTF-8") from exc + if not text.endswith("\n") or "\r" in text: + raise ComponentAdapterError(f"asset {asset_id} manifest encoding changed") + rows: list[tuple[PurePosixPath, int, str]] = [] + for encoded in text.removesuffix("\n").split("\n"): + fields = encoded.split("\t") + if len(fields) != 3: + raise ComponentAdapterError(f"asset {asset_id} manifest row changed") + relative = _tree_relative_path(fields[0], f"asset {asset_id} manifest path") + try: + byte_length = int(fields[1]) + except ValueError as exc: + raise ComponentAdapterError(f"asset {asset_id} manifest byte length changed") from exc + if byte_length < 0: + raise ComponentAdapterError(f"asset {asset_id} manifest byte length changed") + rows.append( + ( + relative, + byte_length, + _digest(fields[2], f"asset {asset_id} manifest sha256"), + ) + ) + names = tuple(row[0].as_posix() for row in rows) + if names != tuple(sorted(names)) or len(names) != len(set(names)): + raise ComponentAdapterError(f"asset {asset_id} manifest order changed") + return tuple(rows) + + +def _canonical_document( + path: Path, + *, + label: str, + maximum: int, + confinement_root: Path | None = None, +) -> tuple[bytes, dict[str, object]]: + payload = _read_bounded_file( + path, + label=label, + maximum=maximum, + confinement_root=confinement_root, + ) + try: + value: object = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ComponentAdapterError(f"{label} is invalid JSON") from exc + document = _object(value, label) + try: + canonical = canonical_json(document) + except (TypeError, ValueError) as exc: + raise ComponentAdapterError(f"{label} is not canonical JSON") from exc + if canonical != payload: + raise ComponentAdapterError(f"{label} is not canonical JSON") + return payload, document + + +def _canonical_or_pretty_document( + path: Path, + *, + label: str, + maximum: int, + confinement_root: Path | None = None, +) -> tuple[bytes, dict[str, object]]: + payload = _read_bounded_file( + path, + label=label, + maximum=maximum, + confinement_root=confinement_root, + ) + try: + value: object = json.loads(payload.decode("utf-8-sig")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ComponentAdapterError(f"{label} is invalid JSON") from exc + return payload, _object(value, label) + + +def _read_bounded_file( + path: Path, + *, + label: str, + maximum: int, + confinement_root: Path | None, +) -> bytes: + root = path.parent if confinement_root is None else confinement_root + resolved = _confined_file(path, root, label) + size = resolved.stat().st_size + if not 0 < size <= maximum: + raise ComponentAdapterError(f"{label} is outside bounds") + try: + return resolved.read_bytes() + except OSError as exc: + raise ComponentAdapterError(f"{label} is unavailable") from exc + + +def _confined_file(path: Path, root: Path, label: str) -> Path: + try: + resolved_root = root.resolve(strict=True) + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise ComponentAdapterError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(resolved_root) + ): + raise ComponentAdapterError(f"{label} is not a confined regular file") + return resolved + + +def _real_file(path: Path, label: str) -> Path: + return _confined_file(path, path.parent, label) + + +def _real_directory(path: Path, label: str) -> Path: + try: + metadata = path.lstat() + resolved = path.resolve(strict=True) + except OSError as exc: + raise ComponentAdapterError(f"{label} is unavailable") from exc + if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISDIR(metadata.st_mode): + raise ComponentAdapterError(f"{label} is not a real directory") + return resolved + + +def _relative_path(value: object, label: str) -> PurePosixPath: + if not isinstance(value, str) or not value or "\\" in value: + raise ComponentAdapterError(f"{label} is invalid") + relative = PurePosixPath(value) + if relative.is_absolute() or ".." in relative.parts or not relative.parts: + raise ComponentAdapterError(f"{label} is unsafe") + if any(_SAFE_COMPONENT.fullmatch(part) is None for part in relative.parts): + raise ComponentAdapterError(f"{label} is unsafe") + return relative + + +def _tree_relative_path(value: object, label: str) -> PurePosixPath: + """Parse an install-time tree member without constraining valid filenames. + + Tree payloads can contain ordinary UTF-8 names with spaces and punctuation. + They still cannot express traversal, platform-dependent separators, control + characters, or filesystem components beyond conservative portable bounds. + """ + + if not isinstance(value, str) or not value or "\\" in value: + raise ComponentAdapterError(f"{label} is invalid") + try: + encoded = value.encode("utf-8") + except UnicodeEncodeError as exc: + raise ComponentAdapterError(f"{label} is invalid UTF-8") from exc + components = value.split("/") + if ( + len(encoded) > MAX_TREE_PATH_BYTES + or not 1 <= len(components) <= MAX_TREE_COMPONENTS + or any( + not component + or component in {".", ".."} + or len(component.encode("utf-8")) > MAX_TREE_COMPONENT_BYTES + or any(unicodedata.category(character) == "Cc" for character in component) + for component in components + ) + ): + raise ComponentAdapterError(f"{label} is unsafe") + relative = PurePosixPath(*components) + if relative.is_absolute() or relative.as_posix() != value: + raise ComponentAdapterError(f"{label} is unsafe") + return relative + + +def _fixed_absolute_path(value: str, label: str) -> None: + path = PurePosixPath(value) + if ( + not path.is_absolute() + or ".." in path.parts + or len(path.parts) < 2 + or any(_SAFE_COMPONENT.fullmatch(part) is None for part in path.parts[1:]) + ): + raise ValueError(f"{label} is invalid") + + +def _object(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise ComponentAdapterError(f"{label} is not an object") + return cast(dict[str, object], value) + + +def _exact_keys(value: Mapping[str, object], expected: set[str], label: str) -> None: + if set(value) != expected: + raise ComponentAdapterError(f"{label} fields changed") + + +def _string(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise ComponentAdapterError(f"{label} is invalid") + return value + + +def _safe_component(value: object, label: str) -> str: + result = _string(value, label) + if _SAFE_COMPONENT.fullmatch(result) is None: + raise ComponentAdapterError(f"{label} is unsafe") + return result + + +def _digest(value: object, label: str) -> str: + if not isinstance(value, str) or _SHA256.fullmatch(value) is None: + raise ComponentAdapterError(f"{label} is invalid") + return value + + +def _positive_int(value: object, label: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ComponentAdapterError(f"{label} is invalid") + return value + + +def _finite_float(value: object, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + ): + raise ComponentAdapterError(f"{label} is invalid") + return float(value) + + +__all__ = [ + "AUTHORITY", + "AssetExpectation", + "CALIBRATION_SHA256", + "CommandRunner", + "ComponentAdapterError", + "ComponentRequest", + "DDRNET_CHECKPOINT_SHA256", + "EOMT_DEPENDENCY_IDENTITY_SHA256", + "EOMT_PIPELINE", + "FIXED_REQUEST_PATH", + "GOOSE_MAPPING_SHA256", + "RuntimeLayout", + "VALID_FOV_IDENTITY_SHA256", + "VALID_FOV_MASK_SHA256", + "available_bytes", + "canonical_json", + "load_canonical_json", + "load_component_request", + "load_json", + "parse_fixed_request_argument", + "run_command", + "sha256_file", + "validate_camera_compute_job", + "validate_critical_tree_file", + "validate_file_asset", + "validate_fixed_result", + "validate_grayscale_png_payload", + "validate_identity_manifest", + "validate_tree_asset", + "validate_tree_receipt", +] diff --git a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py new file mode 100644 index 0000000..2d16642 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py @@ -0,0 +1,828 @@ +#!/usr/bin/env python3 +"""Run the sealed LAB V1 DDRNet component over EoMT-decoded K1 frames.""" + +from __future__ import annotations + +import csv +import hashlib +import io +import json +import os +import shutil +import stat +import sys +import zipfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +from PIL import Image +from portable_lab_v1_component_adapter import ( + DDRNET_CHECKPOINT_SHA256, + EOMT_PIPELINE, + GOOSE_MAPPING_SHA256, + AssetExpectation, + CommandRunner, + ComponentAdapterError, + ComponentRequest, + RuntimeLayout, + canonical_json, + load_canonical_json, + load_component_request, + load_json, + parse_fixed_request_argument, + run_command, + sha256_file, + validate_camera_compute_job, + validate_file_asset, + validate_fixed_result, + validate_grayscale_png_payload, +) + +DDRNET_RESULT_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-run/v1" +EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2" +EFFECTIVE_CONFIG_SCHEMA: Final = "missioncore.lab-v1-goose-vegetation-benchmark/v1" +DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1" +DDRNET_CANDIDATE_ID: Final = "goose-ddrnet-class-512" +DDRNET_LOADED_MODEL: Final = "ddrnet_39" +DDRNET_CHECKPOINT_BYTES: Final = 259_419_077 +DDRNET_STATIC_CONFIG_SHA256: Final = ( + "3c2a60def178d824e6ce612cad9a108da496de769edf0de66099d2ecca6b51bb" +) +RUNNER_SHA256: Final = "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1" +POLICY_SHA256: Final = "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35" +PROVIDER_MAP_SHA256: Final = "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352" +DDRNET_AUTHORITY: Final = { + "navigation_accepted": False, + "safety_accepted": False, + "actuation_accepted": False, + "camera_semantics_can_clear_rigid_geometry": False, +} +EOMT_MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024" +EOMT_MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f" +EOMT_MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation" +SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1" + + +@dataclass(frozen=True, slots=True) +class SourceFrameRow: + sequence: int + path: str + byte_length: int + sha256: str + + +DDRNET_ASSETS: Final = ( + AssetExpectation( + "ddrnet-checkpoint", + "/opt/nodedc/assets/ddrnet-checkpoint", + "file", + "sha256", + DDRNET_CHECKPOINT_SHA256, + DDRNET_CHECKPOINT_BYTES, + ), + AssetExpectation( + "ddrnet-goose-mapping", + "/opt/nodedc/assets/ddrnet-goose-mapping", + "file", + "sha256", + GOOSE_MAPPING_SHA256, + ), + AssetExpectation( + "ddrnet-goose-runner", + "/opt/nodedc/assets/ddrnet-goose-runner", + "file", + "sha256", + RUNNER_SHA256, + 32_877, + ), + AssetExpectation( + "vegetation-policy", + "/opt/nodedc/assets/vegetation-policy", + "file", + "sha256", + POLICY_SHA256, + 3_022, + ), + AssetExpectation( + "vegetation-provider-map", + "/opt/nodedc/assets/vegetation-provider-map", + "file", + "sha256", + PROVIDER_MAP_SHA256, + 2_756, + ), +) + + +def execute_ddrnet_component( + *, + request_path: Path, + layout: RuntimeLayout, + command_runner: CommandRunner = run_command, + work_root: Path = Path("/tmp/nodedc-lab-v1-ddrnet"), +) -> None: + request = load_component_request( + request_path, + component="ddrnet", + expectations=DDRNET_ASSETS, + ) + validate_camera_compute_job(layout.camera_job_root, request.source) + assets = { + item.asset_id: validate_file_asset(request, layout, item.asset_id) for item in DDRNET_ASSETS + } + effective = _validate_effective_config( + layout.effective_ddrnet_config, + request, + ) + eomt_root, source_frames, decode_repair = _validate_eomt_input( + layout.eomt_result_root, + request, + ) + output = _empty_output_root(layout.output_root) + workspace = _prepare_workspace(work_root) + try: + frames_root = _snapshot_source_frames( + eomt_root, + source_frames, + workspace / "source-frames", + ) + mapping_copy = workspace / "goose_label_mapping.csv" + shutil.copyfile(assets["ddrnet-goose-mapping"], mapping_copy) + os.chmod(mapping_copy, 0o400) + if sha256_file(mapping_copy) != GOOSE_MAPPING_SHA256: + raise ComponentAdapterError("DDRNet private mapping copy changed") + staging = output / ".ddrnet-component" + argv = ( + sys.executable, + str(assets["ddrnet-goose-runner"]), + "--mode", + "ravnoves-video", + "--candidate", + "ddrnet", + "--config", + str(layout.effective_ddrnet_config), + "--policy", + str(assets["vegetation-policy"]), + "--provider-map", + str(assets["vegetation-provider-map"]), + "--checkpoint", + str(assets["ddrnet-checkpoint"]), + "--dataset-root", + str(workspace), + "--frames-root", + str(frames_root), + "--output", + str(staging), + "--limit", + "0", + "--visual-count", + "0", + ) + command_runner(argv, _ddrnet_environment()) + shutil.copyfile(eomt_root / "decode-repair.json", staging / "decode-repair.json") + if ( + load_json( + staging / "decode-repair.json", + label="DDRNet decode repair", + maximum=1024 * 1024, + confinement_root=staging, + ) + != decode_repair + ): + raise ComponentAdapterError("DDRNet decode repair copy changed") + _validate_ddrnet_result( + staging, + request, + effective, + mapping_path=assets["ddrnet-goose-mapping"], + ) + _publish_component_children( + staging, + output, + expected_names={ + "decode-repair.json", + "result.json", + "semantic-masks.zip", + }, + ) + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +def _validate_effective_config( + path: Path, + request: ComponentRequest, +) -> dict[str, object]: + config = load_canonical_json( + path, + label="effective DDRNet config", + maximum=1024 * 1024, + ) + if ( + request.effective_ddrnet_config_sha256 is None + or sha256_file(path) != request.effective_ddrnet_config_sha256 + ): + raise ComponentAdapterError("effective DDRNet config identity changed") + expected_keys = { + "schema_version", + "lab_id", + "worker_id", + "runtime", + "dataset", + "candidates", + "vegetation_class_names", + "visual_case_contract", + "policy_action_colors", + "invariants", + "ravnoves", + } + if set(config) != expected_keys: + raise ComponentAdapterError("effective DDRNet config fields changed") + static = dict(config) + ravnoves = static.pop("ravnoves") + if ( + config.get("schema_version") != EFFECTIVE_CONFIG_SCHEMA + or config.get("lab_id") != "LAB-V1" + or config.get("worker_id") != "worker-006" + or hashlib.sha256(canonical_json(static)).hexdigest() != DDRNET_STATIC_CONFIG_SHA256 + ): + raise ComponentAdapterError("effective DDRNet static configuration changed") + source = request.source + expected_ravnoves = { + "source_id": ( + f"portable-k1/{source.source_session_id}/" + f"{source.camera_source_id}@{source.camera_input_sha256}" + ), + "source_sha256": source.camera_input_sha256, + "base_m4_result_id": None, + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": source.frame_count, + "frame_indices": _representative_frame_indices(source.frame_count), + "crop_contract": "center-600-square-to-512; outside-crop-is-undefined", + } + if ravnoves != expected_ravnoves: + raise ComponentAdapterError("effective DDRNet source configuration changed") + return config + + +def _validate_eomt_input( + root: Path, + request: ComponentRequest, +) -> tuple[Path, tuple[SourceFrameRow, ...], dict[str, object]]: + if root.is_symlink(): + raise ComponentAdapterError("EoMT result root is a symbolic link") + resolved = root.resolve(strict=True) + result = validate_fixed_result( + resolved, + schema_version=EOMT_RESULT_SCHEMA, + label="EoMT", + ) + identity = result.get("identity") + if not isinstance(identity, dict): + raise ComponentAdapterError("EoMT result identity is unavailable") + configuration = identity.get("configuration") + models = identity.get("models") + semantic = models.get("semantic") if isinstance(models, dict) else None + observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest() + source = request.source + if ( + not isinstance(configuration, dict) + or result.get("identity_sha256") != observed_identity + or result.get("result_id") != f"result-{observed_identity}" + or result.get("job_id") != source.camera_job_id + or result.get("input_sha256") != source.camera_input_sha256 + or result.get("session_id") != source.source_session_id + or result.get("source_id") != source.camera_source_id + or result.get("codec_epoch") != source.codec_epoch + or result.get("timestamp_basis") != "session-time-seconds" + or result.get("timeline_start_seconds") != source.timeline_start_seconds + or result.get("timeline_end_seconds") != source.timeline_end_seconds + or result.get("frames_processed") != source.frame_count + or result.get("ground_truth") is not False + or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2" + or configuration.get("pipeline") != EOMT_PIPELINE + or not isinstance(semantic, dict) + or semantic.get("id") != EOMT_MODEL_ID + or semantic.get("revision") != EOMT_MODEL_REVISION + or semantic.get("architecture") != EOMT_MODEL_ARCHITECTURE + ): + raise ComponentAdapterError("EoMT source result changed") + artifacts = result.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) != 6: + raise ComponentAdapterError("EoMT source artifact set changed") + expected_artifacts = { + "panoptic-overlay-video": ("perception.mp4", "video/mp4"), + "panoptic-mask-archive": ("masks.tar.gz", "application/gzip"), + "panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"), + "worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"), + "perception-run-report": ("run-report.json", "application/json"), + "decoded-source-frame-manifest": ("source-frames.json", "application/json"), + } + observed_kinds: set[str] = set() + for value in artifacts: + if not isinstance(value, dict): + raise ComponentAdapterError("EoMT source artifact is invalid") + kind = value.get("kind") + encoded = value.get("path") + if ( + not isinstance(kind, str) + or kind in observed_kinds + or expected_artifacts.get(kind) != (encoded, value.get("media_type")) + or not isinstance(encoded, str) + or not encoded + or "/" in encoded + or "\\" in encoded + ): + raise ComponentAdapterError("EoMT source artifact path changed") + observed_kinds.add(kind) + artifact = _confined_regular_file(resolved / encoded, resolved, "EoMT artifact") + if ( + artifact.stat().st_size < 1 + or value.get("byte_length") != artifact.stat().st_size + or value.get("sha256") != sha256_file(artifact) + ): + raise ComponentAdapterError("EoMT source artifact identity changed") + if observed_kinds != set(expected_artifacts): + raise ComponentAdapterError("EoMT source artifact roles changed") + manifest_path = _confined_regular_file( + resolved / "source-frames.json", + resolved, + "EoMT source frame manifest", + ) + source_frame_binding = identity.get("source_frames") + expected_binding = { + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "path": "source-frames.json", + "sha256": sha256_file(manifest_path), + "byte_length": manifest_path.stat().st_size, + "frame_count": source.frame_count, + } + if source_frame_binding != expected_binding: + raise ComponentAdapterError("EoMT source frame manifest binding changed") + source_frames = _source_frame_manifest(manifest_path, resolved, request) + repair = load_json( + resolved / "decode-repair.json", + label="EoMT decode repair", + maximum=1024 * 1024, + confinement_root=resolved, + ) + _validate_decode_repair(repair, source.frame_count) + return resolved, source_frames, repair + + +def _source_frame_manifest( + path: Path, + eomt_root: Path, + request: ComponentRequest, +) -> tuple[SourceFrameRow, ...]: + document = load_canonical_json( + path, + label="EoMT source frame manifest", + maximum=32 * 1024 * 1024, + confinement_root=eomt_root, + ) + source = request.source + if set(document) != { + "schema_version", + "job_id", + "input_sha256", + "source_id", + "codec_epoch", + "frame_count", + "frames", + } or document != { + **document, + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "job_id": source.camera_job_id, + "input_sha256": source.camera_input_sha256, + "source_id": source.camera_source_id, + "codec_epoch": source.codec_epoch, + "frame_count": source.frame_count, + }: + raise ComponentAdapterError("EoMT source frame manifest identity changed") + raw_rows = document["frames"] + if not isinstance(raw_rows, list) or len(raw_rows) != source.frame_count: + raise ComponentAdapterError("EoMT source frame manifest row count changed") + rows: list[SourceFrameRow] = [] + for sequence, value in enumerate(raw_rows, start=1): + if not isinstance(value, dict) or set(value) != { + "sequence", + "path", + "byte_length", + "sha256", + }: + raise ComponentAdapterError("EoMT source frame manifest row changed") + encoded = f"source-frames/frame-{sequence:06d}.png" + byte_length = value.get("byte_length") + digest = value.get("sha256") + if ( + value.get("sequence") != sequence + or value.get("path") != encoded + or isinstance(byte_length, bool) + or not isinstance(byte_length, int) + or not 0 < byte_length <= 16 * 1024 * 1024 + or not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise ComponentAdapterError("EoMT source frame manifest row identity changed") + frame = _confined_regular_file( + eomt_root / encoded, + eomt_root, + "EoMT source frame", + ) + if frame.stat().st_size != byte_length: + raise ComponentAdapterError("EoMT source frame length changed") + rows.append(SourceFrameRow(sequence, encoded, byte_length, digest)) + frames_root = eomt_root / "source-frames" + expected_names = tuple( + f"frame-{sequence:06d}.png" for sequence in range(1, source.frame_count + 1) + ) + try: + observed_names = tuple(sorted(candidate.name for candidate in frames_root.iterdir())) + except OSError as exc: + raise ComponentAdapterError("EoMT source frame set is unavailable") from exc + if observed_names != expected_names: + raise ComponentAdapterError("EoMT source frame set changed") + return tuple(rows) + + +def _snapshot_source_frames( + eomt_root: Path, + rows: Sequence[SourceFrameRow], + snapshot_root: Path, +) -> Path: + snapshot_root.mkdir(mode=0o700) + for row in rows: + source = _confined_regular_file( + eomt_root / row.path, + eomt_root, + "EoMT source frame", + ) + destination = snapshot_root / f"frame-{row.sequence:06d}.png" + digest = hashlib.sha256() + byte_length = 0 + signature = b"" + descriptor = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + try: + opened = os.fstat(descriptor) + admitted = source.stat() + if not stat.S_ISREG(opened.st_mode) or (opened.st_dev, opened.st_ino) != ( + admitted.st_dev, + admitted.st_ino, + ): + raise ComponentAdapterError("EoMT source frame changed during admission") + with ( + os.fdopen(descriptor, "rb", closefd=False) as source_stream, + destination.open("xb") as destination_stream, + ): + while block := source_stream.read(1024 * 1024): + if not signature: + signature = block[:8] + byte_length += len(block) + if byte_length > row.byte_length: + raise ComponentAdapterError("EoMT source frame grew during admission") + digest.update(block) + destination_stream.write(block) + destination_stream.flush() + os.fsync(destination_stream.fileno()) + finally: + os.close(descriptor) + if ( + signature != b"\x89PNG\r\n\x1a\n" + or byte_length != row.byte_length + or digest.hexdigest() != row.sha256 + ): + raise ComponentAdapterError("EoMT source frame identity changed") + destination.chmod(0o400) + return snapshot_root.resolve(strict=True) + + +def _validate_decode_repair(document: Mapping[str, object], frame_count: int) -> None: + if set(document) != { + "schema_version", + "decoder", + "packets_requested", + "frames_decoded", + "repaired_frame_count", + "repairs", + }: + raise ComponentAdapterError("decode repair fields changed") + decoded = document.get("frames_decoded") + repaired = document.get("repaired_frame_count") + repairs = document.get("repairs") + if ( + document.get("schema_version") != DECODE_REPAIR_SCHEMA + or document.get("decoder") != "ffmpeg-h264_cuvid-output-corrupt" + or document.get("packets_requested") != frame_count + or isinstance(decoded, bool) + or not isinstance(decoded, int) + or isinstance(repaired, bool) + or not isinstance(repaired, int) + or repaired not in (0, 1) + or decoded + repaired != frame_count + or not isinstance(repairs, list) + or len(repairs) != repaired + ): + raise ComponentAdapterError("decode repair contract changed") + for value in repairs: + if ( + not isinstance(value, dict) + or set(value) != {"sequence", "packet_pts", "method"} + or isinstance(value.get("sequence"), bool) + or not isinstance(value.get("sequence"), int) + or not 2 <= cast(int, value["sequence"]) <= frame_count + or isinstance(value.get("packet_pts"), bool) + or not isinstance(value.get("packet_pts"), int) + or value.get("method") != "duplicate-previous-decoded-frame" + ): + raise ComponentAdapterError("decode repair entry changed") + + +def _validate_ddrnet_result( + root: Path, + request: ComponentRequest, + effective: Mapping[str, object], + *, + mapping_path: Path, +) -> None: + result = validate_fixed_result( + root, + schema_version=DDRNET_RESULT_SCHEMA, + label="DDRNet", + ) + candidate = result.get("candidate") + source = result.get("source") + semantics = result.get("video_semantics") + provenance = result.get("provenance") + authority = result.get("authority") + ravnoves = effective.get("ravnoves") + if not all( + isinstance(value, dict) for value in (candidate, source, semantics, provenance, ravnoves) + ): + raise ComponentAdapterError("DDRNet result contract is incomplete") + candidate = cast(dict[str, object], candidate) + source = cast(dict[str, object], source) + semantics = cast(dict[str, object], semantics) + provenance = cast(dict[str, object], provenance) + ravnoves = cast(dict[str, object], ravnoves) + archive = semantics.get("mask_archive") + if not isinstance(archive, dict): + raise ComponentAdapterError("DDRNet mask archive contract is unavailable") + if ( + result.get("lab_id") != "LAB-V1" + or result.get("worker_id") != "worker-006" + or result.get("mode") != "ravnoves-video" + or candidate.get("candidate_id") != DDRNET_CANDIDATE_ID + or candidate.get("candidate_key") != "ddrnet" + or candidate.get("loaded_model_name") != DDRNET_LOADED_MODEL + or candidate.get("checkpoint_size_bytes") != DDRNET_CHECKPOINT_BYTES + or candidate.get("checkpoint_sha256") != DDRNET_CHECKPOINT_SHA256 + or source.get("source_id") != ravnoves.get("source_id") + or source.get("input_count") != request.source.frame_count + or source.get("ground_truth_available") is not False + or source.get("mapping_sha256") != GOOSE_MAPPING_SHA256 + or archive.get("path") != "semantic-masks.zip" + or archive.get("media_type") != "application/zip" + or archive.get("frame_count") != request.source.frame_count + or archive.get("width") != 800 + or archive.get("height") != 600 + or archive.get("encoding") != "uint8-class-id-png" + or archive.get("sequence_binding") != "sequence-0-to-masks/frame-000001.png" + or semantics.get("center_crop_xyxy") != [100, 0, 700, 600] + or semantics.get("outside_crop_state") != "undefined" + or semantics.get("base_m4_result_id") is not None + or provenance.get("config_sha256") != request.effective_ddrnet_config_sha256 + or provenance.get("policy_sha256") != POLICY_SHA256 + or provenance.get("provider_map_sha256") != PROVIDER_MAP_SHA256 + or authority != DDRNET_AUTHORITY + or result.get("visual_cases") != [] + ): + raise ComponentAdapterError("DDRNet result identity changed") + archive_path = _confined_regular_file( + root / "semantic-masks.zip", + root, + "DDRNet semantic mask archive", + ) + if archive.get("byte_length") != archive_path.stat().st_size or archive.get( + "sha256" + ) != sha256_file(archive_path): + raise ComponentAdapterError("DDRNet semantic mask archive changed") + observed_histogram = _validate_mask_archive_inventory( + archive_path, + request.source.frame_count, + ) + aggregate = semantics.get("aggregate_prediction_pixels") + if ( + not isinstance(aggregate, list) + or len(aggregate) != 64 + or any(isinstance(value, bool) or not isinstance(value, int) for value in aggregate) + or tuple(aggregate) != observed_histogram + or semantics.get("taxonomy") != _sealed_taxonomy(mapping_path) + ): + raise ComponentAdapterError("DDRNet semantic statistics changed") + identity_value = { + "schema_version": result["schema_version"], + "candidate": result["candidate"], + "source": result["source"], + "video_semantics": result["video_semantics"], + "preprocessing": result["preprocessing"], + "metrics": result["metrics"], + "timing": result["timing"], + "resource": result["resource"], + "visual_cases": result["visual_cases"], + "authority": result["authority"], + "config_sha256": provenance["config_sha256"], + "policy_sha256": provenance["policy_sha256"], + "provider_map_sha256": provenance["provider_map_sha256"], + } + identity_sha256 = hashlib.sha256( + json.dumps(identity_value, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + if result.get("result_id") != f"lab-v1-ravnoves-video-ddrnet-{identity_sha256}": + raise ComponentAdapterError("DDRNet result id changed") + repair = load_json( + root / "decode-repair.json", + label="DDRNet decode repair", + maximum=1024 * 1024, + confinement_root=root, + ) + _validate_decode_repair(repair, request.source.frame_count) + + +def _validate_mask_archive_inventory(path: Path, frame_count: int) -> tuple[int, ...]: + expected = tuple(f"masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1)) + histogram = [0] * 64 + try: + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + names = tuple(info.filename for info in infos) + if ( + names != expected + or len(names) != len(set(names)) + or any( + info.is_dir() + or info.compress_type != zipfile.ZIP_STORED + or info.flag_bits & 0x1 + or stat.S_ISLNK(info.external_attr >> 16) + or not 0 < info.file_size <= 2 * 1024 * 1024 + or info.compress_size != info.file_size + for info in infos + ) + ): + raise ComponentAdapterError("DDRNet semantic mask archive inventory changed") + for info in infos: + with archive.open(info, mode="r") as stream: + payload = stream.read(info.file_size + 1) + if len(payload) != info.file_size: + raise ComponentAdapterError("DDRNet semantic mask archive member changed") + validate_grayscale_png_payload( + payload, + width=800, + height=600, + label=f"DDRNet mask {info.filename}", + ) + with Image.open(io.BytesIO(payload)) as image: + if image.format != "PNG" or image.mode != "L" or image.size != (800, 600): + raise ComponentAdapterError("DDRNet semantic mask raster changed") + image.load() + counts = image.histogram() + if len(counts) != 256 or any(counts[64:]): + raise ComponentAdapterError("DDRNet semantic mask class id escaped 0..63") + for class_id, count in enumerate(counts[:64]): + histogram[class_id] += count + except (OSError, zipfile.BadZipFile) as exc: + raise ComponentAdapterError("DDRNet semantic mask archive is invalid") from exc + if sum(histogram) != frame_count * 800 * 600: + raise ComponentAdapterError("DDRNet semantic mask pixel count changed") + return tuple(histogram) + + +def _sealed_taxonomy(mapping_path: Path) -> dict[str, object]: + classes: dict[int, dict[str, object]] = {} + try: + with mapping_path.open(newline="", encoding="utf-8-sig") as stream: + reader = csv.DictReader(stream) + if reader.fieldnames is None or not {"label_key", "class_name", "hex"}.issubset( + reader.fieldnames + ): + raise ComponentAdapterError("sealed GOOSE mapping columns changed") + for row in reader: + try: + class_id = int(row["label_key"]) + color = bytes.fromhex(row["hex"].removeprefix("#")) + except (KeyError, TypeError, ValueError) as exc: + raise ComponentAdapterError("sealed GOOSE mapping row changed") from exc + label = row.get("class_name") + if ( + not 0 <= class_id < 64 + or class_id in classes + or not isinstance(label, str) + or not label + or len(color) != 3 + ): + raise ComponentAdapterError("sealed GOOSE taxonomy changed") + classes[class_id] = { + "class_id": class_id, + "label": label, + "color_rgb": list(color), + "disposition": "undefined" if class_id == 0 else "prediction", + } + except OSError as exc: + raise ComponentAdapterError("sealed GOOSE mapping is unavailable") from exc + if set(classes) != set(range(64)): + raise ComponentAdapterError("sealed GOOSE mapping does not cover 64 classes") + return { + "schema_version": "missioncore.lab-v1-vegetation-taxonomy/v1", + "classes": [classes[class_id] for class_id in range(64)], + } + + +def _ddrnet_environment() -> dict[str, str]: + retained = { + key: value + for key, value in os.environ.items() + if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"} + } + return { + **retained, + "HOME": "/tmp", + "PYTHONDONTWRITEBYTECODE": "1", + } + + +def _empty_output_root(path: Path) -> Path: + if path.is_symlink(): + raise ComponentAdapterError("DDRNet output root is a symbolic link") + resolved = path.resolve(strict=True) + if not resolved.is_dir() or any(resolved.iterdir()): + raise ComponentAdapterError("DDRNet output root is not an empty real directory") + return resolved + + +def _prepare_workspace(path: Path) -> Path: + if path.exists() or path.is_symlink(): + raise ComponentAdapterError("DDRNet workspace already exists") + path.mkdir(mode=0o700, parents=False) + return path.resolve(strict=True) + + +def _confined_regular_file(path: Path, root: Path, label: str) -> Path: + try: + metadata = path.lstat() + resolved_root = root.resolve(strict=True) + resolved = path.resolve(strict=True) + except OSError as exc: + raise ComponentAdapterError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(resolved_root) + ): + raise ComponentAdapterError(f"{label} is not a confined regular file") + return resolved + + +def _publish_component_children( + staging: Path, + output: Path, + *, + expected_names: set[str], +) -> None: + children = tuple(staging.iterdir()) + if {child.name for child in children} != expected_names or any( + child.is_dir() or child.is_symlink() for child in children + ): + raise ComponentAdapterError("DDRNet staging contains unexpected directories") + for child in children: + destination = output / child.name + if destination.exists() or destination.is_symlink(): + raise ComponentAdapterError("DDRNet publication target already exists") + os.replace(child, destination) + staging.rmdir() + + +def _representative_frame_indices(frame_count: int) -> list[int]: + if frame_count <= 12: + return list(range(frame_count)) + return sorted({round(index * (frame_count - 1) / 11) for index in range(12)}) + + +def main(argv: Sequence[str] | None = None) -> int: + request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv) + execute_ddrnet_component( + request_path=request_path, + layout=RuntimeLayout.fixed(DDRNET_ASSETS), + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ComponentAdapterError as exc: + print(f"portable LAB V1 DDRNet rejected: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py new file mode 100644 index 0000000..99332f5 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py @@ -0,0 +1,987 @@ +#!/usr/bin/env python3 +"""Run the sealed LAB V1 EoMT component over one CameraComputeJob.""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import stat +import sys +import tarfile +import time +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Final, cast + +from portable_lab_v1_component_adapter import ( + EOMT_DEPENDENCY_IDENTITY_SHA256, + EOMT_PIPELINE, + VALID_FOV_IDENTITY_SHA256, + VALID_FOV_MASK_SHA256, + AssetExpectation, + CommandRunner, + ComponentAdapterError, + ComponentRequest, + RuntimeLayout, + available_bytes, + canonical_json, + load_component_request, + load_json, + parse_fixed_request_argument, + run_command, + sha256_file, + validate_camera_compute_job, + validate_critical_tree_file, + validate_fixed_result, + validate_grayscale_png_payload, + validate_identity_manifest, + validate_tree_asset, + validate_tree_receipt, +) + +EOMT_RESULT_SCHEMA: Final = "missioncore.recorded-perception-result/v2" +DEPENDENCY_SCHEMA: Final = "missioncore.k1-e3-dependency-manifest/v1" +VALID_FOV_SCHEMA: Final = "missioncore.k1-valid-fov-mask/v1" +DECODE_REPAIR_SCHEMA: Final = "missioncore.recorded-video-decode-repair/v1" +SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-frames/v1" +MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f" +MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024" +MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation" +DISK_FLOOR_BYTES: Final = 360 * 1024**3 +FFMPEG_TREE_SOURCE_IMAGE_SHA256: Final = ( + "8a364092b03561b9c08ac00730206e363a53d07ea0304f7d543b403b65432b5e" +) +FFMPEG_TREE_IDENTITY_SHA256: Final = ( + "03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69" +) +FFMPEG_TREE_BYTE_LENGTH: Final = 256_208_352 +FFMPEG_SHA256: Final = "7c6e870c57d16a755d7255cb93b65379218f47d3208a3f9b107f3a80646af6af" +FFPROBE_SHA256: Final = "39e7b15b94ef2d37e9fcf052ad1cb1e6550fdb6c6ac6765c5f1aba53cc367929" + +EOMT_ASSETS: Final = ( + AssetExpectation( + "eomt-environment", + "/environment", + "tree", + "identity-sha256", + ), + AssetExpectation( + "eomt-ffmpeg-runtime", + "/opt/ffmpeg", + "tree", + "identity-sha256", + FFMPEG_TREE_IDENTITY_SHA256, + FFMPEG_TREE_BYTE_LENGTH, + ), + AssetExpectation( + "eomt-model-cache", + "/cache", + "tree", + "identity-sha256", + ), + AssetExpectation( + "eomt-python-environment", + "/opt/env", + "tree", + "identity-sha256", + ), + AssetExpectation( + "eomt-runner-bundle", + "/runner", + "tree", + "identity-sha256", + ), + AssetExpectation( + "eomt-transformers-environment", + "/opt/transformers", + "tree", + "identity-sha256", + ), + AssetExpectation( + "k1-valid-fov-root", + "/valid-fov", + "tree", + "identity-sha256", + ), +) + +_RUNNER_FILES: Final = { + "run_e4_full_session_segmentation.py": ( + 30_720, + "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", + ), + "run_e3_rectified_segmentation.py": ( + 45_789, + "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6", + ), + "run_recorded_perception_epoch.py": ( + 34_899, + "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675", + ), + "run_evaluation_prelabels.py": ( + 28_915, + "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4", + ), + "e3_k1_camera1_profile.json": ( + 3_805, + "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875", + ), +} +_MODEL_FILES: Final = { + "config.json": ( + 1_575, + "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650", + ), + "preprocessor_config.json": ( + 666, + "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7", + ), + "model.safetensors": ( + 1_276_175_488, + "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782", + ), +} + + +def execute_eomt_component( + *, + request_path: Path, + layout: RuntimeLayout, + command_runner: CommandRunner = run_command, + work_root: Path | None = None, + disk_floor_bytes: int = DISK_FLOOR_BYTES, +) -> None: + request = load_component_request( + request_path, + component="eomt", + expectations=EOMT_ASSETS, + ) + input_document = validate_camera_compute_job(layout.camera_job_root, request.source) + roots = _validate_release_assets(request, layout) + output = _empty_output_root(layout.output_root) + reserve = request.source.frame_count * 800 * 600 * 7 + request.source.input_byte_length + free_before = available_bytes(output) + if disk_floor_bytes < 0 or free_before < disk_floor_bytes + reserve: + raise ComponentAdapterError("EoMT output does not satisfy its disk reserve") + workspace = _prepare_workspace(output / ".eomt-work" if work_root is None else work_root) + try: + total_started = time.perf_counter() + extract_started = time.perf_counter() + frames_root, timeline_path, decode_repair = _decode_camera_epoch( + request=request, + input_document=input_document, + camera_job_root=layout.camera_job_root, + output_root=output, + work_root=workspace, + ffmpeg_root=roots["eomt-ffmpeg-runtime"], + command_runner=command_runner, + ) + extract_seconds = _elapsed(extract_started) + free_post_extract = available_bytes(output) + if free_post_extract < disk_floor_bytes: + raise ComponentAdapterError("EoMT frame extraction crossed its disk floor") + + inference_root = output / ".eomt-component" + runner = roots["eomt-runner-bundle"] / "run_e4_full_session_segmentation.py" + profile = roots["eomt-runner-bundle"] / "e3_k1_camera1_profile.json" + environment = _eomt_environment(roots) + run_argv = ( + sys.executable, + str(runner), + "run", + "--job", + str(layout.camera_job_root), + "--profile", + str(profile), + "--valid-fov-root", + str(roots["k1-valid-fov-root"]), + "--frames", + str(frames_root), + "--timeline", + str(timeline_path), + "--cache", + str(roots["eomt-model-cache"]), + "--environment", + str(roots["eomt-environment"]), + "--output", + str(inference_root), + "--frame-limit", + str(request.source.frame_count), + "--free-bytes-floor", + str(disk_floor_bytes), + "--orchestrator-sha256", + sha256_file(Path(__file__).resolve(strict=True)), + "--container-image", + f"sha256:{request.component_image_sha256}", + "--telemetry-interval-seconds", + "1", + ) + command_runner(run_argv, environment) + _write_json(inference_root / "decode-repair.json", decode_repair) + free_post_inference = available_bytes(output) + if free_post_inference < disk_floor_bytes: + raise ComponentAdapterError("EoMT inference crossed its disk floor") + + video_path = inference_root / "perception.mp4" + encode_started = time.perf_counter() + duration = request.source.timeline_end_seconds - request.source.timeline_start_seconds + fps = request.source.frame_count / duration + encode_argv = ( + str(roots["eomt-ffmpeg-runtime"] / "bin" / "ffmpeg"), + "-hide_banner", + "-loglevel", + "error", + "-framerate", + _number(fps), + "-i", + str(inference_root / "overlay-frames" / "frame-%06d.png"), + "-t", + _number(duration), + "-c:v", + "h264_nvenc", + "-preset", + "p4", + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + "21", + "-b:v", + "0", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(video_path), + ) + command_runner(encode_argv, None) + encode_seconds = _elapsed(encode_started) + _required_nonempty_file(video_path, inference_root, "EoMT overlay video") + + masks_path = inference_root / "masks.tar.gz" + archive_started = time.perf_counter() + archive_argv = ( + "/usr/bin/tar", + "-czf", + str(masks_path), + "-C", + str(inference_root), + "semantic-masks", + ) + command_runner(archive_argv, None) + archive_seconds = _elapsed(archive_started) + _required_nonempty_file(masks_path, inference_root, "EoMT mask archive") + free_post_artifacts = available_bytes(output) + if free_post_artifacts < disk_floor_bytes: + raise ComponentAdapterError("EoMT publication crossed its disk floor") + + wall_seconds = _elapsed(total_started) + finalize_argv = ( + sys.executable, + str(runner), + "finalize", + "--output", + str(inference_root), + "--video", + str(video_path), + "--masks", + str(masks_path), + "--extract-seconds", + _number(extract_seconds), + "--encode-seconds", + _number(encode_seconds), + "--archive-seconds", + _number(archive_seconds), + "--wall-seconds", + _number(wall_seconds), + "--encoder", + "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart", + "--disk-free-before-bytes", + str(free_before), + "--disk-free-post-extract-bytes", + str(free_post_extract), + "--disk-free-post-inference-bytes", + str(free_post_inference), + "--disk-free-post-artifacts-bytes", + str(free_post_artifacts), + "--disk-floor-bytes", + str(disk_floor_bytes), + "--working-set-reserve-bytes", + str(reserve), + ) + command_runner(finalize_argv, environment) + _bind_source_frame_manifest(inference_root, frames_root, request) + _validate_eomt_result(inference_root, frames_root, request) + for temporary in ("overlay-frames", "semantic-masks"): + path = inference_root / temporary + if path.exists(): + shutil.rmtree(path) + _publish_component_children( + inference_root, + output, + expected_names={ + "decode-repair.json", + "frames.jsonl", + "gpu-telemetry.jsonl", + "masks.tar.gz", + "perception.mp4", + "result.json", + "run-report.json", + "source-frames.json", + }, + ) + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +def _validate_release_assets( + request: ComponentRequest, + layout: RuntimeLayout, +) -> dict[str, Path]: + roots = { + item.asset_id: validate_tree_asset(request, layout, item.asset_id) for item in EOMT_ASSETS + } + for asset_id in ( + "eomt-environment", + "eomt-model-cache", + "eomt-python-environment", + "eomt-transformers-environment", + ): + validate_tree_receipt( + request, + roots[asset_id], + asset_id, + verify_payload=True, + ) + for asset_id in ("eomt-runner-bundle", "k1-valid-fov-root"): + validate_tree_receipt( + request, + roots[asset_id], + asset_id, + verify_payload=True, + ) + ffmpeg_receipt = validate_tree_receipt( + request, + roots["eomt-ffmpeg-runtime"], + "eomt-ffmpeg-runtime", + expected_metadata={ + "source_image_sha256": FFMPEG_TREE_SOURCE_IMAGE_SHA256, + "source_path": "/usr/lib/ffmpeg/7.0", + }, + additional_metadata_keys=frozenset({"binaries"}), + verify_payload=True, + ) + environment_manifest = validate_identity_manifest( + roots["eomt-environment"] / "manifest.json", + expected_identity_sha256=EOMT_DEPENDENCY_IDENTITY_SHA256, + expected_schema=DEPENDENCY_SCHEMA, + label="EoMT dependency manifest", + ) + identity = cast(dict[str, object], environment_manifest["identity"]) + if identity.get("profile_sha256") != _RUNNER_FILES["e3_k1_camera1_profile.json"][1]: + raise ComponentAdapterError("EoMT dependency profile changed") + valid_manifest = validate_identity_manifest( + roots["k1-valid-fov-root"] / "manifest.json", + expected_identity_sha256=VALID_FOV_IDENTITY_SHA256, + expected_schema=VALID_FOV_SCHEMA, + label="valid-FOV manifest", + ) + artifact = valid_manifest.get("artifact") + if not isinstance(artifact, dict) or artifact.get("sha256") != VALID_FOV_MASK_SHA256: + raise ComponentAdapterError("valid-FOV mask binding changed") + mask_byte_length = artifact.get("byte_length") + if ( + isinstance(mask_byte_length, bool) + or not isinstance(mask_byte_length, int) + or mask_byte_length < 1 + ): + raise ComponentAdapterError("valid-FOV mask byte length changed") + validate_critical_tree_file( + roots["k1-valid-fov-root"], + "mask.png", + byte_length=mask_byte_length, + identity_sha256=VALID_FOV_MASK_SHA256, + label="valid-FOV mask", + ) + for name, (byte_length, digest) in _RUNNER_FILES.items(): + validate_critical_tree_file( + roots["eomt-runner-bundle"], + name, + byte_length=byte_length, + identity_sha256=digest, + label=f"EoMT runner file {name}", + ) + snapshot = ( + roots["eomt-model-cache"] + / "huggingface" + / "models--tue-mps--cityscapes_semantic_eomt_large_1024" + / "snapshots" + / MODEL_REVISION + ) + for name, (byte_length, digest) in _MODEL_FILES.items(): + validate_critical_tree_file( + roots["eomt-model-cache"], + (snapshot / name).relative_to(roots["eomt-model-cache"]).as_posix(), + byte_length=byte_length, + identity_sha256=digest, + label=f"EoMT model file {name}", + ) + binaries = ffmpeg_receipt.get("binaries") + if not isinstance(binaries, dict) or set(binaries) != {"ffmpeg", "ffprobe"}: + raise ComponentAdapterError("sealed ffmpeg binary receipt changed") + for name, digest in (("ffmpeg", FFMPEG_SHA256), ("ffprobe", FFPROBE_SHA256)): + row = binaries.get(name) + if not isinstance(row, dict) or set(row) != { + "relative_path", + "byte_length", + "sha256", + }: + raise ComponentAdapterError("sealed ffmpeg binary receipt changed") + path = validate_critical_tree_file( + roots["eomt-ffmpeg-runtime"], + f"bin/{name}", + byte_length=cast(int, row["byte_length"]), + identity_sha256=digest, + label=f"sealed {name}", + ) + if row != { + "relative_path": f"bin/{name}", + "byte_length": path.stat().st_size, + "sha256": digest, + } or not os.access(path, os.X_OK): + raise ComponentAdapterError("sealed ffmpeg binary receipt changed") + return roots + + +def _decode_camera_epoch( + *, + request: ComponentRequest, + input_document: Mapping[str, object], + camera_job_root: Path, + output_root: Path, + work_root: Path, + ffmpeg_root: Path, + command_runner: CommandRunner, +) -> tuple[Path, Path, dict[str, object]]: + source = request.source + epoch = ( + camera_job_root + / "input" + / "camera" + / source.camera_source_id + / f"epoch-{source.codec_epoch}" + ) + stream_path = work_root / "camera.mp4" + with stream_path.open("xb") as stream: + for path in ( + epoch / "init.mp4", + *(epoch / "segments" / f"{index}.m4s" for index in range(1, source.frame_count + 1)), + ): + with path.open("rb") as segment: + shutil.copyfileobj(segment, stream, length=1024 * 1024) + stream.flush() + os.fsync(stream.fileno()) + packets_path = work_root / "packets.csv" + ffprobe_argv = ( + str(ffmpeg_root / "bin" / "ffprobe"), + "-v", + "error", + "-select_streams", + "v:0", + "-show_packets", + "-show_entries", + "packet=pts,flags", + "-of", + "csv=p=0", + "-o", + str(packets_path), + str(stream_path), + ) + command_runner(ffprobe_argv, None) + packet_rows = packets_path.read_text(encoding="utf-8").splitlines() + if len(packet_rows) != source.frame_count: + raise ComponentAdapterError("camera packet count changed") + packet_pts: list[int] = [] + for row in packet_rows: + fields = row.split(",") + try: + packet_pts.append(int(fields[0].strip())) + except (IndexError, ValueError) as exc: + raise ComponentAdapterError("camera packet timestamp is invalid") from exc + decoded_root = work_root / "decoded-by-pts" + decoded_root.mkdir(mode=0o700) + decode_argv = ( + str(ffmpeg_root / "bin" / "ffmpeg"), + "-hide_banner", + "-loglevel", + "error", + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-c:v", + "h264_cuvid", + "-err_detect", + "ignore_err", + "-flags", + "+output_corrupt", + "-copyts", + "-i", + str(stream_path), + "-map", + "0:v:0", + "-vf", + "hwdownload,format=nv12", + "-fps_mode", + "passthrough", + "-enc_time_base", + "demux", + "-frames:v", + str(source.frame_count), + "-frame_pts", + "1", + str(decoded_root / "frame-%d.png"), + ) + command_runner(decode_argv, None) + frames_root = output_root / "source-frames" + frames_root.mkdir(mode=0o700) + repairs: list[dict[str, object]] = [] + decoded_count = len(tuple(decoded_root.glob("frame-*.png"))) + for index, pts in enumerate(packet_pts): + decoded = decoded_root / f"frame-{pts}.png" + canonical = frames_root / f"frame-{index + 1:06d}.png" + if decoded.is_file() and not decoded.is_symlink(): + os.replace(decoded, canonical) + continue + if index == 0 or repairs: + raise ComponentAdapterError("camera decode has more than one recoverable gap") + previous = frames_root / f"frame-{index:06d}.png" + shutil.copyfile(previous, canonical) + repairs.append( + { + "sequence": index + 1, + "packet_pts": pts, + "method": "duplicate-previous-decoded-frame", + } + ) + if any(decoded_root.iterdir()): + raise ComponentAdapterError("camera decoder emitted unexpected frames") + timeline = cast(Mapping[str, object], input_document["timeline"]) + timeline_start = timeline["start_seconds"] + if isinstance(timeline_start, bool) or not isinstance(timeline_start, (int, float)): + raise ComponentAdapterError("camera timeline start changed") + duration = source.timeline_end_seconds - source.timeline_start_seconds + first_pts = packet_pts[0] + previous_seconds = -1.0 + timeline_path = work_root / "timeline.jsonl" + with timeline_path.open("x", encoding="utf-8", newline="\n") as stream: + for index, pts in enumerate(packet_pts): + epoch_seconds = (pts - first_pts) / 90_000.0 + if epoch_seconds <= previous_seconds or epoch_seconds > duration + 0.001: + raise ComponentAdapterError("camera packet timeline changed") + timeline_row = { + "frame_index": index, + "epoch_seconds": epoch_seconds, + "session_seconds": float(timeline_start) + epoch_seconds, + } + stream.write(canonical_json(timeline_row).decode("utf-8") + "\n") + previous_seconds = epoch_seconds + return ( + frames_root, + timeline_path, + { + "schema_version": DECODE_REPAIR_SCHEMA, + "decoder": "ffmpeg-h264_cuvid-output-corrupt", + "packets_requested": source.frame_count, + "frames_decoded": decoded_count, + "repaired_frame_count": len(repairs), + "repairs": repairs, + }, + ) + + +def _validate_eomt_result( + root: Path, + frames_root: Path, + request: ComponentRequest, +) -> None: + document = validate_fixed_result(root, schema_version=EOMT_RESULT_SCHEMA, label="EoMT") + identity = document.get("identity") + if not isinstance(identity, dict): + raise ComponentAdapterError("EoMT result identity is unavailable") + configuration = identity.get("configuration") + models = identity.get("models") + semantic = models.get("semantic") if isinstance(models, dict) else None + if not isinstance(configuration, dict): + raise ComponentAdapterError("EoMT result configuration is unavailable") + observed_identity = hashlib.sha256(canonical_json(identity)).hexdigest() + source = request.source + if ( + document.get("identity_sha256") != observed_identity + or document.get("result_id") != f"result-{observed_identity}" + or document.get("job_id") != source.camera_job_id + or document.get("input_sha256") != source.camera_input_sha256 + or document.get("session_id") != source.source_session_id + or document.get("source_id") != source.camera_source_id + or document.get("codec_epoch") != source.codec_epoch + or document.get("timestamp_basis") != "session-time-seconds" + or document.get("timeline_start_seconds") != source.timeline_start_seconds + or document.get("timeline_end_seconds") != source.timeline_end_seconds + or document.get("frames_processed") != source.frame_count + or document.get("ground_truth") is not False + or identity.get("schema_version") != "missioncore.recorded-perception-identity/v2" + or configuration.get("pipeline") != EOMT_PIPELINE + or configuration.get("dependency_identity_sha256") != EOMT_DEPENDENCY_IDENTITY_SHA256 + or configuration.get("container_image") != f"sha256:{request.component_image_sha256}" + or not isinstance(semantic, dict) + or semantic.get("id") != MODEL_ID + or semantic.get("revision") != MODEL_REVISION + or semantic.get("architecture") != MODEL_ARCHITECTURE + ): + raise ComponentAdapterError("EoMT result source identity changed") + artifacts = document.get("artifacts") + if not isinstance(artifacts, list) or len(artifacts) != 6: + raise ComponentAdapterError("EoMT result artifact set changed") + expected_artifacts = { + "panoptic-overlay-video": ("perception.mp4", "video/mp4"), + "panoptic-mask-archive": ("masks.tar.gz", "application/gzip"), + "panoptic-frame-metadata": ("frames.jsonl", "application/x-ndjson"), + "worker-gpu-telemetry": ("gpu-telemetry.jsonl", "application/x-ndjson"), + "perception-run-report": ("run-report.json", "application/json"), + "decoded-source-frame-manifest": ("source-frames.json", "application/json"), + } + observed_kinds: set[str] = set() + for value in artifacts: + if not isinstance(value, dict): + raise ComponentAdapterError("EoMT result artifact is invalid") + kind = value.get("kind") + encoded = value.get("path") + if ( + not isinstance(kind, str) + or kind in observed_kinds + or expected_artifacts.get(kind) != (encoded, value.get("media_type")) + or not isinstance(encoded, str) + or "/" in encoded + or "\\" in encoded + ): + raise ComponentAdapterError("EoMT result artifact path changed") + observed_kinds.add(kind) + path = _required_nonempty_file(root / encoded, root, "EoMT result artifact") + if value.get("byte_length") != path.stat().st_size or value.get("sha256") != sha256_file( + path + ): + raise ComponentAdapterError("EoMT result artifact identity changed") + if observed_kinds != set(expected_artifacts): + raise ComponentAdapterError("EoMT result artifact roles changed") + manifest_path = _required_nonempty_file( + root / "source-frames.json", + root, + "EoMT source frame manifest", + ) + expected_binding = { + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "path": "source-frames.json", + "sha256": sha256_file(manifest_path), + "byte_length": manifest_path.stat().st_size, + "frame_count": source.frame_count, + } + if identity.get("source_frames") != expected_binding: + raise ComponentAdapterError("EoMT source frame manifest binding changed") + _validate_source_frame_manifest(manifest_path, frames_root, request) + _validate_mask_archive_inventory(root / "masks.tar.gz", source.frame_count) + + +def _bind_source_frame_manifest( + result_root: Path, + frames_root: Path, + request: ComponentRequest, +) -> None: + document = _source_frame_manifest_document(frames_root, request) + manifest_path = result_root / "source-frames.json" + _write_canonical_json(manifest_path, document) + binding = { + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "path": manifest_path.name, + "sha256": sha256_file(manifest_path), + "byte_length": manifest_path.stat().st_size, + "frame_count": request.source.frame_count, + } + result = load_json( + result_root / "result.json", + label="legacy EoMT result", + maximum=8 * 1024 * 1024, + confinement_root=result_root, + ) + identity = result.get("identity") + artifacts = result.get("artifacts") + old_result_id = result.get("result_id") + if ( + result.get("schema_version") != EOMT_RESULT_SCHEMA + or not isinstance(identity, dict) + or "source_frames" in identity + or not isinstance(artifacts, list) + or len(artifacts) != 5 + or not isinstance(old_result_id, str) + ): + raise ComponentAdapterError("legacy EoMT result cannot bind source frames") + identity["source_frames"] = binding + identity_sha256 = hashlib.sha256(canonical_json(identity)).hexdigest() + result_id = f"result-{identity_sha256}" + result["identity_sha256"] = identity_sha256 + result["result_id"] = result_id + + report_path = _required_nonempty_file( + result_root / "run-report.json", + result_root, + "legacy EoMT run report", + ) + report = load_json( + report_path, + label="legacy EoMT run report", + maximum=8 * 1024 * 1024, + confinement_root=result_root, + ) + if report.get("result_id") != old_result_id or "source_frames" in report: + raise ComponentAdapterError("legacy EoMT run report cannot bind source frames") + report["result_id"] = result_id + report["source_frames"] = binding + _write_json(report_path, report) + + observed_kinds: set[str] = set() + for value in artifacts: + if not isinstance(value, dict) or not isinstance(value.get("kind"), str): + raise ComponentAdapterError("legacy EoMT artifact set changed") + kind = cast(str, value["kind"]) + if kind in observed_kinds: + raise ComponentAdapterError("legacy EoMT artifact set changed") + observed_kinds.add(kind) + if kind == "perception-run-report": + value["byte_length"] = report_path.stat().st_size + value["sha256"] = sha256_file(report_path) + if observed_kinds != { + "panoptic-overlay-video", + "panoptic-mask-archive", + "panoptic-frame-metadata", + "worker-gpu-telemetry", + "perception-run-report", + }: + raise ComponentAdapterError("legacy EoMT artifact roles changed") + artifacts.append( + { + "kind": "decoded-source-frame-manifest", + "path": manifest_path.name, + "media_type": "application/json", + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "byte_length": manifest_path.stat().st_size, + "sha256": sha256_file(manifest_path), + } + ) + _write_json(result_root / "result.json", result) + + +def _source_frame_manifest_document( + frames_root: Path, + request: ComponentRequest, +) -> dict[str, object]: + if frames_root.is_symlink(): + raise ComponentAdapterError("EoMT source frame root is a symbolic link") + resolved = frames_root.resolve(strict=True) + expected_names = tuple( + f"frame-{sequence:06d}.png" for sequence in range(1, request.source.frame_count + 1) + ) + try: + observed_names = tuple(sorted(path.name for path in resolved.iterdir())) + except OSError as exc: + raise ComponentAdapterError("EoMT source frame set is unavailable") from exc + if observed_names != expected_names: + raise ComponentAdapterError("EoMT source frame set changed") + rows: list[dict[str, object]] = [] + for sequence, name in enumerate(expected_names, start=1): + path = _required_nonempty_file(resolved / name, resolved, "EoMT source frame") + if path.is_symlink() or path.stat().st_size > 16 * 1024 * 1024: + raise ComponentAdapterError("EoMT source frame is unsafe") + rows.append( + { + "sequence": sequence, + "path": f"source-frames/{name}", + "byte_length": path.stat().st_size, + "sha256": sha256_file(path), + } + ) + source = request.source + return { + "schema_version": SOURCE_FRAME_MANIFEST_SCHEMA, + "job_id": source.camera_job_id, + "input_sha256": source.camera_input_sha256, + "source_id": source.camera_source_id, + "codec_epoch": source.codec_epoch, + "frame_count": source.frame_count, + "frames": rows, + } + + +def _validate_source_frame_manifest( + path: Path, + frames_root: Path, + request: ComponentRequest, +) -> None: + document = load_json( + path, + label="EoMT source frame manifest", + maximum=32 * 1024 * 1024, + confinement_root=path.parent, + ) + if document != _source_frame_manifest_document(frames_root, request): + raise ComponentAdapterError("EoMT source frame manifest changed") + + +def _validate_mask_archive_inventory(path: Path, frame_count: int) -> None: + expected_files = { + f"semantic-masks/frame-{sequence:06d}.png" for sequence in range(1, frame_count + 1) + } + expected_names = {"semantic-masks", *expected_files} + try: + with tarfile.open(path, mode="r:gz") as archive: + members = archive.getmembers() + names = [member.name for member in members] + if len(names) != len(expected_names) or set(names) != expected_names: + raise ComponentAdapterError("EoMT mask archive inventory changed") + if len(names) != len(set(names)): + raise ComponentAdapterError("EoMT mask archive contains duplicate members") + for member in members: + if member.name == "semantic-masks": + if member.type != tarfile.DIRTYPE or member.size != 0: + raise ComponentAdapterError("EoMT mask archive directory type changed") + continue + if ( + member.name not in expected_files + or member.type not in {tarfile.REGTYPE, tarfile.AREGTYPE} + or not 0 < member.size <= 2 * 1024 * 1024 + ): + raise ComponentAdapterError("EoMT mask archive member type changed") + stream = archive.extractfile(member) + if stream is None: + raise ComponentAdapterError("EoMT mask archive member is unavailable") + payload = stream.read(member.size + 1) + if len(payload) != member.size: + raise ComponentAdapterError("EoMT mask archive member length changed") + validate_grayscale_png_payload( + payload, + width=800, + height=600, + label=f"EoMT mask {member.name}", + ) + except (OSError, EOFError, tarfile.TarError) as exc: + raise ComponentAdapterError("EoMT mask archive is invalid") from exc + + +def _eomt_environment(roots: Mapping[str, Path]) -> dict[str, str]: + retained = { + key: value + for key, value in os.environ.items() + if key.startswith("NVIDIA_") or key in {"CUDA_VISIBLE_DEVICES", "LD_LIBRARY_PATH", "PATH"} + } + python_path = f"{roots['eomt-transformers-environment']}:{roots['eomt-python-environment']}" + return { + **retained, + "PYTHONPATH": python_path, + "TORCH_HOME": str(roots["eomt-model-cache"] / "torch"), + "HF_HOME": str(roots["eomt-model-cache"] / "huggingface"), + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "HOME": "/tmp", + } + + +def _empty_output_root(path: Path) -> Path: + resolved = path.resolve(strict=True) + if path.is_symlink() or not resolved.is_dir() or any(resolved.iterdir()): + raise ComponentAdapterError("EoMT output root is not an empty real directory") + return resolved + + +def _prepare_workspace(path: Path) -> Path: + if path.exists() or path.is_symlink(): + raise ComponentAdapterError("EoMT workspace already exists") + path.mkdir(mode=0o700, parents=False) + return path.resolve(strict=True) + + +def _required_nonempty_file(path: Path, root: Path, label: str) -> Path: + resolved = _required_confined_file(path, root, label) + if resolved.stat().st_size < 1: + raise ComponentAdapterError(f"{label} is empty") + return resolved + + +def _required_confined_file(path: Path, root: Path, label: str) -> Path: + try: + metadata = path.lstat() + resolved_root = root.resolve(strict=True) + resolved = path.resolve(strict=True) + except OSError as exc: + raise ComponentAdapterError(f"{label} is unavailable") from exc + if ( + stat.S_ISLNK(metadata.st_mode) + or not stat.S_ISREG(metadata.st_mode) + or not resolved.is_relative_to(resolved_root) + ): + raise ComponentAdapterError(f"{label} is not confined") + return resolved + + +def _write_json(path: Path, value: object) -> None: + payload = json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n" + path.write_text(payload, encoding="utf-8") + + +def _write_canonical_json(path: Path, value: object) -> None: + path.write_bytes(canonical_json(value)) + + +def _publish_component_children( + staging: Path, + output: Path, + *, + expected_names: set[str], +) -> None: + children = tuple(staging.iterdir()) + if {child.name for child in children} != expected_names or any( + child.is_dir() or child.is_symlink() for child in children + ): + raise ComponentAdapterError("EoMT staging contains unexpected directories") + for child in children: + destination = output / child.name + if destination.exists() or destination.is_symlink(): + raise ComponentAdapterError("EoMT publication target already exists") + os.replace(child, destination) + staging.rmdir() + + +def _elapsed(started: float) -> float: + return max(time.perf_counter() - started, 0.000001) + + +def _number(value: float) -> str: + return format(value, ".9f").rstrip("0").rstrip(".") + + +def main(argv: Sequence[str] | None = None) -> int: + request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv) + execute_eomt_component( + request_path=request_path, + layout=RuntimeLayout.fixed(EOMT_ASSETS), + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ComponentAdapterError as exc: + print(f"portable LAB V1 EoMT rejected: {exc}", file=sys.stderr) + raise SystemExit(2) from exc diff --git a/src/k1link/observatory/portable_lab_v1_local_runners.py b/src/k1link/observatory/portable_lab_v1_local_runners.py index 196fe4c..22e0d6d 100644 --- a/src/k1link/observatory/portable_lab_v1_local_runners.py +++ b/src/k1link/observatory/portable_lab_v1_local_runners.py @@ -793,6 +793,7 @@ class InstalledPortableLabV1EomtRunner: try: request = _component_request( component="eomt", + component_image_sha256=component.image_sha256, source=source, plan=plan, assets=component.assets, @@ -854,6 +855,7 @@ class InstalledPortableLabV1DdrnetRunner: try: request = _component_request( component="ddrnet", + component_image_sha256=component.image_sha256, source=source, plan=plan, assets=component.assets, @@ -1022,14 +1024,17 @@ def _prepare_invocation( def _component_request( *, component: PortableLabV1Component, + component_image_sha256: str, source: PortableLabV1MaterializedSource, plan: PortableLabV1OrchestrationPlan, assets: tuple[PortableLabV1HostAsset, ...], effective_config_sha256: str | None, ) -> dict[str, object]: + _digest(component_image_sha256, "component request image sha256") return { "schema_version": PORTABLE_LAB_V1_COMPONENT_REQUEST_SCHEMA, "component": component, + "component_image_sha256": component_image_sha256, "plan_sha256": plan.plan_sha256, "definition_sha256": plan.definition_sha256, "release_candidate_sha256": plan.release_candidate_sha256, diff --git a/tests/test_lab_v1_eomt_tree_asset_sealer.py b/tests/test_lab_v1_eomt_tree_asset_sealer.py index 6c6cdb1..c21db1c 100644 --- a/tests/test_lab_v1_eomt_tree_asset_sealer.py +++ b/tests/test_lab_v1_eomt_tree_asset_sealer.py @@ -22,7 +22,10 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None: assert '"tree-receipt.json"' in script assert '"D:\\NDC_MISSIONCORE\\runtime"' in script assert '"derived\\perception-e3-opencv413092-v1"' in script - assert '"cache\\perception-e3-models-v1"' in script + assert '"eomt-model-cache-cityscapes-large-1024-v1"' in script + assert "Install-SafeEomtModelCache" in script + assert 'New-Item -ItemType HardLink -Path $blobTarget -Target $source' in script + assert 'New-Item -ItemType HardLink -Path $snapshotTarget -Target $source' in script assert '"derived\\perception-p0-env-v1"' in script assert '"derived\\perception-p0-transformers4576-v1"' in script assert '$Utf8.GetByteCount($component) -gt 255' in script @@ -39,15 +42,14 @@ def test_eomt_tree_sealer_is_local_fixed_and_adapter_compatible() -> None: assert "smb" not in lowered -def test_eomt_tree_sealer_has_atomic_commit_marker_and_no_rehash_fast_path() -> None: +def test_eomt_tree_sealer_has_atomic_commit_marker_and_rehashes_existing_seal() -> None: script = SEALER_PATH.read_text(encoding="utf-8") fast_path = script.split("function Test-ExistingSeal", maxsplit=1)[1].split( "function New-SealedReceipt", maxsplit=1 )[0] - assert "-HashPayload:$false" in fast_path - assert "-HashPayload:$true" not in fast_path - assert "Get-FileDigest" not in fast_path + assert "-HashPayload:$true" in fast_path + assert "Assert-InventoryMatches $manifest.records $observed $Asset $false $true" in fast_path publish = script.split("$manifestPath = Join-Path $Root $ManifestName", maxsplit=1)[1] manifest_move = publish.index("[IO.File]::Move($stagedManifest, $manifestPath)") diff --git a/tests/test_lab_v1_portable_static_asset_installer.py b/tests/test_lab_v1_portable_static_asset_installer.py new file mode 100644 index 0000000..1700b64 --- /dev/null +++ b/tests/test_lab_v1_portable_static_asset_installer.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INSTALLER_PATH = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "Install-LabV1PortableStaticAssets.ps1" +) + + +def test_lab_v1_static_installer_is_fixed_offline_and_content_addressed() -> None: + script = INSTALLER_PATH.read_text(encoding="utf-8") + + assert '"D:\\NDC_MISSIONCORE\\runtime"' in script + assert '"staging\\observatory-lab-v1-static-v1"' in script + assert '"eomt-runner-bundle-v1"' in script + assert '"lab-v1-static-files-v1"' in script + assert "New-Item `\n -ItemType HardLink" in script + assert "Get-FileIdentity" in script + assert "tree-manifest.tsv" in script + assert "tree-receipt.json" in script + + lowered = script.lower() + assert "invoke-webrequest" not in lowered + assert "start-bitstransfer" not in lowered + assert "docker " not in lowered + assert "ssh " not in lowered + assert "smb" not in lowered + + +def test_lab_v1_static_installer_seals_expected_exact_payloads() -> None: + script = INSTALLER_PATH.read_text(encoding="utf-8") + + expected = { + "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", + "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6", + "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675", + "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4", + "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875", + "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", + "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f", + "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1", + "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35", + "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352", + "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63", + } + assert all(digest in script for digest in expected) + assert "259419077" in script + assert "32877" in script + assert "1427" in script diff --git a/tests/test_observatory_portable_lab_v1_component_adapters.py b/tests/test_observatory_portable_lab_v1_component_adapters.py new file mode 100644 index 0000000..f789d95 --- /dev/null +++ b/tests/test_observatory_portable_lab_v1_component_adapters.py @@ -0,0 +1,1209 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +import struct +import subprocess +import sys +import tarfile +import zipfile +import zlib +from collections.abc import Mapping, Sequence +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import pytest + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +ADAPTER_ROOT = REPOSITORY_ROOT / "experiments" / "perception" / "worker" / "observatory_portable" +EOMT_COMPONENT_IMAGE_SHA256 = "d" * 64 +DDRNET_COMPONENT_IMAGE_SHA256 = "e" * 64 +sys.path.insert(0, str(ADAPTER_ROOT)) +import portable_lab_v1_component_adapter as contract # noqa: E402 +import run_portable_lab_v1_ddrnet_component as ddrnet # noqa: E402 +import run_portable_lab_v1_eomt_component as eomt # noqa: E402 + + +def _write_canonical(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contract.canonical_json(value)) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _png_payload( + *, + width: int = 800, + height: int = 600, + value: int = 0, + filter_type: int = 0, +) -> bytes: + def chunk(kind: bytes, payload: bytes) -> bytes: + checksum = zlib.crc32(payload, zlib.crc32(kind)) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + kind + payload + struct.pack(">I", checksum) + + if filter_type == 0: + row = b"\x00" + bytes([value]) * width + elif filter_type == 1: + row = b"\x01" + bytes([value]) + bytes(width - 1) + else: + raise ValueError("unsupported test PNG filter") + raw = row * height + return b"".join( + ( + b"\x89PNG\r\n\x1a\n", + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 0, 0, 0, 0)), + chunk(b"IDAT", zlib.compress(raw)), + chunk(b"IEND", b""), + ) + ) + + +def _mapping_payload() -> bytes: + rows = ["label_key,class_name,hex"] + for class_id in range(64): + color = f"{class_id:02x}{(class_id * 3) % 256:02x}{(class_id * 7) % 256:02x}" + rows.append(f"{class_id},class-{class_id},#{color}") + return ("\n".join(rows) + "\n").encode() + + +def _camera_source(root: Path, *, frame_count: int = 2) -> dict[str, object]: + epoch = root / "input" / "camera" / "sensor.camera.right" / "epoch-1" + segments = epoch / "segments" + segments.mkdir(parents=True) + files = { + epoch / "summary.json": b"summary\n", + epoch / "index.jsonl": b"index\n", + epoch / "init.mp4": b"init\n", + **{ + segments / f"{sequence}.m4s": f"segment-{sequence}\n".encode() + for sequence in range(1, frame_count + 1) + }, + } + rows: list[dict[str, object]] = [] + for path, payload in files.items(): + path.write_bytes(payload) + rows.append( + { + "path": path.relative_to(root).as_posix(), + "byte_length": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + ) + input_document = { + "kind": "canonical-camera-epoch", + "session_id": "20260831T083000Z_viewer_live", + "source_id": "sensor.camera.right", + "codec_epoch": 1, + "synchronization": "camera-segment-sequence", + "media_type": "video/mp4", + "timeline": { + "basis": "session-time-seconds", + "start_seconds": 1.0, + "end_seconds": 3.0, + }, + "segment_count": frame_count, + "byte_length": sum(cast(int, row["byte_length"]) for row in rows), + "archive_summary_sha256": "a" * 64, + "archive_index_sha256": "b" * 64, + "files": rows, + } + input_sha256 = hashlib.sha256(contract.canonical_json(input_document)).hexdigest() + camera_job_id = f"recorded-camera-{input_sha256[:24]}" + _write_canonical( + root / "job.json", + { + "schema_version": contract.COMPUTE_JOB_SCHEMA, + "job_id": camera_job_id, + "profile": contract.COMPUTE_PROFILE, + "input_sha256": input_sha256, + "input": input_document, + "result_contract": { + "schema_version": contract.COMPUTE_RESULT_SCHEMA, + "timestamp_basis": "session-time-seconds", + }, + }, + ) + return { + "schema_version": contract.SOURCE_SCHEMA, + "observatory_job": { + "job_id": f"observatory-run-{'1' * 32}", + "request_sha256": "2" * 64, + "identity_sha256": "3" * 64, + }, + "source": { + "session_id": "20260831T083000Z_viewer_live", + "catalog_sha256": "4" * 64, + "bundle_sha256": "5" * 64, + "capability_manifest_sha256": "6" * 64, + "adapter_sha256": "7" * 64, + }, + "camera_compute_job": { + "job_id": camera_job_id, + "input_sha256": input_sha256, + "source_id": "sensor.camera.right", + "codec_epoch": 1, + "input_byte_length": input_document["byte_length"], + "frame_count": frame_count, + "timeline_start_seconds": 1.0, + "timeline_end_seconds": 3.0, + "generation_sha256": "8" * 64, + "calibration_sha256": contract.CALIBRATION_SHA256, + }, + "authority": dict(contract.AUTHORITY), + } + + +def _request_document( + *, + component: contract.Component, + source: Mapping[str, object], + expectations: Sequence[contract.AssetExpectation], + identities: Mapping[str, tuple[str, int | None]], + effective_sha256: str | None = None, +) -> dict[str, object]: + return { + "schema_version": contract.REQUEST_SCHEMA, + "component": component, + "component_image_sha256": ( + EOMT_COMPONENT_IMAGE_SHA256 if component == "eomt" else DDRNET_COMPONENT_IMAGE_SHA256 + ), + "plan_sha256": "9" * 64, + "definition_sha256": "a" * 64, + "release_candidate_sha256": "b" * 64, + "source": dict(source), + "paths": { + "camera_job_root": "/run/nodedc/source", + "request": contract.FIXED_REQUEST_PATH, + "output_root": "/run/nodedc/output", + "effective_ddrnet_config": ( + "/run/nodedc/effective-ddrnet-config.json" if component == "ddrnet" else None + ), + "eomt_result_root": ("/run/nodedc/eomt-result" if component == "ddrnet" else None), + "decoded_frames_root": ( + "/run/nodedc/eomt-result/source-frames" + if component == "ddrnet" + else "/run/nodedc/output/source-frames" + ), + }, + "effective_ddrnet_config_sha256": effective_sha256, + "assets": [ + { + "asset_id": item.asset_id, + "path": item.path, + "kind": item.kind, + "verification": item.verification, + "identity_sha256": identities[item.asset_id][0], + "byte_length": identities[item.asset_id][1], + } + for item in sorted(expectations, key=lambda value: value.asset_id) + ], + "authority": dict(contract.AUTHORITY), + } + + +def _eomt_identity_result( + root: Path, + source: Mapping[str, object], + *, + component_image_sha256: str = EOMT_COMPONENT_IMAGE_SHA256, + source_frames_root: Path | None = None, +) -> None: + camera = cast(dict[str, object], source["camera_compute_job"]) + source_identity = cast(dict[str, object], source["source"]) + identity = { + "schema_version": "missioncore.recorded-perception-identity/v2", + "job_id": camera["job_id"], + "input_sha256": camera["input_sha256"], + "calibration": {}, + "configuration": { + "pipeline": contract.EOMT_PIPELINE, + "dependency_identity_sha256": contract.EOMT_DEPENDENCY_IDENTITY_SHA256, + "container_image": f"sha256:{component_image_sha256}", + }, + "models": { + "semantic": { + "id": eomt.MODEL_ID, + "revision": eomt.MODEL_REVISION, + "architecture": eomt.MODEL_ARCHITECTURE, + } + }, + "publication": {}, + } + source_frame_binding: dict[str, object] | None = None + if source_frames_root is not None: + rows = [ + { + "sequence": sequence, + "path": f"source-frames/frame-{sequence:06d}.png", + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + for sequence in range(1, cast(int, camera["frame_count"]) + 1) + if (path := source_frames_root / f"frame-{sequence:06d}.png").is_file() + ] + manifest = { + "schema_version": eomt.SOURCE_FRAME_MANIFEST_SCHEMA, + "job_id": camera["job_id"], + "input_sha256": camera["input_sha256"], + "source_id": camera["source_id"], + "codec_epoch": camera["codec_epoch"], + "frame_count": camera["frame_count"], + "frames": rows, + } + manifest_path = root / "source-frames.json" + _write_canonical(manifest_path, manifest) + source_frame_binding = { + "schema_version": eomt.SOURCE_FRAME_MANIFEST_SCHEMA, + "path": manifest_path.name, + "sha256": _sha256(manifest_path), + "byte_length": manifest_path.stat().st_size, + "frame_count": camera["frame_count"], + } + identity["source_frames"] = source_frame_binding + identity_sha256 = hashlib.sha256(contract.canonical_json(identity)).hexdigest() + result_id = f"result-{identity_sha256}" + report: dict[str, object] = { + "schema_version": "missioncore.test-eomt-run-report/v1", + "result_id": result_id, + } + if source_frame_binding is not None: + report["source_frames"] = source_frame_binding + _write_canonical(root / "run-report.json", report) + artifacts: list[dict[str, object]] = [] + specs = [ + ("perception.mp4", "panoptic-overlay-video", "video/mp4"), + ("masks.tar.gz", "panoptic-mask-archive", "application/gzip"), + ("frames.jsonl", "panoptic-frame-metadata", "application/x-ndjson"), + ("gpu-telemetry.jsonl", "worker-gpu-telemetry", "application/x-ndjson"), + ("run-report.json", "perception-run-report", "application/json"), + ] + if source_frame_binding is not None: + specs.append(("source-frames.json", "decoded-source-frame-manifest", "application/json")) + for name, kind, media_type in specs: + path = root / name + if not path.exists(): + path.write_bytes(f"{name}\n".encode()) + artifacts.append( + { + "kind": kind, + "path": name, + "media_type": media_type, + "byte_length": path.stat().st_size, + "sha256": _sha256(path), + } + ) + _write_canonical( + root / "result.json", + { + "schema_version": eomt.EOMT_RESULT_SCHEMA, + "result_id": result_id, + "identity_sha256": identity_sha256, + "identity": identity, + "job_id": camera["job_id"], + "input_sha256": camera["input_sha256"], + "session_id": source_identity["session_id"], + "source_id": camera["source_id"], + "codec_epoch": camera["codec_epoch"], + "timestamp_basis": "session-time-seconds", + "timeline_start_seconds": camera["timeline_start_seconds"], + "timeline_end_seconds": camera["timeline_end_seconds"], + "frames_processed": camera["frame_count"], + "ground_truth": False, + "artifacts": artifacts, + }, + ) + + +def _decode_repair(frame_count: int) -> dict[str, object]: + return { + "schema_version": ddrnet.DECODE_REPAIR_SCHEMA, + "decoder": "ffmpeg-h264_cuvid-output-corrupt", + "packets_requested": frame_count, + "frames_decoded": frame_count, + "repaired_frame_count": 0, + "repairs": [], + } + + +def test_component_request_and_camera_job_are_canonical_and_confined( + tmp_path: Path, +) -> None: + camera_root = tmp_path / "camera" + source = _camera_source(camera_root) + expectations = ( + contract.AssetExpectation( + "test-tree", + "/opt/nodedc/assets/test-tree", + "tree", + "identity-sha256", + ), + ) + identities = {"test-tree": ("c" * 64, 7)} + document = _request_document( + component="eomt", + source=source, + expectations=expectations, + identities=identities, + ) + request_path = tmp_path / "request.json" + _write_canonical(request_path, document) + request = contract.load_component_request( + request_path, + component="eomt", + expectations=expectations, + ) + contract.validate_camera_compute_job(camera_root, request.source) + + injected = {**document, "argv": ["/bin/sh", "-c", "malicious"]} + _write_canonical(request_path, injected) + with pytest.raises(contract.ComponentAdapterError, match="fields changed"): + contract.load_component_request( + request_path, + component="eomt", + expectations=expectations, + ) + with pytest.raises(contract.ComponentAdapterError, match="accepts only"): + contract.parse_fixed_request_argument(("--request", "/run/nodedc/other.json")) + + segment = camera_root / "input/camera/sensor.camera.right/epoch-1/segments/1.m4s" + target = tmp_path / "outside-segment.m4s" + target.write_bytes(segment.read_bytes()) + segment.unlink() + segment.symlink_to(target) + with pytest.raises(contract.ComponentAdapterError, match="confined regular file"): + contract.validate_camera_compute_job(camera_root, request.source) + + +def test_sealed_tree_receipt_rehashes_payload_and_rejects_tamper( + tmp_path: Path, +) -> None: + camera_root = tmp_path / "camera" + source = _camera_source(camera_root) + tree = tmp_path / "tree" + tree.mkdir() + payload = tree / "setuptools" / "launcher manifest.xml" + payload.parent.mkdir() + payload.write_bytes(b"sealed-runner\n") + row = ( + f"setuptools/launcher manifest.xml\t{payload.stat().st_size}\t{_sha256(payload)}\n" + ).encode() + identity = hashlib.sha256(row).hexdigest() + (tree / contract.SEALED_TREE_MANIFEST).write_bytes(row) + _write_canonical( + tree / contract.SEALED_TREE_RECEIPT, + { + "schema_version": contract.SEALED_TREE_SCHEMA, + "asset_id": "test-tree", + "identity_algorithm": contract.SEALED_TREE_IDENTITY_ALGORITHM, + "identity_sha256": identity, + "file_count": 1, + "byte_length": payload.stat().st_size, + "manifest_relative_path": contract.SEALED_TREE_MANIFEST, + }, + ) + expectations = ( + contract.AssetExpectation( + "test-tree", + "/opt/nodedc/assets/test-tree", + "tree", + "identity-sha256", + ), + ) + request_path = tmp_path / "request.json" + _write_canonical( + request_path, + _request_document( + component="eomt", + source=source, + expectations=expectations, + identities={"test-tree": (identity, payload.stat().st_size)}, + ), + ) + request = contract.load_component_request( + request_path, + component="eomt", + expectations=expectations, + ) + contract.validate_tree_receipt( + request, + tree, + "test-tree", + verify_payload=True, + ) + payload.write_bytes(b"tampered-data\n") + with pytest.raises(contract.ComponentAdapterError, match="payload changed"): + contract.validate_tree_receipt( + request, + tree, + "test-tree", + verify_payload=True, + ) + + +def test_eomt_release_admission_fully_rehashes_every_mounted_tree( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + roots: dict[str, Path] = {} + for expectation in eomt.EOMT_ASSETS: + root = tmp_path / expectation.asset_id + root.mkdir() + roots[expectation.asset_id] = root + binaries_root = roots["eomt-ffmpeg-runtime"] / "bin" + binaries_root.mkdir() + for name in ("ffmpeg", "ffprobe"): + binary = binaries_root / name + binary.write_bytes(b"x") + binary.chmod(0o500) + + verification: dict[str, bool] = {} + + def fake_receipt( + _request: contract.ComponentRequest, + _root: Path, + asset_id: str, + **kwargs: object, + ) -> dict[str, object]: + verification[asset_id] = cast(bool, kwargs["verify_payload"]) + if asset_id != "eomt-ffmpeg-runtime": + return {} + return { + "binaries": { + name: { + "relative_path": f"bin/{name}", + "byte_length": 1, + "sha256": digest, + } + for name, digest in ( + ("ffmpeg", eomt.FFMPEG_SHA256), + ("ffprobe", eomt.FFPROBE_SHA256), + ) + } + } + + def fake_identity_manifest( + _path: Path, + **kwargs: object, + ) -> dict[str, object]: + if kwargs["label"] == "EoMT dependency manifest": + return { + "identity": {"profile_sha256": eomt._RUNNER_FILES["e3_k1_camera1_profile.json"][1]} + } + return { + "artifact": { + "sha256": contract.VALID_FOV_MASK_SHA256, + "byte_length": 1, + } + } + + monkeypatch.setattr( + eomt, + "validate_tree_asset", + lambda _request, _layout, asset_id: roots[asset_id], + ) + monkeypatch.setattr(eomt, "validate_tree_receipt", fake_receipt) + monkeypatch.setattr(eomt, "validate_identity_manifest", fake_identity_manifest) + monkeypatch.setattr( + eomt, + "validate_critical_tree_file", + lambda *_args, **_kwargs: binaries_root / "ffmpeg", + ) + eomt._validate_release_assets( + cast(contract.ComponentRequest, object()), + cast(contract.RuntimeLayout, object()), + ) + + assert verification == {item.asset_id: True for item in eomt.EOMT_ASSETS} + + +def test_entrypoints_import_shared_module_from_one_readonly_adapter_bundle( + tmp_path: Path, +) -> None: + bundle = tmp_path / "adapter" + bundle.mkdir() + names = ( + "portable_lab_v1_component_adapter.py", + "run_portable_lab_v1_eomt_component.py", + "run_portable_lab_v1_ddrnet_component.py", + ) + for name in names: + destination = bundle / name + shutil.copyfile(ADAPTER_ROOT / name, destination) + destination.chmod(0o444) + bundle.chmod(0o555) + try: + for name in names[1:]: + completed = subprocess.run( + ( + sys.executable, + "-B", + str(bundle / name), + "--request", + "/run/nodedc/not-the-fixed-request.json", + ), + cwd=tmp_path, + env={"PATH": "/usr/bin:/bin", "PYTHONDONTWRITEBYTECODE": "1"}, + check=False, + capture_output=True, + text=True, + ) + assert completed.returncode == 2 + assert "accepts only --request /run/nodedc/request.json" in completed.stderr + assert "ModuleNotFoundError" not in completed.stderr + finally: + bundle.chmod(0o755) + + +def test_adapter_critical_source_identities_match_repository_release_files() -> None: + worker = REPOSITORY_ROOT / "experiments/perception/worker" + for name, (byte_length, identity_sha256) in eomt._RUNNER_FILES.items(): + path = worker / name + assert path.stat().st_size == byte_length + assert _sha256(path) == identity_sha256 + ddrnet_files = { + worker / "lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py": ( + 32_877, + ddrnet.RUNNER_SHA256, + ), + REPOSITORY_ROOT / "config/perception/lab-v1-vegetation-mission-policy-v1.json": ( + 3_022, + ddrnet.POLICY_SHA256, + ), + REPOSITORY_ROOT / "config/perception/lab-v1-vegetation-provider-label-map-v1.json": ( + 2_756, + ddrnet.PROVIDER_MAP_SHA256, + ), + } + for path, (byte_length, identity_sha256) in ddrnet_files.items(): + assert path.stat().st_size == byte_length + assert _sha256(path) == identity_sha256 + profile = json.loads( + (REPOSITORY_ROOT / "config/perception/lab-v1-eomt-ddrnet-portable-v2.json").read_text() + ) + for key in ("profile_id", "source_binding", "effective_config_contract"): + profile.pop(key) + profile.pop("ravnoves", None) + profile["schema_version"] = ddrnet.EFFECTIVE_CONFIG_SCHEMA + assert hashlib.sha256(contract.canonical_json(profile)).hexdigest() == ( + ddrnet.DDRNET_STATIC_CONFIG_SHA256 + ) + + +def test_eomt_adapter_uses_only_fixed_legacy_argv_and_publishes_frames( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + camera_root = tmp_path / "camera" + source = _camera_source(camera_root) + output = tmp_path / "output" + output.mkdir() + asset_roots: dict[str, Path] = {} + identities: dict[str, tuple[str, int | None]] = {} + for index, expectation in enumerate(eomt.EOMT_ASSETS, start=1): + root = tmp_path / "assets" / expectation.asset_id + root.mkdir(parents=True) + asset_roots[expectation.asset_id] = root + identities[expectation.asset_id] = ( + expectation.identity_sha256 or f"{index:x}" * 64, + expectation.byte_length or index, + ) + runner_root = asset_roots["eomt-runner-bundle"] + (runner_root / "run_e4_full_session_segmentation.py").write_text("runner\n") + (runner_root / "e3_k1_camera1_profile.json").write_text("{}\n") + ffmpeg_bin = asset_roots["eomt-ffmpeg-runtime"] / "bin" + ffmpeg_bin.mkdir() + for name in ("ffmpeg", "ffprobe"): + (ffmpeg_bin / name).write_text(f"{name}\n") + request_path = tmp_path / "request.json" + _write_canonical( + request_path, + _request_document( + component="eomt", + source=source, + expectations=eomt.EOMT_ASSETS, + identities=identities, + ), + ) + layout = contract.RuntimeLayout( + request=request_path, + camera_job_root=camera_root, + output_root=output, + effective_ddrnet_config=tmp_path / "unused-config.json", + eomt_result_root=tmp_path / "unused-eomt", + asset_paths=asset_roots, + ) + monkeypatch.setattr(eomt, "_validate_release_assets", lambda _request, _layout: asset_roots) + monkeypatch.setattr(eomt, "available_bytes", lambda _path: 10**12) + clock = iter(float(value) for value in range(8)) + monkeypatch.setattr(eomt, "time", SimpleNamespace(perf_counter=lambda: next(clock))) + commands: list[tuple[tuple[str, ...], Mapping[str, str] | None]] = [] + + def fake_runner( + argv: Sequence[str], + environment: Mapping[str, str] | None, + ) -> None: + command = tuple(argv) + commands.append((command, environment)) + if command[0].endswith("ffprobe"): + Path(command[command.index("-o") + 1]).write_text("0,K_\n90000,__\n") + elif command[0].endswith("ffmpeg") and "-frame_pts" in command: + pattern = command[-1] + Path(pattern.replace("%d", "0")).write_bytes(b"\x89PNG\r\n\x1a\nzero") + Path(pattern.replace("%d", "90000")).write_bytes(b"\x89PNG\r\n\x1a\none") + elif len(command) > 2 and command[2] == "run": + staging = Path(command[command.index("--output") + 1]) + (staging / "overlay-frames").mkdir(parents=True) + (staging / "semantic-masks").mkdir() + for sequence in (1, 2): + (staging / "overlay-frames" / f"frame-{sequence:06d}.png").write_bytes(b"overlay") + (staging / "semantic-masks" / f"frame-{sequence:06d}.png").write_bytes( + _png_payload(value=sequence) + ) + elif command[0].endswith("ffmpeg"): + Path(command[-1]).write_bytes(b"video") + elif command[0] == "/usr/bin/tar": + with tarfile.open(command[2], mode="w:gz") as archive: + archive.add( + Path(command[command.index("-C") + 1]) / "semantic-masks", + arcname="semantic-masks", + ) + elif len(command) > 2 and command[2] == "finalize": + staging = Path(command[command.index("--output") + 1]) + _eomt_identity_result(staging, source) + else: + raise AssertionError(command) + + eomt.execute_eomt_component( + request_path=request_path, + layout=layout, + command_runner=fake_runner, + work_root=tmp_path / "eomt-work", + disk_floor_bytes=0, + ) + + work = tmp_path / "eomt-work" + inference = output / ".eomt-component" + ffmpeg = ffmpeg_bin / "ffmpeg" + ffprobe = ffmpeg_bin / "ffprobe" + orchestrator_sha256 = contract.sha256_file(Path(eomt.__file__).resolve(strict=True)) + camera = cast(dict[str, object], source["camera_compute_job"]) + reserve = 2 * 800 * 600 * 7 + cast(int, camera["input_byte_length"]) + expected_commands = [ + ( + str(ffprobe), + "-v", + "error", + "-select_streams", + "v:0", + "-show_packets", + "-show_entries", + "packet=pts,flags", + "-of", + "csv=p=0", + "-o", + str(work / "packets.csv"), + str(work / "camera.mp4"), + ), + ( + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-hwaccel", + "cuda", + "-hwaccel_output_format", + "cuda", + "-c:v", + "h264_cuvid", + "-err_detect", + "ignore_err", + "-flags", + "+output_corrupt", + "-copyts", + "-i", + str(work / "camera.mp4"), + "-map", + "0:v:0", + "-vf", + "hwdownload,format=nv12", + "-fps_mode", + "passthrough", + "-enc_time_base", + "demux", + "-frames:v", + "2", + "-frame_pts", + "1", + str(work / "decoded-by-pts" / "frame-%d.png"), + ), + ( + sys.executable, + str(runner_root / "run_e4_full_session_segmentation.py"), + "run", + "--job", + str(camera_root), + "--profile", + str(runner_root / "e3_k1_camera1_profile.json"), + "--valid-fov-root", + str(asset_roots["k1-valid-fov-root"]), + "--frames", + str(output / "source-frames"), + "--timeline", + str(work / "timeline.jsonl"), + "--cache", + str(asset_roots["eomt-model-cache"]), + "--environment", + str(asset_roots["eomt-environment"]), + "--output", + str(inference), + "--frame-limit", + "2", + "--free-bytes-floor", + "0", + "--orchestrator-sha256", + orchestrator_sha256, + "--container-image", + f"sha256:{EOMT_COMPONENT_IMAGE_SHA256}", + "--telemetry-interval-seconds", + "1", + ), + ( + str(ffmpeg), + "-hide_banner", + "-loglevel", + "error", + "-framerate", + "1", + "-i", + str(inference / "overlay-frames" / "frame-%06d.png"), + "-t", + "2", + "-c:v", + "h264_nvenc", + "-preset", + "p4", + "-tune", + "hq", + "-rc", + "vbr", + "-cq", + "21", + "-b:v", + "0", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(inference / "perception.mp4"), + ), + ( + "/usr/bin/tar", + "-czf", + str(inference / "masks.tar.gz"), + "-C", + str(inference), + "semantic-masks", + ), + ( + sys.executable, + str(runner_root / "run_e4_full_session_segmentation.py"), + "finalize", + "--output", + str(inference), + "--video", + str(inference / "perception.mp4"), + "--masks", + str(inference / "masks.tar.gz"), + "--extract-seconds", + "1", + "--encode-seconds", + "1", + "--archive-seconds", + "1", + "--wall-seconds", + "7", + "--encoder", + "ffmpeg-h264_nvenc-p4-cq21-yuv420p-faststart", + "--disk-free-before-bytes", + "1000000000000", + "--disk-free-post-extract-bytes", + "1000000000000", + "--disk-free-post-inference-bytes", + "1000000000000", + "--disk-free-post-artifacts-bytes", + "1000000000000", + "--disk-floor-bytes", + "0", + "--working-set-reserve-bytes", + str(reserve), + ), + ] + assert [argv for argv, _environment in commands] == expected_commands + run_argv, run_environment = commands[2] + assert run_argv == expected_commands[2] + assert run_environment is not None + assert run_environment["HF_HUB_OFFLINE"] == "1" + assert run_environment["TRANSFORMERS_OFFLINE"] == "1" + assert all("malicious" not in token for argv, _environment in commands for token in argv) + assert (output / "result.json").is_file() + assert sorted(path.name for path in (output / "source-frames").iterdir()) == [ + "frame-000001.png", + "frame-000002.png", + ] + + +def _effective_ddrnet_config( + source: Mapping[str, object], + *, + checkpoint_sha256: str, + checkpoint_bytes: int, + mapping_sha256: str, +) -> dict[str, object]: + path = REPOSITORY_ROOT / "config/perception/lab-v1-eomt-ddrnet-portable-v2.json" + config = json.loads(path.read_text()) + source_binding = config.pop("source_binding") + config.pop("effective_config_contract") + config.pop("profile_id") + config["schema_version"] = ddrnet.EFFECTIVE_CONFIG_SCHEMA + config["candidates"]["ddrnet"]["checkpoint_sha256"] = checkpoint_sha256 + config["candidates"]["ddrnet"]["checkpoint_size_bytes"] = checkpoint_bytes + config["dataset"]["mapping_sha256"] = mapping_sha256 + camera = cast(dict[str, object], source["camera_compute_job"]) + source_identity = cast(dict[str, object], source["source"]) + frame_count = cast(int, camera["frame_count"]) + config["ravnoves"] = { + "source_id": ( + f"portable-k1/{source_identity['session_id']}/" + f"{camera['source_id']}@{camera['input_sha256']}" + ), + "source_sha256": camera["input_sha256"], + "base_m4_result_id": None, + "expected_width": 800, + "expected_height": 600, + "expected_frame_count": frame_count, + "frame_indices": ddrnet._representative_frame_indices(frame_count), + "crop_contract": source_binding["crop_contract"], + } + return cast(dict[str, object], config) + + +def _prepare_eomt_input(root: Path, source: Mapping[str, object]) -> None: + root.mkdir() + frames = root / "source-frames" + frames.mkdir() + for sequence in (1, 2): + (frames / f"frame-{sequence:06d}.png").write_bytes(b"\x89PNG\r\n\x1a\n" + bytes([sequence])) + _write_canonical(root / "decode-repair.json", _decode_repair(2)) + _eomt_identity_result(root, source, source_frames_root=frames) + + +@pytest.mark.parametrize("tamper_statistics", [False, True], ids=("admitted", "tampered-stats")) +def test_ddrnet_adapter_uses_exact_candidate_argv_and_publishes_legacy_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + tamper_statistics: bool, +) -> None: + camera_root = tmp_path / "camera" + source = _camera_source(camera_root) + output = tmp_path / "output" + output.mkdir() + eomt_root = tmp_path / "eomt-result" + _prepare_eomt_input(eomt_root, source) + payloads = { + "ddrnet-checkpoint": b"checkpoint\n", + "ddrnet-goose-mapping": _mapping_payload(), + "ddrnet-goose-runner": b"runner\n", + "vegetation-policy": b"policy\n", + "vegetation-provider-map": b"provider\n", + } + assets: dict[str, Path] = {} + expectations: list[contract.AssetExpectation] = [] + identities: dict[str, tuple[str, int | None]] = {} + for asset_id, payload in payloads.items(): + path = tmp_path / "assets" / asset_id + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + assets[asset_id] = path + identities[asset_id] = (digest, len(payload)) + expectations.append( + contract.AssetExpectation( + asset_id, + f"/opt/nodedc/assets/{asset_id}", + "file", + "sha256", + digest, + len(payload), + ) + ) + expectations_tuple = tuple(sorted(expectations, key=lambda value: value.asset_id)) + checkpoint_sha256, checkpoint_bytes = identities["ddrnet-checkpoint"] + mapping_sha256, _mapping_bytes = identities["ddrnet-goose-mapping"] + policy_sha256, _policy_bytes = identities["vegetation-policy"] + provider_sha256, _provider_bytes = identities["vegetation-provider-map"] + monkeypatch.setattr(ddrnet, "DDRNET_ASSETS", expectations_tuple) + monkeypatch.setattr(ddrnet, "DDRNET_CHECKPOINT_SHA256", checkpoint_sha256) + monkeypatch.setattr(ddrnet, "DDRNET_CHECKPOINT_BYTES", checkpoint_bytes) + monkeypatch.setattr(ddrnet, "GOOSE_MAPPING_SHA256", mapping_sha256) + monkeypatch.setattr(ddrnet, "POLICY_SHA256", policy_sha256) + monkeypatch.setattr(ddrnet, "PROVIDER_MAP_SHA256", provider_sha256) + effective = _effective_ddrnet_config( + source, + checkpoint_sha256=checkpoint_sha256, + checkpoint_bytes=cast(int, checkpoint_bytes), + mapping_sha256=mapping_sha256, + ) + static = dict(effective) + static.pop("ravnoves") + monkeypatch.setattr( + ddrnet, + "DDRNET_STATIC_CONFIG_SHA256", + hashlib.sha256(contract.canonical_json(static)).hexdigest(), + ) + effective_path = tmp_path / "effective.json" + _write_canonical(effective_path, effective) + effective_sha256 = _sha256(effective_path) + request_path = tmp_path / "request.json" + _write_canonical( + request_path, + _request_document( + component="ddrnet", + source=source, + expectations=expectations_tuple, + identities=identities, + effective_sha256=effective_sha256, + ), + ) + layout = contract.RuntimeLayout( + request=request_path, + camera_job_root=camera_root, + output_root=output, + effective_ddrnet_config=effective_path, + eomt_result_root=eomt_root, + asset_paths=assets, + ) + commands: list[tuple[tuple[str, ...], Mapping[str, str] | None]] = [] + + def fake_runner( + argv: Sequence[str], + environment: Mapping[str, str] | None, + ) -> None: + command = tuple(argv) + commands.append((command, environment)) + staging = Path(command[command.index("--output") + 1]) + staging.mkdir() + archive_path = staging / "semantic-masks.zip" + with zipfile.ZipFile(archive_path, "x", compression=zipfile.ZIP_STORED) as archive: + for sequence in (1, 2): + archive.writestr( + f"masks/frame-{sequence:06d}.png", + _png_payload(value=sequence), + ) + candidate: dict[str, object] = { + "candidate_id": ddrnet.DDRNET_CANDIDATE_ID, + "candidate_key": "ddrnet", + "loaded_model_name": ddrnet.DDRNET_LOADED_MODEL, + "architecture_probe_failures": {}, + "checkpoint_size_bytes": checkpoint_bytes, + "checkpoint_sha256": checkpoint_sha256, + } + source_result = { + "source_id": cast(dict[str, object], effective["ravnoves"])["source_id"], + "input_count": 2, + "ground_truth_available": False, + "mapping_sha256": mapping_sha256, + } + aggregate = [800 * 600 if class_id in {1, 2} else 0 for class_id in range(64)] + if tamper_statistics: + aggregate[0] = 1 + semantics = { + "base_m4_result_id": None, + "mask_archive": { + "path": "semantic-masks.zip", + "sha256": _sha256(archive_path), + "byte_length": archive_path.stat().st_size, + "media_type": "application/zip", + "frame_count": 2, + "width": 800, + "height": 600, + "encoding": "uint8-class-id-png", + "sequence_binding": "sequence-0-to-masks/frame-000001.png", + }, + "taxonomy": ddrnet._sealed_taxonomy(assets["ddrnet-goose-mapping"]), + "aggregate_prediction_pixels": aggregate, + "center_crop_xyxy": [100, 0, 700, 600], + "outside_crop_state": "undefined", + } + result: dict[str, object] = { + "schema_version": ddrnet.DDRNET_RESULT_SCHEMA, + "lab_id": "LAB-V1", + "worker_id": "worker-006", + "mode": "ravnoves-video", + "candidate": candidate, + "source": source_result, + "video_semantics": semantics, + "preprocessing": cast(dict[str, object], effective["dataset"])["preprocessing"], + "metrics": {}, + "timing": {}, + "resource": {}, + "visual_cases": [], + "authority": dict(ddrnet.DDRNET_AUTHORITY), + } + identity_value = { + **{ + key: result[key] + for key in ( + "schema_version", + "candidate", + "source", + "video_semantics", + "preprocessing", + "metrics", + "timing", + "resource", + "visual_cases", + "authority", + ) + }, + "config_sha256": effective_sha256, + "policy_sha256": policy_sha256, + "provider_map_sha256": provider_sha256, + } + identity_sha256 = hashlib.sha256( + json.dumps(identity_value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + result["result_id"] = f"lab-v1-ravnoves-video-ddrnet-{identity_sha256}" + result["provenance"] = { + "config_sha256": effective_sha256, + "policy_sha256": policy_sha256, + "provider_map_sha256": provider_sha256, + "hostname": "sealed-worker", + "pid": 1, + } + (staging / "result.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + + if tamper_statistics: + with pytest.raises(contract.ComponentAdapterError, match="semantic statistics changed"): + ddrnet.execute_ddrnet_component( + request_path=request_path, + layout=layout, + command_runner=fake_runner, + work_root=tmp_path / "ddrnet-work", + ) + assert len(commands) == 1 + return + ddrnet.execute_ddrnet_component( + request_path=request_path, + layout=layout, + command_runner=fake_runner, + work_root=tmp_path / "ddrnet-work", + ) + + assert len(commands) == 1 + argv, environment = commands[0] + assert argv == ( + sys.executable, + str(assets["ddrnet-goose-runner"]), + "--mode", + "ravnoves-video", + "--candidate", + "ddrnet", + "--config", + str(effective_path), + "--policy", + str(assets["vegetation-policy"]), + "--provider-map", + str(assets["vegetation-provider-map"]), + "--checkpoint", + str(assets["ddrnet-checkpoint"]), + "--dataset-root", + str(tmp_path / "ddrnet-work"), + "--frames-root", + str(tmp_path / "ddrnet-work" / "source-frames"), + "--output", + str(output / ".ddrnet-component"), + "--limit", + "0", + "--visual-count", + "0", + ) + assert environment is not None and environment["HOME"] == "/tmp" + assert (output / "result.json").is_file() + assert (output / "semantic-masks.zip").is_file() + assert (output / "decode-repair.json").is_file() + + +def test_ddrnet_mask_archive_rejects_png_with_invalid_internal_crc(tmp_path: Path) -> None: + payload = bytearray(_png_payload()) + idat = payload.index(b"IDAT") + payload[idat + 4] ^= 0x01 + archive_path = tmp_path / "semantic-masks.zip" + with zipfile.ZipFile(archive_path, "x", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("masks/frame-000001.png", payload) + + with pytest.raises(contract.ComponentAdapterError, match="CRC changed"): + ddrnet._validate_mask_archive_inventory(archive_path, 1) + + +def test_ddrnet_mask_archive_decodes_filter_and_counts_classes(tmp_path: Path) -> None: + archive_path = tmp_path / "semantic-masks.zip" + with zipfile.ZipFile(archive_path, "x", compression=zipfile.ZIP_STORED) as archive: + archive.writestr( + "masks/frame-000001.png", + _png_payload(value=7, filter_type=1), + ) + + histogram = ddrnet._validate_mask_archive_inventory(archive_path, 1) + + assert histogram[7] == 800 * 600 + assert sum(histogram) == 800 * 600 + + +def test_ddrnet_mask_archive_rejects_class_id_outside_taxonomy(tmp_path: Path) -> None: + archive_path = tmp_path / "semantic-masks.zip" + with zipfile.ZipFile(archive_path, "x", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("masks/frame-000001.png", _png_payload(value=64)) + + with pytest.raises(contract.ComponentAdapterError, match="class id escaped"): + ddrnet._validate_mask_archive_inventory(archive_path, 1) + + +def test_ddrnet_snapshot_rejects_same_length_source_frame_tamper(tmp_path: Path) -> None: + eomt_root = tmp_path / "eomt-result" + frames_root = eomt_root / "source-frames" + frames_root.mkdir(parents=True) + frame = frames_root / "frame-000001.png" + original = b"\x89PNG\r\n\x1a\noriginal" + frame.write_bytes(original) + row = ddrnet.SourceFrameRow( + sequence=1, + path="source-frames/frame-000001.png", + byte_length=len(original), + sha256=hashlib.sha256(original).hexdigest(), + ) + frame.write_bytes(b"\x89PNG\r\n\x1a\ntampered") + work = tmp_path / "work" + work.mkdir() + + with pytest.raises(contract.ComponentAdapterError, match="source frame identity changed"): + ddrnet._snapshot_source_frames(eomt_root, (row,), work / "source-frames") + + +def test_eomt_mask_archive_rejects_link_member(tmp_path: Path) -> None: + archive_path = tmp_path / "masks.tar.gz" + with tarfile.open(archive_path, mode="w:gz") as archive: + directory = tarfile.TarInfo("semantic-masks") + directory.type = tarfile.DIRTYPE + archive.addfile(directory) + link = tarfile.TarInfo("semantic-masks/frame-000001.png") + link.type = tarfile.SYMTYPE + link.linkname = "/etc/passwd" + archive.addfile(link) + + with pytest.raises(contract.ComponentAdapterError, match="member type changed"): + eomt._validate_mask_archive_inventory(archive_path, 1) diff --git a/tests/test_observatory_portable_lab_v1_local_runners.py b/tests/test_observatory_portable_lab_v1_local_runners.py index ab0852a..f5a117c 100644 --- a/tests/test_observatory_portable_lab_v1_local_runners.py +++ b/tests/test_observatory_portable_lab_v1_local_runners.py @@ -552,7 +552,11 @@ def test_component_adapter_images_are_thin_offline_wrappers() -> None: ).read_text(encoding="utf-8") assert "FROM nvcr.io/nvidia/tritonserver:26.06-py3@sha256:" in eomt - assert "FROM ndc/mission-core-lab-v1-goose:sg3.2.0-cu117-v1" in ddrnet + assert ( + "FROM sha256:" + "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd" + in ddrnet + ) assert "COPY portable_lab_v1_component_adapter.py" in eomt assert "COPY portable_lab_v1_component_adapter.py" in ddrnet assert 'ENTRYPOINT ["python3"]' in eomt