From e958729e9d82278683c940e15f03fb2157805578 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 31 Aug 2026 21:31:45 +0300 Subject: [PATCH] fix(worker): seal coordinator Python dependencies --- .../Dockerfile.worker-006-agent | 45 +- .../Install-Worker006AgentImage.ps1 | 463 ++++++++++++++++-- .../promote_portable_lab_v1_ready.py | 40 +- .../worker-006-agent-build-context.json | 27 +- ...orker-006-agent-install-plan.template.json | 72 ++- ...t_observatory_portable_lab_v1_promotion.py | 101 ++++ ...observatory_worker_agent_image_artifact.py | 150 +++++- .../test_worker_006_agent_image_installer.py | 113 ++++- 8 files changed, 956 insertions(+), 55 deletions(-) diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent b/experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent index fd73e4e..79bbb38 100644 --- a/experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent +++ b/experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent @@ -2,9 +2,14 @@ FROM nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012de ARG NODEDC_SOURCE_REVISION ARG NODEDC_BUILD_CONTEXT_SHA256 +ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_SHA256 +ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_FILE_COUNT +ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_BYTE_LENGTH +ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_SHA256 +ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_BYTE_LENGTH ARG SOURCE_DATE_EPOCH -ENV PYTHONPATH=/opt/nodedc/mission-core/src \ +ENV PYTHONPATH=/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps \ PYTHONNOUSERSITE=1 \ PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 @@ -14,6 +19,12 @@ WORKDIR /opt/nodedc/mission-core COPY src/k1link ./src/k1link COPY experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json \ ./release/worker-006-agent-build-context.json +# Evidence-only mirror of the exact-layer installer contract: these two inputs are +# materialized from the separately sealed build-only dependency bundle. They are +# embedded in the image and must never be supplied as runtime bind mounts. +COPY python-dependency-bundle ./deps +COPY worker-006-agent-python-dependency-bundle.json \ + ./release/worker-006-agent-python-dependency-bundle.json RUN mkdir -p /run/nodedc/registries \ && chmod 0555 /run/nodedc /run/nodedc/registries \ @@ -27,6 +38,31 @@ RUN mkdir -p /run/nodedc/registries \ * ) ;; \ esac \ && test "${#NODEDC_BUILD_CONTEXT_SHA256}" -eq 64 \ + && case "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_SHA256}" in \ + *[!0-9a-f]* ) exit 64 ;; \ + * ) ;; \ + esac \ + && test "${#NODEDC_PYTHON_DEPENDENCY_BUNDLE_SHA256}" -eq 64 \ + && case "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_FILE_COUNT}" in \ + ''|*[!0-9]* ) exit 64 ;; \ + * ) ;; \ + esac \ + && test "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_FILE_COUNT}" -gt 0 \ + && case "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_BYTE_LENGTH}" in \ + ''|*[!0-9]* ) exit 64 ;; \ + * ) ;; \ + esac \ + && test "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_BYTE_LENGTH}" -gt 0 \ + && case "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_SHA256}" in \ + *[!0-9a-f]* ) exit 64 ;; \ + * ) ;; \ + esac \ + && test "${#NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_SHA256}" -eq 64 \ + && case "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_BYTE_LENGTH}" in \ + ''|*[!0-9]* ) exit 64 ;; \ + * ) ;; \ + esac \ + && test "${NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_BYTE_LENGTH}" -gt 0 \ && case "${SOURCE_DATE_EPOCH}" in \ ''|*[!0-9]* ) exit 64 ;; \ * ) ;; \ @@ -48,7 +84,12 @@ LABEL org.opencontainers.image.title="NODE.DC Observatory Worker 006 agent" \ com.nodedc.models="external" \ com.nodedc.runtime-registries="external-read-only" \ com.nodedc.base-image.sha256="58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" \ - com.nodedc.build-context.sha256="${NODEDC_BUILD_CONTEXT_SHA256}" + com.nodedc.build-context.sha256="${NODEDC_BUILD_CONTEXT_SHA256}" \ + com.nodedc.python-dependency-bundle.sha256="${NODEDC_PYTHON_DEPENDENCY_BUNDLE_SHA256}" \ + com.nodedc.python-dependency-bundle.file-count="${NODEDC_PYTHON_DEPENDENCY_BUNDLE_FILE_COUNT}" \ + com.nodedc.python-dependency-bundle.byte-length="${NODEDC_PYTHON_DEPENDENCY_BUNDLE_BYTE_LENGTH}" \ + com.nodedc.python-dependency-bundle.manifest.sha256="${NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_SHA256}" \ + com.nodedc.python-dependency-bundle.manifest.byte-length="${NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_BYTE_LENGTH}" # The current private token and sealed release mounts are provisioned for root. # Runtime confinement is mandatory in the install plan: read-only rootfs, diff --git a/experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1 b/experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1 index f48c965..4ae6084 100644 --- a/experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1 +++ b/experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1 @@ -10,7 +10,15 @@ param( [Parameter(Mandatory = $true)] [ValidatePattern("^[a-f0-9]{64}$")] - [string]$ExpectedStagedSnapshotSha256 + [string]$ExpectedStagedSnapshotSha256, + + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$DependencyBundleRoot, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedDependencyBundleSha256 ) Set-StrictMode -Version Latest @@ -21,6 +29,7 @@ $RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\") $StagedSnapshotRoot = Join-Path ( $RuntimeRoot ) "staging\observatory-worker-agent-$SourceRevision" +$DependencyBundleRoot = [IO.Path]::GetFullPath($DependencyBundleRoot).TrimEnd("\") $ContextManifestRelativePath = ( "experiments/perception/worker/observatory_portable/" + "worker-006-agent-build-context.json" @@ -30,6 +39,9 @@ $DockerfileRelativePath = ( "Dockerfile.worker-006-agent" ) $SourceTreeRelativePath = "src/k1link" +$DependencyManifestImagePath = ( + "/opt/nodedc/mission-core/release/worker-006-agent-python-dependency-bundle.json" +) $BaseImageSha256 = ( "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" ) @@ -40,11 +52,37 @@ $ImageWorkdir = "/opt/nodedc/mission-core" $ImageEntrypoint = '["python3","-m","k1link.observatory.m49_worker_container_main"]' $ImageCommand = '[]' $FixedEnvironment = @( - "PYTHONPATH=/opt/nodedc/mission-core/src", + "PYTHONPATH=/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps", "PYTHONNOUSERSITE=1", "PYTHONDONTWRITEBYTECODE=1", "PYTHONUNBUFFERED=1" ) +$PythonSmokeProgram = ( + "from importlib.metadata import version; " + + "expected={" + + "`"httpx`":`"0.28.1`"," + + "`"lz4`":`"4.4.5`"," + + "`"paho-mqtt`":`"2.1.0`"," + + "`"PyYAML`":`"6.0.3`"," + + "`"typing_extensions`":`"4.16.0`"}; " + + "assert all(version(name) == wanted for name,wanted in expected.items()); " + + "import httpx; import lz4.frame; import paho.mqtt.client as mqtt; " + + "import typing_extensions; import yaml; " + + "payload=b`"worker-006-offline`"; " + + "assert lz4.frame.decompress(lz4.frame.compress(payload)) == payload; " + + "url=httpx.URL(`"https://offline.invalid/probe`",params={`"worker`":`"006`"}); " + + "request=httpx.Request(`"GET`",url); " + + "assert request.url.params[`"worker`"] == `"006`"; " + + "document={`"worker`":6,`"offline`":True}; " + + "assert yaml.safe_load(yaml.safe_dump(document)) == document; " + + "client=mqtt.Client(callback_api_version=mqtt.CallbackAPIVersion.VERSION2); " + + "client.user_data_set(document); " + + "assert typing_extensions.TypeAlias is not None; " + + "import k1link.observatory.m49_worker_container_main as entrypoint; " + + "import k1link.observatory.m49_worker_service as composition; " + + "assert callable(entrypoint.main); " + + "assert callable(composition.compose_installed_m49_worker_service)" +) function Get-FileSha256 { param([string]$Path) @@ -54,7 +92,7 @@ function Get-FileSha256 { -not ($item -is [IO.FileInfo]) -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) ) { - throw "Worker 006 staged source is not a regular file" + throw "Worker 006 installation input is not a regular file" } return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant() } @@ -133,7 +171,7 @@ function Get-StagedSnapshotInspection { $context = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json if ( [string]$context.schema_version -cne - "missioncore.observatory-worker-agent-build-context/v1" -or + "missioncore.observatory-worker-agent-build-context/v2" -or [string]$context.worker_id -cne "worker-006" -or [string]$context.base_image.sha256 -cne $BaseImageSha256 -or [bool]$context.base_image.pull_allowed @@ -153,6 +191,40 @@ function Get-StagedSnapshotInspection { ) { throw "Worker 006 staged build-context entries changed" } + $externalBuildInputs = @($context.external_build_inputs) + if ($externalBuildInputs.Count -ne 1) { + throw "Worker 006 staged dependency build-input contract changed" + } + $dependencyInput = $externalBuildInputs[0] + $expectedDependencyProvenance = ( + "already-proven Python dependency tree from the previously accepted " + + "Worker; build input only" + ) + if ( + [string]$dependencyInput.role -cne "python-dependency-bundle" -or + -not [bool]$dependencyInput.required -or + [string]$dependencyInput.provenance_requirement -cne + $expectedDependencyProvenance -or + [string]$dependencyInput.image_path -cne + "/opt/nodedc/mission-core/deps" -or + [string]$dependencyInput.manifest_image_path -cne + $DependencyManifestImagePath -or + [string]$dependencyInput.manifest_schema_version -cne + "missioncore.observatory-worker-agent-python-dependency-bundle/v1" -or + [string]$dependencyInput.identity.canonicalization -cne + "utf8-path-nul-length-nul-sha256-lf-v1" -or + [string]$dependencyInput.identity.path_order -cne "ordinal" -or + [bool]$dependencyInput.included_in_git_archive -or + -not [bool]$dependencyInput.copied_into_image -or + [bool]$dependencyInput.runtime_bind_allowed -or + [bool]$dependencyInput.network_install_allowed -or + [string]$context.runtime.pythonpath -cne + "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps" -or + -not [bool]$context.runtime.python_dependency_bundle_baked_into_image -or + [bool]$context.runtime.python_dependency_bundle_runtime_bind_allowed + ) { + throw "Worker 006 staged dependency build-input contract is invalid" + } $files = New-Object "Collections.Generic.List[object]" foreach ($item in $sourceItems) { @@ -163,20 +235,28 @@ function Get-StagedSnapshotInspection { $files.Add((Get-Item -LiteralPath $dockerfilePath -Force)) $files.Add((Get-Item -LiteralPath $manifestPath -Force)) - $rows = New-Object "Collections.Generic.List[object]" + $rowsByPath = @{} foreach ($file in $files) { $relative = $file.FullName.Substring($Root.Length).TrimStart("\") $relative = $relative.Replace("\", "/") - if (-not $relative -or $relative -match "[\x00\r\n]") { + if ( + -not $relative -or + $relative -match "[\x00\r\n]" -or + $rowsByPath.ContainsKey($relative) + ) { throw "Worker 006 staged snapshot path is invalid" } - $rows.Add([pscustomobject]@{ + $rowsByPath.Add($relative, [pscustomobject]@{ path = $relative byte_length = [int64]$file.Length sha256 = Get-FileSha256 $file.FullName }) } - $orderedRows = @($rows | Sort-Object -Property path -CaseSensitive) + $orderedPaths = [string[]]@($rowsByPath.Keys) + [Array]::Sort($orderedPaths, [StringComparer]::Ordinal) + $orderedRows = @( + $orderedPaths | ForEach-Object { $rowsByPath[$_] } + ) $canonical = New-Object Text.StringBuilder [int64]$totalBytes = 0 foreach ($row in $orderedRows) { @@ -207,6 +287,92 @@ function Get-StagedSnapshotInspection { } } +function Get-DependencyBundleInspection { + param([string]$Root) + + $rootItem = Get-Item -LiteralPath $Root -Force + if ( + -not ($rootItem -is [IO.DirectoryInfo]) -or + ($rootItem.Attributes -band [IO.FileAttributes]::ReparsePoint) + ) { + throw "Worker 006 Python dependency bundle root is invalid" + } + $items = @(Get-ChildItem -LiteralPath $rootItem.FullName -Recurse -Force) + if ($items.Count -eq 0) { + throw "Worker 006 Python dependency bundle is empty" + } + + $files = New-Object "Collections.Generic.List[object]" + foreach ($item in $items) { + if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Worker 006 Python dependency bundle contains a reparse point" + } + if ($item -is [IO.FileInfo]) { + $files.Add($item) + continue + } + if (-not ($item -is [IO.DirectoryInfo])) { + throw "Worker 006 Python dependency bundle contains a non-file entry" + } + } + if ($files.Count -eq 0) { + throw "Worker 006 Python dependency bundle contains no regular files" + } + + $rowsByPath = @{} + foreach ($file in $files) { + $relative = $file.FullName.Substring($rootItem.FullName.Length).TrimStart("\") + $relative = $relative.Replace("\", "/") + if ( + -not $relative -or + $relative -match "[\x00\r\n]" -or + $rowsByPath.ContainsKey($relative) + ) { + throw "Worker 006 Python dependency bundle path is invalid" + } + $rowsByPath.Add($relative, [pscustomobject]@{ + path = $relative + byte_length = [int64]$file.Length + sha256 = Get-FileSha256 $file.FullName + }) + } + $orderedPaths = [string[]]@($rowsByPath.Keys) + [Array]::Sort($orderedPaths, [StringComparer]::Ordinal) + $orderedRows = @( + $orderedPaths | ForEach-Object { $rowsByPath[$_] } + ) + $canonical = New-Object Text.StringBuilder + [int64]$totalBytes = 0 + foreach ($row in $orderedRows) { + [void]$canonical.Append([string]$row.path) + [void]$canonical.Append([char]0) + [void]$canonical.Append( + ([int64]$row.byte_length).ToString([Globalization.CultureInfo]::InvariantCulture) + ) + [void]$canonical.Append([char]0) + [void]$canonical.Append([string]$row.sha256) + [void]$canonical.Append("`n") + $totalBytes += [int64]$row.byte_length + } + $utf8 = New-Object Text.UTF8Encoding($false) + $canonicalBytes = $utf8.GetBytes($canonical.ToString()) + $hasher = [Security.Cryptography.SHA256]::Create() + try { + $digestBytes = $hasher.ComputeHash($canonicalBytes) + } + finally { + $hasher.Dispose() + } + $digest = ([BitConverter]::ToString($digestBytes)).Replace("-", "").ToLowerInvariant() + return [pscustomobject]@{ + sha256 = $digest + file_count = [int64]$orderedRows.Count + byte_length = $totalBytes + canonicalization = "utf8-path-nul-length-nul-sha256-lf-v1" + files = @($orderedRows) + } +} + function New-StagedSnapshotManifest { param([object]$SnapshotInspection) @@ -240,6 +406,46 @@ function New-StagedSnapshotManifest { } } +function New-DependencyBundleManifest { + param([object]$DependencyInspection) + + $stagingRoot = Split-Path -Parent $StagedSnapshotRoot + $manifestPath = Join-Path ( + $stagingRoot + ) ("observatory-worker-agent-python-deps-{0}.json" -f [Guid]::NewGuid().ToString("N")) + $payload = [ordered]@{ + schema_version = "missioncore.observatory-worker-agent-python-dependency-bundle/v1" + bundle_sha256 = [string]$DependencyInspection.sha256 + canonicalization = [string]$DependencyInspection.canonicalization + file_count = [int64]$DependencyInspection.file_count + byte_length = [int64]$DependencyInspection.byte_length + required_distributions = [ordered]@{ + httpx = "0.28.1" + lz4 = "4.4.5" + "paho-mqtt" = "2.1.0" + PyYAML = "6.0.3" + typing_extensions = "4.16.0" + } + files = @($DependencyInspection.files) + } + $json = $payload | ConvertTo-Json -Compress -Depth 5 + $utf8 = New-Object Text.UTF8Encoding($false) + try { + [IO.File]::WriteAllText($manifestPath, $json, $utf8) + return [pscustomobject]@{ + path = $manifestPath + sha256 = Get-FileSha256 $manifestPath + byte_length = [int64](Get-Item -LiteralPath $manifestPath -Force).Length + } + } + catch { + if (Test-Path -LiteralPath $manifestPath -PathType Leaf) { + Remove-Item -LiteralPath $manifestPath -Force + } + throw + } +} + function Get-ImageInspection { param([string]$Reference) @@ -258,10 +464,21 @@ function Assert-ImageContract { param( [object]$Image, [object]$BaseImage, - [string]$EmbeddedManifestSha256 + [string]$EmbeddedManifestSha256, + [object]$DependencyInspection, + [object]$DependencyManifest ) $labels = $Image.Config.Labels + $dependencyFileCount = ( + [int64]$DependencyInspection.file_count + ).ToString([Globalization.CultureInfo]::InvariantCulture) + $dependencyByteLength = ( + [int64]$DependencyInspection.byte_length + ).ToString([Globalization.CultureInfo]::InvariantCulture) + $dependencyManifestByteLength = ( + [int64]$DependencyManifest.byte_length + ).ToString([Globalization.CultureInfo]::InvariantCulture) $expectedLabels = [ordered]@{ "org.opencontainers.image.title" = "NODE.DC Observatory Worker 006 agent" "org.opencontainers.image.source" = "NODEDC_MISSION_CORE" @@ -278,6 +495,11 @@ function Assert-ImageContract { "com.nodedc.build-context.sha256" = $ExpectedGitArchiveSha256 "com.nodedc.staged-snapshot.sha256" = $ExpectedStagedSnapshotSha256 "com.nodedc.embedded-snapshot-manifest.sha256" = $EmbeddedManifestSha256 + "com.nodedc.python-dependency-bundle.sha256" = [string]$DependencyInspection.sha256 + "com.nodedc.python-dependency-bundle.file-count" = $dependencyFileCount + "com.nodedc.python-dependency-bundle.byte-length" = $dependencyByteLength + "com.nodedc.python-dependency-bundle.manifest.sha256" = [string]$DependencyManifest.sha256 + "com.nodedc.python-dependency-bundle.manifest.byte-length" = $dependencyManifestByteLength "com.nodedc.build-method" = $BuildMethod } foreach ($key in $expectedLabels.Keys) { @@ -342,7 +564,9 @@ function Invoke-InstalledImageSmoke { param( [string]$Tag, [string]$SnapshotRoot, - [object]$EmbeddedManifest + [object]$EmbeddedManifest, + [string]$DependencyRoot, + [object]$DependencyManifest ) $smokeScript = ( @@ -376,11 +600,34 @@ function Invoke-InstalledImageSmoke { "for image_path do relative=`${image_path#/opt/nodedc/mission-core/}; " + "cmp -s `"`$image_path`" `"/nodedc-verify-source/`$relative`" || exit 73; " + "done' sh {} +; " + + "test -d /opt/nodedc/mission-core/deps; " + + "test -z `"`$(find /opt/nodedc/mission-core/deps -mindepth 1 " + + "! -type d ! -type f -print -quit)`"; " + + "test -z `"`$(find /nodedc-verify-deps -mindepth 1 " + + "! -type d ! -type f -print -quit)`"; " + + "cmp -s /nodedc-verify-deps-manifest.json " + + "$DependencyManifestImagePath; " + + "test `"`$(sha256sum $DependencyManifestImagePath | cut -d' ' -f1)`" = " + + "$($DependencyManifest.sha256); " + + "find /nodedc-verify-deps -mindepth 1 -type d -exec sh -c '" + + "for source_path do relative=`${source_path#/nodedc-verify-deps/}; " + + "test -d `"/opt/nodedc/mission-core/deps/`$relative`" || exit 74; " + + "done' sh {} +; " + + "find /opt/nodedc/mission-core/deps -mindepth 1 -type d -exec sh -c '" + + "for image_path do relative=`${image_path#/opt/nodedc/mission-core/deps/}; " + + "test -d `"/nodedc-verify-deps/`$relative`" || exit 75; " + + "done' sh {} +; " + + "find /nodedc-verify-deps -type f -exec sh -c '" + + "for source_path do relative=`${source_path#/nodedc-verify-deps/}; " + + "cmp -s `"`$source_path`" " + + "`"/opt/nodedc/mission-core/deps/`$relative`" || exit 76; " + + "done' sh {} +; " + + "find /opt/nodedc/mission-core/deps -type f -exec sh -c '" + + "for image_path do relative=`${image_path#/opt/nodedc/mission-core/deps/}; " + + "cmp -s `"`$image_path`" `"/nodedc-verify-deps/`$relative`" || exit 77; " + + "done' sh {} +; " + "cd /opt/nodedc/mission-core; " + - "python3 -B -c 'import k1link.observatory.m49_worker_container_main as entrypoint; " + - "import k1link.observatory.m49_worker_service as composition; " + - "assert callable(entrypoint.main); " + - "assert callable(composition.compose_installed_m49_worker_service)'" + "python3 -B -c '$PythonSmokeProgram'" ) $sourceMount = ( "type=bind,source=$SnapshotRoot," + @@ -390,6 +637,14 @@ function Invoke-InstalledImageSmoke { "type=bind,source=$($EmbeddedManifest.path)," + "target=/nodedc-verify-snapshot.json,readonly" ) + $dependencyMount = ( + "type=bind,source=$DependencyRoot," + + "target=/nodedc-verify-deps,readonly" + ) + $dependencyManifestMount = ( + "type=bind,source=$($DependencyManifest.path)," + + "target=/nodedc-verify-deps-manifest.json,readonly" + ) docker run ` --rm ` --network none ` @@ -399,6 +654,8 @@ function Invoke-InstalledImageSmoke { --security-opt no-new-privileges ` --mount $sourceMount ` --mount $manifestMount ` + --mount $dependencyMount ` + --mount $dependencyManifestMount ` --entrypoint /bin/sh ` $Tag ` -c $smokeScript @@ -414,11 +671,13 @@ function New-OutputContract { [object]$Image, [object]$LayerInspection, [object]$SnapshotInspection, - [object]$EmbeddedManifest + [object]$EmbeddedManifest, + [object]$DependencyInspection, + [object]$DependencyManifest ) return [ordered]@{ - schema_version = "missioncore.observatory-worker-agent-image-installation/v1" + schema_version = "missioncore.observatory-worker-agent-image-installation/v2" status = $Status worker_id = "worker-006" build_method = $BuildMethod @@ -432,6 +691,12 @@ function New-OutputContract { staged_snapshot_canonicalization = [string]$SnapshotInspection.canonicalization embedded_snapshot_manifest_sha256 = [string]$EmbeddedManifest.sha256 embedded_snapshot_manifest_byte_length = [int64]$EmbeddedManifest.byte_length + python_dependency_bundle_sha256 = [string]$DependencyInspection.sha256 + python_dependency_bundle_file_count = [int64]$DependencyInspection.file_count + python_dependency_bundle_byte_length = [int64]$DependencyInspection.byte_length + python_dependency_bundle_canonicalization = [string]$DependencyInspection.canonicalization + python_dependency_bundle_manifest_sha256 = [string]$DependencyManifest.sha256 + python_dependency_bundle_manifest_byte_length = [int64]$DependencyManifest.byte_length } base_image_sha256 = $BaseImageSha256 derived_image_sha256 = ([string]$Image.Id).Substring(7) @@ -450,11 +715,13 @@ function New-OutputContract { } runtime_contract = [ordered]@{ workdir = $ImageWorkdir + pythonpath = "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps" entrypoint = @("python3", "-m", "k1link.observatory.m49_worker_container_main") command = @() authority = "observation-only" models = "external" runtime_registries = "external-read-only" + python_dependencies = "embedded-sealed-build-input" } smoke = [ordered]@{ network = "none" @@ -462,6 +729,10 @@ function New-OutputContract { staged_source_bytes = "matched" embedded_context_bytes = "matched" embedded_snapshot_manifest = "matched" + python_dependency_bundle_bytes = "matched" + python_dependency_bundle_manifest = "matched" + required_dependency_versions = "matched" + offline_dependency_operations = "passed" result = "passed" } } @@ -472,10 +743,31 @@ if (-not (Test-Path -LiteralPath $StagedSnapshotRoot -PathType Container)) { } $snapshotBefore = Get-StagedSnapshotInspection $StagedSnapshotRoot if ([string]$snapshotBefore.sha256 -cne $ExpectedStagedSnapshotSha256) { - throw "Worker 006 staged snapshot identity changed" + throw ( + "Worker 006 staged snapshot identity changed: observed=" + + [string]$snapshotBefore.sha256 + ) } -$embeddedManifest = New-StagedSnapshotManifest $snapshotBefore +$dependencyRootItem = Get-Item -LiteralPath $DependencyBundleRoot -Force +if ( + -not ($dependencyRootItem -is [IO.DirectoryInfo]) -or + ($dependencyRootItem.Attributes -band [IO.FileAttributes]::ReparsePoint) +) { + throw "Worker 006 Python dependency bundle root is unavailable" +} +$DependencyBundleRoot = $dependencyRootItem.FullName.TrimEnd("\") +$dependencyBefore = Get-DependencyBundleInspection $DependencyBundleRoot +if ([string]$dependencyBefore.sha256 -cne $ExpectedDependencyBundleSha256) { + throw ( + "Worker 006 Python dependency bundle identity changed: observed=" + + [string]$dependencyBefore.sha256 + ) +} +$embeddedManifest = $null +$dependencyManifest = $null try { + $embeddedManifest = New-StagedSnapshotManifest $snapshotBefore + $dependencyManifest = New-DependencyBundleManifest $dependencyBefore $base = Get-ImageInspection $BaseImageReference if ([string]$base.Id -cne $BaseImageReference) { throw "Worker 006 base image identity changed" @@ -503,16 +795,25 @@ try { $layerInspection = Assert-ImageContract ` $image ` $base ` - ([string]$embeddedManifest.sha256) - Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest + ([string]$embeddedManifest.sha256) ` + $dependencyBefore ` + $dependencyManifest + Invoke-InstalledImageSmoke ` + $tag ` + $StagedSnapshotRoot ` + $embeddedManifest ` + $DependencyBundleRoot ` + $dependencyManifest New-OutputContract ` "already-installed" ` $tag ` $image ` $layerInspection ` $snapshotBefore ` - $embeddedManifest | - ConvertTo-Json -Compress -Depth 9 + $embeddedManifest ` + $dependencyBefore ` + $dependencyManifest | + ConvertTo-Json -Compress -Depth 10 return } @@ -530,13 +831,16 @@ try { $copyScript = ( "set -eu; " + "test ! -e /opt/nodedc/mission-core; " + - "mkdir -p /opt/nodedc/mission-core/src /opt/nodedc/mission-core/release; " + + "mkdir -p /opt/nodedc/mission-core/src /opt/nodedc/mission-core/release " + + "/opt/nodedc/mission-core/deps; " + "mkdir -p /run/nodedc/registries; " + "cp -a /nodedc-build-source/src/k1link /opt/nodedc/mission-core/src/k1link; " + + "cp -a /nodedc-build-deps/. /opt/nodedc/mission-core/deps/; " + "cp /nodedc-build-source/$ContextManifestRelativePath " + "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " + "cp /nodedc-build-snapshot.json " + "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " + + "cp /nodedc-build-deps-manifest.json $DependencyManifestImagePath; " + "find /nodedc-build-source/src/k1link -type f -exec sh -c '" + "for source_path do relative=`${source_path#/nodedc-build-source/}; " + "cmp -s `"`$source_path`" `"/opt/nodedc/mission-core/`$relative`" || exit 70; " + @@ -545,17 +849,36 @@ try { "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " + "cmp -s /nodedc-build-snapshot.json " + "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " + + "cmp -s /nodedc-build-deps-manifest.json $DependencyManifestImagePath; " + + "test -z `"`$(find /nodedc-build-deps -mindepth 1 " + + "! -type d ! -type f -print -quit)`"; " + + "test -z `"`$(find /opt/nodedc/mission-core/deps -mindepth 1 " + + "! -type d ! -type f -print -quit)`"; " + + "find /nodedc-build-deps -mindepth 1 -type d -exec sh -c '" + + "for source_path do relative=`${source_path#/nodedc-build-deps/}; " + + "test -d `"/opt/nodedc/mission-core/deps/`$relative`" || exit 71; " + + "done' sh {} +; " + + "find /opt/nodedc/mission-core/deps -mindepth 1 -type d -exec sh -c '" + + "for image_path do relative=`${image_path#/opt/nodedc/mission-core/deps/}; " + + "test -d `"/nodedc-build-deps/`$relative`" || exit 72; " + + "done' sh {} +; " + + "find /nodedc-build-deps -type f -exec sh -c '" + + "for source_path do relative=`${source_path#/nodedc-build-deps/}; " + + "cmp -s `"`$source_path`" " + + "`"/opt/nodedc/mission-core/deps/`$relative`" || exit 73; " + + "done' sh {} +; " + + "find /opt/nodedc/mission-core/deps -type f -exec sh -c '" + + "for image_path do relative=`${image_path#/opt/nodedc/mission-core/deps/}; " + + "cmp -s `"`$image_path`" `"/nodedc-build-deps/`$relative`" || exit 74; " + + "done' sh {} +; " + "test ! -e /opt/nodedc/mission-core/experiments; " + "find /opt/nodedc/mission-core -type d -exec chmod 0555 {} +; " + "find /opt/nodedc/mission-core -type f -exec chmod 0444 {} +; " + "chmod 0555 /run/nodedc /run/nodedc/registries; " + "cd /opt/nodedc/mission-core; " + - "PYTHONPATH=/opt/nodedc/mission-core/src PYTHONNOUSERSITE=1 " + - "PYTHONDONTWRITEBYTECODE=1 python3 -B -c '" + - "import k1link.observatory.m49_worker_container_main as entrypoint; " + - "import k1link.observatory.m49_worker_service as composition; " + - "assert callable(entrypoint.main); " + - "assert callable(composition.compose_installed_m49_worker_service)'" + "PYTHONPATH=/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps " + + "PYTHONNOUSERSITE=1 PYTHONDONTWRITEBYTECODE=1 " + + "python3 -B -c '$PythonSmokeProgram'" ) $sourceMount = ( "type=bind,source=$StagedSnapshotRoot," + @@ -565,6 +888,14 @@ try { "type=bind,source=$($embeddedManifest.path)," + "target=/nodedc-build-snapshot.json,readonly" ) + $dependencyMount = ( + "type=bind,source=$DependencyBundleRoot," + + "target=/nodedc-build-deps,readonly" + ) + $dependencyManifestMount = ( + "type=bind,source=$($dependencyManifest.path)," + + "target=/nodedc-build-deps-manifest.json,readonly" + ) $containerId = $null $imageCommitted = $false $committedImageId = $null @@ -578,6 +909,8 @@ try { --security-opt no-new-privileges ` --mount $sourceMount ` --mount $manifestMount ` + --mount $dependencyMount ` + --mount $dependencyManifestMount ` --entrypoint /bin/sh ` $BaseImageReference ` -c $copyScript @@ -606,6 +939,35 @@ try { if ((Get-FileSha256 $embeddedManifest.path) -cne [string]$embeddedManifest.sha256) { throw "Worker 006 embedded snapshot manifest changed during installation" } + $dependencyAfter = Get-DependencyBundleInspection $DependencyBundleRoot + if ( + [string]$dependencyAfter.sha256 -cne $ExpectedDependencyBundleSha256 -or + [int64]$dependencyAfter.file_count -ne + [int64]$dependencyBefore.file_count -or + [int64]$dependencyAfter.byte_length -ne + [int64]$dependencyBefore.byte_length + ) { + throw ( + "Worker 006 Python dependency bundle changed during installation: " + + "observed=" + [string]$dependencyAfter.sha256 + ) + } + if ( + (Get-FileSha256 $dependencyManifest.path) -cne + [string]$dependencyManifest.sha256 + ) { + throw "Worker 006 Python dependency manifest changed during installation" + } + + $dependencyFileCount = ( + [int64]$dependencyBefore.file_count + ).ToString([Globalization.CultureInfo]::InvariantCulture) + $dependencyByteLength = ( + [int64]$dependencyBefore.byte_length + ).ToString([Globalization.CultureInfo]::InvariantCulture) + $dependencyManifestByteLength = ( + [int64]$dependencyManifest.byte_length + ).ToString([Globalization.CultureInfo]::InvariantCulture) $changes = @( "--change", "WORKDIR $ImageWorkdir", @@ -630,6 +992,26 @@ try { "LABEL com.nodedc.embedded-snapshot-manifest.sha256=" + [string]$embeddedManifest.sha256 ), + "--change", ( + "LABEL com.nodedc.python-dependency-bundle.sha256=" + + [string]$dependencyBefore.sha256 + ), + "--change", ( + "LABEL com.nodedc.python-dependency-bundle.file-count=" + + $dependencyFileCount + ), + "--change", ( + "LABEL com.nodedc.python-dependency-bundle.byte-length=" + + $dependencyByteLength + ), + "--change", ( + "LABEL com.nodedc.python-dependency-bundle.manifest.sha256=" + + [string]$dependencyManifest.sha256 + ), + "--change", ( + "LABEL com.nodedc.python-dependency-bundle.manifest.byte-length=" + + $dependencyManifestByteLength + ), "--change", "LABEL com.nodedc.build-method=$BuildMethod" ) foreach ($value in $FixedEnvironment) { @@ -663,16 +1045,25 @@ try { $layerInspection = Assert-ImageContract ` $image ` $base ` - ([string]$embeddedManifest.sha256) - Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest + ([string]$embeddedManifest.sha256) ` + $dependencyBefore ` + $dependencyManifest + Invoke-InstalledImageSmoke ` + $tag ` + $StagedSnapshotRoot ` + $embeddedManifest ` + $DependencyBundleRoot ` + $dependencyManifest New-OutputContract ` "installed" ` $tag ` $image ` $layerInspection ` $snapshotBefore ` - $embeddedManifest | - ConvertTo-Json -Compress -Depth 9 + $embeddedManifest ` + $dependencyBefore ` + $dependencyManifest | + ConvertTo-Json -Compress -Depth 10 } catch { $installationFailure = $_ @@ -695,4 +1086,10 @@ finally { ) { Remove-Item -LiteralPath $embeddedManifest.path -Force } + if ( + $null -ne $dependencyManifest -and + (Test-Path -LiteralPath $dependencyManifest.path -PathType Leaf) + ) { + Remove-Item -LiteralPath $dependencyManifest.path -Force + } } diff --git a/experiments/perception/worker/observatory_portable/promote_portable_lab_v1_ready.py b/experiments/perception/worker/observatory_portable/promote_portable_lab_v1_ready.py index 75488be..e723bf9 100644 --- a/experiments/perception/worker/observatory_portable/promote_portable_lab_v1_ready.py +++ b/experiments/perception/worker/observatory_portable/promote_portable_lab_v1_ready.py @@ -78,7 +78,7 @@ INSTALLATION_EVIDENCE_SCHEMA: Final = ( "missioncore.observatory-portable-lab-v1-installation-evidence/v1" ) WORKER_AGENT_IMAGE_INSTALLATION_SCHEMA: Final = ( - "missioncore.observatory-worker-agent-image-installation/v1" + "missioncore.observatory-worker-agent-image-installation/v2" ) LAB_V1_RELEASE_ID: Final = "lab-v1-eomt-ddrnet-worker006-candidate-v2" LAB_V1_WORKER_AGENT_IMAGE_ASSET_ID: Final = "worker-006-agent-image" @@ -1028,6 +1028,12 @@ def _load_installation_evidence( "staged_snapshot_canonicalization", "embedded_snapshot_manifest_sha256", "embedded_snapshot_manifest_byte_length", + "python_dependency_bundle_sha256", + "python_dependency_bundle_file_count", + "python_dependency_bundle_byte_length", + "python_dependency_bundle_canonicalization", + "python_dependency_bundle_manifest_sha256", + "python_dependency_bundle_manifest_byte_length", }, "coordinator provenance", ) @@ -1060,11 +1066,13 @@ def _load_installation_evidence( runtime, { "workdir", + "pythonpath", "entrypoint", "command", "authority", "models", "runtime_registries", + "python_dependencies", }, "coordinator runtime contract", ) @@ -1077,6 +1085,10 @@ def _load_installation_evidence( "staged_source_bytes", "embedded_context_bytes", "embedded_snapshot_manifest", + "python_dependency_bundle_bytes", + "python_dependency_bundle_manifest", + "required_dependency_versions", + "offline_dependency_operations", "result", }, "coordinator installation smoke", @@ -1093,6 +1105,18 @@ def _load_installation_evidence( provenance["embedded_snapshot_manifest_byte_length"], "coordinator embedded snapshot manifest byte length", ) + dependency_file_count = _integer( + provenance["python_dependency_bundle_file_count"], + "coordinator Python dependency bundle file count", + ) + dependency_bytes = _integer( + provenance["python_dependency_bundle_byte_length"], + "coordinator Python dependency bundle byte length", + ) + dependency_manifest_bytes = _integer( + provenance["python_dependency_bundle_manifest_byte_length"], + "coordinator Python dependency bundle manifest byte length", + ) size_bytes = _integer(image["size_bytes"], "coordinator image size") layer_bytes = _integer(image["thin_layer_bytes"], "coordinator thin layer size") maximum_layer_bytes = _integer( @@ -1111,6 +1135,8 @@ def _load_installation_evidence( "git_archive_sha256", "staged_snapshot_sha256", "embedded_snapshot_manifest_sha256", + "python_dependency_bundle_sha256", + "python_dependency_bundle_manifest_sha256", ): _require_digest( _string(provenance[key], f"coordinator {key}"), @@ -1128,9 +1154,14 @@ def _load_installation_evidence( or provenance["git_archive_verification"] != "external-before-extract" or provenance["staged_snapshot_canonicalization"] != "utf8-path-nul-length-nul-sha256-lf-v1" + or provenance["python_dependency_bundle_canonicalization"] + != "utf8-path-nul-length-nul-sha256-lf-v1" or file_count <= 0 or snapshot_bytes <= 0 or embedded_manifest_bytes <= 0 + or dependency_file_count <= 0 + or dependency_bytes <= 0 + or dependency_manifest_bytes <= 0 or image["tag"] != f"ndc/mission-core-observatory-worker-agent:{source_revision[:12]}" or image["id"] != f"sha256:{coordinator_image_sha256}" @@ -1147,12 +1178,15 @@ def _load_installation_evidence( ) is None or runtime["workdir"] != "/opt/nodedc/mission-core" + or runtime["pythonpath"] + != "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps" or runtime["entrypoint"] != ["python3", "-m", "k1link.observatory.m49_worker_container_main"] or runtime["command"] != [] or runtime["authority"] != "observation-only" or runtime["models"] != "external" or runtime["runtime_registries"] != "external-read-only" + or runtime["python_dependencies"] != "embedded-sealed-build-input" or smoke != { "network": "none", @@ -1160,6 +1194,10 @@ def _load_installation_evidence( "staged_source_bytes": "matched", "embedded_context_bytes": "matched", "embedded_snapshot_manifest": "matched", + "python_dependency_bundle_bytes": "matched", + "python_dependency_bundle_manifest": "matched", + "required_dependency_versions": "matched", + "offline_dependency_operations": "passed", "result": "passed", } ): diff --git a/experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json b/experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json index 3a6fe67..04bf3cf 100644 --- a/experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json +++ b/experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json @@ -1,5 +1,5 @@ { - "schema_version": "missioncore.observatory-worker-agent-build-context/v1", + "schema_version": "missioncore.observatory-worker-agent-build-context/v2", "artifact_id": "observatory-worker-006-agent", "worker_id": "worker-006", "source_contract": { @@ -38,6 +38,27 @@ "image_path": "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json" } ], + "external_build_inputs": [ + { + "role": "python-dependency-bundle", + "required": true, + "provenance_requirement": "already-proven Python dependency tree from the previously accepted Worker; build input only", + "source_path": null, + "image_path": "/opt/nodedc/mission-core/deps", + "manifest_image_path": "/opt/nodedc/mission-core/release/worker-006-agent-python-dependency-bundle.json", + "manifest_schema_version": "missioncore.observatory-worker-agent-python-dependency-bundle/v1", + "identity": { + "algorithm": "sha256", + "subject": "complete regular-file inventory", + "canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1", + "path_order": "ordinal" + }, + "included_in_git_archive": false, + "copied_into_image": true, + "runtime_bind_allowed": false, + "network_install_allowed": false + } + ], "embedded_configuration": [], "external_runtime_files": [ { @@ -58,7 +79,7 @@ } ], "runtime": { - "pythonpath": "/opt/nodedc/mission-core/src", + "pythonpath": "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps", "entrypoint": [ "python3", "-m", @@ -67,6 +88,8 @@ "composition_module": "k1link.observatory.m49_worker_service", "configuration_baked_into_image": false, "runtime_registries_baked_into_image": false, + "python_dependency_bundle_baked_into_image": true, + "python_dependency_bundle_runtime_bind_allowed": false, "models_baked_into_image": false, "executor_releases_baked_into_image": false }, diff --git a/experiments/perception/worker/observatory_portable/worker-006-agent-install-plan.template.json b/experiments/perception/worker/observatory_portable/worker-006-agent-install-plan.template.json index 727f96c..efe3d53 100644 --- a/experiments/perception/worker/observatory_portable/worker-006-agent-install-plan.template.json +++ b/experiments/perception/worker/observatory_portable/worker-006-agent-install-plan.template.json @@ -1,5 +1,5 @@ { - "schema_version": "missioncore.observatory-worker-agent-install-plan/v1", + "schema_version": "missioncore.observatory-worker-agent-install-plan/v2", "state": "planned-not-built", "artifact_id": "observatory-worker-006-agent", "worker_id": "worker-006", @@ -32,6 +32,27 @@ ], "embedded_manifest": "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json" }, + "python_dependency_bundle": { + "state": "required-external-build-input", + "provenance_requirement": "already-proven Python dependency tree from the previously accepted Worker; build input only", + "source_root": null, + "expected_sha256": null, + "identity_subject": "complete regular-file inventory", + "canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1", + "path_order": "ordinal", + "reject_reparse_points": true, + "reject_unexpected_non_regular_entries": true, + "verify_timing": [ + "before temporary container creation", + "after dependency installation and before image commit" + ], + "image_path": "/opt/nodedc/mission-core/deps", + "manifest_schema_version": "missioncore.observatory-worker-agent-python-dependency-bundle/v1", + "embedded_manifest": "/opt/nodedc/mission-core/release/worker-006-agent-python-dependency-bundle.json", + "included_in_git_archive": false, + "runtime_bind_allowed": false, + "network_install_allowed": false + }, "base_image": { "reference": "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", "sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", @@ -50,8 +71,10 @@ "maximum_thin_layer_bytes": 33554432, "embedded_payload": [ "/opt/nodedc/mission-core/src/k1link", + "/opt/nodedc/mission-core/deps", "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json", - "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json" + "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json", + "/opt/nodedc/mission-core/release/worker-006-agent-python-dependency-bundle.json" ] }, "required_preflight": [ @@ -61,7 +84,10 @@ "base image inspect ID equals the pinned SHA-256", "context archive contains exactly the context manifest entries", "git archive SHA-256 was verified externally before extraction", - "staged snapshot SHA-256 matches the exact canonical regular-file inventory" + "staged snapshot SHA-256 matches the exact canonical regular-file inventory", + "Python dependency bundle is an external build input with an exact expected SHA-256", + "Python dependency bundle SHA-256 matches its ordinal canonical regular-file inventory", + "Python dependency bundle contains the five exact smoke-tested package versions" ], "image_tag_template": "ndc/mission-core-observatory-worker-agent:" }, @@ -79,6 +105,11 @@ "com.nodedc.build-context.sha256": "", "com.nodedc.staged-snapshot.sha256": "", "com.nodedc.embedded-snapshot-manifest.sha256": "", + "com.nodedc.python-dependency-bundle.sha256": "", + "com.nodedc.python-dependency-bundle.file-count": "", + "com.nodedc.python-dependency-bundle.byte-length": "", + "com.nodedc.python-dependency-bundle.manifest.sha256": "", + "com.nodedc.python-dependency-bundle.manifest.byte-length": "", "com.nodedc.build-method": "docker-commit-exact-layer-v1" }, "rootfs": { @@ -90,7 +121,9 @@ "embedded_identity": { "source_tree": "byte-for-byte and bidirectional path match against staged src/k1link", "context_manifest": "byte-for-byte match against the staged context manifest", - "snapshot_manifest": "byte-for-byte match against the generated manifest covering every staged file, including the Dockerfile" + "snapshot_manifest": "byte-for-byte match against the generated manifest covering every staged file, including the Dockerfile", + "python_dependency_bundle": "byte-for-byte and bidirectional path match against the mandatory external build input", + "python_dependency_bundle_manifest": "byte-for-byte match against the generated manifest covering every embedded dependency file" }, "smoke": { "network": "none", @@ -104,6 +137,16 @@ "staged_source_bytes": "matched", "embedded_context_bytes": "matched", "embedded_snapshot_manifest": "matched", + "embedded_python_dependency_bundle": "matched", + "embedded_python_dependency_bundle_manifest": "matched", + "pythonpath": "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps", + "python_package_versions": { + "httpx": "0.28.1", + "lz4": "4.4.5", + "paho-mqtt": "2.1.0", + "PyYAML": "6.0.3", + "typing_extensions": "4.16.0" + }, "expected_result": "exit-0" } }, @@ -268,7 +311,10 @@ "external_assets": { "model_images": "pre-existing exact local Docker images selected only by installed executor adapters", "model_weights": "pre-existing exact local read-only assets selected only by installed executor adapters", + "python_dependency_bundle": "mandatory external build input copied into /opt/nodedc/mission-core/deps and never mounted at runtime", "baked_into_agent_image": false, + "python_dependency_bundle_baked_into_agent_image": true, + "python_dependency_bundle_runtime_bind_allowed": false, "queued_jobs_may_select_paths_or_commands": false }, "lab_v1_runner_installation_input": { @@ -289,7 +335,7 @@ ] }, "receipt_skeleton": { - "schema_version": "missioncore.observatory-worker-agent-image-installation/v1", + "schema_version": "missioncore.observatory-worker-agent-image-installation/v2", "status": null, "worker_id": "worker-006", "build_method": "docker-commit-exact-layer-v1", @@ -302,7 +348,13 @@ "staged_snapshot_byte_length": null, "staged_snapshot_canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1", "embedded_snapshot_manifest_sha256": null, - "embedded_snapshot_manifest_byte_length": null + "embedded_snapshot_manifest_byte_length": null, + "python_dependency_bundle_sha256": null, + "python_dependency_bundle_file_count": null, + "python_dependency_bundle_byte_length": null, + "python_dependency_bundle_canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1", + "python_dependency_bundle_manifest_sha256": null, + "python_dependency_bundle_manifest_byte_length": null }, "base_image_sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", "derived_image_sha256": null, @@ -327,9 +379,11 @@ "k1link.observatory.m49_worker_container_main" ], "command": [], + "pythonpath": "/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps", "authority": "observation-only", "models": "external", - "runtime_registries": "external-read-only" + "runtime_registries": "external-read-only", + "python_dependencies": "embedded-sealed-build-input" }, "smoke": { "network": "none", @@ -337,6 +391,10 @@ "staged_source_bytes": null, "embedded_context_bytes": null, "embedded_snapshot_manifest": null, + "python_dependency_bundle_bytes": null, + "python_dependency_bundle_manifest": null, + "required_dependency_versions": null, + "offline_dependency_operations": null, "result": "not-run" } } diff --git a/tests/test_observatory_portable_lab_v1_promotion.py b/tests/test_observatory_portable_lab_v1_promotion.py index 70ed3f3..c569c40 100644 --- a/tests/test_observatory_portable_lab_v1_promotion.py +++ b/tests/test_observatory_portable_lab_v1_promotion.py @@ -255,6 +255,14 @@ def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]: ), "embedded_snapshot_manifest_sha256": "a" * 64, "embedded_snapshot_manifest_byte_length": 8192, + "python_dependency_bundle_sha256": "b" * 64, + "python_dependency_bundle_file_count": 128, + "python_dependency_bundle_byte_length": 16_384, + "python_dependency_bundle_canonicalization": ( + "utf8-path-nul-length-nul-sha256-lf-v1" + ), + "python_dependency_bundle_manifest_sha256": "c" * 64, + "python_dependency_bundle_manifest_byte_length": 4096, }, "base_image_sha256": ( "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" @@ -278,6 +286,10 @@ def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]: }, "runtime_contract": { "workdir": "/opt/nodedc/mission-core", + "pythonpath": ( + "/opt/nodedc/mission-core/src:" + "/opt/nodedc/mission-core/deps" + ), "entrypoint": [ "python3", "-m", @@ -287,6 +299,7 @@ def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]: "authority": "observation-only", "models": "external", "runtime_registries": "external-read-only", + "python_dependencies": "embedded-sealed-build-input", }, "smoke": { "network": "none", @@ -294,6 +307,10 @@ def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]: "staged_source_bytes": "matched", "embedded_context_bytes": "matched", "embedded_snapshot_manifest": "matched", + "python_dependency_bundle_bytes": "matched", + "python_dependency_bundle_manifest": "matched", + "required_dependency_versions": "matched", + "offline_dependency_operations": "passed", "result": "passed", }, } @@ -433,3 +450,87 @@ def test_promotion_rejects_bare_or_mismatched_installer_claims( match="does not bind the promotion", ): promotion.load_promotion_input(mismatched) + + +@pytest.mark.parametrize( + ("section", "field", "replacement"), + [ + ("provenance", "python_dependency_bundle_sha256", "not-a-digest"), + ("provenance", "python_dependency_bundle_file_count", 0), + ("provenance", "python_dependency_bundle_byte_length", 0), + ( + "provenance", + "python_dependency_bundle_canonicalization", + "different-canonicalization", + ), + ( + "provenance", + "python_dependency_bundle_manifest_sha256", + "not-a-digest", + ), + ("provenance", "python_dependency_bundle_manifest_byte_length", 0), + ("runtime_contract", "pythonpath", "/tmp/unsealed-dependencies"), + ("runtime_contract", "python_dependencies", "host-mounted"), + ("smoke", "python_dependency_bundle_bytes", "unchecked"), + ("smoke", "python_dependency_bundle_manifest", "unchecked"), + ("smoke", "required_dependency_versions", "unchecked"), + ("smoke", "offline_dependency_operations", "unchecked"), + ], +) +def test_promotion_rejects_invalid_coordinator_dependency_evidence( + tmp_path: Path, + section: str, + field: str, + replacement: object, +) -> None: + work_root = tmp_path / "work" + work_root.mkdir() + component_receipt, coordinator_receipt = _write_installation_evidence(tmp_path) + coordinator = cast( + dict[str, object], + json.loads(coordinator_receipt.read_text(encoding="utf-8")), + ) + nested = cast(dict[str, object], coordinator[section]) + nested[field] = replacement + coordinator_receipt.write_text(json.dumps(coordinator), encoding="utf-8") + input_path = tmp_path / "promotion-input.json" + input_path.write_text( + json.dumps( + _input_document( + work_root, + component_receipt=component_receipt, + coordinator_receipt=coordinator_receipt, + ) + ), + encoding="utf-8", + ) + + with pytest.raises(promotion.PortableLabV1PromotionError, match="coordinator"): + promotion.load_promotion_input(input_path) + + +def test_promotion_rejects_symlinked_installation_evidence_path( + tmp_path: Path, +) -> None: + work_root = tmp_path / "work" + work_root.mkdir() + component_receipt, coordinator_receipt = _write_installation_evidence(tmp_path) + coordinator_link = tmp_path / "coordinator-image-installation-link.json" + coordinator_link.symlink_to(coordinator_receipt) + input_path = tmp_path / "promotion-input.json" + input_path.write_text( + json.dumps( + _input_document( + work_root, + component_receipt=component_receipt, + coordinator_receipt=coordinator_link, + ) + ), + encoding="utf-8", + ) + + with pytest.raises( + promotion.PortableLabV1PromotionError, + match="not a regular file", + ): + promotion.load_promotion_input(input_path) diff --git a/tests/test_observatory_worker_agent_image_artifact.py b/tests/test_observatory_worker_agent_image_artifact.py index ccbcfb8..3bb343b 100644 --- a/tests/test_observatory_worker_agent_image_artifact.py +++ b/tests/test_observatory_worker_agent_image_artifact.py @@ -30,6 +30,22 @@ RUNTIME_REGISTRY_CONTAINER_PATH = ( ) LAB_V1_RECEIPT_CONTAINER_PATH = "/release/lab-v1-worker-installation-receipt.json" LAB_V1_RELEASE_CANDIDATE_CONTAINER_PATH = "/release/lab-v1-executor-release.json" +DEPENDENCY_IMAGE_PATH = "/opt/nodedc/mission-core/deps" +DEPENDENCY_MANIFEST_IMAGE_PATH = ( + "/opt/nodedc/mission-core/release/" + "worker-006-agent-python-dependency-bundle.json" +) +DEPENDENCY_MANIFEST_SCHEMA = ( + "missioncore.observatory-worker-agent-python-dependency-bundle/v1" +) +DEPENDENCY_CANONICALIZATION = "utf8-path-nul-length-nul-sha256-lf-v1" +DEPENDENCY_VERSIONS = { + "httpx": "0.28.1", + "lz4": "4.4.5", + "paho-mqtt": "2.1.0", + "PyYAML": "6.0.3", + "typing_extensions": "4.16.0", +} def _document(path: Path) -> dict[str, object]: @@ -44,19 +60,39 @@ def test_agent_dockerfile_is_offline_commit_bound_and_model_free() -> None: assert payload.splitlines()[0] == f"FROM {BASE_REFERENCE}" assert "ARG NODEDC_SOURCE_REVISION" in payload assert "ARG NODEDC_BUILD_CONTEXT_SHA256" in payload + assert "ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_SHA256" in payload + assert "ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_FILE_COUNT" in payload + assert "ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_BYTE_LENGTH" in payload + assert "ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_SHA256" in payload + assert "ARG NODEDC_PYTHON_DEPENDENCY_BUNDLE_MANIFEST_BYTE_LENGTH" in payload assert "ARG SOURCE_DATE_EPOCH" in payload assert "org.opencontainers.image.revision=\"${NODEDC_SOURCE_REVISION}\"" in payload assert "com.nodedc.build-context.sha256=\"${NODEDC_BUILD_CONTEXT_SHA256}\"" in payload assert "com.nodedc.models=\"external\"" in payload - assert "PYTHONPATH=/opt/nodedc/mission-core/src" in payload + assert f"PYTHONPATH=/opt/nodedc/mission-core/src:{DEPENDENCY_IMAGE_PATH}" in payload assert 'ENTRYPOINT ["python3", "-m", "k1link.observatory.m49_worker_container_main"]' in payload assert "CMD []" in payload assert "COPY src/k1link ./src/k1link" in payload + assert "COPY python-dependency-bundle ./deps" in payload + assert "COPY worker-006-agent-python-dependency-bundle.json" in payload + assert "separately sealed build-only dependency bundle" in payload + assert "must never be supplied as runtime bind mounts" in payload + assert DEPENDENCY_MANIFEST_IMAGE_PATH.removeprefix( + "/opt/nodedc/mission-core/" + ) in payload assert "COPY config" not in payload assert "COPY . ." not in payload assert "mkdir -p /run/nodedc/registries" in payload assert "chmod 0555 /run/nodedc /run/nodedc/registries" in payload assert 'com.nodedc.runtime-registries="external-read-only"' in payload + for label in ( + "sha256", + "file-count", + "byte-length", + "manifest.sha256", + "manifest.byte-length", + ): + assert f"com.nodedc.python-dependency-bundle.{label}" in payload assert re.search(r"(?im)^\s*(?:run\s+)?(?:apt|apk|yum|dnf|pip|uv)\b", payload) is None assert "curl " not in payload.lower() assert "wget " not in payload.lower() @@ -66,7 +102,7 @@ def test_agent_dockerfile_is_offline_commit_bound_and_model_free() -> None: def test_build_context_is_only_repo_source_configuration_and_artifact_contract() -> None: document = _document(CONTEXT_MANIFEST) - assert document["schema_version"] == "missioncore.observatory-worker-agent-build-context/v1" + assert document["schema_version"] == "missioncore.observatory-worker-agent-build-context/v2" assert document["worker_id"] == "worker-006" assert document["authority"] == AUTHORITY source = cast(dict[str, object], document["source_contract"]) @@ -107,6 +143,31 @@ def test_build_context_is_only_repo_source_configuration_and_artifact_contract() ) assert document["embedded_configuration"] == [] + build_inputs = cast(list[dict[str, object]], document["external_build_inputs"]) + assert build_inputs == [ + { + "role": "python-dependency-bundle", + "required": True, + "provenance_requirement": ( + "already-proven Python dependency tree from the previously " + "accepted Worker; build input only" + ), + "source_path": None, + "image_path": DEPENDENCY_IMAGE_PATH, + "manifest_image_path": DEPENDENCY_MANIFEST_IMAGE_PATH, + "manifest_schema_version": DEPENDENCY_MANIFEST_SCHEMA, + "identity": { + "algorithm": "sha256", + "subject": "complete regular-file inventory", + "canonicalization": DEPENDENCY_CANONICALIZATION, + "path_order": "ordinal", + }, + "included_in_git_archive": False, + "copied_into_image": True, + "runtime_bind_allowed": False, + "network_install_allowed": False, + } + ] assert "required_configuration" not in document external_runtime_files = cast( list[dict[str, object]], document["external_runtime_files"] @@ -133,9 +194,14 @@ def test_build_context_is_only_repo_source_configuration_and_artifact_contract() ] runtime = cast(dict[str, object], document["runtime"]) assert runtime["entrypoint"] == ENTRYPOINT + assert runtime["pythonpath"] == ( + f"/opt/nodedc/mission-core/src:{DEPENDENCY_IMAGE_PATH}" + ) assert runtime["composition_module"] == "k1link.observatory.m49_worker_service" assert runtime["configuration_baked_into_image"] is False assert runtime["runtime_registries_baked_into_image"] is False + assert runtime["python_dependency_bundle_baked_into_image"] is True + assert runtime["python_dependency_bundle_runtime_bind_allowed"] is False assert runtime["models_baked_into_image"] is False assert runtime["executor_releases_baked_into_image"] is False forbidden = cast(list[str], document["forbidden_context"]) @@ -146,7 +212,7 @@ def test_build_context_is_only_repo_source_configuration_and_artifact_contract() def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receipt() -> None: document = _document(INSTALL_PLAN) - assert document["schema_version"] == "missioncore.observatory-worker-agent-install-plan/v1" + assert document["schema_version"] == "missioncore.observatory-worker-agent-install-plan/v2" assert document["state"] == "planned-not-built" build = cast(dict[str, object], document["build"]) assert build["source_revision"] is None @@ -181,6 +247,31 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip assert staged["embedded_manifest"] == ( "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json" ) + dependency = cast(dict[str, object], build["python_dependency_bundle"]) + assert dependency == { + "state": "required-external-build-input", + "provenance_requirement": ( + "already-proven Python dependency tree from the previously accepted " + "Worker; build input only" + ), + "source_root": None, + "expected_sha256": None, + "identity_subject": "complete regular-file inventory", + "canonicalization": DEPENDENCY_CANONICALIZATION, + "path_order": "ordinal", + "reject_reparse_points": True, + "reject_unexpected_non_regular_entries": True, + "verify_timing": [ + "before temporary container creation", + "after dependency installation and before image commit", + ], + "image_path": DEPENDENCY_IMAGE_PATH, + "manifest_schema_version": DEPENDENCY_MANIFEST_SCHEMA, + "embedded_manifest": DEPENDENCY_MANIFEST_IMAGE_PATH, + "included_in_git_archive": False, + "runtime_bind_allowed": False, + "network_install_allowed": False, + } base = cast(dict[str, object], build["base_image"]) assert base["reference"] == BASE_REFERENCE assert base["sha256"] == BASE_SHA256 @@ -199,8 +290,10 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip "maximum_thin_layer_bytes": 33554432, "embedded_payload": [ "/opt/nodedc/mission-core/src/k1link", + DEPENDENCY_IMAGE_PATH, "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json", "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json", + DEPENDENCY_MANIFEST_IMAGE_PATH, ], } assert build["required_preflight"] == [ @@ -211,6 +304,9 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip "context archive contains exactly the context manifest entries", "git archive SHA-256 was verified externally before extraction", "staged snapshot SHA-256 matches the exact canonical regular-file inventory", + "Python dependency bundle is an external build input with an exact expected SHA-256", + "Python dependency bundle SHA-256 matches its ordinal canonical regular-file inventory", + "Python dependency bundle contains the five exact smoke-tested package versions", ] acceptance = cast(dict[str, object], document["build_acceptance"]) @@ -225,6 +321,21 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip assert labels["com.nodedc.embedded-snapshot-manifest.sha256"] == ( "" ) + assert labels["com.nodedc.python-dependency-bundle.sha256"] == ( + "" + ) + assert labels["com.nodedc.python-dependency-bundle.file-count"] == ( + "" + ) + assert labels["com.nodedc.python-dependency-bundle.byte-length"] == ( + "" + ) + assert labels["com.nodedc.python-dependency-bundle.manifest.sha256"] == ( + "" + ) + assert labels["com.nodedc.python-dependency-bundle.manifest.byte-length"] == ( + "" + ) assert labels["com.nodedc.build-method"] == "docker-commit-exact-layer-v1" rootfs = cast(dict[str, object], acceptance["rootfs"]) assert rootfs == { @@ -237,6 +348,12 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip assert "byte-for-byte" in cast(str, embedded["source_tree"]) assert "byte-for-byte" in cast(str, embedded["context_manifest"]) assert "every staged file" in cast(str, embedded["snapshot_manifest"]) + assert "mandatory external build input" in cast( + str, embedded["python_dependency_bundle"] + ) + assert "every embedded dependency file" in cast( + str, embedded["python_dependency_bundle_manifest"] + ) smoke = cast(dict[str, object], acceptance["smoke"]) assert smoke["network"] == "none" assert smoke["read_only_rootfs"] is True @@ -244,6 +361,12 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip assert smoke["staged_source_bytes"] == "matched" assert smoke["embedded_context_bytes"] == "matched" assert smoke["embedded_snapshot_manifest"] == "matched" + assert smoke["embedded_python_dependency_bundle"] == "matched" + assert smoke["embedded_python_dependency_bundle_manifest"] == "matched" + assert smoke["pythonpath"] == ( + f"/opt/nodedc/mission-core/src:{DEPENDENCY_IMAGE_PATH}" + ) + assert smoke["python_package_versions"] == DEPENDENCY_VERSIONS assert smoke["expected_result"] == "exit-0" runtime = cast(dict[str, object], document["runtime"]) @@ -367,6 +490,10 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip assert external["baked_into_agent_image"] is False assert external["queued_jobs_may_select_paths_or_commands"] is False mounts = cast(list[dict[str, object]], runtime["mounts"]) + assert not any( + cast(str, row["container_path"]).startswith(DEPENDENCY_IMAGE_PATH) + for row in mounts + ) registry_mounts = [ row for row in mounts @@ -400,10 +527,13 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip "owns_component_image_identities": True, "queued_jobs_may_override_component_images": False, } + external = cast(dict[str, object], runtime["external_assets"]) + assert external["python_dependency_bundle_baked_into_agent_image"] is True + assert external["python_dependency_bundle_runtime_bind_allowed"] is False receipt = cast(dict[str, object], document["receipt_skeleton"]) assert receipt["schema_version"] == ( - "missioncore.observatory-worker-agent-image-installation/v1" + "missioncore.observatory-worker-agent-image-installation/v2" ) assert receipt["status"] is None assert receipt["build_method"] == "docker-commit-exact-layer-v1" @@ -420,6 +550,12 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip ), "embedded_snapshot_manifest_sha256": None, "embedded_snapshot_manifest_byte_length": None, + "python_dependency_bundle_sha256": None, + "python_dependency_bundle_file_count": None, + "python_dependency_bundle_byte_length": None, + "python_dependency_bundle_canonicalization": DEPENDENCY_CANONICALIZATION, + "python_dependency_bundle_manifest_sha256": None, + "python_dependency_bundle_manifest_byte_length": None, } assert receipt["base_image_sha256"] == BASE_SHA256 assert receipt["derived_image_sha256"] is None @@ -442,9 +578,11 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip "workdir": "/opt/nodedc/mission-core", "entrypoint": ENTRYPOINT, "command": [], + "pythonpath": f"/opt/nodedc/mission-core/src:{DEPENDENCY_IMAGE_PATH}", "authority": "observation-only", "models": "external", "runtime_registries": "external-read-only", + "python_dependencies": "embedded-sealed-build-input", } receipt_smoke = cast(dict[str, object], receipt["smoke"]) assert receipt_smoke == { @@ -453,5 +591,9 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip "staged_source_bytes": None, "embedded_context_bytes": None, "embedded_snapshot_manifest": None, + "python_dependency_bundle_bytes": None, + "python_dependency_bundle_manifest": None, + "required_dependency_versions": None, + "offline_dependency_operations": None, "result": "not-run", } diff --git a/tests/test_worker_006_agent_image_installer.py b/tests/test_worker_006_agent_image_installer.py index c7b7473..1fcc04c 100644 --- a/tests/test_worker_006_agent_image_installer.py +++ b/tests/test_worker_006_agent_image_installer.py @@ -73,6 +73,10 @@ def test_worker_agent_installer_separates_archive_and_staged_identities() -> Non ) assert "Dockerfile.worker-006-agent" in script assert "worker-006-agent-build-context.json" in script + assert ( + '"missioncore.observatory-worker-agent-build-context/v2"' + in script + ) assert ( 'schema_version = "missioncore.observatory-worker-agent-embedded-snapshot/v1"' in script @@ -81,6 +85,32 @@ def test_worker_agent_installer_separates_archive_and_staged_identities() -> Non assert "embedded_snapshot_manifest_sha256" in script +def test_worker_agent_installer_uses_ordinal_snapshot_path_order() -> None: + script = _script() + + assert "$rowsByPath = @{}" in script + assert "$rowsByPath.ContainsKey($relative)" in script + assert "$orderedPaths = [string[]]@($rowsByPath.Keys)" in script + assert "[Array]::Sort($orderedPaths, [StringComparer]::Ordinal)" in script + assert "$orderedPaths | ForEach-Object { $rowsByPath[$_] }" in script + assert "Sort-Object -Property path -CaseSensitive" not in script + + +def test_worker_agent_installer_reports_observed_snapshot_sha_on_mismatch() -> None: + script = _script() + + mismatch_guard = ( + "if ([string]$snapshotBefore.sha256 -cne " + "$ExpectedStagedSnapshotSha256)" + ) + diagnostic = ( + '"Worker 006 staged snapshot identity changed: observed=" +\n' + " [string]$snapshotBefore.sha256" + ) + assert mismatch_guard in script + assert diagnostic in script + + def test_worker_agent_installer_copies_only_runtime_source_and_contract() -> None: script = _script() @@ -106,6 +136,60 @@ def test_worker_agent_installer_copies_only_runtime_source_and_contract() -> Non assert "chmod 0555 /run/nodedc /run/nodedc/registries" in script +def test_worker_agent_installer_seals_offline_python_dependency_bundle() -> None: + script = _script() + + assert "[string]$DependencyBundleRoot" in script + assert "[string]$ExpectedDependencyBundleSha256" in script + assert script.count("Get-DependencyBundleInspection $DependencyBundleRoot") == 2 + assert "Python dependency bundle contains a reparse point" in script + assert "Python dependency bundle contains a non-file entry" in script + assert "Python dependency bundle contains no regular files" in script + assert "[Array]::Sort($orderedPaths, [StringComparer]::Ordinal)" in script + assert 'canonicalization = "utf8-path-nul-length-nul-sha256-lf-v1"' in script + assert ( + 'schema_version = "missioncore.observatory-worker-agent-python-' + 'dependency-bundle/v1"' + ) in script + assert "required_distributions = [ordered]@{" in script + assert "$externalBuildInputs = @($context.external_build_inputs)" in script + assert '[string]$dependencyInput.identity.path_order -cne "ordinal"' in script + assert "[bool]$dependencyInput.runtime_bind_allowed" in script + assert "[bool]$dependencyInput.network_install_allowed" in script + assert "cp -a /nodedc-build-deps/. /opt/nodedc/mission-core/deps/" in script + assert "target=/nodedc-build-deps,readonly" in script + assert "target=/nodedc-verify-deps,readonly" in script + assert "target=/opt/nodedc/mission-core/deps" not in script + assert "find /nodedc-build-deps -type f" in script + assert "find /opt/nodedc/mission-core/deps -type f" in script + assert "find /nodedc-verify-deps -type f" in script + assert ( + "/opt/nodedc/mission-core/release/" + "worker-006-agent-python-dependency-bundle.json" + ) in script + + +def test_worker_agent_dependency_smoke_is_exact_and_networkless() -> None: + script = _script() + + for distribution, version in ( + ("httpx", "0.28.1"), + ("lz4", "4.4.5"), + ("paho-mqtt", "2.1.0"), + ("PyYAML", "6.0.3"), + ("typing_extensions", "4.16.0"), + ): + assert f'`"{distribution}`":`"{version}`"' in script + assert "import httpx; import lz4.frame; import paho.mqtt.client as mqtt" in script + assert "lz4.frame.decompress(lz4.frame.compress(payload))" in script + assert "httpx.Request(`\"GET`\",url)" in script + assert "yaml.safe_load(yaml.safe_dump(document))" in script + assert "mqtt.Client(callback_api_version=mqtt.CallbackAPIVersion.VERSION2)" in script + assert "typing_extensions.TypeAlias is not None" in script + assert "compose_installed_m49_worker_service" in script + assert "--network none" in script + + def test_worker_agent_installer_proves_base_chain_and_embedded_bytes() -> None: script = _script() @@ -131,20 +215,20 @@ def test_worker_agent_installer_removes_failed_new_image_and_temporary_files() - script = _script() assert "$imageCommitted = $true" in script - assert ( - "Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest" - in script - ) + assert "Invoke-InstalledImageSmoke `" in script + assert "$DependencyBundleRoot `" in script + assert "$dependencyManifest" in script assert "docker image rm --force $committedImageId" in script assert "image verification failed and committed image cleanup failed" in script assert "Remove-Item -LiteralPath $embeddedManifest.path -Force" in script + assert "Remove-Item -LiteralPath $dependencyManifest.path -Force" in script def test_worker_agent_installer_seals_runtime_contract_smoke_and_output() -> None: script = _script() for value in ( - "PYTHONPATH=/opt/nodedc/mission-core/src", + "PYTHONPATH=/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps", "PYTHONNOUSERSITE=1", "PYTHONDONTWRITEBYTECODE=1", "PYTHONUNBUFFERED=1", @@ -165,8 +249,25 @@ def test_worker_agent_installer_seals_runtime_contract_smoke_and_output() -> Non assert "$MaximumLayerBytes = [int64](32MB)" in script assert "image is not within the thin-layer bound" in script assert ( - 'schema_version = "missioncore.observatory-worker-agent-image-installation/v1"' + 'schema_version = "missioncore.observatory-worker-agent-image-installation/v2"' in script ) + for label in ( + "com.nodedc.python-dependency-bundle.sha256", + "com.nodedc.python-dependency-bundle.file-count", + "com.nodedc.python-dependency-bundle.byte-length", + "com.nodedc.python-dependency-bundle.manifest.sha256", + "com.nodedc.python-dependency-bundle.manifest.byte-length", + ): + assert label in script + for field in ( + "python_dependency_bundle_sha256", + "python_dependency_bundle_file_count", + "python_dependency_bundle_byte_length", + "python_dependency_bundle_canonicalization", + "python_dependency_bundle_manifest_sha256", + "python_dependency_bundle_manifest_byte_length", + ): + assert field in script assert "derived_image_sha256 = ([string]$Image.Id).Substring(7)" in script assert 'result = "passed"' in script