SILVER/NODEDC_MISSION_CORE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(worker): install combined observatory profiles

Browse Source
This commit is contained in:
DCCONSTRUCTIONS
2026-08-31 20:56:29 +03:00
parent 44ebbe7ca2
commit 11a146011f
14 changed files with 3878 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1
+698
View File
@@ -0,0 +1,698 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{40}$")]
[string]$SourceRevision,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedGitArchiveSha256,
[Parameter(Mandatory = $true)]
[ValidatePattern("^[a-f0-9]{64}$")]
[string]$ExpectedStagedSnapshotSha256
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$ProgressPreference = "SilentlyContinue"
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
$StagedSnapshotRoot = Join-Path (
$RuntimeRoot
) "staging\observatory-worker-agent-$SourceRevision"
$ContextManifestRelativePath = (
"experiments/perception/worker/observatory_portable/" +
"worker-006-agent-build-context.json"
)
$DockerfileRelativePath = (
"experiments/perception/worker/observatory_portable/" +
"Dockerfile.worker-006-agent"
)
$SourceTreeRelativePath = "src/k1link"
$BaseImageSha256 = (
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
)
$BaseImageReference = "sha256:$BaseImageSha256"
$BuildMethod = "docker-commit-exact-layer-v1"
$MaximumLayerBytes = [int64](32MB)
$ImageWorkdir = "/opt/nodedc/mission-core"
$ImageEntrypoint = '["python3","-m","k1link.observatory.m49_worker_container_main"]'
$ImageCommand = '[]'
$FixedEnvironment = @(
"PYTHONPATH=/opt/nodedc/mission-core/src",
"PYTHONNOUSERSITE=1",
"PYTHONDONTWRITEBYTECODE=1",
"PYTHONUNBUFFERED=1"
)
function Get-FileSha256 {
param([string]$Path)
$item = Get-Item -LiteralPath $Path -Force
if (
-not ($item -is [IO.FileInfo]) -or
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
) {
throw "Worker 006 staged source is not a regular file"
}
return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
}
function Assert-ExactDirectoryChildren {
param(
[string]$Path,
[string[]]$ExpectedNames
)
$directory = Get-Item -LiteralPath $Path -Force
if (
-not ($directory -is [IO.DirectoryInfo]) -or
($directory.Attributes -band [IO.FileAttributes]::ReparsePoint)
) {
throw "Worker 006 staged snapshot directory is invalid"
}
$actualNames = @(
Get-ChildItem -LiteralPath $directory.FullName -Force |
ForEach-Object { $_.Name }
)
[Array]::Sort($actualNames, [StringComparer]::Ordinal)
$expected = @($ExpectedNames)
[Array]::Sort($expected, [StringComparer]::Ordinal)
if (
$actualNames.Count -ne $expected.Count -or
([string]::Join("`n", $actualNames) -cne [string]::Join("`n", $expected))
) {
throw "Worker 006 staged snapshot contains unexpected entries"
}
}
function Get-StagedSnapshotInspection {
param([string]$Root)
Assert-ExactDirectoryChildren $Root @("experiments", "src")
$srcRoot = Join-Path $Root "src"
Assert-ExactDirectoryChildren $srcRoot @("k1link")
$experimentsRoot = Join-Path $Root "experiments"
Assert-ExactDirectoryChildren $experimentsRoot @("perception")
$perceptionRoot = Join-Path $experimentsRoot "perception"
Assert-ExactDirectoryChildren $perceptionRoot @("worker")
$workerRoot = Join-Path $perceptionRoot "worker"
Assert-ExactDirectoryChildren $workerRoot @("observatory_portable")
$portableRoot = Join-Path $workerRoot "observatory_portable"
Assert-ExactDirectoryChildren $portableRoot @(
"Dockerfile.worker-006-agent",
"worker-006-agent-build-context.json"
)
$sourceRoot = Join-Path $Root ($SourceTreeRelativePath.Replace("/", "\"))
$sourceItems = @(Get-ChildItem -LiteralPath $sourceRoot -Recurse -Force)
foreach ($item in $sourceItems) {
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
throw "Worker 006 staged snapshot contains a reparse point"
}
if (-not ($item -is [IO.FileInfo]) -and -not ($item -is [IO.DirectoryInfo])) {
throw "Worker 006 staged snapshot contains an unsupported entry"
}
}
$manifestPath = Join-Path $Root ($ContextManifestRelativePath.Replace("/", "\"))
$dockerfilePath = Join-Path $Root ($DockerfileRelativePath.Replace("/", "\"))
$requiredSourceFiles = @(
Join-Path $sourceRoot "__init__.py",
Join-Path $sourceRoot "observatory\m49_worker_container_main.py",
Join-Path $sourceRoot "observatory\m49_worker_service.py"
)
foreach ($requiredPath in $requiredSourceFiles + @($manifestPath, $dockerfilePath)) {
if (-not (Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
throw "Worker 006 staged snapshot is incomplete"
}
[void](Get-FileSha256 $requiredPath)
}
$context = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
if (
[string]$context.schema_version -cne
"missioncore.observatory-worker-agent-build-context/v1" -or
[string]$context.worker_id -cne "worker-006" -or
[string]$context.base_image.sha256 -cne $BaseImageSha256 -or
[bool]$context.base_image.pull_allowed
) {
throw "Worker 006 staged build-context contract is invalid"
}
$declaredPaths = @($context.context_entries | ForEach-Object { [string]$_.path })
$expectedDeclaredPaths = @(
$SourceTreeRelativePath,
$DockerfileRelativePath,
$ContextManifestRelativePath
)
if (
$declaredPaths.Count -ne $expectedDeclaredPaths.Count -or
([string]::Join("`n", $declaredPaths) -cne
[string]::Join("`n", $expectedDeclaredPaths))
) {
throw "Worker 006 staged build-context entries changed"
}
$files = New-Object "Collections.Generic.List[object]"
foreach ($item in $sourceItems) {
if ($item -is [IO.FileInfo]) {
$files.Add($item)
}
}
$files.Add((Get-Item -LiteralPath $dockerfilePath -Force))
$files.Add((Get-Item -LiteralPath $manifestPath -Force))
$rows = New-Object "Collections.Generic.List[object]"
foreach ($file in $files) {
$relative = $file.FullName.Substring($Root.Length).TrimStart("\")
$relative = $relative.Replace("\", "/")
if (-not $relative -or $relative -match "[\x00\r\n]") {
throw "Worker 006 staged snapshot path is invalid"
}
$rows.Add([pscustomobject]@{
path = $relative
byte_length = [int64]$file.Length
sha256 = Get-FileSha256 $file.FullName
})
}
$orderedRows = @($rows | Sort-Object -Property path -CaseSensitive)
$canonical = New-Object Text.StringBuilder
[int64]$totalBytes = 0
foreach ($row in $orderedRows) {
[void]$canonical.Append([string]$row.path)
[void]$canonical.Append([char]0)
[void]$canonical.Append(([int64]$row.byte_length).ToString([Globalization.CultureInfo]::InvariantCulture))
[void]$canonical.Append([char]0)
[void]$canonical.Append([string]$row.sha256)
[void]$canonical.Append("`n")
$totalBytes += [int64]$row.byte_length
}
$utf8 = New-Object Text.UTF8Encoding($false)
$canonicalBytes = $utf8.GetBytes($canonical.ToString())
$hasher = [Security.Cryptography.SHA256]::Create()
try {
$digestBytes = $hasher.ComputeHash($canonicalBytes)
}
finally {
$hasher.Dispose()
}
$digest = ([BitConverter]::ToString($digestBytes)).Replace("-", "").ToLowerInvariant()
return [pscustomobject]@{
sha256 = $digest
file_count = [int64]$orderedRows.Count
byte_length = $totalBytes
canonicalization = "utf8-path-nul-length-nul-sha256-lf-v1"
files = @($orderedRows)
}
}
function New-StagedSnapshotManifest {
param([object]$SnapshotInspection)
$stagingRoot = Split-Path -Parent $StagedSnapshotRoot
$manifestPath = Join-Path (
$stagingRoot
) ("observatory-worker-agent-embedded-manifest-{0}.json" -f [Guid]::NewGuid().ToString("N"))
$payload = [ordered]@{
schema_version = "missioncore.observatory-worker-agent-embedded-snapshot/v1"
snapshot_sha256 = [string]$SnapshotInspection.sha256
canonicalization = [string]$SnapshotInspection.canonicalization
file_count = [int64]$SnapshotInspection.file_count
byte_length = [int64]$SnapshotInspection.byte_length
files = @($SnapshotInspection.files)
}
$json = $payload | ConvertTo-Json -Compress -Depth 5
$utf8 = New-Object Text.UTF8Encoding($false)
try {
[IO.File]::WriteAllText($manifestPath, $json, $utf8)
return [pscustomobject]@{
path = $manifestPath
sha256 = Get-FileSha256 $manifestPath
byte_length = [int64](Get-Item -LiteralPath $manifestPath -Force).Length
}
}
catch {
if (Test-Path -LiteralPath $manifestPath -PathType Leaf) {
Remove-Item -LiteralPath $manifestPath -Force
}
throw
}
}
function Get-ImageInspection {
param([string]$Reference)
$payload = docker image inspect $Reference
if ($LASTEXITCODE -ne 0) {
throw "Worker 006 image is unavailable: $Reference"
}
$rows = @($payload | ConvertFrom-Json)
if ($rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") {
throw "Worker 006 image inspection is invalid"
}
return $rows[0]
}
function Assert-ImageContract {
param(
[object]$Image,
[object]$BaseImage,
[string]$EmbeddedManifestSha256
)
$labels = $Image.Config.Labels
$expectedLabels = [ordered]@{
"org.opencontainers.image.title" = "NODE.DC Observatory Worker 006 agent"
"org.opencontainers.image.source" = "NODEDC_MISSION_CORE"
"org.opencontainers.image.revision" = $SourceRevision
"com.nodedc.product" = "mission-core"
"com.nodedc.stack" = "ndc-mission-core-observatory"
"com.nodedc.role" = "observatory-worker-agent"
"com.nodedc.managed-by" = "mission-core-worker-release"
"com.nodedc.worker-contour" = "worker-006"
"com.nodedc.authority" = "observation-only"
"com.nodedc.models" = "external"
"com.nodedc.runtime-registries" = "external-read-only"
"com.nodedc.base-image.sha256" = $BaseImageSha256
"com.nodedc.build-context.sha256" = $ExpectedGitArchiveSha256
"com.nodedc.staged-snapshot.sha256" = $ExpectedStagedSnapshotSha256
"com.nodedc.embedded-snapshot-manifest.sha256" = $EmbeddedManifestSha256
"com.nodedc.build-method" = $BuildMethod
}
foreach ($key in $expectedLabels.Keys) {
if ([string]$labels.$key -cne [string]$expectedLabels[$key]) {
throw "Worker 006 installed image label identity changed: $key"
}
}
if ([string]$Image.Config.WorkingDir -cne $ImageWorkdir) {
throw "Worker 006 installed image workdir changed"
}
if ([string]::Join([char]0, @($Image.Config.Entrypoint)) -cne
[string]::Join([char]0, @("python3", "-m", "k1link.observatory.m49_worker_container_main"))) {
throw "Worker 006 installed image entrypoint changed"
}
if (@($Image.Config.Cmd).Count -ne 0) {
throw "Worker 006 installed image command changed"
}
if ([string]$Image.Config.User -cne "0:0") {
throw "Worker 006 installed image user changed"
}
$actualEnvironment = @($Image.Config.Env)
foreach ($value in $FixedEnvironment) {
if ($actualEnvironment -cnotcontains $value) {
throw "Worker 006 installed image environment changed"
}
}
if (
[string]$BaseImage.RootFS.Type -cne "layers" -or
[string]$Image.RootFS.Type -cne "layers"
) {
throw "Worker 006 image RootFS type changed"
}
$baseLayers = @($BaseImage.RootFS.Layers)
$imageLayers = @($Image.RootFS.Layers)
if ($baseLayers.Count -eq 0 -or $imageLayers.Count -ne ($baseLayers.Count + 1)) {
throw "Worker 006 image is not exactly one layer above the pinned base"
}
for ($index = 0; $index -lt $baseLayers.Count; $index += 1) {
if ([string]$imageLayers[$index] -cne [string]$baseLayers[$index]) {
throw "Worker 006 image RootFS does not extend the pinned base layer chain"
}
}
$derivedLayerDiffId = [string]$imageLayers[$baseLayers.Count]
if ($derivedLayerDiffId -notmatch "^sha256:[a-f0-9]{64}$") {
throw "Worker 006 derived layer identity is invalid"
}
$layerBytes = [int64]$Image.Size - [int64]$BaseImage.Size
if ($layerBytes -lt 0 -or $layerBytes -gt $MaximumLayerBytes) {
throw "Worker 006 image is not within the thin-layer bound"
}
return [pscustomobject]@{
layer_bytes = $layerBytes
base_layer_count = [int64]$baseLayers.Count
derived_layer_count = [int64]1
derived_layer_diff_id = $derivedLayerDiffId
}
}
function Invoke-InstalledImageSmoke {
param(
[string]$Tag,
[string]$SnapshotRoot,
[object]$EmbeddedManifest
)
$smokeScript = (
"set -eu; " +
"test ! -e /opt/nodedc/mission-core/experiments; " +
"test -z `"`$(find /opt/nodedc/mission-core -perm /022 -print -quit)`"; " +
"test -z `"`$(find /opt/nodedc/mission-core/src/k1link " +
"! -type d ! -type f -print -quit)`"; " +
"test -z `"`$(find /nodedc-verify-source/src/k1link " +
"! -type d ! -type f -print -quit)`"; " +
"cmp -s /nodedc-verify-snapshot.json " +
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
"test `"`$(sha256sum " +
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json " +
"| cut -d' ' -f1)`" = $($EmbeddedManifest.sha256); " +
"cmp -s /nodedc-verify-source/$ContextManifestRelativePath " +
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
"find /nodedc-verify-source/src/k1link -type d -exec sh -c '" +
"for source_path do relative=`${source_path#/nodedc-verify-source/}; " +
"test -d `"/opt/nodedc/mission-core/`$relative`" || exit 70; " +
"done' sh {} +; " +
"find /opt/nodedc/mission-core/src/k1link -type d -exec sh -c '" +
"for image_path do relative=`${image_path#/opt/nodedc/mission-core/}; " +
"test -d `"/nodedc-verify-source/`$relative`" || exit 71; " +
"done' sh {} +; " +
"find /nodedc-verify-source/src/k1link -type f -exec sh -c '" +
"for source_path do relative=`${source_path#/nodedc-verify-source/}; " +
"cmp -s `"`$source_path`" `"/opt/nodedc/mission-core/`$relative`" || exit 72; " +
"done' sh {} +; " +
"find /opt/nodedc/mission-core/src/k1link -type f -exec sh -c '" +
"for image_path do relative=`${image_path#/opt/nodedc/mission-core/}; " +
"cmp -s `"`$image_path`" `"/nodedc-verify-source/`$relative`" || exit 73; " +
"done' sh {} +; " +
"cd /opt/nodedc/mission-core; " +
"python3 -B -c 'import k1link.observatory.m49_worker_container_main as entrypoint; " +
"import k1link.observatory.m49_worker_service as composition; " +
"assert callable(entrypoint.main); " +
"assert callable(composition.compose_installed_m49_worker_service)'"
)
$sourceMount = (
"type=bind,source=$SnapshotRoot," +
"target=/nodedc-verify-source,readonly"
)
$manifestMount = (
"type=bind,source=$($EmbeddedManifest.path)," +
"target=/nodedc-verify-snapshot.json,readonly"
)
docker run `
--rm `
--network none `
--read-only `
--tmpfs "/tmp:rw,noexec,nosuid,size=16m" `
--cap-drop ALL `
--security-opt no-new-privileges `
--mount $sourceMount `
--mount $manifestMount `
--entrypoint /bin/sh `
$Tag `
-c $smokeScript
if ($LASTEXITCODE -ne 0) {
throw "Installed Worker 006 image smoke failed"
}
}
function New-OutputContract {
param(
[string]$Status,
[string]$Tag,
[object]$Image,
[object]$LayerInspection,
[object]$SnapshotInspection,
[object]$EmbeddedManifest
)
return [ordered]@{
schema_version = "missioncore.observatory-worker-agent-image-installation/v1"
status = $Status
worker_id = "worker-006"
build_method = $BuildMethod
source_revision = $SourceRevision
provenance = [ordered]@{
git_archive_sha256 = $ExpectedGitArchiveSha256
git_archive_verification = "external-before-extract"
staged_snapshot_sha256 = [string]$SnapshotInspection.sha256
staged_snapshot_file_count = [int64]$SnapshotInspection.file_count
staged_snapshot_byte_length = [int64]$SnapshotInspection.byte_length
staged_snapshot_canonicalization = [string]$SnapshotInspection.canonicalization
embedded_snapshot_manifest_sha256 = [string]$EmbeddedManifest.sha256
embedded_snapshot_manifest_byte_length = [int64]$EmbeddedManifest.byte_length
}
base_image_sha256 = $BaseImageSha256
derived_image_sha256 = ([string]$Image.Id).Substring(7)
image = [ordered]@{
tag = $Tag
id = [string]$Image.Id
size_bytes = [int64]$Image.Size
thin_layer_bytes = [int64]$LayerInspection.layer_bytes
maximum_thin_layer_bytes = $MaximumLayerBytes
rootfs = [ordered]@{
base_layer_count = [int64]$LayerInspection.base_layer_count
derived_layer_count = [int64]$LayerInspection.derived_layer_count
derived_layer_diff_id = [string]$LayerInspection.derived_layer_diff_id
pinned_base_is_exact_prefix = $true
}
}
runtime_contract = [ordered]@{
workdir = $ImageWorkdir
entrypoint = @("python3", "-m", "k1link.observatory.m49_worker_container_main")
command = @()
authority = "observation-only"
models = "external"
runtime_registries = "external-read-only"
}
smoke = [ordered]@{
network = "none"
read_only_rootfs = $true
staged_source_bytes = "matched"
embedded_context_bytes = "matched"
embedded_snapshot_manifest = "matched"
result = "passed"
}
}
}
if (-not (Test-Path -LiteralPath $StagedSnapshotRoot -PathType Container)) {
throw "Worker 006 staged snapshot root is unavailable"
}
$snapshotBefore = Get-StagedSnapshotInspection $StagedSnapshotRoot
if ([string]$snapshotBefore.sha256 -cne $ExpectedStagedSnapshotSha256) {
throw "Worker 006 staged snapshot identity changed"
}
$embeddedManifest = New-StagedSnapshotManifest $snapshotBefore
try {
$base = Get-ImageInspection $BaseImageReference
if ([string]$base.Id -cne $BaseImageReference) {
throw "Worker 006 base image identity changed"
}
$shortRevision = $SourceRevision.Substring(0, 12)
$tag = "ndc/mission-core-observatory-worker-agent:$shortRevision"
$existingIds = @(
docker image ls `
--quiet `
--no-trunc `
--filter "reference=$tag"
)
if ($LASTEXITCODE -ne 0) {
throw "Worker 006 image lookup failed"
}
if ($existingIds.Count -gt 1) {
throw "Worker 006 image tag resolves ambiguously"
}
if ($existingIds.Count -eq 1) {
$image = Get-ImageInspection $tag
if ([string]$image.Id -cne [string]$existingIds[0]) {
throw "Worker 006 existing image lookup changed"
}
$layerInspection = Assert-ImageContract `
$image `
$base `
([string]$embeddedManifest.sha256)
Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest
New-OutputContract `
"already-installed" `
$tag `
$image `
$layerInspection `
$snapshotBefore `
$embeddedManifest |
ConvertTo-Json -Compress -Depth 9
return
}
$containerName = "ndc-observatory-worker-agent-image-build-$shortRevision"
$occupiedContainerIds = @(
docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}"
)
if ($LASTEXITCODE -ne 0) {
throw "Worker 006 temporary build container lookup failed"
}
if ($occupiedContainerIds.Count -ne 0) {
throw "Worker 006 temporary build container name is already occupied"
}
$copyScript = (
"set -eu; " +
"test ! -e /opt/nodedc/mission-core; " +
"mkdir -p /opt/nodedc/mission-core/src /opt/nodedc/mission-core/release; " +
"mkdir -p /run/nodedc/registries; " +
"cp -a /nodedc-build-source/src/k1link /opt/nodedc/mission-core/src/k1link; " +
"cp /nodedc-build-source/$ContextManifestRelativePath " +
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
"cp /nodedc-build-snapshot.json " +
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
"find /nodedc-build-source/src/k1link -type f -exec sh -c '" +
"for source_path do relative=`${source_path#/nodedc-build-source/}; " +
"cmp -s `"`$source_path`" `"/opt/nodedc/mission-core/`$relative`" || exit 70; " +
"done' sh {} +; " +
"cmp -s /nodedc-build-source/$ContextManifestRelativePath " +
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
"cmp -s /nodedc-build-snapshot.json " +
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
"test ! -e /opt/nodedc/mission-core/experiments; " +
"find /opt/nodedc/mission-core -type d -exec chmod 0555 {} +; " +
"find /opt/nodedc/mission-core -type f -exec chmod 0444 {} +; " +
"chmod 0555 /run/nodedc /run/nodedc/registries; " +
"cd /opt/nodedc/mission-core; " +
"PYTHONPATH=/opt/nodedc/mission-core/src PYTHONNOUSERSITE=1 " +
"PYTHONDONTWRITEBYTECODE=1 python3 -B -c '" +
"import k1link.observatory.m49_worker_container_main as entrypoint; " +
"import k1link.observatory.m49_worker_service as composition; " +
"assert callable(entrypoint.main); " +
"assert callable(composition.compose_installed_m49_worker_service)'"
)
$sourceMount = (
"type=bind,source=$StagedSnapshotRoot," +
"target=/nodedc-build-source,readonly"
)
$manifestMount = (
"type=bind,source=$($embeddedManifest.path)," +
"target=/nodedc-build-snapshot.json,readonly"
)
$containerId = $null
$imageCommitted = $false
$committedImageId = $null
try {
try {
$createOutput = @(
docker create `
--name $containerName `
--network none `
--cap-drop ALL `
--security-opt no-new-privileges `
--mount $sourceMount `
--mount $manifestMount `
--entrypoint /bin/sh `
$BaseImageReference `
-c $copyScript
)
$createdIds = @(
$createOutput | Where-Object { $_ -match "^[a-f0-9]{64}$" }
)
if ($createdIds.Count -eq 1) {
$containerId = [string]$createdIds[0]
}
if ($LASTEXITCODE -ne 0 -or $null -eq $containerId) {
throw "Worker 006 temporary build container creation failed"
}
docker start --attach $containerId
if ($LASTEXITCODE -ne 0) {
throw "Worker 006 source installation command failed"
}
$snapshotAfter = Get-StagedSnapshotInspection $StagedSnapshotRoot
if (
[string]$snapshotAfter.sha256 -cne $ExpectedStagedSnapshotSha256 -or
[int64]$snapshotAfter.file_count -ne [int64]$snapshotBefore.file_count -or
[int64]$snapshotAfter.byte_length -ne [int64]$snapshotBefore.byte_length
) {
throw "Worker 006 staged snapshot changed during installation"
}
if ((Get-FileSha256 $embeddedManifest.path) -cne [string]$embeddedManifest.sha256) {
throw "Worker 006 embedded snapshot manifest changed during installation"
}
$changes = @(
"--change", "WORKDIR $ImageWorkdir",
"--change", "USER 0:0",
"--change", "ENTRYPOINT $ImageEntrypoint",
"--change", "CMD $ImageCommand",
"--change", 'LABEL org.opencontainers.image.title="NODE.DC Observatory Worker 006 agent"',
"--change", "LABEL org.opencontainers.image.source=NODEDC_MISSION_CORE",
"--change", "LABEL org.opencontainers.image.revision=$SourceRevision",
"--change", "LABEL com.nodedc.product=mission-core",
"--change", "LABEL com.nodedc.stack=ndc-mission-core-observatory",
"--change", "LABEL com.nodedc.role=observatory-worker-agent",
"--change", "LABEL com.nodedc.managed-by=mission-core-worker-release",
"--change", "LABEL com.nodedc.worker-contour=worker-006",
"--change", "LABEL com.nodedc.authority=observation-only",
"--change", "LABEL com.nodedc.models=external",
"--change", "LABEL com.nodedc.runtime-registries=external-read-only",
"--change", "LABEL com.nodedc.base-image.sha256=$BaseImageSha256",
"--change", "LABEL com.nodedc.build-context.sha256=$ExpectedGitArchiveSha256",
"--change", "LABEL com.nodedc.staged-snapshot.sha256=$ExpectedStagedSnapshotSha256",
"--change", (
"LABEL com.nodedc.embedded-snapshot-manifest.sha256=" +
[string]$embeddedManifest.sha256
),
"--change", "LABEL com.nodedc.build-method=$BuildMethod"
)
foreach ($value in $FixedEnvironment) {
$changes += @("--change", "ENV $value")
}
$commitOutput = docker commit --pause=true @changes $containerId $tag
$committedIds = @(
$commitOutput |
ForEach-Object { ([string]$_).Trim() } |
Where-Object { $_ -match "^sha256:[a-f0-9]{64}$" }
)
if ($LASTEXITCODE -ne 0 -or $committedIds.Count -ne 1) {
throw "Worker 006 image commit failed"
}
$committedImageId = [string]$committedIds[0]
$imageCommitted = $true
}
finally {
if ($null -ne $containerId -and $containerId -match "^[a-f0-9]{64}$") {
docker rm -f $containerId | Out-Null
if ($LASTEXITCODE -ne 0) {
throw "Worker 006 temporary build container cleanup failed"
}
}
}
$image = Get-ImageInspection $tag
if ([string]$image.Id -cne $committedImageId) {
throw "Worker 006 committed image tag identity changed"
}
$layerInspection = Assert-ImageContract `
$image `
$base `
([string]$embeddedManifest.sha256)
Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest
New-OutputContract `
"installed" `
$tag `
$image `
$layerInspection `
$snapshotBefore `
$embeddedManifest |
ConvertTo-Json -Compress -Depth 9
}
catch {
$installationFailure = $_
if ($imageCommitted -and $committedImageId -match "^sha256:[a-f0-9]{64}$") {
docker image rm --force $committedImageId | Out-Null
if ($LASTEXITCODE -ne 0) {
throw (
"Worker 006 image verification failed and committed image cleanup failed: " +
[string]$installationFailure
)
}
}
throw $installationFailure
}
}
finally {
if (
$null -ne $embeddedManifest -and
(Test-Path -LiteralPath $embeddedManifest.path -PathType Leaf)
) {
Remove-Item -LiteralPath $embeddedManifest.path -Force
}
}
experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json
+3 -3
View File
@@ -149,10 +149,10 @@
},
{
"asset_id": "lab-v1-portable-worker",
"byte_length": 47008,
"byte_length": 47392,
"kind": "repository-file",
"repository_path": "src/k1link/observatory/portable_lab_v1_worker.py",
"sha256": "0a954b5d4d2f3cdd588d58ba4234141c32992b4b46380a03b621060c25c4ac23"
"sha256": "56aa40390b8413fdf7f7293329e89420305914abd478821c08bfe8b0a2ec3df8"
},
{
"asset_id": "portable-result-contracts",
@@ -189,7 +189,7 @@
"navigation_or_safety_accepted": false,
"production_accepted": false
},
"candidate_sha256": "546c74fb88b9571996cecc41fa915044e3907365b8891b7fceb297fb74a837c7",
"candidate_sha256": "9d40369e393aea4b1a09a5784336442a082b1b6126afaac379f1ecece39b54f9",
"declared_blockers": [
"combined-executor-entrypoint-uninstalled",
"combined-executor-image-unsealed",
experiments/perception/worker/observatory_portable/promote_portable_lab_v1_ready.py
+1407
View File
File diff suppressed because it is too large Load Diff
experiments/perception/worker/observatory_portable/worker-006-agent-install-plan.template.json
+93 -27
View File
@@ -6,17 +6,31 @@
"build": {
"source_revision": null,
"source_revision_requirement": "full clean committed Git revision",
"source_date_epoch": null,
"build_context_sha256": null,
"git_archive_sha256": null,
"staged_snapshot_sha256": null,
"installer": "experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1",
"context_manifest": "experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json",
"dockerfile": "experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent",
"dockerfile_role": "source-contract evidence included in the staged snapshot; not executed by the exact-layer installer",
"materialization": {
"method": "git archive",
"include_only_context_manifest_entries": true,
"reject_dirty_worktree": true,
"archive_format": "tar",
"archive_paths_source": "context manifest context_entries in declared order",
"build_context_sha256_subject": "exact git-archive tar bytes"
"git_archive_sha256_subject": "exact git-archive tar bytes",
"extraction_target": "D:\\NDC_MISSIONCORE\\runtime\\staging\\observatory-worker-agent-<source-revision>"
},
"staged_snapshot": {
"sha256_subject": "sorted regular-file rows for src/k1link, Dockerfile.worker-006-agent and worker-006-agent-build-context.json",
"canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1",
"reject_reparse_points": true,
"reject_unexpected_entries": true,
"verify_timing": [
"before temporary container creation",
"after source installation and before image commit"
],
"embedded_manifest": "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
},
"base_image": {
"reference": "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
@@ -24,17 +38,20 @@
"must_exist_locally": true,
"pull_allowed": false
},
"docker_build": {
"image_materialization": {
"method": "docker-commit-exact-layer-v1",
"dockerfile_executed": false,
"network": "none",
"pull": false,
"no_cache": true,
"provenance": false,
"platform": "linux/amd64",
"context_input": "exact git-archive tar bytes",
"required_build_args": [
"NODEDC_SOURCE_REVISION",
"NODEDC_BUILD_CONTEXT_SHA256",
"SOURCE_DATE_EPOCH"
"source_input": "exact read-only staged snapshot",
"base_container": "docker create by exact pinned base image ID",
"commit": "docker commit --pause=true with fixed config changes",
"maximum_thin_layer_bytes": 33554432,
"embedded_payload": [
"/opt/nodedc/mission-core/src/k1link",
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json",
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
]
},
"required_preflight": [
@@ -42,7 +59,9 @@
"selected revision resolves to exactly one commit",
"selected revision equals HEAD",
"base image inspect ID equals the pinned SHA-256",
"context archive contains exactly the context manifest entries"
"context archive contains exactly the context manifest entries",
"git archive SHA-256 was verified externally before extraction",
"staged snapshot SHA-256 matches the exact canonical regular-file inventory"
],
"image_tag_template": "ndc/mission-core-observatory-worker-agent:<source-revision-12>"
},
@@ -57,7 +76,21 @@
"com.nodedc.models": "external",
"com.nodedc.runtime-registries": "external-read-only",
"com.nodedc.base-image.sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
"com.nodedc.build-context.sha256": "<build-context-sha256>"
"com.nodedc.build-context.sha256": "<git-archive-sha256>",
"com.nodedc.staged-snapshot.sha256": "<staged-snapshot-sha256>",
"com.nodedc.embedded-snapshot-manifest.sha256": "<embedded-snapshot-manifest-sha256>",
"com.nodedc.build-method": "docker-commit-exact-layer-v1"
},
"rootfs": {
"base_layer_chain": "exact prefix of the pinned base image RootFS.Layers",
"derived_layer_count": 1,
"derived_layer_diff_id_required": true,
"maximum_thin_layer_bytes": 33554432
},
"embedded_identity": {
"source_tree": "byte-for-byte and bidirectional path match against staged src/k1link",
"context_manifest": "byte-for-byte match against the staged context manifest",
"snapshot_manifest": "byte-for-byte match against the generated manifest covering every staged file, including the Dockerfile"
},
"smoke": {
"network": "none",
@@ -68,6 +101,9 @@
"-c",
"import k1link.observatory.m49_worker_container_main as e; import k1link.observatory.m49_worker_service as c; assert callable(e.main); assert callable(c.compose_installed_m49_worker_service)"
],
"staged_source_bytes": "matched",
"embedded_context_bytes": "matched",
"embedded_snapshot_manifest": "matched",
"expected_result": "exit-0"
}
},
@@ -80,7 +116,8 @@
"MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": "/run/nodedc/registries/observatory-portable-run-definitions.json",
"MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": "/run/nodedc/registries/observatory-worker-runtime-candidates.json",
"MISSIONCORE_OBSERVATORY_M49_INSTALLATION_RECEIPT_FILE": "/release/worker-installation-receipt.json",
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE": "/release/lab-v1-worker-installation-receipt.json"
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE": "/release/lab-v1-worker-installation-receipt.json",
"MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE": "/release/lab-v1-executor-release.json"
},
"runtime_registry_files": {
"binding": "individual read-only bind files",
@@ -238,6 +275,9 @@
"state": "external-installed-receipt-required",
"environment_variable": "MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE",
"container_path": "/release/lab-v1-worker-installation-receipt.json",
"release_candidate_environment_variable": "MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE",
"release_candidate_container_path": "/release/lab-v1-executor-release.json",
"release_repository_root": "/release",
"mode": "read-only",
"owns_component_image_identities": true,
"queued_jobs_may_override_component_images": false
@@ -249,29 +289,55 @@
]
},
"receipt_skeleton": {
"schema_version": "missioncore.observatory-worker-agent-image-receipt/v1",
"receipt_state": "not-built",
"schema_version": "missioncore.observatory-worker-agent-image-installation/v1",
"status": null,
"worker_id": "worker-006",
"build_method": "docker-commit-exact-layer-v1",
"source_revision": null,
"source_date_epoch": null,
"build_context_sha256": null,
"provenance": {
"git_archive_sha256": null,
"git_archive_verification": "external-before-extract",
"staged_snapshot_sha256": null,
"staged_snapshot_file_count": null,
"staged_snapshot_byte_length": null,
"staged_snapshot_canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1",
"embedded_snapshot_manifest_sha256": null,
"embedded_snapshot_manifest_byte_length": null
},
"base_image_sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
"derived_image_sha256": null,
"image": {
"tag": null,
"id": null,
"size_bytes": null
"size_bytes": null,
"thin_layer_bytes": null,
"maximum_thin_layer_bytes": 33554432,
"rootfs": {
"base_layer_count": null,
"derived_layer_count": 1,
"derived_layer_diff_id": null,
"pinned_base_is_exact_prefix": true
}
},
"runtime_contract": {
"workdir": "/opt/nodedc/mission-core",
"entrypoint": [
"python3",
"-m",
"k1link.observatory.m49_worker_container_main"
],
"command": [],
"authority": "observation-only",
"models": "external",
"runtime_registries": "external-read-only"
},
"smoke": {
"network": "none",
"read_only_rootfs": true,
"staged_source_bytes": null,
"embedded_context_bytes": null,
"embedded_snapshot_manifest": null,
"result": "not-run"
},
"models_baked_into_image": false,
"runtime_registries_baked_into_image": false,
"authority": {
"commands_enabled": false,
"actuation_allowed": false,
"navigation_or_safety_accepted": false,
"production_accepted": false
}
}
}
src/k1link/observatory/m49_worker_service.py
+435 -4
View File
@@ -1,7 +1,7 @@
"""Fixed POSIX service composition for the sealed portable M4.9 executor.
"""Fixed POSIX service composition for installed M4.9 and LAB V1 executors.
The entrypoint reads only immutable registries, one installation receipt and
path-only Worker service settings. Jobs cannot select commands, providers,
The entrypoint reads only immutable registries, exact external release files
and path-only Worker service settings. Jobs cannot select commands, providers,
modules, images or filesystem locations. Every runtime asset is resolved from
the fixed release layout and re-verified against the ready runtime candidate
before the first queue claim.
@@ -15,6 +15,7 @@ import json
import os
import re
import signal
import socket
import stat
from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager
@@ -35,6 +36,22 @@ from k1link.observatory.m49_portable_executor import (
M49PortableRunnerInstallation,
compose_m49_portable_executor_adapter,
)
from k1link.observatory.portable_lab_v1_executor import (
PortableLabV1ReleaseAsset,
PortableLabV1ReleaseCandidate,
PortableLabV1ReleaseInspection,
)
from k1link.observatory.portable_lab_v1_worker import (
PortableLabV1RunnerInstallation as PortableLabV1ReleaseInstallation,
)
from k1link.observatory.portable_lab_v1_worker_service import (
PORTABLE_LAB_V1_ADAPTER_ID,
PORTABLE_LAB_V1_SETUP_ID,
PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID,
PortableLabV1WorkerInstallationReceipt,
compose_installed_lab_v1_executor_builder,
load_portable_lab_v1_worker_installation_receipt,
)
from k1link.observatory.portable_result_contract import OBSERVATION_ONLY_AUTHORITY
from k1link.observatory.portable_run_definitions import (
PortableRunDefinition,
@@ -64,6 +81,12 @@ M49_WORKER_RUNTIME_REGISTRY_FILE_ENV: Final = "MISSIONCORE_OBSERVATORY_WORKER_RU
M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV: Final = (
"MISSIONCORE_OBSERVATORY_M49_INSTALLATION_RECEIPT_FILE"
)
LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV: Final = (
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE"
)
LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV: Final = (
"MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE"
)
M49_WORKER_INSTALLATION_RECEIPT_SCHEMA: Final = (
"missioncore.m49-tgs-portable-worker-installation-ready-receipt/v1"
@@ -74,12 +97,24 @@ M49_WORKER_RELEASE_ID: Final = "m49-tgs-portable-executor-v1"
M49_EXECUTOR_RELEASE_ASSET_ID: Final = "m49-portable-executor-release"
M49_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = "m49-portable-worker-installation-receipt"
LAB_V1_PORTABLE_CONFIG_ASSET_ID: Final = "ddrnet-portable-config"
LAB_V1_WORKER_AGENT_IMAGE_ASSET_ID: Final = "worker-006-agent-image"
_MAX_RECEIPT_BYTES: Final = 256 * 1024
_MAX_DOCKER_INSPECTION_BYTES: Final = 1024 * 1024
_SHA256: Final = re.compile(r"^[a-f0-9]{64}$")
_CONTAINER_HOSTNAME: Final = re.compile(r"^[a-f0-9]{12,64}$")
_SOURCE_REVISION: Final = re.compile(r"^[a-f0-9]{40}$")
_WORKER_COMPUTER_NAME: Final = "DESKTOP-OPJ8J04"
_DOCKER_SOCKET: Final = Path("/var/run/docker.sock")
_DOCKER_API_VERSION: Final = "v1.47"
_FIXED_RUNTIME_RELEASE_ROOT: Final = Path("/release")
_FIXED_LAB_V1_INSTALLATION_RECEIPT_FILE: Final = (
_FIXED_RUNTIME_RELEASE_ROOT / "lab-v1-worker-installation-receipt.json"
)
_FIXED_LAB_V1_RELEASE_CANDIDATE_FILE: Final = (
_FIXED_RUNTIME_RELEASE_ROOT / "lab-v1-executor-release.json"
)
_PROTECTED_RUNTIME_NAMES: Final = (
"ndc-mission-core-triton",
"ndc-mission-core-perception-worker",
@@ -119,6 +154,8 @@ class M49WorkerEntrypointConfiguration:
definitions_file: Path
runtime_registry_file: Path
installation_receipt_file: Path
lab_v1_installation_receipt_file: Path | None = None
lab_v1_release_candidate_file: Path | None = None
def __post_init__(self) -> None:
for path, label in (
@@ -127,6 +164,33 @@ class M49WorkerEntrypointConfiguration:
(self.installation_receipt_file, "M4.9 installation receipt"),
):
_absolute_path(path, label)
if (self.lab_v1_installation_receipt_file is None) != (
self.lab_v1_release_candidate_file is None
):
raise M49WorkerCompositionError(
"LAB V1 installation receipt and release candidate must be configured together"
)
for optional_path, label in (
(
self.lab_v1_installation_receipt_file,
"LAB V1 installation receipt",
),
(self.lab_v1_release_candidate_file, "LAB V1 release candidate"),
):
if optional_path is not None:
_absolute_path(optional_path, label)
if (
self.lab_v1_installation_receipt_file is not None
and (
self.lab_v1_installation_receipt_file
!= _FIXED_LAB_V1_INSTALLATION_RECEIPT_FILE
or self.lab_v1_release_candidate_file
!= _FIXED_LAB_V1_RELEASE_CANDIDATE_FILE
)
):
raise M49WorkerCompositionError(
"LAB V1 production inputs must use the fixed /release files"
)
@classmethod
def from_environment(
@@ -148,6 +212,14 @@ class M49WorkerEntrypointConfiguration:
values,
M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
),
lab_v1_installation_receipt_file=_required_environment_path(
values,
LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV,
),
lab_v1_release_candidate_file=_required_environment_path(
values,
LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV,
),
)
@@ -483,15 +555,374 @@ def compose_installed_m49_worker_service(
installation=installation,
),
)
installed_builders = list(executor_builders)
if configuration.lab_v1_installation_receipt_file is not None:
installed_builders.insert(
0,
_compose_installed_lab_v1_builder(
configuration=configuration,
definitions=definitions,
runtime_registry=runtime_registry,
),
)
return compose_installed_observatory_worker_service_from_builders(
configuration=configuration.worker,
definitions=definitions,
runtime_registry=runtime_registry,
builders=(m49_builder, *executor_builders),
builders=(m49_builder, *installed_builders),
http_transport=http_transport,
)
def _compose_installed_lab_v1_builder(
*,
configuration: M49WorkerEntrypointConfiguration,
definitions: PortableRunDefinitionRegistry,
runtime_registry: PortableWorkerRuntimeRegistry,
) -> ObservatoryWorkerExecutorBuilderRegistration:
receipt_path = configuration.lab_v1_installation_receipt_file
release_path = configuration.lab_v1_release_candidate_file
if receipt_path is None or release_path is None:
raise M49WorkerCompositionError("LAB V1 production inputs are incomplete")
definition = definitions.resolve_setup(PORTABLE_LAB_V1_SETUP_ID)
candidate = runtime_registry.resolve(
definition.setup_id,
definition.definition_sha256,
)
_verify_ready_lab_v1_identity(definitions, definition, candidate)
receipt = load_portable_lab_v1_worker_installation_receipt(receipt_path)
release_file = _regular_file(release_path, "LAB V1 release candidate")
release = PortableLabV1ReleaseCandidate.from_file(
release_file,
repository_root=_real_directory(
release_file.parent,
"LAB V1 release repository root",
),
)
if receipt.release_candidate_sha256 != release.candidate_sha256:
raise M49WorkerCompositionError(
"LAB V1 installation receipt belongs to another release candidate"
)
release.bind_definition(definition)
inspection = _inspect_installed_lab_v1_release(
release=release,
receipt=receipt,
definition=definition,
running_worker_image_sha256=(
_inspect_running_worker_container_image_sha256()
),
)
bindings = _lab_v1_runtime_asset_bindings(
candidate=candidate,
release=release,
receipt=receipt,
)
admission = inspect_runtime_candidate(candidate, bindings)
if not admission.ready:
raise M49WorkerCompositionError(
"portable LAB V1 local asset admission is not ready"
)
portable_config_path = _lab_v1_release_repository_file(
release,
LAB_V1_PORTABLE_CONFIG_ASSET_ID,
)
release_installation = PortableLabV1ReleaseInstallation(
release=release,
inspection=inspection,
portable_ddrnet_config_path=portable_config_path,
output_parent=(
configuration.worker.work_root / "lab-v1-portable" / "runner-output"
),
)
return compose_installed_lab_v1_executor_builder(
receipt=receipt,
definition=definition,
candidate=candidate,
admission=admission,
release_installation=release_installation,
)
def _verify_ready_lab_v1_identity(
definitions: PortableRunDefinitionRegistry,
definition: PortableRunDefinition,
candidate: PortableWorkerRuntimeCandidate,
) -> None:
ready_identities = {
(item.setup_id, item.definition_sha256)
for item in definitions.ready_recorded_definitions()
}
if (
(definition.setup_id, definition.definition_sha256) not in ready_identities
or definition.setup_id != PORTABLE_LAB_V1_SETUP_ID
or definition.executor.contour_id != WORKER_006_CONTOUR_ID
or candidate.adapter_id != PORTABLE_LAB_V1_ADAPTER_ID
or not definition.executor.ready
or not candidate.ready
):
raise M49WorkerCompositionError(
"fixed Worker requires the exact ready LAB V1 definition"
)
def _lab_v1_runtime_asset_bindings(
*,
candidate: PortableWorkerRuntimeCandidate,
release: PortableLabV1ReleaseCandidate,
receipt: PortableLabV1WorkerInstallationReceipt,
) -> dict[str, PortableWorkerLocalAssetBinding]:
release_assets = {asset.asset_id: asset for asset in release.assets}
bindings: dict[str, PortableWorkerLocalAssetBinding] = {}
for requirement in candidate.reusable_assets:
if (
requirement.asset_id
== PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID
):
if (
requirement.kind != "local-file"
or requirement.sha256 != receipt.file_sha256
or requirement.byte_length != receipt.file_byte_length
):
raise M49WorkerCompositionError(
"LAB V1 runtime installation receipt anchor changed"
)
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
asset_id=requirement.asset_id,
file_path=receipt.path,
)
continue
asset = release_assets.get(requirement.asset_id)
if (
asset is None
or asset.sha256 != requirement.sha256
or asset.byte_length != requirement.byte_length
):
raise M49WorkerCompositionError(
"LAB V1 runtime asset differs from its exact release candidate"
)
if requirement.kind == "container-image" and asset.kind == "container-image":
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
asset_id=requirement.asset_id,
image_sha256=asset.sha256,
)
continue
if asset.repository_path is None:
raise M49WorkerCompositionError(
"LAB V1 non-image runtime asset has no exact release file"
)
bindings[requirement.asset_id] = PortableWorkerLocalAssetBinding(
asset_id=requirement.asset_id,
file_path=_lab_v1_release_repository_file(
release,
requirement.asset_id,
),
)
return bindings
def _inspect_installed_lab_v1_release(
*,
release: PortableLabV1ReleaseCandidate,
receipt: PortableLabV1WorkerInstallationReceipt,
definition: PortableRunDefinition,
running_worker_image_sha256: str,
) -> PortableLabV1ReleaseInspection:
if release.declared_blockers or release.executor_image_sha256 is None:
raise M49WorkerCompositionError("LAB V1 release candidate is not ready")
_digest(
running_worker_image_sha256,
"running Worker container image SHA-256",
)
worker_image_assets = tuple(
asset
for asset in release.assets
if asset.asset_id == LAB_V1_WORKER_AGENT_IMAGE_ASSET_ID
)
if (
len(worker_image_assets) != 1
or worker_image_assets[0].kind != "container-image"
or worker_image_assets[0].sha256 != running_worker_image_sha256
or worker_image_assets[0].byte_length is not None
or worker_image_assets[0].repository_path is not None
or release.executor_image_sha256 != running_worker_image_sha256
):
raise M49WorkerCompositionError(
"LAB V1 executor image differs from the running Worker container"
)
installed_evidence = _lab_v1_installed_release_evidence(receipt)
installed_images = {
receipt.eomt_image_build.base_image_sha256,
receipt.eomt_image_build.derived_image_sha256,
receipt.ddrnet_image_build.base_image_sha256,
receipt.ddrnet_image_build.derived_image_sha256,
running_worker_image_sha256,
}
definition_components = {component.sha256 for component in definition.components}
matched: list[str] = []
for asset in release.assets:
if (
asset.asset_id
== PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID
):
raise M49WorkerCompositionError(
"LAB V1 installation receipt must remain a runtime-only anchor"
)
if asset.kind == "repository-file":
_lab_v1_release_repository_file(release, asset.asset_id)
elif asset.kind == "container-image":
if asset.byte_length is not None or asset.sha256 not in installed_images:
raise M49WorkerCompositionError(
"LAB V1 release contains an uninstalled container image"
)
elif asset.kind == "definition-component":
if asset.byte_length is not None or asset.sha256 not in definition_components:
raise M49WorkerCompositionError(
"LAB V1 release contains an unknown definition component"
)
elif not _lab_v1_release_evidence_matches(asset, installed_evidence):
raise M49WorkerCompositionError(
"LAB V1 release contains an asset absent from its installation receipt"
)
matched.append(asset.asset_id)
return PortableLabV1ReleaseInspection(
candidate_sha256=release.candidate_sha256,
matched_assets=tuple(matched),
blockers=(),
ready=True,
)
def _inspect_running_worker_container_image_sha256(
*,
container_hostname: str | None = None,
transport: httpx.BaseTransport | None = None,
docker_socket: Path = _DOCKER_SOCKET,
) -> str:
"""Return the exact image ID of this running Worker container."""
hostname = socket.gethostname() if container_hostname is None else container_hostname
if _CONTAINER_HOSTNAME.fullmatch(hostname) is None:
raise M49WorkerCompositionError(
"running Worker container hostname is not an exact container ID"
)
selected_transport = transport
if selected_transport is None:
candidate = docker_socket.expanduser().absolute()
try:
metadata = candidate.lstat()
except OSError as exc:
raise M49WorkerCompositionError(
"running Worker Docker Engine socket is unavailable"
) from exc
if stat.S_ISLNK(metadata.st_mode) or not stat.S_ISSOCK(metadata.st_mode):
raise M49WorkerCompositionError(
"running Worker Docker Engine socket is unsafe"
)
selected_transport = httpx.HTTPTransport(uds=str(candidate))
try:
with httpx.Client(
base_url="http://docker",
transport=selected_transport,
timeout=5.0,
) as client:
response = client.get(
f"/{_DOCKER_API_VERSION}/containers/{hostname}/json"
)
payload = response.content
except (httpx.HTTPError, OSError) as exc:
raise M49WorkerCompositionError(
"running Worker container inspection failed"
) from exc
if response.status_code != 200 or not 0 < len(payload) <= _MAX_DOCKER_INSPECTION_BYTES:
raise M49WorkerCompositionError(
"running Worker container inspection is unavailable"
)
try:
document = json.loads(payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise M49WorkerCompositionError(
"running Worker container inspection is invalid"
) from exc
row = _object(document, "running Worker container inspection")
container_id = _string(row.get("Id"), "running Worker container ID")
image_reference = _string(
row.get("Image"),
"running Worker container image ID",
)
state = _object(row.get("State"), "running Worker container state")
image_sha256 = image_reference.removeprefix("sha256:")
if (
_SHA256.fullmatch(container_id) is None
or not container_id.startswith(hostname)
or image_reference != f"sha256:{image_sha256}"
or _SHA256.fullmatch(image_sha256) is None
or state.get("Running") is not True
):
raise M49WorkerCompositionError(
"running Worker container identity changed"
)
return image_sha256
def _lab_v1_installed_release_evidence(
receipt: PortableLabV1WorkerInstallationReceipt,
) -> dict[str, set[int | None]]:
evidence: dict[str, set[int | None]] = {}
def add(digest: str, byte_length: int | None) -> None:
evidence.setdefault(digest, set()).add(byte_length)
for component in (
receipt.runner_installation.eomt,
receipt.runner_installation.ddrnet,
):
add(component.image_sha256, None)
add(component.installation_sha256, None)
for asset in component.assets:
add(asset.identity_sha256, asset.byte_length)
add(receipt.runner_installation.receipt_sha256, None)
for build in (receipt.eomt_image_build, receipt.ddrnet_image_build):
for key, value in build.identity_document().items():
if key.endswith("_sha256") and isinstance(value, str):
add(value, None)
add(build.seal_sha256, None)
return evidence
def _lab_v1_release_evidence_matches(
asset: PortableLabV1ReleaseAsset,
evidence: Mapping[str, set[int | None]],
) -> bool:
lengths = evidence.get(asset.sha256)
if lengths is None:
return False
return asset.byte_length is None or asset.byte_length in lengths
def _lab_v1_release_repository_file(
release: PortableLabV1ReleaseCandidate,
asset_id: str,
) -> Path:
asset = next((row for row in release.assets if row.asset_id == asset_id), None)
if asset is None or asset.kind != "repository-file" or asset.repository_path is None:
raise M49WorkerCompositionError("LAB V1 exact release file is unavailable")
relative = _safe_relative_path(asset.repository_path)
path = _regular_file(
release.repository_root.joinpath(*relative.parts),
"LAB V1 release file",
)
if not path.is_relative_to(release.repository_root):
raise M49WorkerCompositionError("LAB V1 release file escapes its release root")
if (
(asset.byte_length is not None and path.stat().st_size != asset.byte_length)
or _sha256_file(path) != asset.sha256
):
raise M49WorkerCompositionError(
"LAB V1 release file differs from its release candidate"
)
return path
def run_installed_m49_worker(
service: InstalledObservatoryWorkerService,
*,
src/k1link/observatory/portable_lab_v1_worker.py
+9
View File
@@ -94,6 +94,9 @@ _EXPECTED_RESULT_CONTRACT_SHA256: Final = (
"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a"
)
_PORTABLE_CONFIG_ASSET_ID: Final = "ddrnet-portable-config"
_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = (
"lab-v1-worker-installation-receipt"
)
_MAX_SOURCE_DOCUMENT_BYTES: Final = 8 * 1024 * 1024
_MAX_MATERIALIZATION_MANIFEST_BYTES: Final = 64 * 1024 * 1024
_MAX_SOURCE_MEMBERS: Final = 100_000
@@ -582,6 +585,12 @@ def _verify_candidate_release(
):
raise PortableLabV1WorkerError("portable LAB V1 runtime and release candidates disagree")
for requirement in candidate.reusable_assets:
if requirement.asset_id == _WORKER_INSTALLATION_RECEIPT_ASSET_ID:
if requirement.kind != "local-file":
raise PortableLabV1WorkerError(
"portable LAB V1 installation receipt runtime anchor changed"
)
continue
asset = release_assets.get(requirement.asset_id)
if (
asset is None
src/k1link/observatory/portable_lab_v1_worker_service.py
+58 -5
View File
@@ -63,10 +63,13 @@ from k1link.observatory.worker_service import (
)
PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-worker-installation-ready-receipt/v1"
"missioncore.observatory-portable-lab-v1-worker-installation-ready-receipt/v2"
)
PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_SEAL_SCHEMA: Final = (
"missioncore.observatory-portable-lab-v1-component-image-build-seal/v1"
"missioncore.observatory-portable-lab-v1-component-image-build-seal/v2"
)
PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_METHOD: Final = (
"docker-commit-exact-layer-v1"
)
PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID: Final = (
"lab-v1-worker-installation-receipt"
@@ -98,6 +101,8 @@ class PortableLabV1ComponentImageBuildSeal:
base_image_sha256: str
derived_image_sha256: str
dockerfile_sha256: str
build_method: str
installer_sha256: str
shared_adapter_sha256: str
component_adapter_sha256: str
network: str
@@ -112,6 +117,7 @@ class PortableLabV1ComponentImageBuildSeal:
(self.base_image_sha256, "LAB V1 component base image SHA-256"),
(self.derived_image_sha256, "LAB V1 derived component image SHA-256"),
(self.dockerfile_sha256, "LAB V1 component Dockerfile SHA-256"),
(self.installer_sha256, "LAB V1 component image installer SHA-256"),
(self.shared_adapter_sha256, "LAB V1 shared adapter SHA-256"),
(self.component_adapter_sha256, "LAB V1 component adapter SHA-256"),
(self.seal_sha256, "LAB V1 component image build seal SHA-256"),
@@ -120,6 +126,8 @@ class PortableLabV1ComponentImageBuildSeal:
if (
self.base_image_sha256
!= _COMPONENT_BASE_IMAGE_SHA256S[self.component]
or self.build_method
!= PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_METHOD
or self.network != "none"
or self.seal_sha256
!= hashlib.sha256(canonical_json(self.identity_document())).hexdigest()
@@ -136,6 +144,8 @@ class PortableLabV1ComponentImageBuildSeal:
base_image_sha256: str,
derived_image_sha256: str,
dockerfile_sha256: str,
build_method: str,
installer_sha256: str,
shared_adapter_sha256: str,
component_adapter_sha256: str,
) -> PortableLabV1ComponentImageBuildSeal:
@@ -144,6 +154,8 @@ class PortableLabV1ComponentImageBuildSeal:
base_image_sha256=base_image_sha256,
derived_image_sha256=derived_image_sha256,
dockerfile_sha256=dockerfile_sha256,
build_method=build_method,
installer_sha256=installer_sha256,
shared_adapter_sha256=shared_adapter_sha256,
component_adapter_sha256=component_adapter_sha256,
)
@@ -152,6 +164,8 @@ class PortableLabV1ComponentImageBuildSeal:
base_image_sha256=base_image_sha256,
derived_image_sha256=derived_image_sha256,
dockerfile_sha256=dockerfile_sha256,
build_method=build_method,
installer_sha256=installer_sha256,
shared_adapter_sha256=shared_adapter_sha256,
component_adapter_sha256=component_adapter_sha256,
network="none",
@@ -164,6 +178,8 @@ class PortableLabV1ComponentImageBuildSeal:
base_image_sha256=self.base_image_sha256,
derived_image_sha256=self.derived_image_sha256,
dockerfile_sha256=self.dockerfile_sha256,
build_method=self.build_method,
installer_sha256=self.installer_sha256,
shared_adapter_sha256=self.shared_adapter_sha256,
component_adapter_sha256=self.component_adapter_sha256,
)
@@ -181,6 +197,7 @@ class PortableLabV1WorkerInstallationReceipt:
runner_installation: PortableLabV1ComponentRunnerInstallation
eomt_image_build: PortableLabV1ComponentImageBuildSeal
ddrnet_image_build: PortableLabV1ComponentImageBuildSeal
installation_evidence_sha256: str
receipt_sha256: str
def __post_init__(self) -> None:
@@ -193,6 +210,10 @@ class PortableLabV1WorkerInstallationReceipt:
"LAB V1 installation receipt size is invalid"
)
_digest(self.file_sha256, "LAB V1 installation receipt file SHA-256")
_digest(
self.installation_evidence_sha256,
"LAB V1 external installation evidence SHA-256",
)
if _SOURCE_REVISION.fullmatch(self.source_revision) is None:
raise PortableLabV1WorkerCompositionError(
"LAB V1 installation receipt source revision is invalid"
@@ -223,6 +244,7 @@ class PortableLabV1WorkerInstallationReceipt:
runner_installation=self.runner_installation,
eomt_image_build=self.eomt_image_build,
ddrnet_image_build=self.ddrnet_image_build,
installation_evidence_sha256=self.installation_evidence_sha256,
):
raise PortableLabV1WorkerCompositionError(
"LAB V1 installation receipt identity changed"
@@ -309,7 +331,8 @@ def load_portable_lab_v1_worker_installation_receipt(
"runner_installation",
"component_installations",
"component_image_build_seals",
"fixture_smoke",
"installation_evidence_sha256",
"installation_smoke",
"blockers",
"authority",
"receipt_sha256",
@@ -322,7 +345,7 @@ def load_portable_lab_v1_worker_installation_receipt(
!= PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_SCHEMA
or row["receipt_state"] != "installed-ready"
or row["worker_id"] != _WORKER_ID
or row["fixture_smoke"] != "passed"
or row["installation_smoke"] != "offline-import-passed"
or blockers
or row["authority"] != OBSERVATION_ONLY_AUTHORITY
):
@@ -385,6 +408,10 @@ def load_portable_lab_v1_worker_installation_receipt(
runner_installation=runner,
eomt_image_build=eomt_image_build,
ddrnet_image_build=ddrnet_image_build,
installation_evidence_sha256=_string(
row["installation_evidence_sha256"],
"LAB V1 external installation evidence SHA-256",
),
receipt_sha256=_string(
row["receipt_sha256"],
"LAB V1 installation receipt SHA-256",
@@ -501,6 +528,7 @@ def portable_lab_v1_worker_receipt_document(
runner_installation: PortableLabV1ComponentRunnerInstallation,
eomt_image_build: PortableLabV1ComponentImageBuildSeal,
ddrnet_image_build: PortableLabV1ComponentImageBuildSeal,
installation_evidence_sha256: str,
) -> dict[str, object]:
"""Return the canonical serializable receipt document for an installer."""
@@ -510,6 +538,7 @@ def portable_lab_v1_worker_receipt_document(
runner_installation=runner_installation,
eomt_image_build=eomt_image_build,
ddrnet_image_build=ddrnet_image_build,
installation_evidence_sha256=installation_evidence_sha256,
)
return {**identity, "receipt_sha256": hashlib.sha256(canonical_json(identity)).hexdigest()}
@@ -521,6 +550,7 @@ def _receipt_identity_sha256(
runner_installation: PortableLabV1ComponentRunnerInstallation,
eomt_image_build: PortableLabV1ComponentImageBuildSeal,
ddrnet_image_build: PortableLabV1ComponentImageBuildSeal,
installation_evidence_sha256: str,
) -> str:
return hashlib.sha256(
canonical_json(
@@ -530,6 +560,7 @@ def _receipt_identity_sha256(
runner_installation=runner_installation,
eomt_image_build=eomt_image_build,
ddrnet_image_build=ddrnet_image_build,
installation_evidence_sha256=installation_evidence_sha256,
)
)
).hexdigest()
@@ -542,7 +573,12 @@ def _receipt_identity_document(
runner_installation: PortableLabV1ComponentRunnerInstallation,
eomt_image_build: PortableLabV1ComponentImageBuildSeal,
ddrnet_image_build: PortableLabV1ComponentImageBuildSeal,
installation_evidence_sha256: str,
) -> dict[str, object]:
_digest(
installation_evidence_sha256,
"LAB V1 external installation evidence SHA-256",
)
return {
"schema_version": PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_SCHEMA,
"receipt_state": "installed-ready",
@@ -561,7 +597,8 @@ def _receipt_identity_document(
"eomt": _component_image_build_document(eomt_image_build),
"ddrnet": _component_image_build_document(ddrnet_image_build),
},
"fixture_smoke": "passed",
"installation_evidence_sha256": installation_evidence_sha256,
"installation_smoke": "offline-import-passed",
"blockers": [],
"authority": dict(OBSERVATION_ONLY_AUTHORITY),
}
@@ -582,6 +619,8 @@ def _component_image_build_identity(
base_image_sha256: str,
derived_image_sha256: str,
dockerfile_sha256: str,
build_method: str,
installer_sha256: str,
shared_adapter_sha256: str,
component_adapter_sha256: str,
) -> dict[str, object]:
@@ -591,6 +630,8 @@ def _component_image_build_identity(
"base_image_sha256": base_image_sha256,
"derived_image_sha256": derived_image_sha256,
"dockerfile_sha256": dockerfile_sha256,
"build_method": build_method,
"installer_sha256": installer_sha256,
"shared_adapter_sha256": shared_adapter_sha256,
"component_adapter_sha256": component_adapter_sha256,
"network": "none",
@@ -618,6 +659,8 @@ def _component_image_build_seal(
"base_image_sha256",
"derived_image_sha256",
"dockerfile_sha256",
"build_method",
"installer_sha256",
"shared_adapter_sha256",
"component_adapter_sha256",
"network",
@@ -649,6 +692,14 @@ def _component_image_build_seal(
row["dockerfile_sha256"],
"LAB V1 component Dockerfile SHA-256",
),
build_method=_string(
row["build_method"],
"LAB V1 component image build method",
),
installer_sha256=_string(
row["installer_sha256"],
"LAB V1 component image installer SHA-256",
),
shared_adapter_sha256=_string(
row["shared_adapter_sha256"],
"LAB V1 shared adapter SHA-256",
@@ -694,6 +745,7 @@ def _verify_release_covers_runner_installation(
for build in image_builds
for digest in (
build.dockerfile_sha256,
build.installer_sha256,
build.shared_adapter_sha256,
build.component_adapter_sha256,
build.seal_sha256,
@@ -969,6 +1021,7 @@ def _utc_now() -> str:
__all__ = [
"PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_METHOD",
"PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_SEAL_SCHEMA",
"PORTABLE_LAB_V1_ADAPTER_ID",
"PORTABLE_LAB_V1_SETUP_ID",
tests/test_observatory_m49_lab_v1_wiring.py
+237
View File
@@ -0,0 +1,237 @@
from __future__ import annotations
import hashlib
from pathlib import Path
from types import SimpleNamespace
from typing import cast
import httpx
import pytest
from k1link.observatory import m49_worker_service as service_module
from k1link.observatory.portable_lab_v1_executor import (
PortableLabV1ReleaseAsset,
PortableLabV1ReleaseCandidate,
)
from k1link.observatory.portable_lab_v1_worker_service import (
PortableLabV1WorkerInstallationReceipt,
)
from k1link.observatory.portable_run_definitions import PortableRunDefinition
def _sha256(payload: bytes) -> str:
return hashlib.sha256(payload).hexdigest()
def _receipt(asset_sha256: str, byte_length: int) -> object:
host_asset = SimpleNamespace(
identity_sha256=asset_sha256,
byte_length=byte_length,
)
eomt = SimpleNamespace(
image_sha256="1" * 64,
installation_sha256="c" * 64,
assets=(host_asset,),
)
ddrnet = SimpleNamespace(
image_sha256="2" * 64,
installation_sha256="d" * 64,
assets=(),
)
eomt_build = SimpleNamespace(
base_image_sha256="3" * 64,
derived_image_sha256=eomt.image_sha256,
seal_sha256="4" * 64,
identity_document=lambda: {
"base_image_sha256": "3" * 64,
"derived_image_sha256": eomt.image_sha256,
"adapter_sha256": "5" * 64,
},
)
ddrnet_build = SimpleNamespace(
base_image_sha256="6" * 64,
derived_image_sha256=ddrnet.image_sha256,
seal_sha256="7" * 64,
identity_document=lambda: {
"base_image_sha256": "6" * 64,
"derived_image_sha256": ddrnet.image_sha256,
"adapter_sha256": "8" * 64,
},
)
return SimpleNamespace(
runner_installation=SimpleNamespace(
eomt=eomt,
ddrnet=ddrnet,
receipt_sha256="e" * 64,
),
eomt_image_build=eomt_build,
ddrnet_image_build=ddrnet_build,
)
def _release(
tmp_path: Path,
*,
installed_asset_sha256: str,
installed_asset_length: int,
include_unknown: bool = False,
include_worker_image: bool = True,
executor_image_sha256: str = "b" * 64,
) -> object:
config_payload = b'{"profile":"lab-v1"}'
config_path = tmp_path / "config.json"
config_path.write_bytes(config_payload)
assets = [
PortableLabV1ReleaseAsset(
asset_id="ddrnet-portable-config",
kind="repository-file",
sha256=_sha256(config_payload),
byte_length=len(config_payload),
repository_path="config.json",
),
PortableLabV1ReleaseAsset(
asset_id="eomt-installed-tree",
kind="runtime-artifact",
sha256=installed_asset_sha256,
byte_length=installed_asset_length,
repository_path=None,
),
]
if include_worker_image:
assets.append(
PortableLabV1ReleaseAsset(
asset_id=service_module.LAB_V1_WORKER_AGENT_IMAGE_ASSET_ID,
kind="container-image",
sha256=executor_image_sha256,
byte_length=None,
repository_path=None,
)
)
if include_unknown:
assets.append(
PortableLabV1ReleaseAsset(
asset_id="unknown-runtime-artifact",
kind="runtime-artifact",
sha256="9" * 64,
byte_length=None,
repository_path=None,
)
)
assets.sort(key=lambda asset: asset.asset_id)
return SimpleNamespace(
assets=tuple(assets),
candidate_sha256="a" * 64,
declared_blockers=(),
executor_image_sha256=executor_image_sha256,
repository_root=tmp_path,
)
def test_release_admission_requires_exact_files_or_installation_evidence(
tmp_path: Path,
) -> None:
installed_payload = b"sealed EoMT tree identity"
installed_sha256 = _sha256(installed_payload)
receipt = _receipt(installed_sha256, len(installed_payload))
release = _release(
tmp_path,
installed_asset_sha256=installed_sha256,
installed_asset_length=len(installed_payload),
)
definition = SimpleNamespace(components=())
inspection = service_module._inspect_installed_lab_v1_release( # noqa: SLF001
release=cast(PortableLabV1ReleaseCandidate, release),
receipt=cast(PortableLabV1WorkerInstallationReceipt, receipt),
definition=cast(PortableRunDefinition, definition),
running_worker_image_sha256="b" * 64,
)
assert inspection.ready
assert inspection.matched_assets == (
"ddrnet-portable-config",
"eomt-installed-tree",
service_module.LAB_V1_WORKER_AGENT_IMAGE_ASSET_ID,
)
def test_release_admission_rejects_uncovered_runtime_artifact(tmp_path: Path) -> None:
installed_payload = b"sealed EoMT tree identity"
installed_sha256 = _sha256(installed_payload)
receipt = _receipt(installed_sha256, len(installed_payload))
release = _release(
tmp_path,
installed_asset_sha256=installed_sha256,
installed_asset_length=len(installed_payload),
include_unknown=True,
)
with pytest.raises(
service_module.M49WorkerCompositionError,
match="absent from its installation receipt",
):
service_module._inspect_installed_lab_v1_release( # noqa: SLF001
release=cast(PortableLabV1ReleaseCandidate, release),
receipt=cast(PortableLabV1WorkerInstallationReceipt, receipt),
definition=cast(
PortableRunDefinition,
SimpleNamespace(components=()),
),
running_worker_image_sha256="b" * 64,
)
@pytest.mark.parametrize("include_worker_image", [False, True])
def test_release_admission_rejects_missing_or_changed_running_worker_image(
tmp_path: Path,
include_worker_image: bool,
) -> None:
receipt = _receipt("1" * 64, 1)
release = _release(
tmp_path,
installed_asset_sha256="1" * 64,
installed_asset_length=1,
include_worker_image=include_worker_image,
executor_image_sha256="b" * 64,
)
with pytest.raises(
service_module.M49WorkerCompositionError,
match="differs from the running Worker container",
):
service_module._inspect_installed_lab_v1_release( # noqa: SLF001
release=cast(PortableLabV1ReleaseCandidate, release),
receipt=cast(PortableLabV1WorkerInstallationReceipt, receipt),
definition=cast(
PortableRunDefinition,
SimpleNamespace(components=()),
),
running_worker_image_sha256="c" * 64,
)
def test_running_worker_image_uses_exact_docker_container_inspection() -> None:
container_id = "a" * 64
image_sha256 = "b" * 64
requested_paths: list[str] = []
def inspect(request: httpx.Request) -> httpx.Response:
requested_paths.append(request.url.path)
return httpx.Response(
200,
json={
"Id": container_id,
"Image": f"sha256:{image_sha256}",
"State": {"Running": True},
},
)
result = service_module._inspect_running_worker_container_image_sha256( # noqa: SLF001
container_hostname=container_id[:12],
transport=httpx.MockTransport(inspect),
)
assert result == image_sha256
assert requested_paths == [
f"/v1.47/containers/{container_id[:12]}/json"
]
tests/test_observatory_m49_worker_entrypoint.py
+83 -1
View File
@@ -38,7 +38,10 @@ from k1link.observatory.portable_worker_runtime import (
PortableWorkerRuntimeRegistry,
)
from k1link.observatory.worker_http_transport import ObservatoryWorkerHttpGateway
from k1link.observatory.worker_service import ObservatoryWorkerServiceConfiguration
from k1link.observatory.worker_service import (
ObservatoryWorkerExecutorBuilderRegistration,
ObservatoryWorkerServiceConfiguration,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
DEFINITIONS_PATH = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
@@ -424,6 +427,62 @@ def test_fixed_m49_identity_allows_an_additional_ready_worker_profile(
)
def test_configured_lab_v1_builder_is_added_to_the_shared_worker_registry(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
configuration, _receipt = _fixture(tmp_path, monkeypatch)
configuration = replace(
configuration,
lab_v1_installation_receipt_file=(
service_module._FIXED_LAB_V1_INSTALLATION_RECEIPT_FILE # noqa: SLF001
),
lab_v1_release_candidate_file=(
service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001
),
)
lab_builder = ObservatoryWorkerExecutorBuilderRegistration(
setup_id="lab-v1-eomt-ddrnet-portable-v1",
builder=lambda _context: pytest.fail("builder must not be called here"),
)
captured: dict[str, object] = {}
sentinel = object()
def compose_lab_v1(**kwargs: object) -> ObservatoryWorkerExecutorBuilderRegistration:
captured["lab_inputs"] = kwargs
return lab_builder
def compose_worker(**kwargs: object) -> object:
captured["worker_inputs"] = kwargs
return sentinel
monkeypatch.setattr(
service_module,
"_compose_installed_lab_v1_builder",
compose_lab_v1,
)
monkeypatch.setattr(
service_module,
"compose_installed_observatory_worker_service_from_builders",
compose_worker,
)
composed = service_module.compose_installed_m49_worker_service(configuration)
assert composed is sentinel
lab_inputs = cast(dict[str, object], captured["lab_inputs"])
assert lab_inputs["configuration"] is configuration
worker_inputs = cast(dict[str, object], captured["worker_inputs"])
builders = cast(
tuple[ObservatoryWorkerExecutorBuilderRegistration, ...],
worker_inputs["builders"],
)
assert tuple(builder.setup_id for builder in builders) == (
service_module.M49_WORKER_SETUP_ID,
"lab-v1-eomt-ddrnet-portable-v1",
)
def test_fixed_m49_composition_rejects_receipt_asset_drift_before_gateway(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
@@ -467,12 +526,35 @@ def test_entrypoint_environment_requires_all_absolute_fixed_files(tmp_path: Path
service_module.M49_WORKER_DEFINITIONS_FILE_ENV: str(tmp_path / "definitions.json"),
service_module.M49_WORKER_RUNTIME_REGISTRY_FILE_ENV: str(tmp_path / "runtime.json"),
service_module.M49_WORKER_INSTALLATION_RECEIPT_FILE_ENV: str(tmp_path / "receipt.json"),
service_module.LAB_V1_WORKER_INSTALLATION_RECEIPT_FILE_ENV: str(
service_module._FIXED_LAB_V1_INSTALLATION_RECEIPT_FILE # noqa: SLF001
),
service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV: str(
service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001
),
}
configuration = service_module.M49WorkerEntrypointConfiguration.from_environment(environment)
assert configuration.worker.base_url == "http://127.0.0.1:18080"
assert configuration.installation_receipt_file == tmp_path / "receipt.json"
assert configuration.lab_v1_installation_receipt_file == (
service_module._FIXED_LAB_V1_INSTALLATION_RECEIPT_FILE # noqa: SLF001
)
assert configuration.lab_v1_release_candidate_file == (
service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001
)
environment[service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV] = str(
tmp_path / "lab-v1-release.json"
)
with pytest.raises(
service_module.M49WorkerCompositionError,
match="fixed /release files",
):
service_module.M49WorkerEntrypointConfiguration.from_environment(environment)
environment[service_module.LAB_V1_WORKER_RELEASE_CANDIDATE_FILE_ENV] = str(
service_module._FIXED_LAB_V1_RELEASE_CANDIDATE_FILE # noqa: SLF001
)
with pytest.raises(service_module.M49WorkerCompositionError, match="is required"):
service_module.M49WorkerEntrypointConfiguration.from_environment(
{
tests/test_observatory_portable_lab_v1_promotion.py
+435
View File
@@ -0,0 +1,435 @@
from __future__ import annotations
import importlib.util
import json
import sys
from pathlib import Path
from typing import cast
import pytest
from k1link.observatory.portable_lab_v1_executor import (
PortableLabV1ReleaseCandidate,
)
from k1link.observatory.portable_run_definitions import (
PortableRunDefinitionRegistry,
)
from k1link.observatory.portable_worker_runtime import (
PortableWorkerRuntimeRegistry,
)
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
DEFINITIONS = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json"
RUNTIME = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json"
PORTABLE_CONFIG = (
REPOSITORY_ROOT / "config" / "perception" / "lab-v1-eomt-ddrnet-portable-v2.json"
)
PROMOTION_SCRIPT = (
REPOSITORY_ROOT
/ "experiments"
/ "perception"
/ "worker"
/ "observatory_portable"
/ "promote_portable_lab_v1_ready.py"
)
_SPEC = importlib.util.spec_from_file_location(
"observatory_portable_lab_v1_promotion_test",
PROMOTION_SCRIPT,
)
assert _SPEC is not None and _SPEC.loader is not None
promotion = importlib.util.module_from_spec(_SPEC)
sys.modules[_SPEC.name] = promotion
_SPEC.loader.exec_module(promotion)
def _asset(
asset_id: str,
container_path: str,
identity_sha256: str,
byte_length: int | None,
*,
tree: bool,
) -> dict[str, object]:
return {
"asset_id": asset_id,
"host_path": (
"D:\\NDC_MISSIONCORE\\runtime\\assets\\observatory-portable\\"
+ asset_id
),
"container_path": container_path,
"kind": "tree" if tree else "file",
"verification": "identity-sha256" if tree else "sha256",
"identity_sha256": identity_sha256,
"byte_length": byte_length,
}
def _input_document(
work_root: Path,
*,
component_receipt: Path,
coordinator_receipt: Path,
) -> dict[str, object]:
eomt_assets = [
_asset(
"eomt-environment",
"/environment",
"8c8f343a5368ff17edbb58defa1669f6eccfba767aab897a23693872070ab9e0",
211_776_082,
tree=True,
),
_asset(
"eomt-ffmpeg-runtime",
"/opt/ffmpeg",
"03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69",
256_208_352,
tree=True,
),
_asset(
"eomt-model-cache",
"/cache",
"064870e58814b97027d6a7ccd553bf51f5b8e6a8ad82a1cc703584d2dca5690c",
2_552_355_458,
tree=True,
),
_asset(
"eomt-python-environment",
"/opt/env",
"b3f4efc53af491f174b1cff74b3ba03016e67c9c5c74257b49c6e7dd7d853f20",
5_120_848_705,
tree=True,
),
_asset(
"eomt-runner-bundle",
"/runner",
"3bcfb73db5079deffe51173198f7a02e9e4c49f5fc5439d7976757a430fe91d3",
144_128,
tree=True,
),
_asset(
"eomt-transformers-environment",
"/opt/transformers",
"f365de01426a33be51a310923c743655634d0868941bbf3f1aae1647fdeadfc9",
225_272_284,
tree=True,
),
_asset(
"k1-valid-fov-root",
"/valid-fov",
"f4fc2053e4e6213bb364c8773979b755d5682a81b3946c25ff86274bc5f0031e",
6_019,
tree=True,
),
]
ddrnet_assets = [
_asset(
"ddrnet-checkpoint",
"/opt/nodedc/assets/ddrnet-checkpoint",
"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6",
259_419_077,
tree=False,
),
_asset(
"ddrnet-goose-mapping",
"/opt/nodedc/assets/ddrnet-goose-mapping",
"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f",
1_427,
tree=False,
),
_asset(
"ddrnet-goose-runner",
"/opt/nodedc/assets/ddrnet-goose-runner",
"b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1",
32_877,
tree=False,
),
_asset(
"vegetation-policy",
"/opt/nodedc/assets/vegetation-policy",
"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35",
3_022,
tree=False,
),
_asset(
"vegetation-provider-map",
"/opt/nodedc/assets/vegetation-provider-map",
"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352",
2_756,
tree=False,
),
]
return {
"schema_version": promotion.PROMOTION_INPUT_SCHEMA,
"source_revision": "1" * 40,
"component_source_revision": "2" * 40,
"coordinator_image_sha256": "c" * 64,
"work_root": {
"controller_root": str(work_root),
"engine_host_root": "D:\\NDC_MISSIONCORE\\runtime\\work",
},
"components": {
"eomt": {
"base_image_sha256": (
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
),
"derived_image_sha256": "d" * 64,
"dockerfile_sha256": "2" * 64,
"installer_sha256": "3" * 64,
"shared_adapter_sha256": "4" * 64,
"component_adapter_sha256": "5" * 64,
"assets": eomt_assets,
},
"ddrnet": {
"base_image_sha256": (
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
),
"derived_image_sha256": "e" * 64,
"dockerfile_sha256": "6" * 64,
"installer_sha256": "3" * 64,
"shared_adapter_sha256": "4" * 64,
"component_adapter_sha256": "7" * 64,
"assets": ddrnet_assets,
},
},
"installation_evidence": {
"component_image_installer_receipt_file": str(component_receipt),
"coordinator_image_installer_receipt_file": str(coordinator_receipt),
},
}
def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]:
source_revision = "1" * 40
component_source_revision = "2" * 40
component = tmp_path / "component-image-installation.json"
component.write_text(
json.dumps(
[
{
"component": "eomt",
"status": "already-installed",
"tag": (
"ndc/mission-core-lab-v1-eomt-adapter:"
f"{component_source_revision[:12]}"
),
"base_image_sha256": (
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
),
"derived_image_sha256": "d" * 64,
"build_method": "docker-commit-exact-layer-v1",
},
{
"component": "ddrnet",
"status": "already-installed",
"tag": (
"ndc/mission-core-lab-v1-ddrnet-adapter:"
f"{component_source_revision[:12]}"
),
"base_image_sha256": (
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
),
"derived_image_sha256": "e" * 64,
"build_method": "docker-commit-exact-layer-v1",
},
]
),
encoding="utf-8",
)
coordinator = tmp_path / "coordinator-image-installation.json"
coordinator.write_text(
json.dumps(
{
"schema_version": promotion.WORKER_AGENT_IMAGE_INSTALLATION_SCHEMA,
"status": "installed",
"worker_id": "worker-006",
"build_method": "docker-commit-exact-layer-v1",
"source_revision": source_revision,
"provenance": {
"git_archive_sha256": "8" * 64,
"git_archive_verification": "external-before-extract",
"staged_snapshot_sha256": "9" * 64,
"staged_snapshot_file_count": 42,
"staged_snapshot_byte_length": 4096,
"staged_snapshot_canonicalization": (
"utf8-path-nul-length-nul-sha256-lf-v1"
),
"embedded_snapshot_manifest_sha256": "a" * 64,
"embedded_snapshot_manifest_byte_length": 8192,
},
"base_image_sha256": (
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
),
"derived_image_sha256": "c" * 64,
"image": {
"tag": (
"ndc/mission-core-observatory-worker-agent:"
f"{source_revision[:12]}"
),
"id": f"sha256:{'c' * 64}",
"size_bytes": 1_000_000,
"thin_layer_bytes": 10_000,
"maximum_thin_layer_bytes": 33_554_432,
"rootfs": {
"base_layer_count": 42,
"derived_layer_count": 1,
"derived_layer_diff_id": f"sha256:{'b' * 64}",
"pinned_base_is_exact_prefix": True,
},
},
"runtime_contract": {
"workdir": "/opt/nodedc/mission-core",
"entrypoint": [
"python3",
"-m",
"k1link.observatory.m49_worker_container_main",
],
"command": [],
"authority": "observation-only",
"models": "external",
"runtime_registries": "external-read-only",
},
"smoke": {
"network": "none",
"read_only_rootfs": True,
"staged_source_bytes": "matched",
"embedded_context_bytes": "matched",
"embedded_snapshot_manifest": "matched",
"result": "passed",
},
}
),
encoding="utf-8",
)
return component, coordinator
def _files(root: Path) -> dict[str, bytes]:
return {
path.relative_to(root).as_posix(): path.read_bytes()
for path in root.rglob("*")
if path.is_file()
}
def _row(document: dict[str, object], collection: str, setup_id: str) -> object:
rows = cast(list[dict[str, object]], document[collection])
return next(row for row in rows if row["setup_id"] == setup_id)
def test_ready_promotion_is_deterministic_additive_and_round_trips(
tmp_path: Path,
) -> None:
work_root = tmp_path / "work"
work_root.mkdir()
component_receipt, coordinator_receipt = _write_installation_evidence(tmp_path)
input_path = tmp_path / "promotion-input.json"
input_path.write_text(
json.dumps(
_input_document(
work_root,
component_receipt=component_receipt,
coordinator_receipt=coordinator_receipt,
)
),
encoding="utf-8",
)
inputs = promotion.load_promotion_input(input_path)
first = promotion.generate_ready_lab_v1_artifacts(
promotion=inputs,
source_definition_registry=DEFINITIONS,
source_runtime_registry=RUNTIME,
ddrnet_portable_config=PORTABLE_CONFIG,
output_root=tmp_path / "ready-a",
)
second = promotion.generate_ready_lab_v1_artifacts(
promotion=inputs,
source_definition_registry=DEFINITIONS,
source_runtime_registry=RUNTIME,
ddrnet_portable_config=PORTABLE_CONFIG,
output_root=tmp_path / "ready-b",
)
assert _files(first.root) == _files(second.root)
source_definitions = json.loads(DEFINITIONS.read_text(encoding="utf-8"))
ready_definitions_document = json.loads(
first.definition_registry_path.read_text(encoding="utf-8")
)
assert _row(
ready_definitions_document,
"definitions",
"m49-tgs-portable-v2",
) == _row(source_definitions, "definitions", "m49-tgs-portable-v2")
definitions = PortableRunDefinitionRegistry.from_file(
first.definition_registry_path
)
definition = definitions.resolve_setup(promotion.PORTABLE_LAB_V1_SETUP_ID)
runtime = PortableWorkerRuntimeRegistry.from_file(
first.runtime_registry_path,
definitions=definitions,
).resolve(definition.setup_id, definition.definition_sha256)
release = PortableLabV1ReleaseCandidate.from_file(
first.release_candidate_path,
repository_root=first.root,
)
release.bind_definition(definition)
assert first.release_candidate_path.name == "lab-v1-executor-release.json"
assert definition.executor.ready
assert runtime.ready
assert [
asset.asset_id for asset in release.assets if asset.kind == "repository-file"
] == [promotion.DDRNET_PORTABLE_CONFIG_ASSET_ID]
assert promotion.PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID not in {
asset.asset_id for asset in release.assets
}
receipt_requirement = next(
asset
for asset in runtime.reusable_assets
if asset.asset_id
== promotion.PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID
)
assert receipt_requirement.sha256 == first.installation_receipt_file_sha256
assert first.release_candidate_sha256 == second.release_candidate_sha256
assert first.release_sha256 == second.release_sha256
assert first.definition_sha256 == second.definition_sha256
assert first.runtime_candidate_sha256 == second.runtime_candidate_sha256
def test_promotion_rejects_bare_or_mismatched_installer_claims(
tmp_path: Path,
) -> None:
work_root = tmp_path / "work"
work_root.mkdir()
component_receipt, coordinator_receipt = _write_installation_evidence(tmp_path)
bare = _input_document(
work_root,
component_receipt=component_receipt,
coordinator_receipt=coordinator_receipt,
)
del bare["installation_evidence"]
bare_path = tmp_path / "bare.json"
bare_path.write_text(json.dumps(bare), encoding="utf-8")
with pytest.raises(promotion.PortableLabV1PromotionError, match="fields are invalid"):
promotion.load_promotion_input(bare_path)
rows = json.loads(component_receipt.read_text(encoding="utf-8"))
rows[0]["derived_image_sha256"] = "f" * 64
component_receipt.write_text(json.dumps(rows), encoding="utf-8")
mismatched = tmp_path / "mismatched.json"
mismatched.write_text(
json.dumps(
_input_document(
work_root,
component_receipt=component_receipt,
coordinator_receipt=coordinator_receipt,
)
),
encoding="utf-8",
)
with pytest.raises(
promotion.PortableLabV1PromotionError,
match="does not bind the promotion",
):
promotion.load_promotion_input(mismatched)
tests/test_observatory_portable_lab_v1_runtime_anchor.py
+95
View File
@@ -0,0 +1,95 @@
from __future__ import annotations
from types import SimpleNamespace
from typing import cast
import pytest
from k1link.observatory import portable_lab_v1_worker as worker_module
from k1link.observatory.portable_lab_v1_worker import PortableLabV1WorkerError
from k1link.observatory.portable_run_definitions import PortableRunDefinition
from k1link.observatory.portable_worker_runtime import PortableWorkerRuntimeCandidate
def _candidate(*, asset_id: str, kind: str = "local-file") -> object:
return SimpleNamespace(
setup_id="lab-v1-eomt-ddrnet-portable-v1",
definition_id="lab-v1-eomt-ddrnet-portable",
definition_version=2,
definition_sha256="a" * 64,
result_contract_sha256="b" * 64,
phases=tuple(
SimpleNamespace(phase_id=phase_id)
for phase_id in worker_module.PORTABLE_LAB_V1_RUNTIME_PHASES
),
executor=SimpleNamespace(
release_id="lab-v1-eomt-ddrnet-v1",
release_sha256="c" * 64,
image_sha256="d" * 64,
),
reusable_assets=(
SimpleNamespace(
asset_id=asset_id,
kind=kind,
sha256="e" * 64,
byte_length=100,
),
),
)
def _definition() -> object:
return SimpleNamespace(
definition_sha256="a" * 64,
executable_contract_sha256="f" * 64,
)
def _installation() -> object:
seal = SimpleNamespace(
release_id="lab-v1-eomt-ddrnet-v1",
release_sha256="c" * 64,
executor_image_sha256="d" * 64,
)
release = SimpleNamespace(
setup_id="lab-v1-eomt-ddrnet-portable-v1",
definition_id="lab-v1-eomt-ddrnet-portable",
definition_version=2,
definition_contract_sha256="f" * 64,
result_contract_sha256="b" * 64,
assets=(),
seal=lambda _inspection: seal,
)
return SimpleNamespace(release=release, inspection=object())
def test_runtime_only_installation_receipt_anchor_is_not_required_in_release() -> None:
worker_module._verify_candidate_release( # noqa: SLF001
cast(PortableWorkerRuntimeCandidate, _candidate(
asset_id="lab-v1-worker-installation-receipt"
)),
cast(worker_module.PortableLabV1RunnerInstallation, _installation()),
cast(PortableRunDefinition, _definition()),
)
@pytest.mark.parametrize(
("asset_id", "kind"),
[
("unreviewed-runtime-anchor", "local-file"),
("lab-v1-worker-installation-receipt", "container-image"),
],
)
def test_no_other_runtime_asset_bypasses_release_coverage(
asset_id: str,
kind: str,
) -> None:
with pytest.raises(PortableLabV1WorkerError):
worker_module._verify_candidate_release( # noqa: SLF001
cast(
PortableWorkerRuntimeCandidate,
_candidate(asset_id=asset_id, kind=kind),
),
cast(worker_module.PortableLabV1RunnerInstallation, _installation()),
cast(PortableRunDefinition, _definition()),
)
tests/test_observatory_portable_lab_v1_worker_service.py
+43
View File
@@ -49,6 +49,7 @@ RELEASE_SHA256 = "7" * 64
IMAGE_SHA256 = "8" * 64
RELEASE_CANDIDATE_SHA256 = "9" * 64
SOURCE_REVISION = "a" * 40
INSTALLER_SHA256 = hashlib.sha256(b"component-image-installer").hexdigest()
BASE_IMAGE_SHA256S = {
"eomt": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
"ddrnet": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd",
@@ -244,6 +245,10 @@ def _image_build(
dockerfile_sha256=hashlib.sha256(
f"{component}:dockerfile".encode()
).hexdigest(),
build_method=(
service_module.PORTABLE_LAB_V1_COMPONENT_IMAGE_BUILD_METHOD
),
installer_sha256=INSTALLER_SHA256,
shared_adapter_sha256=hashlib.sha256(b"shared-adapter").hexdigest(),
component_adapter_sha256=hashlib.sha256(
f"{component}:adapter".encode()
@@ -293,6 +298,7 @@ def _write_receipt(
"ddrnet",
derived_image_sha256=runner.ddrnet.image_sha256,
),
installation_evidence_sha256="d" * 64,
)
path = tmp_path / path_name
path.write_bytes(canonical_json(document))
@@ -365,6 +371,7 @@ def _composition_inputs(
)
for digest in (
build.dockerfile_sha256,
build.installer_sha256,
build.shared_adapter_sha256,
build.component_adapter_sha256,
build.seal_sha256,
@@ -402,6 +409,11 @@ def test_receipt_loader_round_trips_full_runner_identity(tmp_path: Path) -> None
receipt.runner_installation.eomt.image_sha256
)
assert receipt.ddrnet_image_build.network == "none"
assert receipt.ddrnet_image_build.build_method == (
"docker-commit-exact-layer-v1"
)
assert receipt.ddrnet_image_build.installer_sha256 == INSTALLER_SHA256
assert receipt.installation_evidence_sha256 == "d" * 64
assert receipt.runner_installation.definition_sha256 == (
definition.definition_sha256
)
@@ -411,6 +423,37 @@ def test_receipt_loader_round_trips_full_runner_identity(tmp_path: Path) -> None
assert receipt.file_sha256 == hashlib.sha256(path.read_bytes()).hexdigest()
def test_component_image_build_seal_binds_actual_installer_method() -> None:
seal = _image_build("eomt", derived_image_sha256="b" * 64)
with pytest.raises(
service_module.PortableLabV1WorkerCompositionError,
match="build provenance changed",
):
service_module.PortableLabV1ComponentImageBuildSeal.seal(
component="eomt",
base_image_sha256=BASE_IMAGE_SHA256S["eomt"],
derived_image_sha256="b" * 64,
dockerfile_sha256=seal.dockerfile_sha256,
build_method="dockerfile-build-v1",
installer_sha256=INSTALLER_SHA256,
shared_adapter_sha256=seal.shared_adapter_sha256,
component_adapter_sha256=seal.component_adapter_sha256,
)
with pytest.raises(
service_module.PortableLabV1WorkerCompositionError,
match="image installer SHA-256 is invalid",
):
replace(seal, installer_sha256="invalid")
with pytest.raises(
service_module.PortableLabV1WorkerCompositionError,
match="build provenance changed",
):
replace(seal, installer_sha256="f" * 64)
def test_receipt_loader_rejects_noncanonical_links_and_oversized_files(
tmp_path: Path,
) -> None:
tests/test_observatory_worker_agent_image_artifact.py
+119 -19
View File
@@ -29,6 +29,7 @@ RUNTIME_REGISTRY_CONTAINER_PATH = (
"/run/nodedc/registries/observatory-worker-runtime-candidates.json"
)
LAB_V1_RECEIPT_CONTAINER_PATH = "/release/lab-v1-worker-installation-receipt.json"
LAB_V1_RELEASE_CANDIDATE_CONTAINER_PATH = "/release/lab-v1-executor-release.json"
def _document(path: Path) -> dict[str, object]:
@@ -149,8 +150,13 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
assert document["state"] == "planned-not-built"
build = cast(dict[str, object], document["build"])
assert build["source_revision"] is None
assert build["source_date_epoch"] is None
assert build["build_context_sha256"] is None
assert build["git_archive_sha256"] is None
assert build["staged_snapshot_sha256"] is None
assert build["installer"] == (
"experiments/perception/worker/observatory_portable/"
"Install-Worker006AgentImage.ps1"
)
assert "not executed" in cast(str, build["dockerfile_role"])
materialization = cast(dict[str, object], build["materialization"])
assert materialization == {
"method": "git archive",
@@ -158,25 +164,43 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
"reject_dirty_worktree": True,
"archive_format": "tar",
"archive_paths_source": "context manifest context_entries in declared order",
"build_context_sha256_subject": "exact git-archive tar bytes",
"git_archive_sha256_subject": "exact git-archive tar bytes",
"extraction_target": (
"D:\\NDC_MISSIONCORE\\runtime\\staging\\"
"observatory-worker-agent-<source-revision>"
),
}
staged = cast(dict[str, object], build["staged_snapshot"])
assert staged["canonicalization"] == "utf8-path-nul-length-nul-sha256-lf-v1"
assert staged["reject_reparse_points"] is True
assert staged["reject_unexpected_entries"] is True
assert staged["verify_timing"] == [
"before temporary container creation",
"after source installation and before image commit",
]
assert staged["embedded_manifest"] == (
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
)
base = cast(dict[str, object], build["base_image"])
assert base["reference"] == BASE_REFERENCE
assert base["sha256"] == BASE_SHA256
assert base["must_exist_locally"] is True
assert base["pull_allowed"] is False
docker_build = cast(dict[str, object], build["docker_build"])
assert docker_build == {
image_materialization = cast(dict[str, object], build["image_materialization"])
assert image_materialization == {
"method": "docker-commit-exact-layer-v1",
"dockerfile_executed": False,
"network": "none",
"pull": False,
"no_cache": True,
"provenance": False,
"platform": "linux/amd64",
"context_input": "exact git-archive tar bytes",
"required_build_args": [
"NODEDC_SOURCE_REVISION",
"NODEDC_BUILD_CONTEXT_SHA256",
"SOURCE_DATE_EPOCH",
"source_input": "exact read-only staged snapshot",
"base_container": "docker create by exact pinned base image ID",
"commit": "docker commit --pause=true with fixed config changes",
"maximum_thin_layer_bytes": 33554432,
"embedded_payload": [
"/opt/nodedc/mission-core/src/k1link",
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json",
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json",
],
}
assert build["required_preflight"] == [
@@ -185,6 +209,8 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
"selected revision equals HEAD",
"base image inspect ID equals the pinned SHA-256",
"context archive contains exactly the context manifest entries",
"git archive SHA-256 was verified externally before extraction",
"staged snapshot SHA-256 matches the exact canonical regular-file inventory",
]
acceptance = cast(dict[str, object], document["build_acceptance"])
@@ -193,10 +219,31 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
assert labels["com.nodedc.base-image.sha256"] == BASE_SHA256
assert labels["com.nodedc.models"] == "external"
assert labels["com.nodedc.runtime-registries"] == "external-read-only"
assert labels["com.nodedc.staged-snapshot.sha256"] == (
"<staged-snapshot-sha256>"
)
assert labels["com.nodedc.embedded-snapshot-manifest.sha256"] == (
"<embedded-snapshot-manifest-sha256>"
)
assert labels["com.nodedc.build-method"] == "docker-commit-exact-layer-v1"
rootfs = cast(dict[str, object], acceptance["rootfs"])
assert rootfs == {
"base_layer_chain": "exact prefix of the pinned base image RootFS.Layers",
"derived_layer_count": 1,
"derived_layer_diff_id_required": True,
"maximum_thin_layer_bytes": 33554432,
}
embedded = cast(dict[str, object], acceptance["embedded_identity"])
assert "byte-for-byte" in cast(str, embedded["source_tree"])
assert "byte-for-byte" in cast(str, embedded["context_manifest"])
assert "every staged file" in cast(str, embedded["snapshot_manifest"])
smoke = cast(dict[str, object], acceptance["smoke"])
assert smoke["network"] == "none"
assert smoke["read_only_rootfs"] is True
assert smoke["platform"] == "linux/amd64"
assert smoke["staged_source_bytes"] == "matched"
assert smoke["embedded_context_bytes"] == "matched"
assert smoke["embedded_snapshot_manifest"] == "matched"
assert smoke["expected_result"] == "exit-0"
runtime = cast(dict[str, object], document["runtime"])
@@ -215,6 +262,9 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE": (
LAB_V1_RECEIPT_CONTAINER_PATH
),
"MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE": (
LAB_V1_RELEASE_CANDIDATE_CONTAINER_PATH
),
}
registry_files = cast(dict[str, object], runtime["runtime_registry_files"])
assert registry_files["binding"] == "individual read-only bind files"
@@ -339,19 +389,69 @@ def test_install_plan_requires_offline_build_hardening_smoke_and_unfilled_receip
"state": "external-installed-receipt-required",
"environment_variable": "MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE",
"container_path": LAB_V1_RECEIPT_CONTAINER_PATH,
"release_candidate_environment_variable": (
"MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE"
),
"release_candidate_container_path": (
LAB_V1_RELEASE_CANDIDATE_CONTAINER_PATH
),
"release_repository_root": "/release",
"mode": "read-only",
"owns_component_image_identities": True,
"queued_jobs_may_override_component_images": False,
}
receipt = cast(dict[str, object], document["receipt_skeleton"])
assert receipt["receipt_state"] == "not-built"
assert receipt["schema_version"] == (
"missioncore.observatory-worker-agent-image-installation/v1"
)
assert receipt["status"] is None
assert receipt["build_method"] == "docker-commit-exact-layer-v1"
assert receipt["source_revision"] is None
assert receipt["source_date_epoch"] is None
assert receipt["build_context_sha256"] is None
provenance = cast(dict[str, object], receipt["provenance"])
assert provenance == {
"git_archive_sha256": None,
"git_archive_verification": "external-before-extract",
"staged_snapshot_sha256": None,
"staged_snapshot_file_count": None,
"staged_snapshot_byte_length": None,
"staged_snapshot_canonicalization": (
"utf8-path-nul-length-nul-sha256-lf-v1"
),
"embedded_snapshot_manifest_sha256": None,
"embedded_snapshot_manifest_byte_length": None,
}
assert receipt["base_image_sha256"] == BASE_SHA256
assert receipt["derived_image_sha256"] is None
image = cast(dict[str, object], receipt["image"])
assert image == {"tag": None, "id": None, "size_bytes": None}
assert receipt["models_baked_into_image"] is False
assert receipt["runtime_registries_baked_into_image"] is False
assert receipt["authority"] == AUTHORITY
assert image == {
"tag": None,
"id": None,
"size_bytes": None,
"thin_layer_bytes": None,
"maximum_thin_layer_bytes": 33554432,
"rootfs": {
"base_layer_count": None,
"derived_layer_count": 1,
"derived_layer_diff_id": None,
"pinned_base_is_exact_prefix": True,
},
}
runtime_contract = cast(dict[str, object], receipt["runtime_contract"])
assert runtime_contract == {
"workdir": "/opt/nodedc/mission-core",
"entrypoint": ENTRYPOINT,
"command": [],
"authority": "observation-only",
"models": "external",
"runtime_registries": "external-read-only",
}
receipt_smoke = cast(dict[str, object], receipt["smoke"])
assert receipt_smoke == {
"network": "none",
"read_only_rootfs": True,
"staged_source_bytes": None,
"embedded_context_bytes": None,
"embedded_snapshot_manifest": None,
"result": "not-run",
}
tests/test_worker_006_agent_image_installer.py
+163
View File
@@ -0,0 +1,163 @@
from __future__ import annotations
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
INSTALLER = (
REPOSITORY_ROOT
/ "experiments/perception/worker/observatory_portable"
/ "Install-Worker006AgentImage.ps1"
)
BASE_SHA256 = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
def _script() -> str:
return INSTALLER.read_text(encoding="utf-8")
def test_worker_agent_installer_is_local_offline_and_non_buildkit() -> None:
script = _script()
assert '"docker-commit-exact-layer-v1"' in script
assert BASE_SHA256 in script
assert '"staging\\observatory-worker-agent-$SourceRevision"' in script
assert "docker create `" in script
assert "--network none" in script
assert "--cap-drop ALL" in script
assert "--security-opt no-new-privileges" in script
assert "target=/nodedc-build-source,readonly" in script
assert "docker commit --pause=true @changes $containerId $tag" in script
assert "docker rm -f $containerId" in script
assert "docker rm -f $containerName" not in script
lowered = script.lower()
assert "docker build" not in lowered
assert "docker buildx" not in lowered
assert "docker pull" not in lowered
assert "--pull" not in lowered
assert "invoke-webrequest" not in lowered
assert "start-bitstransfer" not in lowered
assert "curl " not in lowered
assert "wget " not in lowered
assert "smb" not in lowered
def test_worker_agent_installer_separates_archive_and_staged_identities() -> None:
script = _script()
assert "[string]$ExpectedGitArchiveSha256" in script
assert "[string]$ExpectedStagedSnapshotSha256" in script
assert script.count("Get-StagedSnapshotInspection $StagedSnapshotRoot") == 2
assert '"com.nodedc.build-context.sha256" = $ExpectedGitArchiveSha256' in script
assert (
'"com.nodedc.staged-snapshot.sha256" = $ExpectedStagedSnapshotSha256'
in script
)
assert "LABEL com.nodedc.build-context.sha256=$ExpectedGitArchiveSha256" in script
assert (
"LABEL com.nodedc.staged-snapshot.sha256=$ExpectedStagedSnapshotSha256"
in script
)
assert 'git_archive_verification = "external-before-extract"' in script
assert 'canonicalization = "utf8-path-nul-length-nul-sha256-lf-v1"' in script
assert 'Assert-ExactDirectoryChildren $Root @("experiments", "src")' in script
assert 'Assert-ExactDirectoryChildren $srcRoot @("k1link")' in script
assert "Dockerfile.worker-006-agent" in script
assert "worker-006-agent-build-context.json" in script
assert (
'schema_version = "missioncore.observatory-worker-agent-embedded-snapshot/v1"'
in script
)
assert "files = @($SnapshotInspection.files)" in script
assert "embedded_snapshot_manifest_sha256" in script
def test_worker_agent_installer_copies_only_runtime_source_and_contract() -> None:
script = _script()
assert (
"cp -a /nodedc-build-source/src/k1link "
"/opt/nodedc/mission-core/src/k1link"
) in script
assert (
'"cp /nodedc-build-source/$ContextManifestRelativePath " +'
in script
)
assert "/opt/nodedc/mission-core/release/worker-006-agent-build-context.json" in script
assert (
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
in script
)
assert "cp -a /nodedc-build-source/experiments" not in script
assert "cp /nodedc-build-source/$DockerfileRelativePath" not in script
assert "test ! -e /opt/nodedc/mission-core/experiments" in script
assert "cmp -s" in script
assert "find /opt/nodedc/mission-core -type d -exec chmod 0555" in script
assert "find /opt/nodedc/mission-core -type f -exec chmod 0444" in script
assert "chmod 0555 /run/nodedc /run/nodedc/registries" in script
def test_worker_agent_installer_proves_base_chain_and_embedded_bytes() -> None:
script = _script()
assert "$baseLayers = @($BaseImage.RootFS.Layers)" in script
assert "$imageLayers = @($Image.RootFS.Layers)" in script
assert "$imageLayers.Count -ne ($baseLayers.Count + 1)" in script
assert "RootFS does not extend the pinned base layer chain" in script
assert "derived_layer_diff_id = $derivedLayerDiffId" in script
assert "pinned_base_is_exact_prefix = $true" in script
assert "target=/nodedc-verify-source,readonly" in script
assert "target=/nodedc-verify-snapshot.json,readonly" in script
assert "cmp -s /nodedc-verify-snapshot.json" in script
assert (
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
in script
)
assert "find /nodedc-verify-source/src/k1link -type f" in script
assert "find /opt/nodedc/mission-core/src/k1link -type f" in script
assert "cmp -s /nodedc-verify-source/$ContextManifestRelativePath" in script
def test_worker_agent_installer_removes_failed_new_image_and_temporary_files() -> None:
script = _script()
assert "$imageCommitted = $true" in script
assert (
"Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest"
in script
)
assert "docker image rm --force $committedImageId" in script
assert "image verification failed and committed image cleanup failed" in script
assert "Remove-Item -LiteralPath $embeddedManifest.path -Force" in script
def test_worker_agent_installer_seals_runtime_contract_smoke_and_output() -> None:
script = _script()
for value in (
"PYTHONPATH=/opt/nodedc/mission-core/src",
"PYTHONNOUSERSITE=1",
"PYTHONDONTWRITEBYTECODE=1",
"PYTHONUNBUFFERED=1",
"WORKDIR $ImageWorkdir",
"USER 0:0",
"ENTRYPOINT $ImageEntrypoint",
"CMD $ImageCommand",
"com.nodedc.authority=observation-only",
"com.nodedc.models=external",
"com.nodedc.runtime-registries=external-read-only",
):
assert value in script
assert "--read-only" in script
assert '"/tmp:rw,noexec,nosuid,size=16m"' in script
assert "import k1link.observatory.m49_worker_container_main" in script
assert "import k1link.observatory.m49_worker_service" in script
assert "compose_installed_m49_worker_service" in script
assert "$MaximumLayerBytes = [int64](32MB)" in script
assert "image is not within the thin-layer bound" in script
assert (
'schema_version = "missioncore.observatory-worker-agent-image-installation/v1"'
in script
)
assert "derived_image_sha256 = ([string]$Image.Id).Substring(7)" in script
assert 'result = "passed"' in script