feat(worker): install combined observatory profiles
This commit is contained in:
DCCONSTRUCTIONS
@@ -0,0 +1,698 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{40}$")]
|
||||
[string]$SourceRevision,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedGitArchiveSha256,
|
||||
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{64}$")]
|
||||
[string]$ExpectedStagedSnapshotSha256
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
|
||||
$StagedSnapshotRoot = Join-Path (
|
||||
$RuntimeRoot
|
||||
) "staging\observatory-worker-agent-$SourceRevision"
|
||||
$ContextManifestRelativePath = (
|
||||
"experiments/perception/worker/observatory_portable/" +
|
||||
"worker-006-agent-build-context.json"
|
||||
)
|
||||
$DockerfileRelativePath = (
|
||||
"experiments/perception/worker/observatory_portable/" +
|
||||
"Dockerfile.worker-006-agent"
|
||||
)
|
||||
$SourceTreeRelativePath = "src/k1link"
|
||||
$BaseImageSha256 = (
|
||||
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
)
|
||||
$BaseImageReference = "sha256:$BaseImageSha256"
|
||||
$BuildMethod = "docker-commit-exact-layer-v1"
|
||||
$MaximumLayerBytes = [int64](32MB)
|
||||
$ImageWorkdir = "/opt/nodedc/mission-core"
|
||||
$ImageEntrypoint = '["python3","-m","k1link.observatory.m49_worker_container_main"]'
|
||||
$ImageCommand = '[]'
|
||||
$FixedEnvironment = @(
|
||||
"PYTHONPATH=/opt/nodedc/mission-core/src",
|
||||
"PYTHONNOUSERSITE=1",
|
||||
"PYTHONDONTWRITEBYTECODE=1",
|
||||
"PYTHONUNBUFFERED=1"
|
||||
)
|
||||
|
||||
function Get-FileSha256 {
|
||||
param([string]$Path)
|
||||
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (
|
||||
-not ($item -is [IO.FileInfo]) -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||
) {
|
||||
throw "Worker 006 staged source is not a regular file"
|
||||
}
|
||||
return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Assert-ExactDirectoryChildren {
|
||||
param(
|
||||
[string]$Path,
|
||||
[string[]]$ExpectedNames
|
||||
)
|
||||
|
||||
$directory = Get-Item -LiteralPath $Path -Force
|
||||
if (
|
||||
-not ($directory -is [IO.DirectoryInfo]) -or
|
||||
($directory.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||
) {
|
||||
throw "Worker 006 staged snapshot directory is invalid"
|
||||
}
|
||||
$actualNames = @(
|
||||
Get-ChildItem -LiteralPath $directory.FullName -Force |
|
||||
ForEach-Object { $_.Name }
|
||||
)
|
||||
[Array]::Sort($actualNames, [StringComparer]::Ordinal)
|
||||
$expected = @($ExpectedNames)
|
||||
[Array]::Sort($expected, [StringComparer]::Ordinal)
|
||||
if (
|
||||
$actualNames.Count -ne $expected.Count -or
|
||||
([string]::Join("`n", $actualNames) -cne [string]::Join("`n", $expected))
|
||||
) {
|
||||
throw "Worker 006 staged snapshot contains unexpected entries"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-StagedSnapshotInspection {
|
||||
param([string]$Root)
|
||||
|
||||
Assert-ExactDirectoryChildren $Root @("experiments", "src")
|
||||
$srcRoot = Join-Path $Root "src"
|
||||
Assert-ExactDirectoryChildren $srcRoot @("k1link")
|
||||
$experimentsRoot = Join-Path $Root "experiments"
|
||||
Assert-ExactDirectoryChildren $experimentsRoot @("perception")
|
||||
$perceptionRoot = Join-Path $experimentsRoot "perception"
|
||||
Assert-ExactDirectoryChildren $perceptionRoot @("worker")
|
||||
$workerRoot = Join-Path $perceptionRoot "worker"
|
||||
Assert-ExactDirectoryChildren $workerRoot @("observatory_portable")
|
||||
$portableRoot = Join-Path $workerRoot "observatory_portable"
|
||||
Assert-ExactDirectoryChildren $portableRoot @(
|
||||
"Dockerfile.worker-006-agent",
|
||||
"worker-006-agent-build-context.json"
|
||||
)
|
||||
|
||||
$sourceRoot = Join-Path $Root ($SourceTreeRelativePath.Replace("/", "\"))
|
||||
$sourceItems = @(Get-ChildItem -LiteralPath $sourceRoot -Recurse -Force)
|
||||
foreach ($item in $sourceItems) {
|
||||
if ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) {
|
||||
throw "Worker 006 staged snapshot contains a reparse point"
|
||||
}
|
||||
if (-not ($item -is [IO.FileInfo]) -and -not ($item -is [IO.DirectoryInfo])) {
|
||||
throw "Worker 006 staged snapshot contains an unsupported entry"
|
||||
}
|
||||
}
|
||||
|
||||
$manifestPath = Join-Path $Root ($ContextManifestRelativePath.Replace("/", "\"))
|
||||
$dockerfilePath = Join-Path $Root ($DockerfileRelativePath.Replace("/", "\"))
|
||||
$requiredSourceFiles = @(
|
||||
Join-Path $sourceRoot "__init__.py",
|
||||
Join-Path $sourceRoot "observatory\m49_worker_container_main.py",
|
||||
Join-Path $sourceRoot "observatory\m49_worker_service.py"
|
||||
)
|
||||
foreach ($requiredPath in $requiredSourceFiles + @($manifestPath, $dockerfilePath)) {
|
||||
if (-not (Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
|
||||
throw "Worker 006 staged snapshot is incomplete"
|
||||
}
|
||||
[void](Get-FileSha256 $requiredPath)
|
||||
}
|
||||
|
||||
$context = Get-Content -LiteralPath $manifestPath -Raw | ConvertFrom-Json
|
||||
if (
|
||||
[string]$context.schema_version -cne
|
||||
"missioncore.observatory-worker-agent-build-context/v1" -or
|
||||
[string]$context.worker_id -cne "worker-006" -or
|
||||
[string]$context.base_image.sha256 -cne $BaseImageSha256 -or
|
||||
[bool]$context.base_image.pull_allowed
|
||||
) {
|
||||
throw "Worker 006 staged build-context contract is invalid"
|
||||
}
|
||||
$declaredPaths = @($context.context_entries | ForEach-Object { [string]$_.path })
|
||||
$expectedDeclaredPaths = @(
|
||||
$SourceTreeRelativePath,
|
||||
$DockerfileRelativePath,
|
||||
$ContextManifestRelativePath
|
||||
)
|
||||
if (
|
||||
$declaredPaths.Count -ne $expectedDeclaredPaths.Count -or
|
||||
([string]::Join("`n", $declaredPaths) -cne
|
||||
[string]::Join("`n", $expectedDeclaredPaths))
|
||||
) {
|
||||
throw "Worker 006 staged build-context entries changed"
|
||||
}
|
||||
|
||||
$files = New-Object "Collections.Generic.List[object]"
|
||||
foreach ($item in $sourceItems) {
|
||||
if ($item -is [IO.FileInfo]) {
|
||||
$files.Add($item)
|
||||
}
|
||||
}
|
||||
$files.Add((Get-Item -LiteralPath $dockerfilePath -Force))
|
||||
$files.Add((Get-Item -LiteralPath $manifestPath -Force))
|
||||
|
||||
$rows = New-Object "Collections.Generic.List[object]"
|
||||
foreach ($file in $files) {
|
||||
$relative = $file.FullName.Substring($Root.Length).TrimStart("\")
|
||||
$relative = $relative.Replace("\", "/")
|
||||
if (-not $relative -or $relative -match "[\x00\r\n]") {
|
||||
throw "Worker 006 staged snapshot path is invalid"
|
||||
}
|
||||
$rows.Add([pscustomobject]@{
|
||||
path = $relative
|
||||
byte_length = [int64]$file.Length
|
||||
sha256 = Get-FileSha256 $file.FullName
|
||||
})
|
||||
}
|
||||
$orderedRows = @($rows | Sort-Object -Property path -CaseSensitive)
|
||||
$canonical = New-Object Text.StringBuilder
|
||||
[int64]$totalBytes = 0
|
||||
foreach ($row in $orderedRows) {
|
||||
[void]$canonical.Append([string]$row.path)
|
||||
[void]$canonical.Append([char]0)
|
||||
[void]$canonical.Append(([int64]$row.byte_length).ToString([Globalization.CultureInfo]::InvariantCulture))
|
||||
[void]$canonical.Append([char]0)
|
||||
[void]$canonical.Append([string]$row.sha256)
|
||||
[void]$canonical.Append("`n")
|
||||
$totalBytes += [int64]$row.byte_length
|
||||
}
|
||||
$utf8 = New-Object Text.UTF8Encoding($false)
|
||||
$canonicalBytes = $utf8.GetBytes($canonical.ToString())
|
||||
$hasher = [Security.Cryptography.SHA256]::Create()
|
||||
try {
|
||||
$digestBytes = $hasher.ComputeHash($canonicalBytes)
|
||||
}
|
||||
finally {
|
||||
$hasher.Dispose()
|
||||
}
|
||||
$digest = ([BitConverter]::ToString($digestBytes)).Replace("-", "").ToLowerInvariant()
|
||||
return [pscustomobject]@{
|
||||
sha256 = $digest
|
||||
file_count = [int64]$orderedRows.Count
|
||||
byte_length = $totalBytes
|
||||
canonicalization = "utf8-path-nul-length-nul-sha256-lf-v1"
|
||||
files = @($orderedRows)
|
||||
}
|
||||
}
|
||||
|
||||
function New-StagedSnapshotManifest {
|
||||
param([object]$SnapshotInspection)
|
||||
|
||||
$stagingRoot = Split-Path -Parent $StagedSnapshotRoot
|
||||
$manifestPath = Join-Path (
|
||||
$stagingRoot
|
||||
) ("observatory-worker-agent-embedded-manifest-{0}.json" -f [Guid]::NewGuid().ToString("N"))
|
||||
$payload = [ordered]@{
|
||||
schema_version = "missioncore.observatory-worker-agent-embedded-snapshot/v1"
|
||||
snapshot_sha256 = [string]$SnapshotInspection.sha256
|
||||
canonicalization = [string]$SnapshotInspection.canonicalization
|
||||
file_count = [int64]$SnapshotInspection.file_count
|
||||
byte_length = [int64]$SnapshotInspection.byte_length
|
||||
files = @($SnapshotInspection.files)
|
||||
}
|
||||
$json = $payload | ConvertTo-Json -Compress -Depth 5
|
||||
$utf8 = New-Object Text.UTF8Encoding($false)
|
||||
try {
|
||||
[IO.File]::WriteAllText($manifestPath, $json, $utf8)
|
||||
return [pscustomobject]@{
|
||||
path = $manifestPath
|
||||
sha256 = Get-FileSha256 $manifestPath
|
||||
byte_length = [int64](Get-Item -LiteralPath $manifestPath -Force).Length
|
||||
}
|
||||
}
|
||||
catch {
|
||||
if (Test-Path -LiteralPath $manifestPath -PathType Leaf) {
|
||||
Remove-Item -LiteralPath $manifestPath -Force
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Get-ImageInspection {
|
||||
param([string]$Reference)
|
||||
|
||||
$payload = docker image inspect $Reference
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Worker 006 image is unavailable: $Reference"
|
||||
}
|
||||
$rows = @($payload | ConvertFrom-Json)
|
||||
if ($rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") {
|
||||
throw "Worker 006 image inspection is invalid"
|
||||
}
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
function Assert-ImageContract {
|
||||
param(
|
||||
[object]$Image,
|
||||
[object]$BaseImage,
|
||||
[string]$EmbeddedManifestSha256
|
||||
)
|
||||
|
||||
$labels = $Image.Config.Labels
|
||||
$expectedLabels = [ordered]@{
|
||||
"org.opencontainers.image.title" = "NODE.DC Observatory Worker 006 agent"
|
||||
"org.opencontainers.image.source" = "NODEDC_MISSION_CORE"
|
||||
"org.opencontainers.image.revision" = $SourceRevision
|
||||
"com.nodedc.product" = "mission-core"
|
||||
"com.nodedc.stack" = "ndc-mission-core-observatory"
|
||||
"com.nodedc.role" = "observatory-worker-agent"
|
||||
"com.nodedc.managed-by" = "mission-core-worker-release"
|
||||
"com.nodedc.worker-contour" = "worker-006"
|
||||
"com.nodedc.authority" = "observation-only"
|
||||
"com.nodedc.models" = "external"
|
||||
"com.nodedc.runtime-registries" = "external-read-only"
|
||||
"com.nodedc.base-image.sha256" = $BaseImageSha256
|
||||
"com.nodedc.build-context.sha256" = $ExpectedGitArchiveSha256
|
||||
"com.nodedc.staged-snapshot.sha256" = $ExpectedStagedSnapshotSha256
|
||||
"com.nodedc.embedded-snapshot-manifest.sha256" = $EmbeddedManifestSha256
|
||||
"com.nodedc.build-method" = $BuildMethod
|
||||
}
|
||||
foreach ($key in $expectedLabels.Keys) {
|
||||
if ([string]$labels.$key -cne [string]$expectedLabels[$key]) {
|
||||
throw "Worker 006 installed image label identity changed: $key"
|
||||
}
|
||||
}
|
||||
if ([string]$Image.Config.WorkingDir -cne $ImageWorkdir) {
|
||||
throw "Worker 006 installed image workdir changed"
|
||||
}
|
||||
if ([string]::Join([char]0, @($Image.Config.Entrypoint)) -cne
|
||||
[string]::Join([char]0, @("python3", "-m", "k1link.observatory.m49_worker_container_main"))) {
|
||||
throw "Worker 006 installed image entrypoint changed"
|
||||
}
|
||||
if (@($Image.Config.Cmd).Count -ne 0) {
|
||||
throw "Worker 006 installed image command changed"
|
||||
}
|
||||
if ([string]$Image.Config.User -cne "0:0") {
|
||||
throw "Worker 006 installed image user changed"
|
||||
}
|
||||
$actualEnvironment = @($Image.Config.Env)
|
||||
foreach ($value in $FixedEnvironment) {
|
||||
if ($actualEnvironment -cnotcontains $value) {
|
||||
throw "Worker 006 installed image environment changed"
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
[string]$BaseImage.RootFS.Type -cne "layers" -or
|
||||
[string]$Image.RootFS.Type -cne "layers"
|
||||
) {
|
||||
throw "Worker 006 image RootFS type changed"
|
||||
}
|
||||
$baseLayers = @($BaseImage.RootFS.Layers)
|
||||
$imageLayers = @($Image.RootFS.Layers)
|
||||
if ($baseLayers.Count -eq 0 -or $imageLayers.Count -ne ($baseLayers.Count + 1)) {
|
||||
throw "Worker 006 image is not exactly one layer above the pinned base"
|
||||
}
|
||||
for ($index = 0; $index -lt $baseLayers.Count; $index += 1) {
|
||||
if ([string]$imageLayers[$index] -cne [string]$baseLayers[$index]) {
|
||||
throw "Worker 006 image RootFS does not extend the pinned base layer chain"
|
||||
}
|
||||
}
|
||||
$derivedLayerDiffId = [string]$imageLayers[$baseLayers.Count]
|
||||
if ($derivedLayerDiffId -notmatch "^sha256:[a-f0-9]{64}$") {
|
||||
throw "Worker 006 derived layer identity is invalid"
|
||||
}
|
||||
|
||||
$layerBytes = [int64]$Image.Size - [int64]$BaseImage.Size
|
||||
if ($layerBytes -lt 0 -or $layerBytes -gt $MaximumLayerBytes) {
|
||||
throw "Worker 006 image is not within the thin-layer bound"
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
layer_bytes = $layerBytes
|
||||
base_layer_count = [int64]$baseLayers.Count
|
||||
derived_layer_count = [int64]1
|
||||
derived_layer_diff_id = $derivedLayerDiffId
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-InstalledImageSmoke {
|
||||
param(
|
||||
[string]$Tag,
|
||||
[string]$SnapshotRoot,
|
||||
[object]$EmbeddedManifest
|
||||
)
|
||||
|
||||
$smokeScript = (
|
||||
"set -eu; " +
|
||||
"test ! -e /opt/nodedc/mission-core/experiments; " +
|
||||
"test -z `"`$(find /opt/nodedc/mission-core -perm /022 -print -quit)`"; " +
|
||||
"test -z `"`$(find /opt/nodedc/mission-core/src/k1link " +
|
||||
"! -type d ! -type f -print -quit)`"; " +
|
||||
"test -z `"`$(find /nodedc-verify-source/src/k1link " +
|
||||
"! -type d ! -type f -print -quit)`"; " +
|
||||
"cmp -s /nodedc-verify-snapshot.json " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
|
||||
"test `"`$(sha256sum " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json " +
|
||||
"| cut -d' ' -f1)`" = $($EmbeddedManifest.sha256); " +
|
||||
"cmp -s /nodedc-verify-source/$ContextManifestRelativePath " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
|
||||
"find /nodedc-verify-source/src/k1link -type d -exec sh -c '" +
|
||||
"for source_path do relative=`${source_path#/nodedc-verify-source/}; " +
|
||||
"test -d `"/opt/nodedc/mission-core/`$relative`" || exit 70; " +
|
||||
"done' sh {} +; " +
|
||||
"find /opt/nodedc/mission-core/src/k1link -type d -exec sh -c '" +
|
||||
"for image_path do relative=`${image_path#/opt/nodedc/mission-core/}; " +
|
||||
"test -d `"/nodedc-verify-source/`$relative`" || exit 71; " +
|
||||
"done' sh {} +; " +
|
||||
"find /nodedc-verify-source/src/k1link -type f -exec sh -c '" +
|
||||
"for source_path do relative=`${source_path#/nodedc-verify-source/}; " +
|
||||
"cmp -s `"`$source_path`" `"/opt/nodedc/mission-core/`$relative`" || exit 72; " +
|
||||
"done' sh {} +; " +
|
||||
"find /opt/nodedc/mission-core/src/k1link -type f -exec sh -c '" +
|
||||
"for image_path do relative=`${image_path#/opt/nodedc/mission-core/}; " +
|
||||
"cmp -s `"`$image_path`" `"/nodedc-verify-source/`$relative`" || exit 73; " +
|
||||
"done' sh {} +; " +
|
||||
"cd /opt/nodedc/mission-core; " +
|
||||
"python3 -B -c 'import k1link.observatory.m49_worker_container_main as entrypoint; " +
|
||||
"import k1link.observatory.m49_worker_service as composition; " +
|
||||
"assert callable(entrypoint.main); " +
|
||||
"assert callable(composition.compose_installed_m49_worker_service)'"
|
||||
)
|
||||
$sourceMount = (
|
||||
"type=bind,source=$SnapshotRoot," +
|
||||
"target=/nodedc-verify-source,readonly"
|
||||
)
|
||||
$manifestMount = (
|
||||
"type=bind,source=$($EmbeddedManifest.path)," +
|
||||
"target=/nodedc-verify-snapshot.json,readonly"
|
||||
)
|
||||
docker run `
|
||||
--rm `
|
||||
--network none `
|
||||
--read-only `
|
||||
--tmpfs "/tmp:rw,noexec,nosuid,size=16m" `
|
||||
--cap-drop ALL `
|
||||
--security-opt no-new-privileges `
|
||||
--mount $sourceMount `
|
||||
--mount $manifestMount `
|
||||
--entrypoint /bin/sh `
|
||||
$Tag `
|
||||
-c $smokeScript
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Installed Worker 006 image smoke failed"
|
||||
}
|
||||
}
|
||||
|
||||
function New-OutputContract {
|
||||
param(
|
||||
[string]$Status,
|
||||
[string]$Tag,
|
||||
[object]$Image,
|
||||
[object]$LayerInspection,
|
||||
[object]$SnapshotInspection,
|
||||
[object]$EmbeddedManifest
|
||||
)
|
||||
|
||||
return [ordered]@{
|
||||
schema_version = "missioncore.observatory-worker-agent-image-installation/v1"
|
||||
status = $Status
|
||||
worker_id = "worker-006"
|
||||
build_method = $BuildMethod
|
||||
source_revision = $SourceRevision
|
||||
provenance = [ordered]@{
|
||||
git_archive_sha256 = $ExpectedGitArchiveSha256
|
||||
git_archive_verification = "external-before-extract"
|
||||
staged_snapshot_sha256 = [string]$SnapshotInspection.sha256
|
||||
staged_snapshot_file_count = [int64]$SnapshotInspection.file_count
|
||||
staged_snapshot_byte_length = [int64]$SnapshotInspection.byte_length
|
||||
staged_snapshot_canonicalization = [string]$SnapshotInspection.canonicalization
|
||||
embedded_snapshot_manifest_sha256 = [string]$EmbeddedManifest.sha256
|
||||
embedded_snapshot_manifest_byte_length = [int64]$EmbeddedManifest.byte_length
|
||||
}
|
||||
base_image_sha256 = $BaseImageSha256
|
||||
derived_image_sha256 = ([string]$Image.Id).Substring(7)
|
||||
image = [ordered]@{
|
||||
tag = $Tag
|
||||
id = [string]$Image.Id
|
||||
size_bytes = [int64]$Image.Size
|
||||
thin_layer_bytes = [int64]$LayerInspection.layer_bytes
|
||||
maximum_thin_layer_bytes = $MaximumLayerBytes
|
||||
rootfs = [ordered]@{
|
||||
base_layer_count = [int64]$LayerInspection.base_layer_count
|
||||
derived_layer_count = [int64]$LayerInspection.derived_layer_count
|
||||
derived_layer_diff_id = [string]$LayerInspection.derived_layer_diff_id
|
||||
pinned_base_is_exact_prefix = $true
|
||||
}
|
||||
}
|
||||
runtime_contract = [ordered]@{
|
||||
workdir = $ImageWorkdir
|
||||
entrypoint = @("python3", "-m", "k1link.observatory.m49_worker_container_main")
|
||||
command = @()
|
||||
authority = "observation-only"
|
||||
models = "external"
|
||||
runtime_registries = "external-read-only"
|
||||
}
|
||||
smoke = [ordered]@{
|
||||
network = "none"
|
||||
read_only_rootfs = $true
|
||||
staged_source_bytes = "matched"
|
||||
embedded_context_bytes = "matched"
|
||||
embedded_snapshot_manifest = "matched"
|
||||
result = "passed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $StagedSnapshotRoot -PathType Container)) {
|
||||
throw "Worker 006 staged snapshot root is unavailable"
|
||||
}
|
||||
$snapshotBefore = Get-StagedSnapshotInspection $StagedSnapshotRoot
|
||||
if ([string]$snapshotBefore.sha256 -cne $ExpectedStagedSnapshotSha256) {
|
||||
throw "Worker 006 staged snapshot identity changed"
|
||||
}
|
||||
$embeddedManifest = New-StagedSnapshotManifest $snapshotBefore
|
||||
try {
|
||||
$base = Get-ImageInspection $BaseImageReference
|
||||
if ([string]$base.Id -cne $BaseImageReference) {
|
||||
throw "Worker 006 base image identity changed"
|
||||
}
|
||||
|
||||
$shortRevision = $SourceRevision.Substring(0, 12)
|
||||
$tag = "ndc/mission-core-observatory-worker-agent:$shortRevision"
|
||||
$existingIds = @(
|
||||
docker image ls `
|
||||
--quiet `
|
||||
--no-trunc `
|
||||
--filter "reference=$tag"
|
||||
)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Worker 006 image lookup failed"
|
||||
}
|
||||
if ($existingIds.Count -gt 1) {
|
||||
throw "Worker 006 image tag resolves ambiguously"
|
||||
}
|
||||
if ($existingIds.Count -eq 1) {
|
||||
$image = Get-ImageInspection $tag
|
||||
if ([string]$image.Id -cne [string]$existingIds[0]) {
|
||||
throw "Worker 006 existing image lookup changed"
|
||||
}
|
||||
$layerInspection = Assert-ImageContract `
|
||||
$image `
|
||||
$base `
|
||||
([string]$embeddedManifest.sha256)
|
||||
Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest
|
||||
New-OutputContract `
|
||||
"already-installed" `
|
||||
$tag `
|
||||
$image `
|
||||
$layerInspection `
|
||||
$snapshotBefore `
|
||||
$embeddedManifest |
|
||||
ConvertTo-Json -Compress -Depth 9
|
||||
return
|
||||
}
|
||||
|
||||
$containerName = "ndc-observatory-worker-agent-image-build-$shortRevision"
|
||||
$occupiedContainerIds = @(
|
||||
docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}"
|
||||
)
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Worker 006 temporary build container lookup failed"
|
||||
}
|
||||
if ($occupiedContainerIds.Count -ne 0) {
|
||||
throw "Worker 006 temporary build container name is already occupied"
|
||||
}
|
||||
|
||||
$copyScript = (
|
||||
"set -eu; " +
|
||||
"test ! -e /opt/nodedc/mission-core; " +
|
||||
"mkdir -p /opt/nodedc/mission-core/src /opt/nodedc/mission-core/release; " +
|
||||
"mkdir -p /run/nodedc/registries; " +
|
||||
"cp -a /nodedc-build-source/src/k1link /opt/nodedc/mission-core/src/k1link; " +
|
||||
"cp /nodedc-build-source/$ContextManifestRelativePath " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
|
||||
"cp /nodedc-build-snapshot.json " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
|
||||
"find /nodedc-build-source/src/k1link -type f -exec sh -c '" +
|
||||
"for source_path do relative=`${source_path#/nodedc-build-source/}; " +
|
||||
"cmp -s `"`$source_path`" `"/opt/nodedc/mission-core/`$relative`" || exit 70; " +
|
||||
"done' sh {} +; " +
|
||||
"cmp -s /nodedc-build-source/$ContextManifestRelativePath " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json; " +
|
||||
"cmp -s /nodedc-build-snapshot.json " +
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json; " +
|
||||
"test ! -e /opt/nodedc/mission-core/experiments; " +
|
||||
"find /opt/nodedc/mission-core -type d -exec chmod 0555 {} +; " +
|
||||
"find /opt/nodedc/mission-core -type f -exec chmod 0444 {} +; " +
|
||||
"chmod 0555 /run/nodedc /run/nodedc/registries; " +
|
||||
"cd /opt/nodedc/mission-core; " +
|
||||
"PYTHONPATH=/opt/nodedc/mission-core/src PYTHONNOUSERSITE=1 " +
|
||||
"PYTHONDONTWRITEBYTECODE=1 python3 -B -c '" +
|
||||
"import k1link.observatory.m49_worker_container_main as entrypoint; " +
|
||||
"import k1link.observatory.m49_worker_service as composition; " +
|
||||
"assert callable(entrypoint.main); " +
|
||||
"assert callable(composition.compose_installed_m49_worker_service)'"
|
||||
)
|
||||
$sourceMount = (
|
||||
"type=bind,source=$StagedSnapshotRoot," +
|
||||
"target=/nodedc-build-source,readonly"
|
||||
)
|
||||
$manifestMount = (
|
||||
"type=bind,source=$($embeddedManifest.path)," +
|
||||
"target=/nodedc-build-snapshot.json,readonly"
|
||||
)
|
||||
$containerId = $null
|
||||
$imageCommitted = $false
|
||||
$committedImageId = $null
|
||||
try {
|
||||
try {
|
||||
$createOutput = @(
|
||||
docker create `
|
||||
--name $containerName `
|
||||
--network none `
|
||||
--cap-drop ALL `
|
||||
--security-opt no-new-privileges `
|
||||
--mount $sourceMount `
|
||||
--mount $manifestMount `
|
||||
--entrypoint /bin/sh `
|
||||
$BaseImageReference `
|
||||
-c $copyScript
|
||||
)
|
||||
$createdIds = @(
|
||||
$createOutput | Where-Object { $_ -match "^[a-f0-9]{64}$" }
|
||||
)
|
||||
if ($createdIds.Count -eq 1) {
|
||||
$containerId = [string]$createdIds[0]
|
||||
}
|
||||
if ($LASTEXITCODE -ne 0 -or $null -eq $containerId) {
|
||||
throw "Worker 006 temporary build container creation failed"
|
||||
}
|
||||
docker start --attach $containerId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Worker 006 source installation command failed"
|
||||
}
|
||||
$snapshotAfter = Get-StagedSnapshotInspection $StagedSnapshotRoot
|
||||
if (
|
||||
[string]$snapshotAfter.sha256 -cne $ExpectedStagedSnapshotSha256 -or
|
||||
[int64]$snapshotAfter.file_count -ne [int64]$snapshotBefore.file_count -or
|
||||
[int64]$snapshotAfter.byte_length -ne [int64]$snapshotBefore.byte_length
|
||||
) {
|
||||
throw "Worker 006 staged snapshot changed during installation"
|
||||
}
|
||||
if ((Get-FileSha256 $embeddedManifest.path) -cne [string]$embeddedManifest.sha256) {
|
||||
throw "Worker 006 embedded snapshot manifest changed during installation"
|
||||
}
|
||||
|
||||
$changes = @(
|
||||
"--change", "WORKDIR $ImageWorkdir",
|
||||
"--change", "USER 0:0",
|
||||
"--change", "ENTRYPOINT $ImageEntrypoint",
|
||||
"--change", "CMD $ImageCommand",
|
||||
"--change", 'LABEL org.opencontainers.image.title="NODE.DC Observatory Worker 006 agent"',
|
||||
"--change", "LABEL org.opencontainers.image.source=NODEDC_MISSION_CORE",
|
||||
"--change", "LABEL org.opencontainers.image.revision=$SourceRevision",
|
||||
"--change", "LABEL com.nodedc.product=mission-core",
|
||||
"--change", "LABEL com.nodedc.stack=ndc-mission-core-observatory",
|
||||
"--change", "LABEL com.nodedc.role=observatory-worker-agent",
|
||||
"--change", "LABEL com.nodedc.managed-by=mission-core-worker-release",
|
||||
"--change", "LABEL com.nodedc.worker-contour=worker-006",
|
||||
"--change", "LABEL com.nodedc.authority=observation-only",
|
||||
"--change", "LABEL com.nodedc.models=external",
|
||||
"--change", "LABEL com.nodedc.runtime-registries=external-read-only",
|
||||
"--change", "LABEL com.nodedc.base-image.sha256=$BaseImageSha256",
|
||||
"--change", "LABEL com.nodedc.build-context.sha256=$ExpectedGitArchiveSha256",
|
||||
"--change", "LABEL com.nodedc.staged-snapshot.sha256=$ExpectedStagedSnapshotSha256",
|
||||
"--change", (
|
||||
"LABEL com.nodedc.embedded-snapshot-manifest.sha256=" +
|
||||
[string]$embeddedManifest.sha256
|
||||
),
|
||||
"--change", "LABEL com.nodedc.build-method=$BuildMethod"
|
||||
)
|
||||
foreach ($value in $FixedEnvironment) {
|
||||
$changes += @("--change", "ENV $value")
|
||||
}
|
||||
$commitOutput = docker commit --pause=true @changes $containerId $tag
|
||||
$committedIds = @(
|
||||
$commitOutput |
|
||||
ForEach-Object { ([string]$_).Trim() } |
|
||||
Where-Object { $_ -match "^sha256:[a-f0-9]{64}$" }
|
||||
)
|
||||
if ($LASTEXITCODE -ne 0 -or $committedIds.Count -ne 1) {
|
||||
throw "Worker 006 image commit failed"
|
||||
}
|
||||
$committedImageId = [string]$committedIds[0]
|
||||
$imageCommitted = $true
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $containerId -and $containerId -match "^[a-f0-9]{64}$") {
|
||||
docker rm -f $containerId | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Worker 006 temporary build container cleanup failed"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$image = Get-ImageInspection $tag
|
||||
if ([string]$image.Id -cne $committedImageId) {
|
||||
throw "Worker 006 committed image tag identity changed"
|
||||
}
|
||||
$layerInspection = Assert-ImageContract `
|
||||
$image `
|
||||
$base `
|
||||
([string]$embeddedManifest.sha256)
|
||||
Invoke-InstalledImageSmoke $tag $StagedSnapshotRoot $embeddedManifest
|
||||
New-OutputContract `
|
||||
"installed" `
|
||||
$tag `
|
||||
$image `
|
||||
$layerInspection `
|
||||
$snapshotBefore `
|
||||
$embeddedManifest |
|
||||
ConvertTo-Json -Compress -Depth 9
|
||||
}
|
||||
catch {
|
||||
$installationFailure = $_
|
||||
if ($imageCommitted -and $committedImageId -match "^sha256:[a-f0-9]{64}$") {
|
||||
docker image rm --force $committedImageId | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw (
|
||||
"Worker 006 image verification failed and committed image cleanup failed: " +
|
||||
[string]$installationFailure
|
||||
)
|
||||
}
|
||||
}
|
||||
throw $installationFailure
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if (
|
||||
$null -ne $embeddedManifest -and
|
||||
(Test-Path -LiteralPath $embeddedManifest.path -PathType Leaf)
|
||||
) {
|
||||
Remove-Item -LiteralPath $embeddedManifest.path -Force
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -149,10 +149,10 @@
|
||||
},
|
||||
{
|
||||
"asset_id": "lab-v1-portable-worker",
|
||||
"byte_length": 47008,
|
||||
"byte_length": 47392,
|
||||
"kind": "repository-file",
|
||||
"repository_path": "src/k1link/observatory/portable_lab_v1_worker.py",
|
||||
"sha256": "0a954b5d4d2f3cdd588d58ba4234141c32992b4b46380a03b621060c25c4ac23"
|
||||
"sha256": "56aa40390b8413fdf7f7293329e89420305914abd478821c08bfe8b0a2ec3df8"
|
||||
},
|
||||
{
|
||||
"asset_id": "portable-result-contracts",
|
||||
@@ -189,7 +189,7 @@
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
},
|
||||
"candidate_sha256": "546c74fb88b9571996cecc41fa915044e3907365b8891b7fceb297fb74a837c7",
|
||||
"candidate_sha256": "9d40369e393aea4b1a09a5784336442a082b1b6126afaac379f1ecece39b54f9",
|
||||
"declared_blockers": [
|
||||
"combined-executor-entrypoint-uninstalled",
|
||||
"combined-executor-image-unsealed",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+93
-27
@@ -6,17 +6,31 @@
|
||||
"build": {
|
||||
"source_revision": null,
|
||||
"source_revision_requirement": "full clean committed Git revision",
|
||||
"source_date_epoch": null,
|
||||
"build_context_sha256": null,
|
||||
"git_archive_sha256": null,
|
||||
"staged_snapshot_sha256": null,
|
||||
"installer": "experiments/perception/worker/observatory_portable/Install-Worker006AgentImage.ps1",
|
||||
"context_manifest": "experiments/perception/worker/observatory_portable/worker-006-agent-build-context.json",
|
||||
"dockerfile": "experiments/perception/worker/observatory_portable/Dockerfile.worker-006-agent",
|
||||
"dockerfile_role": "source-contract evidence included in the staged snapshot; not executed by the exact-layer installer",
|
||||
"materialization": {
|
||||
"method": "git archive",
|
||||
"include_only_context_manifest_entries": true,
|
||||
"reject_dirty_worktree": true,
|
||||
"archive_format": "tar",
|
||||
"archive_paths_source": "context manifest context_entries in declared order",
|
||||
"build_context_sha256_subject": "exact git-archive tar bytes"
|
||||
"git_archive_sha256_subject": "exact git-archive tar bytes",
|
||||
"extraction_target": "D:\\NDC_MISSIONCORE\\runtime\\staging\\observatory-worker-agent-<source-revision>"
|
||||
},
|
||||
"staged_snapshot": {
|
||||
"sha256_subject": "sorted regular-file rows for src/k1link, Dockerfile.worker-006-agent and worker-006-agent-build-context.json",
|
||||
"canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1",
|
||||
"reject_reparse_points": true,
|
||||
"reject_unexpected_entries": true,
|
||||
"verify_timing": [
|
||||
"before temporary container creation",
|
||||
"after source installation and before image commit"
|
||||
],
|
||||
"embedded_manifest": "/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
|
||||
},
|
||||
"base_image": {
|
||||
"reference": "nvcr.io/nvidia/tritonserver:26.06-py3@sha256:58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
@@ -24,17 +38,20 @@
|
||||
"must_exist_locally": true,
|
||||
"pull_allowed": false
|
||||
},
|
||||
"docker_build": {
|
||||
"image_materialization": {
|
||||
"method": "docker-commit-exact-layer-v1",
|
||||
"dockerfile_executed": false,
|
||||
"network": "none",
|
||||
"pull": false,
|
||||
"no_cache": true,
|
||||
"provenance": false,
|
||||
"platform": "linux/amd64",
|
||||
"context_input": "exact git-archive tar bytes",
|
||||
"required_build_args": [
|
||||
"NODEDC_SOURCE_REVISION",
|
||||
"NODEDC_BUILD_CONTEXT_SHA256",
|
||||
"SOURCE_DATE_EPOCH"
|
||||
"source_input": "exact read-only staged snapshot",
|
||||
"base_container": "docker create by exact pinned base image ID",
|
||||
"commit": "docker commit --pause=true with fixed config changes",
|
||||
"maximum_thin_layer_bytes": 33554432,
|
||||
"embedded_payload": [
|
||||
"/opt/nodedc/mission-core/src/k1link",
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-build-context.json",
|
||||
"/opt/nodedc/mission-core/release/worker-006-agent-staged-snapshot.json"
|
||||
]
|
||||
},
|
||||
"required_preflight": [
|
||||
@@ -42,7 +59,9 @@
|
||||
"selected revision resolves to exactly one commit",
|
||||
"selected revision equals HEAD",
|
||||
"base image inspect ID equals the pinned SHA-256",
|
||||
"context archive contains exactly the context manifest entries"
|
||||
"context archive contains exactly the context manifest entries",
|
||||
"git archive SHA-256 was verified externally before extraction",
|
||||
"staged snapshot SHA-256 matches the exact canonical regular-file inventory"
|
||||
],
|
||||
"image_tag_template": "ndc/mission-core-observatory-worker-agent:<source-revision-12>"
|
||||
},
|
||||
@@ -57,7 +76,21 @@
|
||||
"com.nodedc.models": "external",
|
||||
"com.nodedc.runtime-registries": "external-read-only",
|
||||
"com.nodedc.base-image.sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
"com.nodedc.build-context.sha256": "<build-context-sha256>"
|
||||
"com.nodedc.build-context.sha256": "<git-archive-sha256>",
|
||||
"com.nodedc.staged-snapshot.sha256": "<staged-snapshot-sha256>",
|
||||
"com.nodedc.embedded-snapshot-manifest.sha256": "<embedded-snapshot-manifest-sha256>",
|
||||
"com.nodedc.build-method": "docker-commit-exact-layer-v1"
|
||||
},
|
||||
"rootfs": {
|
||||
"base_layer_chain": "exact prefix of the pinned base image RootFS.Layers",
|
||||
"derived_layer_count": 1,
|
||||
"derived_layer_diff_id_required": true,
|
||||
"maximum_thin_layer_bytes": 33554432
|
||||
},
|
||||
"embedded_identity": {
|
||||
"source_tree": "byte-for-byte and bidirectional path match against staged src/k1link",
|
||||
"context_manifest": "byte-for-byte match against the staged context manifest",
|
||||
"snapshot_manifest": "byte-for-byte match against the generated manifest covering every staged file, including the Dockerfile"
|
||||
},
|
||||
"smoke": {
|
||||
"network": "none",
|
||||
@@ -68,6 +101,9 @@
|
||||
"-c",
|
||||
"import k1link.observatory.m49_worker_container_main as e; import k1link.observatory.m49_worker_service as c; assert callable(e.main); assert callable(c.compose_installed_m49_worker_service)"
|
||||
],
|
||||
"staged_source_bytes": "matched",
|
||||
"embedded_context_bytes": "matched",
|
||||
"embedded_snapshot_manifest": "matched",
|
||||
"expected_result": "exit-0"
|
||||
}
|
||||
},
|
||||
@@ -80,7 +116,8 @@
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE": "/run/nodedc/registries/observatory-portable-run-definitions.json",
|
||||
"MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE": "/run/nodedc/registries/observatory-worker-runtime-candidates.json",
|
||||
"MISSIONCORE_OBSERVATORY_M49_INSTALLATION_RECEIPT_FILE": "/release/worker-installation-receipt.json",
|
||||
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE": "/release/lab-v1-worker-installation-receipt.json"
|
||||
"MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE": "/release/lab-v1-worker-installation-receipt.json",
|
||||
"MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE": "/release/lab-v1-executor-release.json"
|
||||
},
|
||||
"runtime_registry_files": {
|
||||
"binding": "individual read-only bind files",
|
||||
@@ -238,6 +275,9 @@
|
||||
"state": "external-installed-receipt-required",
|
||||
"environment_variable": "MISSIONCORE_OBSERVATORY_LAB_V1_INSTALLATION_RECEIPT_FILE",
|
||||
"container_path": "/release/lab-v1-worker-installation-receipt.json",
|
||||
"release_candidate_environment_variable": "MISSIONCORE_OBSERVATORY_LAB_V1_RELEASE_CANDIDATE_FILE",
|
||||
"release_candidate_container_path": "/release/lab-v1-executor-release.json",
|
||||
"release_repository_root": "/release",
|
||||
"mode": "read-only",
|
||||
"owns_component_image_identities": true,
|
||||
"queued_jobs_may_override_component_images": false
|
||||
@@ -249,29 +289,55 @@
|
||||
]
|
||||
},
|
||||
"receipt_skeleton": {
|
||||
"schema_version": "missioncore.observatory-worker-agent-image-receipt/v1",
|
||||
"receipt_state": "not-built",
|
||||
"schema_version": "missioncore.observatory-worker-agent-image-installation/v1",
|
||||
"status": null,
|
||||
"worker_id": "worker-006",
|
||||
"build_method": "docker-commit-exact-layer-v1",
|
||||
"source_revision": null,
|
||||
"source_date_epoch": null,
|
||||
"build_context_sha256": null,
|
||||
"provenance": {
|
||||
"git_archive_sha256": null,
|
||||
"git_archive_verification": "external-before-extract",
|
||||
"staged_snapshot_sha256": null,
|
||||
"staged_snapshot_file_count": null,
|
||||
"staged_snapshot_byte_length": null,
|
||||
"staged_snapshot_canonicalization": "utf8-path-nul-length-nul-sha256-lf-v1",
|
||||
"embedded_snapshot_manifest_sha256": null,
|
||||
"embedded_snapshot_manifest_byte_length": null
|
||||
},
|
||||
"base_image_sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
"derived_image_sha256": null,
|
||||
"image": {
|
||||
"tag": null,
|
||||
"id": null,
|
||||
"size_bytes": null
|
||||
"size_bytes": null,
|
||||
"thin_layer_bytes": null,
|
||||
"maximum_thin_layer_bytes": 33554432,
|
||||
"rootfs": {
|
||||
"base_layer_count": null,
|
||||
"derived_layer_count": 1,
|
||||
"derived_layer_diff_id": null,
|
||||
"pinned_base_is_exact_prefix": true
|
||||
}
|
||||
},
|
||||
"runtime_contract": {
|
||||
"workdir": "/opt/nodedc/mission-core",
|
||||
"entrypoint": [
|
||||
"python3",
|
||||
"-m",
|
||||
"k1link.observatory.m49_worker_container_main"
|
||||
],
|
||||
"command": [],
|
||||
"authority": "observation-only",
|
||||
"models": "external",
|
||||
"runtime_registries": "external-read-only"
|
||||
},
|
||||
"smoke": {
|
||||
"network": "none",
|
||||
"read_only_rootfs": true,
|
||||
"staged_source_bytes": null,
|
||||
"embedded_context_bytes": null,
|
||||
"embedded_snapshot_manifest": null,
|
||||
"result": "not-run"
|
||||
},
|
||||
"models_baked_into_image": false,
|
||||
"runtime_registries_baked_into_image": false,
|
||||
"authority": {
|
||||
"commands_enabled": false,
|
||||
"actuation_allowed": false,
|
||||
"navigation_or_safety_accepted": false,
|
||||
"production_accepted": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user