From b2ae29b8d8d40a40fd9f0469d60115840fd02601 Mon Sep 17 00:00:00 2001 From: DCCONSTRUCTIONS Date: Mon, 31 Aug 2026 19:51:51 +0300 Subject: [PATCH] feat(lab): install thin adapter layers offline --- .../Install-LabV1ComponentAdapterImages.ps1 | 253 ++++++++++++++++++ ...ab_v1_component_adapter_image_installer.py | 50 ++++ 2 files changed, 303 insertions(+) create mode 100644 experiments/perception/worker/observatory_portable/Install-LabV1ComponentAdapterImages.ps1 create mode 100644 tests/test_lab_v1_component_adapter_image_installer.py diff --git a/experiments/perception/worker/observatory_portable/Install-LabV1ComponentAdapterImages.ps1 b/experiments/perception/worker/observatory_portable/Install-LabV1ComponentAdapterImages.ps1 new file mode 100644 index 0000000..56f928d --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Install-LabV1ComponentAdapterImages.ps1 @@ -0,0 +1,253 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{40}$")] + [string]$SourceRevision +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\") +$BuildRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-build-$SourceRevision" +$SharedAdapterSha256 = "fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b" +$BuildMethod = "docker-commit-exact-layer-v1" +$MaximumLayerBytes = [int64](16MB) + +$Components = [ordered]@{ + "eomt" = [pscustomobject]@{ + base_image_sha256 = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" + adapter_name = "run_portable_lab_v1_eomt_component.py" + adapter_sha256 = "b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5" + python_check = "python3 -B -m py_compile" + entrypoint = '["python3"]' + command = '["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]' + environment = @( + "HF_HUB_OFFLINE=1", + "TRANSFORMERS_OFFLINE=1", + "PYTHONNOUSERSITE=1", + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONUNBUFFERED=1" + ) + } + "ddrnet" = [pscustomobject]@{ + base_image_sha256 = "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd" + adapter_name = "run_portable_lab_v1_ddrnet_component.py" + adapter_sha256 = "d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c" + python_check = "conda run --no-capture-output --name goose python -B -m py_compile" + entrypoint = '["conda","run","--no-capture-output","--name","goose","python"]' + command = '["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]' + environment = @( + "PYTHONDONTWRITEBYTECODE=1", + "PYTHONUNBUFFERED=1" + ) + } +} + +function Get-FileSha256 { + param([string]$Path) + + $item = Get-Item -LiteralPath $Path -Force + if ( + -not ($item -is [IO.FileInfo]) -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) + ) { + throw "LAB V1 adapter source is not a regular file" + } + return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Get-ImageInspection { + param([string]$Reference) + + $payload = docker image inspect $Reference + if ($LASTEXITCODE -ne 0) { + throw "LAB V1 component image is unavailable: $Reference" + } + $rows = @($payload | ConvertFrom-Json) + if ($rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") { + throw "LAB V1 component image inspection is invalid" + } + return $rows[0] +} + +function Assert-ComponentSource { + param( + [string]$ComponentRoot, + [object]$Descriptor + ) + + $shared = Join-Path $ComponentRoot "portable_lab_v1_component_adapter.py" + $component = Join-Path $ComponentRoot $Descriptor.adapter_name + if ((Get-FileSha256 $shared) -cne $SharedAdapterSha256) { + throw "Shared LAB V1 component adapter identity changed" + } + if ((Get-FileSha256 $component) -cne [string]$Descriptor.adapter_sha256) { + throw "LAB V1 component adapter identity changed" + } +} + +function Invoke-InstalledImageSmoke { + param( + [string]$Tag, + [object]$Descriptor + ) + + $script = ( + "set -eu; " + + "test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " + + "| cut -d' ' -f1) = $SharedAdapterSha256; " + + "test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " + + "| cut -d' ' -f1) = $($Descriptor.adapter_sha256)" + ) + docker run ` + --rm ` + --network none ` + --cap-drop ALL ` + --security-opt no-new-privileges ` + --entrypoint /bin/sh ` + $Tag ` + -c $script + if ($LASTEXITCODE -ne 0) { + throw "Installed LAB V1 component image smoke failed" + } +} + +function Install-ComponentImage { + param( + [string]$Component, + [object]$Descriptor + ) + + $componentRoot = Join-Path $BuildRoot $Component + if (-not (Test-Path -LiteralPath $componentRoot -PathType Container)) { + throw "LAB V1 component build source is unavailable" + } + Assert-ComponentSource $componentRoot $Descriptor + $baseReference = "sha256:$($Descriptor.base_image_sha256)" + $base = Get-ImageInspection $baseReference + if ([string]$base.Id -cne $baseReference) { + throw "LAB V1 component base image identity changed" + } + $shortRevision = $SourceRevision.Substring(0, 12) + $tag = "ndc/mission-core-lab-v1-$Component-adapter:$shortRevision" + $existing = docker image inspect $tag 2>$null + if ($LASTEXITCODE -eq 0) { + $image = @($existing | ConvertFrom-Json)[0] + $labels = $image.Config.Labels + if ( + [string]$labels."org.opencontainers.image.revision" -cne $SourceRevision -or + [string]$labels."com.nodedc.base-image.sha256" -cne + [string]$Descriptor.base_image_sha256 -or + [string]$labels."com.nodedc.shared-adapter.sha256" -cne + $SharedAdapterSha256 -or + [string]$labels."com.nodedc.component-adapter.sha256" -cne + [string]$Descriptor.adapter_sha256 -or + [string]$labels."com.nodedc.build-method" -cne $BuildMethod + ) { + throw "Existing LAB V1 component image label identity changed" + } + Invoke-InstalledImageSmoke $tag $Descriptor + return [ordered]@{ + component = $Component + status = "already-installed" + tag = $tag + base_image_sha256 = [string]$Descriptor.base_image_sha256 + derived_image_sha256 = ([string]$image.Id).Substring(7) + build_method = $BuildMethod + } + } + + $containerName = "ndc-lab-v1-$Component-image-build-$shortRevision" + $existingContainer = docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}" + if ($existingContainer) { + throw "LAB V1 temporary build container name is already occupied" + } + $copyScript = ( + "set -eu; " + + "mkdir -p /opt/nodedc/adapter; " + + "cp /nodedc-build-source/portable_lab_v1_component_adapter.py " + + "/opt/nodedc/adapter/portable_lab_v1_component_adapter.py; " + + "cp /nodedc-build-source/$($Descriptor.adapter_name) " + + "/opt/nodedc/adapter/$($Descriptor.adapter_name); " + + "test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " + + "| cut -d' ' -f1) = $SharedAdapterSha256; " + + "test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " + + "| cut -d' ' -f1) = $($Descriptor.adapter_sha256); " + + "chmod 0444 /opt/nodedc/adapter/*.py; " + + "$($Descriptor.python_check) " + + "/opt/nodedc/adapter/portable_lab_v1_component_adapter.py " + + "/opt/nodedc/adapter/$($Descriptor.adapter_name); " + + "rm -rf /opt/nodedc/adapter/__pycache__" + ) + $mount = "type=bind,source=$componentRoot,target=/nodedc-build-source,readonly" + $containerId = docker create ` + --name $containerName ` + --network none ` + --cap-drop ALL ` + --security-opt no-new-privileges ` + --mount $mount ` + --entrypoint /bin/sh ` + $baseReference ` + -c $copyScript + if ($LASTEXITCODE -ne 0 -or $containerId -notmatch "^[a-f0-9]{64}$") { + throw "LAB V1 temporary component build container creation failed" + } + try { + docker start --attach $containerId + if ($LASTEXITCODE -ne 0) { + throw "LAB V1 component adapter installation command failed" + } + $changes = @( + "--change", "ENTRYPOINT $($Descriptor.entrypoint)", + "--change", "CMD $($Descriptor.command)", + "--change", "LABEL org.opencontainers.image.revision=$SourceRevision", + "--change", "LABEL com.nodedc.product=mission-core", + "--change", "LABEL com.nodedc.stack=observatory", + "--change", "LABEL com.nodedc.component=lab-v1-$Component", + "--change", "LABEL com.nodedc.authority=observation-only", + "--change", "LABEL com.nodedc.base-image.sha256=$($Descriptor.base_image_sha256)", + "--change", "LABEL com.nodedc.shared-adapter.sha256=$SharedAdapterSha256", + "--change", "LABEL com.nodedc.component-adapter.sha256=$($Descriptor.adapter_sha256)", + "--change", "LABEL com.nodedc.build-method=$BuildMethod" + ) + foreach ($value in $Descriptor.environment) { + $changes += @("--change", "ENV $value") + } + docker commit @changes $containerId $tag | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "LAB V1 component image commit failed" + } + } + finally { + docker rm -f $containerId | Out-Null + } + + $image = Get-ImageInspection $tag + if ( + [int64]$image.Size -lt [int64]$base.Size -or + ([int64]$image.Size - [int64]$base.Size) -gt $MaximumLayerBytes + ) { + throw "LAB V1 adapter image is not a thin layer" + } + Invoke-InstalledImageSmoke $tag $Descriptor + return [ordered]@{ + component = $Component + status = "installed" + tag = $tag + base_image_sha256 = [string]$Descriptor.base_image_sha256 + derived_image_sha256 = ([string]$image.Id).Substring(7) + build_method = $BuildMethod + } +} + +if (-not (Test-Path -LiteralPath $BuildRoot -PathType Container)) { + throw "LAB V1 component build root is unavailable" +} + +$outputs = New-Object "Collections.Generic.List[object]" +foreach ($component in $Components.Keys) { + $outputs.Add((Install-ComponentImage $component $Components[$component])) +} +$outputs.ToArray() | ConvertTo-Json -Compress -Depth 5 diff --git a/tests/test_lab_v1_component_adapter_image_installer.py b/tests/test_lab_v1_component_adapter_image_installer.py new file mode 100644 index 0000000..6e67d0d --- /dev/null +++ b/tests/test_lab_v1_component_adapter_image_installer.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INSTALLER = ( + REPOSITORY_ROOT + / "experiments/perception/worker/observatory_portable" + / "Install-LabV1ComponentAdapterImages.ps1" +) + + +def test_component_image_installer_is_exact_local_and_offline() -> None: + script = INSTALLER.read_text(encoding="utf-8") + + assert '"docker-commit-exact-layer-v1"' in script + assert "--network none" in script + assert "--cap-drop ALL" in script + assert "--security-opt no-new-privileges" in script + assert "target=/nodedc-build-source,readonly" in script + assert "Get-FileSha256" in script + assert "Invoke-InstalledImageSmoke" in script + assert "docker commit @changes $containerId $tag" in script + assert "docker rm -f $containerId" in script + assert "com.nodedc.build-method" in script + + lowered = script.lower() + assert "docker pull" not in lowered + assert "invoke-webrequest" not in lowered + assert "start-bitstransfer" not in lowered + assert "curl " not in lowered + assert "wget " not in lowered + assert "smb" not in lowered + + +def test_component_image_installer_binds_bases_and_adapter_sources() -> None: + script = INSTALLER.read_text(encoding="utf-8") + + for digest in ( + "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794", + "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd", + "fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b", + "b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5", + "d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c", + ): + assert digest in script + assert 'entrypoint = \'["python3"]\'' in script + assert '"goose","python"' in script + assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]\'' in script + assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]\'' in script