feat(lab): install thin adapter layers offline
This commit is contained in:
DCCONSTRUCTIONS
+253
@@ -0,0 +1,253 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidatePattern("^[a-f0-9]{40}$")]
|
||||
[string]$SourceRevision
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
$ProgressPreference = "SilentlyContinue"
|
||||
|
||||
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
|
||||
$BuildRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-build-$SourceRevision"
|
||||
$SharedAdapterSha256 = "fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b"
|
||||
$BuildMethod = "docker-commit-exact-layer-v1"
|
||||
$MaximumLayerBytes = [int64](16MB)
|
||||
|
||||
$Components = [ordered]@{
|
||||
"eomt" = [pscustomobject]@{
|
||||
base_image_sha256 = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||
adapter_name = "run_portable_lab_v1_eomt_component.py"
|
||||
adapter_sha256 = "b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5"
|
||||
python_check = "python3 -B -m py_compile"
|
||||
entrypoint = '["python3"]'
|
||||
command = '["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]'
|
||||
environment = @(
|
||||
"HF_HUB_OFFLINE=1",
|
||||
"TRANSFORMERS_OFFLINE=1",
|
||||
"PYTHONNOUSERSITE=1",
|
||||
"PYTHONDONTWRITEBYTECODE=1",
|
||||
"PYTHONUNBUFFERED=1"
|
||||
)
|
||||
}
|
||||
"ddrnet" = [pscustomobject]@{
|
||||
base_image_sha256 = "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||
adapter_name = "run_portable_lab_v1_ddrnet_component.py"
|
||||
adapter_sha256 = "d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c"
|
||||
python_check = "conda run --no-capture-output --name goose python -B -m py_compile"
|
||||
entrypoint = '["conda","run","--no-capture-output","--name","goose","python"]'
|
||||
command = '["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]'
|
||||
environment = @(
|
||||
"PYTHONDONTWRITEBYTECODE=1",
|
||||
"PYTHONUNBUFFERED=1"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function Get-FileSha256 {
|
||||
param([string]$Path)
|
||||
|
||||
$item = Get-Item -LiteralPath $Path -Force
|
||||
if (
|
||||
-not ($item -is [IO.FileInfo]) -or
|
||||
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||
) {
|
||||
throw "LAB V1 adapter source is not a regular file"
|
||||
}
|
||||
return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Get-ImageInspection {
|
||||
param([string]$Reference)
|
||||
|
||||
$payload = docker image inspect $Reference
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 component image is unavailable: $Reference"
|
||||
}
|
||||
$rows = @($payload | ConvertFrom-Json)
|
||||
if ($rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") {
|
||||
throw "LAB V1 component image inspection is invalid"
|
||||
}
|
||||
return $rows[0]
|
||||
}
|
||||
|
||||
function Assert-ComponentSource {
|
||||
param(
|
||||
[string]$ComponentRoot,
|
||||
[object]$Descriptor
|
||||
)
|
||||
|
||||
$shared = Join-Path $ComponentRoot "portable_lab_v1_component_adapter.py"
|
||||
$component = Join-Path $ComponentRoot $Descriptor.adapter_name
|
||||
if ((Get-FileSha256 $shared) -cne $SharedAdapterSha256) {
|
||||
throw "Shared LAB V1 component adapter identity changed"
|
||||
}
|
||||
if ((Get-FileSha256 $component) -cne [string]$Descriptor.adapter_sha256) {
|
||||
throw "LAB V1 component adapter identity changed"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-InstalledImageSmoke {
|
||||
param(
|
||||
[string]$Tag,
|
||||
[object]$Descriptor
|
||||
)
|
||||
|
||||
$script = (
|
||||
"set -eu; " +
|
||||
"test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||
"| cut -d' ' -f1) = $SharedAdapterSha256; " +
|
||||
"test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " +
|
||||
"| cut -d' ' -f1) = $($Descriptor.adapter_sha256)"
|
||||
)
|
||||
docker run `
|
||||
--rm `
|
||||
--network none `
|
||||
--cap-drop ALL `
|
||||
--security-opt no-new-privileges `
|
||||
--entrypoint /bin/sh `
|
||||
$Tag `
|
||||
-c $script
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Installed LAB V1 component image smoke failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Install-ComponentImage {
|
||||
param(
|
||||
[string]$Component,
|
||||
[object]$Descriptor
|
||||
)
|
||||
|
||||
$componentRoot = Join-Path $BuildRoot $Component
|
||||
if (-not (Test-Path -LiteralPath $componentRoot -PathType Container)) {
|
||||
throw "LAB V1 component build source is unavailable"
|
||||
}
|
||||
Assert-ComponentSource $componentRoot $Descriptor
|
||||
$baseReference = "sha256:$($Descriptor.base_image_sha256)"
|
||||
$base = Get-ImageInspection $baseReference
|
||||
if ([string]$base.Id -cne $baseReference) {
|
||||
throw "LAB V1 component base image identity changed"
|
||||
}
|
||||
$shortRevision = $SourceRevision.Substring(0, 12)
|
||||
$tag = "ndc/mission-core-lab-v1-$Component-adapter:$shortRevision"
|
||||
$existing = docker image inspect $tag 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
$image = @($existing | ConvertFrom-Json)[0]
|
||||
$labels = $image.Config.Labels
|
||||
if (
|
||||
[string]$labels."org.opencontainers.image.revision" -cne $SourceRevision -or
|
||||
[string]$labels."com.nodedc.base-image.sha256" -cne
|
||||
[string]$Descriptor.base_image_sha256 -or
|
||||
[string]$labels."com.nodedc.shared-adapter.sha256" -cne
|
||||
$SharedAdapterSha256 -or
|
||||
[string]$labels."com.nodedc.component-adapter.sha256" -cne
|
||||
[string]$Descriptor.adapter_sha256 -or
|
||||
[string]$labels."com.nodedc.build-method" -cne $BuildMethod
|
||||
) {
|
||||
throw "Existing LAB V1 component image label identity changed"
|
||||
}
|
||||
Invoke-InstalledImageSmoke $tag $Descriptor
|
||||
return [ordered]@{
|
||||
component = $Component
|
||||
status = "already-installed"
|
||||
tag = $tag
|
||||
base_image_sha256 = [string]$Descriptor.base_image_sha256
|
||||
derived_image_sha256 = ([string]$image.Id).Substring(7)
|
||||
build_method = $BuildMethod
|
||||
}
|
||||
}
|
||||
|
||||
$containerName = "ndc-lab-v1-$Component-image-build-$shortRevision"
|
||||
$existingContainer = docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}"
|
||||
if ($existingContainer) {
|
||||
throw "LAB V1 temporary build container name is already occupied"
|
||||
}
|
||||
$copyScript = (
|
||||
"set -eu; " +
|
||||
"mkdir -p /opt/nodedc/adapter; " +
|
||||
"cp /nodedc-build-source/portable_lab_v1_component_adapter.py " +
|
||||
"/opt/nodedc/adapter/portable_lab_v1_component_adapter.py; " +
|
||||
"cp /nodedc-build-source/$($Descriptor.adapter_name) " +
|
||||
"/opt/nodedc/adapter/$($Descriptor.adapter_name); " +
|
||||
"test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||
"| cut -d' ' -f1) = $SharedAdapterSha256; " +
|
||||
"test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " +
|
||||
"| cut -d' ' -f1) = $($Descriptor.adapter_sha256); " +
|
||||
"chmod 0444 /opt/nodedc/adapter/*.py; " +
|
||||
"$($Descriptor.python_check) " +
|
||||
"/opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||
"/opt/nodedc/adapter/$($Descriptor.adapter_name); " +
|
||||
"rm -rf /opt/nodedc/adapter/__pycache__"
|
||||
)
|
||||
$mount = "type=bind,source=$componentRoot,target=/nodedc-build-source,readonly"
|
||||
$containerId = docker create `
|
||||
--name $containerName `
|
||||
--network none `
|
||||
--cap-drop ALL `
|
||||
--security-opt no-new-privileges `
|
||||
--mount $mount `
|
||||
--entrypoint /bin/sh `
|
||||
$baseReference `
|
||||
-c $copyScript
|
||||
if ($LASTEXITCODE -ne 0 -or $containerId -notmatch "^[a-f0-9]{64}$") {
|
||||
throw "LAB V1 temporary component build container creation failed"
|
||||
}
|
||||
try {
|
||||
docker start --attach $containerId
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 component adapter installation command failed"
|
||||
}
|
||||
$changes = @(
|
||||
"--change", "ENTRYPOINT $($Descriptor.entrypoint)",
|
||||
"--change", "CMD $($Descriptor.command)",
|
||||
"--change", "LABEL org.opencontainers.image.revision=$SourceRevision",
|
||||
"--change", "LABEL com.nodedc.product=mission-core",
|
||||
"--change", "LABEL com.nodedc.stack=observatory",
|
||||
"--change", "LABEL com.nodedc.component=lab-v1-$Component",
|
||||
"--change", "LABEL com.nodedc.authority=observation-only",
|
||||
"--change", "LABEL com.nodedc.base-image.sha256=$($Descriptor.base_image_sha256)",
|
||||
"--change", "LABEL com.nodedc.shared-adapter.sha256=$SharedAdapterSha256",
|
||||
"--change", "LABEL com.nodedc.component-adapter.sha256=$($Descriptor.adapter_sha256)",
|
||||
"--change", "LABEL com.nodedc.build-method=$BuildMethod"
|
||||
)
|
||||
foreach ($value in $Descriptor.environment) {
|
||||
$changes += @("--change", "ENV $value")
|
||||
}
|
||||
docker commit @changes $containerId $tag | Out-Null
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "LAB V1 component image commit failed"
|
||||
}
|
||||
}
|
||||
finally {
|
||||
docker rm -f $containerId | Out-Null
|
||||
}
|
||||
|
||||
$image = Get-ImageInspection $tag
|
||||
if (
|
||||
[int64]$image.Size -lt [int64]$base.Size -or
|
||||
([int64]$image.Size - [int64]$base.Size) -gt $MaximumLayerBytes
|
||||
) {
|
||||
throw "LAB V1 adapter image is not a thin layer"
|
||||
}
|
||||
Invoke-InstalledImageSmoke $tag $Descriptor
|
||||
return [ordered]@{
|
||||
component = $Component
|
||||
status = "installed"
|
||||
tag = $tag
|
||||
base_image_sha256 = [string]$Descriptor.base_image_sha256
|
||||
derived_image_sha256 = ([string]$image.Id).Substring(7)
|
||||
build_method = $BuildMethod
|
||||
}
|
||||
}
|
||||
|
||||
if (-not (Test-Path -LiteralPath $BuildRoot -PathType Container)) {
|
||||
throw "LAB V1 component build root is unavailable"
|
||||
}
|
||||
|
||||
$outputs = New-Object "Collections.Generic.List[object]"
|
||||
foreach ($component in $Components.Keys) {
|
||||
$outputs.Add((Install-ComponentImage $component $Components[$component]))
|
||||
}
|
||||
$outputs.ToArray() | ConvertTo-Json -Compress -Depth 5
|
||||
@@ -0,0 +1,50 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||
INSTALLER = (
|
||||
REPOSITORY_ROOT
|
||||
/ "experiments/perception/worker/observatory_portable"
|
||||
/ "Install-LabV1ComponentAdapterImages.ps1"
|
||||
)
|
||||
|
||||
|
||||
def test_component_image_installer_is_exact_local_and_offline() -> None:
|
||||
script = INSTALLER.read_text(encoding="utf-8")
|
||||
|
||||
assert '"docker-commit-exact-layer-v1"' in script
|
||||
assert "--network none" in script
|
||||
assert "--cap-drop ALL" in script
|
||||
assert "--security-opt no-new-privileges" in script
|
||||
assert "target=/nodedc-build-source,readonly" in script
|
||||
assert "Get-FileSha256" in script
|
||||
assert "Invoke-InstalledImageSmoke" in script
|
||||
assert "docker commit @changes $containerId $tag" in script
|
||||
assert "docker rm -f $containerId" in script
|
||||
assert "com.nodedc.build-method" in script
|
||||
|
||||
lowered = script.lower()
|
||||
assert "docker pull" not in lowered
|
||||
assert "invoke-webrequest" not in lowered
|
||||
assert "start-bitstransfer" not in lowered
|
||||
assert "curl " not in lowered
|
||||
assert "wget " not in lowered
|
||||
assert "smb" not in lowered
|
||||
|
||||
|
||||
def test_component_image_installer_binds_bases_and_adapter_sources() -> None:
|
||||
script = INSTALLER.read_text(encoding="utf-8")
|
||||
|
||||
for digest in (
|
||||
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd",
|
||||
"fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b",
|
||||
"b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5",
|
||||
"d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c",
|
||||
):
|
||||
assert digest in script
|
||||
assert 'entrypoint = \'["python3"]\'' in script
|
||||
assert '"goose","python"' in script
|
||||
assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]\'' in script
|
||||
assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]\'' in script
|
||||
Reference in New Issue
Block a user