feat(lab): install thin adapter layers offline
This commit is contained in:
DCCONSTRUCTIONS
+253
@@ -0,0 +1,253 @@
|
|||||||
|
[CmdletBinding()]
|
||||||
|
param(
|
||||||
|
[Parameter(Mandatory = $true)]
|
||||||
|
[ValidatePattern("^[a-f0-9]{40}$")]
|
||||||
|
[string]$SourceRevision
|
||||||
|
)
|
||||||
|
|
||||||
|
Set-StrictMode -Version Latest
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
$ProgressPreference = "SilentlyContinue"
|
||||||
|
|
||||||
|
$RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\")
|
||||||
|
$BuildRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-build-$SourceRevision"
|
||||||
|
$SharedAdapterSha256 = "fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b"
|
||||||
|
$BuildMethod = "docker-commit-exact-layer-v1"
|
||||||
|
$MaximumLayerBytes = [int64](16MB)
|
||||||
|
|
||||||
|
$Components = [ordered]@{
|
||||||
|
"eomt" = [pscustomobject]@{
|
||||||
|
base_image_sha256 = "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794"
|
||||||
|
adapter_name = "run_portable_lab_v1_eomt_component.py"
|
||||||
|
adapter_sha256 = "b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5"
|
||||||
|
python_check = "python3 -B -m py_compile"
|
||||||
|
entrypoint = '["python3"]'
|
||||||
|
command = '["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]'
|
||||||
|
environment = @(
|
||||||
|
"HF_HUB_OFFLINE=1",
|
||||||
|
"TRANSFORMERS_OFFLINE=1",
|
||||||
|
"PYTHONNOUSERSITE=1",
|
||||||
|
"PYTHONDONTWRITEBYTECODE=1",
|
||||||
|
"PYTHONUNBUFFERED=1"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
"ddrnet" = [pscustomobject]@{
|
||||||
|
base_image_sha256 = "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd"
|
||||||
|
adapter_name = "run_portable_lab_v1_ddrnet_component.py"
|
||||||
|
adapter_sha256 = "d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c"
|
||||||
|
python_check = "conda run --no-capture-output --name goose python -B -m py_compile"
|
||||||
|
entrypoint = '["conda","run","--no-capture-output","--name","goose","python"]'
|
||||||
|
command = '["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]'
|
||||||
|
environment = @(
|
||||||
|
"PYTHONDONTWRITEBYTECODE=1",
|
||||||
|
"PYTHONUNBUFFERED=1"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-FileSha256 {
|
||||||
|
param([string]$Path)
|
||||||
|
|
||||||
|
$item = Get-Item -LiteralPath $Path -Force
|
||||||
|
if (
|
||||||
|
-not ($item -is [IO.FileInfo]) -or
|
||||||
|
($item.Attributes -band [IO.FileAttributes]::ReparsePoint)
|
||||||
|
) {
|
||||||
|
throw "LAB V1 adapter source is not a regular file"
|
||||||
|
}
|
||||||
|
return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-ImageInspection {
|
||||||
|
param([string]$Reference)
|
||||||
|
|
||||||
|
$payload = docker image inspect $Reference
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "LAB V1 component image is unavailable: $Reference"
|
||||||
|
}
|
||||||
|
$rows = @($payload | ConvertFrom-Json)
|
||||||
|
if ($rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") {
|
||||||
|
throw "LAB V1 component image inspection is invalid"
|
||||||
|
}
|
||||||
|
return $rows[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
function Assert-ComponentSource {
|
||||||
|
param(
|
||||||
|
[string]$ComponentRoot,
|
||||||
|
[object]$Descriptor
|
||||||
|
)
|
||||||
|
|
||||||
|
$shared = Join-Path $ComponentRoot "portable_lab_v1_component_adapter.py"
|
||||||
|
$component = Join-Path $ComponentRoot $Descriptor.adapter_name
|
||||||
|
if ((Get-FileSha256 $shared) -cne $SharedAdapterSha256) {
|
||||||
|
throw "Shared LAB V1 component adapter identity changed"
|
||||||
|
}
|
||||||
|
if ((Get-FileSha256 $component) -cne [string]$Descriptor.adapter_sha256) {
|
||||||
|
throw "LAB V1 component adapter identity changed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-InstalledImageSmoke {
|
||||||
|
param(
|
||||||
|
[string]$Tag,
|
||||||
|
[object]$Descriptor
|
||||||
|
)
|
||||||
|
|
||||||
|
$script = (
|
||||||
|
"set -eu; " +
|
||||||
|
"test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||||
|
"| cut -d' ' -f1) = $SharedAdapterSha256; " +
|
||||||
|
"test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " +
|
||||||
|
"| cut -d' ' -f1) = $($Descriptor.adapter_sha256)"
|
||||||
|
)
|
||||||
|
docker run `
|
||||||
|
--rm `
|
||||||
|
--network none `
|
||||||
|
--cap-drop ALL `
|
||||||
|
--security-opt no-new-privileges `
|
||||||
|
--entrypoint /bin/sh `
|
||||||
|
$Tag `
|
||||||
|
-c $script
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "Installed LAB V1 component image smoke failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function Install-ComponentImage {
|
||||||
|
param(
|
||||||
|
[string]$Component,
|
||||||
|
[object]$Descriptor
|
||||||
|
)
|
||||||
|
|
||||||
|
$componentRoot = Join-Path $BuildRoot $Component
|
||||||
|
if (-not (Test-Path -LiteralPath $componentRoot -PathType Container)) {
|
||||||
|
throw "LAB V1 component build source is unavailable"
|
||||||
|
}
|
||||||
|
Assert-ComponentSource $componentRoot $Descriptor
|
||||||
|
$baseReference = "sha256:$($Descriptor.base_image_sha256)"
|
||||||
|
$base = Get-ImageInspection $baseReference
|
||||||
|
if ([string]$base.Id -cne $baseReference) {
|
||||||
|
throw "LAB V1 component base image identity changed"
|
||||||
|
}
|
||||||
|
$shortRevision = $SourceRevision.Substring(0, 12)
|
||||||
|
$tag = "ndc/mission-core-lab-v1-$Component-adapter:$shortRevision"
|
||||||
|
$existing = docker image inspect $tag 2>$null
|
||||||
|
if ($LASTEXITCODE -eq 0) {
|
||||||
|
$image = @($existing | ConvertFrom-Json)[0]
|
||||||
|
$labels = $image.Config.Labels
|
||||||
|
if (
|
||||||
|
[string]$labels."org.opencontainers.image.revision" -cne $SourceRevision -or
|
||||||
|
[string]$labels."com.nodedc.base-image.sha256" -cne
|
||||||
|
[string]$Descriptor.base_image_sha256 -or
|
||||||
|
[string]$labels."com.nodedc.shared-adapter.sha256" -cne
|
||||||
|
$SharedAdapterSha256 -or
|
||||||
|
[string]$labels."com.nodedc.component-adapter.sha256" -cne
|
||||||
|
[string]$Descriptor.adapter_sha256 -or
|
||||||
|
[string]$labels."com.nodedc.build-method" -cne $BuildMethod
|
||||||
|
) {
|
||||||
|
throw "Existing LAB V1 component image label identity changed"
|
||||||
|
}
|
||||||
|
Invoke-InstalledImageSmoke $tag $Descriptor
|
||||||
|
return [ordered]@{
|
||||||
|
component = $Component
|
||||||
|
status = "already-installed"
|
||||||
|
tag = $tag
|
||||||
|
base_image_sha256 = [string]$Descriptor.base_image_sha256
|
||||||
|
derived_image_sha256 = ([string]$image.Id).Substring(7)
|
||||||
|
build_method = $BuildMethod
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$containerName = "ndc-lab-v1-$Component-image-build-$shortRevision"
|
||||||
|
$existingContainer = docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}"
|
||||||
|
if ($existingContainer) {
|
||||||
|
throw "LAB V1 temporary build container name is already occupied"
|
||||||
|
}
|
||||||
|
$copyScript = (
|
||||||
|
"set -eu; " +
|
||||||
|
"mkdir -p /opt/nodedc/adapter; " +
|
||||||
|
"cp /nodedc-build-source/portable_lab_v1_component_adapter.py " +
|
||||||
|
"/opt/nodedc/adapter/portable_lab_v1_component_adapter.py; " +
|
||||||
|
"cp /nodedc-build-source/$($Descriptor.adapter_name) " +
|
||||||
|
"/opt/nodedc/adapter/$($Descriptor.adapter_name); " +
|
||||||
|
"test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||||
|
"| cut -d' ' -f1) = $SharedAdapterSha256; " +
|
||||||
|
"test `$(sha256sum /opt/nodedc/adapter/$($Descriptor.adapter_name) " +
|
||||||
|
"| cut -d' ' -f1) = $($Descriptor.adapter_sha256); " +
|
||||||
|
"chmod 0444 /opt/nodedc/adapter/*.py; " +
|
||||||
|
"$($Descriptor.python_check) " +
|
||||||
|
"/opt/nodedc/adapter/portable_lab_v1_component_adapter.py " +
|
||||||
|
"/opt/nodedc/adapter/$($Descriptor.adapter_name); " +
|
||||||
|
"rm -rf /opt/nodedc/adapter/__pycache__"
|
||||||
|
)
|
||||||
|
$mount = "type=bind,source=$componentRoot,target=/nodedc-build-source,readonly"
|
||||||
|
$containerId = docker create `
|
||||||
|
--name $containerName `
|
||||||
|
--network none `
|
||||||
|
--cap-drop ALL `
|
||||||
|
--security-opt no-new-privileges `
|
||||||
|
--mount $mount `
|
||||||
|
--entrypoint /bin/sh `
|
||||||
|
$baseReference `
|
||||||
|
-c $copyScript
|
||||||
|
if ($LASTEXITCODE -ne 0 -or $containerId -notmatch "^[a-f0-9]{64}$") {
|
||||||
|
throw "LAB V1 temporary component build container creation failed"
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
docker start --attach $containerId
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "LAB V1 component adapter installation command failed"
|
||||||
|
}
|
||||||
|
$changes = @(
|
||||||
|
"--change", "ENTRYPOINT $($Descriptor.entrypoint)",
|
||||||
|
"--change", "CMD $($Descriptor.command)",
|
||||||
|
"--change", "LABEL org.opencontainers.image.revision=$SourceRevision",
|
||||||
|
"--change", "LABEL com.nodedc.product=mission-core",
|
||||||
|
"--change", "LABEL com.nodedc.stack=observatory",
|
||||||
|
"--change", "LABEL com.nodedc.component=lab-v1-$Component",
|
||||||
|
"--change", "LABEL com.nodedc.authority=observation-only",
|
||||||
|
"--change", "LABEL com.nodedc.base-image.sha256=$($Descriptor.base_image_sha256)",
|
||||||
|
"--change", "LABEL com.nodedc.shared-adapter.sha256=$SharedAdapterSha256",
|
||||||
|
"--change", "LABEL com.nodedc.component-adapter.sha256=$($Descriptor.adapter_sha256)",
|
||||||
|
"--change", "LABEL com.nodedc.build-method=$BuildMethod"
|
||||||
|
)
|
||||||
|
foreach ($value in $Descriptor.environment) {
|
||||||
|
$changes += @("--change", "ENV $value")
|
||||||
|
}
|
||||||
|
docker commit @changes $containerId $tag | Out-Null
|
||||||
|
if ($LASTEXITCODE -ne 0) {
|
||||||
|
throw "LAB V1 component image commit failed"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
docker rm -f $containerId | Out-Null
|
||||||
|
}
|
||||||
|
|
||||||
|
$image = Get-ImageInspection $tag
|
||||||
|
if (
|
||||||
|
[int64]$image.Size -lt [int64]$base.Size -or
|
||||||
|
([int64]$image.Size - [int64]$base.Size) -gt $MaximumLayerBytes
|
||||||
|
) {
|
||||||
|
throw "LAB V1 adapter image is not a thin layer"
|
||||||
|
}
|
||||||
|
Invoke-InstalledImageSmoke $tag $Descriptor
|
||||||
|
return [ordered]@{
|
||||||
|
component = $Component
|
||||||
|
status = "installed"
|
||||||
|
tag = $tag
|
||||||
|
base_image_sha256 = [string]$Descriptor.base_image_sha256
|
||||||
|
derived_image_sha256 = ([string]$image.Id).Substring(7)
|
||||||
|
build_method = $BuildMethod
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not (Test-Path -LiteralPath $BuildRoot -PathType Container)) {
|
||||||
|
throw "LAB V1 component build root is unavailable"
|
||||||
|
}
|
||||||
|
|
||||||
|
$outputs = New-Object "Collections.Generic.List[object]"
|
||||||
|
foreach ($component in $Components.Keys) {
|
||||||
|
$outputs.Add((Install-ComponentImage $component $Components[$component]))
|
||||||
|
}
|
||||||
|
$outputs.ToArray() | ConvertTo-Json -Compress -Depth 5
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
INSTALLER = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "experiments/perception/worker/observatory_portable"
|
||||||
|
/ "Install-LabV1ComponentAdapterImages.ps1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_component_image_installer_is_exact_local_and_offline() -> None:
|
||||||
|
script = INSTALLER.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert '"docker-commit-exact-layer-v1"' in script
|
||||||
|
assert "--network none" in script
|
||||||
|
assert "--cap-drop ALL" in script
|
||||||
|
assert "--security-opt no-new-privileges" in script
|
||||||
|
assert "target=/nodedc-build-source,readonly" in script
|
||||||
|
assert "Get-FileSha256" in script
|
||||||
|
assert "Invoke-InstalledImageSmoke" in script
|
||||||
|
assert "docker commit @changes $containerId $tag" in script
|
||||||
|
assert "docker rm -f $containerId" in script
|
||||||
|
assert "com.nodedc.build-method" in script
|
||||||
|
|
||||||
|
lowered = script.lower()
|
||||||
|
assert "docker pull" not in lowered
|
||||||
|
assert "invoke-webrequest" not in lowered
|
||||||
|
assert "start-bitstransfer" not in lowered
|
||||||
|
assert "curl " not in lowered
|
||||||
|
assert "wget " not in lowered
|
||||||
|
assert "smb" not in lowered
|
||||||
|
|
||||||
|
|
||||||
|
def test_component_image_installer_binds_bases_and_adapter_sources() -> None:
|
||||||
|
script = INSTALLER.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
for digest in (
|
||||||
|
"58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794",
|
||||||
|
"591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd",
|
||||||
|
"fb208c6cd337153827147006cd26bf4c1c00963f91eb801280485dbc7213d08b",
|
||||||
|
"b48ef255403abcfd177a6550688410a52b0fbd29d2a54c8417cb5a0daa330ad5",
|
||||||
|
"d2af887b065b0f63890768d59247114e49b70ad490c027f20eefd98682636c3c",
|
||||||
|
):
|
||||||
|
assert digest in script
|
||||||
|
assert 'entrypoint = \'["python3"]\'' in script
|
||||||
|
assert '"goose","python"' in script
|
||||||
|
assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"]\'' in script
|
||||||
|
assert 'command = \'["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"]\'' in script
|
||||||
Reference in New Issue
Block a user