diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-ddrnet-step b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-ddrnet-step new file mode 100644 index 0000000..4a417e5 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-ddrnet-step @@ -0,0 +1,27 @@ +FROM ndc/mission-core-lab-v1-ddrnet-adapter:44ebbe7ca2e6@sha256:104896278a762e34c1242686992af9e59b1b4c56e5291c8694a34470eefa5d35 + +ARG NODEDC_SHARED_ADAPTER_SHA256 +ARG NODEDC_COMPONENT_ADAPTER_SHA256 + +COPY portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py +COPY run_portable_lab_v1_ddrnet_component.py /opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py + +RUN test "$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1)" = "${NODEDC_SHARED_ADAPTER_SHA256}" \ + && test "$(sha256sum /opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py | cut -d' ' -f1)" = "${NODEDC_COMPONENT_ADAPTER_SHA256}" \ + && chmod 0444 /opt/nodedc/adapter/portable_lab_v1_component_adapter.py \ + /opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py \ + && cd /opt/nodedc/adapter \ + && conda run --no-capture-output --name goose python -B -m py_compile \ + portable_lab_v1_component_adapter.py run_portable_lab_v1_ddrnet_component.py \ + && rm -rf /opt/nodedc/adapter/__pycache__ + +LABEL com.nodedc.product="mission-core" \ + com.nodedc.stack="observatory" \ + com.nodedc.component="installed-lab-v1-ddrnet-step" \ + com.nodedc.authority="observation-only" \ + com.nodedc.base-image.sha256="104896278a762e34c1242686992af9e59b1b4c56e5291c8694a34470eefa5d35" \ + com.nodedc.shared-adapter.sha256="${NODEDC_SHARED_ADAPTER_SHA256}" \ + com.nodedc.component-adapter.sha256="${NODEDC_COMPONENT_ADAPTER_SHA256}" + +ENTRYPOINT ["conda", "run", "--no-capture-output", "--name", "goose", "python"] +CMD ["/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py"] diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-eomt-step b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-eomt-step new file mode 100644 index 0000000..3af2379 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-v1-eomt-step @@ -0,0 +1,27 @@ +FROM ndc/mission-core-lab-v1-eomt-adapter:44ebbe7ca2e6@sha256:1621b274967079b8e30aca93eb82a281d532710328de1f0a2887b7b25393c04c + +ARG NODEDC_SHARED_ADAPTER_SHA256 +ARG NODEDC_COMPONENT_ADAPTER_SHA256 + +COPY portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py +COPY run_portable_lab_v1_eomt_component.py /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py + +RUN test "$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1)" = "${NODEDC_SHARED_ADAPTER_SHA256}" \ + && test "$(sha256sum /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py | cut -d' ' -f1)" = "${NODEDC_COMPONENT_ADAPTER_SHA256}" \ + && chmod 0444 /opt/nodedc/adapter/portable_lab_v1_component_adapter.py \ + /opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py \ + && cd /opt/nodedc/adapter \ + && python3 -B -m py_compile portable_lab_v1_component_adapter.py \ + run_portable_lab_v1_eomt_component.py \ + && rm -rf /opt/nodedc/adapter/__pycache__ + +LABEL com.nodedc.product="mission-core" \ + com.nodedc.stack="observatory" \ + com.nodedc.component="installed-lab-v1-eomt-step" \ + com.nodedc.authority="observation-only" \ + com.nodedc.base-image.sha256="1621b274967079b8e30aca93eb82a281d532710328de1f0a2887b7b25393c04c" \ + com.nodedc.shared-adapter.sha256="${NODEDC_SHARED_ADAPTER_SHA256}" \ + com.nodedc.component-adapter.sha256="${NODEDC_COMPONENT_ADAPTER_SHA256}" + +ENTRYPOINT ["python3"] +CMD ["/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py"] diff --git a/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-worker-agent b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-worker-agent new file mode 100644 index 0000000..01a5fd5 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Dockerfile.installed-lab-worker-agent @@ -0,0 +1,29 @@ +FROM ndc/mission-core-observatory-worker-agent:7025e173a337@sha256:be38fe430e6047714882c2538dbb47a05fe0d7d4f6cb715703d18bb8066bc09d + +ARG NODEDC_SOURCE_TREE_SHA256 + +ENV PYTHONPATH=/opt/nodedc/mission-core/src:/opt/nodedc/mission-core/deps \ + PYTHONNOUSERSITE=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /opt/nodedc/mission-core + +COPY src/k1link ./src/k1link + +RUN case "${NODEDC_SOURCE_TREE_SHA256}" in *[!0-9a-f]*|'') exit 64 ;; esac \ + && test "${#NODEDC_SOURCE_TREE_SHA256}" -eq 64 \ + && find /opt/nodedc/mission-core/src/k1link -type d -exec chmod 0555 {} + \ + && find /opt/nodedc/mission-core/src/k1link -type f -exec chmod 0444 {} + \ + && python3 -B -c "import k1link.observatory.installed_lab_worker_container_main as entrypoint; import k1link.observatory.installed_lab_worker_service as worker; import k1link.observatory.lab_v1_installed_package_steps as steps; assert callable(entrypoint.main); assert callable(worker.main); assert callable(steps.main)" + +LABEL com.nodedc.product="mission-core" \ + com.nodedc.stack="observatory" \ + com.nodedc.role="installed-lab-worker-agent" \ + com.nodedc.worker-contour="worker-006" \ + com.nodedc.authority="observation-only" \ + com.nodedc.source-tree.sha256="${NODEDC_SOURCE_TREE_SHA256}" \ + com.nodedc.base-image.sha256="be38fe430e6047714882c2538dbb47a05fe0d7d4f6cb715703d18bb8066bc09d" + +ENTRYPOINT ["python3"] +CMD ["-m", "k1link.observatory.installed_lab_worker_container_main"] diff --git a/experiments/perception/worker/observatory_portable/Initialize-ObservatoryWorkerTokenVolume.ps1 b/experiments/perception/worker/observatory_portable/Initialize-ObservatoryWorkerTokenVolume.ps1 new file mode 100644 index 0000000..73c13c5 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Initialize-ObservatoryWorkerTokenVolume.ps1 @@ -0,0 +1,92 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^ndc-observatory-m49-worker-token-[a-f0-9]{7,8}$")] + [string]$SourceTokenVolume, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$AgentImageSha256, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$ProvisionScriptPath, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedProvisionScriptSha256 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$ExpectedComputer = "DESKTOP-OPJ8J04" +$TargetTokenVolume = "ndc-observatory-worker-token-v1" + +function Resolve-DFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: file" + } + return $item.FullName +} + +if ($env:COMPUTERNAME -cne $ExpectedComputer) { + throw "Observatory Worker token migration is pinned to Worker 006" +} +$provisionScript = Resolve-DFile $ProvisionScriptPath "token provision script" +if ( + (Get-FileHash -LiteralPath $provisionScript -Algorithm SHA256).Hash.ToLowerInvariant() -cne + $ExpectedProvisionScriptSha256 +) { + throw "token provision script identity changed" +} +$image = @((& docker image inspect "sha256:$AgentImageSha256") | ConvertFrom-Json) +if ( + $LASTEXITCODE -ne 0 -or + $image.Count -ne 1 -or + [string]$image[0].Id -cne "sha256:$AgentImageSha256" +) { + throw "Observatory Worker agent image identity changed" +} +$source = @(& docker volume inspect $SourceTokenVolume 2>$null) +if ($LASTEXITCODE -ne 0 -or $source.Count -eq 0) { + throw "source Observatory Worker token volume is absent" +} +$existing = @(& docker volume inspect $TargetTokenVolume 2>$null) +if ($LASTEXITCODE -ne 0) { + $created = [string](& docker volume create $TargetTokenVolume) + if ($LASTEXITCODE -ne 0 -or $created -cne $TargetTokenVolume) { + throw "shared Observatory Worker token volume creation failed" + } + $state = "created" +} +elseif ($existing.Count -eq 0) { + throw "shared Observatory Worker token volume inspection failed" +} +else { + $state = "existing" +} + +& docker run --rm --network none --read-only --cap-drop ALL ` + --security-opt no-new-privileges --pids-limit 32 ` + --mount "type=volume,source=$SourceTokenVolume,target=/source-secret,readonly" ` + --mount "type=volume,source=$TargetTokenVolume,target=/target-secret" ` + --mount "type=bind,source=$provisionScript,target=/provision-token.sh,readonly" ` + --entrypoint /bin/sh "sha256:$AgentImageSha256" /provision-token.sh +if ($LASTEXITCODE -ne 0) { + throw "shared Observatory Worker token volume provisioning failed" +} + +[ordered]@{ + schema_version = "missioncore.observatory-worker-token-volume-initialization/v1" + state = $state + target_volume = $TargetTokenVolume + token_exposed = $false + worker_id = "worker-006" +} | ConvertTo-Json -Compress diff --git a/experiments/perception/worker/observatory_portable/Install-InstalledLabV1PackageImages.ps1 b/experiments/perception/worker/observatory_portable/Install-InstalledLabV1PackageImages.ps1 new file mode 100644 index 0000000..a60ad25 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Install-InstalledLabV1PackageImages.ps1 @@ -0,0 +1,315 @@ +[CmdletBinding(DefaultParameterSetName = "Full")] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$StagingRoot, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$SourceTreeSha256, + + [Parameter(Mandatory = $true, ParameterSetName = "Full")] + [Parameter(Mandatory = $true, ParameterSetName = "ComponentsOnly")] + [Parameter(Mandatory = $true, ParameterSetName = "EomtOnly")] + [Parameter(Mandatory = $true, ParameterSetName = "DdrnetOnly")] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$SharedAdapterSha256, + + [Parameter(Mandatory = $true, ParameterSetName = "Full")] + [Parameter(Mandatory = $true, ParameterSetName = "ComponentsOnly")] + [Parameter(Mandatory = $true, ParameterSetName = "EomtOnly")] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$EomtAdapterSha256, + + [Parameter(Mandatory = $true, ParameterSetName = "Full")] + [Parameter(Mandatory = $true, ParameterSetName = "ComponentsOnly")] + [Parameter(Mandatory = $true, ParameterSetName = "DdrnetOnly")] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$DdrnetAdapterSha256, + + [Parameter(Mandatory = $true, ParameterSetName = "AgentOnly")] + [switch]$AgentOnly, + + [Parameter(Mandatory = $true, ParameterSetName = "ComponentsOnly")] + [switch]$ComponentsOnly, + + [Parameter(Mandatory = $true, ParameterSetName = "EomtOnly")] + [switch]$EomtOnly, + + [Parameter(Mandatory = $true, ParameterSetName = "DdrnetOnly")] + [switch]$DdrnetOnly +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$StagingRoot = [IO.Path]::GetFullPath($StagingRoot).TrimEnd("\") +$AllowedRoot = "D:\NDC_MISSIONCORE\runtime\staging\observatory-installed-lab-v1-" +if (-not $StagingRoot.StartsWith($AllowedRoot, [StringComparison]::Ordinal)) { + throw "Installed LAB V1 staging root is outside the allowlist" +} +$PortableRoot = Join-Path $StagingRoot "experiments\perception\worker\observatory_portable" +$ShortIdentity = $SourceTreeSha256.Substring(0, 12) +$BuildMethod = "docker-commit-exact-layer-v1" + +function Get-ExactImage { + param([string]$Reference) + + $rows = @(docker image inspect $Reference | ConvertFrom-Json) + if ($LASTEXITCODE -ne 0 -or $rows.Count -ne 1 -or [string]$rows[0].Id -notmatch "^sha256:[a-f0-9]{64}$") { + throw "Installed LAB V1 image is unavailable: $Reference" + } + return $rows[0] +} + +function Get-ExactFileSha256 { + param([string]$Path) + + $item = Get-Item -LiteralPath $Path -Force + if (-not ($item -is [IO.FileInfo]) -or ($item.Attributes -band [IO.FileAttributes]::ReparsePoint)) { + throw "Installed LAB V1 build input is not a regular file" + } + return (Get-FileHash -LiteralPath $item.FullName -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Get-ExactSourceTreeSha256 { + param([string]$Path) + + $root = [IO.Path]::GetFullPath($Path).TrimEnd("\") + "\" + $files = @(Get-ChildItem -LiteralPath $root -File -Force -Recurse) + [string[]]$relativePaths = @( + $files | ForEach-Object { + if ($_.Attributes -band [IO.FileAttributes]::ReparsePoint) { + throw "Installed LAB V1 source tree contains a reparse point" + } + $_.FullName.Substring($root.Length).Replace("\", "/") + } + ) + [Array]::Sort($relativePaths, [StringComparer]::Ordinal) + $tab = [char]9 + $lineFeed = [char]10 + $rows = foreach ($relativePath in $relativePaths) { + $file = Get-Item -LiteralPath (Join-Path $root $relativePath.Replace("/", "\")) -Force + $sha256 = (Get-FileHash -LiteralPath $file.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + $relativePath + $tab + $file.Length + $tab + $sha256 + } + $payload = [string]::Join($lineFeed, $rows) + $lineFeed + $utf8 = New-Object System.Text.UTF8Encoding($false) + $digest = [Security.Cryptography.SHA256]::Create().ComputeHash($utf8.GetBytes($payload)) + return ([BitConverter]::ToString($digest)).Replace("-", "").ToLowerInvariant() +} + +function Invoke-ContainerBuild { + param( + [string]$Name, + [string]$BaseReference, + [string]$CopyScript, + [string]$Tag, + [string[]]$Changes, + [string]$MountSource + ) + + if (docker ps -a --filter "name=^/$Name$" --format "{{.ID}}") { + throw "Installed LAB V1 temporary container name is occupied" + } + $mount = "type=bind,source=$MountSource,target=/nodedc-build-source,readonly" + $containerId = docker create ` + --name $Name ` + --label "com.nodedc.build=installed-lab-v1" ` + --network none ` + --cap-drop ALL ` + --security-opt no-new-privileges ` + --mount $mount ` + --entrypoint /bin/sh ` + $BaseReference ` + -c $CopyScript + if ($LASTEXITCODE -ne 0 -or $containerId -notmatch "^[a-f0-9]{64}$") { + throw "Installed LAB V1 temporary container creation failed" + } + try { + docker start --attach $containerId + if ($LASTEXITCODE -ne 0) { + throw "Installed LAB V1 exact layer command failed" + } + docker commit $Changes $containerId $Tag | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Installed LAB V1 image commit failed" + } + } + finally { + docker rm -f $containerId | Out-Null + } + return Get-ExactImage $Tag +} + +function Install-Component { + param( + [string]$Component, + [string]$BaseSha256, + [string]$AdapterName, + [string]$AdapterSha256 + ) + + $shared = Join-Path $PortableRoot "portable_lab_v1_component_adapter.py" + $adapter = Join-Path $PortableRoot $AdapterName + if ((Get-ExactFileSha256 $shared) -cne $SharedAdapterSha256 -or (Get-ExactFileSha256 $adapter) -cne $AdapterSha256) { + throw "Installed LAB V1 component source identity changed" + } + $identityPayload = "$Component`n$BaseSha256`n$SharedAdapterSha256`n$AdapterSha256`n" + $utf8 = New-Object System.Text.UTF8Encoding($false) + $identityDigest = [Security.Cryptography.SHA256]::Create().ComputeHash( + $utf8.GetBytes($identityPayload) + ) + $componentIdentity = ([BitConverter]::ToString($identityDigest)).Replace( + "-", "" + ).ToLowerInvariant() + $componentShortIdentity = $componentIdentity.Substring(0, 12) + $tag = "ndc/mission-core-installed-lab-v1-$Component-step:$componentShortIdentity" + $existing = docker image ls --quiet --no-trunc --filter "reference=$tag" + if ($existing) { + throw "Installed LAB V1 target image tag already exists" + } + $pythonCheck = "python3 -B -m py_compile" + $entrypoint = '[\"python3\"]' + if ($Component -ceq "ddrnet") { + $pythonCheck = "conda run --no-capture-output --name goose python -B -m py_compile" + $entrypoint = '[\"conda\",\"run\",\"--no-capture-output\",\"--name\",\"goose\",\"python\"]' + } + $copyScript = ( + "set -eu; mkdir -p /opt/nodedc/adapter; " + + "rm -f /opt/nodedc/adapter/portable_lab_v1_component_adapter.py /opt/nodedc/adapter/$AdapterName; " + + "cp /nodedc-build-source/portable_lab_v1_component_adapter.py /opt/nodedc/adapter/portable_lab_v1_component_adapter.py; " + + "cp /nodedc-build-source/$AdapterName /opt/nodedc/adapter/$AdapterName; " + + "test `$(sha256sum /opt/nodedc/adapter/portable_lab_v1_component_adapter.py | cut -d' ' -f1) = $SharedAdapterSha256; " + + "test `$(sha256sum /opt/nodedc/adapter/$AdapterName | cut -d' ' -f1) = $AdapterSha256; " + + "chmod 0444 /opt/nodedc/adapter/*.py; cd /opt/nodedc/adapter; " + + "$pythonCheck portable_lab_v1_component_adapter.py $AdapterName; rm -rf __pycache__" + ) + $changes = @( + "--change", "ENTRYPOINT $entrypoint", + "--change", ('CMD [\"/opt/nodedc/adapter/{0}\"]' -f $AdapterName), + "--change", "LABEL com.nodedc.product=mission-core", + "--change", "LABEL com.nodedc.stack=observatory", + "--change", "LABEL com.nodedc.component=installed-lab-v1-$Component-step", + "--change", "LABEL com.nodedc.authority=observation-only", + "--change", "LABEL com.nodedc.base-image.sha256=$BaseSha256", + "--change", "LABEL com.nodedc.shared-adapter.sha256=$SharedAdapterSha256", + "--change", "LABEL com.nodedc.component-adapter.sha256=$AdapterSha256", + "--change", "LABEL com.nodedc.build-method=$BuildMethod" + ) + $image = Invoke-ContainerBuild ` + -Name "ndc-installed-lab-v1-$Component-build-$componentShortIdentity" ` + -BaseReference "sha256:$BaseSha256" ` + -CopyScript $copyScript ` + -Tag $tag ` + -Changes $changes ` + -MountSource $PortableRoot + return [ordered]@{ + component = $Component + tag = $tag + base_image_sha256 = $BaseSha256 + image_sha256 = ([string]$image.Id).Substring(7) + build_method = $BuildMethod + } +} + +function Install-Agent { + $baseSha256 = "be38fe430e6047714882c2538dbb47a05fe0d7d4f6cb715703d18bb8066bc09d" + $sourceRoot = Join-Path $StagingRoot "src" + if ((Get-ExactSourceTreeSha256 $sourceRoot) -cne $SourceTreeSha256) { + throw "Installed LAB V1 agent source tree identity changed" + } + $tag = "ndc/mission-core-installed-lab-worker-agent:$ShortIdentity" + $existing = docker image ls --quiet --no-trunc --filter "reference=$tag" + if ($existing) { + throw "Installed LAB V1 agent image tag already exists" + } + $copyScript = ( + "set -eu; mkdir -p /opt/nodedc/installed-lab/src; " + + "cp -a /nodedc-build-source/src/k1link /opt/nodedc/installed-lab/src/k1link; " + + "find /opt/nodedc/installed-lab/src/k1link -type d -exec chmod 0555 {} +; " + + "find /opt/nodedc/installed-lab/src/k1link -type f -exec chmod 0444 {} +; " + + "PYTHONPATH=/opt/nodedc/installed-lab/src:/opt/nodedc/mission-core/deps PYTHONDONTWRITEBYTECODE=1 python3 -B -c 'import k1link.observatory.installed_lab_worker_container_main as entrypoint; import k1link.observatory.installed_lab_worker_service as worker; import k1link.observatory.lab_v1_installed_package_steps as steps; assert callable(entrypoint.main); assert callable(worker.main); assert callable(steps.main)'" + ) + $changes = @( + "--change", 'ENTRYPOINT [\"python3\"]', + "--change", 'CMD [\"-m\",\"k1link.observatory.installed_lab_worker_container_main\"]', + "--change", "ENV PYTHONPATH=/opt/nodedc/installed-lab/src:/opt/nodedc/mission-core/deps", + "--change", "LABEL com.nodedc.product=mission-core", + "--change", "LABEL com.nodedc.stack=observatory", + "--change", "LABEL com.nodedc.role=installed-lab-worker-agent", + "--change", "LABEL com.nodedc.worker-contour=worker-006", + "--change", "LABEL com.nodedc.authority=observation-only", + "--change", "LABEL com.nodedc.source-tree.sha256=$SourceTreeSha256", + "--change", "LABEL com.nodedc.base-image.sha256=$baseSha256", + "--change", "LABEL com.nodedc.build-method=$BuildMethod" + ) + $image = Invoke-ContainerBuild ` + -Name "ndc-installed-lab-agent-build-$ShortIdentity" ` + -BaseReference "sha256:$baseSha256" ` + -CopyScript $copyScript ` + -Tag $tag ` + -Changes $changes ` + -MountSource $StagingRoot + return [ordered]@{ + component = "agent" + tag = $tag + base_image_sha256 = $baseSha256 + image_sha256 = ([string]$image.Id).Substring(7) + build_method = $BuildMethod + } +} + +if ($AgentOnly) { + $receipts = @(Install-Agent) +} +elseif ($EomtOnly) { + $receipts = @( + Install-Component ` + -Component "eomt" ` + -BaseSha256 "1621b274967079b8e30aca93eb82a281d532710328de1f0a2887b7b25393c04c" ` + -AdapterName "run_portable_lab_v1_eomt_component.py" ` + -AdapterSha256 $EomtAdapterSha256 + ) +} +elseif ($DdrnetOnly) { + $receipts = @( + Install-Component ` + -Component "ddrnet" ` + -BaseSha256 "104896278a762e34c1242686992af9e59b1b4c56e5291c8694a34470eefa5d35" ` + -AdapterName "run_portable_lab_v1_ddrnet_component.py" ` + -AdapterSha256 $DdrnetAdapterSha256 + ) +} +elseif ($ComponentsOnly) { + $receipts = @( + Install-Component ` + -Component "eomt" ` + -BaseSha256 "1621b274967079b8e30aca93eb82a281d532710328de1f0a2887b7b25393c04c" ` + -AdapterName "run_portable_lab_v1_eomt_component.py" ` + -AdapterSha256 $EomtAdapterSha256 + Install-Component ` + -Component "ddrnet" ` + -BaseSha256 "104896278a762e34c1242686992af9e59b1b4c56e5291c8694a34470eefa5d35" ` + -AdapterName "run_portable_lab_v1_ddrnet_component.py" ` + -AdapterSha256 $DdrnetAdapterSha256 + ) +} +else { + $receipts = @( + Install-Component ` + -Component "eomt" ` + -BaseSha256 "1621b274967079b8e30aca93eb82a281d532710328de1f0a2887b7b25393c04c" ` + -AdapterName "run_portable_lab_v1_eomt_component.py" ` + -AdapterSha256 $EomtAdapterSha256 + Install-Component ` + -Component "ddrnet" ` + -BaseSha256 "104896278a762e34c1242686992af9e59b1b4c56e5291c8694a34470eefa5d35" ` + -AdapterName "run_portable_lab_v1_ddrnet_component.py" ` + -AdapterSha256 $DdrnetAdapterSha256 + Install-Agent + ) +} + +$receipts | ConvertTo-Json -Depth 4 diff --git a/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 b/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 index 25fc6f6..dbe5c1c 100644 --- a/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 +++ b/experiments/perception/worker/observatory_portable/Install-LabV1PortableStaticAssets.ps1 @@ -6,9 +6,9 @@ $ErrorActionPreference = "Stop" $ProgressPreference = "SilentlyContinue" $RuntimeRoot = [IO.Path]::GetFullPath("D:\NDC_MISSIONCORE\runtime").TrimEnd("\") -$SourceRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-static-v1" +$SourceRoot = Join-Path $RuntimeRoot "staging\observatory-lab-v1-static-v2" $AssetRoot = Join-Path $RuntimeRoot "assets\observatory-portable" -$RunnerRoot = Join-Path $AssetRoot "eomt-runner-bundle-v1" +$RunnerRoot = Join-Path $AssetRoot "eomt-runner-bundle-v2" $ValidFovRoot = Join-Path $AssetRoot ( "k1-valid-fov-root-" + "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1" @@ -42,8 +42,8 @@ $RunnerFiles = [ordered]@{ } "run_e4_full_session_segmentation.py" = [pscustomobject]@{ source = (Join-Path $SourceRoot "run_e4_full_session_segmentation.py") - byte_length = [int64]30720 - sha256 = "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" + byte_length = [int64]31733 + sha256 = "1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff" } "run_evaluation_prelabels.py" = [pscustomobject]@{ source = (Join-Path $SourceRoot "run_evaluation_prelabels.py") diff --git a/experiments/perception/worker/observatory_portable/Install-ObservatoryInstalledLabPackage.ps1 b/experiments/perception/worker/observatory_portable/Install-ObservatoryInstalledLabPackage.ps1 new file mode 100644 index 0000000..58d8b27 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Install-ObservatoryInstalledLabPackage.ps1 @@ -0,0 +1,601 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$SourceReleaseRoot, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedPromotionSummarySha256, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedPackageSha256, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$ValidationReceiptPath, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedValidationReceiptFileSha256 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$WorkerId = "worker-006" +$ExpectedComputer = "DESKTOP-OPJ8J04" +$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime" +$StagingPrefix = "$RuntimeRoot\staging\observatory-installed-lab-" +$ReleaseParent = "$RuntimeRoot\releases\observatory-installed-labs" +$StateParent = "$RuntimeRoot\state\observatory-installed-labs" +$WorkParent = "$RuntimeRoot\services\observatory-installed-labs" +$InstallationReceiptName = "installation-receipt.json" +$ActivationName = "activation.json" +$PackageRegistryName = "observatory-installed-lab-packages.json" +$AssetBindingsName = "observatory-installed-lab-asset-bindings.json" +$PromotionSummaryName = "promotion-summary.json" +$RequiredReleaseFiles = @( + $AssetBindingsName, + $PackageRegistryName, + "observatory-portable-run-definitions.json", + "observatory-worker-runtime-candidates.json", + $PromotionSummaryName +) +$Authority = [ordered]@{ + actuation_allowed = $false + commands_enabled = $false + navigation_or_safety_accepted = $false + production_accepted = $false +} + +function Resolve-DDirectory([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName.TrimEnd("\") +} + +function Resolve-DFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: file" + } + return $item.FullName +} + +function Resolve-OrCreateDDirectory([string]$Path, [string]$Label) { + if (-not (Test-Path -LiteralPath $Path)) { + $null = New-Item -ItemType Directory -Path $Path + } + return Resolve-DDirectory $Path $Label +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Get-PayloadSha256([string]$Value) { + $utf8 = New-Object System.Text.UTF8Encoding($false) + $algorithm = [Security.Cryptography.SHA256]::Create() + try { + $digest = $algorithm.ComputeHash($utf8.GetBytes($Value)) + return ([BitConverter]::ToString($digest)).Replace("-", "").ToLowerInvariant() + } + finally { + $algorithm.Dispose() + } +} + +function ConvertTo-CompactJson([object]$Value) { + return $Value | ConvertTo-Json -Compress -Depth 16 +} + +function Write-Utf8NoBom([string]$Path, [string]$Value) { + $utf8 = New-Object System.Text.UTF8Encoding($false) + [IO.File]::WriteAllText($Path, $Value, $utf8) +} + +function Assert-ExactFields([object]$Value, [string[]]$Expected, [string]$Label) { + if ($null -eq $Value) { + throw "$Label is absent" + } + [string[]]$actual = @($Value.PSObject.Properties.Name) + [string[]]$wanted = @($Expected) + [Array]::Sort($actual, [StringComparer]::Ordinal) + [Array]::Sort($wanted, [StringComparer]::Ordinal) + if ([string]::Join([char]10, $actual) -cne [string]::Join([char]10, $wanted)) { + throw "$Label fields changed" + } +} + +function Assert-Authority([object]$Value, [string]$Label) { + Assert-ExactFields $Value @( + "actuation_allowed", + "commands_enabled", + "navigation_or_safety_accepted", + "production_accepted" + ) $Label + if ( + [bool]$Value.actuation_allowed -or + [bool]$Value.commands_enabled -or + [bool]$Value.navigation_or_safety_accepted -or + [bool]$Value.production_accepted + ) { + throw "$Label grants forbidden authority" + } +} + +function Read-ExactJson([string]$Path, [string]$Sha256, [string]$Label) { + $resolved = Resolve-DFile $Path $Label + if ((Get-Sha256 $resolved) -cne $Sha256) { + throw "$Label identity changed" + } + try { + return Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + } + catch { + throw "$Label is not valid JSON" + } +} + +function Get-ReleaseInventory([string]$Root) { + $children = @(Get-ChildItem -LiteralPath $Root -Force) + if ( + $children.Count -lt $RequiredReleaseFiles.Count -or + $children.Count -gt 64 + ) { + throw "installed LAB source release inventory changed" + } + [string[]]$names = @($children | ForEach-Object { $_.Name }) + [string[]]$expected = @($RequiredReleaseFiles) + [Array]::Sort($names, [StringComparer]::Ordinal) + [Array]::Sort($expected, [StringComparer]::Ordinal) + foreach ($required in $expected) { + if ($names -cnotcontains $required) { + throw "installed LAB source release files changed" + } + } + $rows = foreach ($name in $names) { + $file = Resolve-DFile (Join-Path $Root $name) "installed LAB release file" + $item = Get-Item -LiteralPath $file -Force + [ordered]@{ + byte_length = [long]$item.Length + relative_path = $name + sha256 = Get-Sha256 $file + } + } + return @($rows) +} + +function Assert-ExactImage([string]$Sha256) { + $rows = @(((& docker image inspect "sha256:$Sha256") | ConvertFrom-Json)) + if ( + $LASTEXITCODE -ne 0 -or + $rows.Count -ne 1 -or + [string]$rows[0].Id -cne "sha256:$Sha256" + ) { + throw "installed LAB image identity changed" + } +} + +function New-ReleaseAnchor( + [string]$PackageId, + [string]$PackageSha256, + [string]$ReleaseRoot, + [string]$ReceiptPath, + [string]$ReceiptFileSha256, + [string]$ReceiptSha256 +) { + return [ordered]@{ + installation_receipt_file_sha256 = $ReceiptFileSha256 + installation_receipt_path = $ReceiptPath + installation_receipt_sha256 = $ReceiptSha256 + package_id = $PackageId + package_sha256 = $PackageSha256 + release_root = $ReleaseRoot + } +} + +function New-ActivationIdentity( + [object]$Current, + [object]$Rollback, + [int]$Generation +) { + return [ordered]@{ + authority = $Authority + current = $Current + generation = $Generation + rollback = $Rollback + schema_version = "missioncore.observatory-installed-lab-activation/v1" + state = "installed-ready-offline" + worker_id = $WorkerId + } +} + +function Assert-ReleaseAnchor([object]$Value, [string]$Label) { + Assert-ExactFields $Value @( + "installation_receipt_file_sha256", + "installation_receipt_path", + "installation_receipt_sha256", + "package_id", + "package_sha256", + "release_root" + ) $Label + foreach ($name in @( + "installation_receipt_file_sha256", + "installation_receipt_sha256", + "package_sha256" + )) { + if ([string]$Value.$name -notmatch "^[a-f0-9]{64}$") { + throw "$Label digest is invalid" + } + } +} + +function Read-Activation([string]$Path) { + $resolved = Resolve-DFile $Path "installed LAB activation pointer" + $document = Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + Assert-ExactFields $document @( + "activation_sha256", + "authority", + "current", + "generation", + "rollback", + "schema_version", + "state", + "worker_id" + ) "installed LAB activation pointer" + Assert-Authority $document.authority "installed LAB activation authority" + if ( + [string]$document.schema_version -cne "missioncore.observatory-installed-lab-activation/v1" -or + [string]$document.state -cne "installed-ready-offline" -or + [string]$document.worker_id -cne $WorkerId -or + [int]$document.generation -lt 1 + ) { + throw "installed LAB activation pointer is invalid" + } + Assert-ReleaseAnchor $document.current "installed LAB current release" + if ($null -ne $document.rollback) { + Assert-ReleaseAnchor $document.rollback "installed LAB rollback release" + } + $identity = New-ActivationIdentity ` + -Current $document.current ` + -Rollback $document.rollback ` + -Generation ([int]$document.generation) + if ((Get-PayloadSha256 (ConvertTo-CompactJson $identity)) -cne [string]$document.activation_sha256) { + throw "installed LAB activation identity changed" + } + return $document +} + +function Assert-InstalledRoot( + [string]$Root, + [object[]]$Inventory, + [string]$ExpectedReceiptFileSha256 +) { + $installed = Resolve-DDirectory $Root "installed LAB release root" + $children = @(Get-ChildItem -LiteralPath $installed -Force) + if ($children.Count -ne ($Inventory.Count + 1)) { + throw "installed LAB release root inventory changed" + } + foreach ($row in $Inventory) { + $file = Resolve-DFile ` + (Join-Path $installed ([string]$row.relative_path)) ` + "installed LAB release file" + $item = Get-Item -LiteralPath $file -Force + if ( + [long]$item.Length -ne [long]$row.byte_length -or + (Get-Sha256 $file) -cne [string]$row.sha256 + ) { + throw "installed LAB release file identity changed" + } + } + $receipt = Resolve-DFile ` + (Join-Path $installed $InstallationReceiptName) ` + "installed LAB installation receipt" + if ((Get-Sha256 $receipt) -cne $ExpectedReceiptFileSha256) { + throw "installed LAB installation receipt identity changed" + } + return $installed +} + +if ($env:COMPUTERNAME -cne $ExpectedComputer) { + throw "installed LAB package installation is pinned to Worker 006" +} + +$sourceRoot = Resolve-DDirectory $SourceReleaseRoot "installed LAB source release" +if (-not $sourceRoot.StartsWith($StagingPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "installed LAB source release is outside the staging allowlist" +} +$validationPath = Resolve-DFile $ValidationReceiptPath "installed LAB validation receipt" +if (-not $validationPath.StartsWith($StagingPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "installed LAB validation receipt is outside the staging allowlist" +} +$inventory = @(Get-ReleaseInventory $sourceRoot) +$summary = Read-ExactJson ` + (Join-Path $sourceRoot $PromotionSummaryName) ` + $ExpectedPromotionSummarySha256 ` + "installed LAB promotion summary" +$packageRegistryPath = Resolve-DFile ` + (Join-Path $sourceRoot $PackageRegistryName) ` + "installed LAB package registry" +$packageRegistry = Get-Content -LiteralPath $packageRegistryPath -Raw | ConvertFrom-Json +if ( + [string]$packageRegistry.schema_version -cne "missioncore.observatory-installed-lab-package-registry/v1" -or + @($packageRegistry.packages).Count -ne 1 +) { + throw "installed LAB package registry must contain one package" +} +$package = @($packageRegistry.packages)[0] +$packageId = [string]$package.package_id +$packageSha256 = [string]$package.package_sha256 +if ( + $packageId -notmatch "^[a-z0-9][a-z0-9.-]{0,126}[a-z0-9]$" -or + $packageSha256 -cne $ExpectedPackageSha256 -or + $packageSha256 -cne [string]$summary.package_sha256 +) { + throw "installed LAB package identity changed" +} +Assert-Authority $package.authority "installed LAB package authority" + +$validation = Read-ExactJson ` + $validationPath ` + $ExpectedValidationReceiptFileSha256 ` + "installed LAB validation receipt" +if ( + [string]$validation.schema_version -cne "missioncore.observatory-installed-lab-worker-validation/v1" -or + [string]$validation.state -cne "ready" -or + [bool]$validation.checks.backend_contacted -or + [bool]$validation.checks.claim_attempted -or + [string]$validation.checks.registry_binding -cne "passed" -or + [string]$validation.checks.local_asset_identity -cne "passed" -or + [string]$validation.checks.docker_image_identity -cne "passed" -or + @($validation.packages).Count -ne 1 -or + [string]$validation.packages[0].package_sha256 -cne $packageSha256 -or + [string]$validation.packages[0].definition_sha256 -cne [string]$package.definition_sha256 -or + [string]$validation.packages[0].runtime_candidate_sha256 -cne [string]$package.runtime_candidate_sha256 +) { + throw "installed LAB validation receipt does not admit the package" +} +Assert-Authority $validation.authority "installed LAB validation authority" + +[string[]]$packageImages = @( + $package.containers | ForEach-Object { [string]$_.image_sha256 } | Select-Object -Unique +) +[string[]]$validatedImages = @($validation.verified_image_sha256s | ForEach-Object { [string]$_ }) +[Array]::Sort($packageImages, [StringComparer]::Ordinal) +[Array]::Sort($validatedImages, [StringComparer]::Ordinal) +if ( + $packageImages.Count -lt 1 -or + [string]::Join([char]10, $packageImages) -cne [string]::Join([char]10, $validatedImages) +) { + throw "installed LAB validated image inventory changed" +} +foreach ($imageSha256 in $packageImages) { + Assert-ExactImage $imageSha256 +} + +$runtime = Resolve-DDirectory $RuntimeRoot "Mission Core Worker runtime root" +$releaseBase = Resolve-OrCreateDDirectory $ReleaseParent "installed LAB release parent" +$stateBase = Resolve-OrCreateDDirectory $StateParent "installed LAB state parent" +$workBase = Resolve-OrCreateDDirectory $WorkParent "installed LAB work parent" +$packageReleaseParent = Resolve-OrCreateDDirectory ` + (Join-Path $releaseBase $packageId) ` + "installed LAB package release parent" +$packageStateRoot = Resolve-OrCreateDDirectory ` + (Join-Path $stateBase $packageId) ` + "installed LAB package state root" +$packageWorkParent = Resolve-OrCreateDDirectory ` + (Join-Path $workBase $packageId) ` + "installed LAB package work parent" +$targetRoot = Join-Path $packageReleaseParent $packageSha256 +$workRoot = Resolve-OrCreateDDirectory ` + (Join-Path $packageWorkParent $packageSha256) ` + "installed LAB package work root" + +$bindings = Get-Content -LiteralPath (Join-Path $sourceRoot $AssetBindingsName) -Raw | ConvertFrom-Json +if ( + [string]$bindings.schema_version -cne "missioncore.observatory-installed-lab-asset-bindings/v1" -or + [string]$bindings.engine_work_root -cne $workRoot +) { + throw "installed LAB binding work root is not canonical" +} +[string[]]$boundAssetIds = @($bindings.assets | ForEach-Object { [string]$_.asset_id }) +[string[]]$packageAssetIds = @($package.asset_ids | ForEach-Object { [string]$_ }) +[Array]::Sort($boundAssetIds, [StringComparer]::Ordinal) +[Array]::Sort($packageAssetIds, [StringComparer]::Ordinal) +if ([string]::Join([char]10, $boundAssetIds) -cne [string]::Join([char]10, $packageAssetIds)) { + throw "installed LAB bound asset inventory changed" +} +foreach ($binding in $bindings.assets) { + $controllerPath = $binding.controller_path + $enginePath = $binding.engine_path + $imageSha256 = $binding.image_sha256 + if ($null -ne $imageSha256) { + if ($null -ne $controllerPath -or $null -ne $enginePath) { + throw "installed LAB image binding is ambiguous" + } + continue + } + if ($null -eq $controllerPath -or $null -eq $enginePath) { + throw "installed LAB file binding is incomplete" + } + $controller = [string]$controllerPath + $engine = [string]$enginePath + if ($controller.StartsWith("/release/", [StringComparison]::Ordinal)) { + $relative = $controller.Substring(9).Replace("/", "\") + if ($relative.Split("\") -contains ".." -or $engine -cne (Join-Path $targetRoot $relative)) { + throw "installed LAB release binding is not canonical" + } + } + elseif ($controller.StartsWith("/runtime/", [StringComparison]::Ordinal)) { + $relative = $controller.Substring(9).Replace("/", "\") + if ($relative.Split("\") -contains ".." -or $engine -cne (Join-Path $runtime $relative)) { + throw "installed LAB runtime binding is not canonical" + } + } + else { + throw "installed LAB controller binding root changed" + } +} + +$receiptIdentity = [ordered]@{ + authority = $Authority + computer_name = $ExpectedComputer + files = $inventory + package_id = $packageId + package_sha256 = $packageSha256 + package_version = [int]$package.package_version + promotion_summary_sha256 = $ExpectedPromotionSummarySha256 + release_root = $targetRoot + schema_version = "missioncore.observatory-installed-lab-installation/v1" + state = "installed-ready-offline" + validation_receipt_file_sha256 = $ExpectedValidationReceiptFileSha256 + validation_receipt_sha256 = [string]$validation.receipt_sha256 + verified_image_sha256s = $validatedImages + work_root = $workRoot + worker_id = $WorkerId +} +$receiptSha256 = Get-PayloadSha256 (ConvertTo-CompactJson $receiptIdentity) +$receiptDocument = [ordered]@{} +foreach ($key in $receiptIdentity.Keys) { + $receiptDocument[$key] = $receiptIdentity[$key] +} +$receiptDocument["receipt_sha256"] = $receiptSha256 +$receiptPayload = ConvertTo-CompactJson $receiptDocument +$receiptFileSha256 = Get-PayloadSha256 $receiptPayload +$receiptPath = Join-Path $targetRoot $InstallationReceiptName + +$alreadyInstalled = $false +if (Test-Path -LiteralPath $targetRoot) { + $targetRoot = Assert-InstalledRoot $targetRoot $inventory $receiptFileSha256 + $alreadyInstalled = $true +} +else { + $stageRoot = Join-Path $packageReleaseParent ".install-$packageSha256-$PID" + if (Test-Path -LiteralPath $stageRoot) { + throw "installed LAB package staging requires reconciliation" + } + $null = New-Item -ItemType Directory -Path $stageRoot + try { + $stageRoot = Resolve-DDirectory $stageRoot "installed LAB package staging" + foreach ($row in $inventory) { + Copy-Item ` + -LiteralPath (Join-Path $sourceRoot ([string]$row.relative_path)) ` + -Destination (Join-Path $stageRoot ([string]$row.relative_path)) + } + Write-Utf8NoBom (Join-Path $stageRoot $InstallationReceiptName) $receiptPayload + $null = Assert-InstalledRoot $stageRoot $inventory $receiptFileSha256 + foreach ($file in @(Get-ChildItem -LiteralPath $stageRoot -File -Force)) { + $file.IsReadOnly = $true + } + if (Test-Path -LiteralPath $targetRoot) { + throw "installed LAB package target collided during installation" + } + Move-Item -LiteralPath $stageRoot -Destination $targetRoot + $targetRoot = Assert-InstalledRoot $targetRoot $inventory $receiptFileSha256 + } + finally { + if (Test-Path -LiteralPath $stageRoot) { + [IO.Directory]::Delete($stageRoot, $true) + } + } +} + +$current = New-ReleaseAnchor ` + -PackageId $packageId ` + -PackageSha256 $packageSha256 ` + -ReleaseRoot $targetRoot ` + -ReceiptPath $receiptPath ` + -ReceiptFileSha256 $receiptFileSha256 ` + -ReceiptSha256 $receiptSha256 +$activationPath = Join-Path $packageStateRoot $ActivationName +$existingActivation = $null +if (Test-Path -LiteralPath $activationPath) { + $existingActivation = Read-Activation $activationPath +} +$activationChanged = $true +if ( + $null -ne $existingActivation -and + [string]$existingActivation.current.package_sha256 -ceq $packageSha256 -and + [string]$existingActivation.current.release_root -ceq $targetRoot -and + [string]$existingActivation.current.installation_receipt_file_sha256 -ceq $receiptFileSha256 +) { + $activationChanged = $false + $activationDocument = $existingActivation +} +else { + $generation = 1 + $rollback = $null + if ($null -ne $existingActivation) { + $generation = [int]$existingActivation.generation + 1 + $rollback = New-ReleaseAnchor ` + -PackageId ([string]$existingActivation.current.package_id) ` + -PackageSha256 ([string]$existingActivation.current.package_sha256) ` + -ReleaseRoot ([string]$existingActivation.current.release_root) ` + -ReceiptPath ([string]$existingActivation.current.installation_receipt_path) ` + -ReceiptFileSha256 ([string]$existingActivation.current.installation_receipt_file_sha256) ` + -ReceiptSha256 ([string]$existingActivation.current.installation_receipt_sha256) + } + $activationIdentity = New-ActivationIdentity ` + -Current $current ` + -Rollback $rollback ` + -Generation $generation + $activationSha256 = Get-PayloadSha256 (ConvertTo-CompactJson $activationIdentity) + $activationDocument = [ordered]@{} + foreach ($key in $activationIdentity.Keys) { + $activationDocument[$key] = $activationIdentity[$key] + } + $activationDocument["activation_sha256"] = $activationSha256 + $activationPayload = ConvertTo-CompactJson $activationDocument + $temporaryActivation = Join-Path $packageStateRoot ".activation-$PID.tmp" + $backupActivation = Join-Path $packageStateRoot ".activation-$PID.backup" + if ( + (Test-Path -LiteralPath $temporaryActivation) -or + (Test-Path -LiteralPath $backupActivation) + ) { + throw "installed LAB activation staging requires reconciliation" + } + Write-Utf8NoBom $temporaryActivation $activationPayload + if (Test-Path -LiteralPath $activationPath) { + [IO.File]::Replace($temporaryActivation, $activationPath, $backupActivation, $true) + [IO.File]::Delete($backupActivation) + } + else { + [IO.File]::Move($temporaryActivation, $activationPath) + } + $activationDocument = Read-Activation $activationPath +} + +[ordered]@{ + activation_changed = $activationChanged + activation_path = $activationPath + activation_sha256 = [string]$activationDocument.activation_sha256 + already_installed = $alreadyInstalled + backend_changed = $false + claim_attempted = $false + installation_receipt_file_sha256 = $receiptFileSha256 + installation_receipt_path = $receiptPath + installation_receipt_sha256 = $receiptSha256 + package_id = $packageId + package_sha256 = $packageSha256 + queue_started = $false + release_root = $targetRoot + rollback_available = $null -ne $activationDocument.rollback + schema_version = "missioncore.observatory-installed-lab-installation-result/v1" + state = "installed-ready-offline" + work_root = $workRoot + worker_id = $WorkerId +} | ConvertTo-Json -Compress -Depth 8 diff --git a/experiments/perception/worker/observatory_portable/Invoke-ObservatoryInstalledLabOfflineValidation.ps1 b/experiments/perception/worker/observatory_portable/Invoke-ObservatoryInstalledLabOfflineValidation.ps1 new file mode 100644 index 0000000..364452d --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Invoke-ObservatoryInstalledLabOfflineValidation.ps1 @@ -0,0 +1,553 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$ActivationPath, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedActivationFileSha256 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$WorkerId = "worker-006" +$ExpectedComputer = "DESKTOP-OPJ8J04" +$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime" +$ReleaseParent = "$RuntimeRoot\releases\observatory-installed-labs" +$StateParent = "$RuntimeRoot\state\observatory-installed-labs" +$WorkParent = "$RuntimeRoot\services\observatory-installed-labs" +$ActivationName = "activation.json" +$InstallationReceiptName = "installation-receipt.json" +$PackageRegistryName = "observatory-installed-lab-packages.json" +$DefinitionsName = "observatory-portable-run-definitions.json" +$RuntimeRegistryName = "observatory-worker-runtime-candidates.json" +$AssetBindingsName = "observatory-installed-lab-asset-bindings.json" +$Authority = [ordered]@{ + actuation_allowed = $false + commands_enabled = $false + navigation_or_safety_accepted = $false + production_accepted = $false +} + +function Resolve-DDirectory([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName.TrimEnd("\") +} + +function Resolve-DFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: file" + } + return $item.FullName +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Get-PayloadSha256([string]$Value) { + $utf8 = New-Object System.Text.UTF8Encoding($false) + $algorithm = [Security.Cryptography.SHA256]::Create() + try { + $digest = $algorithm.ComputeHash($utf8.GetBytes($Value)) + return ([BitConverter]::ToString($digest)).Replace("-", "").ToLowerInvariant() + } + finally { + $algorithm.Dispose() + } +} + +function ConvertTo-CompactJson([object]$Value) { + return $Value | ConvertTo-Json -Compress -Depth 16 +} + +function Write-ExactFile([string]$Path, [string]$Payload, [string]$Label) { + $expectedSha256 = Get-PayloadSha256 $Payload + if (Test-Path -LiteralPath $Path) { + $existing = Resolve-DFile $Path $Label + if ((Get-Sha256 $existing) -cne $expectedSha256) { + throw "$Label identity changed" + } + return $existing + } + $utf8 = New-Object System.Text.UTF8Encoding($false) + $temporary = "$Path.$PID.tmp" + if (Test-Path -LiteralPath $temporary) { + throw "$Label staging requires reconciliation" + } + [IO.File]::WriteAllText($temporary, $Payload, $utf8) + [IO.File]::Move($temporary, $Path) + $written = Resolve-DFile $Path $Label + if ((Get-Sha256 $written) -cne $expectedSha256) { + throw "$Label write verification failed" + } + return $written +} + +function Assert-ExactFields([object]$Value, [string[]]$Expected, [string]$Label) { + if ($null -eq $Value) { + throw "$Label is absent" + } + [string[]]$actual = @($Value.PSObject.Properties.Name) + [string[]]$wanted = @($Expected) + [Array]::Sort($actual, [StringComparer]::Ordinal) + [Array]::Sort($wanted, [StringComparer]::Ordinal) + if ([string]::Join([char]10, $actual) -cne [string]::Join([char]10, $wanted)) { + throw "$Label fields changed" + } +} + +function Assert-Authority([object]$Value, [string]$Label) { + Assert-ExactFields $Value @( + "actuation_allowed", + "commands_enabled", + "navigation_or_safety_accepted", + "production_accepted" + ) $Label + if ( + [bool]$Value.actuation_allowed -or + [bool]$Value.commands_enabled -or + [bool]$Value.navigation_or_safety_accepted -or + [bool]$Value.production_accepted + ) { + throw "$Label grants forbidden authority" + } +} + +function Read-Json([string]$Path, [string]$Label) { + $resolved = Resolve-DFile $Path $Label + try { + return Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + } + catch { + throw "$Label is not valid JSON" + } +} + +function New-ReleaseAnchor([object]$Value, [string]$Label) { + Assert-ExactFields $Value @( + "installation_receipt_file_sha256", + "installation_receipt_path", + "installation_receipt_sha256", + "package_id", + "package_sha256", + "release_root" + ) $Label + foreach ($name in @( + "installation_receipt_file_sha256", + "installation_receipt_sha256", + "package_sha256" + )) { + if ([string]$Value.$name -notmatch "^[a-f0-9]{64}$") { + throw "$Label digest is invalid" + } + } + return [ordered]@{ + installation_receipt_file_sha256 = [string]$Value.installation_receipt_file_sha256 + installation_receipt_path = [string]$Value.installation_receipt_path + installation_receipt_sha256 = [string]$Value.installation_receipt_sha256 + package_id = [string]$Value.package_id + package_sha256 = [string]$Value.package_sha256 + release_root = [string]$Value.release_root + } +} + +function Read-Activation([string]$Path, [string]$ExpectedFileSha256) { + $resolved = Resolve-DFile $Path "installed LAB activation pointer" + if ((Get-Sha256 $resolved) -cne $ExpectedFileSha256) { + throw "installed LAB activation file identity changed" + } + $document = Read-Json $resolved "installed LAB activation pointer" + Assert-ExactFields $document @( + "activation_sha256", + "authority", + "current", + "generation", + "rollback", + "schema_version", + "state", + "worker_id" + ) "installed LAB activation pointer" + Assert-Authority $document.authority "installed LAB activation authority" + if ( + [string]$document.schema_version -cne "missioncore.observatory-installed-lab-activation/v1" -or + [string]$document.state -cne "installed-ready-offline" -or + [string]$document.worker_id -cne $WorkerId -or + [int]$document.generation -lt 1 -or + [string]$document.activation_sha256 -notmatch "^[a-f0-9]{64}$" + ) { + throw "installed LAB activation pointer is invalid" + } + $current = New-ReleaseAnchor $document.current "installed LAB current release" + $rollback = $null + if ($null -ne $document.rollback) { + $rollback = New-ReleaseAnchor $document.rollback "installed LAB rollback release" + } + $identity = [ordered]@{ + authority = $Authority + current = $current + generation = [int]$document.generation + rollback = $rollback + schema_version = "missioncore.observatory-installed-lab-activation/v1" + state = "installed-ready-offline" + worker_id = $WorkerId + } + if ((Get-PayloadSha256 (ConvertTo-CompactJson $identity)) -cne [string]$document.activation_sha256) { + throw "installed LAB activation identity changed" + } + return [ordered]@{ + activation_sha256 = [string]$document.activation_sha256 + current = $current + generation = [int]$document.generation + rollback = $rollback + } +} + +function Read-InstallationReceipt([string]$Path, [string]$ExpectedFileSha256) { + $resolved = Resolve-DFile $Path "installed LAB installation receipt" + if ((Get-Sha256 $resolved) -cne $ExpectedFileSha256) { + throw "installed LAB installation receipt file identity changed" + } + $document = Read-Json $resolved "installed LAB installation receipt" + Assert-ExactFields $document @( + "authority", + "computer_name", + "files", + "package_id", + "package_sha256", + "package_version", + "promotion_summary_sha256", + "receipt_sha256", + "release_root", + "schema_version", + "state", + "validation_receipt_file_sha256", + "validation_receipt_sha256", + "verified_image_sha256s", + "work_root", + "worker_id" + ) "installed LAB installation receipt" + Assert-Authority $document.authority "installed LAB installation authority" + if ( + [string]$document.schema_version -cne "missioncore.observatory-installed-lab-installation/v1" -or + [string]$document.state -cne "installed-ready-offline" -or + [string]$document.worker_id -cne $WorkerId -or + [string]$document.computer_name -cne $ExpectedComputer -or + [string]$document.receipt_sha256 -notmatch "^[a-f0-9]{64}$" -or + @($document.files).Count -lt 5 -or + @($document.verified_image_sha256s).Count -lt 1 + ) { + throw "installed LAB installation receipt is invalid" + } + $files = @( + $document.files | ForEach-Object { + Assert-ExactFields $_ @("byte_length", "relative_path", "sha256") "installed LAB file receipt" + if ( + [string]$_.relative_path -notmatch "^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$" -or + [long]$_.byte_length -lt 1 -or + [string]$_.sha256 -notmatch "^[a-f0-9]{64}$" + ) { + throw "installed LAB file receipt is invalid" + } + [ordered]@{ + byte_length = [long]$_.byte_length + relative_path = [string]$_.relative_path + sha256 = [string]$_.sha256 + } + } + ) + [string[]]$images = @($document.verified_image_sha256s | ForEach-Object { [string]$_ }) + [Array]::Sort($images, [StringComparer]::Ordinal) + $identity = [ordered]@{ + authority = $Authority + computer_name = [string]$document.computer_name + files = $files + package_id = [string]$document.package_id + package_sha256 = [string]$document.package_sha256 + package_version = [int]$document.package_version + promotion_summary_sha256 = [string]$document.promotion_summary_sha256 + release_root = [string]$document.release_root + schema_version = "missioncore.observatory-installed-lab-installation/v1" + state = "installed-ready-offline" + validation_receipt_file_sha256 = [string]$document.validation_receipt_file_sha256 + validation_receipt_sha256 = [string]$document.validation_receipt_sha256 + verified_image_sha256s = $images + work_root = [string]$document.work_root + worker_id = $WorkerId + } + if ((Get-PayloadSha256 (ConvertTo-CompactJson $identity)) -cne [string]$document.receipt_sha256) { + throw "installed LAB installation receipt identity changed" + } + return [ordered]@{ + document = $document + files = $files + verified_images = $images + } +} + +function Assert-ExactImage([string]$Sha256) { + $rows = @(((& docker image inspect "sha256:$Sha256") | ConvertFrom-Json)) + if ( + $LASTEXITCODE -ne 0 -or + $rows.Count -ne 1 -or + [string]$rows[0].Id -cne "sha256:$Sha256" + ) { + throw "installed LAB image identity changed" + } +} + +if ($env:COMPUTERNAME -cne $ExpectedComputer) { + throw "installed LAB offline validation is pinned to Worker 006" +} + +$runtime = Resolve-DDirectory $RuntimeRoot "Mission Core Worker runtime root" +$stateBase = Resolve-DDirectory $StateParent "installed LAB state parent" +$releaseBase = Resolve-DDirectory $ReleaseParent "installed LAB release parent" +$workBase = Resolve-DDirectory $WorkParent "installed LAB work parent" +$activationFile = Resolve-DFile $ActivationPath "installed LAB activation pointer" +if ( + (Split-Path -Leaf $activationFile) -cne $ActivationName -or + -not $activationFile.StartsWith("$stateBase\", [StringComparison]::OrdinalIgnoreCase) +) { + throw "installed LAB activation pointer is outside the canonical state root" +} +$packageStateRoot = Resolve-DDirectory ` + (Split-Path -Parent $activationFile) ` + "installed LAB package state root" +$activation = Read-Activation $activationFile $ExpectedActivationFileSha256 +$current = $activation.current +$packageId = [string]$current.package_id +$packageSha256 = [string]$current.package_sha256 +if ((Split-Path -Leaf $packageStateRoot) -cne $packageId) { + throw "installed LAB activation package path changed" +} +$expectedReleaseRoot = Join-Path (Join-Path $releaseBase $packageId) $packageSha256 +$releaseRoot = Resolve-DDirectory ([string]$current.release_root) "installed LAB current release root" +if ($releaseRoot -cne $expectedReleaseRoot) { + throw "installed LAB current release root is not canonical" +} +$receiptPath = Resolve-DFile ` + ([string]$current.installation_receipt_path) ` + "installed LAB installation receipt" +if ($receiptPath -cne (Join-Path $releaseRoot $InstallationReceiptName)) { + throw "installed LAB installation receipt path changed" +} +$receipt = Read-InstallationReceipt ` + $receiptPath ` + ([string]$current.installation_receipt_file_sha256) +$receiptDocument = $receipt.document +if ( + [string]$receiptDocument.receipt_sha256 -cne [string]$current.installation_receipt_sha256 -or + [string]$receiptDocument.package_id -cne $packageId -or + [string]$receiptDocument.package_sha256 -cne $packageSha256 -or + [string]$receiptDocument.release_root -cne $releaseRoot +) { + throw "installed LAB activation and installation receipt disagree" +} +$expectedWorkRoot = Join-Path (Join-Path $workBase $packageId) $packageSha256 +$workRoot = Resolve-DDirectory ([string]$receiptDocument.work_root) "installed LAB package work root" +if ($workRoot -cne $expectedWorkRoot) { + throw "installed LAB work root is not canonical" +} + +$releaseChildren = @(Get-ChildItem -LiteralPath $releaseRoot -Force) +if ($releaseChildren.Count -ne ($receipt.files.Count + 1)) { + throw "installed LAB release inventory changed" +} +foreach ($row in $receipt.files) { + $file = Resolve-DFile ` + (Join-Path $releaseRoot ([string]$row.relative_path)) ` + "installed LAB release file" + $item = Get-Item -LiteralPath $file -Force + if ( + -not $item.IsReadOnly -or + [long]$item.Length -ne [long]$row.byte_length -or + (Get-Sha256 $file) -cne [string]$row.sha256 + ) { + throw "installed LAB release file identity changed" + } +} +if (-not (Get-Item -LiteralPath $receiptPath -Force).IsReadOnly) { + throw "installed LAB installation receipt is not read-only" +} + +$packageRegistry = Read-Json ` + (Join-Path $releaseRoot $PackageRegistryName) ` + "installed LAB package registry" +if ( + [string]$packageRegistry.schema_version -cne "missioncore.observatory-installed-lab-package-registry/v1" -or + @($packageRegistry.packages).Count -ne 1 +) { + throw "installed LAB package registry must contain one package" +} +$package = @($packageRegistry.packages)[0] +if ( + [string]$package.package_id -cne $packageId -or + [string]$package.package_sha256 -cne $packageSha256 +) { + throw "installed LAB package registry identity changed" +} +Assert-Authority $package.authority "installed LAB package authority" +$agentImageSha256 = [string]$package.executor_identity.image_sha256 +if ($agentImageSha256 -notmatch "^[a-f0-9]{64}$") { + throw "installed LAB agent image identity is invalid" +} +[string[]]$packageImages = @( + $package.containers | ForEach-Object { [string]$_.image_sha256 } | Select-Object -Unique +) +[Array]::Sort($packageImages, [StringComparer]::Ordinal) +if ( + $packageImages -cnotcontains $agentImageSha256 -or + [string]::Join([char]10, $packageImages) -cne + [string]::Join([char]10, [string[]]$receipt.verified_images) +) { + throw "installed LAB package and receipt image inventories disagree" +} +foreach ($imageSha256 in $packageImages) { + Assert-ExactImage $imageSha256 +} + +$containerName = "ndc-installed-lab-offline-validation-$($packageSha256.Substring(0, 16))" +$mounts = @( + [ordered]@{ read_only = $true; source = $releaseRoot; target = "/release" }, + [ordered]@{ read_only = $true; source = $runtime; target = "/runtime" }, + [ordered]@{ read_only = $false; source = $workRoot; target = "/work" }, + [ordered]@{ read_only = $false; source = "/var/run/docker.sock"; target = "/var/run/docker.sock" } +) +$environment = @( + [ordered]@{ name = "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE"; value = "/release/$DefinitionsName" }, + [ordered]@{ name = "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE"; value = "/release/$AssetBindingsName" }, + [ordered]@{ name = "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE"; value = "/release/$PackageRegistryName" }, + [ordered]@{ name = "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE"; value = "/release/$RuntimeRegistryName" }, + [ordered]@{ name = "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT"; value = "/work" }, + [ordered]@{ name = "PYTHONDONTWRITEBYTECODE"; value = "1" } +) +$command = @("-m", "k1link.observatory.installed_lab_worker_service", "--validate-only") +$planIdentity = [ordered]@{ + activation_file_sha256 = $ExpectedActivationFileSha256 + activation_sha256 = [string]$activation.activation_sha256 + agent_image_sha256 = $agentImageSha256 + authority = $Authority + backend_contacted = $false + claim_attempted = $false + command = $command + container_name = $containerName + environment = $environment + hardening = [ordered]@{ + capabilities = "none" + network = "none" + no_new_privileges = $true + pids_limit = 128 + root_filesystem = "read-only" + } + mode = "validation-only" + mounts = $mounts + package_id = $packageId + package_sha256 = $packageSha256 + queue_started = $false + schema_version = "missioncore.observatory-installed-lab-offline-launch-plan/v1" + worker_id = $WorkerId +} +$planSha256 = Get-PayloadSha256 (ConvertTo-CompactJson $planIdentity) +$planDocument = [ordered]@{} +foreach ($key in $planIdentity.Keys) { + $planDocument[$key] = $planIdentity[$key] +} +$planDocument["plan_sha256"] = $planSha256 +$planPayload = ConvertTo-CompactJson $planDocument +$planPath = Join-Path $packageStateRoot "offline-launch-plan-$packageSha256.json" +$planFile = Write-ExactFile $planPath $planPayload "installed LAB offline launch plan" + +$occupied = @(& docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}") +if ($LASTEXITCODE -ne 0 -or $occupied.Count -ne 0) { + throw "installed LAB offline validation container name is occupied" +} +$dockerArguments = @( + "run", "--rm", + "--name", $containerName, + "--network", "none", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", + "--read-only", + "--pids-limit", "128" +) +foreach ($mount in $mounts) { + $specification = "type=bind,source=$([string]$mount.source),target=$([string]$mount.target)" + if ([bool]$mount.read_only) { + $specification += ",readonly" + } + $dockerArguments += @("--mount", $specification) +} +foreach ($variable in $environment) { + $dockerArguments += @("-e", "$([string]$variable.name)=$([string]$variable.value)") +} +$dockerArguments += "sha256:$agentImageSha256" +$dockerArguments += $command +$output = @(& docker @dockerArguments) +if ($LASTEXITCODE -ne 0) { + throw "installed LAB offline validation container failed" +} +# Worker receipts are canonical UTF-8 JSON followed by one LF. Rebuilding +# Docker stdout with the host newline would make the receipt identity depend +# on whether the launcher runs on Windows or Linux. +$receiptPayload = [string]::Join("`n", $output) + "`n" +try { + $validation = $receiptPayload | ConvertFrom-Json +} +catch { + throw "installed LAB offline validation receipt is not valid JSON" +} +if ( + [string]$validation.schema_version -cne "missioncore.observatory-installed-lab-worker-validation/v1" -or + [string]$validation.state -cne "ready" -or + [bool]$validation.checks.backend_contacted -or + [bool]$validation.checks.claim_attempted -or + @($validation.packages).Count -ne 1 -or + [string]$validation.packages[0].package_sha256 -cne $packageSha256 -or + @($validation.supported_executor_identities).Count -ne 1 -or + (Get-PayloadSha256 $receiptPayload) -cne [string]$receiptDocument.validation_receipt_file_sha256 -or + [string]$validation.receipt_sha256 -cne [string]$receiptDocument.validation_receipt_sha256 +) { + throw "installed LAB offline validation receipt changed" +} +$validationPath = Join-Path $packageStateRoot "offline-validation-$packageSha256.json" +$validationFile = Write-ExactFile ` + $validationPath ` + $receiptPayload ` + "installed LAB offline validation receipt" +if (@(& docker ps -a --filter "name=^/$containerName$" --format "{{.ID}}")) { + throw "installed LAB offline validation container was not removed" +} + +[ordered]@{ + backend_contacted = $false + claim_attempted = $false + package_id = $packageId + package_sha256 = $packageSha256 + plan_file_sha256 = Get-Sha256 $planFile + plan_path = $planFile + plan_sha256 = $planSha256 + queue_started = $false + schema_version = "missioncore.observatory-installed-lab-offline-launch-result/v1" + state = "ready" + validation_receipt_file_sha256 = Get-Sha256 $validationFile + validation_receipt_path = $validationFile + validation_receipt_sha256 = [string]$validation.receipt_sha256 + verified_image_count = @($validation.verified_image_sha256s).Count + worker_id = $WorkerId +} | ConvertTo-Json -Compress -Depth 6 diff --git a/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 index 05cbbc0..34882b3 100644 --- a/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 +++ b/experiments/perception/worker/observatory_portable/Seal-LabV1EomtTreeAssets.ps1 @@ -64,7 +64,7 @@ $AssetRelativeRoots = [ordered]@{ "assets\observatory-portable\eomt-model-cache-cityscapes-large-1024-v1" ) "eomt-python-environment" = "derived\perception-p0-env-v1" - "eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v1" + "eomt-runner-bundle" = "assets\observatory-portable\eomt-runner-bundle-v2" "eomt-transformers-environment" = "derived\perception-p0-transformers4576-v1" "k1-valid-fov-root" = ( "assets\observatory-portable\" + diff --git a/experiments/perception/worker/observatory_portable/Start-ObservatoryInstalledLabWorker.ps1 b/experiments/perception/worker/observatory_portable/Start-ObservatoryInstalledLabWorker.ps1 new file mode 100644 index 0000000..0614a20 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/Start-ObservatoryInstalledLabWorker.ps1 @@ -0,0 +1,302 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$ActivationPath, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedActivationFileSha256, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[A-Za-z]:\\")] + [string]$OfflineValidationScriptPath, + + [Parameter(Mandatory = $true)] + [ValidatePattern("^[a-f0-9]{64}$")] + [string]$ExpectedOfflineValidationScriptSha256 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" +$ProgressPreference = "SilentlyContinue" + +$WorkerId = "worker-006" +$ExpectedComputer = "DESKTOP-OPJ8J04" +$RuntimeRoot = "D:\NDC_MISSIONCORE\runtime" +$ContainerName = "ndc-observatory-installed-lab-worker-agent" +$TokenVolume = "ndc-observatory-worker-token-v1" +$TokenName = "observatory-worker.token" +$DefinitionsName = "observatory-portable-run-definitions.json" +$RuntimeRegistryName = "observatory-worker-runtime-candidates.json" +$PackageRegistryName = "observatory-installed-lab-packages.json" +$AssetBindingsName = "observatory-installed-lab-asset-bindings.json" +$InstallationReceiptName = "installation-receipt.json" + +function Resolve-DFile([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: file" + } + return $item.FullName +} + +function Resolve-DDirectory([string]$Path, [string]$Label) { + $item = Get-Item -LiteralPath (Resolve-Path -LiteralPath $Path).Path -Force + if ( + -not $item.PSIsContainer -or + ($item.Attributes -band [IO.FileAttributes]::ReparsePoint) -or + [IO.Path]::GetPathRoot($item.FullName).TrimEnd("\") -ine "D:" + ) { + throw "$Label must be a real D: directory" + } + return $item.FullName.TrimEnd("\") +} + +function Get-Sha256([string]$Path) { + return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() +} + +function Read-Json([string]$Path, [string]$Label) { + $resolved = Resolve-DFile $Path $Label + try { + return Get-Content -LiteralPath $resolved -Raw | ConvertFrom-Json + } + catch { + throw "$Label is not valid JSON" + } +} + +function Assert-Authority([object]$Value, [string]$Label) { + if ( + $null -eq $Value -or + [bool]$Value.actuation_allowed -or + [bool]$Value.commands_enabled -or + [bool]$Value.navigation_or_safety_accepted -or + [bool]$Value.production_accepted + ) { + throw "$Label grants forbidden authority" + } +} + +if ($env:COMPUTERNAME -cne $ExpectedComputer) { + throw "installed LAB online launcher is pinned to Worker 006" +} + +$validationScript = Resolve-DFile ` + $OfflineValidationScriptPath ` + "installed LAB offline validation script" +if ((Get-Sha256 $validationScript) -cne $ExpectedOfflineValidationScriptSha256) { + throw "installed LAB offline validation script identity changed" +} +$activationFile = Resolve-DFile $ActivationPath "installed LAB activation pointer" +if ((Get-Sha256 $activationFile) -cne $ExpectedActivationFileSha256) { + throw "installed LAB activation file identity changed" +} + +# The online service is allowed to start only after the same immutable activation +# passes the credential-free, network-free package and image validation boundary. +$validationOutput = @( + & powershell.exe -NoProfile -NonInteractive -File $validationScript ` + -ActivationPath $activationFile ` + -ExpectedActivationFileSha256 $ExpectedActivationFileSha256 +) +if ($LASTEXITCODE -ne 0) { + throw "installed LAB offline validation failed" +} +try { + $validation = [string]::Join([Environment]::NewLine, $validationOutput) | + ConvertFrom-Json +} +catch { + throw "installed LAB offline validation output is invalid" +} +if ( + [string]$validation.schema_version -cne + "missioncore.observatory-installed-lab-offline-launch-result/v1" -or + [string]$validation.state -cne "ready" -or + [bool]$validation.backend_contacted -or + [bool]$validation.claim_attempted -or + [bool]$validation.queue_started +) { + throw "installed LAB offline validation did not remain offline" +} + +$activation = Read-Json $activationFile "installed LAB activation pointer" +Assert-Authority $activation.authority "installed LAB activation authority" +if ( + [string]$activation.schema_version -cne + "missioncore.observatory-installed-lab-activation/v1" -or + [string]$activation.state -cne "installed-ready-offline" -or + [string]$activation.worker_id -cne $WorkerId -or + [string]$activation.current.package_sha256 -cne + [string]$validation.package_sha256 +) { + throw "installed LAB activation pointer changed after validation" +} + +$releaseRoot = Resolve-DDirectory ` + ([string]$activation.current.release_root) ` + "installed LAB release root" +$expectedReleasePrefix = "$RuntimeRoot\releases\observatory-installed-labs\" +if (-not $releaseRoot.StartsWith($expectedReleasePrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "installed LAB release root is outside the canonical runtime" +} +$receiptPath = Resolve-DFile ` + ([string]$activation.current.installation_receipt_path) ` + "installed LAB installation receipt" +if ( + $receiptPath -cne (Join-Path $releaseRoot $InstallationReceiptName) -or + (Get-Sha256 $receiptPath) -cne + [string]$activation.current.installation_receipt_file_sha256 +) { + throw "installed LAB installation receipt identity changed" +} +$receipt = Read-Json $receiptPath "installed LAB installation receipt" +Assert-Authority $receipt.authority "installed LAB installation authority" +if ( + [string]$receipt.schema_version -cne + "missioncore.observatory-installed-lab-installation/v1" -or + [string]$receipt.state -cne "installed-ready-offline" -or + [string]$receipt.worker_id -cne $WorkerId -or + [string]$receipt.package_sha256 -cne [string]$validation.package_sha256 -or + [string]$receipt.receipt_sha256 -cne + [string]$activation.current.installation_receipt_sha256 +) { + throw "installed LAB installation receipt changed after validation" +} +$workRoot = Resolve-DDirectory ([string]$receipt.work_root) "installed LAB work root" +$expectedWorkPrefix = "$RuntimeRoot\services\observatory-installed-labs\" +if (-not $workRoot.StartsWith($expectedWorkPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "installed LAB work root is outside the canonical runtime" +} + +$packageRegistry = Read-Json ` + (Join-Path $releaseRoot $PackageRegistryName) ` + "installed LAB package registry" +if ( + [string]$packageRegistry.schema_version -cne + "missioncore.observatory-installed-lab-package-registry/v1" -or + @($packageRegistry.packages).Count -ne 1 +) { + throw "installed LAB package registry must contain one package" +} +$package = @($packageRegistry.packages)[0] +Assert-Authority $package.authority "installed LAB package authority" +$packageSha256 = [string]$package.package_sha256 +$agentImageSha256 = [string]$package.executor_identity.image_sha256 +if ( + $packageSha256 -cne [string]$validation.package_sha256 -or + $agentImageSha256 -notmatch "^[a-f0-9]{64}$" +) { + throw "installed LAB package identity changed after validation" +} +$image = @((& docker image inspect "sha256:$agentImageSha256") | ConvertFrom-Json) +if ( + $LASTEXITCODE -ne 0 -or + $image.Count -ne 1 -or + [string]$image[0].Id -cne "sha256:$agentImageSha256" +) { + throw "installed LAB agent image identity changed" +} + +$volume = @(& docker volume inspect $TokenVolume 2>$null) +if ($LASTEXITCODE -ne 0 -or $volume.Count -eq 0) { + throw "shared Observatory Worker token volume is absent" +} +$secretCheck = @( + & docker run --rm --network none --read-only --cap-drop ALL ` + --security-opt no-new-privileges --pids-limit 32 ` + --mount "type=volume,source=$TokenVolume,target=/run/secrets,readonly" ` + --entrypoint /bin/sh "sha256:$agentImageSha256" -ec ` + "test -f /run/secrets/$TokenName; test ! -L /run/secrets/$TokenName; test -s /run/secrets/$TokenName" +) +if ($LASTEXITCODE -ne 0 -or $secretCheck.Count -ne 0) { + throw "shared Observatory Worker token volume is invalid" +} + +$existing = @(& docker ps -a --filter "name=^/$ContainerName$" --format "{{.ID}}") +if ($LASTEXITCODE -ne 0) { + throw "installed LAB Worker container lookup failed" +} +if ($existing.Count -ne 0) { + $inspection = @((& docker inspect $ContainerName) | ConvertFrom-Json) + if ( + $inspection.Count -ne 1 -or + [string]$inspection[0].Image -cne "sha256:$agentImageSha256" -or + [string]$inspection[0].Config.Labels."com.nodedc.package-sha256" -cne + $packageSha256 -or + -not [bool]$inspection[0].State.Running + ) { + throw "installed LAB Worker container requires explicit reconciliation" + } + [ordered]@{ + container_id = [string]$inspection[0].Id + container_name = $ContainerName + package_sha256 = $packageSha256 + schema_version = "missioncore.observatory-installed-lab-online-launch-result/v1" + state = "already-running" + worker_id = $WorkerId + } | ConvertTo-Json -Compress + exit 0 +} + +$dockerArguments = @( + "run", "-d", + "--name", $ContainerName, + "--restart", "unless-stopped", + "--read-only", + "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", + "--pids-limit", "256", + "--mount", "type=bind,source=$releaseRoot,target=/release,readonly", + "--mount", "type=bind,source=$RuntimeRoot,target=/runtime,readonly", + "--mount", "type=bind,source=$workRoot,target=/work", + "--mount", "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock", + "--mount", "type=volume,source=$TokenVolume,target=/run/secrets,readonly", + "--label", "com.nodedc.authority=observation-only", + "--label", "com.nodedc.component=installed-lab-worker-agent", + "--label", "com.nodedc.managed-by=mission-core-worker", + "--label", "com.nodedc.package-sha256=$packageSha256", + "--label", "com.nodedc.product=mission-core", + "--label", "com.nodedc.stack=observatory", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL=http://127.0.0.1:18080", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_TOKEN_FILE=/run/secrets/$TokenName", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_WORK_ROOT=/work", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_DEFINITIONS_FILE=/release/$DefinitionsName", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_RUNTIME_REGISTRY_FILE=/release/$RuntimeRegistryName", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_REGISTRY_FILE=/release/$PackageRegistryName", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_PACKAGE_ASSET_BINDINGS_FILE=/release/$AssetBindingsName", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_IDLE_POLL_SECONDS=1", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_TRANSPORT_BACKOFF_SECONDS=5", + "-e", "MISSIONCORE_OBSERVATORY_WORKER_MAX_TRANSPORT_FAILURES=120", + "-e", "PYTHONDONTWRITEBYTECODE=1", + "sha256:$agentImageSha256" +) +$containerId = [string](& docker @dockerArguments) +if ($LASTEXITCODE -ne 0 -or $containerId -notmatch "^[a-f0-9]{12,128}$") { + throw "installed LAB Worker container failed to start" +} +Start-Sleep -Seconds 2 +$inspection = @((& docker inspect $ContainerName) | ConvertFrom-Json) +if ( + $inspection.Count -ne 1 -or + [string]$inspection[0].Id -cne $containerId -or + -not [bool]$inspection[0].State.Running -or + [int]$inspection[0].RestartCount -ne 0 +) { + throw "installed LAB Worker container did not remain running" +} + +[ordered]@{ + container_id = $containerId + container_name = $ContainerName + package_sha256 = $packageSha256 + schema_version = "missioncore.observatory-installed-lab-online-launch-result/v1" + state = "running" + worker_id = $WorkerId +} | ConvertTo-Json -Compress diff --git a/experiments/perception/worker/observatory_portable/initialize_observatory_worker_token_volume.sh b/experiments/perception/worker/observatory_portable/initialize_observatory_worker_token_volume.sh new file mode 100644 index 0000000..d209905 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/initialize_observatory_worker_token_volume.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu + +source=/source-secret/observatory-worker.token +target=/target-secret/observatory-worker.token +test -f "$source" +test ! -L "$source" +bytes="$(wc -c <"$source")" +test "$bytes" -ge 32 +test "$bytes" -le 512 +test "$(tr -d '\r\n' <"$source" | wc -c)" = "$bytes" +if test -e "$target"; then + test -f "$target" + test ! -L "$target" + cmp -s "$source" "$target" +else + cp "$source" "$target" +fi +chmod 0400 "$target" +test "$(wc -c <"$target")" = "$bytes" diff --git a/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json b/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json index 146a908..57d4cc8 100644 --- a/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json +++ b/experiments/perception/worker/observatory_portable/lab-v1-eomt-ddrnet-executor-candidate.json @@ -1,218 +1 @@ -{ - "assets": [ - { - "asset_id": "ddrnet-checkpoint", - "byte_length": 259419077, - "kind": "model-artifact", - "repository_path": null, - "sha256": "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" - }, - { - "asset_id": "ddrnet-goose-image", - "byte_length": null, - "kind": "container-image", - "repository_path": null, - "sha256": "591cb382c099eeb05e7ec16e2371e0b2da54d2bb5c49ec0f4ac88dbf72b0f0cd" - }, - { - "asset_id": "ddrnet-goose-mapping", - "byte_length": null, - "kind": "runtime-artifact", - "repository_path": null, - "sha256": "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f" - }, - { - "asset_id": "ddrnet-goose-runner", - "byte_length": 32877, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/run_goose_vegetation_benchmark.py", - "sha256": "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1" - }, - { - "asset_id": "ddrnet-image-dockerfile", - "byte_length": 1793, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/lab_v1_vegetation_goose/Dockerfile", - "sha256": "8203fd01e05d8f5bcce11c328dd39dbb6b97df54ca5d9690706536bafe3d3bad" - }, - { - "asset_id": "ddrnet-portable-config", - "byte_length": 4324, - "kind": "repository-file", - "repository_path": "config/perception/lab-v1-eomt-ddrnet-portable-v2.json", - "sha256": "c0ff657dadc86607d77d232e84d041fbf2d8b63e86d02319e9cd607220d00f21" - }, - { - "asset_id": "eomt-config-json", - "byte_length": 1575, - "kind": "model-artifact", - "repository_path": null, - "sha256": "7f4aa94fa4e43c0dbd79a5420edb511120aef62bd82bfbcbcece79948286a650" - }, - { - "asset_id": "eomt-dependency-set", - "byte_length": null, - "kind": "definition-component", - "repository_path": null, - "sha256": "4eb1f8d33236806e74f9e5bb96b7dce2ac37623dc39b2184be2aa8d7d00e983e" - }, - { - "asset_id": "eomt-evaluation-helper", - "byte_length": 28915, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/run_evaluation_prelabels.py", - "sha256": "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4" - }, - { - "asset_id": "eomt-image", - "byte_length": null, - "kind": "container-image", - "repository_path": null, - "sha256": "58df7489c3f2276f9591d500a012dee03e23d35543ce3c390b4c001e6bf90794" - }, - { - "asset_id": "eomt-model-weights", - "byte_length": 1276175488, - "kind": "model-artifact", - "repository_path": null, - "sha256": "c265da9a74f58f5c3f4826d23ca4ca78beac0b106cca5842beca61580de5b782" - }, - { - "asset_id": "eomt-orchestrator", - "byte_length": 21489, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/Invoke-E4FullSessionSegmentation.ps1", - "sha256": "d3e9435939444ab35b27a744ac314e289ebd66a13fa56e3d59f121e088d22774" - }, - { - "asset_id": "eomt-preprocessor-config", - "byte_length": 666, - "kind": "model-artifact", - "repository_path": null, - "sha256": "97e2fbf7f0bdba2cfc90251c5133bae9c27ddc9c4410509f40670be2332854e7" - }, - { - "asset_id": "eomt-profile", - "byte_length": 3805, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/e3_k1_camera1_profile.json", - "sha256": "ea583966bc3409f5cf563cbf4fad05e366907e67187082eb692aff53d9f5d875" - }, - { - "asset_id": "eomt-profile-runtime", - "byte_length": 45789, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/run_e3_rectified_segmentation.py", - "sha256": "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6" - }, - { - "asset_id": "eomt-recorded-runtime", - "byte_length": 34899, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/run_recorded_perception_epoch.py", - "sha256": "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675" - }, - { - "asset_id": "eomt-runner", - "byte_length": 30720, - "kind": "repository-file", - "repository_path": "experiments/perception/worker/run_e4_full_session_segmentation.py", - "sha256": "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4" - }, - { - "asset_id": "k1-calibration", - "byte_length": null, - "kind": "definition-component", - "repository_path": null, - "sha256": "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" - }, - { - "asset_id": "k1-valid-fov-identity", - "byte_length": null, - "kind": "definition-component", - "repository_path": null, - "sha256": "b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2" - }, - { - "asset_id": "k1-valid-fov-mask", - "byte_length": null, - "kind": "definition-component", - "repository_path": null, - "sha256": "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" - }, - { - "asset_id": "lab-v1-portable-contracts", - "byte_length": 115196, - "kind": "repository-file", - "repository_path": "src/k1link/observatory/portable_lab_v1_executor.py", - "sha256": "2a18fe862f0934c78e86156708d8b80a3c31644a68829c8e673a0f48538f3473" - }, - { - "asset_id": "lab-v1-portable-worker", - "byte_length": 47392, - "kind": "repository-file", - "repository_path": "src/k1link/observatory/portable_lab_v1_worker.py", - "sha256": "56aa40390b8413fdf7f7293329e89420305914abd478821c08bfe8b0a2ec3df8" - }, - { - "asset_id": "portable-result-contracts", - "byte_length": 22008, - "kind": "repository-file", - "repository_path": "src/k1link/observatory/portable_result_contract.py", - "sha256": "936d6f20789c26e9bed1c9e34ee51259368eb95bc7e3aa549821d1f7de79030c" - }, - { - "asset_id": "portable-worker-runtime", - "byte_length": 39120, - "kind": "repository-file", - "repository_path": "src/k1link/observatory/portable_worker_runtime.py", - "sha256": "205d32116f1ba75d5257d15cc79b5146574d93acd52028a5950f6fca4d8129cb" - }, - { - "asset_id": "vegetation-policy", - "byte_length": 3022, - "kind": "repository-file", - "repository_path": "config/perception/lab-v1-vegetation-mission-policy-v1.json", - "sha256": "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35" - }, - { - "asset_id": "vegetation-provider-map", - "byte_length": 2756, - "kind": "repository-file", - "repository_path": "config/perception/lab-v1-vegetation-provider-label-map-v1.json", - "sha256": "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352" - } - ], - "authority": { - "actuation_allowed": false, - "commands_enabled": false, - "navigation_or_safety_accepted": false, - "production_accepted": false - }, - "candidate_sha256": "9d40369e393aea4b1a09a5784336442a082b1b6126afaac379f1ecece39b54f9", - "declared_blockers": [ - "combined-executor-entrypoint-uninstalled", - "combined-executor-image-unsealed", - "commit-bound-source-unavailable", - "ddrnet-component-port-uninstalled", - "eomt-component-port-uninstalled", - "fixture-smoke-unaccepted", - "worker-installation-receipt-unavailable" - ], - "definition_id": "lab-v1-eomt-ddrnet-portable", - "definition_contract_sha256": "504e197db72e8bad258a2e4ec117d6d8247297847bd651470f8d43601e2e1fc7", - "definition_version": 2, - "executor_image_sha256": null, - "phases": [ - "source-materialization", - "eomt-full-session", - "ddrnet-full-session", - "result-v2-assembly", - "result-v2-validation", - "portable-result-packaging" - ], - "release_id": "lab-v1-eomt-ddrnet-worker006-candidate-v2", - "result_contract_sha256": "b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a", - "schema_version": "missioncore.observatory-portable-lab-v1-executor-candidate/v2", - "setup_id": "lab-v1-eomt-ddrnet-portable-v1" -} +{"assets":[{"asset_id":"agent-image","byte_length":null,"kind":"container-image","repository_path":null,"sha256":"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373"},{"asset_id":"ddrnet-checkpoint","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6"},{"asset_id":"ddrnet-goose-mapping","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f"},{"asset_id":"ddrnet-goose-runner","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1"},{"asset_id":"ddrnet-step-image","byte_length":null,"kind":"container-image","repository_path":null,"sha256":"e6c986100613ec804f0e0076ca8695abf43ff88ef9d5d85f6857e0b41db74051"},{"asset_id":"eomt-environment","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"8c8f343a5368ff17edbb58defa1669f6eccfba767aab897a23693872070ab9e0"},{"asset_id":"eomt-ffmpeg-runtime","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69"},{"asset_id":"eomt-model-cache","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"064870e58814b97027d6a7ccd553bf51f5b8e6a8ad82a1cc703584d2dca5690c"},{"asset_id":"eomt-python-environment","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"b3f4efc53af491f174b1cff74b3ba03016e67c9c5c74257b49c6e7dd7d853f20"},{"asset_id":"eomt-runner-bundle","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"0d08f0492d5ad62903874ea224c505e54a6f6059c8283f586bc79e42d55156b7"},{"asset_id":"eomt-step-image","byte_length":null,"kind":"container-image","repository_path":null,"sha256":"adba3dc8c97b161ba261ec44fca9ebe1680f117d1bcb1481440172cc3331a174"},{"asset_id":"eomt-transformers-environment","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"f365de01426a33be51a310923c743655634d0868941bbf3f1aae1647fdeadfc9"},{"asset_id":"k1-valid-fov-root","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"f4fc2053e4e6213bb364c8773979b755d5682a81b3946c25ff86274bc5f0031e"},{"asset_id":"result-assembler","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"00e041071bf605dcc90439b74e63043585608043a811c7e97304003f254b6f89"},{"asset_id":"source-tree","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"d464a498cf0d17a86dd6b5282a5bba2d80c75a49a48ad9e92a045e55dc06b2e3"},{"asset_id":"vegetation-policy","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35"},{"asset_id":"vegetation-provider-map","byte_length":null,"kind":"runtime-artifact","repository_path":null,"sha256":"f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352"}],"authority":{"actuation_allowed":false,"commands_enabled":false,"navigation_or_safety_accepted":false,"production_accepted":false},"candidate_sha256":"2a83b79b88c2c449ce3e82f43dbf52e7c350f61cc8dc094c4b1e62cd8437b1e6","declared_blockers":[],"definition_contract_sha256":"cb9cafaf88c5a48f8f2cafb1c4a8c5a3b297168f6da27d57df527e8e30028271","definition_id":"lab-v1-eomt-ddrnet-portable","definition_version":2,"executor_image_sha256":"5ad7d95baac63af13812cb693d492add4e806a333aba8e60edb2ea1aba754373","phases":["prepare-source","eomt-step","ddrnet-step","assemble-result"],"release_id":"lab-v1-installed-package-v1","result_contract_sha256":"b3dfaa8e20a0f22fc510d062ac469f010a3281c650059d9ea134f0b3ccb38d9a","schema_version":"missioncore.observatory-portable-lab-v1-executor-candidate/v2","setup_id":"lab-v1-eomt-ddrnet-portable-v1"} diff --git a/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py b/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py index f2226dc..aed37ac 100644 --- a/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py +++ b/experiments/perception/worker/observatory_portable/portable_lab_v1_component_adapter.py @@ -37,7 +37,6 @@ AUTHORITY: Final = { "navigation_or_safety_accepted": False, "production_accepted": False, } -CAMERA_SOURCE_ID: Final = "sensor.camera.right" CALIBRATION_SHA256: Final = "05f3ad9b38b3a4fc95388a8ec83da83c745e217709e51787b3d5aad0969f6fa9" EOMT_PIPELINE: Final = "recorded-semantic-eomt-fisheye-mask/v1" DDRNET_CHECKPOINT_SHA256: Final = "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6" @@ -51,6 +50,8 @@ VALID_FOV_IDENTITY_SHA256: Final = ( VALID_FOV_MASK_SHA256: Final = "a40cee06b7c6f69b6a09a11563dcfd237f3de833b1ccd31459e66692e528ba63" FIXED_REQUEST_PATH: Final = "/run/nodedc/request.json" +PACKAGE_OUTPUT_ROOT: Final = "/missioncore/output" +PACKAGE_STEP_INPUT_ROOT: Final = "/missioncore/input/steps" MAX_REQUEST_BYTES: Final = 2 * 1024 * 1024 MAX_JOB_BYTES: Final = 32 * 1024 * 1024 MAX_JOB_FILES: Final = 100_003 @@ -161,6 +162,24 @@ class RuntimeLayout: asset_paths={item.asset_id: Path(item.path) for item in expectations}, ) + @classmethod + def installed_package( + cls, + component: Component, + expectations: Sequence[AssetExpectation], + ) -> RuntimeLayout: + if component not in ("eomt", "ddrnet"): + raise ComponentAdapterError("installed package component is invalid") + prepared = Path(PACKAGE_STEP_INPUT_ROOT) / "prepare" + return cls( + request=prepared / f"{component}-request.json", + camera_job_root=prepared / "camera-job", + output_root=Path(PACKAGE_OUTPUT_ROOT), + effective_ddrnet_config=prepared / "effective-ddrnet-config.json", + eomt_result_root=Path(PACKAGE_STEP_INPUT_ROOT) / "eomt", + asset_paths={item.asset_id: Path(item.path) for item in expectations}, + ) + def canonical_json(value: object) -> bytes: return json.dumps( @@ -210,6 +229,18 @@ def parse_fixed_request_argument(argv: Sequence[str]) -> Path: return Path(FIXED_REQUEST_PATH) +def resolve_runtime_layout( + argv: Sequence[str], + *, + component: Component, + expectations: Sequence[AssetExpectation], +) -> RuntimeLayout: + if tuple(argv) == ("--package-step", component): + return RuntimeLayout.installed_package(component, expectations) + parse_fixed_request_argument(argv) + return RuntimeLayout.fixed(expectations) + + def load_component_request( request_path: Path, *, @@ -750,9 +781,13 @@ def _source_binding(value: object) -> SourceBinding: frame_count = _positive_int(camera["frame_count"], "camera frame count") start = _finite_float(camera["timeline_start_seconds"], "camera timeline start") end = _finite_float(camera["timeline_end_seconds"], "camera timeline end") + # ``camera_source_id`` is the public, archive-owned channel identity. It + # can legitimately differ between recordings; the admitted equipment and + # physical channel were already checked by the backend and prepare step. + # At this boundary it must be safe and match the sealed CameraComputeJob, + # but the model package must not carry a device-specific allowlist. if ( - camera_source_id != CAMERA_SOURCE_ID - or camera.get("calibration_sha256") != CALIBRATION_SHA256 + camera.get("calibration_sha256") != CALIBRATION_SHA256 or frame_count > MAX_FRAME_COUNT or input_bytes > MAX_SOURCE_BYTES or end <= start @@ -775,7 +810,7 @@ def _source_binding(value: object) -> SourceBinding: def _paths(value: object, component: Component) -> dict[str, str | None]: document = _object(value, "component paths") - expected = { + legacy = { "camera_job_root": "/run/nodedc/source", "request": FIXED_REQUEST_PATH, "output_root": "/run/nodedc/output", @@ -789,8 +824,25 @@ def _paths(value: object, component: Component) -> dict[str, str | None]: else "/run/nodedc/eomt-result/source-frames" ), } - _exact_keys(document, set(expected), "component paths") - if document != expected: + prepared = f"{PACKAGE_STEP_INPUT_ROOT}/prepare" + installed = { + "camera_job_root": f"{prepared}/camera-job", + "request": f"{prepared}/{component}-request.json", + "output_root": PACKAGE_OUTPUT_ROOT, + "effective_ddrnet_config": ( + f"{prepared}/effective-ddrnet-config.json" if component == "ddrnet" else None + ), + "eomt_result_root": ( + f"{PACKAGE_STEP_INPUT_ROOT}/eomt" if component == "ddrnet" else None + ), + "decoded_frames_root": ( + f"{PACKAGE_OUTPUT_ROOT}/source-frames" + if component == "eomt" + else f"{PACKAGE_STEP_INPUT_ROOT}/eomt/source-frames" + ), + } + _exact_keys(document, set(legacy), "component paths") + if document not in (legacy, installed): raise ComponentAdapterError("component path contract changed") return cast(dict[str, Optional[str]], document) # noqa: UP045 diff --git a/experiments/perception/worker/observatory_portable/promote_installed_lab_v1_package.py b/experiments/perception/worker/observatory_portable/promote_installed_lab_v1_package.py new file mode 100644 index 0000000..84ae111 --- /dev/null +++ b/experiments/perception/worker/observatory_portable/promote_installed_lab_v1_package.py @@ -0,0 +1,1008 @@ +"""Generate one ready, generic installed-package release for portable LAB V1.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Final, cast + +import httpx + +from k1link.observatory.installed_lab_packages import ( + INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA, + InstalledLabContainer, + InstalledLabPackageMount, + seal_installed_lab_package, +) +from k1link.observatory.installed_lab_worker_service import ( + INSTALLED_LAB_ASSET_BINDINGS_SCHEMA, + InstalledLabWorkerAssetBindings, + InstalledLabWorkerCompositionError, +) +from k1link.observatory.portable_lab_v1_executor import ( + PORTABLE_LAB_V1_RELEASE_SCHEMA, + PortableLabV1ReleaseAsset, + PortableLabV1ReleaseCandidate, +) +from k1link.observatory.portable_result_contract import ( + OBSERVATION_ONLY_AUTHORITY, + canonical_json, +) +from k1link.observatory.portable_run_definitions import ( + PORTABLE_RUN_DEFINITION_REGISTRY_SCHEMA, + PortableExecutorAvailability, + PortableRunDefinition, + PortableRunDefinitionRegistry, + canonical_sha256, +) +from k1link.observatory.portable_worker_runtime import ( + PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA, + PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA, + PortableWorkerAssetRequirement, + PortableWorkerExecutorSeal, + PortableWorkerRuntimeCandidate, + PortableWorkerRuntimePhase, + PortableWorkerRuntimeRegistry, + RuntimeAssetKind, +) + +LAB_V1_SETUP_ID: Final = "lab-v1-eomt-ddrnet-portable-v1" +LAB_V1_RELEASE_ID: Final = "lab-v1-installed-package-v1" +LAB_V1_PACKAGE_ID: Final = "lab-v1-eomt-ddrnet-package-v1" +LAB_V1_ADAPTER_ID: Final = "lab-v1-installed-package-worker006-v1" +LAB_V1_CONTRACT_SCHEMA: Final = "missioncore.observatory-lab-v1-installed-package-contract/v1" +_SHA256 = re.compile(r"^[a-f0-9]{64}$") +_ENGINE_PATH_PLACEHOLDER = re.compile( + r"(?:\$[A-Za-z_][A-Za-z0-9_]*|%[A-Za-z_][A-Za-z0-9_]*%|" + r"\{[A-Za-z_][A-Za-z0-9_]*\})" +) +ImageResolver = Callable[[str], str] +_DOCKER_SOCKET: Final = Path("/var/run/docker.sock") +_DOCKER_API_VERSION: Final = "v1.45" + +_EOMT_ASSETS: Final = ( + ( + "eomt-environment", + "/environment", + "8c8f343a5368ff17edbb58defa1669f6eccfba767aab897a23693872070ab9e0", + 211_776_082, + ), + ( + "eomt-ffmpeg-runtime", + "/opt/ffmpeg", + "03651449fdcccec847a0f1241e1663a82cf374bd94e7470b4ddb0c0e46d88c69", + 256_208_352, + ), + ( + "eomt-model-cache", + "/cache", + "064870e58814b97027d6a7ccd553bf51f5b8e6a8ad82a1cc703584d2dca5690c", + 2_552_355_458, + ), + ( + "eomt-python-environment", + "/opt/env", + "b3f4efc53af491f174b1cff74b3ba03016e67c9c5c74257b49c6e7dd7d853f20", + 5_120_848_705, + ), + ( + "eomt-runner-bundle", + "/runner", + "0d08f0492d5ad62903874ea224c505e54a6f6059c8283f586bc79e42d55156b7", + 145_141, + ), + ( + "eomt-transformers-environment", + "/opt/transformers", + "f365de01426a33be51a310923c743655634d0868941bbf3f1aae1647fdeadfc9", + 225_272_284, + ), + ( + "k1-valid-fov-root", + "/valid-fov", + "f4fc2053e4e6213bb364c8773979b755d5682a81b3946c25ff86274bc5f0031e", + 6_019, + ), +) +_DDRNET_ASSETS: Final = ( + ( + "ddrnet-checkpoint", + "/opt/nodedc/assets/ddrnet-checkpoint", + "b99c2838051bcd7b092fd3970aa62a77d5c0bbb809c9b9afb2ff4b0ebdaa4ee6", + 259_419_077, + ), + ( + "ddrnet-goose-mapping", + "/opt/nodedc/assets/ddrnet-goose-mapping", + "88ae319ba5a3877dd3ae0773f693a6a5fdc283934140de9dfaff029108aefd7f", + 1_427, + ), + ( + "ddrnet-goose-runner", + "/opt/nodedc/assets/ddrnet-goose-runner", + "b18ad60f277eea69a240a28f290611b94627fb9707faf1bb3e6e22102dad67c1", + 32_877, + ), + ( + "vegetation-policy", + "/opt/nodedc/assets/vegetation-policy", + "b75c4ac841d7b4bcc57f7a9c8417ca2317d8ecfa499e72a9af8a8591a2ec0d35", + 3_022, + ), + ( + "vegetation-provider-map", + "/opt/nodedc/assets/vegetation-provider-map", + "f2b69046b6a740fd9532d2d88e7fabae7c20fb662f783c9502adc9026406f352", + 2_756, + ), +) +_RUNTIME_ASSET_RELATIVE_PATHS: Final = { + "eomt-environment": "derived\\perception-e3-opencv413092-v1", + "eomt-ffmpeg-runtime": ( + "assets\\observatory-portable\\eomt-ffmpeg-runtime-frigate-8a364092b03561b9-v1" + ), + "eomt-model-cache": ("assets\\observatory-portable\\eomt-model-cache-cityscapes-large-1024-v1"), + "eomt-python-environment": "derived\\perception-p0-env-v1", + "eomt-runner-bundle": "assets\\observatory-portable\\eomt-runner-bundle-v2", + "eomt-transformers-environment": "derived\\perception-p0-transformers4576-v1", + "k1-valid-fov-root": ( + "assets\\observatory-portable\\" + "k1-valid-fov-root-b4dd8ddf2b87c1d520ee8a0868c4fea062d7c14d1bae73ccabd3abe1f3acbac2-v1" + ), + "ddrnet-checkpoint": ( + "assets\\observatory-portable\\lab-v1-static-files-v1\\ddrnet-checkpoint.pth" + ), + "ddrnet-goose-mapping": ( + "assets\\observatory-portable\\lab-v1-static-files-v1\\ddrnet-goose-mapping.csv" + ), + "ddrnet-goose-runner": ( + "assets\\observatory-portable\\lab-v1-static-files-v1\\ddrnet-goose-runner.py" + ), + "vegetation-policy": ( + "assets\\observatory-portable\\lab-v1-static-files-v1\\vegetation-policy.json" + ), + "vegetation-provider-map": ( + "assets\\observatory-portable\\lab-v1-static-files-v1\\vegetation-provider-map.json" + ), +} + + +class InstalledLabV1PromotionError(RuntimeError): + """The generic LAB V1 release could not be sealed.""" + + +def generate( + *, + source_definitions: Path, + source_runtime: Path, + ddrnet_config: Path, + result_assembler_source: Path, + output_root: Path, + source_tree_sha256: str, + agent_image_sha256: str, + eomt_image_sha256: str, + ddrnet_image_sha256: str, + engine_release_root: str, + engine_runtime_root: str, + engine_work_root: str, + image_resolver: ImageResolver, +) -> None: + if not output_root.is_absolute() or output_root.exists(): + raise InstalledLabV1PromotionError("output root must be a new absolute path") + for value, label in ( + (source_tree_sha256, "source tree"), + (agent_image_sha256, "agent image"), + (eomt_image_sha256, "EoMT image"), + (ddrnet_image_sha256, "DDRNet image"), + ): + _digest(value, label) + for value, label in ( + (engine_release_root, "Engine release root"), + (engine_runtime_root, "Engine runtime root"), + (engine_work_root, "Engine work root"), + ): + _engine_root(value, label) + for value, label in ( + (agent_image_sha256, "agent image"), + (eomt_image_sha256, "EoMT image"), + (ddrnet_image_sha256, "DDRNet image"), + ): + _verify_installed_image(value, label, image_resolver) + definitions_document = _read_object(source_definitions) + runtime_document = _read_object(source_runtime) + definitions = PortableRunDefinitionRegistry.from_file(source_definitions) + PortableWorkerRuntimeRegistry.from_file(source_runtime, definitions=definitions) + # Promotion is also the immutable upgrade path. A previously ready source + # definition keeps its executable contract while the new release replaces + # only the executor identity and derives fresh cross-registry digests. + blocked = definitions.resolve_setup(LAB_V1_SETUP_ID) + config_payload = ddrnet_config.read_bytes() + config_sha256 = hashlib.sha256(config_payload).hexdigest() + config_component = next( + item + for item in blocked.components + if item.component_id == "ddrnet-portable-runtime-config-v2" + ) + if config_sha256 != config_component.sha256: + raise InstalledLabV1PromotionError("DDRNet config differs from its definition") + result_assembler_sha256 = _sha256_file(result_assembler_source) + output_root.mkdir(mode=0o700) + + release_assets = tuple( + sorted( + ( + PortableLabV1ReleaseAsset( + "agent-image", "container-image", agent_image_sha256, None, None + ), + PortableLabV1ReleaseAsset( + "ddrnet-step-image", "container-image", ddrnet_image_sha256, None, None + ), + PortableLabV1ReleaseAsset( + "eomt-step-image", "container-image", eomt_image_sha256, None, None + ), + PortableLabV1ReleaseAsset( + "source-tree", "runtime-artifact", source_tree_sha256, None, None + ), + PortableLabV1ReleaseAsset( + "result-assembler", "runtime-artifact", result_assembler_sha256, None, None + ), + *( + PortableLabV1ReleaseAsset(asset_id, "runtime-artifact", sha256, None, None) + for asset_id, _target, sha256, _length in (*_EOMT_ASSETS, *_DDRNET_ASSETS) + ), + ), + key=lambda item: item.asset_id, + ) + ) + release_identity = { + "schema_version": "missioncore.observatory-portable-lab-v1-executor-candidate-identity/v2", + "release_id": LAB_V1_RELEASE_ID, + "setup_id": blocked.setup_id, + "definition_id": blocked.definition_id, + "definition_version": blocked.version, + "definition_contract_sha256": blocked.executable_contract_sha256, + "result_contract_sha256": blocked.result_contract.contract_sha256, + "executor_image_sha256": agent_image_sha256, + "assets": [item.as_dict() for item in release_assets], + "phases": ["prepare-source", "eomt-step", "ddrnet-step", "assemble-result"], + "declared_blockers": [], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + release = PortableLabV1ReleaseCandidate( + release_id=LAB_V1_RELEASE_ID, + setup_id=blocked.setup_id, + definition_id=blocked.definition_id, + definition_version=blocked.version, + definition_contract_sha256=blocked.executable_contract_sha256, + result_contract_sha256=blocked.result_contract.contract_sha256, + executor_image_sha256=agent_image_sha256, + assets=release_assets, + phases=("prepare-source", "eomt-step", "ddrnet-step", "assemble-result"), + declared_blockers=(), + candidate_sha256=canonical_sha256(release_identity), + repository_root=output_root, + ) + release.bind_definition(blocked) + inspection = release.inspect({item.asset_id: item.sha256 for item in release.assets}) + seal = release.seal(inspection) + ready_definition = _ready_definition(blocked, seal.release_sha256, agent_image_sha256) + ready_definitions_document = _replace_row( + definitions_document, + collection="definitions", + setup_id=LAB_V1_SETUP_ID, + replacement=_definition_document(ready_definition), + schema=PORTABLE_RUN_DEFINITION_REGISTRY_SCHEMA, + ) + definitions_path = output_root / "observatory-portable-run-definitions.json" + _write(definitions_path, ready_definitions_document) + + contract = { + "schema_version": LAB_V1_CONTRACT_SCHEMA, + "package_release_sha256": release.candidate_sha256, + "ddrnet_runner_sha256": next( + item[2] for item in _DDRNET_ASSETS if item[0] == "ddrnet-goose-runner" + ), + "result_assembler_sha256": result_assembler_sha256, + "executor": { + "release_id": LAB_V1_RELEASE_ID, + "release_sha256": seal.release_sha256, + "image_sha256": agent_image_sha256, + }, + "component_images": {"eomt": eomt_image_sha256, "ddrnet": ddrnet_image_sha256}, + "component_assets": { + "eomt": [ + _component_asset(item, kind="tree", verification="identity-sha256") + for item in _EOMT_ASSETS + ], + "ddrnet": [ + _component_asset(item, kind="file", verification="sha256") + for item in _DDRNET_ASSETS + ], + }, + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + contract_path = output_root / "lab-v1-installed-package-contract.json" + _write(contract_path, contract) + config_path = output_root / "lab-v1-eomt-ddrnet-portable-v2.json" + config_path.write_bytes(config_payload) + + requirements = _runtime_requirements( + definition=ready_definition, + agent_image_sha256=agent_image_sha256, + eomt_image_sha256=eomt_image_sha256, + ddrnet_image_sha256=ddrnet_image_sha256, + contract_path=contract_path, + definitions_path=definitions_path, + config_path=config_path, + ) + executor = PortableWorkerExecutorSeal( + release_id=LAB_V1_RELEASE_ID, + release_sha256=seal.release_sha256, + image_sha256=agent_image_sha256, + ) + phases = tuple( + PortableWorkerRuntimePhase(phase_id=value, state="implemented") + for value in ( + "source-delivery", + "prepare-source", + "eomt-step", + "ddrnet-step", + "assemble-result", + "result-publication", + ) + ) + runtime_identity = { + "schema_version": PORTABLE_WORKER_RUNTIME_CANDIDATE_SCHEMA, + "adapter_id": LAB_V1_ADAPTER_ID, + "setup_id": ready_definition.setup_id, + "definition_id": ready_definition.definition_id, + "definition_version": ready_definition.version, + "definition_sha256": ready_definition.definition_sha256, + "source_adapter_sha256": ready_definition.source_adapter.contract_sha256, + "model_manifest_sha256": ready_definition.model_manifest_sha256, + "resource_profile_sha256": ready_definition.resource_profile.profile_sha256, + "result_contract_sha256": ready_definition.result_contract.contract_sha256, + "state": "ready", + "executor": executor.as_dict(), + "reusable_assets": [item.as_dict() for item in requirements], + "phases": [item.as_dict() for item in phases], + "blockers": [], + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + } + candidate = PortableWorkerRuntimeCandidate( + adapter_id=LAB_V1_ADAPTER_ID, + setup_id=ready_definition.setup_id, + definition_id=ready_definition.definition_id, + definition_version=ready_definition.version, + definition_sha256=ready_definition.definition_sha256, + source_adapter_sha256=ready_definition.source_adapter.contract_sha256, + model_manifest_sha256=ready_definition.model_manifest_sha256, + resource_profile_sha256=ready_definition.resource_profile.profile_sha256, + result_contract_sha256=ready_definition.result_contract.contract_sha256, + state="ready", + executor=executor, + reusable_assets=requirements, + phases=phases, + blockers=(), + candidate_sha256=canonical_sha256(runtime_identity), + ) + ready_runtime_document = _replace_row( + runtime_document, + collection="candidates", + setup_id=LAB_V1_SETUP_ID, + replacement={ + **candidate.identity_document(), + "candidate_sha256": candidate.candidate_sha256, + }, + schema=PORTABLE_WORKER_RUNTIME_REGISTRY_SCHEMA, + ) + runtime_path = output_root / "observatory-worker-runtime-candidates.json" + _write(runtime_path, ready_runtime_document) + + package = seal_installed_lab_package( + package_id=LAB_V1_PACKAGE_ID, + package_version=1, + setup_id=ready_definition.setup_id, + definition_id=ready_definition.definition_id, + definition_version=ready_definition.version, + definition_sha256=ready_definition.definition_sha256, + runtime_candidate_sha256=candidate.candidate_sha256, + source_adapter_sha256=ready_definition.source_adapter.contract_sha256, + result_contract_sha256=ready_definition.result_contract.contract_sha256, + executor_identity=candidate.executor_identity(), + execution_mode="fixed-stack", + asset_ids=tuple(item.asset_id for item in requirements), + containers=_containers( + agent_image_sha256=agent_image_sha256, + eomt_image_sha256=eomt_image_sha256, + ddrnet_image_sha256=ddrnet_image_sha256, + ), + ) + packages_path = output_root / "observatory-installed-lab-packages.json" + _write( + packages_path, + { + "schema_version": INSTALLED_LAB_PACKAGE_REGISTRY_SCHEMA, + "packages": [{**package.identity_document(), "package_sha256": package.package_sha256}], + }, + ) + _write(output_root / "lab-v1-executor-release.json", _release_document(release)) + _write(output_root / "lab-v1-executor-seal.json", seal.as_dict()) + + ready_definitions = PortableRunDefinitionRegistry.from_file(definitions_path) + ready_runtime = PortableWorkerRuntimeRegistry.from_file( + runtime_path, definitions=ready_definitions + ) + from k1link.observatory.installed_lab_packages import InstalledLabPackageRegistry + + InstalledLabPackageRegistry.from_file( + packages_path, + definitions=ready_definitions, + runtime_registry=ready_runtime, + ) + _write_asset_bindings( + output_root=output_root, + requirements=requirements, + engine_release_root=engine_release_root, + engine_runtime_root=engine_runtime_root, + engine_work_root=engine_work_root, + ) + _validate_generated_bindings( + path=output_root / "observatory-installed-lab-asset-bindings.json", + requirements=requirements, + engine_work_root=engine_work_root, + ) + _write( + output_root / "promotion-summary.json", + { + "definition_sha256": ready_definition.definition_sha256, + "runtime_candidate_sha256": candidate.candidate_sha256, + "package_sha256": package.package_sha256, + "release_candidate_sha256": release.candidate_sha256, + "executor_release_sha256": seal.release_sha256, + "agent_image_sha256": agent_image_sha256, + "eomt_image_sha256": eomt_image_sha256, + "ddrnet_image_sha256": ddrnet_image_sha256, + "source_tree_sha256": source_tree_sha256, + }, + ) + + +def _runtime_requirements( + *, + definition: PortableRunDefinition, + agent_image_sha256: str, + eomt_image_sha256: str, + ddrnet_image_sha256: str, + contract_path: Path, + definitions_path: Path, + config_path: Path, +) -> tuple[PortableWorkerAssetRequirement, ...]: + requirements = [ + _requirement("agent-image", "container-image", agent_image_sha256), + _requirement("ddrnet-step-image", "container-image", ddrnet_image_sha256), + _requirement("eomt-step-image", "container-image", eomt_image_sha256), + _requirement( + "lab-v1-package-contract", + "local-file", + _sha256_file(contract_path), + contract_path.stat().st_size, + ), + _requirement( + "lab-v1-definition-registry", + "local-file", + _sha256_file(definitions_path), + definitions_path.stat().st_size, + ), + PortableWorkerAssetRequirement( + asset_id="ddrnet-portable-config", + kind="definition-component", + sha256=_sha256_file(config_path), + byte_length=config_path.stat().st_size, + component_id="ddrnet-portable-runtime-config-v2", + model_release_id=None, + model_artifact_role=None, + ), + ] + requirements.extend( + _requirement(asset_id, "local-tree", sha256, byte_length) + for asset_id, _target, sha256, byte_length in _EOMT_ASSETS + ) + for asset_id, _target, sha256, byte_length in _DDRNET_ASSETS: + if asset_id == "ddrnet-checkpoint": + requirements.append( + PortableWorkerAssetRequirement( + asset_id=asset_id, + kind="model-artifact", + sha256=sha256, + byte_length=byte_length, + component_id=None, + model_release_id="lab-v1-ddrnet-39-goose-fine-64-v1", + model_artifact_role="checkpoint", + ) + ) + elif asset_id in {"vegetation-policy", "vegetation-provider-map"}: + component_id = ( + "vegetation-mission-policy-v1" + if asset_id == "vegetation-policy" + else "vegetation-provider-label-map-v1" + ) + requirements.append( + PortableWorkerAssetRequirement( + asset_id=asset_id, + kind="definition-component", + sha256=sha256, + byte_length=byte_length, + component_id=component_id, + model_release_id=None, + model_artifact_role=None, + ) + ) + else: + requirements.append(_requirement(asset_id, "local-file", sha256, byte_length)) + ordered = tuple(sorted(requirements, key=lambda item: item.asset_id)) + for requirement in ordered: + if requirement.kind == "definition-component": + assert any( + item.component_id == requirement.component_id and item.sha256 == requirement.sha256 + for item in definition.components + ) + return ordered + + +def _requirement( + asset_id: str, + kind: RuntimeAssetKind, + sha256: str, + byte_length: int | None = None, +) -> PortableWorkerAssetRequirement: + return PortableWorkerAssetRequirement( + asset_id=asset_id, + kind=kind, + sha256=sha256, + byte_length=byte_length, + component_id=None, + model_release_id=None, + model_artifact_role=None, + ) + + +def _containers( + *, agent_image_sha256: str, eomt_image_sha256: str, ddrnet_image_sha256: str +) -> tuple[InstalledLabContainer, ...]: + return ( + InstalledLabContainer( + container_id="assemble", + role="result-writer", + image_sha256=agent_image_sha256, + argv=("-m", "k1link.observatory.lab_v1_installed_package_steps", "assemble"), + depends_on=("ddrnet", "eomt", "prepare"), + mounts=( + InstalledLabPackageMount( + "lab-v1-package-contract", "/opt/nodedc/package/contract.json" + ), + InstalledLabPackageMount( + "lab-v1-definition-registry", + "/opt/nodedc/package/portable-run-definitions.json", + ), + ), + gpu_count=0, + memory_bytes=8 * 1024**3, + nano_cpus=4_000_000_000, + network="none", + pids_limit=2048, + shm_bytes=4 * 1024**3, + tmpfs_bytes=4 * 1024**3, + timeout_seconds=7200, + ), + InstalledLabContainer( + container_id="ddrnet", + role="step", + image_sha256=ddrnet_image_sha256, + argv=( + "/opt/nodedc/adapter/run_portable_lab_v1_ddrnet_component.py", + "--package-step", + "ddrnet", + ), + depends_on=("eomt", "prepare"), + mounts=tuple( + sorted( + (InstalledLabPackageMount(item[0], item[1]) for item in _DDRNET_ASSETS), + key=lambda item: (item.target, item.asset_id), + ) + ), + gpu_count=1, + memory_bytes=16 * 1024**3, + nano_cpus=4_000_000_000, + network="none", + pids_limit=2048, + shm_bytes=4 * 1024**3, + tmpfs_bytes=4 * 1024**3, + timeout_seconds=7200, + ), + InstalledLabContainer( + container_id="eomt", + role="step", + image_sha256=eomt_image_sha256, + argv=( + "/opt/nodedc/adapter/run_portable_lab_v1_eomt_component.py", + "--package-step", + "eomt", + ), + depends_on=("prepare",), + mounts=tuple( + sorted( + (InstalledLabPackageMount(item[0], item[1]) for item in _EOMT_ASSETS), + key=lambda item: (item.target, item.asset_id), + ) + ), + gpu_count=1, + memory_bytes=24 * 1024**3, + nano_cpus=8_000_000_000, + network="none", + pids_limit=2048, + shm_bytes=4 * 1024**3, + tmpfs_bytes=4 * 1024**3, + timeout_seconds=7200, + ), + InstalledLabContainer( + container_id="prepare", + role="step", + image_sha256=agent_image_sha256, + argv=("-m", "k1link.observatory.lab_v1_installed_package_steps", "prepare"), + depends_on=(), + mounts=( + InstalledLabPackageMount( + "lab-v1-package-contract", "/opt/nodedc/package/contract.json" + ), + InstalledLabPackageMount( + "ddrnet-portable-config", + "/opt/nodedc/package/lab-v1-eomt-ddrnet-portable-v2.json", + ), + InstalledLabPackageMount( + "lab-v1-definition-registry", + "/opt/nodedc/package/portable-run-definitions.json", + ), + ), + gpu_count=0, + memory_bytes=4 * 1024**3, + nano_cpus=2_000_000_000, + network="none", + pids_limit=2048, + shm_bytes=4 * 1024**3, + tmpfs_bytes=4 * 1024**3, + timeout_seconds=7200, + ), + ) + + +def _ready_definition( + blocked: PortableRunDefinition, release_sha256: str, image_sha256: str +) -> PortableRunDefinition: + executor = PortableExecutorAvailability( + contour_id=blocked.executor.contour_id, + state="ready", + release_id=LAB_V1_RELEASE_ID, + release_sha256=release_sha256, + image_sha256=image_sha256, + reason_code=None, + reason=None, + ) + identity = {**blocked.identity_document(), "executor": executor.identity_document()} + return PortableRunDefinition( + setup_id=blocked.setup_id, + definition_id=blocked.definition_id, + version=blocked.version, + definition_sha256=canonical_sha256(identity), + source_requirements=blocked.source_requirements, + source_adapter=blocked.source_adapter, + components=blocked.components, + models=blocked.models, + resource_profile=blocked.resource_profile, + result_contract=blocked.result_contract, + executor=executor, + authority=blocked.authority, + ) + + +def _component_asset( + item: tuple[str, str, str, int], *, kind: str, verification: str +) -> dict[str, object]: + asset_id, target, sha256, byte_length = item + return { + "asset_id": asset_id, + "path": target, + "kind": kind, + "verification": verification, + "identity_sha256": sha256, + "byte_length": byte_length, + } + + +def _definition_document(value: PortableRunDefinition) -> dict[str, object]: + return { + "setup_id": value.setup_id, + "definition_id": value.definition_id, + "version": value.version, + "definition_sha256": value.definition_sha256, + "source_requirements": value.source_requirements.as_dict(), + "source_adapter": value.source_adapter.as_dict(), + "components": [item.as_dict() for item in value.components], + "models": [item.as_dict() for item in value.models], + "resource_profile": value.resource_profile.as_dict(), + "result_contract": value.result_contract.as_dict(), + "executor": { + **value.executor.identity_document(), + "reason_code": value.executor.reason_code, + "reason": value.executor.reason, + }, + "authority": value.authority.as_dict(), + } + + +def _release_document(value: PortableLabV1ReleaseCandidate) -> dict[str, object]: + return { + "schema_version": PORTABLE_LAB_V1_RELEASE_SCHEMA, + "release_id": value.release_id, + "setup_id": value.setup_id, + "definition_id": value.definition_id, + "definition_version": value.definition_version, + "definition_contract_sha256": value.definition_contract_sha256, + "result_contract_sha256": value.result_contract_sha256, + "executor_image_sha256": value.executor_image_sha256, + "assets": [item.as_dict() for item in value.assets], + "phases": list(value.phases), + "declared_blockers": list(value.declared_blockers), + "authority": dict(OBSERVATION_ONLY_AUTHORITY), + "candidate_sha256": value.candidate_sha256, + } + + +def _replace_row( + source: Mapping[str, object], + *, + collection: str, + setup_id: str, + replacement: Mapping[str, object], + schema: str, +) -> dict[str, object]: + if source.get("schema_version") != schema or not isinstance(source.get(collection), list): + raise InstalledLabV1PromotionError("source registry changed") + rows = cast(list[object], source[collection]) + if sum(isinstance(item, dict) and item.get("setup_id") == setup_id for item in rows) != 1: + raise InstalledLabV1PromotionError("source LAB V1 row is not unique") + return { + "schema_version": schema, + collection: [ + dict(replacement) + if isinstance(item, dict) and item.get("setup_id") == setup_id + else item + for item in rows + ], + } + + +def _read_object(path: Path) -> dict[str, object]: + value: object = json.loads(path.read_bytes()) + if not isinstance(value, dict): + raise InstalledLabV1PromotionError("registry is not an object") + return cast(dict[str, object], value) + + +def _write(path: Path, value: Mapping[str, object]) -> None: + path.write_bytes(canonical_json(value)) + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_asset_bindings( + *, + output_root: Path, + requirements: tuple[PortableWorkerAssetRequirement, ...], + engine_release_root: str, + engine_runtime_root: str, + engine_work_root: str, +) -> None: + for value, label in ( + (engine_release_root, "Engine release root"), + (engine_runtime_root, "Engine runtime root"), + (engine_work_root, "Engine work root"), + ): + _engine_root(value, label) + release_files = { + "ddrnet-portable-config": "lab-v1-eomt-ddrnet-portable-v2.json", + "lab-v1-definition-registry": "observatory-portable-run-definitions.json", + "lab-v1-package-contract": "lab-v1-installed-package-contract.json", + } + rows: list[dict[str, object]] = [] + for requirement in requirements: + if requirement.kind == "container-image": + rows.append( + { + "asset_id": requirement.asset_id, + "controller_path": None, + "engine_path": None, + "image_sha256": requirement.sha256, + } + ) + continue + release_name = release_files.get(requirement.asset_id) + if release_name is not None: + rows.append( + { + "asset_id": requirement.asset_id, + "controller_path": f"/release/{release_name}", + "engine_path": f"{engine_release_root}\\{release_name}", + "image_sha256": None, + } + ) + continue + relative = _RUNTIME_ASSET_RELATIVE_PATHS.get(requirement.asset_id) + if relative is None: + raise InstalledLabV1PromotionError("runtime asset has no Worker 006 binding") + rows.append( + { + "asset_id": requirement.asset_id, + "controller_path": f"/runtime/{relative.replace('\\', '/')}", + "engine_path": f"{engine_runtime_root}\\{relative}", + "image_sha256": None, + } + ) + _write( + output_root / "observatory-installed-lab-asset-bindings.json", + { + "schema_version": INSTALLED_LAB_ASSET_BINDINGS_SCHEMA, + "engine_work_root": engine_work_root, + "assets": rows, + }, + ) + + +def _validate_generated_bindings( + *, + path: Path, + requirements: tuple[PortableWorkerAssetRequirement, ...], + engine_work_root: str, +) -> None: + try: + bindings = InstalledLabWorkerAssetBindings.from_file(path) + except (InstalledLabWorkerCompositionError, ValueError) as exc: + raise InstalledLabV1PromotionError( + "generated installed LAB asset bindings are invalid" + ) from exc + if bindings.engine_work_root != engine_work_root: + raise InstalledLabV1PromotionError("generated Engine work root changed") + expected = {item.asset_id: item for item in requirements} + actual = {item.asset_id: item for item in bindings.assets} + if set(actual) != set(expected): + raise InstalledLabV1PromotionError("generated asset binding inventory changed") + for asset_id, requirement in expected.items(): + binding = actual[asset_id] + if requirement.kind == "container-image": + if binding.image_sha256 != requirement.sha256: + raise InstalledLabV1PromotionError( + "generated image asset binding changed" + ) + elif binding.image_sha256 is not None: + raise InstalledLabV1PromotionError("generated file asset binding changed") + + +def _engine_root(value: str, label: str) -> None: + if ( + re.fullmatch(r"[A-Za-z]:\\[^\x00\r\n]+", value) is None + or _ENGINE_PATH_PLACEHOLDER.search(value) is not None + or ".." in value.replace("\\", "/").split("/") + ): + raise InstalledLabV1PromotionError(f"{label} is invalid") + + +def _verify_installed_image( + expected_sha256: str, + label: str, + image_resolver: ImageResolver, +) -> None: + try: + actual_sha256 = image_resolver(expected_sha256) + except InstalledLabV1PromotionError: + raise + except Exception as exc: + raise InstalledLabV1PromotionError(f"{label} is unavailable") from exc + _digest(actual_sha256, f"resolved {label}") + if actual_sha256 != expected_sha256: + raise InstalledLabV1PromotionError(f"{label} identity changed") + + +def _resolve_local_docker_image_sha256(expected_sha256: str) -> str: + if _DOCKER_SOCKET.exists(): + return _resolve_docker_engine_image_sha256(expected_sha256) + reference = f"sha256:{expected_sha256}" + try: + completed = subprocess.run( + ["docker", "image", "inspect", "--format", "{{.Id}}", reference], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise InstalledLabV1PromotionError("local Docker image inspection failed") from exc + values = completed.stdout.splitlines() + if completed.returncode != 0 or values != [reference]: + raise InstalledLabV1PromotionError("local Docker image is unavailable") + return expected_sha256 + + +def _resolve_docker_engine_image_sha256( + expected_sha256: str, + *, + transport_factory: Callable[[], httpx.BaseTransport] | None = None, +) -> str: + reference = f"sha256:{expected_sha256}" + transport = ( + httpx.HTTPTransport(uds=str(_DOCKER_SOCKET)) + if transport_factory is None + else transport_factory() + ) + try: + with httpx.Client( + base_url="http://docker", + transport=transport, + timeout=httpx.Timeout(10.0, connect=5.0), + ) as client: + response = client.get(f"/{_DOCKER_API_VERSION}/images/{reference}/json") + response.raise_for_status() + document: object = response.json() + except (httpx.HTTPError, ValueError) as exc: + raise InstalledLabV1PromotionError("local Docker image is unavailable") from exc + if not isinstance(document, dict) or document.get("Id") != reference: + raise InstalledLabV1PromotionError("local Docker image identity changed") + return expected_sha256 + + +def _digest(value: str, label: str) -> None: + if _SHA256.fullmatch(value) is None: + raise InstalledLabV1PromotionError(f"{label} SHA-256 is invalid") + + +def main(arguments: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-definitions", type=Path, required=True) + parser.add_argument("--source-runtime", type=Path, required=True) + parser.add_argument("--ddrnet-config", type=Path, required=True) + parser.add_argument("--result-assembler-source", type=Path, required=True) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--source-tree-sha256", required=True) + parser.add_argument("--agent-image-sha256", required=True) + parser.add_argument("--eomt-image-sha256", required=True) + parser.add_argument("--ddrnet-image-sha256", required=True) + parser.add_argument("--engine-release-root", required=True) + parser.add_argument("--engine-runtime-root", required=True) + parser.add_argument("--engine-work-root", required=True) + options = parser.parse_args(arguments) + generate( + source_definitions=options.source_definitions, + source_runtime=options.source_runtime, + ddrnet_config=options.ddrnet_config, + result_assembler_source=options.result_assembler_source, + output_root=options.output_root, + source_tree_sha256=options.source_tree_sha256, + agent_image_sha256=options.agent_image_sha256, + eomt_image_sha256=options.eomt_image_sha256, + ddrnet_image_sha256=options.ddrnet_image_sha256, + engine_release_root=options.engine_release_root, + engine_runtime_root=options.engine_runtime_root, + engine_work_root=options.engine_work_root, + image_resolver=_resolve_local_docker_image_sha256, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py index 63a83aa..b1e8721 100644 --- a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py +++ b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_ddrnet_component.py @@ -31,7 +31,7 @@ from portable_lab_v1_component_adapter import ( load_canonical_json, load_component_request, load_json, - parse_fixed_request_argument, + resolve_runtime_layout, run_command, sha256_file, validate_camera_compute_job, @@ -143,11 +143,12 @@ def execute_ddrnet_component( output = _empty_output_root(layout.output_root) workspace = _prepare_workspace(work_root) try: - frames_root = _snapshot_source_frames( - eomt_root, - source_frames, - workspace / "source-frames", - ) + # The EoMT step output is already mounted read-only by the package + # runner. Re-copying every decoded frame into the 4 GiB container + # tmpfs makes a full 6,830-frame K1 run fail even though the immutable + # input is valid. Verify every manifest digest through an O_NOFOLLOW + # descriptor, then let the sealed runner read that same read-only tree. + frames_root = _verify_source_frames(eomt_root, source_frames) mapping_copy = workspace / "goose_label_mapping.csv" shutil.copyfile(assets["ddrnet-goose-mapping"], mapping_copy) os.chmod(mapping_copy, 0o400) @@ -450,19 +451,17 @@ def _source_frame_manifest( return tuple(rows) -def _snapshot_source_frames( +def _verify_source_frames( eomt_root: Path, rows: Sequence[SourceFrameRow], - snapshot_root: Path, ) -> Path: - snapshot_root.mkdir(mode=0o700) + frames_root = eomt_root / "source-frames" for row in rows: source = _confined_regular_file( eomt_root / row.path, eomt_root, "EoMT source frame", ) - destination = snapshot_root / f"frame-{row.sequence:06d}.png" digest = hashlib.sha256() byte_length = 0 signature = b"" @@ -475,10 +474,7 @@ def _snapshot_source_frames( admitted.st_ino, ): raise ComponentAdapterError("EoMT source frame changed during admission") - with ( - os.fdopen(descriptor, "rb", closefd=False) as source_stream, - destination.open("xb") as destination_stream, - ): + with os.fdopen(descriptor, "rb", closefd=False) as source_stream: while block := source_stream.read(1024 * 1024): if not signature: signature = block[:8] @@ -486,19 +482,21 @@ def _snapshot_source_frames( if byte_length > row.byte_length: raise ComponentAdapterError("EoMT source frame grew during admission") digest.update(block) - destination_stream.write(block) - destination_stream.flush() - os.fsync(destination_stream.fileno()) + completed = os.fstat(descriptor) + current = source.stat() finally: os.close(descriptor) if ( signature != b"\x89PNG\r\n\x1a\n" or byte_length != row.byte_length or digest.hexdigest() != row.sha256 + or (completed.st_dev, completed.st_ino, completed.st_size) + != (opened.st_dev, opened.st_ino, opened.st_size) + or (current.st_dev, current.st_ino, current.st_size) + != (opened.st_dev, opened.st_ino, opened.st_size) ): raise ComponentAdapterError("EoMT source frame identity changed") - destination.chmod(0o400) - return snapshot_root.resolve(strict=True) + return frames_root.resolve(strict=True) def _validate_decode_repair(document: Mapping[str, object], frame_count: int) -> None: @@ -812,10 +810,14 @@ def _representative_frame_indices(frame_count: int) -> list[int]: def main(argv: Sequence[str] | None = None) -> int: - request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv) + layout = resolve_runtime_layout( + sys.argv[1:] if argv is None else argv, + component="ddrnet", + expectations=DDRNET_ASSETS, + ) execute_ddrnet_component( - request_path=request_path, - layout=RuntimeLayout.fixed(DDRNET_ASSETS), + request_path=layout.request, + layout=layout, ) return 0 diff --git a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py index 99332f5..83decd4 100644 --- a/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py +++ b/experiments/perception/worker/observatory_portable/run_portable_lab_v1_eomt_component.py @@ -29,7 +29,7 @@ from portable_lab_v1_component_adapter import ( canonical_json, load_component_request, load_json, - parse_fixed_request_argument, + resolve_runtime_layout, run_command, sha256_file, validate_camera_compute_job, @@ -49,7 +49,12 @@ SOURCE_FRAME_MANIFEST_SCHEMA: Final = "missioncore.recorded-perception-source-fr MODEL_REVISION: Final = "8d6b6d1a3f7b50d441afd7d247c2ed10db186e8f" MODEL_ID: Final = "tue-mps/cityscapes_semantic_eomt_large_1024" MODEL_ARCHITECTURE: Final = "EomtForUniversalSegmentation" -DISK_FLOOR_BYTES: Final = 360 * 1024**3 +PHYSICAL_CAMERA_SOURCE_ID: Final = "sensor.camera.right" +# Keep a large post-run floor while admitting the full 6,830-frame K1 record on +# Worker 006. The independent ``reserve`` below already accounts for the +# complete worst-case working set, so adding the historical 360 GiB floor made +# the real job miss admission by about 1.4 GB despite 408 GB being free. +DISK_FLOOR_BYTES: Final = 350 * 1024**3 FFMPEG_TREE_SOURCE_IMAGE_SHA256: Final = ( "8a364092b03561b9c08ac00730206e363a53d07ea0304f7d543b403b65432b5e" ) @@ -109,8 +114,8 @@ EOMT_ASSETS: Final = ( _RUNNER_FILES: Final = { "run_e4_full_session_segmentation.py": ( - 30_720, - "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", + 31_733, + "1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff", ), "run_e3_rectified_segmentation.py": ( 45_789, @@ -195,6 +200,8 @@ def execute_eomt_component( str(layout.camera_job_root), "--profile", str(profile), + "--physical-source-id", + PHYSICAL_CAMERA_SOURCE_ID, "--valid-fov-root", str(roots["k1-valid-fov-root"]), "--frames", @@ -971,10 +978,14 @@ def _number(value: float) -> str: def main(argv: Sequence[str] | None = None) -> int: - request_path = parse_fixed_request_argument(sys.argv[1:] if argv is None else argv) + layout = resolve_runtime_layout( + sys.argv[1:] if argv is None else argv, + component="eomt", + expectations=EOMT_ASSETS, + ) execute_eomt_component( - request_path=request_path, - layout=RuntimeLayout.fixed(EOMT_ASSETS), + request_path=layout.request, + layout=layout, ) return 0 diff --git a/experiments/perception/worker/run_e4_full_session_segmentation.py b/experiments/perception/worker/run_e4_full_session_segmentation.py index 8b27e80..0467e5e 100644 --- a/experiments/perception/worker/run_e4_full_session_segmentation.py +++ b/experiments/perception/worker/run_e4_full_session_segmentation.py @@ -9,6 +9,7 @@ import importlib.metadata import json import os import platform +import re import resource import shutil import time @@ -46,6 +47,7 @@ VALID_FOV_SCHEMA = "missioncore.k1-valid-fov-mask/v1" PIPELINE_ID = "recorded-semantic-eomt-fisheye-mask/v1" SEMANTIC_ALPHA = 0.48 TARGET_CLASS_COUNT = 16 +_SAFE_SOURCE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") def _arguments() -> argparse.Namespace: @@ -58,6 +60,7 @@ def _arguments() -> argparse.Namespace: preflight.add_argument("--valid-fov-root", type=Path, required=True) preflight.add_argument("--cache", type=Path, required=True) preflight.add_argument("--environment", type=Path, required=True) + preflight.add_argument("--physical-source-id") run = commands.add_parser("run") run.add_argument("--job", type=Path, required=True) @@ -67,6 +70,7 @@ def _arguments() -> argparse.Namespace: run.add_argument("--timeline", type=Path, required=True) run.add_argument("--cache", type=Path, required=True) run.add_argument("--environment", type=Path, required=True) + run.add_argument("--physical-source-id") run.add_argument("--output", type=Path, required=True) run.add_argument("--frame-limit", type=int, default=0) run.add_argument("--free-bytes-floor", type=int, default=0) @@ -99,21 +103,36 @@ def _read_object(path: Path) -> dict[str, Any]: return value -def _validate_source(job: dict[str, Any], profile: dict[str, Any]) -> None: +def _validate_source( + job: dict[str, Any], + profile: dict[str, Any], + physical_source_id: str | None = None, +) -> None: source = profile["source"] input_document = job["input"] + job_source_id = input_document.get("source_id") + profile_source_id = source.get("source_id") if ( input_document.get("kind") != "canonical-camera-epoch" - or input_document.get("source_id") != source["source_id"] + or not isinstance(job_source_id, str) + or _SAFE_SOURCE_ID.fullmatch(job_source_id) is None + or not isinstance(profile_source_id, str) + or _SAFE_SOURCE_ID.fullmatch(profile_source_id) is None or source.get("resolution") != [800, 600] ): raise RuntimeError("LAB E4 profile does not match the camera job") + if physical_source_id is None: + if job_source_id != profile_source_id: + raise RuntimeError("LAB E4 profile does not match the camera job") + elif physical_source_id != profile_source_id: + raise RuntimeError("LAB E4 physical camera binding is invalid") def _load_valid_fov( root: Path, job: dict[str, Any], profile: dict[str, Any], + physical_source_id: str | None = None, ) -> tuple[Any, dict[str, Any]]: import numpy as np from PIL import Image @@ -124,6 +143,7 @@ def _load_valid_fov( identity_sha256 = manifest.get("identity_sha256") artifact = manifest.get("artifact") source = profile["source"] + expected_source_id = physical_source_id or job["input"]["source_id"] if ( manifest.get("schema_version") != VALID_FOV_SCHEMA or not isinstance(identity, dict) @@ -132,7 +152,7 @@ def _load_valid_fov( or manifest.get("generation_id") != f"valid-fov-mask-{identity_sha256}" or identity.get("calibration_sha256") != source["calibration_sha256"] or identity.get("calibration_slot") != source["calibration_slot"] - or identity.get("source_id") != job["input"]["source_id"] + or identity.get("source_id") != expected_source_id or identity.get("admitted_resolution") != source["resolution"] or not isinstance(artifact, dict) or artifact.get("path") != "mask.png" @@ -212,11 +232,11 @@ def _preflight(args: argparse.Namespace) -> int: job = _validate_job(args.job.resolve(strict=True)) profile, profile_sha256 = _profile(args.profile) - _validate_source(job, profile) + _validate_source(job, profile, args.physical_source_id) dependency = _dependency_manifest(args.environment.resolve(strict=True)) if dependency["identity"]["profile_sha256"] != profile_sha256: raise RuntimeError("LAB E4 dependencies belong to another profile") - _load_valid_fov(args.valid_fov_root, job, profile) + _load_valid_fov(args.valid_fov_root, job, profile, args.physical_source_id) if not torch.cuda.is_available() or torch.cuda.device_count() < 1: raise RuntimeError("CUDA device 0 is unavailable") device = torch.device("cuda:0") @@ -314,11 +334,16 @@ def _run(args: argparse.Namespace) -> int: raise RuntimeError("LAB E4 container image identity is invalid") job = _validate_job(job_root) profile, profile_sha256 = _profile(args.profile) - _validate_source(job, profile) + _validate_source(job, profile, args.physical_source_id) dependency = _dependency_manifest(args.environment.resolve(strict=True)) if dependency["identity"]["profile_sha256"] != profile_sha256: raise RuntimeError("LAB E4 dependencies belong to another profile") - valid_mask, valid_fov = _load_valid_fov(args.valid_fov_root, job, profile) + valid_mask, valid_fov = _load_valid_fov( + args.valid_fov_root, + job, + profile, + args.physical_source_id, + ) input_document = job["input"] full_frame_count = int(input_document["segment_count"]) diff --git a/tests/test_e4_full_session_segmentation.py b/tests/test_e4_full_session_segmentation.py index ab9a605..a013a6c 100644 --- a/tests/test_e4_full_session_segmentation.py +++ b/tests/test_e4_full_session_segmentation.py @@ -37,6 +37,25 @@ def _profile() -> dict[str, object]: return value +def test_e4_portable_binding_distinguishes_public_and_physical_camera_ids() -> None: + worker = _worker_module() + profile = _profile() + public_job = { + "input": { + "kind": "canonical-camera-epoch", + "source_id": "recorded.camera.6a3945242828a038", + } + } + + with pytest.raises(RuntimeError, match="profile does not match"): + worker._validate_source(public_job, profile) + + worker._validate_source(public_job, profile, "sensor.camera.right") + + with pytest.raises(RuntimeError, match="physical camera binding"): + worker._validate_source(public_job, profile, "sensor.camera.left") + + def test_e4_accepts_the_sealed_camera1_valid_fov() -> None: worker = _worker_module() root = ( diff --git a/tests/test_lab_v1_portable_static_asset_installer.py b/tests/test_lab_v1_portable_static_asset_installer.py index 1700b64..8162694 100644 --- a/tests/test_lab_v1_portable_static_asset_installer.py +++ b/tests/test_lab_v1_portable_static_asset_installer.py @@ -17,8 +17,8 @@ def test_lab_v1_static_installer_is_fixed_offline_and_content_addressed() -> Non script = INSTALLER_PATH.read_text(encoding="utf-8") assert '"D:\\NDC_MISSIONCORE\\runtime"' in script - assert '"staging\\observatory-lab-v1-static-v1"' in script - assert '"eomt-runner-bundle-v1"' in script + assert '"staging\\observatory-lab-v1-static-v2"' in script + assert '"eomt-runner-bundle-v2"' in script assert '"lab-v1-static-files-v1"' in script assert "New-Item `\n -ItemType HardLink" in script assert "Get-FileIdentity" in script @@ -37,7 +37,7 @@ def test_lab_v1_static_installer_seals_expected_exact_payloads() -> None: script = INSTALLER_PATH.read_text(encoding="utf-8") expected = { - "651e8e06c3912dffb036b7fd08f2c0623f7563d8306cc7aee05db562798518f4", + "1e64869de48d10f1531c742e6067c4c3ae2a709c5eb0d770d1fab74b4a2431ff", "01881862d4eaa218955f776a948124bf19c34be2b5ec282115daeacb15c53ae6", "4dcc4fc8bdf33702651a199be69d0dd4fadb243d2e65aee1c3d1ae7a58fdf675", "25baf30c0df564734e08f38ace88cc4bc147cacf240c761622279511e361daa4", diff --git a/tests/test_observatory_installed_lab_offline_launch.py b/tests/test_observatory_installed_lab_offline_launch.py new file mode 100644 index 0000000..2edb8c8 --- /dev/null +++ b/tests/test_observatory_installed_lab_offline_launch.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +LAUNCHER = ( + REPOSITORY_ROOT + / "experiments/perception/worker/observatory_portable" + / "Invoke-ObservatoryInstalledLabOfflineValidation.ps1" +) + + +def _script() -> str: + return LAUNCHER.read_text(encoding="utf-8") + + +def test_offline_launcher_uses_only_the_canonical_activation_chain() -> None: + script = _script() + + assert "ExpectedActivationFileSha256" in script + assert "Read-Activation $activationFile" in script + assert "Read-InstallationReceipt" in script + assert "$packageId = [string]$current.package_id" in script + assert "$packageSha256 = [string]$current.package_sha256" in script + assert '"$RuntimeRoot\\releases\\observatory-installed-labs"' in script + assert '"$RuntimeRoot\\state\\observatory-installed-labs"' in script + assert '"$RuntimeRoot\\services\\observatory-installed-labs"' in script + assert '"lab-v1-eomt-ddrnet-portable-v2"' not in script + + +def test_offline_launcher_has_one_fixed_non_claiming_command() -> None: + script = _script() + + assert ( + '$command = @("-m", "k1link.observatory.installed_lab_worker_service", ' + '"--validate-only")' + ) in script + assert 'mode = "validation-only"' in script + assert 'backend_contacted = $false' in script + assert 'claim_attempted = $false' in script + assert 'queue_started = $false' in script + assert "--once" not in script + assert "MISSIONCORE_OBSERVATORY_WORKER_TOKEN" not in script + assert "MISSIONCORE_OBSERVATORY_WORKER_BASE_URL" not in script + + +def test_offline_launcher_builds_a_hardened_exact_docker_run() -> None: + script = _script() + + assert '"--network", "none"' in script + assert '"--cap-drop", "ALL"' in script + assert '"--security-opt", "no-new-privileges"' in script + assert '"--read-only"' in script + assert '"--pids-limit", "128"' in script + assert 'source = "/var/run/docker.sock"' in script + assert "Assert-ExactImage $imageSha256" in script + assert "$packageImages -cnotcontains $agentImageSha256" in script + assert "docker ps -a" in script + assert '"missioncore.observatory-installed-lab-offline-launch-plan/v1"' in script + assert '"missioncore.observatory-installed-lab-offline-launch-result/v1"' in script + + +def test_offline_launcher_writes_only_content_checked_evidence() -> None: + script = _script() + + assert "function Write-ExactFile" in script + assert '$receiptPayload = [string]::Join("`n", $output) + "`n"' in script + assert "[Environment]::NewLine" not in script + assert "installed LAB offline launch plan" in script + assert "installed LAB offline validation receipt" in script + assert "validation_receipt_file_sha256" in script + assert "validation_receipt_sha256" in script + assert "[IO.File]::Move(" in script + assert "Invoke-WebRequest" not in script + assert "Start-Service" not in script + assert "Start-ScheduledTask" not in script diff --git a/tests/test_observatory_installed_lab_package_installer.py b/tests/test_observatory_installed_lab_package_installer.py new file mode 100644 index 0000000..c515795 --- /dev/null +++ b/tests/test_observatory_installed_lab_package_installer.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INSTALLER = ( + REPOSITORY_ROOT + / "experiments/perception/worker/observatory_portable" + / "Install-ObservatoryInstalledLabPackage.ps1" +) + + +def _script() -> str: + return INSTALLER.read_text(encoding="utf-8") + + +def test_installer_uses_generic_content_addressed_package_roots() -> None: + script = _script() + + assert '"$RuntimeRoot\\releases\\observatory-installed-labs"' in script + assert '"$RuntimeRoot\\state\\observatory-installed-labs"' in script + assert '"$RuntimeRoot\\services\\observatory-installed-labs"' in script + assert "$packageId = [string]$package.package_id" in script + assert "$targetRoot = Join-Path $packageReleaseParent $packageSha256" in script + assert '"missioncore.observatory-installed-lab-installation/v1"' in script + assert '"missioncore.observatory-installed-lab-activation/v1"' in script + assert '"missioncore.observatory-installed-lab-installation-result/v1"' in script + assert '"lab-v1-eomt-ddrnet-portable-v2.json"' not in script + assert "$packageImages.Count -lt 1" in script + + +def test_installer_is_atomic_idempotent_and_retains_one_rollback_anchor() -> None: + script = _script() + + assert '".install-$packageSha256-$PID"' in script + assert "Assert-InstalledRoot $stageRoot" in script + assert "Move-Item -LiteralPath $stageRoot -Destination $targetRoot" in script + assert "$alreadyInstalled = $true" in script + assert "New-ReleaseAnchor" in script + assert "$rollback = New-ReleaseAnchor" in script + assert '".activation-$PID.tmp"' in script + assert "[IO.File]::Replace(" in script + assert "[IO.File]::Move(" in script + assert "Read-Activation $activationPath" in script + + +def test_installer_fails_closed_before_writing_canonical_state() -> None: + script = _script() + write_boundary = script.index("$alreadyInstalled = $false") + preflight = script[:write_boundary] + + assert "$StagingPrefix" in preflight + assert "ExpectedPromotionSummarySha256" in preflight + assert "ExpectedPackageSha256" in preflight + assert "ExpectedValidationReceiptFileSha256" in preflight + assert "backend_contacted" in preflight + assert "claim_attempted" in preflight + assert "Assert-ExactImage $imageSha256" in preflight + assert "installed LAB binding work root is not canonical" in preflight + assert "installed LAB release binding is not canonical" in preflight + assert "installed LAB runtime binding is not canonical" in preflight + + +def test_installer_never_starts_services_or_contacts_a_backend() -> None: + script = _script() + lowered = script.lower() + + assert 'backend_changed = $false' in script + assert 'claim_attempted = $false' in script + assert 'queue_started = $false' in script + assert "docker image inspect" in lowered + assert "docker run" not in lowered + assert "docker start" not in lowered + assert "start-service" not in lowered + assert "start-scheduledtask" not in lowered + assert "invoke-webrequest" not in lowered + assert "curl " not in lowered + assert "http://" not in lowered + assert "https://" not in lowered diff --git a/tests/test_observatory_installed_lab_v1_image_installer.py b/tests/test_observatory_installed_lab_v1_image_installer.py new file mode 100644 index 0000000..4c5572d --- /dev/null +++ b/tests/test_observatory_installed_lab_v1_image_installer.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +INSTALLER = ( + REPOSITORY_ROOT + / "experiments/perception/worker/observatory_portable" + / "Install-InstalledLabV1PackageImages.ps1" +) + + +def _script() -> str: + return INSTALLER.read_text(encoding="utf-8") + + +def test_installed_lab_v1_image_installer_is_local_and_offline() -> None: + script = _script() + + assert '"docker-commit-exact-layer-v1"' in script + assert "--network none" in script + assert "--cap-drop ALL" in script + assert "--security-opt no-new-privileges" in script + assert "target=/nodedc-build-source,readonly" in script + assert "docker rm -f $containerId" in script + + lowered = script.lower() + assert "docker build" not in lowered + assert "docker buildx" not in lowered + assert "docker pull" not in lowered + assert "invoke-webrequest" not in lowered + assert "start-bitstransfer" not in lowered + assert "curl " not in lowered + assert "wget " not in lowered + assert "smb" not in lowered + + +def test_agent_only_parameter_set_requires_no_component_inputs() -> None: + script = _script() + + assert '[CmdletBinding(DefaultParameterSetName = "Full")]' in script + assert script.count( + '[Parameter(Mandatory = $true, ParameterSetName = "Full")]' + ) == 3 + assert script.count('ParameterSetName = "AgentOnly"') == 1 + assert script.count('ParameterSetName = "ComponentsOnly"') == 4 + assert script.count('ParameterSetName = "EomtOnly"') == 3 + assert script.count('ParameterSetName = "DdrnetOnly"') == 3 + assert "[switch]$AgentOnly" in script + assert "[switch]$ComponentsOnly" in script + assert "[switch]$EomtOnly" in script + assert "[switch]$DdrnetOnly" in script + + agent_only_branch = script.split("if ($AgentOnly) {", 1)[1].split( + "elseif ($EomtOnly) {", 1 + )[0] + assert "$receipts = @(Install-Agent)" in agent_only_branch + assert "Install-Component" not in agent_only_branch + assert "$SharedAdapterSha256" not in agent_only_branch + assert "$EomtAdapterSha256" not in agent_only_branch + assert "$DdrnetAdapterSha256" not in agent_only_branch + assert "installed_lab_worker_container_main" in script + + +def test_components_only_parameter_set_builds_no_agent() -> None: + script = _script() + component_branch = script.split("elseif ($ComponentsOnly) {", 1)[1].split( + "else {", 1 + )[0] + + assert component_branch.count("Install-Component `") == 2 + assert '-Component "eomt"' in component_branch + assert '-Component "ddrnet"' in component_branch + assert "Install-Agent" not in component_branch + assert "$componentIdentity" in script + + +def test_single_component_parameter_sets_build_only_requested_image() -> None: + script = _script() + eomt_branch = script.split("elseif ($EomtOnly) {", 1)[1].split( + "elseif ($DdrnetOnly) {", 1 + )[0] + ddrnet_branch = script.split("elseif ($DdrnetOnly) {", 1)[1].split( + "elseif ($ComponentsOnly) {", 1 + )[0] + + assert eomt_branch.count("Install-Component `") == 1 + assert '-Component "eomt"' in eomt_branch + assert '-Component "ddrnet"' not in eomt_branch + assert "Install-Agent" not in eomt_branch + assert ddrnet_branch.count("Install-Component `") == 1 + assert '-Component "ddrnet"' in ddrnet_branch + assert '-Component "eomt"' not in ddrnet_branch + assert "Install-Agent" not in ddrnet_branch + + +def test_full_parameter_set_still_builds_both_components_and_agent() -> None: + script = _script() + full_branch = script.rsplit("else {", 1)[1] + + assert '-Component "eomt"' in full_branch + assert '-Component "ddrnet"' in full_branch + assert full_branch.count("Install-Component `") == 2 + assert "Install-Agent" in full_branch diff --git a/tests/test_observatory_installed_lab_v1_promotion.py b/tests/test_observatory_installed_lab_v1_promotion.py new file mode 100644 index 0000000..237e45e --- /dev/null +++ b/tests/test_observatory_installed_lab_v1_promotion.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from collections.abc import Callable +from pathlib import Path + +import httpx +import pytest + +from k1link.observatory.installed_lab_packages import InstalledLabPackageRegistry +from k1link.observatory.portable_run_definitions import PortableRunDefinitionRegistry +from k1link.observatory.portable_worker_runtime import PortableWorkerRuntimeRegistry + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +PROMOTION_SCRIPT = ( + REPOSITORY_ROOT + / "experiments" + / "perception" + / "worker" + / "observatory_portable" + / "promote_installed_lab_v1_package.py" +) +_SPEC = importlib.util.spec_from_file_location( + "observatory_installed_lab_v1_promotion_test", + PROMOTION_SCRIPT, +) +assert _SPEC is not None and _SPEC.loader is not None +promotion = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = promotion +_SPEC.loader.exec_module(promotion) + + +def test_generic_lab_v1_promotion_round_trips_all_three_registries( + tmp_path: Path, +) -> None: + output = tmp_path / "release" + + promotion.generate( + source_definitions=( + REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" + ), + source_runtime=( + REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" + ), + ddrnet_config=( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-eomt-ddrnet-portable-v2.json" + ), + result_assembler_source=( + REPOSITORY_ROOT / "src" / "k1link" / "observatory" / "portable_lab_v1_executor.py" + ), + output_root=output, + source_tree_sha256="1" * 64, + agent_image_sha256="2" * 64, + eomt_image_sha256="3" * 64, + ddrnet_image_sha256="4" * 64, + engine_release_root="D:\\NDC_MISSIONCORE\\runtime\\releases\\lab-v1", + engine_runtime_root="D:\\NDC_MISSIONCORE\\runtime", + engine_work_root="D:\\NDC_MISSIONCORE\\runtime\\services\\lab-v1\\work", + image_resolver=lambda value: value, + ) + + definitions = PortableRunDefinitionRegistry.from_file( + output / "observatory-portable-run-definitions.json" + ) + runtime = PortableWorkerRuntimeRegistry.from_file( + output / "observatory-worker-runtime-candidates.json", + definitions=definitions, + ) + packages = InstalledLabPackageRegistry.from_file( + output / "observatory-installed-lab-packages.json", + definitions=definitions, + runtime_registry=runtime, + ) + definition = definitions.resolve_setup("lab-v1-eomt-ddrnet-portable-v1") + candidate = runtime.resolve(definition.setup_id, definition.definition_sha256) + package = packages.resolve(definition.setup_id, definition.definition_sha256) + bindings = json.loads( + (output / "observatory-installed-lab-asset-bindings.json").read_text() + ) + + assert definition.executor.ready is True + assert candidate.ready is True + assert package.runtime_candidate_sha256 == candidate.candidate_sha256 + assert [item.container_id for item in package.containers] == [ + "assemble", + "ddrnet", + "eomt", + "prepare", + ] + assert len(bindings["assets"]) == len(candidate.reusable_assets) + assert (output / "promotion-summary.json").is_file() + + +def test_generic_lab_v1_promotion_rejects_uninstalled_image_before_writing( + tmp_path: Path, +) -> None: + output = tmp_path / "release" + + def resolve_image(value: str) -> str: + if value == "3" * 64: + raise RuntimeError("image is absent") + return value + + with pytest.raises( + promotion.InstalledLabV1PromotionError, + match="EoMT image is unavailable", + ): + _generate(output=output, image_resolver=resolve_image) + + assert not output.exists() + + +def test_generic_lab_v1_promotion_rejects_image_identity_drift_before_writing( + tmp_path: Path, +) -> None: + output = tmp_path / "release" + + with pytest.raises( + promotion.InstalledLabV1PromotionError, + match="agent image identity changed", + ): + _generate( + output=output, + image_resolver=lambda value: "9" * 64 if value == "2" * 64 else value, + ) + + assert not output.exists() + + +def test_docker_engine_image_resolver_requires_exact_identity() -> None: + digest = "2" * 64 + + def exact(request: httpx.Request) -> httpx.Response: + assert request.url.path.endswith(f"/images/sha256:{digest}/json") + return httpx.Response(200, json={"Id": f"sha256:{digest}"}) + + assert ( + promotion._resolve_docker_engine_image_sha256( + digest, + transport_factory=lambda: httpx.MockTransport(exact), + ) + == digest + ) + + +def test_docker_engine_image_resolver_rejects_identity_drift() -> None: + digest = "2" * 64 + + def changed(_request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json={"Id": f"sha256:{'9' * 64}"}) + + with pytest.raises( + promotion.InstalledLabV1PromotionError, + match="local Docker image identity changed", + ): + promotion._resolve_docker_engine_image_sha256( + digest, + transport_factory=lambda: httpx.MockTransport(changed), + ) + + +@pytest.mark.parametrize( + "placeholder", + ( + "D:\\NDC_MISSIONCORE\\runtime\\releases\\lab-v1$release", + "D:\\NDC_MISSIONCORE\\runtime\\releases\\{release}", + "D:\\NDC_MISSIONCORE\\runtime\\releases\\%RELEASE%", + ), +) +def test_generic_lab_v1_promotion_rejects_placeholder_path_before_writing( + tmp_path: Path, + placeholder: str, +) -> None: + output = tmp_path / "release" + + with pytest.raises( + promotion.InstalledLabV1PromotionError, + match="Engine release root is invalid", + ): + _generate( + output=output, + image_resolver=lambda value: value, + engine_release_root=placeholder, + ) + + assert not output.exists() + + +def _generate( + *, + output: Path, + image_resolver: Callable[[str], str], + engine_release_root: str = "D:\\NDC_MISSIONCORE\\runtime\\releases\\lab-v1", +) -> None: + promotion.generate( + source_definitions=( + REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" + ), + source_runtime=( + REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" + ), + ddrnet_config=( + REPOSITORY_ROOT + / "config" + / "perception" + / "lab-v1-eomt-ddrnet-portable-v2.json" + ), + result_assembler_source=( + REPOSITORY_ROOT / "src" / "k1link" / "observatory" / "portable_lab_v1_executor.py" + ), + output_root=output, + source_tree_sha256="1" * 64, + agent_image_sha256="2" * 64, + eomt_image_sha256="3" * 64, + ddrnet_image_sha256="4" * 64, + engine_release_root=engine_release_root, + engine_runtime_root="D:\\NDC_MISSIONCORE\\runtime", + engine_work_root="D:\\NDC_MISSIONCORE\\runtime\\services\\lab-v1\\work", + image_resolver=image_resolver, + ) diff --git a/tests/test_observatory_portable_lab_v1_component_adapters.py b/tests/test_observatory_portable_lab_v1_component_adapters.py index edd0670..e0a8cb9 100644 --- a/tests/test_observatory_portable_lab_v1_component_adapters.py +++ b/tests/test_observatory_portable_lab_v1_component_adapters.py @@ -99,8 +99,13 @@ def _mapping_payload() -> bytes: return ("\n".join(rows) + "\n").encode() -def _camera_source(root: Path, *, frame_count: int = 2) -> dict[str, object]: - epoch = root / "input" / "camera" / "sensor.camera.right" / "epoch-1" +def _camera_source( + root: Path, + *, + frame_count: int = 2, + source_id: str = "sensor.camera.right", +) -> dict[str, object]: + epoch = root / "input" / "camera" / source_id / "epoch-1" segments = epoch / "segments" segments.mkdir(parents=True) files = { @@ -125,7 +130,7 @@ def _camera_source(root: Path, *, frame_count: int = 2) -> dict[str, object]: input_document = { "kind": "canonical-camera-epoch", "session_id": "20260831T083000Z_viewer_live", - "source_id": "sensor.camera.right", + "source_id": source_id, "codec_epoch": 1, "synchronization": "camera-segment-sequence", "media_type": "video/mp4", @@ -173,7 +178,7 @@ def _camera_source(root: Path, *, frame_count: int = 2) -> dict[str, object]: "camera_compute_job": { "job_id": camera_job_id, "input_sha256": input_sha256, - "source_id": "sensor.camera.right", + "source_id": source_id, "codec_epoch": 1, "input_byte_length": input_document["byte_length"], "frame_count": frame_count, @@ -407,6 +412,82 @@ def test_component_request_and_camera_job_are_canonical_and_confined( contract.validate_camera_compute_job(camera_root, request.source) +def test_component_request_accepts_the_persisted_public_camera_identity( + tmp_path: Path, +) -> None: + camera_root = tmp_path / "camera" + source = _camera_source( + camera_root, + source_id="recorded.camera.6a3945242828a038", + ) + expectations = ( + contract.AssetExpectation( + "test-tree", + "/opt/nodedc/assets/test-tree", + "tree", + "identity-sha256", + ), + ) + request_path = tmp_path / "request.json" + _write_canonical( + request_path, + _request_document( + component="eomt", + source=source, + expectations=expectations, + identities={"test-tree": ("c" * 64, 7)}, + ), + ) + + request = contract.load_component_request( + request_path, + component="eomt", + expectations=expectations, + ) + + assert request.source.camera_source_id == "recorded.camera.6a3945242828a038" + contract.validate_camera_compute_job(camera_root, request.source) + + +def test_component_adapter_accepts_only_exact_legacy_or_installed_package_layout() -> None: + expectations = ( + contract.AssetExpectation( + "test-tree", + "/opt/nodedc/assets/test-tree", + "tree", + "identity-sha256", + ), + ) + + legacy = contract.resolve_runtime_layout( + ("--request", contract.FIXED_REQUEST_PATH), + component="eomt", + expectations=expectations, + ) + assert legacy.request == Path(contract.FIXED_REQUEST_PATH) + + installed = contract.resolve_runtime_layout( + ("--package-step", "ddrnet"), + component="ddrnet", + expectations=expectations, + ) + assert installed.request == Path( + "/missioncore/input/steps/prepare/ddrnet-request.json" + ) + assert installed.camera_job_root == Path( + "/missioncore/input/steps/prepare/camera-job" + ) + assert installed.output_root == Path("/missioncore/output") + assert installed.eomt_result_root == Path("/missioncore/input/steps/eomt") + + with pytest.raises(contract.ComponentAdapterError, match="accepts only"): + contract.resolve_runtime_layout( + ("--package-step", "eomt"), + component="ddrnet", + expectations=expectations, + ) + + def test_sealed_tree_receipt_rehashes_payload_and_rejects_tamper( tmp_path: Path, ) -> None: @@ -777,6 +858,8 @@ def test_eomt_adapter_uses_only_fixed_legacy_argv_and_publishes_frames( str(camera_root), "--profile", str(runner_root / "e3_k1_camera1_profile.json"), + "--physical-source-id", + "sensor.camera.right", "--valid-fov-root", str(asset_roots["k1-valid-fov-root"]), "--frames", @@ -885,6 +968,13 @@ def test_eomt_adapter_uses_only_fixed_legacy_argv_and_publishes_frames( ] +def test_eomt_default_disk_floor_retains_large_post_run_reserve() -> None: + assert eomt.DISK_FLOOR_BYTES == 350 * 1024**3 + full_record_working_set = 6_830 * 800 * 600 * 7 + 556_912_640 + assert full_record_working_set < 22 * 1024**3 + assert (eomt.DISK_FLOOR_BYTES + full_record_working_set) < 372 * 1024**3 + + def _effective_ddrnet_config( source: Mapping[str, object], *, @@ -1152,7 +1242,7 @@ def test_ddrnet_adapter_uses_exact_candidate_argv_and_publishes_legacy_result( "--dataset-root", str(tmp_path / "ddrnet-work"), "--frames-root", - str(tmp_path / "ddrnet-work" / "source-frames"), + str(eomt_root / "source-frames"), "--output", str(output / ".ddrnet-component"), "--limit", @@ -1201,7 +1291,9 @@ def test_ddrnet_mask_archive_rejects_class_id_outside_taxonomy(tmp_path: Path) - ddrnet._validate_mask_archive_inventory(archive_path, 1) -def test_ddrnet_snapshot_rejects_same_length_source_frame_tamper(tmp_path: Path) -> None: +def test_ddrnet_read_only_frame_verification_rejects_same_length_tamper( + tmp_path: Path, +) -> None: eomt_root = tmp_path / "eomt-result" frames_root = eomt_root / "source-frames" frames_root.mkdir(parents=True) @@ -1215,11 +1307,28 @@ def test_ddrnet_snapshot_rejects_same_length_source_frame_tamper(tmp_path: Path) sha256=hashlib.sha256(original).hexdigest(), ) frame.write_bytes(b"\x89PNG\r\n\x1a\ntampered") - work = tmp_path / "work" - work.mkdir() - with pytest.raises(contract.ComponentAdapterError, match="source frame identity changed"): - ddrnet._snapshot_source_frames(eomt_root, (row,), work / "source-frames") + ddrnet._verify_source_frames(eomt_root, (row,)) + + +def test_ddrnet_uses_verified_eomt_frames_without_tmpfs_snapshot(tmp_path: Path) -> None: + eomt_root = tmp_path / "eomt-result" + frames_root = eomt_root / "source-frames" + frames_root.mkdir(parents=True) + frame = frames_root / "frame-000001.png" + payload = b"\x89PNG\r\n\x1a\nverified" + frame.write_bytes(payload) + row = ddrnet.SourceFrameRow( + sequence=1, + path="source-frames/frame-000001.png", + byte_length=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), + ) + + admitted = ddrnet._verify_source_frames(eomt_root, (row,)) + + assert admitted == frames_root.resolve(strict=True) + assert tuple(frames_root.iterdir()) == (frame,) def test_eomt_mask_archive_rejects_link_member(tmp_path: Path) -> None: diff --git a/tests/test_observatory_portable_lab_v1_promotion.py b/tests/test_observatory_portable_lab_v1_promotion.py index c569c40..1bf3575 100644 --- a/tests/test_observatory_portable_lab_v1_promotion.py +++ b/tests/test_observatory_portable_lab_v1_promotion.py @@ -8,16 +8,6 @@ from typing import cast import pytest -from k1link.observatory.portable_lab_v1_executor import ( - PortableLabV1ReleaseCandidate, -) -from k1link.observatory.portable_run_definitions import ( - PortableRunDefinitionRegistry, -) -from k1link.observatory.portable_worker_runtime import ( - PortableWorkerRuntimeRegistry, -) - REPOSITORY_ROOT = Path(__file__).resolve().parents[1] DEFINITIONS = REPOSITORY_ROOT / "config" / "observatory-portable-run-definitions.json" RUNTIME = REPOSITORY_ROOT / "config" / "observatory-worker-runtime-candidates.json" @@ -102,8 +92,8 @@ def _input_document( _asset( "eomt-runner-bundle", "/runner", - "3bcfb73db5079deffe51173198f7a02e9e4c49f5fc5439d7976757a430fe91d3", - 144_128, + "0d08f0492d5ad62903874ea224c505e54a6f6059c8283f586bc79e42d55156b7", + 145_141, tree=True, ), _asset( @@ -320,20 +310,7 @@ def _write_installation_evidence(tmp_path: Path) -> tuple[Path, Path]: return component, coordinator -def _files(root: Path) -> dict[str, bytes]: - return { - path.relative_to(root).as_posix(): path.read_bytes() - for path in root.rglob("*") - if path.is_file() - } - - -def _row(document: dict[str, object], collection: str, setup_id: str) -> object: - rows = cast(list[dict[str, object]], document[collection]) - return next(row for row in rows if row["setup_id"] == setup_id) - - -def test_ready_promotion_is_deterministic_additive_and_round_trips( +def test_legacy_ready_promotion_refuses_an_already_promoted_source( tmp_path: Path, ) -> None: work_root = tmp_path / "work" @@ -352,66 +329,20 @@ def test_ready_promotion_is_deterministic_additive_and_round_trips( ) inputs = promotion.load_promotion_input(input_path) - first = promotion.generate_ready_lab_v1_artifacts( - promotion=inputs, - source_definition_registry=DEFINITIONS, - source_runtime_registry=RUNTIME, - ddrnet_portable_config=PORTABLE_CONFIG, - output_root=tmp_path / "ready-a", - ) - second = promotion.generate_ready_lab_v1_artifacts( - promotion=inputs, - source_definition_registry=DEFINITIONS, - source_runtime_registry=RUNTIME, - ddrnet_portable_config=PORTABLE_CONFIG, - output_root=tmp_path / "ready-b", - ) + output_root = tmp_path / "must-not-regenerate" + with pytest.raises( + promotion.PortableLabV1PromotionError, + match="source LAB V1 definition is already ready", + ): + promotion.generate_ready_lab_v1_artifacts( + promotion=inputs, + source_definition_registry=DEFINITIONS, + source_runtime_registry=RUNTIME, + ddrnet_portable_config=PORTABLE_CONFIG, + output_root=output_root, + ) - assert _files(first.root) == _files(second.root) - source_definitions = json.loads(DEFINITIONS.read_text(encoding="utf-8")) - ready_definitions_document = json.loads( - first.definition_registry_path.read_text(encoding="utf-8") - ) - assert _row( - ready_definitions_document, - "definitions", - "m49-tgs-portable-v2", - ) == _row(source_definitions, "definitions", "m49-tgs-portable-v2") - - definitions = PortableRunDefinitionRegistry.from_file( - first.definition_registry_path - ) - definition = definitions.resolve_setup(promotion.PORTABLE_LAB_V1_SETUP_ID) - runtime = PortableWorkerRuntimeRegistry.from_file( - first.runtime_registry_path, - definitions=definitions, - ).resolve(definition.setup_id, definition.definition_sha256) - release = PortableLabV1ReleaseCandidate.from_file( - first.release_candidate_path, - repository_root=first.root, - ) - release.bind_definition(definition) - - assert first.release_candidate_path.name == "lab-v1-executor-release.json" - assert definition.executor.ready - assert runtime.ready - assert [ - asset.asset_id for asset in release.assets if asset.kind == "repository-file" - ] == [promotion.DDRNET_PORTABLE_CONFIG_ASSET_ID] - assert promotion.PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID not in { - asset.asset_id for asset in release.assets - } - receipt_requirement = next( - asset - for asset in runtime.reusable_assets - if asset.asset_id - == promotion.PORTABLE_LAB_V1_WORKER_INSTALLATION_RECEIPT_ASSET_ID - ) - assert receipt_requirement.sha256 == first.installation_receipt_file_sha256 - assert first.release_candidate_sha256 == second.release_candidate_sha256 - assert first.release_sha256 == second.release_sha256 - assert first.definition_sha256 == second.definition_sha256 - assert first.runtime_candidate_sha256 == second.runtime_candidate_sha256 + assert not output_root.exists() def test_promotion_rejects_bare_or_mismatched_installer_claims(