From 3c538ad98c2714be69e7f3c83d875f6912e1fb8c Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 26 Jul 2026 00:09:55 +0300 Subject: [PATCH] fix(deploy): publish device plane loopback health --- ...ane-foundation-network-publication-v1.json | 24 + device-plane/docker-compose.device-plane.yml | 8 + infra/deploy-runner/README.md | 36 + .../build-device-plane-artifact.mjs | 17 +- ...oundation-network-publication-artifact.mjs | 267 ++++++ ...ice-plane-foundation-recovery-artifact.mjs | 11 +- ...vice-plane-foundation-internal-only-v1.yml | 118 +++ infra/deploy-runner/nodedc-deploy | 783 +++++++++++++++++- ...foundation_network_publication_artifact.py | 302 +++++++ ...vice_plane_foundation_recovery_artifact.py | 6 +- 10 files changed, 1565 insertions(+), 7 deletions(-) create mode 100644 device-plane/deployment/device-plane-foundation-network-publication-v1.json create mode 100644 infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs create mode 100644 infra/deploy-runner/fixtures/device-plane-foundation-internal-only-v1.yml create mode 100644 infra/deploy-runner/test_device_plane_foundation_network_publication_artifact.py diff --git a/device-plane/deployment/device-plane-foundation-network-publication-v1.json b/device-plane/deployment/device-plane-foundation-network-publication-v1.json new file mode 100644 index 0000000..a4ae825 --- /dev/null +++ b/device-plane/deployment/device-plane-foundation-network-publication-v1.json @@ -0,0 +1,24 @@ +{ + "schemaVersion": "nodedc.device-plane.foundation-network-publication.v1", + "mode": "failed-foundation-network-publication-correction", + "failedRecoveryPatchId": "device-plane-foundation-recovery-20260725-002", + "failedRecoveryArtifactSha256": "9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf", + "failedRecoveryBackupId": "device-plane-device-plane-foundation-recovery-20260725-002-20260725-232447", + "sourceAction": "publish-network-corrected-foundation-source", + "runtimeAction": "recreate-stateless-services-no-build", + "selectedServices": [ + "device-control-core", + "device-gateway" + ], + "preservedServices": [ + "device-postgres" + ], + "privateNetwork": "nodedc-device-plane-private", + "controlNetwork": "nodedc-device-plane-control", + "publishedLoopbackPorts": [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121" + ], + "databaseVolume": "nodedc-device-plane-postgres-data", + "rollback": "restore-partial-source-and-internal-only-stateless-runtime" +} diff --git a/device-plane/docker-compose.device-plane.yml b/device-plane/docker-compose.device-plane.yml index 15c444c..4943189 100644 --- a/device-plane/docker-compose.device-plane.yml +++ b/device-plane/docker-compose.device-plane.yml @@ -55,6 +55,7 @@ services: - "127.0.0.1:18120:18120" networks: - device-plane-private + - device-plane-control depends_on: device-postgres: condition: service_healthy @@ -93,6 +94,7 @@ services: - "127.0.0.1:18121:18121" networks: - device-plane-private + - device-plane-control security_opt: - no-new-privileges:true cap_drop: @@ -112,6 +114,12 @@ networks: device-plane-private: name: nodedc-device-plane-private internal: true + device-plane-control: + name: nodedc-device-plane-control + driver: bridge + internal: false + driver_opts: + com.docker.network.bridge.enable_ip_masquerade: "false" volumes: device-plane-postgres-data: diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 230a07e..a3aa99c 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -500,6 +500,42 @@ node infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs \ device-plane-foundation-recovery-20260725-002 ``` +The recovery `device-plane-foundation-recovery-20260725-002` is also terminal +failed. Docker Engine 24 does not materialize requested host port mappings when +the container is attached only to an `internal` network. Its registered +successor is the exact +`deployment/device-plane-foundation-network-publication-v1.json` transition. +It keeps PostgreSQL exclusively on the existing internal private network and +adds a second non-internal bridge only to Control Core and Gateway. Masquerade +is disabled on that bridge; the only published ports remain +`127.0.0.1:18120` and `127.0.0.1:18121`. Device TCP `9921`, public ingress, +discovery ingest and command transport remain disabled. + +This successor has an empty build set and force-recreates only +`device-control-core` and `device-gateway` with `--no-deps` from their exact +existing `pull_policy: never` images. Its predecessor barrier validates the +terminal failed archive/journal/backup, partial source, exact running +generations, absent actual port mappings, internal private network and absence +of the control network. Acceptance requires new stateless container +generations, the unchanged PostgreSQL generation and volume, exact actual +loopback mappings, healthy HTTP contracts and a closed `9921`. + +Rollback removes only the two stateless candidate containers and the newly +created unused control network, restores the partial source predecessor and +recreates the internal-only Core/Gateway runtime from the sealed predecessor +Compose bytes. PostgreSQL and its volume are never selected. + +Build and test the deterministic network-publication successor: + +```bash +python3 -m unittest -v \ + infra.deploy-runner.test_device_plane_foundation_network_publication_artifact + +node \ + infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs \ + device-plane-foundation-network-publication-20260725-003 +``` + After the runner is promoted and freshly verified, bootstrap the durable prerequisite with a separate artifact before planning the application: diff --git a/infra/deploy-runner/build-device-plane-artifact.mjs b/infra/deploy-runner/build-device-plane-artifact.mjs index 575e54b..0339436 100644 --- a/infra/deploy-runner/build-device-plane-artifact.mjs +++ b/infra/deploy-runner/build-device-plane-artifact.mjs @@ -18,6 +18,10 @@ import { fileURLToPath } from "node:url"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const platformRoot = resolve(scriptDir, "../.."); const sourceRoot = resolve(platformRoot, "device-plane"); +const failedFoundationCompose = resolve( + scriptDir, + "fixtures/device-plane-foundation-internal-only-v1.yml", +); const artifactDir = resolve( process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"), @@ -54,8 +58,14 @@ await assertSourceBoundary(); try { await mkdir(payload, { recursive: true }); for (const sourceRelative of files) { + const source = ( + patchId === "device-plane-foundation-20260725-001" + && sourceRelative === "docker-compose.device-plane.yml" + ) + ? failedFoundationCompose + : resolve(sourceRoot, sourceRelative); await copySafe( - resolve(sourceRoot, sourceRelative), + source, join(payload, sourceRelative), ); } @@ -109,8 +119,11 @@ try { } async function assertSourceBoundary() { + const composeSource = patchId === "device-plane-foundation-20260725-001" + ? failedFoundationCompose + : resolve(sourceRoot, "docker-compose.device-plane.yml"); const compose = await readFile( - resolve(sourceRoot, "docker-compose.device-plane.yml"), + composeSource, "utf8", ); for (const fragment of [ diff --git a/infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs b/infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs new file mode 100644 index 0000000..ac8c5f2 --- /dev/null +++ b/infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs @@ -0,0 +1,267 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-plane-foundation-network-publication-20260725-003", + ...extra +] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-plane-foundation-network-publication-artifact.mjs " + + "[patch-id]", + ); +} + +const files = [ + ".dockerignore", + "package.json", + "package-lock.json", + "docker-compose.device-plane.yml", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway", + "deployment/device-plane-foundation-network-publication-v1.json", +]; +const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); +const ignoredDirectoryNames = new Set(["test"]); +const stage = await mkdtemp( + join(tmpdir(), "nodedc-device-plane-network-publication-"), +); +const payload = join(stage, "payload"); +const target = join( + artifactDir, + `nodedc-device-plane-${patchId}.tgz`, +); + +await assertBoundary(); + +try { + await mkdir(payload, { recursive: true }); + for (const sourceRelative of files) { + await copySafe( + resolve(sourceRoot, sourceRelative), + join(payload, sourceRelative), + ); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile( + join(stage, "files.txt"), + `${files.join("\n")}\n`, + "utf8", + ); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + }, + ); + if (tar.status !== 0) { + throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + } + + const digest = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + sha256: digest, + component: "device-plane", + transition: "failed-foundation-network-publication-correction", + entries: files, + build: [], + services: ["device-control-core", "device-gateway"], + preservedRuntime: [ + "device-postgres", + "nodedc-device-plane-postgres-data", + "Gelios", + ], + networkChange: { + private: "preserved:internal", + control: "create:non-internal:no-masquerade", + published: [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121", + ], + disabled: ["9921", "public-ingress", "command-transport"], + }, + rollback: + "restore-partial-source-and-internal-only-stateless-runtime", + excluded: [ + ".env*", + "node_modules", + "**/test", + "docs", + "runtime", + "secrets", + ], + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertBoundary() { + const compose = await readFile( + resolve(sourceRoot, "docker-compose.device-plane.yml"), + "utf8", + ); + for (const fragment of [ + 'DEVICE_DISCOVERY_INGEST_ENABLED: "false"', + 'DEVICE_GATEWAY_LISTEN_ENABLED: "false"', + '"127.0.0.1:18120:18120"', + '"127.0.0.1:18121:18121"', + "name: nodedc-device-plane-private", + "internal: true", + "name: nodedc-device-plane-control", + "internal: false", + 'com.docker.network.bridge.enable_ip_masquerade: "false"', + "name: nodedc-device-plane-postgres-data", + "pull_policy: never", + ]) { + if (!compose.includes(fragment)) { + throw new Error( + `device_plane_network_publication_boundary_missing:${fragment}`, + ); + } + } + for (const forbidden of [ + "9921:9921", + "0.0.0.0:9921", + 'DEVICE_DISCOVERY_INGEST_ENABLED: "true"', + 'DEVICE_GATEWAY_LISTEN_ENABLED: "true"', + "POSTGRES_PASSWORD:", + ]) { + if (compose.includes(forbidden)) { + throw new Error( + `device_plane_network_publication_boundary_violation:${forbidden}`, + ); + } + } + + const descriptor = JSON.parse(await readFile( + resolve( + sourceRoot, + "deployment/device-plane-foundation-network-publication-v1.json", + ), + "utf8", + )); + const expected = { + schemaVersion: + "nodedc.device-plane.foundation-network-publication.v1", + mode: "failed-foundation-network-publication-correction", + failedRecoveryPatchId: + "device-plane-foundation-recovery-20260725-002", + failedRecoveryArtifactSha256: + "9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf", + failedRecoveryBackupId: + "device-plane-device-plane-foundation-recovery-20260725-002-20260725-232447", + sourceAction: "publish-network-corrected-foundation-source", + runtimeAction: "recreate-stateless-services-no-build", + selectedServices: ["device-control-core", "device-gateway"], + preservedServices: ["device-postgres"], + privateNetwork: "nodedc-device-plane-private", + controlNetwork: "nodedc-device-plane-control", + publishedLoopbackPorts: [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121", + ], + databaseVolume: "nodedc-device-plane-postgres-data", + rollback: + "restore-partial-source-and-internal-only-stateless-runtime", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error( + "device_plane_network_publication_descriptor_mismatch", + ); + } +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(sourceRoot, source)}`, + ); + } + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { + force: true, + verbatimSymlinks: true, + }); + return; + } + if (!sourceStat.isDirectory()) { + throw new Error(`source_type_rejected:${source}`); + } + + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + ignoredBasenames.has(entry.name) + || entry.name.startsWith(".env") + || ( + entry.isDirectory() + && ignoredDirectoryNames.has(entry.name) + ) + ) { + continue; + } + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(sourceRoot, childSource)}`, + ); + } + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs b/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs index f3beea3..c82a43b 100755 --- a/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs +++ b/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs @@ -18,6 +18,10 @@ import { fileURLToPath } from "node:url"; const scriptDir = dirname(fileURLToPath(import.meta.url)); const platformRoot = resolve(scriptDir, "../.."); const sourceRoot = resolve(platformRoot, "device-plane"); +const predecessorCompose = resolve( + scriptDir, + "fixtures/device-plane-foundation-internal-only-v1.yml", +); const artifactDir = resolve( process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"), @@ -64,8 +68,11 @@ await assertRecoveryBoundary(); try { await mkdir(payload, { recursive: true }); for (const sourceRelative of files) { + const source = sourceRelative === "docker-compose.device-plane.yml" + ? predecessorCompose + : resolve(sourceRoot, sourceRelative); await copySafe( - resolve(sourceRoot, sourceRelative), + source, join(payload, sourceRelative), ); } @@ -129,7 +136,7 @@ try { async function assertRecoveryBoundary() { const compose = await readFile( - resolve(sourceRoot, "docker-compose.device-plane.yml"), + predecessorCompose, "utf8", ); for (const fragment of [ diff --git a/infra/deploy-runner/fixtures/device-plane-foundation-internal-only-v1.yml b/infra/deploy-runner/fixtures/device-plane-foundation-internal-only-v1.yml new file mode 100644 index 0000000..15c444c --- /dev/null +++ b/infra/deploy-runner/fixtures/device-plane-foundation-internal-only-v1.yml @@ -0,0 +1,118 @@ +services: + device-postgres: + image: postgres:16-alpine + pull_policy: missing + restart: unless-stopped + environment: + POSTGRES_DB: device_plane + POSTGRES_USER: device_plane + POSTGRES_PASSWORD_FILE: /run/nodedc-secrets/postgres-password + volumes: + - type: volume + source: device-plane-postgres-data + target: /var/lib/postgresql/data + - type: bind + source: /volume1/docker/nodedc-device-plane/secrets/postgres-password + target: /run/nodedc-secrets/postgres-password + read_only: true + bind: + create_host_path: false + networks: + - device-plane-private + healthcheck: + test: ["CMD-SHELL", "pg_isready -U device_plane -d device_plane"] + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + + device-control-core: + image: nodedc/device-control-core:local + pull_policy: never + restart: unless-stopped + user: "1000:1000" + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + environment: + HOST: 0.0.0.0 + PORT: "18120" + DEVICE_DATABASE_HOST: device-postgres + DEVICE_DATABASE_PORT: "5432" + DEVICE_DATABASE_NAME: device_plane + DEVICE_DATABASE_USER: device_plane + DEVICE_DATABASE_PASSWORD_FILE: /run/nodedc-secrets/postgres-password + DEVICE_DATABASE_POOL_SIZE: "10" + DEVICE_DISCOVERY_INGEST_ENABLED: "false" + volumes: + - type: bind + source: /volume1/docker/nodedc-device-plane/secrets/postgres-password + target: /run/nodedc-secrets/postgres-password + read_only: true + bind: + create_host_path: false + ports: + - "127.0.0.1:18120:18120" + networks: + - device-plane-private + depends_on: + device-postgres: + condition: service_healthy + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD + - node + - -e + - fetch('http://127.0.0.1:18120/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1)) + interval: 10s + timeout: 5s + retries: 12 + start_period: 20s + + device-gateway: + image: nodedc/device-gateway:local + pull_policy: never + restart: unless-stopped + user: "1000:1000" + read_only: true + tmpfs: + - /tmp:size=16m,mode=1777 + environment: + DEVICE_GATEWAY_HEALTH_HOST: 0.0.0.0 + DEVICE_GATEWAY_HEALTH_PORT: "18121" + DEVICE_GATEWAY_LISTEN_ENABLED: "false" + DEVICE_GATEWAY_TCP_HOST: 127.0.0.1 + DEVICE_GATEWAY_TCP_PORT: "9921" + DEVICE_GATEWAY_MAX_SESSIONS: "100" + DEVICE_GATEWAY_SESSION_TIMEOUT_MS: "10000" + ports: + - "127.0.0.1:18121:18121" + networks: + - device-plane-private + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + healthcheck: + test: + - CMD + - node + - -e + - fetch('http://127.0.0.1:18121/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1)) + interval: 10s + timeout: 5s + retries: 12 + start_period: 10s + +networks: + device-plane-private: + name: nodedc-device-plane-private + internal: true + +volumes: + device-plane-postgres-data: + name: nodedc-device-plane-postgres-data diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 4f48be7..6b67477 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -7,6 +7,7 @@ import os import re import secrets import shutil +import socket import stat import subprocess import sys @@ -69,6 +70,13 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES = ( *DEVICE_PLANE_FOUNDATION_ENTRIES, DEVICE_PLANE_FOUNDATION_RECOVERY_REL, ) +DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL = ( + "deployment/device-plane-foundation-network-publication-v1.json" +) +DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES = ( + *DEVICE_PLANE_FOUNDATION_ENTRIES, + DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, +) DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = ( "device-plane-foundation-20260725-001" ) @@ -116,6 +124,56 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS = { "sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d" ), } +DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID = ( + "device-plane-foundation-recovery-20260725-002" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-plane-foundation-recovery-20260725-002.tgz." + "20260725-232447" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 = ( + "9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf" +) +DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID = ( + "device-plane-device-plane-foundation-recovery-20260725-002-" + "20260725-232447" +) +DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256 = { + "manifest.env": ( + "e1c31f28474d66c33f8b91e890c112e605ffe376ed3e7292d2f8504ea18fd9c1" + ), + "files.txt": ( + "bb0e3deec5952811dd413afdf24d116e5bf3fd3e041f78bfe5cc5ca87166b8ff" + ), + "existing-files.txt": ( + "f2e2c6e7501d3a383cd7e909a2e4e4cfbb3b0cda132f508f54e478bfb455585b" + ), + "missing-files.txt": ( + "e57cda977037cabbe01de04ece1a1e9f15cb9eb3c0ad2996811d6ea3d4f4b3d2" + ), + "runtime-before.json": ( + "f500964078c5814d2130fe57b5c67fb89dea000355591836e1b72088b37170ef" + ), + "source-before.tgz": ( + "eed474b04608e206f09ebe68ca80da675a2318fa3efaf3be90f2d983429f9d9f" + ), +} +DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256 = ( + "47d4d153d27bc6f418c1f787ea5ff0153b1ee082f96b92e28486805cb46b3f15" +) +DEVICE_PLANE_PRIVATE_NETWORK = "nodedc-device-plane-private" +DEVICE_PLANE_CONTROL_NETWORK = "nodedc-device-plane-control" +DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS = { + "device-control-core": ( + "ccdaeee71472a557ccecb485e81b31f5a4ee37b1b3f3cb4d5edbcf12d02bb08b" + ), + "device-gateway": ( + "d59258eb86ae8ec728ceaaed5b4976dd1359bdcecd6abc7950e73ffabf1d8eba" + ), + "device-postgres": ( + "44702b995b4397131646ed8076144efd116cb38d00d9a3d641252b7ad635d7a1" + ), +} DEVICE_PLANE_RUNTIME_SERVICES = ( "device-control-core", "device-gateway", @@ -2806,6 +2864,7 @@ def allowed_payload_path(component, rel): "docker-compose.device-plane.yml", DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, DEVICE_PLANE_FOUNDATION_RECOVERY_REL, + DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, "packages/device-protocol-contract", "packages/arusnavi-b2-adapter", "services/device-control-core", @@ -7835,6 +7894,13 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_foundation_recovery_payload(payload_dir) + if is_device_plane_foundation_network_publication_slice( + manifest["component"], + entries, + ): + validate_device_plane_foundation_network_publication_payload( + payload_dir + ) if manifest["component"] == "n8n-private-extension": validate_n8n_private_extension_release(payload_dir, entries) if manifest["component"] == "engine": @@ -8118,6 +8184,16 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256): "Device Plane foundation 001 is terminal failed; " "use the exact registered foundation recovery successor" ) + if ( + manifest.get("id") + == DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID + or sha256 + == DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 + ): + die( + "Device Plane foundation recovery 002 is terminal failed; " + "use the exact registered network-publication successor" + ) class DeployLock: @@ -8153,6 +8229,15 @@ def is_device_plane_foundation_recovery_slice(component, entries): ) +def is_device_plane_foundation_network_publication_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) + == DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES + ) + + def expected_device_plane_foundation_recovery_descriptor(): return { "schemaVersion": "nodedc.device-plane.foundation-recovery.v1", @@ -8185,6 +8270,65 @@ def validate_device_plane_foundation_recovery_payload(payload_dir): return descriptor +def expected_device_plane_foundation_network_publication_descriptor(): + return { + "schemaVersion": ( + "nodedc.device-plane.foundation-network-publication.v1" + ), + "mode": "failed-foundation-network-publication-correction", + "failedRecoveryPatchId": ( + DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID + ), + "failedRecoveryArtifactSha256": ( + DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 + ), + "failedRecoveryBackupId": ( + DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID + ), + "sourceAction": "publish-network-corrected-foundation-source", + "runtimeAction": "recreate-stateless-services-no-build", + "selectedServices": [ + "device-control-core", + "device-gateway", + ], + "preservedServices": ["device-postgres"], + "privateNetwork": DEVICE_PLANE_PRIVATE_NETWORK, + "controlNetwork": DEVICE_PLANE_CONTROL_NETWORK, + "publishedLoopbackPorts": [ + "127.0.0.1:18120:18120", + "127.0.0.1:18121:18121", + ], + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "rollback": ( + "restore-partial-source-and-internal-only-stateless-runtime" + ), + } + + +def validate_device_plane_foundation_network_publication_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, + "Device Plane foundation network-publication descriptor", + max_bytes=16 * 1024, + ) + expected = ( + expected_device_plane_foundation_network_publication_descriptor() + ) + if descriptor != expected: + die( + "Device Plane foundation network-publication descriptor mismatch" + ) + compose = payload_dir / "docker-compose.device-plane.yml" + if ( + compose.is_symlink() + or not compose.is_file() + or sha256_file(compose) + != DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256 + ): + die("Device Plane network-publication Compose mismatch") + return descriptor + + def device_plane_service_container_ids(service): if service not in DEVICE_PLANE_RUNTIME_SERVICES: die(f"Device Plane runtime service is not registered: {service}") @@ -8464,6 +8608,263 @@ def validate_device_plane_foundation_recovery_evidence(payload_dir): } +def inspect_device_plane_network_optional(name): + result = subprocess.run( + [str(DOCKER), "network", "inspect", name], + check=False, + capture_output=True, + text=True, + ) + if result.returncode == 1: + return None + if result.returncode != 0: + die(f"Device Plane network inspect failed: {name}") + try: + value = json.loads(result.stdout) + except json.JSONDecodeError: + die(f"Device Plane network inspect returned invalid JSON: {name}") + if ( + not isinstance(value, list) + or len(value) != 1 + or not isinstance(value[0], dict) + ): + die(f"Device Plane network inspect shape mismatch: {name}") + return value[0] + + +def validate_device_plane_network_contract( + name, + *, + internal, + expected_container_ids, +): + network = inspect_device_plane_network_optional(name) + if network is None: + die(f"Device Plane network is missing: {name}") + options = network.get("Options") or {} + if ( + network.get("Name") != name + or network.get("Driver") != "bridge" + or network.get("Internal") is not internal + or set((network.get("Containers") or {}).keys()) + != set(expected_container_ids) + ): + die(f"Device Plane network boundary mismatch: {name}") + if ( + name == DEVICE_PLANE_CONTROL_NETWORK + and options.get( + "com.docker.network.bridge.enable_ip_masquerade" + ) + != "false" + ): + die("Device Plane control network masquerade boundary mismatch") + return network + + +def validate_device_plane_foundation_network_publication_evidence( + payload_dir, +): + validate_device_plane_foundation_network_publication_payload(payload_dir) + + backup_dir = ( + BACKUPS_DIR + / DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID + ) + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Plane failed recovery backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Plane failed recovery backup is unsafe") + backup_names = {child.name for child in backup_dir.iterdir()} + expected_backup = ( + DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256 + ) + if backup_names != set(expected_backup): + die("Device Plane failed recovery backup file set mismatch") + for name, expected_sha256 in expected_backup.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Device Plane failed recovery backup drift detected: " + f"{name}" + ) + + failed_artifact = ( + FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT + ) + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Plane failed recovery artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 + ): + die("Device Plane failed recovery artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text( + encoding="utf-8" + ).splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Device Plane failed recovery journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( + state_stat.st_mode + ): + die("Device Plane failed recovery journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Device Plane failed recovery journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") + == DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Device Plane failed recovery journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") + != DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") + != ( + "ok:device-plane-overlay:" + "source-restored-runtime-unchanged:9" + ) + or record.get("status") != "failed" + or record.get("message") + != ( + "healthcheck failed for http://127.0.0.1:18120/healthz: " + "" + ) + ): + die("Device Plane failed recovery journal evidence mismatch") + + comparable_entries = tuple( + rel + for rel in DEVICE_PLANE_FOUNDATION_ENTRIES + if rel != "docker-compose.device-plane.yml" + ) + with tempfile.TemporaryDirectory( + prefix="device-plane-failed-recovery-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) + != DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES + ): + die("Device Plane failed recovery artifact contract mismatch") + failed_source = collect_exact_files( + failed_payload, + comparable_entries, + "Device Plane failed recovery source", + ) + candidate_source = collect_exact_files( + payload_dir, + comparable_entries, + "Device Plane network-publication candidate source", + ) + if candidate_source != failed_source: + die( + "Device Plane network-publication candidate source drift " + "detected" + ) + + root = component_root("device-plane") + compose_path = root / "docker-compose.device-plane.yml" + if ( + compose_path.is_symlink() + or not compose_path.is_file() + or sha256_file(compose_path) + != DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256 + ): + die("Device Plane network-publication source predecessor mismatch") + for rel in DEVICE_PLANE_FOUNDATION_ENTRIES: + if rel == "docker-compose.device-plane.yml": + continue + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Device Plane network-publication predecessor retained " + f"source: {rel}" + ) + for descriptor_rel in ( + DEVICE_PLANE_FOUNDATION_RECOVERY_REL, + DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, + ): + descriptor_path = root / descriptor_rel + if descriptor_path.exists() or descriptor_path.is_symlink(): + die( + "Device Plane network-publication descriptor predecessor " + "mismatch" + ) + bootstrap_descriptor = root / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL + if ( + bootstrap_descriptor.is_symlink() + or not bootstrap_descriptor.is_file() + or sha256_file(bootstrap_descriptor) + != DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256 + ): + die("Device Plane PostgreSQL bootstrap descriptor drift detected") + + runtime = validate_device_plane_foundation_runtime() + for service, expected_id in ( + DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS.items() + ): + if runtime[service]["containerId"] != expected_id: + die( + "Device Plane network-publication runtime predecessor " + f"generation mismatch: {service}" + ) + expected_network_ids = { + item["containerId"] + for item in runtime.values() + } + validate_device_plane_network_contract( + DEVICE_PLANE_PRIVATE_NETWORK, + internal=True, + expected_container_ids=expected_network_ids, + ) + if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK): + die("Device Plane control network already exists") + + return { + "mode": "failed-foundation-network-publication-correction", + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def validate_device_plane_postgres_bootstrap_payload(payload_dir): descriptor = read_strict_json( payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, @@ -8537,7 +8938,7 @@ def device_plane_postgres_plan_selection(postgres_preflight): return "bootstrap-selected:create-if-absent" -def validate_device_plane_foundation_runtime(): +def validate_device_plane_foundation_runtime(network_publication=False): validate_device_plane_runtime_secret_metadata() expected = { "device-control-core": { @@ -8552,6 +8953,24 @@ def validate_device_plane_foundation_runtime(): "HostPort": "18120", }], }, + "actualPorts": ( + { + "18120/tcp": [{ + "HostIp": "127.0.0.1", + "HostPort": "18120", + }], + } + if network_publication + else {} + ), + "networks": ( + { + DEVICE_PLANE_PRIVATE_NETWORK, + DEVICE_PLANE_CONTROL_NETWORK, + } + if network_publication + else {DEVICE_PLANE_PRIVATE_NETWORK} + ), }, "device-gateway": { "image": DEVICE_PLANE_GATEWAY_IMAGE, @@ -8565,6 +8984,24 @@ def validate_device_plane_foundation_runtime(): "HostPort": "18121", }], }, + "actualPorts": ( + { + "18121/tcp": [{ + "HostIp": "127.0.0.1", + "HostPort": "18121", + }], + } + if network_publication + else {} + ), + "networks": ( + { + DEVICE_PLANE_PRIVATE_NETWORK, + DEVICE_PLANE_CONTROL_NETWORK, + } + if network_publication + else {DEVICE_PLANE_PRIVATE_NETWORK} + ), }, "device-postgres": { "image": "postgres:16-alpine", @@ -8573,6 +9010,8 @@ def validate_device_plane_foundation_runtime(): ], "user": "", "ports": {}, + "actualPorts": {}, + "networks": {DEVICE_PLANE_PRIVATE_NETWORK}, }, } accepted = {} @@ -8591,6 +9030,9 @@ def validate_device_plane_foundation_runtime(): networks = (container.get("NetworkSettings") or {}).get( "Networks" ) or {} + actual_ports = (container.get("NetworkSettings") or {}).get( + "Ports" + ) or {} if ( state.get("Status") != "running" or state.get("Running") is not True @@ -8614,7 +9056,8 @@ def validate_device_plane_foundation_runtime(): or labels.get("com.docker.compose.project") != "nodedc-device-plane" or labels.get("com.docker.compose.service") != service - or set(networks) != {"nodedc-device-plane-private"} + or set(networks) != contract["networks"] + or actual_ports != contract["actualPorts"] ): die( "Device Plane foundation runtime boundary mismatch: " @@ -8627,6 +9070,24 @@ def validate_device_plane_foundation_runtime(): "restartCount": 0, } + if network_publication: + for service in ("device-control-core", "device-gateway"): + if ( + accepted[service]["containerId"] + == DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[service] + ): + die( + "Device Plane stateless generation was not recreated: " + f"{service}" + ) + if ( + accepted["device-postgres"]["containerId"] + != DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[ + "device-postgres" + ] + ): + die("Device Plane PostgreSQL generation changed") + core = inspect_device_plane_container( device_plane_service_container_ids("device-control-core")[0] ) @@ -8752,6 +9213,23 @@ def validate_device_plane_foundation_runtime(): or postgres_volume[0].get("RW") is not True ): die("Device Plane PostgreSQL preserved mount mismatch") + if network_publication: + validate_device_plane_network_contract( + DEVICE_PLANE_PRIVATE_NETWORK, + internal=True, + expected_container_ids={ + accepted[service]["containerId"] + for service in DEVICE_PLANE_RUNTIME_SERVICES + }, + ) + validate_device_plane_network_contract( + DEVICE_PLANE_CONTROL_NETWORK, + internal=False, + expected_container_ids={ + accepted["device-control-core"]["containerId"], + accepted["device-gateway"]["containerId"], + }, + ) return accepted @@ -8834,6 +9312,75 @@ def validate_device_plane_foundation_installed_source(): return actual +def validate_device_plane_foundation_network_publication_installed_source(): + failed_artifact = ( + FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT + ) + comparable_entries = tuple( + rel + for rel in DEVICE_PLANE_FOUNDATION_ENTRIES + if rel != "docker-compose.device-plane.yml" + ) + with tempfile.TemporaryDirectory( + prefix="device-plane-installed-network-publication-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID + or tuple(failed_entries) + != DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES + ): + die("Device Plane failed recovery source contract mismatch") + expected = collect_exact_files( + failed_payload, + comparable_entries, + "Device Plane failed recovery source", + ) + actual = collect_exact_files( + component_root("device-plane"), + comparable_entries, + "Device Plane installed network-publication source", + ) + if actual != expected: + die("Device Plane installed network-publication source mismatch") + compose = ( + component_root("device-plane") + / "docker-compose.device-plane.yml" + ) + if ( + compose.is_symlink() + or not compose.is_file() + or sha256_file(compose) + != DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256 + ): + die("Device Plane installed network-publication Compose mismatch") + descriptor = read_strict_json( + component_root("device-plane") + / DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL, + "installed Device Plane network-publication descriptor", + max_bytes=16 * 1024, + ) + if ( + descriptor + != expected_device_plane_foundation_network_publication_descriptor() + ): + die( + "installed Device Plane network-publication descriptor mismatch" + ) + recovery_descriptor = ( + component_root("device-plane") + / DEVICE_PLANE_FOUNDATION_RECOVERY_REL + ) + if recovery_descriptor.exists() or recovery_descriptor.is_symlink(): + die("terminal Device Plane recovery descriptor is installed") + return actual + + def component_compose_root(component): return COMPONENTS[component].get("compose_root", component_root(component)) @@ -10366,6 +10913,14 @@ def component_services(component, entries=None): # runtime without build/recreate/restart. return () + if is_device_plane_foundation_network_publication_slice( + component, + entries, + ): + # Recreate only the two stateless services from their exact existing + # image IDs. PostgreSQL is a preserved prerequisite, never selected. + return ("device-control-core", "device-gateway") + if is_engine_l2_closed_loop_slice(component, entries): # The failed 030 apply published both the backend source and the built # UI before Compose rejected the descriptor/source mismatch. The exact @@ -10703,6 +11258,12 @@ def component_builds(component, entries=None): if is_device_plane_foundation_recovery_slice(component, entries): return () + if is_device_plane_foundation_network_publication_slice( + component, + entries, + ): + return () + if component == "device-plane" and entries is not None: selected_services = component_services(component, entries) builds = [] @@ -12675,6 +13236,7 @@ def plan_artifact(artifact): mcp_autonomy_provider_v5_preflight = None l2_closed_loop_preflight = None device_plane_foundation_recovery_preflight = None + device_plane_network_publication_preflight = None device_plane_runtime_before = None composite_provider_v4_preflight = None provider_rotating_slot_preflight = None @@ -12849,6 +13411,15 @@ def plan_artifact(artifact): payload_dir ) ) + if is_device_plane_foundation_network_publication_slice( + manifest["component"], + entries, + ): + device_plane_network_publication_preflight = ( + validate_device_plane_foundation_network_publication_evidence( + payload_dir + ) + ) component = manifest["component"] root = component_root(component) @@ -14257,6 +14828,53 @@ def plan_artifact(artifact): ) print("device_plane_source_action=publish-exact-failed-source") print("device_plane_rollback=source-only-runtime-unchanged") + if device_plane_network_publication_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_network_publication_preflight['mode']}" + ) + print( + "failed_recovery_patch=" + f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID}" + ) + print( + "failed_recovery_artifact_sha256=" + f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256}" + ) + print( + "failed_recovery_backup=" + f"{device_plane_network_publication_preflight['backup'].name}" + ) + print("device_plane_build=none") + print( + "device_plane_runtime_mutation=" + "recreate:device-control-core,device-gateway" + ) + print( + "device_plane_runtime_services=" + "preserved:device-postgres" + ) + print( + "device_plane_private_network=" + "preserved:internal:true" + ) + print( + "device_plane_control_network=" + "create:internal:false:masquerade:false" + ) + print( + "device_plane_actual_ports=" + "required:127.0.0.1:18120,127.0.0.1:18121" + ) + print("device_gateway_tcp_9921=disabled:unpublished") + print( + "device_plane_source_action=" + "publish-network-corrected-foundation-source" + ) + print( + "device_plane_rollback=" + "partial-source+internal-only-stateless-runtime" + ) if device_plane_postgres_preflight is not None: print( "device_postgres_bootstrap=" @@ -14545,6 +15163,83 @@ def rollback_platform_apply(root, backup_dir, entries, current_stamp, runtime_st return f"source+runtime-restored:{restored_count}" +def remove_device_plane_control_network_if_unused(): + network = inspect_device_plane_network_optional( + DEVICE_PLANE_CONTROL_NETWORK + ) + if network is None: + return "absent" + if ( + network.get("Name") != DEVICE_PLANE_CONTROL_NETWORK + or network.get("Driver") != "bridge" + or network.get("Internal") is not False + or (network.get("Containers") or {}) + or (network.get("Options") or {}).get( + "com.docker.network.bridge.enable_ip_masquerade" + ) + != "false" + ): + die("Device Plane control network is unsafe to remove") + subprocess.run( + [ + str(DOCKER), + "network", + "rm", + DEVICE_PLANE_CONTROL_NETWORK, + ], + check=True, + ) + if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK): + die("Device Plane control network removal failed") + return "removed" + + +def rollback_device_plane_network_publication_apply( + root, + backup_dir, + entries, + current_stamp, + runtime_started, + applied_services, +): + if tuple(applied_services) != ( + "device-control-core", + "device-gateway", + ): + die("Device Plane network-publication rollback service mismatch") + if runtime_started: + stop_and_remove_compose_services( + "device-plane", + applied_services, + ) + restored_count = restore_platform_overlay( + root, + backup_dir, + entries, + current_stamp, + ) + remove_device_plane_control_network_if_unused() + run_compose( + "device-plane", + applied_services, + ("docker-compose.device-plane.yml",), + ) + for service in DEVICE_PLANE_RUNTIME_SERVICES: + healthcheck_compose_service("device-plane", service) + runtime = validate_device_plane_foundation_runtime() + if ( + runtime["device-postgres"]["containerId"] + != DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[ + "device-postgres" + ] + ): + die("Device Plane rollback changed PostgreSQL generation") + assert_loopback_tcp_port_closed(18120) + assert_loopback_tcp_port_closed(18121) + assert_loopback_tcp_port_closed(9921) + return f"source+internal-runtime-restored:{restored_count}" + + def rollback_device_plane_apply( root, backup_dir, @@ -15313,6 +16008,13 @@ def run_component_runtime(component, entries, services): return if is_device_plane_foundation_recovery_slice(component, entries): return + if is_device_plane_foundation_network_publication_slice( + component, + entries, + ): + prepare_component_runtime(component, entries) + run_compose(component, services, entries) + return run_build(component, entries) prepare_component_runtime(component, entries) if is_engine_node_intelligence_transition(component, entries): @@ -15365,6 +16067,15 @@ def healthcheck_url(check): die(f"healthcheck failed for {url}: {last_error}") +def assert_loopback_tcp_port_closed(port): + try: + connection = socket.create_connection(("127.0.0.1", port), timeout=3) + except OSError: + return + connection.close() + die(f"unexpected loopback TCP listener is open: {port}") + + def external_data_plane_healthcheck(require_managed=True): expected_json = { "ok": True, @@ -15896,6 +16607,31 @@ def run_healthchecks(component, entries=None, services=None): validate_device_plane_foundation_installed_source() validate_device_plane_foundation_runtime() return + if is_device_plane_foundation_network_publication_slice( + component, + entries, + ): + if tuple(services or ()) != ( + "device-control-core", + "device-gateway", + ): + die( + "Device Plane network-publication service set mismatch" + ) + for service in ( + "device-control-core", + "device-gateway", + "device-postgres", + ): + healthcheck_compose_service("device-plane", service) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + validate_device_plane_foundation_network_publication_installed_source() + validate_device_plane_foundation_runtime( + network_publication=True + ) + assert_loopback_tcp_port_closed(9921) + return if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): return if is_engine_l2_closed_loop_slice(component, entries): @@ -16706,6 +17442,13 @@ def apply_artifact(artifact): payload_dir ) ) + if is_device_plane_foundation_network_publication_slice( + component, + entries, + ): + validate_device_plane_foundation_network_publication_evidence( + payload_dir + ) if not root.is_dir(): if bootstrap_root: root.mkdir(parents=True, exist_ok=True) @@ -17289,6 +18032,42 @@ def apply_artifact(artifact): except Exception as rollback_exc: rollback_status = f"failed:{type(rollback_exc).__name__}" print("platform-automatic-rollback=failed", file=sys.stderr) + elif ( + is_device_plane_foundation_network_publication_slice( + component, + entries, + ) + and services is not None + ): + try: + restored_state = ( + rollback_device_plane_network_publication_apply( + root, + backup_dir, + entries, + current_stamp, + runtime_started, + services, + ) + ) + rollback_status = ( + "ok:device-plane-network-publication:" + f"{restored_state}" + ) + print( + "device-plane-network-publication-" + f"automatic-rollback={rollback_status}", + file=sys.stderr, + ) + except Exception as rollback_exc: + rollback_status = ( + f"failed:{type(rollback_exc).__name__}" + ) + print( + "device-plane-network-publication-" + "automatic-rollback=failed", + file=sys.stderr, + ) elif ( component == "device-plane" and entries is not None diff --git a/infra/deploy-runner/test_device_plane_foundation_network_publication_artifact.py b/infra/deploy-runner/test_device_plane_foundation_network_publication_artifact.py new file mode 100644 index 0000000..cfab9ea --- /dev/null +++ b/infra/deploy-runner/test_device_plane_foundation_network_publication_artifact.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +BUILDER = ( + SCRIPT_DIR + / "build-device-plane-foundation-network-publication-artifact.mjs" +) +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +COMPOSE = ( + SCRIPT_DIR.parent.parent + / "device-plane/docker-compose.device-plane.yml" +) +PREDECESSOR_COMPOSE = ( + SCRIPT_DIR + / "fixtures/device-plane-foundation-internal-only-v1.yml" +) + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_device_plane_network_publication_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DevicePlaneFoundationNetworkPublicationArtifactTest( + unittest.TestCase +): + def build(self, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(BUILDER), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_artifact_is_exact_deterministic_and_database_free(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-network-publication-", + ) as directory: + artifact_dir = Path(directory) + patch_id = ( + "device-plane-foundation-network-publication-unit-003" + ) + first = self.build(artifact_dir, patch_id) + first_bytes = Path(first["artifact"]).read_bytes() + second = self.build(artifact_dir, patch_id) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + self.assertEqual( + first["transition"], + "failed-foundation-network-publication-correction", + ) + self.assertEqual(first["build"], []) + self.assertEqual( + first["services"], + ["device-control-core", "device-gateway"], + ) + self.assertNotIn("device-postgres", first["services"]) + self.assertEqual( + first["entries"], + list( + RUNNER + .DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES + ), + ) + + with tarfile.open(first["artifact"], "r:gz") as archive: + files = ( + archive.extractfile("files.txt") + .read() + .decode("utf-8") + .splitlines() + ) + compose = ( + archive.extractfile( + "payload/docker-compose.device-plane.yml" + ) + .read() + ) + descriptor = json.loads( + archive.extractfile( + "payload/deployment/" + "device-plane-foundation-network-publication-v1.json" + ) + .read() + .decode("utf-8") + ) + + self.assertEqual(files, first["entries"]) + self.assertEqual( + hashlib.sha256(compose).hexdigest(), + RUNNER + .DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256, + ) + self.assertEqual( + descriptor, + RUNNER + .expected_device_plane_foundation_network_publication_descriptor(), + ) + + def test_compose_preserves_database_and_adds_control_network_only_to_stateless_services( + self, + ): + compose = COMPOSE.read_text(encoding="utf-8") + predecessor = PREDECESSOR_COMPOSE.read_text(encoding="utf-8") + self.assertEqual( + hashlib.sha256(PREDECESSOR_COMPOSE.read_bytes()).hexdigest(), + RUNNER.DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256, + ) + self.assertEqual( + hashlib.sha256(COMPOSE.read_bytes()).hexdigest(), + RUNNER + .DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256, + ) + self.assertIn("name: nodedc-device-plane-control", compose) + self.assertIn( + 'com.docker.network.bridge.enable_ip_masquerade: "false"', + compose, + ) + self.assertNotIn("nodedc-device-plane-control", predecessor) + postgres_section, stateless = compose.split( + " device-control-core:", + 1, + ) + self.assertNotIn("device-plane-control", postgres_section) + self.assertEqual(stateless.count(" - device-plane-control"), 2) + self.assertNotIn("9921:9921", compose) + + def test_runner_selects_exact_services_no_build_and_no_deps(self): + entries = ( + RUNNER.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES + ) + services = RUNNER.component_services("device-plane", entries) + self.assertEqual( + services, + ("device-control-core", "device-gateway"), + ) + self.assertEqual( + RUNNER.component_builds("device-plane", entries), + (), + ) + + with ( + mock.patch.object(RUNNER, "run_build") as run_build, + mock.patch.object( + RUNNER, + "prepare_component_runtime", + ) as prepare, + mock.patch.object(RUNNER, "run_compose") as run_compose, + ): + RUNNER.run_component_runtime( + "device-plane", + entries, + services, + ) + run_build.assert_not_called() + prepare.assert_called_once_with("device-plane", entries) + run_compose.assert_called_once_with( + "device-plane", + services, + entries, + ) + + with mock.patch.object(RUNNER.subprocess, "run") as run: + RUNNER.run_compose("device-plane", services, entries) + command = run.call_args_list[0].args[0] + self.assertIn("--force-recreate", command) + self.assertIn("--no-deps", command) + self.assertEqual( + command[-2:], + ["device-control-core", "device-gateway"], + ) + + def test_terminal_failed_recovery_is_rejected(self): + with self.assertRaisesRegex( + RUNNER.DeployError, + "recovery 002 is terminal failed", + ): + RUNNER.reject_terminal_device_plane_foundation_artifact( + { + "id": + RUNNER + .DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID, + }, + "0" * 64, + ) + with self.assertRaisesRegex( + RUNNER.DeployError, + "recovery 002 is terminal failed", + ): + RUNNER.reject_terminal_device_plane_foundation_artifact( + {"id": "unrelated"}, + RUNNER + .DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256, + ) + + def test_rollback_is_stateless_and_removes_only_control_network(self): + entries = ( + RUNNER.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES + ) + services = ("device-control-core", "device-gateway") + runtime = { + "device-control-core": {"containerId": "1" * 64}, + "device-gateway": {"containerId": "2" * 64}, + "device-postgres": { + "containerId": + RUNNER + .DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[ + "device-postgres" + ], + }, + } + with ( + mock.patch.object( + RUNNER, + "stop_and_remove_compose_services", + ) as stop, + mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=9, + ) as restore, + mock.patch.object( + RUNNER, + "remove_device_plane_control_network_if_unused", + return_value="removed", + ) as remove_network, + mock.patch.object(RUNNER, "run_compose") as compose, + mock.patch.object( + RUNNER, + "healthcheck_compose_service", + ) as health, + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_runtime", + return_value=runtime, + ) as validate_runtime, + mock.patch.object( + RUNNER, + "assert_loopback_tcp_port_closed", + ) as closed, + ): + result = ( + RUNNER + .rollback_device_plane_network_publication_apply( + Path("/live"), + Path("/backup"), + entries, + "stamp", + True, + services, + ) + ) + + self.assertEqual(result, "source+internal-runtime-restored:9") + stop.assert_called_once_with("device-plane", services) + restore.assert_called_once() + remove_network.assert_called_once_with() + compose.assert_called_once_with( + "device-plane", + services, + ("docker-compose.device-plane.yml",), + ) + self.assertEqual(health.call_count, 3) + validate_runtime.assert_called_once_with() + self.assertEqual( + [call.args[0] for call in closed.call_args_list], + [18120, 18121, 9921], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py b/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py index 9135130..31b2c11 100644 --- a/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py +++ b/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py @@ -18,6 +18,10 @@ RECOVERY_BUILDER = ( SCRIPT_DIR / "build-device-plane-foundation-recovery-artifact.mjs" ) FOUNDATION_BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs" +PREDECESSOR_COMPOSE = ( + SCRIPT_DIR + / "fixtures/device-plane-foundation-internal-only-v1.yml" +) RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" RECOVERY_ENTRIES = [ ".dockerignore", @@ -246,7 +250,7 @@ class DevicePlaneFoundationRecoveryArtifactTest(unittest.TestCase): source_root = SCRIPT_DIR.parent.parent / "device-plane" shutil.copy2( - source_root / "docker-compose.device-plane.yml", + PREDECESSOR_COMPOSE, live_root / "docker-compose.device-plane.yml", ) shutil.copy2(