SILVER
/
NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(deploy): publish device plane loopback health

This commit is contained in:
Codex 2026-07-26 00:09:55 +03:00
parent aca47c6143
commit 3c538ad98c
10 changed files with 1565 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
device-plane/deployment/device-plane-foundation-network-publication-v1.json Normal file
View File

@ -0,0 +1,24 @@
{
"schemaVersion": "nodedc.device-plane.foundation-network-publication.v1",
"mode": "failed-foundation-network-publication-correction",
"failedRecoveryPatchId": "device-plane-foundation-recovery-20260725-002",
"failedRecoveryArtifactSha256": "9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf",
"failedRecoveryBackupId": "device-plane-device-plane-foundation-recovery-20260725-002-20260725-232447",
"sourceAction": "publish-network-corrected-foundation-source",
"runtimeAction": "recreate-stateless-services-no-build",
"selectedServices": [
"device-control-core",
"device-gateway"
],
"preservedServices": [
"device-postgres"
],
"privateNetwork": "nodedc-device-plane-private",
"controlNetwork": "nodedc-device-plane-control",
"publishedLoopbackPorts": [
"127.0.0.1:18120:18120",
"127.0.0.1:18121:18121"
],
"databaseVolume": "nodedc-device-plane-postgres-data",
"rollback": "restore-partial-source-and-internal-only-stateless-runtime"
}

8
device-plane/docker-compose.device-plane.yml
View File

@ -55,6 +55,7 @@ services:
- "127.0.0.1:18120:18120" - "127.0.0.1:18120:18120"
networks: networks:
- device-plane-private - device-plane-private
- device-plane-control
depends_on: depends_on:
device-postgres: device-postgres:
condition: service_healthy condition: service_healthy
@ -93,6 +94,7 @@ services:
- "127.0.0.1:18121:18121" - "127.0.0.1:18121:18121"
networks: networks:
- device-plane-private - device-plane-private
- device-plane-control
security_opt: security_opt:
- no-new-privileges:true - no-new-privileges:true
cap_drop: cap_drop:
@ -112,6 +114,12 @@ networks:
device-plane-private: device-plane-private:
name: nodedc-device-plane-private name: nodedc-device-plane-private
internal: true internal: true
device-plane-control:
name: nodedc-device-plane-control
driver: bridge
internal: false
driver_opts:
com.docker.network.bridge.enable_ip_masquerade: "false"
volumes: volumes:
device-plane-postgres-data: device-plane-postgres-data:

36
infra/deploy-runner/README.md
View File

@ -500,6 +500,42 @@ node infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs \
device-plane-foundation-recovery-20260725-002 device-plane-foundation-recovery-20260725-002
``` ```
The recovery `device-plane-foundation-recovery-20260725-002` is also terminal
failed. Docker Engine 24 does not materialize requested host port mappings when
the container is attached only to an `internal` network. Its registered
successor is the exact
`deployment/device-plane-foundation-network-publication-v1.json` transition.
It keeps PostgreSQL exclusively on the existing internal private network and
adds a second non-internal bridge only to Control Core and Gateway. Masquerade
is disabled on that bridge; the only published ports remain
`127.0.0.1:18120` and `127.0.0.1:18121`. Device TCP `9921`, public ingress,
discovery ingest and command transport remain disabled.
This successor has an empty build set and force-recreates only
`device-control-core` and `device-gateway` with `--no-deps` from their exact
existing `pull_policy: never` images. Its predecessor barrier validates the
terminal failed archive/journal/backup, partial source, exact running
generations, absent actual port mappings, internal private network and absence
of the control network. Acceptance requires new stateless container
generations, the unchanged PostgreSQL generation and volume, exact actual
loopback mappings, healthy HTTP contracts and a closed `9921`.
Rollback removes only the two stateless candidate containers and the newly
created unused control network, restores the partial source predecessor and
recreates the internal-only Core/Gateway runtime from the sealed predecessor
Compose bytes. PostgreSQL and its volume are never selected.
Build and test the deterministic network-publication successor:
```bash
python3 -m unittest -v \
infra.deploy-runner.test_device_plane_foundation_network_publication_artifact
node \
infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs \
device-plane-foundation-network-publication-20260725-003
```
After the runner is promoted and freshly verified, bootstrap the durable After the runner is promoted and freshly verified, bootstrap the durable
prerequisite with a separate artifact before planning the application: prerequisite with a separate artifact before planning the application:

17
infra/deploy-runner/build-device-plane-artifact.mjs
View File

@ -18,6 +18,10 @@ import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url)); const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../.."); const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane"); const sourceRoot = resolve(platformRoot, "device-plane");
const failedFoundationCompose = resolve(
scriptDir,
"fixtures/device-plane-foundation-internal-only-v1.yml",
);
const artifactDir = resolve( const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"), || resolve(scriptDir, "../deploy-artifacts"),
@ -54,8 +58,14 @@ await assertSourceBoundary();
try { try {
await mkdir(payload, { recursive: true }); await mkdir(payload, { recursive: true });
for (const sourceRelative of files) { for (const sourceRelative of files) {
const source = (
patchId === "device-plane-foundation-20260725-001"
&& sourceRelative === "docker-compose.device-plane.yml"
)
? failedFoundationCompose
: resolve(sourceRoot, sourceRelative);
await copySafe( await copySafe(
resolve(sourceRoot, sourceRelative), source,
join(payload, sourceRelative), join(payload, sourceRelative),
); );
} }
@ -109,8 +119,11 @@ try {
} }
async function assertSourceBoundary() { async function assertSourceBoundary() {
const composeSource = patchId === "device-plane-foundation-20260725-001"
? failedFoundationCompose
: resolve(sourceRoot, "docker-compose.device-plane.yml");
const compose = await readFile( const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"), composeSource,
"utf8", "utf8",
); );
for (const fragment of [ for (const fragment of [

267
infra/deploy-runner/build-device-plane-foundation-network-publication-artifact.mjs Normal file
View File

@ -0,0 +1,267 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import {
cp,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
rm,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [
patchId = "device-plane-foundation-network-publication-20260725-003",
...extra
] = process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error(
"usage: build-device-plane-foundation-network-publication-artifact.mjs "
+ "[patch-id]",
);
}
const files = [
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
"deployment/device-plane-foundation-network-publication-v1.json",
];
const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]);
const ignoredDirectoryNames = new Set(["test"]);
const stage = await mkdtemp(
join(tmpdir(), "nodedc-device-plane-network-publication-"),
);
const payload = join(stage, "payload");
const target = join(
artifactDir,
`nodedc-device-plane-${patchId}.tgz`,
);
await assertBoundary();
try {
await mkdir(payload, { recursive: true });
for (const sourceRelative of files) {
await copySafe(
resolve(sourceRoot, sourceRelative),
join(payload, sourceRelative),
);
}
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(
join(stage, "files.txt"),
`${files.join("\n")}\n`,
"utf8",
);
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{
encoding: "utf8",
maxBuffer: 128 * 1024 * 1024,
},
);
if (tar.status !== 0) {
throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
}
const digest = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
artifact: target,
sha256: digest,
component: "device-plane",
transition: "failed-foundation-network-publication-correction",
entries: files,
build: [],
services: ["device-control-core", "device-gateway"],
preservedRuntime: [
"device-postgres",
"nodedc-device-plane-postgres-data",
"Gelios",
],
networkChange: {
private: "preserved:internal",
control: "create:non-internal:no-masquerade",
published: [
"127.0.0.1:18120:18120",
"127.0.0.1:18121:18121",
],
disabled: ["9921", "public-ingress", "command-transport"],
},
rollback:
"restore-partial-source-and-internal-only-stateless-runtime",
excluded: [
".env*",
"node_modules",
"**/test",
"docs",
"runtime",
"secrets",
],
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
async function assertBoundary() {
const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"),
"utf8",
);
for (const fragment of [
'DEVICE_DISCOVERY_INGEST_ENABLED: "false"',
'DEVICE_GATEWAY_LISTEN_ENABLED: "false"',
'"127.0.0.1:18120:18120"',
'"127.0.0.1:18121:18121"',
"name: nodedc-device-plane-private",
"internal: true",
"name: nodedc-device-plane-control",
"internal: false",
'com.docker.network.bridge.enable_ip_masquerade: "false"',
"name: nodedc-device-plane-postgres-data",
"pull_policy: never",
]) {
if (!compose.includes(fragment)) {
throw new Error(
`device_plane_network_publication_boundary_missing:${fragment}`,
);
}
}
for (const forbidden of [
"9921:9921",
"0.0.0.0:9921",
'DEVICE_DISCOVERY_INGEST_ENABLED: "true"',
'DEVICE_GATEWAY_LISTEN_ENABLED: "true"',
"POSTGRES_PASSWORD:",
]) {
if (compose.includes(forbidden)) {
throw new Error(
`device_plane_network_publication_boundary_violation:${forbidden}`,
);
}
}
const descriptor = JSON.parse(await readFile(
resolve(
sourceRoot,
"deployment/device-plane-foundation-network-publication-v1.json",
),
"utf8",
));
const expected = {
schemaVersion:
"nodedc.device-plane.foundation-network-publication.v1",
mode: "failed-foundation-network-publication-correction",
failedRecoveryPatchId:
"device-plane-foundation-recovery-20260725-002",
failedRecoveryArtifactSha256:
"9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf",
failedRecoveryBackupId:
"device-plane-device-plane-foundation-recovery-20260725-002-20260725-232447",
sourceAction: "publish-network-corrected-foundation-source",
runtimeAction: "recreate-stateless-services-no-build",
selectedServices: ["device-control-core", "device-gateway"],
preservedServices: ["device-postgres"],
privateNetwork: "nodedc-device-plane-private",
controlNetwork: "nodedc-device-plane-control",
publishedLoopbackPorts: [
"127.0.0.1:18120:18120",
"127.0.0.1:18121:18121",
],
databaseVolume: "nodedc-device-plane-postgres-data",
rollback:
"restore-partial-source-and-internal-only-stateless-runtime",
};
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
throw new Error(
"device_plane_network_publication_descriptor_mismatch",
);
}
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
async function copySafe(source, destination) {
const sourceStat = await lstat(source);
if (sourceStat.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, source)}`,
);
}
if (sourceStat.isFile()) {
await mkdir(dirname(destination), { recursive: true });
await cp(source, destination, {
force: true,
verbatimSymlinks: true,
});
return;
}
if (!sourceStat.isDirectory()) {
throw new Error(`source_type_rejected:${source}`);
}
await mkdir(destination, { recursive: true });
for (const entry of await readdir(source, { withFileTypes: true })) {
if (
ignoredBasenames.has(entry.name)
|| entry.name.startsWith(".env")
|| (
entry.isDirectory()
&& ignoredDirectoryNames.has(entry.name)
)
) {
continue;
}
const childSource = join(source, entry.name);
const childDestination = join(destination, entry.name);
if (entry.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, childSource)}`,
);
}
await copySafe(childSource, childDestination);
}
}

11
infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs
View File

@ -18,6 +18,10 @@ import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url)); const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../.."); const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane"); const sourceRoot = resolve(platformRoot, "device-plane");
const predecessorCompose = resolve(
scriptDir,
"fixtures/device-plane-foundation-internal-only-v1.yml",
);
const artifactDir = resolve( const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"), || resolve(scriptDir, "../deploy-artifacts"),
@ -64,8 +68,11 @@ await assertRecoveryBoundary();
try { try {
await mkdir(payload, { recursive: true }); await mkdir(payload, { recursive: true });
for (const sourceRelative of files) { for (const sourceRelative of files) {
const source = sourceRelative === "docker-compose.device-plane.yml"
? predecessorCompose
: resolve(sourceRoot, sourceRelative);
await copySafe( await copySafe(
resolve(sourceRoot, sourceRelative), source,
join(payload, sourceRelative), join(payload, sourceRelative),
); );
} }
@ -129,7 +136,7 @@ try {
async function assertRecoveryBoundary() { async function assertRecoveryBoundary() {
const compose = await readFile( const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"), predecessorCompose,
"utf8", "utf8",
); );
for (const fragment of [ for (const fragment of [

118
infra/deploy-runner/fixtures/device-plane-foundation-internal-only-v1.yml Normal file
View File

@ -0,0 +1,118 @@
services:
device-postgres:
image: postgres:16-alpine
pull_policy: missing
restart: unless-stopped
environment:
POSTGRES_DB: device_plane
POSTGRES_USER: device_plane
POSTGRES_PASSWORD_FILE: /run/nodedc-secrets/postgres-password
volumes:
- type: volume
source: device-plane-postgres-data
target: /var/lib/postgresql/data
- type: bind
source: /volume1/docker/nodedc-device-plane/secrets/postgres-password
target: /run/nodedc-secrets/postgres-password
read_only: true
bind:
create_host_path: false
networks:
- device-plane-private
healthcheck:
test: ["CMD-SHELL", "pg_isready -U device_plane -d device_plane"]
interval: 10s
timeout: 5s
retries: 12
start_period: 20s
device-control-core:
image: nodedc/device-control-core:local
pull_policy: never
restart: unless-stopped
user: "1000:1000"
read_only: true
tmpfs:
- /tmp:size=16m,mode=1777
environment:
HOST: 0.0.0.0
PORT: "18120"
DEVICE_DATABASE_HOST: device-postgres
DEVICE_DATABASE_PORT: "5432"
DEVICE_DATABASE_NAME: device_plane
DEVICE_DATABASE_USER: device_plane
DEVICE_DATABASE_PASSWORD_FILE: /run/nodedc-secrets/postgres-password
DEVICE_DATABASE_POOL_SIZE: "10"
DEVICE_DISCOVERY_INGEST_ENABLED: "false"
volumes:
- type: bind
source: /volume1/docker/nodedc-device-plane/secrets/postgres-password
target: /run/nodedc-secrets/postgres-password
read_only: true
bind:
create_host_path: false
ports:
- "127.0.0.1:18120:18120"
networks:
- device-plane-private
depends_on:
device-postgres:
condition: service_healthy
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
healthcheck:
test:
- CMD
- node
- -e
- fetch('http://127.0.0.1:18120/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))
interval: 10s
timeout: 5s
retries: 12
start_period: 20s
device-gateway:
image: nodedc/device-gateway:local
pull_policy: never
restart: unless-stopped
user: "1000:1000"
read_only: true
tmpfs:
- /tmp:size=16m,mode=1777
environment:
DEVICE_GATEWAY_HEALTH_HOST: 0.0.0.0
DEVICE_GATEWAY_HEALTH_PORT: "18121"
DEVICE_GATEWAY_LISTEN_ENABLED: "false"
DEVICE_GATEWAY_TCP_HOST: 127.0.0.1
DEVICE_GATEWAY_TCP_PORT: "9921"
DEVICE_GATEWAY_MAX_SESSIONS: "100"
DEVICE_GATEWAY_SESSION_TIMEOUT_MS: "10000"
ports:
- "127.0.0.1:18121:18121"
networks:
- device-plane-private
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
healthcheck:
test:
- CMD
- node
- -e
- fetch('http://127.0.0.1:18121/healthz').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))
interval: 10s
timeout: 5s
retries: 12
start_period: 10s
networks:
device-plane-private:
name: nodedc-device-plane-private
internal: true
volumes:
device-plane-postgres-data:
name: nodedc-device-plane-postgres-data

783
infra/deploy-runner/nodedc-deploy
View File

@ -7,6 +7,7 @@ import os
import re import re
import secrets import secrets
import shutil import shutil
import socket
import stat import stat
import subprocess import subprocess
import sys import sys
@ -69,6 +70,13 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES = (
*DEVICE_PLANE_FOUNDATION_ENTRIES, *DEVICE_PLANE_FOUNDATION_ENTRIES,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL, DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
) )
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL = (
"deployment/device-plane-foundation-network-publication-v1.json"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES = (
*DEVICE_PLANE_FOUNDATION_ENTRIES,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
)
DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = ( DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = (
"device-plane-foundation-20260725-001" "device-plane-foundation-20260725-001"
) )
@ -116,6 +124,56 @@ DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS = {
"sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d" "sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d"
), ),
} }
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID = (
"device-plane-foundation-recovery-20260725-002"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT = (
"nodedc-device-plane-device-plane-foundation-recovery-20260725-002.tgz."
"20260725-232447"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256 = (
"9183cc385142584bfd12510bb0a3e6b833b2fd26607436f2486a564c628ea1bf"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID = (
"device-plane-device-plane-foundation-recovery-20260725-002-"
"20260725-232447"
)
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256 = {
"manifest.env": (
"e1c31f28474d66c33f8b91e890c112e605ffe376ed3e7292d2f8504ea18fd9c1"
),
"files.txt": (
"bb0e3deec5952811dd413afdf24d116e5bf3fd3e041f78bfe5cc5ca87166b8ff"
),
"existing-files.txt": (
"f2e2c6e7501d3a383cd7e909a2e4e4cfbb3b0cda132f508f54e478bfb455585b"
),
"missing-files.txt": (
"e57cda977037cabbe01de04ece1a1e9f15cb9eb3c0ad2996811d6ea3d4f4b3d2"
),
"runtime-before.json": (
"f500964078c5814d2130fe57b5c67fb89dea000355591836e1b72088b37170ef"
),
"source-before.tgz": (
"eed474b04608e206f09ebe68ca80da675a2318fa3efaf3be90f2d983429f9d9f"
),
}
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256 = (
"47d4d153d27bc6f418c1f787ea5ff0153b1ee082f96b92e28486805cb46b3f15"
)
DEVICE_PLANE_PRIVATE_NETWORK = "nodedc-device-plane-private"
DEVICE_PLANE_CONTROL_NETWORK = "nodedc-device-plane-control"
DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS = {
"device-control-core": (
"ccdaeee71472a557ccecb485e81b31f5a4ee37b1b3f3cb4d5edbcf12d02bb08b"
),
"device-gateway": (
"d59258eb86ae8ec728ceaaed5b4976dd1359bdcecd6abc7950e73ffabf1d8eba"
),
"device-postgres": (
"44702b995b4397131646ed8076144efd116cb38d00d9a3d641252b7ad635d7a1"
),
}
DEVICE_PLANE_RUNTIME_SERVICES = ( DEVICE_PLANE_RUNTIME_SERVICES = (
"device-control-core", "device-control-core",
"device-gateway", "device-gateway",
@ -2806,6 +2864,7 @@ def allowed_payload_path(component, rel):
"docker-compose.device-plane.yml", "docker-compose.device-plane.yml",
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL, DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"packages/device-protocol-contract", "packages/device-protocol-contract",
"packages/arusnavi-b2-adapter", "packages/arusnavi-b2-adapter",
"services/device-control-core", "services/device-control-core",
@ -7835,6 +7894,13 @@ def load_artifact(artifact, work_dir):
entries, entries,
): ):
validate_device_plane_foundation_recovery_payload(payload_dir) validate_device_plane_foundation_recovery_payload(payload_dir)
if is_device_plane_foundation_network_publication_slice(
manifest["component"],
entries,
):
validate_device_plane_foundation_network_publication_payload(
payload_dir
)
if manifest["component"] == "n8n-private-extension": if manifest["component"] == "n8n-private-extension":
validate_n8n_private_extension_release(payload_dir, entries) validate_n8n_private_extension_release(payload_dir, entries)
if manifest["component"] == "engine": if manifest["component"] == "engine":
@ -8118,6 +8184,16 @@ def reject_terminal_device_plane_foundation_artifact(manifest, sha256):
"Device Plane foundation 001 is terminal failed; " "Device Plane foundation 001 is terminal failed; "
"use the exact registered foundation recovery successor" "use the exact registered foundation recovery successor"
) )
if (
manifest.get("id")
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or sha256
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
):
die(
"Device Plane foundation recovery 002 is terminal failed; "
"use the exact registered network-publication successor"
)
class DeployLock: class DeployLock:
@ -8153,6 +8229,15 @@ def is_device_plane_foundation_recovery_slice(component, entries):
) )
def is_device_plane_foundation_network_publication_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries)
== DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES
)
def expected_device_plane_foundation_recovery_descriptor(): def expected_device_plane_foundation_recovery_descriptor():
return { return {
"schemaVersion": "nodedc.device-plane.foundation-recovery.v1", "schemaVersion": "nodedc.device-plane.foundation-recovery.v1",
@ -8185,6 +8270,65 @@ def validate_device_plane_foundation_recovery_payload(payload_dir):
return descriptor return descriptor
def expected_device_plane_foundation_network_publication_descriptor():
return {
"schemaVersion": (
"nodedc.device-plane.foundation-network-publication.v1"
),
"mode": "failed-foundation-network-publication-correction",
"failedRecoveryPatchId": (
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
),
"failedRecoveryArtifactSha256": (
DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
),
"failedRecoveryBackupId": (
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
),
"sourceAction": "publish-network-corrected-foundation-source",
"runtimeAction": "recreate-stateless-services-no-build",
"selectedServices": [
"device-control-core",
"device-gateway",
],
"preservedServices": ["device-postgres"],
"privateNetwork": DEVICE_PLANE_PRIVATE_NETWORK,
"controlNetwork": DEVICE_PLANE_CONTROL_NETWORK,
"publishedLoopbackPorts": [
"127.0.0.1:18120:18120",
"127.0.0.1:18121:18121",
],
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"rollback": (
"restore-partial-source-and-internal-only-stateless-runtime"
),
}
def validate_device_plane_foundation_network_publication_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"Device Plane foundation network-publication descriptor",
max_bytes=16 * 1024,
)
expected = (
expected_device_plane_foundation_network_publication_descriptor()
)
if descriptor != expected:
die(
"Device Plane foundation network-publication descriptor mismatch"
)
compose = payload_dir / "docker-compose.device-plane.yml"
if (
compose.is_symlink()
or not compose.is_file()
or sha256_file(compose)
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256
):
die("Device Plane network-publication Compose mismatch")
return descriptor
def device_plane_service_container_ids(service): def device_plane_service_container_ids(service):
if service not in DEVICE_PLANE_RUNTIME_SERVICES: if service not in DEVICE_PLANE_RUNTIME_SERVICES:
die(f"Device Plane runtime service is not registered: {service}") die(f"Device Plane runtime service is not registered: {service}")
@ -8464,6 +8608,263 @@ def validate_device_plane_foundation_recovery_evidence(payload_dir):
} }
def inspect_device_plane_network_optional(name):
result = subprocess.run(
[str(DOCKER), "network", "inspect", name],
check=False,
capture_output=True,
text=True,
)
if result.returncode == 1:
return None
if result.returncode != 0:
die(f"Device Plane network inspect failed: {name}")
try:
value = json.loads(result.stdout)
except json.JSONDecodeError:
die(f"Device Plane network inspect returned invalid JSON: {name}")
if (
not isinstance(value, list)
or len(value) != 1
or not isinstance(value[0], dict)
):
die(f"Device Plane network inspect shape mismatch: {name}")
return value[0]
def validate_device_plane_network_contract(
name,
*,
internal,
expected_container_ids,
):
network = inspect_device_plane_network_optional(name)
if network is None:
die(f"Device Plane network is missing: {name}")
options = network.get("Options") or {}
if (
network.get("Name") != name
or network.get("Driver") != "bridge"
or network.get("Internal") is not internal
or set((network.get("Containers") or {}).keys())
!= set(expected_container_ids)
):
die(f"Device Plane network boundary mismatch: {name}")
if (
name == DEVICE_PLANE_CONTROL_NETWORK
and options.get(
"com.docker.network.bridge.enable_ip_masquerade"
)
!= "false"
):
die("Device Plane control network masquerade boundary mismatch")
return network
def validate_device_plane_foundation_network_publication_evidence(
payload_dir,
):
validate_device_plane_foundation_network_publication_payload(payload_dir)
backup_dir = (
BACKUPS_DIR
/ DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
)
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Plane failed recovery backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Plane failed recovery backup is unsafe")
backup_names = {child.name for child in backup_dir.iterdir()}
expected_backup = (
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_SHA256
)
if backup_names != set(expected_backup):
die("Device Plane failed recovery backup file set mismatch")
for name, expected_sha256 in expected_backup.items():
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected_sha256
):
die(
"Device Plane failed recovery backup drift detected: "
f"{name}"
)
failed_artifact = (
FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
)
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Plane failed recovery artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
):
die("Device Plane failed recovery artifact evidence mismatch")
try:
state_stat = FAILED_STATE_FILE.lstat()
state_lines = FAILED_STATE_FILE.read_text(
encoding="utf-8"
).splitlines()
except (FileNotFoundError, OSError, UnicodeDecodeError):
die("Device Plane failed recovery journal is unreadable")
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
state_stat.st_mode
):
die("Device Plane failed recovery journal is unsafe")
records = []
for line in state_lines:
try:
value = json.loads(line)
except json.JSONDecodeError:
die("Device Plane failed recovery journal contains invalid JSON")
if (
isinstance(value, dict)
and value.get("id")
== DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
):
records.append(value)
if len(records) != 1:
die("Device Plane failed recovery journal evidence count mismatch")
record = records[0]
if (
record.get("artifact")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status")
!= (
"ok:device-plane-overlay:"
"source-restored-runtime-unchanged:9"
)
or record.get("status") != "failed"
or record.get("message")
!= (
"healthcheck failed for http://127.0.0.1:18120/healthz: "
"<urlopen error [Errno 111] Connection refused>"
)
):
die("Device Plane failed recovery journal evidence mismatch")
comparable_entries = tuple(
rel
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES
if rel != "docker-compose.device-plane.yml"
)
with tempfile.TemporaryDirectory(
prefix="device-plane-failed-recovery-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES
):
die("Device Plane failed recovery artifact contract mismatch")
failed_source = collect_exact_files(
failed_payload,
comparable_entries,
"Device Plane failed recovery source",
)
candidate_source = collect_exact_files(
payload_dir,
comparable_entries,
"Device Plane network-publication candidate source",
)
if candidate_source != failed_source:
die(
"Device Plane network-publication candidate source drift "
"detected"
)
root = component_root("device-plane")
compose_path = root / "docker-compose.device-plane.yml"
if (
compose_path.is_symlink()
or not compose_path.is_file()
or sha256_file(compose_path)
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256
):
die("Device Plane network-publication source predecessor mismatch")
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES:
if rel == "docker-compose.device-plane.yml":
continue
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Plane network-publication predecessor retained "
f"source: {rel}"
)
for descriptor_rel in (
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
):
descriptor_path = root / descriptor_rel
if descriptor_path.exists() or descriptor_path.is_symlink():
die(
"Device Plane network-publication descriptor predecessor "
"mismatch"
)
bootstrap_descriptor = root / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL
if (
bootstrap_descriptor.is_symlink()
or not bootstrap_descriptor.is_file()
or sha256_file(bootstrap_descriptor)
!= DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256
):
die("Device Plane PostgreSQL bootstrap descriptor drift detected")
runtime = validate_device_plane_foundation_runtime()
for service, expected_id in (
DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS.items()
):
if runtime[service]["containerId"] != expected_id:
die(
"Device Plane network-publication runtime predecessor "
f"generation mismatch: {service}"
)
expected_network_ids = {
item["containerId"]
for item in runtime.values()
}
validate_device_plane_network_contract(
DEVICE_PLANE_PRIVATE_NETWORK,
internal=True,
expected_container_ids=expected_network_ids,
)
if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK):
die("Device Plane control network already exists")
return {
"mode": "failed-foundation-network-publication-correction",
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def validate_device_plane_postgres_bootstrap_payload(payload_dir): def validate_device_plane_postgres_bootstrap_payload(payload_dir):
descriptor = read_strict_json( descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
@ -8537,7 +8938,7 @@ def device_plane_postgres_plan_selection(postgres_preflight):
return "bootstrap-selected:create-if-absent" return "bootstrap-selected:create-if-absent"
def validate_device_plane_foundation_runtime(): def validate_device_plane_foundation_runtime(network_publication=False):
validate_device_plane_runtime_secret_metadata() validate_device_plane_runtime_secret_metadata()
expected = { expected = {
"device-control-core": { "device-control-core": {
@ -8552,6 +8953,24 @@ def validate_device_plane_foundation_runtime():
"HostPort": "18120", "HostPort": "18120",
}], }],
}, },
"actualPorts": (
{
"18120/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18120",
}],
}
if network_publication
else {}
),
"networks": (
{
DEVICE_PLANE_PRIVATE_NETWORK,
DEVICE_PLANE_CONTROL_NETWORK,
}
if network_publication
else {DEVICE_PLANE_PRIVATE_NETWORK}
),
}, },
"device-gateway": { "device-gateway": {
"image": DEVICE_PLANE_GATEWAY_IMAGE, "image": DEVICE_PLANE_GATEWAY_IMAGE,
@ -8565,6 +8984,24 @@ def validate_device_plane_foundation_runtime():
"HostPort": "18121", "HostPort": "18121",
}], }],
}, },
"actualPorts": (
{
"18121/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18121",
}],
}
if network_publication
else {}
),
"networks": (
{
DEVICE_PLANE_PRIVATE_NETWORK,
DEVICE_PLANE_CONTROL_NETWORK,
}
if network_publication
else {DEVICE_PLANE_PRIVATE_NETWORK}
),
}, },
"device-postgres": { "device-postgres": {
"image": "postgres:16-alpine", "image": "postgres:16-alpine",
@ -8573,6 +9010,8 @@ def validate_device_plane_foundation_runtime():
], ],
"user": "", "user": "",
"ports": {}, "ports": {},
"actualPorts": {},
"networks": {DEVICE_PLANE_PRIVATE_NETWORK},
}, },
} }
accepted = {} accepted = {}
@ -8591,6 +9030,9 @@ def validate_device_plane_foundation_runtime():
networks = (container.get("NetworkSettings") or {}).get( networks = (container.get("NetworkSettings") or {}).get(
"Networks" "Networks"
) or {} ) or {}
actual_ports = (container.get("NetworkSettings") or {}).get(
"Ports"
) or {}
if ( if (
state.get("Status") != "running" state.get("Status") != "running"
or state.get("Running") is not True or state.get("Running") is not True
@ -8614,7 +9056,8 @@ def validate_device_plane_foundation_runtime():
or labels.get("com.docker.compose.project") or labels.get("com.docker.compose.project")
!= "nodedc-device-plane" != "nodedc-device-plane"
or labels.get("com.docker.compose.service") != service or labels.get("com.docker.compose.service") != service
or set(networks) != {"nodedc-device-plane-private"} or set(networks) != contract["networks"]
or actual_ports != contract["actualPorts"]
): ):
die( die(
"Device Plane foundation runtime boundary mismatch: " "Device Plane foundation runtime boundary mismatch: "
@ -8627,6 +9070,24 @@ def validate_device_plane_foundation_runtime():
"restartCount": 0, "restartCount": 0,
} }
if network_publication:
for service in ("device-control-core", "device-gateway"):
if (
accepted[service]["containerId"]
== DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[service]
):
die(
"Device Plane stateless generation was not recreated: "
f"{service}"
)
if (
accepted["device-postgres"]["containerId"]
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[
"device-postgres"
]
):
die("Device Plane PostgreSQL generation changed")
core = inspect_device_plane_container( core = inspect_device_plane_container(
device_plane_service_container_ids("device-control-core")[0] device_plane_service_container_ids("device-control-core")[0]
) )
@ -8752,6 +9213,23 @@ def validate_device_plane_foundation_runtime():
or postgres_volume[0].get("RW") is not True or postgres_volume[0].get("RW") is not True
): ):
die("Device Plane PostgreSQL preserved mount mismatch") die("Device Plane PostgreSQL preserved mount mismatch")
if network_publication:
validate_device_plane_network_contract(
DEVICE_PLANE_PRIVATE_NETWORK,
internal=True,
expected_container_ids={
accepted[service]["containerId"]
for service in DEVICE_PLANE_RUNTIME_SERVICES
},
)
validate_device_plane_network_contract(
DEVICE_PLANE_CONTROL_NETWORK,
internal=False,
expected_container_ids={
accepted["device-control-core"]["containerId"],
accepted["device-gateway"]["containerId"],
},
)
return accepted return accepted
@ -8834,6 +9312,75 @@ def validate_device_plane_foundation_installed_source():
return actual return actual
def validate_device_plane_foundation_network_publication_installed_source():
failed_artifact = (
FAILED_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT
)
comparable_entries = tuple(
rel
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES
if rel != "docker-compose.device-plane.yml"
)
with tempfile.TemporaryDirectory(
prefix="device-plane-installed-network-publication-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID
or tuple(failed_entries)
!= DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES
):
die("Device Plane failed recovery source contract mismatch")
expected = collect_exact_files(
failed_payload,
comparable_entries,
"Device Plane failed recovery source",
)
actual = collect_exact_files(
component_root("device-plane"),
comparable_entries,
"Device Plane installed network-publication source",
)
if actual != expected:
die("Device Plane installed network-publication source mismatch")
compose = (
component_root("device-plane")
/ "docker-compose.device-plane.yml"
)
if (
compose.is_symlink()
or not compose.is_file()
or sha256_file(compose)
!= DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256
):
die("Device Plane installed network-publication Compose mismatch")
descriptor = read_strict_json(
component_root("device-plane")
/ DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_REL,
"installed Device Plane network-publication descriptor",
max_bytes=16 * 1024,
)
if (
descriptor
!= expected_device_plane_foundation_network_publication_descriptor()
):
die(
"installed Device Plane network-publication descriptor mismatch"
)
recovery_descriptor = (
component_root("device-plane")
/ DEVICE_PLANE_FOUNDATION_RECOVERY_REL
)
if recovery_descriptor.exists() or recovery_descriptor.is_symlink():
die("terminal Device Plane recovery descriptor is installed")
return actual
def component_compose_root(component): def component_compose_root(component):
return COMPONENTS[component].get("compose_root", component_root(component)) return COMPONENTS[component].get("compose_root", component_root(component))
@ -10366,6 +10913,14 @@ def component_services(component, entries=None):
# runtime without build/recreate/restart. # runtime without build/recreate/restart.
return () return ()
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
# Recreate only the two stateless services from their exact existing
# image IDs. PostgreSQL is a preserved prerequisite, never selected.
return ("device-control-core", "device-gateway")
if is_engine_l2_closed_loop_slice(component, entries): if is_engine_l2_closed_loop_slice(component, entries):
# The failed 030 apply published both the backend source and the built # The failed 030 apply published both the backend source and the built
# UI before Compose rejected the descriptor/source mismatch. The exact # UI before Compose rejected the descriptor/source mismatch. The exact
@ -10703,6 +11258,12 @@ def component_builds(component, entries=None):
if is_device_plane_foundation_recovery_slice(component, entries): if is_device_plane_foundation_recovery_slice(component, entries):
return () return ()
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
return ()
if component == "device-plane" and entries is not None: if component == "device-plane" and entries is not None:
selected_services = component_services(component, entries) selected_services = component_services(component, entries)
builds = [] builds = []
@ -12675,6 +13236,7 @@ def plan_artifact(artifact):
mcp_autonomy_provider_v5_preflight = None mcp_autonomy_provider_v5_preflight = None
l2_closed_loop_preflight = None l2_closed_loop_preflight = None
device_plane_foundation_recovery_preflight = None device_plane_foundation_recovery_preflight = None
device_plane_network_publication_preflight = None
device_plane_runtime_before = None device_plane_runtime_before = None
composite_provider_v4_preflight = None composite_provider_v4_preflight = None
provider_rotating_slot_preflight = None provider_rotating_slot_preflight = None
@ -12849,6 +13411,15 @@ def plan_artifact(artifact):
payload_dir payload_dir
) )
) )
if is_device_plane_foundation_network_publication_slice(
manifest["component"],
entries,
):
device_plane_network_publication_preflight = (
validate_device_plane_foundation_network_publication_evidence(
payload_dir
)
)
component = manifest["component"] component = manifest["component"]
root = component_root(component) root = component_root(component)
@ -14257,6 +14828,53 @@ def plan_artifact(artifact):
) )
print("device_plane_source_action=publish-exact-failed-source") print("device_plane_source_action=publish-exact-failed-source")
print("device_plane_rollback=source-only-runtime-unchanged") print("device_plane_rollback=source-only-runtime-unchanged")
if device_plane_network_publication_preflight is not None:
print(
"device_plane_transition="
f"{device_plane_network_publication_preflight['mode']}"
)
print(
"failed_recovery_patch="
f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID}"
)
print(
"failed_recovery_artifact_sha256="
f"{DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256}"
)
print(
"failed_recovery_backup="
f"{device_plane_network_publication_preflight['backup'].name}"
)
print("device_plane_build=none")
print(
"device_plane_runtime_mutation="
"recreate:device-control-core,device-gateway"
)
print(
"device_plane_runtime_services="
"preserved:device-postgres"
)
print(
"device_plane_private_network="
"preserved:internal:true"
)
print(
"device_plane_control_network="
"create:internal:false:masquerade:false"
)
print(
"device_plane_actual_ports="
"required:127.0.0.1:18120,127.0.0.1:18121"
)
print("device_gateway_tcp_9921=disabled:unpublished")
print(
"device_plane_source_action="
"publish-network-corrected-foundation-source"
)
print(
"device_plane_rollback="
"partial-source+internal-only-stateless-runtime"
)
if device_plane_postgres_preflight is not None: if device_plane_postgres_preflight is not None:
print( print(
"device_postgres_bootstrap=" "device_postgres_bootstrap="
@ -14545,6 +15163,83 @@ def rollback_platform_apply(root, backup_dir, entries, current_stamp, runtime_st
return f"source+runtime-restored:{restored_count}" return f"source+runtime-restored:{restored_count}"
def remove_device_plane_control_network_if_unused():
network = inspect_device_plane_network_optional(
DEVICE_PLANE_CONTROL_NETWORK
)
if network is None:
return "absent"
if (
network.get("Name") != DEVICE_PLANE_CONTROL_NETWORK
or network.get("Driver") != "bridge"
or network.get("Internal") is not False
or (network.get("Containers") or {})
or (network.get("Options") or {}).get(
"com.docker.network.bridge.enable_ip_masquerade"
)
!= "false"
):
die("Device Plane control network is unsafe to remove")
subprocess.run(
[
str(DOCKER),
"network",
"rm",
DEVICE_PLANE_CONTROL_NETWORK,
],
check=True,
)
if inspect_device_plane_network_optional(DEVICE_PLANE_CONTROL_NETWORK):
die("Device Plane control network removal failed")
return "removed"
def rollback_device_plane_network_publication_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
applied_services,
):
if tuple(applied_services) != (
"device-control-core",
"device-gateway",
):
die("Device Plane network-publication rollback service mismatch")
if runtime_started:
stop_and_remove_compose_services(
"device-plane",
applied_services,
)
restored_count = restore_platform_overlay(
root,
backup_dir,
entries,
current_stamp,
)
remove_device_plane_control_network_if_unused()
run_compose(
"device-plane",
applied_services,
("docker-compose.device-plane.yml",),
)
for service in DEVICE_PLANE_RUNTIME_SERVICES:
healthcheck_compose_service("device-plane", service)
runtime = validate_device_plane_foundation_runtime()
if (
runtime["device-postgres"]["containerId"]
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[
"device-postgres"
]
):
die("Device Plane rollback changed PostgreSQL generation")
assert_loopback_tcp_port_closed(18120)
assert_loopback_tcp_port_closed(18121)
assert_loopback_tcp_port_closed(9921)
return f"source+internal-runtime-restored:{restored_count}"
def rollback_device_plane_apply( def rollback_device_plane_apply(
root, root,
backup_dir, backup_dir,
@ -15313,6 +16008,13 @@ def run_component_runtime(component, entries, services):
return return
if is_device_plane_foundation_recovery_slice(component, entries): if is_device_plane_foundation_recovery_slice(component, entries):
return return
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
prepare_component_runtime(component, entries)
run_compose(component, services, entries)
return
run_build(component, entries) run_build(component, entries)
prepare_component_runtime(component, entries) prepare_component_runtime(component, entries)
if is_engine_node_intelligence_transition(component, entries): if is_engine_node_intelligence_transition(component, entries):
@ -15365,6 +16067,15 @@ def healthcheck_url(check):
die(f"healthcheck failed for {url}: {last_error}") die(f"healthcheck failed for {url}: {last_error}")
def assert_loopback_tcp_port_closed(port):
try:
connection = socket.create_connection(("127.0.0.1", port), timeout=3)
except OSError:
return
connection.close()
die(f"unexpected loopback TCP listener is open: {port}")
def external_data_plane_healthcheck(require_managed=True): def external_data_plane_healthcheck(require_managed=True):
expected_json = { expected_json = {
"ok": True, "ok": True,
@ -15896,6 +16607,31 @@ def run_healthchecks(component, entries=None, services=None):
validate_device_plane_foundation_installed_source() validate_device_plane_foundation_installed_source()
validate_device_plane_foundation_runtime() validate_device_plane_foundation_runtime()
return return
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
if tuple(services or ()) != (
"device-control-core",
"device-gateway",
):
die(
"Device Plane network-publication service set mismatch"
)
for service in (
"device-control-core",
"device-gateway",
"device-postgres",
):
healthcheck_compose_service("device-plane", service)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
validate_device_plane_foundation_network_publication_installed_source()
validate_device_plane_foundation_runtime(
network_publication=True
)
assert_loopback_tcp_port_closed(9921)
return
if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries):
return return
if is_engine_l2_closed_loop_slice(component, entries): if is_engine_l2_closed_loop_slice(component, entries):
@ -16706,6 +17442,13 @@ def apply_artifact(artifact):
payload_dir payload_dir
) )
) )
if is_device_plane_foundation_network_publication_slice(
component,
entries,
):
validate_device_plane_foundation_network_publication_evidence(
payload_dir
)
if not root.is_dir(): if not root.is_dir():
if bootstrap_root: if bootstrap_root:
root.mkdir(parents=True, exist_ok=True) root.mkdir(parents=True, exist_ok=True)
@ -17289,6 +18032,42 @@ def apply_artifact(artifact):
except Exception as rollback_exc: except Exception as rollback_exc:
rollback_status = f"failed:{type(rollback_exc).__name__}" rollback_status = f"failed:{type(rollback_exc).__name__}"
print("platform-automatic-rollback=failed", file=sys.stderr) print("platform-automatic-rollback=failed", file=sys.stderr)
elif (
is_device_plane_foundation_network_publication_slice(
component,
entries,
)
and services is not None
):
try:
restored_state = (
rollback_device_plane_network_publication_apply(
root,
backup_dir,
entries,
current_stamp,
runtime_started,
services,
)
)
rollback_status = (
"ok:device-plane-network-publication:"
f"{restored_state}"
)
print(
"device-plane-network-publication-"
f"automatic-rollback={rollback_status}",
file=sys.stderr,
)
except Exception as rollback_exc:
rollback_status = (
f"failed:{type(rollback_exc).__name__}"
)
print(
"device-plane-network-publication-"
"automatic-rollback=failed",
file=sys.stderr,
)
elif ( elif (
component == "device-plane" component == "device-plane"
and entries is not None and entries is not None

302
infra/deploy-runner/test_device_plane_foundation_network_publication_artifact.py Normal file
View File

@ -0,0 +1,302 @@
#!/usr/bin/env python3
import hashlib
import importlib.machinery
import importlib.util
import json
import os
import subprocess
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest import mock
SCRIPT_DIR = Path(__file__).resolve().parent
BUILDER = (
SCRIPT_DIR
/ "build-device-plane-foundation-network-publication-artifact.mjs"
)
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
COMPOSE = (
SCRIPT_DIR.parent.parent
/ "device-plane/docker-compose.device-plane.yml"
)
PREDECESSOR_COMPOSE = (
SCRIPT_DIR
/ "fixtures/device-plane-foundation-internal-only-v1.yml"
)
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_device_plane_network_publication_runner_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
class DevicePlaneFoundationNetworkPublicationArtifactTest(
unittest.TestCase
):
def build(self, artifact_dir, patch_id):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
["node", str(BUILDER), patch_id],
check=True,
capture_output=True,
text=True,
env=environment,
)
return json.loads(result.stdout)
def test_artifact_is_exact_deterministic_and_database_free(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-network-publication-",
) as directory:
artifact_dir = Path(directory)
patch_id = (
"device-plane-foundation-network-publication-unit-003"
)
first = self.build(artifact_dir, patch_id)
first_bytes = Path(first["artifact"]).read_bytes()
second = self.build(artifact_dir, patch_id)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first_bytes, second_bytes)
self.assertEqual(
first["sha256"],
hashlib.sha256(first_bytes).hexdigest(),
)
self.assertEqual(
first["transition"],
"failed-foundation-network-publication-correction",
)
self.assertEqual(first["build"], [])
self.assertEqual(
first["services"],
["device-control-core", "device-gateway"],
)
self.assertNotIn("device-postgres", first["services"])
self.assertEqual(
first["entries"],
list(
RUNNER
.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES
),
)
with tarfile.open(first["artifact"], "r:gz") as archive:
files = (
archive.extractfile("files.txt")
.read()
.decode("utf-8")
.splitlines()
)
compose = (
archive.extractfile(
"payload/docker-compose.device-plane.yml"
)
.read()
)
descriptor = json.loads(
archive.extractfile(
"payload/deployment/"
"device-plane-foundation-network-publication-v1.json"
)
.read()
.decode("utf-8")
)
self.assertEqual(files, first["entries"])
self.assertEqual(
hashlib.sha256(compose).hexdigest(),
RUNNER
.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256,
)
self.assertEqual(
descriptor,
RUNNER
.expected_device_plane_foundation_network_publication_descriptor(),
)
def test_compose_preserves_database_and_adds_control_network_only_to_stateless_services(
self,
):
compose = COMPOSE.read_text(encoding="utf-8")
predecessor = PREDECESSOR_COMPOSE.read_text(encoding="utf-8")
self.assertEqual(
hashlib.sha256(PREDECESSOR_COMPOSE.read_bytes()).hexdigest(),
RUNNER.DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256,
)
self.assertEqual(
hashlib.sha256(COMPOSE.read_bytes()).hexdigest(),
RUNNER
.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_COMPOSE_SHA256,
)
self.assertIn("name: nodedc-device-plane-control", compose)
self.assertIn(
'com.docker.network.bridge.enable_ip_masquerade: "false"',
compose,
)
self.assertNotIn("nodedc-device-plane-control", predecessor)
postgres_section, stateless = compose.split(
" device-control-core:",
1,
)
self.assertNotIn("device-plane-control", postgres_section)
self.assertEqual(stateless.count(" - device-plane-control"), 2)
self.assertNotIn("9921:9921", compose)
def test_runner_selects_exact_services_no_build_and_no_deps(self):
entries = (
RUNNER.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES
)
services = RUNNER.component_services("device-plane", entries)
self.assertEqual(
services,
("device-control-core", "device-gateway"),
)
self.assertEqual(
RUNNER.component_builds("device-plane", entries),
(),
)
with (
mock.patch.object(RUNNER, "run_build") as run_build,
mock.patch.object(
RUNNER,
"prepare_component_runtime",
) as prepare,
mock.patch.object(RUNNER, "run_compose") as run_compose,
):
RUNNER.run_component_runtime(
"device-plane",
entries,
services,
)
run_build.assert_not_called()
prepare.assert_called_once_with("device-plane", entries)
run_compose.assert_called_once_with(
"device-plane",
services,
entries,
)
with mock.patch.object(RUNNER.subprocess, "run") as run:
RUNNER.run_compose("device-plane", services, entries)
command = run.call_args_list[0].args[0]
self.assertIn("--force-recreate", command)
self.assertIn("--no-deps", command)
self.assertEqual(
command[-2:],
["device-control-core", "device-gateway"],
)
def test_terminal_failed_recovery_is_rejected(self):
with self.assertRaisesRegex(
RUNNER.DeployError,
"recovery 002 is terminal failed",
):
RUNNER.reject_terminal_device_plane_foundation_artifact(
{
"id":
RUNNER
.DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_PATCH_ID,
},
"0" * 64,
)
with self.assertRaisesRegex(
RUNNER.DeployError,
"recovery 002 is terminal failed",
):
RUNNER.reject_terminal_device_plane_foundation_artifact(
{"id": "unrelated"},
RUNNER
.DEVICE_PLANE_FOUNDATION_RECOVERY_FAILED_ARTIFACT_SHA256,
)
def test_rollback_is_stateless_and_removes_only_control_network(self):
entries = (
RUNNER.DEVICE_PLANE_FOUNDATION_NETWORK_PUBLICATION_ENTRIES
)
services = ("device-control-core", "device-gateway")
runtime = {
"device-control-core": {"containerId": "1" * 64},
"device-gateway": {"containerId": "2" * 64},
"device-postgres": {
"containerId":
RUNNER
.DEVICE_PLANE_FOUNDATION_PREDECESSOR_CONTAINER_IDS[
"device-postgres"
],
},
}
with (
mock.patch.object(
RUNNER,
"stop_and_remove_compose_services",
) as stop,
mock.patch.object(
RUNNER,
"restore_platform_overlay",
return_value=9,
) as restore,
mock.patch.object(
RUNNER,
"remove_device_plane_control_network_if_unused",
return_value="removed",
) as remove_network,
mock.patch.object(RUNNER, "run_compose") as compose,
mock.patch.object(
RUNNER,
"healthcheck_compose_service",
) as health,
mock.patch.object(
RUNNER,
"validate_device_plane_foundation_runtime",
return_value=runtime,
) as validate_runtime,
mock.patch.object(
RUNNER,
"assert_loopback_tcp_port_closed",
) as closed,
):
result = (
RUNNER
.rollback_device_plane_network_publication_apply(
Path("/live"),
Path("/backup"),
entries,
"stamp",
True,
services,
)
)
self.assertEqual(result, "source+internal-runtime-restored:9")
stop.assert_called_once_with("device-plane", services)
restore.assert_called_once()
remove_network.assert_called_once_with()
compose.assert_called_once_with(
"device-plane",
services,
("docker-compose.device-plane.yml",),
)
self.assertEqual(health.call_count, 3)
validate_runtime.assert_called_once_with()
self.assertEqual(
[call.args[0] for call in closed.call_args_list],
[18120, 18121, 9921],
)
if __name__ == "__main__":
unittest.main()

6
infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py
View File

@ -18,6 +18,10 @@ RECOVERY_BUILDER = (
SCRIPT_DIR / "build-device-plane-foundation-recovery-artifact.mjs" SCRIPT_DIR / "build-device-plane-foundation-recovery-artifact.mjs"
) )
FOUNDATION_BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs" FOUNDATION_BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs"
PREDECESSOR_COMPOSE = (
SCRIPT_DIR
/ "fixtures/device-plane-foundation-internal-only-v1.yml"
)
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
RECOVERY_ENTRIES = [ RECOVERY_ENTRIES = [
".dockerignore", ".dockerignore",
@ -246,7 +250,7 @@ class DevicePlaneFoundationRecoveryArtifactTest(unittest.TestCase):
source_root = SCRIPT_DIR.parent.parent / "device-plane" source_root = SCRIPT_DIR.parent.parent / "device-plane"
shutil.copy2( shutil.copy2(
source_root / "docker-compose.device-plane.yml", PREDECESSOR_COMPOSE,
live_root / "docker-compose.device-plane.yml", live_root / "docker-compose.device-plane.yml",
) )
shutil.copy2( shutil.copy2(