fix(deploy): reconcile device plane foundation

This commit is contained in:
Codex 2026-07-25 23:11:14 +03:00
parent 1102e25e6e
commit aca47c6143
8 changed files with 1692 additions and 15 deletions

View File

@ -0,0 +1,16 @@
{
"schemaVersion": "nodedc.device-plane.foundation-recovery.v1",
"mode": "failed-foundation-live-runtime-adoption",
"failedPatchId": "device-plane-foundation-20260725-001",
"failedArtifactSha256": "23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24",
"backupId": "device-plane-device-plane-foundation-20260725-001-20260725-223441",
"sourceAction": "publish-exact-failed-artifact-source",
"runtimeAction": "read-only-acceptance",
"preservedServices": [
"device-control-core",
"device-gateway",
"device-postgres"
],
"databaseVolume": "nodedc-device-plane-postgres-data",
"rollback": "source-only-runtime-unchanged"
}

View File

@ -1,7 +1,11 @@
# Device Plane Implementation Baseline # Device Plane Implementation Baseline
Status: local fail-closed foundation implemented; no runtime or production Status: PostgreSQL, Control Core and Gateway foundation are running healthy on
mutation. Synology with public ingress, discovery ingest and commands disabled. The first
application artifact is terminal-failed because its health gate timed out and
its automatic rollback could not model the DB-bootstrap predecessor. Recovery
is a separate source-only canonical transition; the failed artifact is never
retried.
## Product boundary ## Product boundary
@ -72,6 +76,14 @@ The canonical runner selects only `device-control-core` and `device-gateway`
with `--no-deps`. Its health acceptance is scoped to the selected services and with `--no-deps`. Its health acceptance is scoped to the selected services and
requires the fail-closed fields to remain disabled. A failed first activation requires the fail-closed fields to remain disabled. A failed first activation
removes only candidate stateless services and never requests volume removal. removes only candidate stateless services and never requests volume removal.
Rollback now records an explicit pre-apply service inventory in the backup;
the existence of the shared Compose file does not imply that Core or Gateway
existed before apply.
The exact foundation recovery validates the failed archive, journal, backup,
partial live source and observed healthy image/container generations. It then
publishes the matching source and performs read-only runtime acceptance. It
does not build, restart, recreate or remove any service.
## Network boundary ## Network boundary
@ -157,15 +169,13 @@ raw TCP remain forbidden until separate reviewed acceptance slices.
## Next source slice ## Next source slice
1. Review, hash and promote the canonical runner candidate through the 1. Promote the reviewed recovery-capable runner through the standalone root
standalone root administrative gate; do not stage an application artifact administrative gate and run a fresh `verify-install`.
before fresh `verify-install`. 2. Plan and apply the exact source-only foundation recovery artifact once, then
2. Build, plan and accept the separate one-time PostgreSQL prerequisite reconcile its journal/backup and repeat runtime acceptance.
bootstrap artifact; ordinary application artifacts must continue to exclude
the database service.
3. Obtain or capture the exact official B2 INTERNAL framing and acknowledgement 3. Obtain or capture the exact official B2 INTERNAL framing and acknowledgement
contract; keep ingress disabled until its fixtures pass. contract; keep ingress disabled until its fixtures pass.
4. Add an authenticated internal Core/Gateway discovery boundary and explicit 4. Add an authenticated internal Core/Gateway discovery boundary and explicit
platform-admin claim operation. platform-admin claim operation.
5. Build, audit and stage the first deterministic application artifact only 5. Open raw TCP ingress only in a later reviewed slice after quarantine,
after the runner extension is installed and verified. framing, rate-limit and audit acceptance.

View File

@ -20,6 +20,11 @@
| Database preservation | Ordinary application artifacts never select `device-postgres` | | Database preservation | Ordinary application artifacts never select `device-postgres` |
| Database bootstrap | Exact descriptor selects PostgreSQL only when both container and volume are absent | | Database bootstrap | Exact descriptor selects PostgreSQL only when both container and volume are absent |
| Bootstrap rollback | Candidate container may be removed; named volume is never removed | | Bootstrap rollback | Candidate container may be removed; named volume is never removed |
| Rollback predecessor | Backup records actual pre-apply services; Compose presence cannot invent Core/Gateway |
| Failed-001 evidence | Recovery requires exact failed archive, journal and backup digests |
| Partial source | Recovery accepts only DB-bootstrap source plus the observed healthy foundation runtime |
| Recovery mutation | Source is published without build, restart, recreate or service removal |
| Recovery rollback | Failed acceptance restores source only and leaves runtime unchanged |
| Artifact policy | `.env`, secrets, runtime state, tests, logs and `node_modules` are excluded | | Artifact policy | `.env`, secrets, runtime state, tests, logs and `node_modules` are excluded |
| Artifact reproducibility | Repeated builds for the same patch id are byte-identical | | Artifact reproducibility | Repeated builds for the same patch id are byte-identical |
| Runner compatibility | Existing canonical Platform registry tests remain green | | Runner compatibility | Existing canonical Platform registry tests remain green |

View File

@ -476,6 +476,29 @@ node infra/deploy-runner/build-device-plane-artifact.mjs \
Any failed first activation removes only candidate Core/Gateway containers, Any failed first activation removes only candidate Core/Gateway containers,
never volumes, restores the source overlay and retains PostgreSQL state. never volumes, restores the source overlay and retains PostgreSQL state.
The rollback baseline is taken from an explicit pre-apply Docker service
inventory stored as `runtime-before.json` in the backup. The presence of the
shared Compose file never implies that Core or Gateway existed before apply.
The exact `device-plane-foundation-20260725-001` failed activation is recovered
only through the registered
`deployment/device-plane-foundation-recovery-v1.json` transition. Its artifact
must reproduce the failed foundation source byte-for-byte. The runner validates
the exact failed archive, journal, backup partition, partial live source and
healthy observed Core/Gateway/PostgreSQL generations. Recovery publishes source
and performs read-only runtime acceptance; it does not build, recreate, restart
or remove containers. A failed recovery restores source only and leaves runtime
unchanged.
Build and test that incident-specific deterministic recovery artifact:
```bash
python3 -m unittest -v \
infra.deploy-runner.test_device_plane_foundation_recovery_artifact
node infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs \
device-plane-foundation-recovery-20260725-002
```
After the runner is promoted and freshly verified, bootstrap the durable After the runner is promoted and freshly verified, bootstrap the durable
prerequisite with a separate artifact before planning the application: prerequisite with a separate artifact before planning the application:

View File

@ -0,0 +1,248 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { spawnSync } from "node:child_process";
import {
cp,
lstat,
mkdir,
mkdtemp,
readFile,
readdir,
rm,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const scriptDir = dirname(fileURLToPath(import.meta.url));
const platformRoot = resolve(scriptDir, "../..");
const sourceRoot = resolve(platformRoot, "device-plane");
const artifactDir = resolve(
process.env.NODEDC_DEPLOY_ARTIFACT_DIR
|| resolve(scriptDir, "../deploy-artifacts"),
);
const [
patchId = "device-plane-foundation-recovery-20260725-002",
...extra
] = process.argv.slice(2);
if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) {
throw new Error(
"usage: build-device-plane-foundation-recovery-artifact.mjs [patch-id]",
);
}
const files = [
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
"deployment/device-plane-foundation-recovery-v1.json",
];
const ignoredBasenames = new Set([
".DS_Store",
".git",
"node_modules",
]);
const ignoredDirectoryNames = new Set(["test"]);
const stage = await mkdtemp(
join(tmpdir(), "nodedc-device-plane-foundation-recovery-"),
);
const payload = join(stage, "payload");
const target = join(
artifactDir,
`nodedc-device-plane-${patchId}.tgz`,
);
await assertRecoveryBoundary();
try {
await mkdir(payload, { recursive: true });
for (const sourceRelative of files) {
await copySafe(
resolve(sourceRoot, sourceRelative),
join(payload, sourceRelative),
);
}
await writeFile(
join(stage, "manifest.env"),
`id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`,
"utf8",
);
await writeFile(
join(stage, "files.txt"),
`${files.join("\n")}\n`,
"utf8",
);
await mkdir(artifactDir, { recursive: true });
const tar = spawnSync(
"python3",
["-c", canonicalTarScript(), target, stage],
{
encoding: "utf8",
maxBuffer: 128 * 1024 * 1024,
},
);
if (tar.status !== 0) {
throw new Error(`tar_failed:${tar.stderr || tar.stdout}`);
}
const digest = createHash("sha256")
.update(await readFile(target))
.digest("hex");
console.log(JSON.stringify({
ok: true,
patchId,
artifact: target,
sha256: digest,
component: "device-plane",
transition: "failed-foundation-live-runtime-adoption",
entries: files,
build: [],
services: [],
preservedRuntime: [
"device-control-core",
"device-gateway",
"device-postgres",
"nodedc-device-plane-postgres-data",
],
sourceAction: "publish-exact-failed-artifact-source",
runtimeAction: "read-only-acceptance",
excluded: [
".env*",
"node_modules",
"**/test",
"docs",
"runtime",
"secrets",
],
}, null, 2));
} finally {
await rm(stage, { recursive: true, force: true });
}
async function assertRecoveryBoundary() {
const compose = await readFile(
resolve(sourceRoot, "docker-compose.device-plane.yml"),
"utf8",
);
for (const fragment of [
'DEVICE_DISCOVERY_INGEST_ENABLED: "false"',
'DEVICE_GATEWAY_LISTEN_ENABLED: "false"',
'"127.0.0.1:18120:18120"',
'"127.0.0.1:18121:18121"',
"name: nodedc-device-plane-postgres-data",
]) {
if (!compose.includes(fragment)) {
throw new Error(
`device_plane_recovery_compose_boundary_missing:${fragment}`,
);
}
}
for (const forbidden of [
"9921:9921",
"0.0.0.0:9921",
'DEVICE_DISCOVERY_INGEST_ENABLED: "true"',
'DEVICE_GATEWAY_LISTEN_ENABLED: "true"',
"POSTGRES_PASSWORD:",
]) {
if (compose.includes(forbidden)) {
throw new Error(
`device_plane_recovery_compose_boundary_violation:${forbidden}`,
);
}
}
const descriptor = JSON.parse(await readFile(
resolve(
sourceRoot,
"deployment/device-plane-foundation-recovery-v1.json",
),
"utf8",
));
const expected = {
schemaVersion: "nodedc.device-plane.foundation-recovery.v1",
mode: "failed-foundation-live-runtime-adoption",
failedPatchId: "device-plane-foundation-20260725-001",
failedArtifactSha256:
"23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24",
backupId:
"device-plane-device-plane-foundation-20260725-001-20260725-223441",
sourceAction: "publish-exact-failed-artifact-source",
runtimeAction: "read-only-acceptance",
preservedServices: [
"device-control-core",
"device-gateway",
"device-postgres",
],
databaseVolume: "nodedc-device-plane-postgres-data",
rollback: "source-only-runtime-unchanged",
};
if (JSON.stringify(descriptor) !== JSON.stringify(expected)) {
throw new Error("device_plane_recovery_descriptor_mismatch");
}
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
"root=pathlib.Path(sys.argv[2])",
"with open(sys.argv[1],'wb') as out:",
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
" for top in ('manifest.env','files.txt','payload'):",
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
" for x in paths:",
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
].join("\n");
}
async function copySafe(source, destination) {
const sourceStat = await lstat(source);
if (sourceStat.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, source)}`,
);
}
if (sourceStat.isFile()) {
await mkdir(dirname(destination), { recursive: true });
await cp(source, destination, {
force: true,
verbatimSymlinks: true,
});
return;
}
if (!sourceStat.isDirectory()) {
throw new Error(`source_type_rejected:${source}`);
}
await mkdir(destination, { recursive: true });
for (const entry of await readdir(source, { withFileTypes: true })) {
if (
ignoredBasenames.has(entry.name)
|| entry.name.startsWith(".env")
|| (
entry.isDirectory()
&& ignoredDirectoryNames.has(entry.name)
)
) {
continue;
}
const childSource = join(source, entry.name);
const childDestination = join(destination, entry.name);
if (entry.isSymbolicLink()) {
throw new Error(
`source_symlink_rejected:${relative(sourceRoot, childSource)}`,
);
}
await copySafe(childSource, childDestination);
}
}

View File

@ -52,6 +52,75 @@ DEVICE_PLANE_POSTGRES_BOOTSTRAP_ENTRIES = (
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
) )
DEVICE_PLANE_POSTGRES_VOLUME = "nodedc-device-plane-postgres-data" DEVICE_PLANE_POSTGRES_VOLUME = "nodedc-device-plane-postgres-data"
DEVICE_PLANE_FOUNDATION_ENTRIES = (
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
)
DEVICE_PLANE_FOUNDATION_RECOVERY_REL = (
"deployment/device-plane-foundation-recovery-v1.json"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES = (
*DEVICE_PLANE_FOUNDATION_ENTRIES,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
)
DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = (
"device-plane-foundation-20260725-001"
)
DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT = (
"nodedc-device-plane-device-plane-foundation-20260725-001.tgz."
"20260725-223441"
)
DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 = (
"23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID = (
"device-plane-device-plane-foundation-20260725-001-20260725-223441"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256 = {
"manifest.env": (
"21ff4250147f99fd1461191e1f05aee99e92da9cdedce4513749046c26327c77"
),
"files.txt": (
"3b7dda5cc42ce460e4a74f6df986d4a4fa8aa28511e9d9b123deb445a0382083"
),
"existing-files.txt": (
"f2e2c6e7501d3a383cd7e909a2e4e4cfbb3b0cda132f508f54e478bfb455585b"
),
"missing-files.txt": (
"308d86656e0a0863d5fa16833ec67d1698c0cef7e009a7bd987e62a9e6469c63"
),
"source-before.tgz": (
"df451cdf7a5798c332d03f15473358b1e7fdb7b96212037aaed2d8d00dfecf68"
),
}
DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256 = (
"e3afbb1ab252914e21dfb0acc1a2016fd8af471aa13bb14fb578f106f2cdae36"
)
DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256 = (
"9475f12d151aee386cba74a4c0f9e7485416d5616d3ad71dfd8963fe114abf61"
)
DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS = {
"device-control-core": (
"sha256:78a9fdfbe46f6e9531536abb997ae105fc7ea5c9a8176cb88ca8c0644900c254"
),
"device-gateway": (
"sha256:9be91a3496d1ac627b6b621ade0c2ae77afa10e9ddb9b2071823769bc5a5a472"
),
"device-postgres": (
"sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d"
),
}
DEVICE_PLANE_RUNTIME_SERVICES = (
"device-control-core",
"device-gateway",
"device-postgres",
)
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner" EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner"
EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token" EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token"
ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "engine-credential-provisioner-ed25519.pem" ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "engine-credential-provisioner-ed25519.pem"
@ -2736,6 +2805,7 @@ def allowed_payload_path(component, rel):
"package-lock.json", "package-lock.json",
"docker-compose.device-plane.yml", "docker-compose.device-plane.yml",
DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
"packages/device-protocol-contract", "packages/device-protocol-contract",
"packages/arusnavi-b2-adapter", "packages/arusnavi-b2-adapter",
"services/device-control-core", "services/device-control-core",
@ -7760,6 +7830,11 @@ def load_artifact(artifact, work_dir):
entries, entries,
): ):
validate_device_plane_postgres_bootstrap_payload(payload_dir) validate_device_plane_postgres_bootstrap_payload(payload_dir)
if is_device_plane_foundation_recovery_slice(
manifest["component"],
entries,
):
validate_device_plane_foundation_recovery_payload(payload_dir)
if manifest["component"] == "n8n-private-extension": if manifest["component"] == "n8n-private-extension":
validate_n8n_private_extension_release(payload_dir, entries) validate_n8n_private_extension_release(payload_dir, entries)
if manifest["component"] == "engine": if manifest["component"] == "engine":
@ -8034,6 +8109,17 @@ def reject_terminal_engine_l2_failed_artifact(manifest, sha256):
) )
def reject_terminal_device_plane_foundation_artifact(manifest, sha256):
if (
manifest.get("id") == DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID
or sha256 == DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256
):
die(
"Device Plane foundation 001 is terminal failed; "
"use the exact registered foundation recovery successor"
)
class DeployLock: class DeployLock:
def __enter__(self): def __enter__(self):
try: try:
@ -8059,6 +8145,325 @@ def is_device_plane_postgres_bootstrap_slice(component, entries):
) )
def is_device_plane_foundation_recovery_slice(component, entries):
return (
component == "device-plane"
and entries is not None
and tuple(entries) == DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES
)
def expected_device_plane_foundation_recovery_descriptor():
return {
"schemaVersion": "nodedc.device-plane.foundation-recovery.v1",
"mode": "failed-foundation-live-runtime-adoption",
"failedPatchId": DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID,
"failedArtifactSha256": (
DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256
),
"backupId": DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID,
"sourceAction": "publish-exact-failed-artifact-source",
"runtimeAction": "read-only-acceptance",
"preservedServices": [
"device-control-core",
"device-gateway",
"device-postgres",
],
"databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME,
"rollback": "source-only-runtime-unchanged",
}
def validate_device_plane_foundation_recovery_payload(payload_dir):
descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
"Device Plane foundation recovery descriptor",
max_bytes=16 * 1024,
)
if descriptor != expected_device_plane_foundation_recovery_descriptor():
die("Device Plane foundation recovery descriptor mismatch")
return descriptor
def device_plane_service_container_ids(service):
if service not in DEVICE_PLANE_RUNTIME_SERVICES:
die(f"Device Plane runtime service is not registered: {service}")
result = subprocess.run(
[
str(DOCKER),
"container",
"ls",
"-a",
"--filter",
"label=com.docker.compose.project=nodedc-device-plane",
"--filter",
f"label=com.docker.compose.service={service}",
"--format",
"{{.ID}}",
],
check=False,
capture_output=True,
text=True,
)
container_ids = [
line.strip()
for line in result.stdout.splitlines()
if line.strip()
]
if (
result.returncode != 0
or any(
not re.fullmatch(r"[a-f0-9]{12,64}", container_id)
for container_id in container_ids
)
or len(container_ids) > 1
):
die(f"Device Plane runtime inventory failed: {service}")
return tuple(container_ids)
def inspect_device_plane_container(container_id):
containers = docker_json(
["container", "inspect", container_id],
"Device Plane container inspect",
)
if (
not isinstance(containers, list)
or len(containers) != 1
or not isinstance(containers[0], dict)
):
die("Device Plane container inspect shape mismatch")
return containers[0]
def device_plane_runtime_inventory(services):
inventory = []
for service in services:
container_ids = device_plane_service_container_ids(service)
if not container_ids:
continue
container = inspect_device_plane_container(container_ids[0])
state = container.get("State") or {}
health = (state.get("Health") or {}).get("Status")
inventory.append({
"service": service,
"containerId": container.get("Id"),
"imageId": container.get("Image"),
"status": state.get("Status"),
"running": state.get("Running") is True,
"health": health,
"restartCount": int(container.get("RestartCount") or 0),
})
return {
"schemaVersion": "nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": inventory,
}
def device_plane_inventory_service_names(inventory):
if (
not isinstance(inventory, dict)
or inventory.get("schemaVersion")
!= "nodedc.device-plane.runtime-inventory.v1"
or inventory.get("composeProject") != "nodedc-device-plane"
or not isinstance(inventory.get("services"), list)
):
die("Device Plane runtime inventory shape mismatch")
names = []
for item in inventory["services"]:
if (
not isinstance(item, dict)
or set(item) != {
"service",
"containerId",
"imageId",
"status",
"running",
"health",
"restartCount",
}
or item.get("service") not in DEVICE_PLANE_RUNTIME_SERVICES
or not isinstance(item.get("containerId"), str)
or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"])
or not isinstance(item.get("imageId"), str)
or not re.fullmatch(r"sha256:[a-f0-9]{64}", item["imageId"])
or not isinstance(item.get("status"), str)
or not isinstance(item.get("running"), bool)
or item.get("health") not in (
None,
"starting",
"healthy",
"unhealthy",
)
or not isinstance(item.get("restartCount"), int)
or item["restartCount"] < 0
):
die("Device Plane runtime inventory service mismatch")
service = item["service"]
if service in names:
die("Device Plane runtime inventory contains duplicates")
names.append(service)
return tuple(names)
def validate_device_plane_foundation_recovery_evidence(payload_dir):
backup_dir = BACKUPS_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID
try:
backup_stat = backup_dir.lstat()
except FileNotFoundError:
die("Device Plane foundation recovery backup is missing")
if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(
backup_stat.st_mode
):
die("Device Plane foundation recovery backup is unsafe")
backup_names = {child.name for child in backup_dir.iterdir()}
if backup_names != set(DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256):
die("Device Plane foundation recovery backup file set mismatch")
for name, expected in (
DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256.items()
):
path = backup_dir / name
path_stat = path.lstat()
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or sha256_file(path) != expected
):
die(
"Device Plane foundation recovery backup drift detected: "
f"{name}"
)
failed_artifact = FAILED_DIR / DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT
try:
failed_stat = failed_artifact.lstat()
except FileNotFoundError:
die("Device Plane foundation failed artifact is missing")
if (
stat.S_ISLNK(failed_stat.st_mode)
or not stat.S_ISREG(failed_stat.st_mode)
or sha256_file(failed_artifact)
!= DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256
):
die("Device Plane foundation failed artifact evidence mismatch")
try:
state_stat = FAILED_STATE_FILE.lstat()
state_lines = FAILED_STATE_FILE.read_text(
encoding="utf-8"
).splitlines()
except (FileNotFoundError, OSError, UnicodeDecodeError):
die("Device Plane foundation failed journal is unreadable")
if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(
state_stat.st_mode
):
die("Device Plane foundation failed journal is unsafe")
records = []
for line in state_lines:
try:
value = json.loads(line)
except json.JSONDecodeError:
die("Device Plane foundation failed journal contains invalid JSON")
if (
isinstance(value, dict)
and value.get("id") == DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID
):
records.append(value)
if len(records) != 1:
die("Device Plane foundation failed journal evidence count mismatch")
record = records[0]
if (
record.get("artifact") != DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT
or record.get("backup_id")
!= DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID
or record.get("component") != "device-plane"
or record.get("sha256")
!= DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256
or record.get("started_apply") is not True
or record.get("rollback_status") != "failed:DeployError"
or record.get("status") != "failed"
or record.get("message")
!= (
"healthcheck failed for http://127.0.0.1:18120/healthz: "
"<urlopen error [Errno 111] Connection refused>"
)
):
die("Device Plane foundation failed journal evidence mismatch")
with tempfile.TemporaryDirectory(
prefix="device-plane-failed-foundation-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID
or failed_manifest.get("component") != "device-plane"
or failed_manifest.get("type") != "app-overlay"
or tuple(failed_entries) != DEVICE_PLANE_FOUNDATION_ENTRIES
):
die("Device Plane foundation failed artifact contract mismatch")
failed_source = collect_exact_files(
failed_payload,
DEVICE_PLANE_FOUNDATION_ENTRIES,
"Device Plane failed foundation source",
)
candidate_source = collect_exact_files(
payload_dir,
DEVICE_PLANE_FOUNDATION_ENTRIES,
"Device Plane recovery candidate source",
)
if candidate_source != failed_source:
die(
"Device Plane recovery candidate does not match failed "
"foundation source"
)
root = component_root("device-plane")
compose_path = root / "docker-compose.device-plane.yml"
if (
compose_path.is_symlink()
or not compose_path.is_file()
or sha256_file(compose_path)
!= DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256
):
die("Device Plane foundation partial Compose predecessor mismatch")
for rel in DEVICE_PLANE_FOUNDATION_ENTRIES:
if rel == "docker-compose.device-plane.yml":
continue
path = root / rel
if path.exists() or path.is_symlink():
die(
"Device Plane foundation partial predecessor retained source: "
f"{rel}"
)
recovery_descriptor = root / DEVICE_PLANE_FOUNDATION_RECOVERY_REL
if recovery_descriptor.exists() or recovery_descriptor.is_symlink():
die("Device Plane foundation recovery descriptor already exists")
bootstrap_descriptor = (
root / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL
)
if (
bootstrap_descriptor.is_symlink()
or not bootstrap_descriptor.is_file()
or sha256_file(bootstrap_descriptor)
!= DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256
):
die("Device Plane PostgreSQL bootstrap descriptor drift detected")
runtime = validate_device_plane_foundation_runtime()
return {
"mode": "failed-foundation-live-runtime-adoption",
"backup": backup_dir,
"failedArtifact": failed_artifact,
"runtime": runtime,
}
def validate_device_plane_postgres_bootstrap_payload(payload_dir): def validate_device_plane_postgres_bootstrap_payload(payload_dir):
descriptor = read_strict_json( descriptor = read_strict_json(
payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL,
@ -8132,6 +8537,303 @@ def device_plane_postgres_plan_selection(postgres_preflight):
return "bootstrap-selected:create-if-absent" return "bootstrap-selected:create-if-absent"
def validate_device_plane_foundation_runtime():
validate_device_plane_runtime_secret_metadata()
expected = {
"device-control-core": {
"image": DEVICE_PLANE_CONTROL_CORE_IMAGE,
"imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[
"device-control-core"
],
"user": "1000:1000",
"ports": {
"18120/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18120",
}],
},
},
"device-gateway": {
"image": DEVICE_PLANE_GATEWAY_IMAGE,
"imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[
"device-gateway"
],
"user": "1000:1000",
"ports": {
"18121/tcp": [{
"HostIp": "127.0.0.1",
"HostPort": "18121",
}],
},
},
"device-postgres": {
"image": "postgres:16-alpine",
"imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[
"device-postgres"
],
"user": "",
"ports": {},
},
}
accepted = {}
for service, contract in expected.items():
container_ids = device_plane_service_container_ids(service)
if len(container_ids) != 1:
die(
"Device Plane foundation runtime service count mismatch: "
f"{service}"
)
container = inspect_device_plane_container(container_ids[0])
state = container.get("State") or {}
config = container.get("Config") or {}
host_config = container.get("HostConfig") or {}
labels = config.get("Labels") or {}
networks = (container.get("NetworkSettings") or {}).get(
"Networks"
) or {}
if (
state.get("Status") != "running"
or state.get("Running") is not True
or state.get("Restarting") is True
or state.get("ExitCode") != 0
or state.get("Error") not in ("", None)
or (state.get("Health") or {}).get("Status") != "healthy"
or int(container.get("RestartCount") or 0) != 0
):
die(
"Device Plane foundation runtime state mismatch: "
f"{service}"
)
if (
container.get("Image") != contract["imageId"]
or config.get("Image") != contract["image"]
or config.get("User", "") != contract["user"]
or (host_config.get("PortBindings") or {}) != contract["ports"]
or (host_config.get("RestartPolicy") or {}).get("Name")
!= "unless-stopped"
or labels.get("com.docker.compose.project")
!= "nodedc-device-plane"
or labels.get("com.docker.compose.service") != service
or set(networks) != {"nodedc-device-plane-private"}
):
die(
"Device Plane foundation runtime boundary mismatch: "
f"{service}"
)
accepted[service] = {
"containerId": container.get("Id"),
"imageId": container.get("Image"),
"health": "healthy",
"restartCount": 0,
}
core = inspect_device_plane_container(
device_plane_service_container_ids("device-control-core")[0]
)
gateway = inspect_device_plane_container(
device_plane_service_container_ids("device-gateway")[0]
)
postgres = inspect_device_plane_container(
device_plane_service_container_ids("device-postgres")[0]
)
for service, container in (
("device-control-core", core),
("device-gateway", gateway),
):
host_config = container.get("HostConfig") or {}
if (
host_config.get("ReadonlyRootfs") is not True
or set(host_config.get("CapDrop") or ()) != {"ALL"}
or "no-new-privileges:true"
not in set(host_config.get("SecurityOpt") or ())
):
die(
"Device Plane stateless hardening mismatch: "
f"{service}"
)
core_environment = container_environment(
core,
"Device Plane Control Core",
)
core_required_environment = {
"HOST": "0.0.0.0",
"PORT": "18120",
"DEVICE_DATABASE_HOST": "device-postgres",
"DEVICE_DATABASE_PORT": "5432",
"DEVICE_DATABASE_NAME": "device_plane",
"DEVICE_DATABASE_USER": "device_plane",
"DEVICE_DATABASE_PASSWORD_FILE": (
"/run/nodedc-secrets/postgres-password"
),
"DEVICE_DATABASE_POOL_SIZE": "10",
"DEVICE_DISCOVERY_INGEST_ENABLED": "false",
}
if any(
core_environment.get(key) != value
for key, value in core_required_environment.items()
):
die("Device Plane Control Core environment mismatch")
for forbidden in (
"DEVICE_DATABASE_URL",
"DEVICE_DATABASE_PASSWORD",
"DEVICE_GATEWAY_CORE_TOKEN",
"DEVICE_IDENTIFIER_PEPPER",
):
if forbidden in core_environment:
die("Device Plane Control Core plaintext secret boundary mismatch")
gateway_environment = container_environment(
gateway,
"Device Plane Gateway",
)
gateway_required_environment = {
"DEVICE_GATEWAY_HEALTH_HOST": "0.0.0.0",
"DEVICE_GATEWAY_HEALTH_PORT": "18121",
"DEVICE_GATEWAY_LISTEN_ENABLED": "false",
"DEVICE_GATEWAY_TCP_HOST": "127.0.0.1",
"DEVICE_GATEWAY_TCP_PORT": "9921",
"DEVICE_GATEWAY_MAX_SESSIONS": "100",
"DEVICE_GATEWAY_SESSION_TIMEOUT_MS": "10000",
}
if any(
gateway_environment.get(key) != value
for key, value in gateway_required_environment.items()
):
die("Device Plane Gateway environment mismatch")
if any(
key in gateway_environment
for key in (
"DEVICE_GATEWAY_CORE_TOKEN",
"DEVICE_GATEWAY_COMMAND_TOKEN",
)
):
die("Device Plane Gateway plaintext secret boundary mismatch")
core_mounts = core.get("Mounts") or []
if len(core_mounts) != 1:
die("Device Plane Control Core mount count mismatch")
core_secret = core_mounts[0]
if (
core_secret.get("Type") != "bind"
or core_secret.get("Source")
!= str(DEVICE_PLANE_POSTGRES_PASSWORD_FILE)
or core_secret.get("Destination")
!= "/run/nodedc-secrets/postgres-password"
or core_secret.get("RW") is not False
):
die("Device Plane Control Core secret mount mismatch")
if gateway.get("Mounts") not in (None, []):
die("Device Plane Gateway unexpected mount")
postgres_mounts = postgres.get("Mounts") or []
postgres_secret = [
mount
for mount in postgres_mounts
if mount.get("Destination")
== "/run/nodedc-secrets/postgres-password"
]
postgres_volume = [
mount
for mount in postgres_mounts
if mount.get("Destination") == "/var/lib/postgresql/data"
]
if (
len(postgres_mounts) != 2
or len(postgres_secret) != 1
or postgres_secret[0].get("Type") != "bind"
or postgres_secret[0].get("Source")
!= str(DEVICE_PLANE_POSTGRES_PASSWORD_FILE)
or postgres_secret[0].get("RW") is not False
or len(postgres_volume) != 1
or postgres_volume[0].get("Type") != "volume"
or postgres_volume[0].get("Name")
!= DEVICE_PLANE_POSTGRES_VOLUME
or postgres_volume[0].get("RW") is not True
):
die("Device Plane PostgreSQL preserved mount mismatch")
return accepted
def validate_device_plane_runtime_secret_metadata():
try:
directory_stat = DEVICE_PLANE_SECRET_DIR.lstat()
except FileNotFoundError:
die("Device Plane runtime secret directory is missing")
if (
stat.S_ISLNK(directory_stat.st_mode)
or not stat.S_ISDIR(directory_stat.st_mode)
or directory_stat.st_uid != 0
or directory_stat.st_gid != MAP_GATEWAY_RUNTIME_GID
or stat.S_IMODE(directory_stat.st_mode) != 0o710
):
die("Device Plane runtime secret directory boundary mismatch")
for path, label in (
(DEVICE_PLANE_POSTGRES_PASSWORD_FILE, "PostgreSQL"),
(DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE, "Gateway to Core"),
(DEVICE_PLANE_IDENTIFIER_PEPPER_FILE, "identifier pepper"),
):
try:
path_stat = path.lstat()
except FileNotFoundError:
die(f"Device Plane {label} runtime secret is missing")
if (
stat.S_ISLNK(path_stat.st_mode)
or not stat.S_ISREG(path_stat.st_mode)
or path_stat.st_uid != 0
or path_stat.st_gid != MAP_GATEWAY_RUNTIME_GID
or stat.S_IMODE(path_stat.st_mode) != 0o640
or path_stat.st_size > 512
):
die(f"Device Plane {label} runtime secret boundary mismatch")
try:
value = path.read_text(encoding="ascii").strip()
except (OSError, UnicodeDecodeError):
die(f"Device Plane {label} runtime secret is unreadable")
if not MAP_GATEWAY_SECRET_RE.fullmatch(value):
die(f"Device Plane {label} runtime secret format mismatch")
return "exact"
def validate_device_plane_foundation_installed_source():
failed_artifact = FAILED_DIR / DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT
with tempfile.TemporaryDirectory(
prefix="device-plane-installed-foundation-",
dir=TMP_DIR,
) as directory:
failed_manifest, failed_entries, failed_payload = load_artifact(
failed_artifact,
Path(directory),
)
if (
failed_manifest.get("id")
!= DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID
or tuple(failed_entries) != DEVICE_PLANE_FOUNDATION_ENTRIES
):
die("Device Plane failed foundation source contract mismatch")
expected = collect_exact_files(
failed_payload,
DEVICE_PLANE_FOUNDATION_ENTRIES,
"Device Plane failed foundation source",
)
actual = collect_exact_files(
component_root("device-plane"),
DEVICE_PLANE_FOUNDATION_ENTRIES,
"Device Plane installed foundation source",
)
if actual != expected:
die("Device Plane installed foundation source mismatch")
descriptor = read_strict_json(
component_root("device-plane")
/ DEVICE_PLANE_FOUNDATION_RECOVERY_REL,
"installed Device Plane foundation recovery descriptor",
max_bytes=16 * 1024,
)
if descriptor != expected_device_plane_foundation_recovery_descriptor():
die("installed Device Plane foundation recovery descriptor mismatch")
return actual
def component_compose_root(component): def component_compose_root(component):
return COMPONENTS[component].get("compose_root", component_root(component)) return COMPONENTS[component].get("compose_root", component_root(component))
@ -9658,6 +10360,12 @@ def component_services(component, entries=None):
# installed database. # installed database.
return ("device-postgres",) return ("device-postgres",)
if is_device_plane_foundation_recovery_slice(component, entries):
# The failed apply already built and started the exact reviewed images.
# Recovery publishes their matching source and accepts the current
# runtime without build/recreate/restart.
return ()
if is_engine_l2_closed_loop_slice(component, entries): if is_engine_l2_closed_loop_slice(component, entries):
# The failed 030 apply published both the backend source and the built # The failed 030 apply published both the backend source and the built
# UI before Compose rejected the descriptor/source mismatch. The exact # UI before Compose rejected the descriptor/source mismatch. The exact
@ -9992,6 +10700,9 @@ def component_builds(component, entries=None):
if is_device_plane_postgres_bootstrap_slice(component, entries): if is_device_plane_postgres_bootstrap_slice(component, entries):
return () return ()
if is_device_plane_foundation_recovery_slice(component, entries):
return ()
if component == "device-plane" and entries is not None: if component == "device-plane" and entries is not None:
selected_services = component_services(component, entries) selected_services = component_services(component, entries)
builds = [] builds = []
@ -11963,6 +12674,8 @@ def plan_artifact(artifact):
mcp_ontology_sdk_preflight = None mcp_ontology_sdk_preflight = None
mcp_autonomy_provider_v5_preflight = None mcp_autonomy_provider_v5_preflight = None
l2_closed_loop_preflight = None l2_closed_loop_preflight = None
device_plane_foundation_recovery_preflight = None
device_plane_runtime_before = None
composite_provider_v4_preflight = None composite_provider_v4_preflight = None
provider_rotating_slot_preflight = None provider_rotating_slot_preflight = None
provider_authority_diagnostics_preflight = None provider_authority_diagnostics_preflight = None
@ -11982,9 +12695,11 @@ def plan_artifact(artifact):
mcp_gelios_units_items_preflight = None mcp_gelios_units_items_preflight = None
provider_catalog_preflight = None provider_catalog_preflight = None
device_plane_postgres_preflight = None device_plane_postgres_preflight = None
device_plane_foundation_recovery_preflight = None
with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp:
manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) manifest, entries, payload_dir = load_artifact(artifact, Path(tmp))
reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_engine_l2_failed_artifact(manifest, sha)
reject_terminal_device_plane_foundation_artifact(manifest, sha)
transition_descriptor = None transition_descriptor = None
transition_preflight = None transition_preflight = None
if is_engine_n8n_transition(manifest["component"], entries): if is_engine_n8n_transition(manifest["component"], entries):
@ -12125,6 +12840,15 @@ def plan_artifact(artifact):
device_plane_postgres_preflight = ( device_plane_postgres_preflight = (
preflight_device_plane_postgres_bootstrap() preflight_device_plane_postgres_bootstrap()
) )
if is_device_plane_foundation_recovery_slice(
manifest["component"],
entries,
):
device_plane_foundation_recovery_preflight = (
validate_device_plane_foundation_recovery_evidence(
payload_dir
)
)
component = manifest["component"] component = manifest["component"]
root = component_root(component) root = component_root(component)
@ -13508,6 +14232,31 @@ def plan_artifact(artifact):
print("device_gateway_public_ingress=disabled") print("device_gateway_public_ingress=disabled")
print("device_gateway_command_transport=disabled") print("device_gateway_command_transport=disabled")
print("gelios=untouched") print("gelios=untouched")
if device_plane_foundation_recovery_preflight is not None:
print(
"device_plane_transition="
f"{device_plane_foundation_recovery_preflight['mode']}"
)
print(
"failed_patch="
f"{DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID}"
)
print(
"failed_artifact_sha256="
f"{DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256}"
)
print(
"recovery_backup="
f"{device_plane_foundation_recovery_preflight['backup'].name}"
)
print("device_plane_build=none")
print("device_plane_runtime_mutation=none")
print(
"device_plane_runtime_services="
"preserved:device-control-core,device-gateway,device-postgres"
)
print("device_plane_source_action=publish-exact-failed-source")
print("device_plane_rollback=source-only-runtime-unchanged")
if device_plane_postgres_preflight is not None: if device_plane_postgres_preflight is not None:
print( print(
"device_postgres_bootstrap=" "device_postgres_bootstrap="
@ -13813,11 +14562,18 @@ def rollback_device_plane_apply(
"Device Plane runtime rollback", "Device Plane runtime rollback",
) )
baseline_entries = [rel for rel in entries if rel in existing_set] baseline_entries = [rel for rel in entries if rel in existing_set]
compose_was_installed = "docker-compose.device-plane.yml" in existing_set runtime_inventory = read_strict_json(
baseline_services = ( backup_dir / "runtime-before.json",
component_services("device-plane", baseline_entries) "Device Plane pre-apply runtime inventory",
if compose_was_installed max_bytes=64 * 1024,
else () )
preapply_services = set(
device_plane_inventory_service_names(runtime_inventory)
)
baseline_services = tuple(
service
for service in applied_services
if service in preapply_services
) )
candidate_only_services = tuple( candidate_only_services = tuple(
service service
@ -14555,6 +15311,8 @@ def run_component_runtime(component, entries, services):
return return
if component_artifact_only(component): if component_artifact_only(component):
return return
if is_device_plane_foundation_recovery_slice(component, entries):
return
run_build(component, entries) run_build(component, entries)
prepare_component_runtime(component, entries) prepare_component_runtime(component, entries)
if is_engine_node_intelligence_transition(component, entries): if is_engine_node_intelligence_transition(component, entries):
@ -14663,6 +15421,29 @@ def module_foundry_healthcheck():
def component_healthchecks(component, entries=None, services=None): def component_healthchecks(component, entries=None, services=None):
if is_device_plane_postgres_bootstrap_slice(component, entries): if is_device_plane_postgres_bootstrap_slice(component, entries):
return () return ()
if is_device_plane_foundation_recovery_slice(component, entries):
return (
{
"url": "http://127.0.0.1:18120/healthz",
"expected_json": {
"ok": True,
"service": "nodedc-device-control-core",
"database": "ready",
"discoveryIngest": "disabled",
"commandTransport": "disabled",
},
},
{
"url": "http://127.0.0.1:18121/healthz",
"expected_json": {
"ok": True,
"service": "nodedc-device-gateway",
"tcpListener": "disabled",
"publicIngress": "disabled",
"commandTransport": "disabled",
},
},
)
if is_engine_n8n_transition(component, entries): if is_engine_n8n_transition(component, entries):
# The transition does not restart the Engine UI/backend generation. # The transition does not restart the Engine UI/backend generation.
# Its own acceptance below verifies n8n readiness, image, mount, logs, # Its own acceptance below verifies n8n readiness, image, mount, logs,
@ -15101,6 +15882,20 @@ def run_healthchecks(component, entries=None, services=None):
die("Device Plane PostgreSQL bootstrap service set mismatch") die("Device Plane PostgreSQL bootstrap service set mismatch")
healthcheck_compose_service("device-plane", "device-postgres") healthcheck_compose_service("device-plane", "device-postgres")
return return
if is_device_plane_foundation_recovery_slice(component, entries):
if tuple(services or ()) != ():
die("Device Plane foundation recovery service set mismatch")
for service in (
"device-control-core",
"device-gateway",
"device-postgres",
):
healthcheck_compose_service("device-plane", service)
for check in component_healthchecks(component, entries, services):
healthcheck_url(check)
validate_device_plane_foundation_installed_source()
validate_device_plane_foundation_runtime()
return
if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries):
return return
if is_engine_l2_closed_loop_slice(component, entries): if is_engine_l2_closed_loop_slice(component, entries):
@ -15881,6 +16676,10 @@ def apply_artifact(artifact):
work = Path(tmp) work = Path(tmp)
manifest, entries, payload_dir = load_artifact(artifact, work) manifest, entries, payload_dir = load_artifact(artifact, work)
reject_terminal_engine_l2_failed_artifact(manifest, sha) reject_terminal_engine_l2_failed_artifact(manifest, sha)
reject_terminal_device_plane_foundation_artifact(
manifest,
sha,
)
patch_id = manifest["id"] patch_id = manifest["id"]
component = manifest["component"] component = manifest["component"]
root = component_root(component) root = component_root(component)
@ -15898,6 +16697,15 @@ def apply_artifact(artifact):
entries, entries,
): ):
preflight_device_plane_postgres_bootstrap() preflight_device_plane_postgres_bootstrap()
if is_device_plane_foundation_recovery_slice(
component,
entries,
):
device_plane_foundation_recovery_preflight = (
validate_device_plane_foundation_recovery_evidence(
payload_dir
)
)
if not root.is_dir(): if not root.is_dir():
if bootstrap_root: if bootstrap_root:
root.mkdir(parents=True, exist_ok=True) root.mkdir(parents=True, exist_ok=True)
@ -16176,6 +16984,12 @@ def apply_artifact(artifact):
die(f"compose env file not found: {compose_env_file}") die(f"compose env file not found: {compose_env_file}")
if not artifact_only and not DOCKER.is_file(): if not artifact_only and not DOCKER.is_file():
die(f"docker not found: {DOCKER}") die(f"docker not found: {DOCKER}")
if component == "device-plane":
device_plane_runtime_before = (
device_plane_runtime_inventory(
DEVICE_PLANE_RUNTIME_SERVICES
)
)
if is_engine_n8n_transition(component, entries): if is_engine_n8n_transition(component, entries):
transition_descriptor = read_engine_n8n_transition_descriptor( transition_descriptor = read_engine_n8n_transition_descriptor(
@ -16202,6 +17016,23 @@ def apply_artifact(artifact):
include_nginx_html = component == "engine" and component_publish_dist(component, entries) include_nginx_html = component == "engine" and component_publish_dist(component, entries)
create_backup(root, backup_dir, entries, include_nginx_html) create_backup(root, backup_dir, entries, include_nginx_html)
if component == "device-plane":
if device_plane_runtime_before is None:
die("Device Plane pre-apply runtime inventory is missing")
runtime_inventory_path = (
backup_dir / "runtime-before.json"
)
runtime_inventory_path.write_text(
json.dumps(
device_plane_runtime_before,
ensure_ascii=False,
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
runtime_inventory_path.chmod(0o600)
if component == "platform" and touches_external_data_plane_files(entries): if component == "platform" and touches_external_data_plane_files(entries):
validate_backup_partition( validate_backup_partition(
entries, entries,

View File

@ -0,0 +1,375 @@
#!/usr/bin/env python3
import hashlib
import importlib.machinery
import importlib.util
import json
import os
import shutil
import subprocess
import tarfile
import tempfile
import unittest
from pathlib import Path
from unittest import mock
SCRIPT_DIR = Path(__file__).resolve().parent
RECOVERY_BUILDER = (
SCRIPT_DIR / "build-device-plane-foundation-recovery-artifact.mjs"
)
FOUNDATION_BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs"
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
RECOVERY_ENTRIES = [
".dockerignore",
"package.json",
"package-lock.json",
"docker-compose.device-plane.yml",
"packages/device-protocol-contract",
"packages/arusnavi-b2-adapter",
"services/device-control-core",
"services/device-gateway",
"deployment/device-plane-foundation-recovery-v1.json",
]
def load_runner():
loader = importlib.machinery.SourceFileLoader(
"nodedc_device_plane_recovery_runner_under_test",
str(RUNNER_PATH),
)
spec = importlib.util.spec_from_loader(loader.name, loader)
module = importlib.util.module_from_spec(spec)
loader.exec_module(module)
return module
RUNNER = load_runner()
class DevicePlaneFoundationRecoveryArtifactTest(unittest.TestCase):
def build(self, builder, artifact_dir, patch_id):
environment = os.environ.copy()
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
result = subprocess.run(
["node", str(builder), patch_id],
check=True,
capture_output=True,
text=True,
env=environment,
)
return json.loads(result.stdout)
def test_recovery_artifact_is_source_only_exact_and_deterministic(self):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-recovery-artifact-",
) as directory:
artifact_dir = Path(directory)
first = self.build(
RECOVERY_BUILDER,
artifact_dir,
"device-plane-foundation-recovery-unit-002",
)
artifact = Path(first["artifact"])
first_bytes = artifact.read_bytes()
second = self.build(
RECOVERY_BUILDER,
artifact_dir,
"device-plane-foundation-recovery-unit-002",
)
second_bytes = Path(second["artifact"]).read_bytes()
self.assertEqual(first["component"], "device-plane")
self.assertEqual(first["entries"], RECOVERY_ENTRIES)
self.assertEqual(first["build"], [])
self.assertEqual(first["services"], [])
self.assertEqual(
first["transition"],
"failed-foundation-live-runtime-adoption",
)
self.assertEqual(first_bytes, second_bytes)
self.assertEqual(
first["sha256"],
hashlib.sha256(first_bytes).hexdigest(),
)
with tarfile.open(artifact, "r:gz") as archive:
names = {member.name for member in archive.getmembers()}
files = (
archive.extractfile("files.txt")
.read()
.decode("utf-8")
.splitlines()
)
descriptor = json.loads(
archive.extractfile(
"payload/deployment/"
"device-plane-foundation-recovery-v1.json"
)
.read()
.decode("utf-8")
)
self.assertEqual(files, RECOVERY_ENTRIES)
self.assertIn(
"payload/services/device-control-core/src/server.mjs",
names,
)
self.assertIn(
"payload/services/device-gateway/src/server.mjs",
names,
)
self.assertFalse(any(
"/test/" in name
or "/node_modules/" in name
or Path(name).name.startswith(".env")
or name.startswith("payload/runtime/")
or name.startswith("payload/secrets/")
for name in names
))
self.assertEqual(
descriptor,
RUNNER.expected_device_plane_foundation_recovery_descriptor(),
)
self.assertEqual(
RUNNER.component_services(
"device-plane",
tuple(RECOVERY_ENTRIES),
),
(),
)
self.assertEqual(
RUNNER.component_builds(
"device-plane",
tuple(RECOVERY_ENTRIES),
),
(),
)
def test_preflight_requires_exact_failed_evidence_partial_source_and_runtime(
self,
):
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-recovery-preflight-",
) as directory:
workspace = Path(directory)
artifacts = workspace / "artifacts"
failed_root = workspace / "failed"
backups_root = workspace / "backups"
state_root = workspace / "state"
temp_root = workspace / "tmp"
live_root = workspace / "live"
for path in (
artifacts,
failed_root,
backups_root,
state_root,
temp_root,
live_root / "deployment",
):
path.mkdir(parents=True, exist_ok=True)
failed_build = self.build(
FOUNDATION_BUILDER,
artifacts,
RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID,
)
self.assertEqual(
failed_build["sha256"],
RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256,
)
failed_artifact = (
failed_root / RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT
)
shutil.copy2(failed_build["artifact"], failed_artifact)
recovery_build = self.build(
RECOVERY_BUILDER,
artifacts,
"device-plane-foundation-recovery-unit-002",
)
recovery_extract = workspace / "recovery-extract"
recovery_extract.mkdir()
_manifest, entries, payload = RUNNER.load_artifact(
Path(recovery_build["artifact"]),
recovery_extract,
)
self.assertEqual(entries, RECOVERY_ENTRIES)
backup = (
backups_root
/ RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID
)
backup.mkdir()
for name in (
"manifest.env",
"files.txt",
"existing-files.txt",
"missing-files.txt",
"source-before.tgz",
):
(backup / name).write_text(
f"fixture:{name}\n",
encoding="utf-8",
)
backup_hashes = {
name: hashlib.sha256((backup / name).read_bytes()).hexdigest()
for name in (
"manifest.env",
"files.txt",
"existing-files.txt",
"missing-files.txt",
"source-before.tgz",
)
}
(state_root / "failed.jsonl").write_text(
json.dumps({
"artifact":
RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT,
"backup_id":
RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID,
"component": "device-plane",
"id": RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID,
"message":
"healthcheck failed for "
"http://127.0.0.1:18120/healthz: "
"<urlopen error [Errno 111] Connection refused>",
"rollback_status": "failed:DeployError",
"sha256":
RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256,
"started_apply": True,
"status": "failed",
})
+ "\n",
encoding="utf-8",
)
source_root = SCRIPT_DIR.parent.parent / "device-plane"
shutil.copy2(
source_root / "docker-compose.device-plane.yml",
live_root / "docker-compose.device-plane.yml",
)
shutil.copy2(
source_root
/ "deployment/device-postgres-bootstrap-v1.json",
live_root
/ "deployment/device-postgres-bootstrap-v1.json",
)
runtime = {
service: {
"containerId": service,
"imageId":
RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[
service
],
"health": "healthy",
"restartCount": 0,
}
for service in (
"device-control-core",
"device-gateway",
"device-postgres",
)
}
with (
mock.patch.object(RUNNER, "BACKUPS_DIR", backups_root),
mock.patch.object(RUNNER, "FAILED_DIR", failed_root),
mock.patch.object(
RUNNER,
"FAILED_STATE_FILE",
state_root / "failed.jsonl",
),
mock.patch.object(RUNNER, "TMP_DIR", temp_root),
mock.patch.object(
RUNNER,
"component_root",
return_value=live_root,
),
mock.patch.dict(
RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256,
backup_hashes,
clear=True,
),
mock.patch.object(
RUNNER,
"validate_device_plane_foundation_runtime",
return_value=runtime,
),
):
result = (
RUNNER
.validate_device_plane_foundation_recovery_evidence(
payload
)
)
self.assertEqual(
result["mode"],
"failed-foundation-live-runtime-adoption",
)
self.assertEqual(result["runtime"], runtime)
def test_recovery_health_acceptance_never_mutates_runtime(self):
entries = tuple(RECOVERY_ENTRIES)
checks = ("core-health", "gateway-health")
with (
mock.patch.object(
RUNNER,
"healthcheck_compose_service",
) as compose_health,
mock.patch.object(
RUNNER,
"component_healthchecks",
return_value=checks,
),
mock.patch.object(RUNNER, "healthcheck_url") as url_health,
mock.patch.object(
RUNNER,
"validate_device_plane_foundation_installed_source",
) as source_acceptance,
mock.patch.object(
RUNNER,
"validate_device_plane_foundation_runtime",
) as runtime_acceptance,
mock.patch.object(RUNNER, "run_compose") as compose_mutation,
mock.patch.object(RUNNER, "run_build") as build_mutation,
):
RUNNER.run_healthchecks("device-plane", entries, ())
self.assertEqual(
[call.args for call in compose_health.call_args_list],
[
("device-plane", "device-control-core"),
("device-plane", "device-gateway"),
("device-plane", "device-postgres"),
],
)
self.assertEqual(
[call.args[0] for call in url_health.call_args_list],
list(checks),
)
source_acceptance.assert_called_once_with()
runtime_acceptance.assert_called_once_with()
compose_mutation.assert_not_called()
build_mutation.assert_not_called()
def test_recovery_runtime_phase_has_no_runtime_mutation(self):
entries = tuple(RECOVERY_ENTRIES)
with (
mock.patch.object(
RUNNER,
"prepare_component_runtime",
) as prepare,
mock.patch.object(RUNNER, "run_compose") as compose_mutation,
mock.patch.object(RUNNER, "run_build") as build_mutation,
):
RUNNER.run_component_runtime("device-plane", entries, ())
prepare.assert_not_called()
compose_mutation.assert_not_called()
build_mutation.assert_not_called()
if __name__ == "__main__":
unittest.main(verbosity=2)

View File

@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import importlib.machinery import importlib.machinery
import importlib.util import importlib.util
import json
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
@ -207,6 +208,15 @@ class DevicePlaneRegistryTest(unittest.TestCase):
"\n".join(entries) + "\n", "\n".join(entries) + "\n",
encoding="utf-8", encoding="utf-8",
) )
(backup / "runtime-before.json").write_text(
json.dumps({
"schemaVersion":
"nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": [],
}),
encoding="utf-8",
)
with ( with (
mock.patch.object( mock.patch.object(
RUNNER, RUNNER,
@ -234,6 +244,165 @@ class DevicePlaneRegistryTest(unittest.TestCase):
restore.assert_called_once() restore.assert_called_once()
self.assertEqual(result, "source-restored-runtime-unchanged:3") self.assertEqual(result, "source-restored-runtime-unchanged:3")
def test_db_bootstrap_compose_does_not_invent_stateless_baseline(self):
entries = (
"docker-compose.device-plane.yml",
"services/device-control-core",
"services/device-gateway",
)
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-compose-predecessor-",
) as directory:
root = Path(directory) / "live"
backup = Path(directory) / "backup"
root.mkdir()
backup.mkdir()
(backup / "existing-files.txt").write_text(
"docker-compose.device-plane.yml\n",
encoding="utf-8",
)
(backup / "missing-files.txt").write_text(
"services/device-control-core\n"
"services/device-gateway\n",
encoding="utf-8",
)
(backup / "runtime-before.json").write_text(
json.dumps({
"schemaVersion":
"nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": [{
"service": "device-postgres",
"containerId": "a" * 64,
"imageId": "sha256:" + "b" * 64,
"status": "running",
"running": True,
"health": "healthy",
"restartCount": 0,
}],
}),
encoding="utf-8",
)
with (
mock.patch.object(
RUNNER,
"stop_and_remove_compose_services",
) as stop,
mock.patch.object(
RUNNER,
"restore_platform_overlay",
return_value=3,
),
mock.patch.object(
RUNNER,
"run_component_runtime",
) as run_runtime,
):
result = RUNNER.rollback_device_plane_apply(
root,
backup,
entries,
"test-stamp",
True,
("device-control-core", "device-gateway"),
)
stop.assert_called_once_with(
"device-plane",
("device-control-core", "device-gateway"),
)
run_runtime.assert_not_called()
self.assertEqual(result, "source-restored-runtime-unchanged:3")
def test_installed_stateless_baseline_is_restored_from_runtime_inventory(
self,
):
entries = (
"docker-compose.device-plane.yml",
"services/device-control-core",
"services/device-gateway",
)
with tempfile.TemporaryDirectory(
prefix="nodedc-device-plane-installed-predecessor-",
) as directory:
root = Path(directory) / "live"
backup = Path(directory) / "backup"
root.mkdir()
backup.mkdir()
(backup / "existing-files.txt").write_text(
"\n".join(entries) + "\n",
encoding="utf-8",
)
(backup / "missing-files.txt").write_text(
"",
encoding="utf-8",
)
(backup / "runtime-before.json").write_text(
json.dumps({
"schemaVersion":
"nodedc.device-plane.runtime-inventory.v1",
"composeProject": "nodedc-device-plane",
"services": [
{
"service": "device-control-core",
"containerId": "a" * 64,
"imageId": "sha256:" + "b" * 64,
"status": "running",
"running": True,
"health": "healthy",
"restartCount": 0,
},
{
"service": "device-gateway",
"containerId": "c" * 64,
"imageId": "sha256:" + "d" * 64,
"status": "running",
"running": True,
"health": "healthy",
"restartCount": 0,
},
],
}),
encoding="utf-8",
)
with (
mock.patch.object(
RUNNER,
"stop_and_remove_compose_services",
) as stop,
mock.patch.object(
RUNNER,
"restore_platform_overlay",
return_value=3,
),
mock.patch.object(
RUNNER,
"run_component_runtime",
) as run_runtime,
mock.patch.object(RUNNER, "run_healthchecks") as healthchecks,
):
result = RUNNER.rollback_device_plane_apply(
root,
backup,
entries,
"test-stamp",
True,
("device-control-core", "device-gateway"),
)
stop.assert_not_called()
run_runtime.assert_called_once_with(
"device-plane",
list(entries),
("device-control-core", "device-gateway"),
)
healthchecks.assert_called_once_with(
"device-plane",
list(entries),
("device-control-core", "device-gateway"),
)
self.assertEqual(result, "source+runtime-restored:3")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main(verbosity=2) unittest.main(verbosity=2)