From aca47c6143b382a6f497f06386369d969ed04914 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 23:11:14 +0300 Subject: [PATCH] fix(deploy): reconcile device plane foundation --- .../device-plane-foundation-recovery-v1.json | 16 + device-plane/docs/IMPLEMENTATION_BASELINE.md | 30 +- device-plane/docs/TEST_MATRIX.md | 5 + infra/deploy-runner/README.md | 23 + ...ice-plane-foundation-recovery-artifact.mjs | 248 ++++++ infra/deploy-runner/nodedc-deploy | 841 +++++++++++++++++- ...vice_plane_foundation_recovery_artifact.py | 375 ++++++++ .../test_device_plane_registry.py | 169 ++++ 8 files changed, 1692 insertions(+), 15 deletions(-) create mode 100644 device-plane/deployment/device-plane-foundation-recovery-v1.json create mode 100755 infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs create mode 100644 infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py diff --git a/device-plane/deployment/device-plane-foundation-recovery-v1.json b/device-plane/deployment/device-plane-foundation-recovery-v1.json new file mode 100644 index 0000000..fbdb05b --- /dev/null +++ b/device-plane/deployment/device-plane-foundation-recovery-v1.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": "nodedc.device-plane.foundation-recovery.v1", + "mode": "failed-foundation-live-runtime-adoption", + "failedPatchId": "device-plane-foundation-20260725-001", + "failedArtifactSha256": "23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24", + "backupId": "device-plane-device-plane-foundation-20260725-001-20260725-223441", + "sourceAction": "publish-exact-failed-artifact-source", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres" + ], + "databaseVolume": "nodedc-device-plane-postgres-data", + "rollback": "source-only-runtime-unchanged" +} diff --git a/device-plane/docs/IMPLEMENTATION_BASELINE.md b/device-plane/docs/IMPLEMENTATION_BASELINE.md index 87d79b1..5f9cf07 100644 --- a/device-plane/docs/IMPLEMENTATION_BASELINE.md +++ b/device-plane/docs/IMPLEMENTATION_BASELINE.md @@ -1,7 +1,11 @@ # Device Plane Implementation Baseline -Status: local fail-closed foundation implemented; no runtime or production -mutation. +Status: PostgreSQL, Control Core and Gateway foundation are running healthy on +Synology with public ingress, discovery ingest and commands disabled. The first +application artifact is terminal-failed because its health gate timed out and +its automatic rollback could not model the DB-bootstrap predecessor. Recovery +is a separate source-only canonical transition; the failed artifact is never +retried. ## Product boundary @@ -72,6 +76,14 @@ The canonical runner selects only `device-control-core` and `device-gateway` with `--no-deps`. Its health acceptance is scoped to the selected services and requires the fail-closed fields to remain disabled. A failed first activation removes only candidate stateless services and never requests volume removal. +Rollback now records an explicit pre-apply service inventory in the backup; +the existence of the shared Compose file does not imply that Core or Gateway +existed before apply. + +The exact foundation recovery validates the failed archive, journal, backup, +partial live source and observed healthy image/container generations. It then +publishes the matching source and performs read-only runtime acceptance. It +does not build, restart, recreate or remove any service. ## Network boundary @@ -157,15 +169,13 @@ raw TCP remain forbidden until separate reviewed acceptance slices. ## Next source slice -1. Review, hash and promote the canonical runner candidate through the - standalone root administrative gate; do not stage an application artifact - before fresh `verify-install`. -2. Build, plan and accept the separate one-time PostgreSQL prerequisite - bootstrap artifact; ordinary application artifacts must continue to exclude - the database service. +1. Promote the reviewed recovery-capable runner through the standalone root + administrative gate and run a fresh `verify-install`. +2. Plan and apply the exact source-only foundation recovery artifact once, then + reconcile its journal/backup and repeat runtime acceptance. 3. Obtain or capture the exact official B2 INTERNAL framing and acknowledgement contract; keep ingress disabled until its fixtures pass. 4. Add an authenticated internal Core/Gateway discovery boundary and explicit platform-admin claim operation. -5. Build, audit and stage the first deterministic application artifact only - after the runner extension is installed and verified. +5. Open raw TCP ingress only in a later reviewed slice after quarantine, + framing, rate-limit and audit acceptance. diff --git a/device-plane/docs/TEST_MATRIX.md b/device-plane/docs/TEST_MATRIX.md index f7d0690..28bb7bc 100644 --- a/device-plane/docs/TEST_MATRIX.md +++ b/device-plane/docs/TEST_MATRIX.md @@ -20,6 +20,11 @@ | Database preservation | Ordinary application artifacts never select `device-postgres` | | Database bootstrap | Exact descriptor selects PostgreSQL only when both container and volume are absent | | Bootstrap rollback | Candidate container may be removed; named volume is never removed | +| Rollback predecessor | Backup records actual pre-apply services; Compose presence cannot invent Core/Gateway | +| Failed-001 evidence | Recovery requires exact failed archive, journal and backup digests | +| Partial source | Recovery accepts only DB-bootstrap source plus the observed healthy foundation runtime | +| Recovery mutation | Source is published without build, restart, recreate or service removal | +| Recovery rollback | Failed acceptance restores source only and leaves runtime unchanged | | Artifact policy | `.env`, secrets, runtime state, tests, logs and `node_modules` are excluded | | Artifact reproducibility | Repeated builds for the same patch id are byte-identical | | Runner compatibility | Existing canonical Platform registry tests remain green | diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 9155f8d..230a07e 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -476,6 +476,29 @@ node infra/deploy-runner/build-device-plane-artifact.mjs \ Any failed first activation removes only candidate Core/Gateway containers, never volumes, restores the source overlay and retains PostgreSQL state. +The rollback baseline is taken from an explicit pre-apply Docker service +inventory stored as `runtime-before.json` in the backup. The presence of the +shared Compose file never implies that Core or Gateway existed before apply. + +The exact `device-plane-foundation-20260725-001` failed activation is recovered +only through the registered +`deployment/device-plane-foundation-recovery-v1.json` transition. Its artifact +must reproduce the failed foundation source byte-for-byte. The runner validates +the exact failed archive, journal, backup partition, partial live source and +healthy observed Core/Gateway/PostgreSQL generations. Recovery publishes source +and performs read-only runtime acceptance; it does not build, recreate, restart +or remove containers. A failed recovery restores source only and leaves runtime +unchanged. + +Build and test that incident-specific deterministic recovery artifact: + +```bash +python3 -m unittest -v \ + infra.deploy-runner.test_device_plane_foundation_recovery_artifact + +node infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs \ + device-plane-foundation-recovery-20260725-002 +``` After the runner is promoted and freshly verified, bootstrap the durable prerequisite with a separate artifact before planning the application: diff --git a/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs b/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs new file mode 100755 index 0000000..f3beea3 --- /dev/null +++ b/infra/deploy-runner/build-device-plane-foundation-recovery-artifact.mjs @@ -0,0 +1,248 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + cp, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(scriptDir, "../.."); +const sourceRoot = resolve(platformRoot, "device-plane"); +const artifactDir = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR + || resolve(scriptDir, "../deploy-artifacts"), +); +const [ + patchId = "device-plane-foundation-recovery-20260725-002", + ...extra +] = process.argv.slice(2); + +if (extra.length || !/^[A-Za-z0-9._-]{1,96}$/.test(patchId)) { + throw new Error( + "usage: build-device-plane-foundation-recovery-artifact.mjs [patch-id]", + ); +} + +const files = [ + ".dockerignore", + "package.json", + "package-lock.json", + "docker-compose.device-plane.yml", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway", + "deployment/device-plane-foundation-recovery-v1.json", +]; +const ignoredBasenames = new Set([ + ".DS_Store", + ".git", + "node_modules", +]); +const ignoredDirectoryNames = new Set(["test"]); +const stage = await mkdtemp( + join(tmpdir(), "nodedc-device-plane-foundation-recovery-"), +); +const payload = join(stage, "payload"); +const target = join( + artifactDir, + `nodedc-device-plane-${patchId}.tgz`, +); + +await assertRecoveryBoundary(); + +try { + await mkdir(payload, { recursive: true }); + for (const sourceRelative of files) { + await copySafe( + resolve(sourceRoot, sourceRelative), + join(payload, sourceRelative), + ); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=device-plane\ntype=app-overlay\n`, + "utf8", + ); + await writeFile( + join(stage, "files.txt"), + `${files.join("\n")}\n`, + "utf8", + ); + await mkdir(artifactDir, { recursive: true }); + + const tar = spawnSync( + "python3", + ["-c", canonicalTarScript(), target, stage], + { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + }, + ); + if (tar.status !== 0) { + throw new Error(`tar_failed:${tar.stderr || tar.stdout}`); + } + + const digest = createHash("sha256") + .update(await readFile(target)) + .digest("hex"); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact: target, + sha256: digest, + component: "device-plane", + transition: "failed-foundation-live-runtime-adoption", + entries: files, + build: [], + services: [], + preservedRuntime: [ + "device-control-core", + "device-gateway", + "device-postgres", + "nodedc-device-plane-postgres-data", + ], + sourceAction: "publish-exact-failed-artifact-source", + runtimeAction: "read-only-acceptance", + excluded: [ + ".env*", + "node_modules", + "**/test", + "docs", + "runtime", + "secrets", + ], + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertRecoveryBoundary() { + const compose = await readFile( + resolve(sourceRoot, "docker-compose.device-plane.yml"), + "utf8", + ); + for (const fragment of [ + 'DEVICE_DISCOVERY_INGEST_ENABLED: "false"', + 'DEVICE_GATEWAY_LISTEN_ENABLED: "false"', + '"127.0.0.1:18120:18120"', + '"127.0.0.1:18121:18121"', + "name: nodedc-device-plane-postgres-data", + ]) { + if (!compose.includes(fragment)) { + throw new Error( + `device_plane_recovery_compose_boundary_missing:${fragment}`, + ); + } + } + for (const forbidden of [ + "9921:9921", + "0.0.0.0:9921", + 'DEVICE_DISCOVERY_INGEST_ENABLED: "true"', + 'DEVICE_GATEWAY_LISTEN_ENABLED: "true"', + "POSTGRES_PASSWORD:", + ]) { + if (compose.includes(forbidden)) { + throw new Error( + `device_plane_recovery_compose_boundary_violation:${forbidden}`, + ); + } + } + + const descriptor = JSON.parse(await readFile( + resolve( + sourceRoot, + "deployment/device-plane-foundation-recovery-v1.json", + ), + "utf8", + )); + const expected = { + schemaVersion: "nodedc.device-plane.foundation-recovery.v1", + mode: "failed-foundation-live-runtime-adoption", + failedPatchId: "device-plane-foundation-20260725-001", + failedArtifactSha256: + "23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24", + backupId: + "device-plane-device-plane-foundation-20260725-001-20260725-223441", + sourceAction: "publish-exact-failed-artifact-source", + runtimeAction: "read-only-acceptance", + preservedServices: [ + "device-control-core", + "device-gateway", + "device-postgres", + ], + databaseVolume: "nodedc-device-plane-postgres-data", + rollback: "source-only-runtime-unchanged", + }; + if (JSON.stringify(descriptor) !== JSON.stringify(expected)) { + throw new Error("device_plane_recovery_descriptor_mismatch"); + } +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +async function copySafe(source, destination) { + const sourceStat = await lstat(source); + if (sourceStat.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(sourceRoot, source)}`, + ); + } + if (sourceStat.isFile()) { + await mkdir(dirname(destination), { recursive: true }); + await cp(source, destination, { + force: true, + verbatimSymlinks: true, + }); + return; + } + if (!sourceStat.isDirectory()) { + throw new Error(`source_type_rejected:${source}`); + } + + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source, { withFileTypes: true })) { + if ( + ignoredBasenames.has(entry.name) + || entry.name.startsWith(".env") + || ( + entry.isDirectory() + && ignoredDirectoryNames.has(entry.name) + ) + ) { + continue; + } + const childSource = join(source, entry.name); + const childDestination = join(destination, entry.name); + if (entry.isSymbolicLink()) { + throw new Error( + `source_symlink_rejected:${relative(sourceRoot, childSource)}`, + ); + } + await copySafe(childSource, childDestination); + } +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index b83f288..4f48be7 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -52,6 +52,75 @@ DEVICE_PLANE_POSTGRES_BOOTSTRAP_ENTRIES = ( DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, ) DEVICE_PLANE_POSTGRES_VOLUME = "nodedc-device-plane-postgres-data" +DEVICE_PLANE_FOUNDATION_ENTRIES = ( + ".dockerignore", + "package.json", + "package-lock.json", + "docker-compose.device-plane.yml", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway", +) +DEVICE_PLANE_FOUNDATION_RECOVERY_REL = ( + "deployment/device-plane-foundation-recovery-v1.json" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES = ( + *DEVICE_PLANE_FOUNDATION_ENTRIES, + DEVICE_PLANE_FOUNDATION_RECOVERY_REL, +) +DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID = ( + "device-plane-foundation-20260725-001" +) +DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT = ( + "nodedc-device-plane-device-plane-foundation-20260725-001.tgz." + "20260725-223441" +) +DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 = ( + "23d428de547854ad8b1a026671e2f850386ab0be98bde80f016f1e9db631ee24" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID = ( + "device-plane-device-plane-foundation-20260725-001-20260725-223441" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256 = { + "manifest.env": ( + "21ff4250147f99fd1461191e1f05aee99e92da9cdedce4513749046c26327c77" + ), + "files.txt": ( + "3b7dda5cc42ce460e4a74f6df986d4a4fa8aa28511e9d9b123deb445a0382083" + ), + "existing-files.txt": ( + "f2e2c6e7501d3a383cd7e909a2e4e4cfbb3b0cda132f508f54e478bfb455585b" + ), + "missing-files.txt": ( + "308d86656e0a0863d5fa16833ec67d1698c0cef7e009a7bd987e62a9e6469c63" + ), + "source-before.tgz": ( + "df451cdf7a5798c332d03f15473358b1e7fdb7b96212037aaed2d8d00dfecf68" + ), +} +DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256 = ( + "e3afbb1ab252914e21dfb0acc1a2016fd8af471aa13bb14fb578f106f2cdae36" +) +DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256 = ( + "9475f12d151aee386cba74a4c0f9e7485416d5616d3ad71dfd8963fe114abf61" +) +DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS = { + "device-control-core": ( + "sha256:78a9fdfbe46f6e9531536abb997ae105fc7ea5c9a8176cb88ca8c0644900c254" + ), + "device-gateway": ( + "sha256:9be91a3496d1ac627b6b621ade0c2ae77afa10e9ddb9b2071823769bc5a5a472" + ), + "device-postgres": ( + "sha256:0001755f47e38bfd163788ccc543387b7f3da35a01b38a7c2ca1cd0277f06b5d" + ), +} +DEVICE_PLANE_RUNTIME_SERVICES = ( + "device-control-core", + "device-gateway", + "device-postgres", +) EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-provisioner" EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "token" ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE = EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_DIR / "engine-credential-provisioner-ed25519.pem" @@ -2736,6 +2805,7 @@ def allowed_payload_path(component, rel): "package-lock.json", "docker-compose.device-plane.yml", DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, + DEVICE_PLANE_FOUNDATION_RECOVERY_REL, "packages/device-protocol-contract", "packages/arusnavi-b2-adapter", "services/device-control-core", @@ -7760,6 +7830,11 @@ def load_artifact(artifact, work_dir): entries, ): validate_device_plane_postgres_bootstrap_payload(payload_dir) + if is_device_plane_foundation_recovery_slice( + manifest["component"], + entries, + ): + validate_device_plane_foundation_recovery_payload(payload_dir) if manifest["component"] == "n8n-private-extension": validate_n8n_private_extension_release(payload_dir, entries) if manifest["component"] == "engine": @@ -8034,6 +8109,17 @@ def reject_terminal_engine_l2_failed_artifact(manifest, sha256): ) +def reject_terminal_device_plane_foundation_artifact(manifest, sha256): + if ( + manifest.get("id") == DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID + or sha256 == DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 + ): + die( + "Device Plane foundation 001 is terminal failed; " + "use the exact registered foundation recovery successor" + ) + + class DeployLock: def __enter__(self): try: @@ -8059,6 +8145,325 @@ def is_device_plane_postgres_bootstrap_slice(component, entries): ) +def is_device_plane_foundation_recovery_slice(component, entries): + return ( + component == "device-plane" + and entries is not None + and tuple(entries) == DEVICE_PLANE_FOUNDATION_RECOVERY_ENTRIES + ) + + +def expected_device_plane_foundation_recovery_descriptor(): + return { + "schemaVersion": "nodedc.device-plane.foundation-recovery.v1", + "mode": "failed-foundation-live-runtime-adoption", + "failedPatchId": DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID, + "failedArtifactSha256": ( + DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 + ), + "backupId": DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID, + "sourceAction": "publish-exact-failed-artifact-source", + "runtimeAction": "read-only-acceptance", + "preservedServices": [ + "device-control-core", + "device-gateway", + "device-postgres", + ], + "databaseVolume": DEVICE_PLANE_POSTGRES_VOLUME, + "rollback": "source-only-runtime-unchanged", + } + + +def validate_device_plane_foundation_recovery_payload(payload_dir): + descriptor = read_strict_json( + payload_dir / DEVICE_PLANE_FOUNDATION_RECOVERY_REL, + "Device Plane foundation recovery descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_foundation_recovery_descriptor(): + die("Device Plane foundation recovery descriptor mismatch") + return descriptor + + +def device_plane_service_container_ids(service): + if service not in DEVICE_PLANE_RUNTIME_SERVICES: + die(f"Device Plane runtime service is not registered: {service}") + result = subprocess.run( + [ + str(DOCKER), + "container", + "ls", + "-a", + "--filter", + "label=com.docker.compose.project=nodedc-device-plane", + "--filter", + f"label=com.docker.compose.service={service}", + "--format", + "{{.ID}}", + ], + check=False, + capture_output=True, + text=True, + ) + container_ids = [ + line.strip() + for line in result.stdout.splitlines() + if line.strip() + ] + if ( + result.returncode != 0 + or any( + not re.fullmatch(r"[a-f0-9]{12,64}", container_id) + for container_id in container_ids + ) + or len(container_ids) > 1 + ): + die(f"Device Plane runtime inventory failed: {service}") + return tuple(container_ids) + + +def inspect_device_plane_container(container_id): + containers = docker_json( + ["container", "inspect", container_id], + "Device Plane container inspect", + ) + if ( + not isinstance(containers, list) + or len(containers) != 1 + or not isinstance(containers[0], dict) + ): + die("Device Plane container inspect shape mismatch") + return containers[0] + + +def device_plane_runtime_inventory(services): + inventory = [] + for service in services: + container_ids = device_plane_service_container_ids(service) + if not container_ids: + continue + container = inspect_device_plane_container(container_ids[0]) + state = container.get("State") or {} + health = (state.get("Health") or {}).get("Status") + inventory.append({ + "service": service, + "containerId": container.get("Id"), + "imageId": container.get("Image"), + "status": state.get("Status"), + "running": state.get("Running") is True, + "health": health, + "restartCount": int(container.get("RestartCount") or 0), + }) + return { + "schemaVersion": "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": inventory, + } + + +def device_plane_inventory_service_names(inventory): + if ( + not isinstance(inventory, dict) + or inventory.get("schemaVersion") + != "nodedc.device-plane.runtime-inventory.v1" + or inventory.get("composeProject") != "nodedc-device-plane" + or not isinstance(inventory.get("services"), list) + ): + die("Device Plane runtime inventory shape mismatch") + names = [] + for item in inventory["services"]: + if ( + not isinstance(item, dict) + or set(item) != { + "service", + "containerId", + "imageId", + "status", + "running", + "health", + "restartCount", + } + or item.get("service") not in DEVICE_PLANE_RUNTIME_SERVICES + or not isinstance(item.get("containerId"), str) + or not re.fullmatch(r"[a-f0-9]{64}", item["containerId"]) + or not isinstance(item.get("imageId"), str) + or not re.fullmatch(r"sha256:[a-f0-9]{64}", item["imageId"]) + or not isinstance(item.get("status"), str) + or not isinstance(item.get("running"), bool) + or item.get("health") not in ( + None, + "starting", + "healthy", + "unhealthy", + ) + or not isinstance(item.get("restartCount"), int) + or item["restartCount"] < 0 + ): + die("Device Plane runtime inventory service mismatch") + service = item["service"] + if service in names: + die("Device Plane runtime inventory contains duplicates") + names.append(service) + return tuple(names) + + +def validate_device_plane_foundation_recovery_evidence(payload_dir): + backup_dir = BACKUPS_DIR / DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Device Plane foundation recovery backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR( + backup_stat.st_mode + ): + die("Device Plane foundation recovery backup is unsafe") + backup_names = {child.name for child in backup_dir.iterdir()} + if backup_names != set(DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256): + die("Device Plane foundation recovery backup file set mismatch") + for name, expected in ( + DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256.items() + ): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected + ): + die( + "Device Plane foundation recovery backup drift detected: " + f"{name}" + ) + + failed_artifact = FAILED_DIR / DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Device Plane foundation failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 + ): + die("Device Plane foundation failed artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text( + encoding="utf-8" + ).splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Device Plane foundation failed journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG( + state_stat.st_mode + ): + die("Device Plane foundation failed journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Device Plane foundation failed journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") == DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Device Plane foundation failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT + or record.get("backup_id") + != DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID + or record.get("component") != "device-plane" + or record.get("sha256") + != DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "failed:DeployError" + or record.get("status") != "failed" + or record.get("message") + != ( + "healthcheck failed for http://127.0.0.1:18120/healthz: " + "" + ) + ): + die("Device Plane foundation failed journal evidence mismatch") + + with tempfile.TemporaryDirectory( + prefix="device-plane-failed-foundation-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID + or failed_manifest.get("component") != "device-plane" + or failed_manifest.get("type") != "app-overlay" + or tuple(failed_entries) != DEVICE_PLANE_FOUNDATION_ENTRIES + ): + die("Device Plane foundation failed artifact contract mismatch") + failed_source = collect_exact_files( + failed_payload, + DEVICE_PLANE_FOUNDATION_ENTRIES, + "Device Plane failed foundation source", + ) + candidate_source = collect_exact_files( + payload_dir, + DEVICE_PLANE_FOUNDATION_ENTRIES, + "Device Plane recovery candidate source", + ) + if candidate_source != failed_source: + die( + "Device Plane recovery candidate does not match failed " + "foundation source" + ) + + root = component_root("device-plane") + compose_path = root / "docker-compose.device-plane.yml" + if ( + compose_path.is_symlink() + or not compose_path.is_file() + or sha256_file(compose_path) + != DEVICE_PLANE_FOUNDATION_PREDECESSOR_COMPOSE_SHA256 + ): + die("Device Plane foundation partial Compose predecessor mismatch") + for rel in DEVICE_PLANE_FOUNDATION_ENTRIES: + if rel == "docker-compose.device-plane.yml": + continue + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Device Plane foundation partial predecessor retained source: " + f"{rel}" + ) + recovery_descriptor = root / DEVICE_PLANE_FOUNDATION_RECOVERY_REL + if recovery_descriptor.exists() or recovery_descriptor.is_symlink(): + die("Device Plane foundation recovery descriptor already exists") + bootstrap_descriptor = ( + root / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL + ) + if ( + bootstrap_descriptor.is_symlink() + or not bootstrap_descriptor.is_file() + or sha256_file(bootstrap_descriptor) + != DEVICE_PLANE_POSTGRES_BOOTSTRAP_DESCRIPTOR_SHA256 + ): + die("Device Plane PostgreSQL bootstrap descriptor drift detected") + + runtime = validate_device_plane_foundation_runtime() + return { + "mode": "failed-foundation-live-runtime-adoption", + "backup": backup_dir, + "failedArtifact": failed_artifact, + "runtime": runtime, + } + + def validate_device_plane_postgres_bootstrap_payload(payload_dir): descriptor = read_strict_json( payload_dir / DEVICE_PLANE_POSTGRES_BOOTSTRAP_REL, @@ -8132,6 +8537,303 @@ def device_plane_postgres_plan_selection(postgres_preflight): return "bootstrap-selected:create-if-absent" +def validate_device_plane_foundation_runtime(): + validate_device_plane_runtime_secret_metadata() + expected = { + "device-control-core": { + "image": DEVICE_PLANE_CONTROL_CORE_IMAGE, + "imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[ + "device-control-core" + ], + "user": "1000:1000", + "ports": { + "18120/tcp": [{ + "HostIp": "127.0.0.1", + "HostPort": "18120", + }], + }, + }, + "device-gateway": { + "image": DEVICE_PLANE_GATEWAY_IMAGE, + "imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[ + "device-gateway" + ], + "user": "1000:1000", + "ports": { + "18121/tcp": [{ + "HostIp": "127.0.0.1", + "HostPort": "18121", + }], + }, + }, + "device-postgres": { + "image": "postgres:16-alpine", + "imageId": DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[ + "device-postgres" + ], + "user": "", + "ports": {}, + }, + } + accepted = {} + for service, contract in expected.items(): + container_ids = device_plane_service_container_ids(service) + if len(container_ids) != 1: + die( + "Device Plane foundation runtime service count mismatch: " + f"{service}" + ) + container = inspect_device_plane_container(container_ids[0]) + state = container.get("State") or {} + config = container.get("Config") or {} + host_config = container.get("HostConfig") or {} + labels = config.get("Labels") or {} + networks = (container.get("NetworkSettings") or {}).get( + "Networks" + ) or {} + if ( + state.get("Status") != "running" + or state.get("Running") is not True + or state.get("Restarting") is True + or state.get("ExitCode") != 0 + or state.get("Error") not in ("", None) + or (state.get("Health") or {}).get("Status") != "healthy" + or int(container.get("RestartCount") or 0) != 0 + ): + die( + "Device Plane foundation runtime state mismatch: " + f"{service}" + ) + if ( + container.get("Image") != contract["imageId"] + or config.get("Image") != contract["image"] + or config.get("User", "") != contract["user"] + or (host_config.get("PortBindings") or {}) != contract["ports"] + or (host_config.get("RestartPolicy") or {}).get("Name") + != "unless-stopped" + or labels.get("com.docker.compose.project") + != "nodedc-device-plane" + or labels.get("com.docker.compose.service") != service + or set(networks) != {"nodedc-device-plane-private"} + ): + die( + "Device Plane foundation runtime boundary mismatch: " + f"{service}" + ) + accepted[service] = { + "containerId": container.get("Id"), + "imageId": container.get("Image"), + "health": "healthy", + "restartCount": 0, + } + + core = inspect_device_plane_container( + device_plane_service_container_ids("device-control-core")[0] + ) + gateway = inspect_device_plane_container( + device_plane_service_container_ids("device-gateway")[0] + ) + postgres = inspect_device_plane_container( + device_plane_service_container_ids("device-postgres")[0] + ) + for service, container in ( + ("device-control-core", core), + ("device-gateway", gateway), + ): + host_config = container.get("HostConfig") or {} + if ( + host_config.get("ReadonlyRootfs") is not True + or set(host_config.get("CapDrop") or ()) != {"ALL"} + or "no-new-privileges:true" + not in set(host_config.get("SecurityOpt") or ()) + ): + die( + "Device Plane stateless hardening mismatch: " + f"{service}" + ) + + core_environment = container_environment( + core, + "Device Plane Control Core", + ) + core_required_environment = { + "HOST": "0.0.0.0", + "PORT": "18120", + "DEVICE_DATABASE_HOST": "device-postgres", + "DEVICE_DATABASE_PORT": "5432", + "DEVICE_DATABASE_NAME": "device_plane", + "DEVICE_DATABASE_USER": "device_plane", + "DEVICE_DATABASE_PASSWORD_FILE": ( + "/run/nodedc-secrets/postgres-password" + ), + "DEVICE_DATABASE_POOL_SIZE": "10", + "DEVICE_DISCOVERY_INGEST_ENABLED": "false", + } + if any( + core_environment.get(key) != value + for key, value in core_required_environment.items() + ): + die("Device Plane Control Core environment mismatch") + for forbidden in ( + "DEVICE_DATABASE_URL", + "DEVICE_DATABASE_PASSWORD", + "DEVICE_GATEWAY_CORE_TOKEN", + "DEVICE_IDENTIFIER_PEPPER", + ): + if forbidden in core_environment: + die("Device Plane Control Core plaintext secret boundary mismatch") + + gateway_environment = container_environment( + gateway, + "Device Plane Gateway", + ) + gateway_required_environment = { + "DEVICE_GATEWAY_HEALTH_HOST": "0.0.0.0", + "DEVICE_GATEWAY_HEALTH_PORT": "18121", + "DEVICE_GATEWAY_LISTEN_ENABLED": "false", + "DEVICE_GATEWAY_TCP_HOST": "127.0.0.1", + "DEVICE_GATEWAY_TCP_PORT": "9921", + "DEVICE_GATEWAY_MAX_SESSIONS": "100", + "DEVICE_GATEWAY_SESSION_TIMEOUT_MS": "10000", + } + if any( + gateway_environment.get(key) != value + for key, value in gateway_required_environment.items() + ): + die("Device Plane Gateway environment mismatch") + if any( + key in gateway_environment + for key in ( + "DEVICE_GATEWAY_CORE_TOKEN", + "DEVICE_GATEWAY_COMMAND_TOKEN", + ) + ): + die("Device Plane Gateway plaintext secret boundary mismatch") + + core_mounts = core.get("Mounts") or [] + if len(core_mounts) != 1: + die("Device Plane Control Core mount count mismatch") + core_secret = core_mounts[0] + if ( + core_secret.get("Type") != "bind" + or core_secret.get("Source") + != str(DEVICE_PLANE_POSTGRES_PASSWORD_FILE) + or core_secret.get("Destination") + != "/run/nodedc-secrets/postgres-password" + or core_secret.get("RW") is not False + ): + die("Device Plane Control Core secret mount mismatch") + if gateway.get("Mounts") not in (None, []): + die("Device Plane Gateway unexpected mount") + + postgres_mounts = postgres.get("Mounts") or [] + postgres_secret = [ + mount + for mount in postgres_mounts + if mount.get("Destination") + == "/run/nodedc-secrets/postgres-password" + ] + postgres_volume = [ + mount + for mount in postgres_mounts + if mount.get("Destination") == "/var/lib/postgresql/data" + ] + if ( + len(postgres_mounts) != 2 + or len(postgres_secret) != 1 + or postgres_secret[0].get("Type") != "bind" + or postgres_secret[0].get("Source") + != str(DEVICE_PLANE_POSTGRES_PASSWORD_FILE) + or postgres_secret[0].get("RW") is not False + or len(postgres_volume) != 1 + or postgres_volume[0].get("Type") != "volume" + or postgres_volume[0].get("Name") + != DEVICE_PLANE_POSTGRES_VOLUME + or postgres_volume[0].get("RW") is not True + ): + die("Device Plane PostgreSQL preserved mount mismatch") + return accepted + + +def validate_device_plane_runtime_secret_metadata(): + try: + directory_stat = DEVICE_PLANE_SECRET_DIR.lstat() + except FileNotFoundError: + die("Device Plane runtime secret directory is missing") + if ( + stat.S_ISLNK(directory_stat.st_mode) + or not stat.S_ISDIR(directory_stat.st_mode) + or directory_stat.st_uid != 0 + or directory_stat.st_gid != MAP_GATEWAY_RUNTIME_GID + or stat.S_IMODE(directory_stat.st_mode) != 0o710 + ): + die("Device Plane runtime secret directory boundary mismatch") + for path, label in ( + (DEVICE_PLANE_POSTGRES_PASSWORD_FILE, "PostgreSQL"), + (DEVICE_PLANE_GATEWAY_CORE_TOKEN_FILE, "Gateway to Core"), + (DEVICE_PLANE_IDENTIFIER_PEPPER_FILE, "identifier pepper"), + ): + try: + path_stat = path.lstat() + except FileNotFoundError: + die(f"Device Plane {label} runtime secret is missing") + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or path_stat.st_uid != 0 + or path_stat.st_gid != MAP_GATEWAY_RUNTIME_GID + or stat.S_IMODE(path_stat.st_mode) != 0o640 + or path_stat.st_size > 512 + ): + die(f"Device Plane {label} runtime secret boundary mismatch") + try: + value = path.read_text(encoding="ascii").strip() + except (OSError, UnicodeDecodeError): + die(f"Device Plane {label} runtime secret is unreadable") + if not MAP_GATEWAY_SECRET_RE.fullmatch(value): + die(f"Device Plane {label} runtime secret format mismatch") + return "exact" + + +def validate_device_plane_foundation_installed_source(): + failed_artifact = FAILED_DIR / DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT + with tempfile.TemporaryDirectory( + prefix="device-plane-installed-foundation-", + dir=TMP_DIR, + ) as directory: + failed_manifest, failed_entries, failed_payload = load_artifact( + failed_artifact, + Path(directory), + ) + if ( + failed_manifest.get("id") + != DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID + or tuple(failed_entries) != DEVICE_PLANE_FOUNDATION_ENTRIES + ): + die("Device Plane failed foundation source contract mismatch") + expected = collect_exact_files( + failed_payload, + DEVICE_PLANE_FOUNDATION_ENTRIES, + "Device Plane failed foundation source", + ) + actual = collect_exact_files( + component_root("device-plane"), + DEVICE_PLANE_FOUNDATION_ENTRIES, + "Device Plane installed foundation source", + ) + if actual != expected: + die("Device Plane installed foundation source mismatch") + descriptor = read_strict_json( + component_root("device-plane") + / DEVICE_PLANE_FOUNDATION_RECOVERY_REL, + "installed Device Plane foundation recovery descriptor", + max_bytes=16 * 1024, + ) + if descriptor != expected_device_plane_foundation_recovery_descriptor(): + die("installed Device Plane foundation recovery descriptor mismatch") + return actual + + def component_compose_root(component): return COMPONENTS[component].get("compose_root", component_root(component)) @@ -9658,6 +10360,12 @@ def component_services(component, entries=None): # installed database. return ("device-postgres",) + if is_device_plane_foundation_recovery_slice(component, entries): + # The failed apply already built and started the exact reviewed images. + # Recovery publishes their matching source and accepts the current + # runtime without build/recreate/restart. + return () + if is_engine_l2_closed_loop_slice(component, entries): # The failed 030 apply published both the backend source and the built # UI before Compose rejected the descriptor/source mismatch. The exact @@ -9992,6 +10700,9 @@ def component_builds(component, entries=None): if is_device_plane_postgres_bootstrap_slice(component, entries): return () + if is_device_plane_foundation_recovery_slice(component, entries): + return () + if component == "device-plane" and entries is not None: selected_services = component_services(component, entries) builds = [] @@ -11963,6 +12674,8 @@ def plan_artifact(artifact): mcp_ontology_sdk_preflight = None mcp_autonomy_provider_v5_preflight = None l2_closed_loop_preflight = None + device_plane_foundation_recovery_preflight = None + device_plane_runtime_before = None composite_provider_v4_preflight = None provider_rotating_slot_preflight = None provider_authority_diagnostics_preflight = None @@ -11982,9 +12695,11 @@ def plan_artifact(artifact): mcp_gelios_units_items_preflight = None provider_catalog_preflight = None device_plane_postgres_preflight = None + device_plane_foundation_recovery_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) reject_terminal_engine_l2_failed_artifact(manifest, sha) + reject_terminal_device_plane_foundation_artifact(manifest, sha) transition_descriptor = None transition_preflight = None if is_engine_n8n_transition(manifest["component"], entries): @@ -12125,6 +12840,15 @@ def plan_artifact(artifact): device_plane_postgres_preflight = ( preflight_device_plane_postgres_bootstrap() ) + if is_device_plane_foundation_recovery_slice( + manifest["component"], + entries, + ): + device_plane_foundation_recovery_preflight = ( + validate_device_plane_foundation_recovery_evidence( + payload_dir + ) + ) component = manifest["component"] root = component_root(component) @@ -13508,6 +14232,31 @@ def plan_artifact(artifact): print("device_gateway_public_ingress=disabled") print("device_gateway_command_transport=disabled") print("gelios=untouched") + if device_plane_foundation_recovery_preflight is not None: + print( + "device_plane_transition=" + f"{device_plane_foundation_recovery_preflight['mode']}" + ) + print( + "failed_patch=" + f"{DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256}" + ) + print( + "recovery_backup=" + f"{device_plane_foundation_recovery_preflight['backup'].name}" + ) + print("device_plane_build=none") + print("device_plane_runtime_mutation=none") + print( + "device_plane_runtime_services=" + "preserved:device-control-core,device-gateway,device-postgres" + ) + print("device_plane_source_action=publish-exact-failed-source") + print("device_plane_rollback=source-only-runtime-unchanged") if device_plane_postgres_preflight is not None: print( "device_postgres_bootstrap=" @@ -13813,11 +14562,18 @@ def rollback_device_plane_apply( "Device Plane runtime rollback", ) baseline_entries = [rel for rel in entries if rel in existing_set] - compose_was_installed = "docker-compose.device-plane.yml" in existing_set - baseline_services = ( - component_services("device-plane", baseline_entries) - if compose_was_installed - else () + runtime_inventory = read_strict_json( + backup_dir / "runtime-before.json", + "Device Plane pre-apply runtime inventory", + max_bytes=64 * 1024, + ) + preapply_services = set( + device_plane_inventory_service_names(runtime_inventory) + ) + baseline_services = tuple( + service + for service in applied_services + if service in preapply_services ) candidate_only_services = tuple( service @@ -14555,6 +15311,8 @@ def run_component_runtime(component, entries, services): return if component_artifact_only(component): return + if is_device_plane_foundation_recovery_slice(component, entries): + return run_build(component, entries) prepare_component_runtime(component, entries) if is_engine_node_intelligence_transition(component, entries): @@ -14663,6 +15421,29 @@ def module_foundry_healthcheck(): def component_healthchecks(component, entries=None, services=None): if is_device_plane_postgres_bootstrap_slice(component, entries): return () + if is_device_plane_foundation_recovery_slice(component, entries): + return ( + { + "url": "http://127.0.0.1:18120/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-control-core", + "database": "ready", + "discoveryIngest": "disabled", + "commandTransport": "disabled", + }, + }, + { + "url": "http://127.0.0.1:18121/healthz", + "expected_json": { + "ok": True, + "service": "nodedc-device-gateway", + "tcpListener": "disabled", + "publicIngress": "disabled", + "commandTransport": "disabled", + }, + }, + ) if is_engine_n8n_transition(component, entries): # The transition does not restart the Engine UI/backend generation. # Its own acceptance below verifies n8n readiness, image, mount, logs, @@ -15101,6 +15882,20 @@ def run_healthchecks(component, entries=None, services=None): die("Device Plane PostgreSQL bootstrap service set mismatch") healthcheck_compose_service("device-plane", "device-postgres") return + if is_device_plane_foundation_recovery_slice(component, entries): + if tuple(services or ()) != (): + die("Device Plane foundation recovery service set mismatch") + for service in ( + "device-control-core", + "device-gateway", + "device-postgres", + ): + healthcheck_compose_service("device-plane", service) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + validate_device_plane_foundation_installed_source() + validate_device_plane_foundation_runtime() + return if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): return if is_engine_l2_closed_loop_slice(component, entries): @@ -15881,6 +16676,10 @@ def apply_artifact(artifact): work = Path(tmp) manifest, entries, payload_dir = load_artifact(artifact, work) reject_terminal_engine_l2_failed_artifact(manifest, sha) + reject_terminal_device_plane_foundation_artifact( + manifest, + sha, + ) patch_id = manifest["id"] component = manifest["component"] root = component_root(component) @@ -15898,6 +16697,15 @@ def apply_artifact(artifact): entries, ): preflight_device_plane_postgres_bootstrap() + if is_device_plane_foundation_recovery_slice( + component, + entries, + ): + device_plane_foundation_recovery_preflight = ( + validate_device_plane_foundation_recovery_evidence( + payload_dir + ) + ) if not root.is_dir(): if bootstrap_root: root.mkdir(parents=True, exist_ok=True) @@ -16176,6 +16984,12 @@ def apply_artifact(artifact): die(f"compose env file not found: {compose_env_file}") if not artifact_only and not DOCKER.is_file(): die(f"docker not found: {DOCKER}") + if component == "device-plane": + device_plane_runtime_before = ( + device_plane_runtime_inventory( + DEVICE_PLANE_RUNTIME_SERVICES + ) + ) if is_engine_n8n_transition(component, entries): transition_descriptor = read_engine_n8n_transition_descriptor( @@ -16202,6 +17016,23 @@ def apply_artifact(artifact): include_nginx_html = component == "engine" and component_publish_dist(component, entries) create_backup(root, backup_dir, entries, include_nginx_html) + if component == "device-plane": + if device_plane_runtime_before is None: + die("Device Plane pre-apply runtime inventory is missing") + runtime_inventory_path = ( + backup_dir / "runtime-before.json" + ) + runtime_inventory_path.write_text( + json.dumps( + device_plane_runtime_before, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + runtime_inventory_path.chmod(0o600) if component == "platform" and touches_external_data_plane_files(entries): validate_backup_partition( entries, diff --git a/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py b/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py new file mode 100644 index 0000000..9135130 --- /dev/null +++ b/infra/deploy-runner/test_device_plane_foundation_recovery_artifact.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RECOVERY_BUILDER = ( + SCRIPT_DIR / "build-device-plane-foundation-recovery-artifact.mjs" +) +FOUNDATION_BUILDER = SCRIPT_DIR / "build-device-plane-artifact.mjs" +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +RECOVERY_ENTRIES = [ + ".dockerignore", + "package.json", + "package-lock.json", + "docker-compose.device-plane.yml", + "packages/device-protocol-contract", + "packages/arusnavi-b2-adapter", + "services/device-control-core", + "services/device-gateway", + "deployment/device-plane-foundation-recovery-v1.json", +] + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_device_plane_recovery_runner_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class DevicePlaneFoundationRecoveryArtifactTest(unittest.TestCase): + def build(self, builder, artifact_dir, patch_id): + environment = os.environ.copy() + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + result = subprocess.run( + ["node", str(builder), patch_id], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(result.stdout) + + def test_recovery_artifact_is_source_only_exact_and_deterministic(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-recovery-artifact-", + ) as directory: + artifact_dir = Path(directory) + first = self.build( + RECOVERY_BUILDER, + artifact_dir, + "device-plane-foundation-recovery-unit-002", + ) + artifact = Path(first["artifact"]) + first_bytes = artifact.read_bytes() + second = self.build( + RECOVERY_BUILDER, + artifact_dir, + "device-plane-foundation-recovery-unit-002", + ) + second_bytes = Path(second["artifact"]).read_bytes() + + self.assertEqual(first["component"], "device-plane") + self.assertEqual(first["entries"], RECOVERY_ENTRIES) + self.assertEqual(first["build"], []) + self.assertEqual(first["services"], []) + self.assertEqual( + first["transition"], + "failed-foundation-live-runtime-adoption", + ) + self.assertEqual(first_bytes, second_bytes) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_bytes).hexdigest(), + ) + + with tarfile.open(artifact, "r:gz") as archive: + names = {member.name for member in archive.getmembers()} + files = ( + archive.extractfile("files.txt") + .read() + .decode("utf-8") + .splitlines() + ) + descriptor = json.loads( + archive.extractfile( + "payload/deployment/" + "device-plane-foundation-recovery-v1.json" + ) + .read() + .decode("utf-8") + ) + + self.assertEqual(files, RECOVERY_ENTRIES) + self.assertIn( + "payload/services/device-control-core/src/server.mjs", + names, + ) + self.assertIn( + "payload/services/device-gateway/src/server.mjs", + names, + ) + self.assertFalse(any( + "/test/" in name + or "/node_modules/" in name + or Path(name).name.startswith(".env") + or name.startswith("payload/runtime/") + or name.startswith("payload/secrets/") + for name in names + )) + self.assertEqual( + descriptor, + RUNNER.expected_device_plane_foundation_recovery_descriptor(), + ) + self.assertEqual( + RUNNER.component_services( + "device-plane", + tuple(RECOVERY_ENTRIES), + ), + (), + ) + self.assertEqual( + RUNNER.component_builds( + "device-plane", + tuple(RECOVERY_ENTRIES), + ), + (), + ) + + def test_preflight_requires_exact_failed_evidence_partial_source_and_runtime( + self, + ): + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-recovery-preflight-", + ) as directory: + workspace = Path(directory) + artifacts = workspace / "artifacts" + failed_root = workspace / "failed" + backups_root = workspace / "backups" + state_root = workspace / "state" + temp_root = workspace / "tmp" + live_root = workspace / "live" + for path in ( + artifacts, + failed_root, + backups_root, + state_root, + temp_root, + live_root / "deployment", + ): + path.mkdir(parents=True, exist_ok=True) + + failed_build = self.build( + FOUNDATION_BUILDER, + artifacts, + RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID, + ) + self.assertEqual( + failed_build["sha256"], + RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256, + ) + failed_artifact = ( + failed_root / RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT + ) + shutil.copy2(failed_build["artifact"], failed_artifact) + + recovery_build = self.build( + RECOVERY_BUILDER, + artifacts, + "device-plane-foundation-recovery-unit-002", + ) + recovery_extract = workspace / "recovery-extract" + recovery_extract.mkdir() + _manifest, entries, payload = RUNNER.load_artifact( + Path(recovery_build["artifact"]), + recovery_extract, + ) + self.assertEqual(entries, RECOVERY_ENTRIES) + + backup = ( + backups_root + / RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID + ) + backup.mkdir() + for name in ( + "manifest.env", + "files.txt", + "existing-files.txt", + "missing-files.txt", + "source-before.tgz", + ): + (backup / name).write_text( + f"fixture:{name}\n", + encoding="utf-8", + ) + backup_hashes = { + name: hashlib.sha256((backup / name).read_bytes()).hexdigest() + for name in ( + "manifest.env", + "files.txt", + "existing-files.txt", + "missing-files.txt", + "source-before.tgz", + ) + } + + (state_root / "failed.jsonl").write_text( + json.dumps({ + "artifact": + RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT, + "backup_id": + RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_ID, + "component": "device-plane", + "id": RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_PATCH_ID, + "message": + "healthcheck failed for " + "http://127.0.0.1:18120/healthz: " + "", + "rollback_status": "failed:DeployError", + "sha256": + RUNNER.DEVICE_PLANE_FOUNDATION_FAILED_ARTIFACT_SHA256, + "started_apply": True, + "status": "failed", + }) + + "\n", + encoding="utf-8", + ) + + source_root = SCRIPT_DIR.parent.parent / "device-plane" + shutil.copy2( + source_root / "docker-compose.device-plane.yml", + live_root / "docker-compose.device-plane.yml", + ) + shutil.copy2( + source_root + / "deployment/device-postgres-bootstrap-v1.json", + live_root + / "deployment/device-postgres-bootstrap-v1.json", + ) + runtime = { + service: { + "containerId": service, + "imageId": + RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_IMAGE_IDS[ + service + ], + "health": "healthy", + "restartCount": 0, + } + for service in ( + "device-control-core", + "device-gateway", + "device-postgres", + ) + } + + with ( + mock.patch.object(RUNNER, "BACKUPS_DIR", backups_root), + mock.patch.object(RUNNER, "FAILED_DIR", failed_root), + mock.patch.object( + RUNNER, + "FAILED_STATE_FILE", + state_root / "failed.jsonl", + ), + mock.patch.object(RUNNER, "TMP_DIR", temp_root), + mock.patch.object( + RUNNER, + "component_root", + return_value=live_root, + ), + mock.patch.dict( + RUNNER.DEVICE_PLANE_FOUNDATION_RECOVERY_BACKUP_SHA256, + backup_hashes, + clear=True, + ), + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_runtime", + return_value=runtime, + ), + ): + result = ( + RUNNER + .validate_device_plane_foundation_recovery_evidence( + payload + ) + ) + + self.assertEqual( + result["mode"], + "failed-foundation-live-runtime-adoption", + ) + self.assertEqual(result["runtime"], runtime) + + def test_recovery_health_acceptance_never_mutates_runtime(self): + entries = tuple(RECOVERY_ENTRIES) + checks = ("core-health", "gateway-health") + with ( + mock.patch.object( + RUNNER, + "healthcheck_compose_service", + ) as compose_health, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=checks, + ), + mock.patch.object(RUNNER, "healthcheck_url") as url_health, + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_installed_source", + ) as source_acceptance, + mock.patch.object( + RUNNER, + "validate_device_plane_foundation_runtime", + ) as runtime_acceptance, + mock.patch.object(RUNNER, "run_compose") as compose_mutation, + mock.patch.object(RUNNER, "run_build") as build_mutation, + ): + RUNNER.run_healthchecks("device-plane", entries, ()) + + self.assertEqual( + [call.args for call in compose_health.call_args_list], + [ + ("device-plane", "device-control-core"), + ("device-plane", "device-gateway"), + ("device-plane", "device-postgres"), + ], + ) + self.assertEqual( + [call.args[0] for call in url_health.call_args_list], + list(checks), + ) + source_acceptance.assert_called_once_with() + runtime_acceptance.assert_called_once_with() + compose_mutation.assert_not_called() + build_mutation.assert_not_called() + + def test_recovery_runtime_phase_has_no_runtime_mutation(self): + entries = tuple(RECOVERY_ENTRIES) + with ( + mock.patch.object( + RUNNER, + "prepare_component_runtime", + ) as prepare, + mock.patch.object(RUNNER, "run_compose") as compose_mutation, + mock.patch.object(RUNNER, "run_build") as build_mutation, + ): + RUNNER.run_component_runtime("device-plane", entries, ()) + + prepare.assert_not_called() + compose_mutation.assert_not_called() + build_mutation.assert_not_called() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/infra/deploy-runner/test_device_plane_registry.py b/infra/deploy-runner/test_device_plane_registry.py index 067f908..9a2b3c5 100644 --- a/infra/deploy-runner/test_device_plane_registry.py +++ b/infra/deploy-runner/test_device_plane_registry.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import importlib.machinery import importlib.util +import json import tempfile import unittest from pathlib import Path @@ -207,6 +208,15 @@ class DevicePlaneRegistryTest(unittest.TestCase): "\n".join(entries) + "\n", encoding="utf-8", ) + (backup / "runtime-before.json").write_text( + json.dumps({ + "schemaVersion": + "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [], + }), + encoding="utf-8", + ) with ( mock.patch.object( RUNNER, @@ -234,6 +244,165 @@ class DevicePlaneRegistryTest(unittest.TestCase): restore.assert_called_once() self.assertEqual(result, "source-restored-runtime-unchanged:3") + def test_db_bootstrap_compose_does_not_invent_stateless_baseline(self): + entries = ( + "docker-compose.device-plane.yml", + "services/device-control-core", + "services/device-gateway", + ) + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-compose-predecessor-", + ) as directory: + root = Path(directory) / "live" + backup = Path(directory) / "backup" + root.mkdir() + backup.mkdir() + (backup / "existing-files.txt").write_text( + "docker-compose.device-plane.yml\n", + encoding="utf-8", + ) + (backup / "missing-files.txt").write_text( + "services/device-control-core\n" + "services/device-gateway\n", + encoding="utf-8", + ) + (backup / "runtime-before.json").write_text( + json.dumps({ + "schemaVersion": + "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [{ + "service": "device-postgres", + "containerId": "a" * 64, + "imageId": "sha256:" + "b" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }], + }), + encoding="utf-8", + ) + with ( + mock.patch.object( + RUNNER, + "stop_and_remove_compose_services", + ) as stop, + mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=3, + ), + mock.patch.object( + RUNNER, + "run_component_runtime", + ) as run_runtime, + ): + result = RUNNER.rollback_device_plane_apply( + root, + backup, + entries, + "test-stamp", + True, + ("device-control-core", "device-gateway"), + ) + + stop.assert_called_once_with( + "device-plane", + ("device-control-core", "device-gateway"), + ) + run_runtime.assert_not_called() + self.assertEqual(result, "source-restored-runtime-unchanged:3") + + def test_installed_stateless_baseline_is_restored_from_runtime_inventory( + self, + ): + entries = ( + "docker-compose.device-plane.yml", + "services/device-control-core", + "services/device-gateway", + ) + with tempfile.TemporaryDirectory( + prefix="nodedc-device-plane-installed-predecessor-", + ) as directory: + root = Path(directory) / "live" + backup = Path(directory) / "backup" + root.mkdir() + backup.mkdir() + (backup / "existing-files.txt").write_text( + "\n".join(entries) + "\n", + encoding="utf-8", + ) + (backup / "missing-files.txt").write_text( + "", + encoding="utf-8", + ) + (backup / "runtime-before.json").write_text( + json.dumps({ + "schemaVersion": + "nodedc.device-plane.runtime-inventory.v1", + "composeProject": "nodedc-device-plane", + "services": [ + { + "service": "device-control-core", + "containerId": "a" * 64, + "imageId": "sha256:" + "b" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }, + { + "service": "device-gateway", + "containerId": "c" * 64, + "imageId": "sha256:" + "d" * 64, + "status": "running", + "running": True, + "health": "healthy", + "restartCount": 0, + }, + ], + }), + encoding="utf-8", + ) + with ( + mock.patch.object( + RUNNER, + "stop_and_remove_compose_services", + ) as stop, + mock.patch.object( + RUNNER, + "restore_platform_overlay", + return_value=3, + ), + mock.patch.object( + RUNNER, + "run_component_runtime", + ) as run_runtime, + mock.patch.object(RUNNER, "run_healthchecks") as healthchecks, + ): + result = RUNNER.rollback_device_plane_apply( + root, + backup, + entries, + "test-stamp", + True, + ("device-control-core", "device-gateway"), + ) + + stop.assert_not_called() + run_runtime.assert_called_once_with( + "device-plane", + list(entries), + ("device-control-core", "device-gateway"), + ) + healthchecks.assert_called_once_with( + "device-plane", + list(entries), + ("device-control-core", "device-gateway"), + ) + self.assertEqual(result, "source+runtime-restored:3") + if __name__ == "__main__": unittest.main(verbosity=2)