diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 5f422d1..a323764 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -253,6 +253,65 @@ become `custom` and remain exact; they are never elevated. The artifact does not touch UI, n8n, credentials, agent tokens, workflow graphs, databases or runtime payloads. +## Engine L2 node-intelligence transition + +The node-intelligence transition is an additive Engine-owned deployment domain. +It does not merge Ontology, provider APIs or Ops into the Engine MCP. It pins the +reviewed upstream `n8n-mcp` implementation at package `2.33.2`, commit +`974a9fb3492fe2c4984ee0549085d531cdc6242a`, and exposes only the safe NDC L2 +projection through the existing Engine Agent gateway. Upstream management and +write tools are not forwarded. + +Production never clones, pulls, installs or builds this dependency. The builder +saves the reviewed `linux/amd64` image once, embeds that exact archive in a +data-only activation artifact and records both the archive and image-config +digests. The runner validates every inner blob, revision label, entrypoint, +command and platform before an offline `docker image load`; Compose then uses +the fixed tag with `pull_policy: never` and `--pull never`. + +Build a fresh activation/rollback pair: + +```bash +node infra/deploy-runner/build-engine-node-intelligence-artifacts.mjs \ + YYYYMMDD-NNN +``` + +The activation exact set is: + +- `nodedc-source/server/nodeIntelligence` +- `nodedc-source/server/routes/engineAgentGateway.js` +- `nodedc-source/services/node-intelligence` + +The runtime adds only `nodedc-node-intelligence` and recreates the existing +`nodedc-backend`; n8n, UI, databases, L1, provider services and volumes are not +selected. The sidecar has no host port, runs as `11007:11007`, has a read-only +root filesystem, drops all capabilities and receives no Engine or provider +credential. Its independent MCP bearer is created by the root-owned runner as a +read-only file and mounted only into the sidecar and backend. It never appears +in an artifact, shared environment file, log or MCP response. + +Acceptance requires the exact image/container/mount/security inventory, +immutable backend identity and live authenticated `get_node`, `validate_node` +and `validate_workflow` calls. Any apply failure restores source and runtime +automatically. The separate rollback artifact first removes only the sidecar +without volumes, restores the pinned inactive gateway, and recreates only the +verified backend. Loaded images and failed candidate source are retained for +audit rather than destructively deleted. + +Stage the reviewed runner under a unique candidate name first: + +```text +/volume1/docker/nodedc-deploy/runner-install/candidates/ + nodedc-deploy.engine-node-intelligence-YYYYMMDD-NNN +``` + +Do not overwrite the canonical staging candidate while another deploy may be in +flight. After the no-lock/no-process gate, promote the exact candidate in a +standalone root step, run `verify-install`, then invoke a fresh process for +activation `plan` and only then `apply`. The generated plan/apply runbook carries +the runner, activation and rollback SHA-256 values and is staged beside the +unique runner candidate. + `module-foundry` is an independent, authenticated application component. Its artifact contains source and compose infrastructure only; its live `/volume1/docker/nodedc-platform/module-foundry/source/.env` is root-owned and diff --git a/infra/deploy-runner/build-engine-node-intelligence-artifacts.mjs b/infra/deploy-runner/build-engine-node-intelligence-artifacts.mjs new file mode 100644 index 0000000..723b2b0 --- /dev/null +++ b/infra/deploy-runner/build-engine-node-intelligence-artifacts.mjs @@ -0,0 +1,414 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + copyFile, + lstat, + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { createReadStream } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(here, "../.."); +const engineRoot = resolve( + process.env.NODEDC_ENGINE_ROOT || join(platformRoot, "../NODEDC_ENGINE_INFRA"), +); +const artifactRoot = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR || join(here, "../deploy-artifacts"), +); + +const releaseId = "2.33.2-974a9fb3492f"; +const upstreamCommit = "974a9fb3492fe2c4984ee0549085d531cdc6242a"; +const imageTag = `nodedc/engine-node-intelligence:${releaseId}`; +const predecessorGatewaySha256 = "6b8c80fa997ef7c438d199a6ee942c5e37aebc4057667af417897fa636131db4"; +const predecessorComposeSha256 = "258cebb64ff1943c939655cc55bdce00fc5c4dced67ec291d84d6df066ace50e"; +const sourceRootRel = "nodedc-source/server/nodeIntelligence"; +const gatewayRel = "nodedc-source/server/routes/engineAgentGateway.js"; +const serviceRootRel = "nodedc-source/services/node-intelligence"; +const overrideRel = `${serviceRootRel}/docker-compose.immutable-runtime.yml`; +const descriptorRel = `${serviceRootRel}/activation.json`; +const imageArchiveRel = `${serviceRootRel}/image/engine-node-intelligence.tar`; +const activationEntries = [sourceRootRel, gatewayRel, serviceRootRel]; +const rollbackEntries = [gatewayRel, serviceRootRel]; +const transitionId = readTransitionId(process.argv.slice(2)); +const activationId = `engine-node-intelligence-${transitionId}`; +const rollbackId = `engine-node-intelligence-rollback-${transitionId}`; +const activationTarget = join(artifactRoot, `nodedc-${activationId}.tgz`); +const rollbackTarget = join(artifactRoot, `nodedc-${rollbackId}.tgz`); +const commandsTarget = join( + artifactRoot, + `engine-node-intelligence-${transitionId}-plan-apply.txt`, +); + +await mkdir(artifactRoot, { recursive: true }); +for (const path of [ + activationTarget, + `${activationTarget}.sha256`, + rollbackTarget, + `${rollbackTarget}.sha256`, + commandsTarget, +]) { + await assertFresh(path); +} +await assertEngineInputs(); +const image = inspectLocalImage(); + +const work = await mkdtemp(join(tmpdir(), "nodedc-engine-node-intelligence-")); +try { + const imageArchive = join(work, "engine-node-intelligence.tar"); + run("docker", ["image", "save", "--output", imageArchive, imageTag]); + const archive = inspectImageArchive(imageArchive); + const archiveSha256 = await hashFile(imageArchive); + + const source = { + gatewaySha256: await hashFile(join(engineRoot, gatewayRel)), + catalogSha256: await hashFile(join(engineRoot, sourceRootRel, "catalog.js")), + upstreamClientSha256: await hashFile(join(engineRoot, sourceRootRel, "upstreamMcpClient.js")), + upstreamProjectionSha256: await hashFile(join(engineRoot, sourceRootRel, "upstreamProjection.js")), + composeOverrideSha256: await hashFile(join(engineRoot, overrideRel)), + readmeSha256: await hashFile(join(engineRoot, serviceRootRel, "README.md")), + }; + const activationDescriptor = descriptor({ + action: "activate", + expectedCurrent: "inactive", + gatewayPredecessor: predecessorGatewaySha256, + source, + image: { + tag: imageTag, + archiveRelativePath: imageArchiveRel, + archiveSha256, + configSha256: archive.configSha256, + architecture: "amd64", + os: "linux", + }, + }); + + await buildArtifact( + work, + activationId, + activationEntries, + activationTarget, + async (payload) => { + await copyExactDirectory(join(engineRoot, sourceRootRel), join(payload, sourceRootRel)); + await copyExactFile(join(engineRoot, gatewayRel), join(payload, gatewayRel)); + await copyExactFile(join(engineRoot, serviceRootRel, "README.md"), join(payload, serviceRootRel, "README.md")); + await copyExactFile(join(engineRoot, overrideRel), join(payload, overrideRel)); + await copyExactFile(imageArchive, join(payload, imageArchiveRel)); + await writeJson(join(payload, descriptorRel), activationDescriptor); + }, + ); + + const rollbackReadme = [ + "# NDC Engine node intelligence (inactive rollback)", + "", + `This exact runner-owned descriptor deactivates ${releaseId}.`, + "The pinned image and source are retained for audit; the Compose overlay is absent,", + "the sidecar is removed, and the verified immutable backend is recreated alone.", + "", + ].join("\n"); + const rollbackReadmeSha256 = hashBytes(Buffer.from(rollbackReadme, "utf8")); + const rollbackDescriptor = descriptor({ + action: "rollback-inactive", + expectedCurrent: releaseId, + gatewayPredecessor: source.gatewaySha256, + source: { + gatewaySha256: predecessorGatewaySha256, + readmeSha256: rollbackReadmeSha256, + }, + image: { + tag: imageTag, + configSha256: archive.configSha256, + architecture: "amd64", + os: "linux", + }, + }); + await buildArtifact( + work, + rollbackId, + rollbackEntries, + rollbackTarget, + async (payload) => { + await writeGitFile(gatewayRel, join(payload, gatewayRel)); + await mkdir(join(payload, serviceRootRel), { recursive: true }); + await writeFile(join(payload, serviceRootRel, "README.md"), rollbackReadme, "utf8"); + await writeJson(join(payload, descriptorRel), rollbackDescriptor); + }, + ); + + const runnerSha256 = await hashFile(join(here, "nodedc-deploy")); + const activationSha256 = await hashFile(activationTarget); + const rollbackSha256 = await hashFile(rollbackTarget); + await writeFile( + `${activationTarget}.sha256`, + `${activationSha256} ${activationTarget.split("/").at(-1)}\n`, + "utf8", + ); + await writeFile( + `${rollbackTarget}.sha256`, + `${rollbackSha256} ${rollbackTarget.split("/").at(-1)}\n`, + "utf8", + ); + await writeFile( + commandsTarget, + commandPlan({ runnerSha256, activationSha256, rollbackSha256 }), + "utf8", + ); + + process.stdout.write(`${JSON.stringify({ + ok: true, + transitionId, + releaseId, + upstreamCommit, + image: { + tag: imageTag, + localId: image.Id, + archiveSha256, + configSha256: archive.configSha256, + architecture: archive.architecture, + os: archive.os, + }, + runner: { sha256: runnerSha256 }, + activation: { artifact: activationTarget, sha256: activationSha256, entries: activationEntries }, + rollback: { artifact: rollbackTarget, sha256: rollbackSha256, entries: rollbackEntries }, + commands: commandsTarget, + }, null, 2)}\n`); +} finally { + await rm(work, { recursive: true, force: true }); +} + +function readTransitionId(args) { + if (args.length !== 1) throw new Error("usage: build-engine-node-intelligence-artifacts.mjs YYYYMMDD-NNN"); + const value = String(args[0] || "").trim(); + const match = /^(\d{4})(\d{2})(\d{2})-([0-9]{3})$/.exec(value); + if (!match || match[4] === "000") throw new Error("transition_id_invalid"); + const parsed = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]))); + if ( + parsed.getUTCFullYear() !== Number(match[1]) + || parsed.getUTCMonth() !== Number(match[2]) - 1 + || parsed.getUTCDate() !== Number(match[3]) + ) throw new Error("transition_id_invalid"); + return value; +} + +async function assertFresh(path) { + try { + await lstat(path); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + throw new Error(`target_already_exists:${path}`); +} + +async function assertEngineInputs() { + const baselineGateway = Buffer.from(run("git", ["show", `HEAD:${gatewayRel}`], engineRoot).stdout, "utf8"); + if (hashBytes(baselineGateway) !== predecessorGatewaySha256) throw new Error("gateway_predecessor_mismatch"); + if (await hashFile(join(engineRoot, "docker-compose.yml")) !== predecessorComposeSha256) { + throw new Error("compose_predecessor_mismatch"); + } + const sourceFiles = (await readdir(join(engineRoot, sourceRootRel))).sort(); + if (JSON.stringify(sourceFiles) !== JSON.stringify(["catalog.js", "upstreamMcpClient.js", "upstreamProjection.js"])) { + throw new Error("node_intelligence_source_exact_set_mismatch"); + } + const serviceFiles = (await readdir(join(engineRoot, serviceRootRel))).sort(); + if (JSON.stringify(serviceFiles) !== JSON.stringify(["README.md", "docker-compose.immutable-runtime.yml"])) { + throw new Error("node_intelligence_service_exact_set_mismatch"); + } + const gateway = await readFile(join(engineRoot, gatewayRel), "utf8"); + for (const tool of [ + "engine_get_node_intelligence_status", + "engine_get_node_guidance", + "engine_validate_node_configuration", + "engine_validate_l2_deep", + ]) { + if (!gateway.includes(tool)) throw new Error(`gateway_tool_missing:${tool}`); + } + const override = await readFile(join(engineRoot, overrideRel), "utf8"); + for (const exact of [ + `image: ${imageTag}`, + "pull_policy: never", + 'user: "11007:11007"', + "AUTH_TOKEN_FILE: /run/nodedc-secrets/engine-node-intelligence-auth-token", + "ENGINE_NODE_INTELLIGENCE_AUTH_TOKEN_FILE: /run/nodedc-secrets/engine-node-intelligence-auth-token", + "read_only: true", + "no-new-privileges:true", + ]) { + if (!override.includes(exact)) throw new Error(`compose_contract_missing:${exact}`); + } + if (/^\s*(?:AUTH_TOKEN|N8N_API_URL|N8N_API_KEY):/m.test(override)) { + throw new Error("compose_runtime_authority_forbidden"); + } +} + +function inspectLocalImage() { + const raw = run("docker", ["image", "inspect", imageTag]).stdout; + const images = JSON.parse(raw); + if (!Array.isArray(images) || images.length !== 1) throw new Error("image_inspect_shape_mismatch"); + const image = images[0]; + const config = image.Config || {}; + const labels = config.Labels || {}; + if ( + !/^sha256:[a-f0-9]{64}$/.test(String(image.Id || "")) + || image.Architecture !== "amd64" + || image.Os !== "linux" + || !(image.RepoTags || []).includes(imageTag) + || labels["org.opencontainers.image.revision"] !== upstreamCommit + || JSON.stringify(config.Entrypoint) !== JSON.stringify(["/usr/local/bin/docker-entrypoint.sh"]) + || JSON.stringify(config.Cmd) !== JSON.stringify(["node", "dist/mcp/index.js"]) + || (config.Env || []).some((value) => /^(?:AUTH_TOKEN|N8N_API_URL|N8N_API_KEY)=/.test(String(value))) + ) throw new Error("image_identity_mismatch"); + return image; +} + +function inspectImageArchive(path) { + const script = [ + "import hashlib,json,pathlib,sys,tarfile", + "p=pathlib.Path(sys.argv[1])", + "with tarfile.open(p,'r:') as tf:", + " m=json.loads(tf.extractfile('manifest.json').read())", + " if not isinstance(m,list) or len(m)!=1: raise SystemExit('manifest-set')", + " r=m[0]", + ` if r.get('RepoTags')!=[${JSON.stringify(imageTag)}]: raise SystemExit('tag')`, + " n=r.get('Config','')", + " raw=tf.extractfile(n).read()", + " digest=n.rsplit('/',1)[-1]", + " if hashlib.sha256(raw).hexdigest()!=digest: raise SystemExit('config-digest')", + " c=json.loads(raw)", + " cfg=c.get('config') or {}", + ` if c.get('architecture')!='amd64' or c.get('os')!='linux' or (cfg.get('Labels') or {}).get('org.opencontainers.image.revision')!=${JSON.stringify(upstreamCommit)}: raise SystemExit('identity')`, + " if cfg.get('Entrypoint')!=['/usr/local/bin/docker-entrypoint.sh'] or cfg.get('Cmd')!=['node','dist/mcp/index.js']: raise SystemExit('command')", + " print(json.dumps({'configSha256':digest,'architecture':c['architecture'],'os':c['os']}))", + ].join("\n"); + return JSON.parse(run("python3", ["-c", script, path]).stdout); +} + +function descriptor({ action, expectedCurrent, gatewayPredecessor, source, image }) { + return { + schemaVersion: "nodedc.engine-node-intelligence-transition/v1", + action, + releaseId, + expectedCurrent, + upstream: { + package: "n8n-mcp", + version: "2.33.2", + commit: upstreamCommit, + }, + image, + source, + predecessor: { + gatewaySha256: gatewayPredecessor, + composeSha256: predecessorComposeSha256, + backendRuntime: "verified-derived-retry", + }, + }; +} + +async function buildArtifact(workRoot, id, entries, target, populate) { + const stage = join(workRoot, id); + const payload = join(stage, "payload"); + await mkdir(payload, { recursive: true }); + await populate(payload); + await writeFile(join(stage, "manifest.env"), `id=${id}\ncomponent=engine\ntype=app-overlay\n`, "utf8"); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, "utf8"); + run("python3", ["-c", canonicalTarScript(), target, stage]); +} + +async function copyExactDirectory(source, target) { + await mkdir(target, { recursive: true }); + for (const name of (await readdir(source)).sort()) { + await copyExactFile(join(source, name), join(target, name)); + } +} + +async function copyExactFile(source, target) { + const sourceStat = await lstat(source); + if (!sourceStat.isFile() || sourceStat.isSymbolicLink()) throw new Error(`source_file_unsafe:${source}`); + await mkdir(dirname(target), { recursive: true }); + await copyFile(source, target, 0); +} + +async function writeGitFile(relativePath, target) { + const value = run("git", ["show", `HEAD:${relativePath}`], engineRoot).stdout; + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, value, "utf8"); +} + +async function writeJson(path, value) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'wb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +function commandPlan({ runnerSha256, activationSha256, rollbackSha256 }) { + return [ + `transition=${transitionId}`, + `runner_sha256=${runnerSha256}`, + `activation_sha256=${activationSha256}`, + `rollback_sha256=${rollbackSha256}`, + "", + "# 1. Read-only gate: no deploy process and no state/deploy.lock.", + "# 2. Promote the exact staged runner in a standalone sudo step:", + `sudo sha256sum /volume1/docker/nodedc-deploy/runner-install/candidates/nodedc-deploy.${activationId}`, + "sudo install -o root -g root -m 0755 \\", + ` /volume1/docker/nodedc-deploy/runner-install/candidates/nodedc-deploy.${activationId} \\`, + " /usr/local/sbin/nodedc-deploy", + "sudo /usr/local/sbin/nodedc-deploy verify-install", + "", + "# 3. Activation is always plan, then explicit apply:", + `sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/${activationTarget.split("/").at(-1)}`, + `sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/${activationTarget.split("/").at(-1)}`, + "", + "# 4. Rollback is operator-invoked only after its own plan:", + `sudo /usr/local/sbin/nodedc-deploy plan /volume1/docker/nodedc-deploy/inbox/${rollbackTarget.split("/").at(-1)}`, + `sudo /usr/local/sbin/nodedc-deploy apply /volume1/docker/nodedc-deploy/inbox/${rollbackTarget.split("/").at(-1)}`, + "", + ].join("\n"); +} + +function hashBytes(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function hashFile(path) { + return new Promise((resolveHash, rejectHash) => { + const digest = createHash("sha256"); + const input = createReadStream(path); + input.on("data", (chunk) => digest.update(chunk)); + input.on("error", rejectHash); + input.on("end", () => resolveHash(digest.digest("hex"))); + }); +} + +function run(command, args, cwd) { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + return result; +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index fe2e034..f4ea532 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -74,6 +74,47 @@ ENGINE_N8N_CREDENTIALS_CATALOG_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/credentials. ENGINE_N8N_SCHEMA_META_REL = f"{ENGINE_N8N_SCHEMA_ROOT_REL}/meta.json" ENGINE_N8N_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.svg" ENGINE_N8N_DARK_ICON_REL = "nodedc-source/server/assets/n8n/icons/ndc.dark.svg" +ENGINE_NODE_INTELLIGENCE_RELEASE_ID = "2.33.2-974a9fb3492f" +ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT = "974a9fb3492fe2c4984ee0549085d531cdc6242a" +ENGINE_NODE_INTELLIGENCE_IMAGE = ( + f"nodedc/engine-node-intelligence:{ENGINE_NODE_INTELLIGENCE_RELEASE_ID}" +) +ENGINE_NODE_INTELLIGENCE_SERVICE = "nodedc-node-intelligence" +ENGINE_NODE_INTELLIGENCE_RUNTIME_UID = 11007 +ENGINE_NODE_INTELLIGENCE_RUNTIME_GID = 11007 +ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR = RUNTIME_DIR / "engine-node-intelligence" +ENGINE_NODE_INTELLIGENCE_SECRET_FILE = ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR / "auth-token" +ENGINE_NODE_INTELLIGENCE_RELEASES_DIR = ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR / "releases" +ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH = ( + "/run/nodedc-secrets/engine-node-intelligence-auth-token" +) +ENGINE_NODE_INTELLIGENCE_SOURCE_REL = "nodedc-source/server/nodeIntelligence" +ENGINE_NODE_INTELLIGENCE_GATEWAY_REL = "nodedc-source/server/routes/engineAgentGateway.js" +ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL = "nodedc-source/services/node-intelligence" +ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL = ( + f"{ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL}/docker-compose.immutable-runtime.yml" +) +ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL = ( + f"{ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL}/activation.json" +) +ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL = ( + f"{ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL}/image/engine-node-intelligence.tar" +) +ENGINE_NODE_INTELLIGENCE_PREDECESSOR_GATEWAY_SHA256 = ( + "6b8c80fa997ef7c438d199a6ee942c5e37aebc4057667af417897fa636131db4" +) +ENGINE_NODE_INTELLIGENCE_PREDECESSOR_COMPOSE_SHA256 = ( + "258cebb64ff1943c939655cc55bdce00fc5c4dced67ec291d84d6df066ace50e" +) +ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES = ( + ENGINE_NODE_INTELLIGENCE_SOURCE_REL, + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL, + ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL, +) +ENGINE_NODE_INTELLIGENCE_ROLLBACK_ENTRIES = ( + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL, + ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL, +) ENGINE_CONTROL_PLANE_STATE_REL = "nodedc-control-plane" ENGINE_PUBLISH_GRANT_STATE_REL = f"{ENGINE_CONTROL_PLANE_STATE_REL}/publish-grants" ENGINE_CONTROL_PLANE_STATE_PATH = Path("/volume2/nodedc-demo/nodedc-control-plane") @@ -113,6 +154,7 @@ PATCH_ID_RE = re.compile(r"^[A-Za-z0-9._-]{1,96}$") MANIFEST_KEYS = {"id", "component", "type"} MAP_GATEWAY_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$") EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$") +ENGINE_NODE_INTELLIGENCE_SECRET_RE = re.compile(r"^[A-Za-z0-9_-]{48,256}$") N8N_PRIVATE_EXTENSION_RELEASE_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+-[a-f0-9]{16}$") N8N_PRIVATE_EXTENSION_NODES = ( "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", @@ -1523,6 +1565,10 @@ def allowed_payload_path(component, rel): return True if rel == ENGINE_DATA_PRODUCT_PUBLISH_GRANT_OVERRIDE_REL: return True + if rel == ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL or rel.startswith( + ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + "/" + ): + return True if rel == "nodedc-source/dist": return True @@ -2279,6 +2325,386 @@ def validate_engine_n8n_transition(payload_dir, entries): return descriptor +def is_engine_node_intelligence_transition(component, entries): + if component != "engine" or entries is None: + return False + return tuple(entries) in ( + ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ENGINE_NODE_INTELLIGENCE_ROLLBACK_ENTRIES, + ) + + +def expected_engine_node_intelligence_compose_override(): + return "\n".join(( + "services:", + f" {ENGINE_NODE_INTELLIGENCE_SERVICE}:", + f" image: {ENGINE_NODE_INTELLIGENCE_IMAGE}", + " pull_policy: never", + " restart: unless-stopped", + f' user: "{ENGINE_NODE_INTELLIGENCE_RUNTIME_UID}:{ENGINE_NODE_INTELLIGENCE_RUNTIME_GID}"', + " read_only: true", + " environment:", + " HOME: /tmp", + " NODE_ENV: production", + " MCP_MODE: http", + ' PORT: "3000"', + f" AUTH_TOKEN_FILE: {ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH}", + " NODE_DB_PATH: /app/data/nodes.db", + ' REBUILD_ON_START: "false"', + ' N8N_MCP_TELEMETRY_DISABLED: "true"', + " LOG_LEVEL: warn", + " volumes:", + " - type: bind", + f" source: {ENGINE_NODE_INTELLIGENCE_SECRET_FILE}", + f" target: {ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH}", + " read_only: true", + " expose:", + ' - "3000"', + " tmpfs:", + " - /tmp:mode=1777", + " - /app/logs:mode=0755", + " cap_drop:", + " - ALL", + " security_opt:", + " - no-new-privileges:true", + " healthcheck:", + " test:", + " - CMD-SHELL", + " - curl -fsS http://127.0.0.1:3000/health >/dev/null", + " interval: 15s", + " timeout: 5s", + " retries: 10", + " start_period: 30s", + "", + " nodedc-backend:", + " depends_on:", + f" {ENGINE_NODE_INTELLIGENCE_SERVICE}:", + " condition: service_healthy", + " environment:", + f" ENGINE_NODE_INTELLIGENCE_MCP_URL: http://{ENGINE_NODE_INTELLIGENCE_SERVICE}:3000/mcp", + f" ENGINE_NODE_INTELLIGENCE_AUTH_TOKEN_FILE: {ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH}", + f" ENGINE_NODE_INTELLIGENCE_ASSET_ID: nodedc-node-intelligence@{ENGINE_NODE_INTELLIGENCE_RELEASE_ID.replace('-', '+', 1)}", + " volumes:", + " - type: bind", + f" source: {ENGINE_NODE_INTELLIGENCE_SECRET_FILE}", + f" target: {ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH}", + " read_only: true", + "", + )) + + +def read_engine_node_intelligence_descriptor(path, label="Engine node-intelligence descriptor"): + descriptor = read_strict_json(path, label) + require_exact_json_keys( + descriptor, + ( + "schemaVersion", + "action", + "releaseId", + "expectedCurrent", + "upstream", + "image", + "source", + "predecessor", + ), + label, + ) + if descriptor.get("schemaVersion") != "nodedc.engine-node-intelligence-transition/v1": + die("Engine node-intelligence transition schema mismatch") + action = descriptor.get("action") + if action not in ("activate", "rollback-inactive"): + die("Engine node-intelligence transition action mismatch") + if descriptor.get("releaseId") != ENGINE_NODE_INTELLIGENCE_RELEASE_ID: + die("Engine node-intelligence release mismatch") + require_exact_json_keys( + descriptor.get("upstream"), + ("package", "version", "commit"), + f"{label} upstream", + ) + if descriptor["upstream"] != { + "package": "n8n-mcp", + "version": "2.33.2", + "commit": ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT, + }: + die("Engine node-intelligence upstream pin mismatch") + require_exact_json_keys( + descriptor.get("predecessor"), + ("gatewaySha256", "composeSha256", "backendRuntime"), + f"{label} predecessor", + ) + predecessor = descriptor["predecessor"] + if predecessor.get("composeSha256") != ENGINE_NODE_INTELLIGENCE_PREDECESSOR_COMPOSE_SHA256: + die("Engine node-intelligence predecessor Compose mismatch") + if predecessor.get("backendRuntime") != "verified-derived-retry": + die("Engine node-intelligence backend predecessor mismatch") + + if action == "activate": + if descriptor.get("expectedCurrent") != "inactive": + die("Engine node-intelligence activation expected-current mismatch") + if predecessor.get("gatewaySha256") != ENGINE_NODE_INTELLIGENCE_PREDECESSOR_GATEWAY_SHA256: + die("Engine node-intelligence activation gateway predecessor mismatch") + require_exact_json_keys( + descriptor.get("image"), + ( + "tag", + "archiveRelativePath", + "archiveSha256", + "configSha256", + "architecture", + "os", + ), + f"{label} image", + ) + image = descriptor["image"] + if ( + image.get("tag") != ENGINE_NODE_INTELLIGENCE_IMAGE + or image.get("archiveRelativePath") != ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL + or not re.fullmatch(r"[a-f0-9]{64}", str(image.get("archiveSha256") or "")) + or not re.fullmatch(r"[a-f0-9]{64}", str(image.get("configSha256") or "")) + or image.get("architecture") != "amd64" + or image.get("os") != "linux" + ): + die("Engine node-intelligence image descriptor mismatch") + require_exact_json_keys( + descriptor.get("source"), + ( + "gatewaySha256", + "catalogSha256", + "upstreamClientSha256", + "upstreamProjectionSha256", + "composeOverrideSha256", + "readmeSha256", + ), + f"{label} source", + ) + for value in descriptor["source"].values(): + if not re.fullmatch(r"[a-f0-9]{64}", str(value or "")): + die("Engine node-intelligence source digest is invalid") + else: + if descriptor.get("expectedCurrent") != ENGINE_NODE_INTELLIGENCE_RELEASE_ID: + die("Engine node-intelligence rollback expected-current mismatch") + require_exact_json_keys( + descriptor.get("image"), + ("tag", "configSha256", "architecture", "os"), + f"{label} image", + ) + image = descriptor["image"] + if ( + image.get("tag") != ENGINE_NODE_INTELLIGENCE_IMAGE + or not re.fullmatch(r"[a-f0-9]{64}", str(image.get("configSha256") or "")) + or image.get("architecture") != "amd64" + or image.get("os") != "linux" + ): + die("Engine node-intelligence rollback image descriptor mismatch") + require_exact_json_keys( + descriptor.get("source"), + ("gatewaySha256", "readmeSha256"), + f"{label} source", + ) + if descriptor["source"].get("gatewaySha256") != ENGINE_NODE_INTELLIGENCE_PREDECESSOR_GATEWAY_SHA256: + die("Engine node-intelligence rollback gateway target mismatch") + if any( + not re.fullmatch(r"[a-f0-9]{64}", str(value or "")) + for value in descriptor["source"].values() + ): + die("Engine node-intelligence rollback source digest is invalid") + if not re.fullmatch(r"[a-f0-9]{64}", str(predecessor.get("gatewaySha256") or "")): + die("Engine node-intelligence rollback predecessor gateway is invalid") + return descriptor + + +def validate_engine_node_intelligence_image_archive(path, descriptor): + image = descriptor["image"] + try: + archive_stat = path.lstat() + except FileNotFoundError: + die("Engine node-intelligence image archive is missing") + if stat.S_ISLNK(archive_stat.st_mode) or not stat.S_ISREG(archive_stat.st_mode): + die("Engine node-intelligence image archive is unsafe") + if archive_stat.st_size < 1024 or archive_stat.st_size > MAX_ARTIFACT_BYTES: + die("Engine node-intelligence image archive size is invalid") + if sha256_file(path) != image["archiveSha256"]: + die("Engine node-intelligence image archive sha256 mismatch") + + names = set() + manifest_raw = None + config_raw = None + total = 0 + try: + with tarfile.open(path, "r:") as archive: + members = archive.getmembers() + if not members or len(members) > MAX_MEMBER_COUNT: + die("Engine node-intelligence image archive member count is invalid") + for member in members: + validate_posix_path(member.name.rstrip("/")) + if member.name in names: + die(f"Engine node-intelligence image archive duplicate member: {member.name}") + names.add(member.name) + if not (member.isfile() or member.isdir()): + die(f"Engine node-intelligence image archive special member: {member.name}") + if member.isdir(): + continue + if member.size > MAX_FILE_BYTES: + die(f"Engine node-intelligence image archive member too large: {member.name}") + total += member.size + if total > MAX_ARTIFACT_BYTES: + die("Engine node-intelligence image archive expanded size is too large") + source = archive.extractfile(member) + if source is None: + die(f"Engine node-intelligence image archive member unreadable: {member.name}") + data = source.read(MAX_FILE_BYTES + 1) + if len(data) != member.size: + die(f"Engine node-intelligence image archive member size mismatch: {member.name}") + if member.name.startswith("blobs/sha256/"): + digest = member.name.rsplit("/", 1)[-1] + if not re.fullmatch(r"[a-f0-9]{64}", digest): + die("Engine node-intelligence image blob name is invalid") + if hashlib.sha256(data).hexdigest() != digest: + die("Engine node-intelligence image blob digest mismatch") + elif member.name not in ("index.json", "manifest.json", "oci-layout"): + die(f"Engine node-intelligence image archive unexpected member: {member.name}") + if member.name == "manifest.json": + manifest_raw = data + if manifest_raw is None: + die("Engine node-intelligence image manifest is missing") + try: + manifest = json.loads(manifest_raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + die("Engine node-intelligence image manifest is invalid") + if not isinstance(manifest, list) or len(manifest) != 1: + die("Engine node-intelligence image manifest exact set mismatch") + record = manifest[0] + if not isinstance(record, dict) or set(record) != {"Config", "RepoTags", "Layers"}: + die("Engine node-intelligence image manifest record mismatch") + if record.get("RepoTags") != [ENGINE_NODE_INTELLIGENCE_IMAGE]: + die("Engine node-intelligence image tag mismatch") + config_name = record.get("Config") + expected_config_name = f"blobs/sha256/{image['configSha256']}" + if config_name != expected_config_name: + die("Engine node-intelligence image config digest mismatch") + try: + config_member = archive.getmember(config_name) + config_file = archive.extractfile(config_member) + config_raw = config_file.read(MAX_FILE_BYTES + 1) if config_file else None + except KeyError: + config_raw = None + except (tarfile.TarError, OSError): + die("Engine node-intelligence image archive cannot be read") + if config_raw is None: + die("Engine node-intelligence image config is missing") + try: + config = json.loads(config_raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + die("Engine node-intelligence image config is invalid") + if config.get("architecture") != "amd64" or config.get("os") != "linux": + die("Engine node-intelligence image platform mismatch") + runtime_config = config.get("config") or {} + labels = runtime_config.get("Labels") or {} + if labels.get("org.opencontainers.image.revision") != ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT: + die("Engine node-intelligence image revision label mismatch") + if runtime_config.get("Entrypoint") != ["/usr/local/bin/docker-entrypoint.sh"]: + die("Engine node-intelligence image entrypoint mismatch") + if runtime_config.get("Cmd") != ["node", "dist/mcp/index.js"]: + die("Engine node-intelligence image command mismatch") + env = runtime_config.get("Env") or [] + if any( + str(value).startswith(("AUTH_TOKEN=", "N8N_API_URL=", "N8N_API_KEY=")) + for value in env + ): + die("Engine node-intelligence image contains runtime authority") + return image["configSha256"] + + +def validate_engine_node_intelligence_transition(payload_dir, entries): + service_root = payload_dir / ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + descriptor = read_engine_node_intelligence_descriptor( + service_root / "activation.json" + ) + expected_entries = ( + ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES + if descriptor["action"] == "activate" + else ENGINE_NODE_INTELLIGENCE_ROLLBACK_ENTRIES + ) + if tuple(entries) != expected_entries: + die("Engine node-intelligence files.txt exact set/order mismatch") + gateway = payload_dir / ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + if sha256_file(gateway) != descriptor["source"]["gatewaySha256"]: + die("Engine node-intelligence gateway sha256 mismatch") + readme = service_root / "README.md" + if sha256_file(readme) != descriptor["source"]["readmeSha256"]: + die("Engine node-intelligence README sha256 mismatch") + + if descriptor["action"] == "activate": + expected_service_files = { + "README.md", + "activation.json", + "docker-compose.immutable-runtime.yml", + "image/engine-node-intelligence.tar", + } + actual_service_files = { + child.relative_to(service_root).as_posix() + for child in service_root.rglob("*") + if child.is_file() + } + if actual_service_files != expected_service_files: + die("Engine node-intelligence service payload file set mismatch") + source_root = payload_dir / ENGINE_NODE_INTELLIGENCE_SOURCE_REL + expected_source_files = { + "catalog.js", + "upstreamMcpClient.js", + "upstreamProjection.js", + } + actual_source_files = { + child.relative_to(source_root).as_posix() + for child in source_root.rglob("*") + if child.is_file() + } + if actual_source_files != expected_source_files: + die("Engine node-intelligence source file set mismatch") + source_digests = { + "catalogSha256": sha256_file(source_root / "catalog.js"), + "upstreamClientSha256": sha256_file(source_root / "upstreamMcpClient.js"), + "upstreamProjectionSha256": sha256_file(source_root / "upstreamProjection.js"), + } + if any(descriptor["source"][key] != value for key, value in source_digests.items()): + die("Engine node-intelligence source digest mismatch") + override = service_root / "docker-compose.immutable-runtime.yml" + try: + override_text = override.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + die("Engine node-intelligence Compose override is unreadable") + if override_text != expected_engine_node_intelligence_compose_override(): + die("Engine node-intelligence Compose override mismatch") + if sha256_file(override) != descriptor["source"]["composeOverrideSha256"]: + die("Engine node-intelligence Compose override sha256 mismatch") + gateway_text = gateway.read_text(encoding="utf-8") + for required in ( + "engine_get_node_intelligence_status", + "engine_get_node_guidance", + "engine_validate_node_configuration", + "engine_validate_l2_deep", + "const ENGINE_AGENT_MCP_VERSION = '0.3.0'", + ): + if required not in gateway_text: + die(f"Engine node-intelligence gateway contract missing: {required}") + validate_engine_node_intelligence_image_archive( + payload_dir / ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL, + descriptor, + ) + else: + expected_service_files = {"README.md", "activation.json"} + actual_service_files = { + child.relative_to(service_root).as_posix() + for child in service_root.rglob("*") + if child.is_file() + } + if actual_service_files != expected_service_files: + die("Engine node-intelligence rollback payload file set mismatch") + if sha256_file(gateway) != ENGINE_NODE_INTELLIGENCE_PREDECESSOR_GATEWAY_SHA256: + die("Engine node-intelligence rollback gateway baseline mismatch") + return descriptor + + def validate_no_lifecycle_scripts(payload_dir, label): forbidden = ("preinstall", "install", "postinstall", "prepare", "prepack", "postpack") for package_path in payload_dir.rglob("package.json"): @@ -2484,6 +2910,127 @@ def current_engine_n8n_transition_descriptor(): return read_engine_n8n_transition_descriptor(path, "installed Engine n8n transition descriptor") +def current_engine_node_intelligence_descriptor(): + path = component_root("engine") / ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL + if not path.exists(): + return None + if path.is_symlink() or not path.is_file(): + die("installed Engine node-intelligence descriptor is unsafe") + return read_engine_node_intelligence_descriptor( + path, + "installed Engine node-intelligence descriptor", + ) + + +def validate_installed_engine_node_intelligence_source(descriptor): + if descriptor is None or descriptor.get("action") != "activate": + die("active Engine node-intelligence descriptor is required") + root = component_root("engine") + source_root = root / ENGINE_NODE_INTELLIGENCE_SOURCE_REL + expected_source_files = { + "catalog.js": descriptor["source"]["catalogSha256"], + "upstreamMcpClient.js": descriptor["source"]["upstreamClientSha256"], + "upstreamProjection.js": descriptor["source"]["upstreamProjectionSha256"], + } + try: + source_stat = source_root.lstat() + except FileNotFoundError: + die("installed Engine node-intelligence source is missing") + if stat.S_ISLNK(source_stat.st_mode) or not stat.S_ISDIR(source_stat.st_mode): + die("installed Engine node-intelligence source is unsafe") + actual_files = { + child.relative_to(source_root).as_posix() + for child in source_root.rglob("*") + if child.is_file() + } + if actual_files != set(expected_source_files): + die("installed Engine node-intelligence source file set mismatch") + for rel, expected_sha256 in expected_source_files.items(): + path = source_root / rel + if path.is_symlink() or sha256_file(path) != expected_sha256: + die(f"installed Engine node-intelligence source drift detected: {rel}") + gateway = root / ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + if gateway.is_symlink() or sha256_file(gateway) != descriptor["source"]["gatewaySha256"]: + die("installed Engine node-intelligence gateway drift detected") + override = root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL + try: + override_stat = override.lstat() + override_text = override.read_text(encoding="utf-8") + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("installed Engine node-intelligence Compose override is missing") + if stat.S_ISLNK(override_stat.st_mode) or not stat.S_ISREG(override_stat.st_mode): + die("installed Engine node-intelligence Compose override is unsafe") + if ( + override_text != expected_engine_node_intelligence_compose_override() + or sha256_file(override) != descriptor["source"]["composeOverrideSha256"] + ): + die("installed Engine node-intelligence Compose override drift detected") + readme = root / ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL / "README.md" + if readme.is_symlink() or sha256_file(readme) != descriptor["source"]["readmeSha256"]: + die("installed Engine node-intelligence README drift detected") + return descriptor["source"]["gatewaySha256"] + + +def preflight_engine_node_intelligence_predecessor(payload_dir): + candidate = read_engine_node_intelligence_descriptor( + payload_dir / ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL, + "candidate Engine node-intelligence descriptor", + ) + root = component_root("engine") + compose_path = root / "docker-compose.yml" + gateway_path = root / ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + for path, label in ( + (compose_path, "Compose"), + (gateway_path, "gateway"), + ): + try: + path_stat = path.lstat() + except FileNotFoundError: + die(f"Engine node-intelligence predecessor {label} is missing") + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISREG(path_stat.st_mode): + die(f"Engine node-intelligence predecessor {label} is unsafe") + if sha256_file(compose_path) != candidate["predecessor"]["composeSha256"]: + die("Engine node-intelligence predecessor Compose drift detected") + + current = current_engine_node_intelligence_descriptor() + current_state = ( + current["releaseId"] + if current is not None and current["action"] == "activate" + else "inactive" + ) + if current_state != candidate["expectedCurrent"]: + die( + "Engine node-intelligence predecessor state mismatch: " + f"expected={candidate['expectedCurrent']} actual={current_state}" + ) + actual_gateway_sha256 = sha256_file(gateway_path) + if actual_gateway_sha256 != candidate["predecessor"]["gatewaySha256"]: + die( + "Engine node-intelligence predecessor gateway drift detected: " + f"expected={candidate['predecessor']['gatewaySha256']} " + f"actual={actual_gateway_sha256}" + ) + if candidate["action"] == "activate": + if current is not None and current["action"] == "activate": + die("Engine node-intelligence activation requires an inactive predecessor") + override = root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL + if override.exists() or override.is_symlink(): + die("inactive Engine node-intelligence predecessor still has a Compose override") + else: + validate_installed_engine_node_intelligence_source(current) + + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die("Engine node-intelligence requires the active immutable backend") + return { + "action": candidate["action"], + "current_state": current_state, + "gateway_sha256": actual_gateway_sha256, + "backend_mode": backend["mode"], + "descriptor": candidate, + } + + def validate_engine_n8n_staged_release(descriptor): release_rel = f"releases/n8n-nodes-ndc/{descriptor['releaseId']}" release_dir = N8N_PRIVATE_EXTENSION_RELEASES_ROOT / release_rel @@ -2549,6 +3096,19 @@ def load_artifact(artifact, work_dir): die("Engine private-extension payload requires the canonical transition descriptor") if is_engine_n8n_transition(manifest["component"], entries): validate_engine_n8n_transition(payload_dir, entries) + touches_node_intelligence = any( + rel == ENGINE_NODE_INTELLIGENCE_SOURCE_REL + or rel.startswith(ENGINE_NODE_INTELLIGENCE_SOURCE_REL + "/") + or rel == ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + or rel.startswith(ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + "/") + for rel in entries + ) + if touches_node_intelligence and not is_engine_node_intelligence_transition( + manifest["component"], entries + ): + die("Engine node-intelligence payload requires the canonical transition") + if is_engine_node_intelligence_transition(manifest["component"], entries): + validate_engine_node_intelligence_transition(payload_dir, entries) if touches_engine_credential_sink(manifest["component"], entries): validate_engine_credential_sink_slice(payload_dir, entries) if ( @@ -2902,6 +3462,11 @@ def is_platform_provider_catalog_only(entries): def component_services(component, entries=None): + if is_engine_node_intelligence_transition(component, entries): + if tuple(entries) == ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES: + return (ENGINE_NODE_INTELLIGENCE_SERVICE, "nodedc-backend") + return ("nodedc-backend",) + if is_engine_n8n_transition(component, entries): # The actual Engine topology has one n8n process and no separately # deployed worker/webhook services. The release barrier therefore @@ -3088,6 +3653,15 @@ def component_compose_files(component, allow_prepared_engine_backend=False): # Canonical order is immutable: base, active L2 extension, credential # sink, then the additive Publish grant overlay. files.append(publish_grant_override) + node_intelligence_descriptor = current_engine_node_intelligence_descriptor() + if ( + node_intelligence_descriptor is not None + and node_intelligence_descriptor["action"] == "activate" + ): + validate_installed_engine_node_intelligence_source( + node_intelligence_descriptor + ) + files.append(root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL) return tuple(files) files = COMPONENTS[component].get("compose_files", ()) if component == "platform": @@ -3266,6 +3840,201 @@ def inspect_optional_local_image(image_ref, label): return image_id +def ensure_engine_node_intelligence_secret(): + try: + runtime_stat = ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR.lstat() + except FileNotFoundError: + ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR.mkdir(parents=False, exist_ok=False) + runtime_stat = ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR.lstat() + if stat.S_ISLNK(runtime_stat.st_mode) or not stat.S_ISDIR(runtime_stat.st_mode): + die("Engine node-intelligence runtime directory is unsafe") + os.chown( + ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR, + 0, + ENGINE_NODE_INTELLIGENCE_RUNTIME_GID, + ) + ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR.chmod(0o710) + + try: + secret_stat = ENGINE_NODE_INTELLIGENCE_SECRET_FILE.lstat() + except FileNotFoundError: + value = secrets.token_urlsafe(48) + temporary = ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR / ( + f".{ENGINE_NODE_INTELLIGENCE_SECRET_FILE.name}." + f"{os.getpid()}.{time.time_ns()}.tmp" + ) + descriptor = None + try: + descriptor = os.open( + str(temporary), + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o440, + ) + os.write(descriptor, f"{value}\n".encode("ascii")) + os.fsync(descriptor) + os.fchown(descriptor, 0, ENGINE_NODE_INTELLIGENCE_RUNTIME_GID) + os.fchmod(descriptor, 0o440) + os.close(descriptor) + descriptor = None + os.replace(temporary, ENGINE_NODE_INTELLIGENCE_SECRET_FILE) + fsync_directory(ENGINE_NODE_INTELLIGENCE_RUNTIME_DIR) + finally: + if descriptor is not None: + os.close(descriptor) + if temporary.exists(): + temporary.unlink() + return "created" + + if ( + stat.S_ISLNK(secret_stat.st_mode) + or not stat.S_ISREG(secret_stat.st_mode) + or secret_stat.st_uid != 0 + or secret_stat.st_gid != ENGINE_NODE_INTELLIGENCE_RUNTIME_GID + or stat.S_IMODE(secret_stat.st_mode) != 0o440 + or secret_stat.st_size < 49 + or secret_stat.st_size > 512 + ): + die("Engine node-intelligence secret file is unsafe") + try: + value = ENGINE_NODE_INTELLIGENCE_SECRET_FILE.read_text( + encoding="ascii" + ).strip() + except (OSError, UnicodeDecodeError): + die("Engine node-intelligence secret file is unreadable") + if not ENGINE_NODE_INTELLIGENCE_SECRET_RE.fullmatch(value): + die("Engine node-intelligence secret file has invalid format") + return "reused" + + +def inspect_engine_node_intelligence_image(descriptor, required=True): + result = subprocess.run( + [str(DOCKER), "image", "inspect", ENGINE_NODE_INTELLIGENCE_IMAGE], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0: + if not required: + return None + die("Engine node-intelligence image is missing") + try: + images = json.loads(result.stdout) + except json.JSONDecodeError: + die("Engine node-intelligence image inspect returned invalid JSON") + if not isinstance(images, list) or len(images) != 1 or not isinstance(images[0], dict): + die("Engine node-intelligence image inspect shape mismatch") + image = images[0] + image_id = image.get("Id") + config = image.get("Config") or {} + labels = config.get("Labels") or {} + repo_tags = image.get("RepoTags") or [] + if ( + not isinstance(image_id, str) + or not re.fullmatch(r"sha256:[a-f0-9]{64}", image_id) + or image.get("Architecture") != descriptor["image"]["architecture"] + or image.get("Os") != descriptor["image"]["os"] + or ENGINE_NODE_INTELLIGENCE_IMAGE not in repo_tags + or not isinstance(labels, dict) + or labels.get("org.opencontainers.image.revision") + != ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT + or config.get("Entrypoint") != ["/usr/local/bin/docker-entrypoint.sh"] + or config.get("Cmd") != ["node", "dist/mcp/index.js"] + ): + die("Engine node-intelligence image identity mismatch") + env = config.get("Env") or [] + if any( + str(value).startswith(("AUTH_TOKEN=", "N8N_API_URL=", "N8N_API_KEY=")) + for value in env + ): + die("Engine node-intelligence image contains runtime authority") + return image + + +def install_engine_node_intelligence_image(descriptor): + if descriptor.get("action") != "activate": + die("Engine node-intelligence image install requires activation") + source_archive = ( + component_root("engine") / ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL + ) + validate_engine_node_intelligence_image_archive(source_archive, descriptor) + + try: + releases_stat = ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.lstat() + except FileNotFoundError: + ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.mkdir(parents=False, exist_ok=False) + releases_stat = ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.lstat() + if stat.S_ISLNK(releases_stat.st_mode) or not stat.S_ISDIR(releases_stat.st_mode): + die("Engine node-intelligence releases directory is unsafe") + os.chown(ENGINE_NODE_INTELLIGENCE_RELEASES_DIR, 0, 0) + ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.chmod(0o500) + + release_dir = ( + ENGINE_NODE_INTELLIGENCE_RELEASES_DIR / descriptor["releaseId"] + ) + try: + release_stat = release_dir.lstat() + except FileNotFoundError: + ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.chmod(0o700) + try: + release_dir.mkdir(mode=0o500, exist_ok=False) + finally: + ENGINE_NODE_INTELLIGENCE_RELEASES_DIR.chmod(0o500) + release_stat = release_dir.lstat() + if ( + stat.S_ISLNK(release_stat.st_mode) + or not stat.S_ISDIR(release_stat.st_mode) + or release_stat.st_uid != 0 + ): + die("Engine node-intelligence release directory is unsafe") + os.chown(release_dir, 0, 0) + release_dir.chmod(0o500) + + release_archive = release_dir / "engine-node-intelligence.tar" + if release_archive.exists() or release_archive.is_symlink(): + release_archive_stat = release_archive.lstat() + if ( + stat.S_ISLNK(release_archive_stat.st_mode) + or not stat.S_ISREG(release_archive_stat.st_mode) + or release_archive_stat.st_uid != 0 + or release_archive_stat.st_gid != 0 + or stat.S_IMODE(release_archive_stat.st_mode) != 0o400 + or sha256_file(release_archive) != descriptor["image"]["archiveSha256"] + ): + die("Engine node-intelligence sealed release collision") + else: + release_dir.chmod(0o700) + temporary = release_dir / ( + f".engine-node-intelligence.tar.{os.getpid()}.{time.time_ns()}.tmp" + ) + try: + with source_archive.open("rb") as source, temporary.open("xb") as output: + shutil.copyfileobj(source, output, 1024 * 1024) + output.flush() + os.fsync(output.fileno()) + os.chown(temporary, 0, 0) + temporary.chmod(0o400) + if sha256_file(temporary) != descriptor["image"]["archiveSha256"]: + die("Engine node-intelligence sealed release copy mismatch") + os.replace(temporary, release_archive) + fsync_directory(release_dir) + finally: + if temporary.exists(): + temporary.unlink() + release_dir.chmod(0o500) + + load = subprocess.run( + [str(DOCKER), "image", "load", "--input", str(release_archive)], + check=False, + capture_output=True, + text=True, + timeout=600, + ) + if load.returncode != 0: + die("Engine node-intelligence offline image load failed") + image = inspect_engine_node_intelligence_image(descriptor) + return image["Id"] + + def engine_backend_container_id(): result = subprocess.run( [*compose_base_cmd("engine"), "ps", "-q", "nodedc-backend"], @@ -3426,6 +4195,60 @@ def validate_engine_backend_mounts_for_installed_runtime( return validate_engine_backend_mounts(projected, node_modules_read_only) +def validate_engine_backend_node_intelligence_mounts_for_installed_runtime( + container, + node_modules_read_only, +): + descriptor = current_engine_node_intelligence_descriptor() + active = descriptor is not None and descriptor["action"] == "activate" + if not active: + # The pre-existing runtime guard remains authoritative when this + # independent overlay is inactive. Any stray secret mount therefore + # still fails its exact baseline inventory. + return validate_engine_backend_mounts_for_installed_runtime( + container, + node_modules_read_only, + ) + + validate_installed_engine_node_intelligence_source(descriptor) + mounts = container.get("Mounts") + if not isinstance(mounts, list): + die("Engine backend mount inventory is missing") + baseline_mounts = [] + observed = 0 + for mount in mounts: + if not isinstance(mount, dict) or not isinstance(mount.get("Destination"), str): + die("Engine backend mount inventory is invalid") + if mount["Destination"] != ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH: + baseline_mounts.append(mount) + continue + observed += 1 + actual = ( + mount.get("Type"), + bool(mount.get("RW")), + mount.get("Source"), + ) + expected = ( + "bind", + False, + str(ENGINE_NODE_INTELLIGENCE_SECRET_FILE), + ) + if observed != 1 or actual != expected: + die("Engine backend node-intelligence mount barrier mismatch") + if observed != 1: + die("Engine backend node-intelligence mount inventory mismatch") + + # Remove only the independently proven additive mount, then delegate the + # remaining inventory to the existing Publish-aware guard and ultimately + # the byte-protected legacy baseline validator. + projected = dict(container) + projected["Mounts"] = baseline_mounts + return validate_engine_backend_mounts_for_installed_runtime( + projected, + node_modules_read_only, + ) + + def validate_engine_backend_dependencies(container_id): package_path = component_root("engine") / "nodedc-source/package.json" try: @@ -3859,7 +4682,7 @@ def preflight_engine_credential_backend_runtime(): die("Engine backend source container is not stable and healthy") config_image = (container.get("Config") or {}).get("Image") current_image_id = container.get("Image") - validate_engine_backend_mounts_for_installed_runtime( + validate_engine_backend_node_intelligence_mounts_for_installed_runtime( container, node_modules_read_only=config_image == ENGINE_CREDENTIAL_BACKEND_IMAGE, ) @@ -4734,6 +5557,7 @@ def plan_artifact(artifact): sha = sha256_file(artifact) publish_grant_preflight = None + node_intelligence_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) transition_descriptor = None @@ -4750,6 +5574,10 @@ def plan_artifact(artifact): publish_grant_preflight = ( preflight_engine_data_product_publish_grant_predecessor(payload_dir) ) + if is_engine_node_intelligence_transition(manifest["component"], entries): + node_intelligence_preflight = preflight_engine_node_intelligence_predecessor( + payload_dir + ) component = manifest["component"] root = component_root(component) @@ -4800,6 +5628,22 @@ def plan_artifact(artifact): print(f"build_root={build_root}") print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}") print(f"services={' '.join(services)}") + if node_intelligence_preflight: + descriptor = node_intelligence_preflight["descriptor"] + print(f"node_intelligence_transition={descriptor['action']}") + print(f"node_intelligence_release={descriptor['releaseId']}") + print(f"node_intelligence_current_state={node_intelligence_preflight['current_state']}") + print(f"node_intelligence_upstream_commit={descriptor['upstream']['commit']}") + print(f"node_intelligence_image={descriptor['image']['tag']}") + print(f"node_intelligence_image_config_sha256={descriptor['image']['configSha256']}") + if descriptor["action"] == "activate": + print(f"node_intelligence_image_archive_sha256={descriptor['image']['archiveSha256']}") + print("node_intelligence_build=offline-image-load") + print("node_intelligence_pull=never") + print(f"node_intelligence_secret=runner-managed:{ENGINE_NODE_INTELLIGENCE_SECRET_FILE}") + print("node_intelligence_host_ports=none") + print(f"predecessor_gateway_sha256={node_intelligence_preflight['gateway_sha256']}") + print(f"backend_current_barrier={node_intelligence_preflight['backend_mode']}") if transition_descriptor: print(f"n8n_transition={transition_descriptor['action']}") print(f"n8n_version={transition_descriptor['n8nVersion']}") @@ -5307,6 +6151,63 @@ def rollback_engine_credential_bridge( return "engine-sink-source-restored" +def rollback_engine_node_intelligence( + root, + backup_dir, + entries, + current_stamp, + runtime_started=False, + node_intelligence_service_stopped=False, +): + candidate_descriptor = current_engine_node_intelligence_descriptor() + candidate_active = ( + candidate_descriptor is not None + and candidate_descriptor["action"] == "activate" + ) + cleanup_failed = False + if runtime_started and candidate_active: + try: + stop_and_remove_compose_services( + "engine", + (ENGINE_NODE_INTELLIGENCE_SERVICE,), + ) + except Exception: + cleanup_failed = True + + restore_overlay_source(root, backup_dir, entries, current_stamp) + restored = current_engine_node_intelligence_descriptor() + restored_action = restored["action"] if restored is not None else "inactive" + if cleanup_failed: + die("Engine node-intelligence candidate runtime cleanup failed after source restore") + + reconcile_runtime = runtime_started or node_intelligence_service_stopped + if not reconcile_runtime: + return f"source-restored-runtime-unchanged:{restored_action}" + + if restored is not None and restored["action"] == "activate": + validate_installed_engine_node_intelligence_source(restored) + ensure_engine_node_intelligence_secret() + install_engine_node_intelligence_image(restored) + run_engine_node_intelligence_compose( + (ENGINE_NODE_INTELLIGENCE_SERVICE, "nodedc-backend"), + ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + accept_engine_node_intelligence_runtime(restored) + return "active" + + run_engine_node_intelligence_compose( + ("nodedc-backend",), + ENGINE_NODE_INTELLIGENCE_ROLLBACK_ENTRIES, + ) + healthcheck_compose_service("engine", "nodedc-backend") + healthcheck_url("http://127.0.0.1:3001/health") + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die("Engine node-intelligence rollback backend barrier failed") + assert_engine_node_intelligence_container_absent() + return restored_action + + def seal_n8n_private_extension_release(root, rel): release_dir = root / rel if not release_dir.is_dir() or release_dir.is_symlink(): @@ -5434,6 +6335,42 @@ def run_compose(component, services, entries=None): ) +def run_engine_node_intelligence_compose(services, entries): + if not is_engine_node_intelligence_transition("engine", entries): + die("Engine node-intelligence Compose called for unrelated artifact") + compose_root = component_compose_root("engine") + cmd = [ + *compose_base_cmd("engine"), + "up", + "-d", + "--force-recreate", + "--pull", + "never", + "--no-deps", + *services, + ] + try: + subprocess.run(cmd, cwd=str(compose_root), check=True) + except subprocess.CalledProcessError: + subprocess.run( + [ + *compose_base_cmd("engine"), + "logs", + "--no-color", + "--tail=180", + *services, + ], + cwd=str(compose_root), + check=False, + ) + raise + subprocess.run( + [*compose_base_cmd("engine"), "ps"], + cwd=str(compose_root), + check=True, + ) + + def stop_and_remove_compose_services(component, services): if not services: return @@ -5458,6 +6395,15 @@ def prepare_component_runtime(component, entries=None): ensure_engine_n8n_sealed_release(descriptor, stamp()) return + if is_engine_node_intelligence_transition(component, entries): + ensure_engine_node_intelligence_secret() + descriptor = current_engine_node_intelligence_descriptor() + if descriptor is None: + die("installed Engine node-intelligence descriptor is missing") + if descriptor["action"] == "activate": + install_engine_node_intelligence_image(descriptor) + return + if touches_engine_credential_sink(component, entries): ensure_engine_credential_issuer_keypair() prepare_engine_credential_backend_runtime() @@ -5549,7 +6495,10 @@ def run_component_runtime(component, entries, services): return run_build(component, entries) prepare_component_runtime(component, entries) - run_compose(component, services, entries) + if is_engine_node_intelligence_transition(component, entries): + run_engine_node_intelligence_compose(services, entries) + else: + run_compose(component, services, entries) def healthcheck_url(check): @@ -5616,6 +6565,11 @@ def component_healthchecks(component, entries=None, services=None): # Its own acceptance below verifies n8n readiness, image, mount, logs, # live node export and the pinned Engine MCP catalogs. return () + if is_engine_node_intelligence_transition(component, entries): + # The exact sidecar/backend topology and live MCP capability calls are + # accepted separately below. Keep the generic HTTP barrier scoped to + # the backend generation recreated by this transition. + return ("http://127.0.0.1:3001/health",) if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): return () if ( @@ -5685,7 +6639,7 @@ def healthcheck_container(container_name): die(f"container healthcheck failed for {container_name}: {last_status}") -def healthcheck_compose_service(component, service): +def compose_service_container_id(component, service): result = subprocess.run( [*compose_base_cmd(component), "ps", "-q", service], cwd=str(component_compose_root(component)), @@ -5694,9 +6648,167 @@ def healthcheck_compose_service(component, service): text=True, ) container_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()] - if result.returncode != 0 or len(container_ids) != 1: + if ( + result.returncode != 0 + or len(container_ids) != 1 + or not re.fullmatch(r"[a-f0-9]{12,64}", container_ids[0]) + ): die(f"Compose service container lookup failed: {service}") - healthcheck_container(container_ids[0]) + return container_ids[0] + + +def healthcheck_compose_service(component, service): + healthcheck_container(compose_service_container_id(component, service)) + + +def container_environment(container, label): + environment = {} + for raw in (container.get("Config") or {}).get("Env") or []: + if not isinstance(raw, str) or "=" not in raw: + die(f"{label} environment inventory is invalid") + key, value = raw.split("=", 1) + if not key or key in environment: + die(f"{label} environment inventory is invalid") + environment[key] = value + return environment + + +def validate_engine_node_intelligence_secret_mount(container, label): + matches = [ + mount + for mount in (container.get("Mounts") or []) + if isinstance(mount, dict) + and mount.get("Destination") == ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH + ] + if len(matches) != 1: + die(f"{label} secret mount inventory mismatch") + mount = matches[0] + if ( + mount.get("Type") != "bind" + or mount.get("Source") != str(ENGINE_NODE_INTELLIGENCE_SECRET_FILE) + or mount.get("RW") is not False + ): + die(f"{label} secret mount boundary mismatch") + + +def assert_engine_node_intelligence_container_absent(): + result = subprocess.run( + [ + str(DOCKER), + "container", + "ls", + "--all", + "--filter", + f"label=com.docker.compose.service={ENGINE_NODE_INTELLIGENCE_SERVICE}", + "--format", + "{{.ID}}", + ], + check=False, + capture_output=True, + text=True, + ) + if result.returncode != 0 or result.stdout.strip(): + die("Engine node-intelligence inactive runtime still has a sidecar container") + + +def engine_node_intelligence_live_probe_script(): + return """ +const {createNodeIntelligenceMcpClient}=await import('file:///app/server/nodeIntelligence/upstreamMcpClient.js'); +const client=createNodeIntelligenceMcpClient(); +const status=await client.status(); +if(!status.ok||status.status!=='ready'||status.authSource!=='file'||status.server?.version!=='2.33.2'||!Object.values(status.capabilities||{}).every(Boolean))throw new Error('status'); +const guidance=await client.callTool('get_node',{nodeType:'nodes-base.httpRequest',detail:'minimal'}); +if(!guidance||typeof guidance!=='object')throw new Error('guidance'); +const nodeValidation=await client.callTool('validate_node',{nodeType:'nodes-base.httpRequest',config:{url:'https://example.com'},mode:'minimal',profile:'ai-friendly'}); +if(!nodeValidation||typeof nodeValidation!=='object'||nodeValidation.valid!==true)throw new Error('node-validation'); +const workflowValidation=await client.callTool('validate_workflow',{workflow:{name:'NDC acceptance',nodes:[{id:'probe-1',name:'Start',type:'nodes-base.manualTrigger',typeVersion:1,position:[0,0],parameters:{}},{id:'probe-2',name:'Output',type:'nodes-base.set',typeVersion:3.4,position:[240,0],parameters:{}}],connections:{Start:{main:[[{node:'Output',type:'main',index:0}]]}},settings:{}},options:{validateNodes:true,validateConnections:true,validateExpressions:true,profile:'runtime'}}); +if(!workflowValidation||typeof workflowValidation!=='object'||workflowValidation.valid!==true)throw new Error('workflow-validation'); +process.stdout.write('node-intelligence-live:2.33.2:guidance+node+workflow'); +""".strip() + + +def accept_engine_node_intelligence_runtime(descriptor): + if descriptor is None or descriptor.get("action") != "activate": + die("Engine node-intelligence activation descriptor is required") + validate_installed_engine_node_intelligence_source(descriptor) + ensure_engine_node_intelligence_secret() + image = inspect_engine_node_intelligence_image(descriptor) + + sidecar_id = compose_service_container_id( + "engine", + ENGINE_NODE_INTELLIGENCE_SERVICE, + ) + backend_id = compose_service_container_id("engine", "nodedc-backend") + healthcheck_container(sidecar_id) + healthcheck_container(backend_id) + sidecar = inspect_container(sidecar_id) + backend = inspect_container(backend_id) + + sidecar_config = sidecar.get("Config") or {} + sidecar_host = sidecar.get("HostConfig") or {} + sidecar_environment = container_environment( + sidecar, + "Engine node-intelligence sidecar", + ) + if ( + sidecar.get("Image") != image["Id"] + or sidecar_config.get("Image") != ENGINE_NODE_INTELLIGENCE_IMAGE + or sidecar_config.get("User") + != f"{ENGINE_NODE_INTELLIGENCE_RUNTIME_UID}:{ENGINE_NODE_INTELLIGENCE_RUNTIME_GID}" + or sidecar_host.get("ReadonlyRootfs") is not True + or sidecar_host.get("PortBindings") not in (None, {}) + or set(sidecar_host.get("CapDrop") or []) != {"ALL"} + or "no-new-privileges:true" not in (sidecar_host.get("SecurityOpt") or []) + or sidecar_environment.get("AUTH_TOKEN_FILE") + != ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH + or sidecar_environment.get("NODE_DB_PATH") != "/app/data/nodes.db" + or sidecar_environment.get("REBUILD_ON_START") != "false" + or sidecar_environment.get("N8N_MCP_TELEMETRY_DISABLED") != "true" + ): + die("Engine node-intelligence sidecar runtime contract mismatch") + if any( + key in sidecar_environment + for key in ("AUTH_TOKEN", "N8N_API_URL", "N8N_API_KEY") + ): + die("Engine node-intelligence sidecar runtime authority leak") + validate_engine_node_intelligence_secret_mount( + sidecar, + "Engine node-intelligence sidecar", + ) + + backend_environment = container_environment(backend, "Engine backend") + if ( + backend_environment.get("ENGINE_NODE_INTELLIGENCE_MCP_URL") + != f"http://{ENGINE_NODE_INTELLIGENCE_SERVICE}:3000/mcp" + or backend_environment.get("ENGINE_NODE_INTELLIGENCE_AUTH_TOKEN_FILE") + != ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH + or backend_environment.get("ENGINE_NODE_INTELLIGENCE_ASSET_ID") + != f"nodedc-node-intelligence@{ENGINE_NODE_INTELLIGENCE_RELEASE_ID.replace('-', '+', 1)}" + or "ENGINE_NODE_INTELLIGENCE_AUTH_TOKEN" in backend_environment + ): + die("Engine backend node-intelligence boundary mismatch") + validate_engine_node_intelligence_secret_mount( + backend, + "Engine backend node-intelligence", + ) + + result = run_engine_backend_probe( + ( + "node", + "--input-type=module", + "-e", + engine_node_intelligence_live_probe_script(), + ), + "node-intelligence live capability", + container_id=backend_id, + ) + expected = "node-intelligence-live:2.33.2:guidance+node+workflow" + if result != expected: + die("Engine node-intelligence live capability acceptance mismatch") + backend_runtime = preflight_engine_credential_backend_runtime() + if backend_runtime["mode"] != "verified-derived-retry": + die("Engine node-intelligence backend immutable runtime acceptance failed") + return result def accept_engine_agent_full_grant_migration_runtime(): @@ -5750,6 +6862,24 @@ def run_healthchecks(component, entries=None, services=None): die("installed Engine n8n transition descriptor is missing during acceptance") accept_engine_n8n_runtime(descriptor) return + if is_engine_node_intelligence_transition(component, entries): + descriptor = current_engine_node_intelligence_descriptor() + if descriptor is None: + die( + "installed Engine node-intelligence transition descriptor " + "is missing during acceptance" + ) + if descriptor["action"] == "activate": + accept_engine_node_intelligence_runtime(descriptor) + healthcheck_url("http://127.0.0.1:3001/health") + return + healthcheck_compose_service("engine", "nodedc-backend") + healthcheck_url("http://127.0.0.1:3001/health") + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die("Engine node-intelligence rollback backend acceptance failed") + assert_engine_node_intelligence_container_absent() + return touches_publish_grant = is_engine_data_product_publish_grant_slice( component, entries, @@ -5808,6 +6938,8 @@ def apply_artifact(artifact): root = None services = None transition_descriptor = None + node_intelligence_descriptor = None + node_intelligence_service_stopped = False apply_started = False engine_backend_recreated = False engine_backend_initial_mode = None @@ -5846,6 +6978,11 @@ def apply_artifact(artifact): migration_backend_preflight = preflight_engine_credential_backend_runtime() if migration_backend_preflight["mode"] != "verified-derived-retry": die("Engine agent full grant migration requires the active immutable credential backend") + if is_engine_node_intelligence_transition(component, entries): + node_intelligence_preflight = preflight_engine_node_intelligence_predecessor( + payload_dir + ) + node_intelligence_descriptor = node_intelligence_preflight["descriptor"] if not artifact_only and not compose_root.is_dir(): if bootstrap_root and is_relative_to(compose_root.resolve(strict=False), root.resolve(strict=False)): compose_root.mkdir(parents=True, exist_ok=True) @@ -5909,6 +7046,15 @@ def apply_artifact(artifact): ) apply_started = True + if ( + node_intelligence_descriptor is not None + and node_intelligence_descriptor["action"] == "rollback-inactive" + ): + stop_and_remove_compose_services( + "engine", + (ENGINE_NODE_INTELLIGENCE_SERVICE,), + ) + node_intelligence_service_stopped = True for rel in entries: copy_payload_path(payload_dir, root, rel, current_stamp) @@ -5966,6 +7112,30 @@ def apply_artifact(artifact): except Exception as rollback_exc: rollback_status = f"failed:{type(rollback_exc).__name__}" print("engine-n8n-automatic-rollback=failed", file=sys.stderr) + elif ( + node_intelligence_descriptor is not None + and is_engine_node_intelligence_transition(component, entries) + ): + try: + restored_action = rollback_engine_node_intelligence( + root, + backup_dir, + entries, + current_stamp, + runtime_started=runtime_started, + node_intelligence_service_stopped=node_intelligence_service_stopped, + ) + rollback_status = f"ok:{restored_action}" + print( + f"engine-node-intelligence-automatic-rollback={rollback_status}", + file=sys.stderr, + ) + except Exception as rollback_exc: + rollback_status = f"failed:{type(rollback_exc).__name__}" + print( + "engine-node-intelligence-automatic-rollback=failed", + file=sys.stderr, + ) elif touches_engine_credential_sink(component, entries): try: restored_action = rollback_engine_credential_bridge( diff --git a/infra/deploy-runner/test_engine_node_intelligence.py b/infra/deploy-runner/test_engine_node_intelligence.py new file mode 100644 index 0000000..7899117 --- /dev/null +++ b/infra/deploy-runner/test_engine_node_intelligence.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +import hashlib +import importlib.machinery +import importlib.util +import json +import shutil +import tarfile +import tempfile +import unittest +from io import BytesIO +from pathlib import Path +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +PLATFORM_ROOT = SCRIPT_DIR.parent.parent +ENGINE_ROOT = PLATFORM_ROOT.parent / "NODEDC_ENGINE_INFRA" +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_engine_node_intelligence_under_test", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class EngineNodeIntelligenceTest(unittest.TestCase): + def make_image_archive(self, root, runtime_env=None): + runtime_env = list(runtime_env or ["PATH=/usr/local/bin"]) + config = { + "architecture": "amd64", + "os": "linux", + "config": { + "Labels": { + "org.opencontainers.image.revision": RUNNER.ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT, + }, + "Entrypoint": ["/usr/local/bin/docker-entrypoint.sh"], + "Cmd": ["node", "dist/mcp/index.js"], + "Env": runtime_env, + }, + } + config_raw = json.dumps(config, separators=(",", ":")).encode() + config_sha = hashlib.sha256(config_raw).hexdigest() + layer_raw = b"nodedc-node-intelligence-test-layer\n" * 64 + layer_sha = hashlib.sha256(layer_raw).hexdigest() + manifest = [{ + "Config": f"blobs/sha256/{config_sha}", + "RepoTags": [RUNNER.ENGINE_NODE_INTELLIGENCE_IMAGE], + "Layers": [f"blobs/sha256/{layer_sha}"], + }] + archive = root / "engine-node-intelligence.tar" + with tarfile.open(archive, "w") as output: + for name, value in ( + ("manifest.json", json.dumps(manifest, separators=(",", ":")).encode()), + (f"blobs/sha256/{config_sha}", config_raw), + (f"blobs/sha256/{layer_sha}", layer_raw), + ): + member = tarfile.TarInfo(name) + member.size = len(value) + member.mode = 0o644 + output.addfile(member, BytesIO(value)) + return archive, config_sha + + def make_activation_payload(self, root, runtime_env=None): + payload = root / "payload" + source = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_SOURCE_REL + service = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + gateway = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + source.mkdir(parents=True) + service.joinpath("image").mkdir(parents=True) + gateway.parent.mkdir(parents=True) + for name in ("catalog.js", "upstreamMcpClient.js", "upstreamProjection.js"): + shutil.copy2( + ENGINE_ROOT / RUNNER.ENGINE_NODE_INTELLIGENCE_SOURCE_REL / name, + source / name, + ) + shutil.copy2( + ENGINE_ROOT / RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL, + gateway, + ) + readme = service / "README.md" + readme.write_text("# exact node intelligence test\n", encoding="utf-8") + override = service / "docker-compose.immutable-runtime.yml" + override.write_text( + RUNNER.expected_engine_node_intelligence_compose_override(), + encoding="utf-8", + ) + archive, config_sha = self.make_image_archive(root, runtime_env) + target_archive = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL + shutil.copy2(archive, target_archive) + descriptor = { + "schemaVersion": "nodedc.engine-node-intelligence-transition/v1", + "action": "activate", + "releaseId": RUNNER.ENGINE_NODE_INTELLIGENCE_RELEASE_ID, + "expectedCurrent": "inactive", + "upstream": { + "package": "n8n-mcp", + "version": "2.33.2", + "commit": RUNNER.ENGINE_NODE_INTELLIGENCE_UPSTREAM_COMMIT, + }, + "image": { + "tag": RUNNER.ENGINE_NODE_INTELLIGENCE_IMAGE, + "archiveRelativePath": RUNNER.ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL, + "archiveSha256": sha256(target_archive), + "configSha256": config_sha, + "architecture": "amd64", + "os": "linux", + }, + "source": { + "gatewaySha256": sha256(gateway), + "catalogSha256": sha256(source / "catalog.js"), + "upstreamClientSha256": sha256(source / "upstreamMcpClient.js"), + "upstreamProjectionSha256": sha256(source / "upstreamProjection.js"), + "composeOverrideSha256": sha256(override), + "readmeSha256": sha256(readme), + }, + "predecessor": { + "gatewaySha256": RUNNER.ENGINE_NODE_INTELLIGENCE_PREDECESSOR_GATEWAY_SHA256, + "composeSha256": RUNNER.ENGINE_NODE_INTELLIGENCE_PREDECESSOR_COMPOSE_SHA256, + "backendRuntime": "verified-derived-retry", + }, + } + (payload / RUNNER.ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL).write_text( + json.dumps(descriptor, indent=2) + "\n", + encoding="utf-8", + ) + return payload, descriptor + + def test_activation_payload_and_offline_image_are_exact(self): + with tempfile.TemporaryDirectory() as directory: + payload, descriptor = self.make_activation_payload(Path(directory)) + actual = RUNNER.validate_engine_node_intelligence_transition( + payload, + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + self.assertEqual(actual, descriptor) + + def test_archive_tamper_and_embedded_authority_fail_closed(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload, descriptor = self.make_activation_payload(root) + archive = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_IMAGE_ARCHIVE_REL + archive.write_bytes(archive.read_bytes() + b"tamper") + with self.assertRaisesRegex(RUNNER.DeployError, "archive sha256 mismatch"): + RUNNER.validate_engine_node_intelligence_transition( + payload, + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + + with tempfile.TemporaryDirectory() as directory: + payload, _descriptor = self.make_activation_payload( + Path(directory), + runtime_env=["AUTH_TOKEN=forbidden"], + ) + with self.assertRaisesRegex(RUNNER.DeployError, "contains runtime authority"): + RUNNER.validate_engine_node_intelligence_transition( + payload, + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + + def test_payload_extra_file_and_wrong_entry_order_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + payload, _descriptor = self.make_activation_payload(Path(directory)) + service = payload / RUNNER.ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL + (service / "unexpected.txt").write_text("no\n", encoding="utf-8") + with self.assertRaisesRegex(RUNNER.DeployError, "service payload file set mismatch"): + RUNNER.validate_engine_node_intelligence_transition( + payload, + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + with tempfile.TemporaryDirectory() as directory: + payload, _descriptor = self.make_activation_payload(Path(directory)) + with self.assertRaisesRegex(RUNNER.DeployError, "files.txt exact set/order mismatch"): + RUNNER.validate_engine_node_intelligence_transition( + payload, + tuple(reversed(RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES)), + ) + + def test_predecessor_drift_is_rejected_before_apply(self): + with tempfile.TemporaryDirectory() as payload_dir, tempfile.TemporaryDirectory() as live_dir: + payload, _descriptor = self.make_activation_payload(Path(payload_dir)) + live = Path(live_dir) + (live / "nodedc-source/server/routes").mkdir(parents=True) + (live / "docker-compose.yml").write_bytes( + (ENGINE_ROOT / "docker-compose.yml").read_bytes() + ) + baseline = bytes.fromhex("00") + gateway = live / RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + gateway.write_bytes(baseline) + original_root = RUNNER.COMPONENTS["engine"]["payload_root"] + RUNNER.COMPONENTS["engine"]["payload_root"] = live + try: + with mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value={"mode": "verified-derived-retry"}, + ): + with self.assertRaisesRegex(RUNNER.DeployError, "gateway drift detected"): + RUNNER.preflight_engine_node_intelligence_predecessor(payload) + finally: + RUNNER.COMPONENTS["engine"]["payload_root"] = original_root + + def test_service_selection_and_compose_dispatch_are_narrow(self): + self.assertEqual( + RUNNER.component_services( + "engine", + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ), + (RUNNER.ENGINE_NODE_INTELLIGENCE_SERVICE, "nodedc-backend"), + ) + self.assertEqual( + RUNNER.component_services( + "engine", + RUNNER.ENGINE_NODE_INTELLIGENCE_ROLLBACK_ENTRIES, + ), + ("nodedc-backend",), + ) + with ( + mock.patch.object(RUNNER, "run_build") as build, + mock.patch.object(RUNNER, "prepare_component_runtime") as prepare, + mock.patch.object(RUNNER, "run_engine_node_intelligence_compose") as exact_compose, + mock.patch.object(RUNNER, "run_compose") as legacy_compose, + ): + RUNNER.run_component_runtime( + "engine", + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + (RUNNER.ENGINE_NODE_INTELLIGENCE_SERVICE, "nodedc-backend"), + ) + build.assert_called_once() + prepare.assert_called_once() + exact_compose.assert_called_once() + legacy_compose.assert_not_called() + + def test_exact_compose_never_recreates_dependencies(self): + services = (RUNNER.ENGINE_NODE_INTELLIGENCE_SERVICE, "nodedc-backend") + with ( + mock.patch.object( + RUNNER, + "compose_base_cmd", + return_value=["docker", "compose", "-f", "exact.yml"], + ), + mock.patch.object( + RUNNER, + "component_compose_root", + return_value=Path("/exact-engine-root"), + ), + mock.patch.object(RUNNER.subprocess, "run") as execute, + ): + RUNNER.run_engine_node_intelligence_compose( + services, + RUNNER.ENGINE_NODE_INTELLIGENCE_ARTIFACT_ENTRIES, + ) + command = execute.call_args_list[0].args[0] + self.assertIn("--pull", command) + self.assertIn("never", command) + self.assertIn("--no-deps", command) + self.assertEqual(command[-2:], list(services)) + + def test_backend_additive_mount_projects_into_legacy_guard(self): + descriptor = {"action": "activate"} + baseline_mounts = [ + { + "Type": "bind", + "Source": "/baseline/app", + "Destination": "/app", + "RW": True, + }, + ] + node_mount = { + "Type": "bind", + "Source": str(RUNNER.ENGINE_NODE_INTELLIGENCE_SECRET_FILE), + "Destination": RUNNER.ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH, + "RW": False, + } + container = {"Mounts": [*baseline_mounts, node_mount]} + with ( + mock.patch.object( + RUNNER, + "current_engine_node_intelligence_descriptor", + return_value=descriptor, + ), + mock.patch.object( + RUNNER, + "validate_installed_engine_node_intelligence_source", + ) as source_guard, + mock.patch.object( + RUNNER, + "validate_engine_backend_mounts_for_installed_runtime", + ) as legacy_guard, + ): + RUNNER.validate_engine_backend_node_intelligence_mounts_for_installed_runtime( + container, + node_modules_read_only=True, + ) + source_guard.assert_called_once_with(descriptor) + projected = legacy_guard.call_args.args[0] + self.assertEqual(projected["Mounts"], baseline_mounts) + self.assertTrue(legacy_guard.call_args.args[1]) + + def test_backend_additive_mount_rejects_source_or_mode_drift(self): + descriptor = {"action": "activate"} + for source, read_write in (("/wrong/source", False), (str(RUNNER.ENGINE_NODE_INTELLIGENCE_SECRET_FILE), True)): + container = { + "Mounts": [{ + "Type": "bind", + "Source": source, + "Destination": RUNNER.ENGINE_NODE_INTELLIGENCE_SECRET_CONTAINER_PATH, + "RW": read_write, + }], + } + with ( + mock.patch.object( + RUNNER, + "current_engine_node_intelligence_descriptor", + return_value=descriptor, + ), + mock.patch.object( + RUNNER, + "validate_installed_engine_node_intelligence_source", + ), + ): + with self.assertRaisesRegex( + RUNNER.DeployError, + "node-intelligence mount barrier mismatch", + ): + RUNNER.validate_engine_backend_node_intelligence_mounts_for_installed_runtime( + container, + node_modules_read_only=True, + ) + + +if __name__ == "__main__": + unittest.main()