diff --git a/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs b/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs new file mode 100644 index 0000000..2e66751 --- /dev/null +++ b/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs @@ -0,0 +1,231 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + copyFile, + cp, + lstat, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const platformRoot = resolve(here, "../.."); +const engineRoot = resolve( + process.env.NODEDC_ENGINE_SOURCE_ROOT || resolve(platformRoot, "../NODEDC_ENGINE_INFRA"), +); +const artifactRoot = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(here, "../deploy-artifacts"), +); +const [patchId = "", ...extra] = process.argv.slice(2); +if (extra.length || !/^engine-l2-closed-loop-\d{8}-\d{3}$/.test(patchId)) { + throw new Error( + "usage: build-engine-l2-closed-loop-artifact.mjs " + + "", + ); +} + +const sourceCommit = "dda405cff27977622af0c218abb4d4420c408536"; +const entries = Object.freeze([ + "nodedc-source/server/index.js", + "nodedc-source/server/l2/graphRepository.js", + "nodedc-source/server/realtime/ws.js", + "nodedc-source/server/routes/engineAgentGateway.js", + "nodedc-source/server/routes/n8n.js", + "nodedc-source/server/routes/ndcAgentMcp.js", + "nodedc-source/src/App.tsx", + "nodedc-source/src/n8n/N8nSubworkflowHost.tsx", + "nodedc-source/src/realtime/useMultiplayer.ts", + "nodedc-source/src/utils/n8nApi.ts", + "nodedc-source/dist/index.html", + "nodedc-source/dist/assets", +]); +const targetSha256 = Object.freeze({ + "nodedc-source/server/index.js": + "1896e1cade61579863c50ff3f52f2e81ff27f2a511a9d362db81925ee21cefad", + "nodedc-source/server/l2/graphRepository.js": + "94ad08e1bb7e7be04854f1f911e06ee1acd6e09631f33f4c01e42e230665ac33", + "nodedc-source/server/realtime/ws.js": + "82cfa833e05c2fc6d6049dd4564af06164b5c784fdfb82c243ca67519f4509c2", + "nodedc-source/server/routes/engineAgentGateway.js": + "4f600a2781be9118bec891fa2a6f20d7f55e0892ef2f06af24059193cebd28f4", + "nodedc-source/server/routes/n8n.js": + "752cb1524160adc1c95163c34e139b615c063d2ce8980f2a63879bddbd0f1e08", + "nodedc-source/server/routes/ndcAgentMcp.js": + "353a10291ceb93c3641ece60ec6e809a394be86f5ceb97cb44755178940409dd", + "nodedc-source/src/App.tsx": + "b3e61a89330b774f309660208d05143d299ab582f18765c1c14e2122a62c4d5e", + "nodedc-source/src/n8n/N8nSubworkflowHost.tsx": + "683aa44ba92aeac4879889e2bdb5319282976d7680d620701578830ce9677087", + "nodedc-source/src/realtime/useMultiplayer.ts": + "a4d001d9914098e50a78d8abe0a66344d23680b7a19b3573aa7b6f48c8bb9c35", + "nodedc-source/src/utils/n8nApi.ts": + "6f16d4992c51ffcc1e71a7bd172fd9ceb440d6e1a74d69ef1db62e0ac4230b3c", + "nodedc-source/dist/index.html": + "90d72f8790dc8cf951066b1210447c84d640373117bae23f3a4cb3227fb96d6d", + "nodedc-source/dist/assets/index-Bim2pv1P.css": + "18322addd45c126a7b8396f36b005f3085fddba3e9b346dd2c910f6fa6987ebf", + "nodedc-source/dist/assets/index-CqvJfRRS.js": + "06e6c23b03ea2ba8a4890e1f1714fd08ebaf18d735834200af6ab430af360ca8", +}); +const artifact = join(artifactRoot, `nodedc-${patchId}.tgz`); + +await assertFresh(artifact); +assertSourceCommit(); +await assertExactSources(); + +const stage = await mkdtemp(join(tmpdir(), "nodedc-engine-l2-closed-loop-")); +try { + const payload = join(stage, "payload"); + for (const relativePath of entries) { + const source = join(engineRoot, relativePath); + const destination = join(payload, relativePath); + await mkdir(dirname(destination), { recursive: true }); + const info = await lstat(source); + if (info.isSymbolicLink()) { + throw new Error(`engine_l2_closed_loop_source_symlink:${relativePath}`); + } + if (info.isDirectory()) { + await cp(source, destination, { + recursive: true, + force: false, + errorOnExist: true, + }); + } else if (info.isFile()) { + await copyFile(source, destination); + } else { + throw new Error(`engine_l2_closed_loop_source_unsafe:${relativePath}`); + } + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=engine\ntype=app-overlay\n`, + { encoding: "utf8", flag: "wx", mode: 0o644 }, + ); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o644, + }); + await mkdir(artifactRoot, { recursive: true }); + run("python3", ["-c", canonicalTarScript(), artifact, stage]); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact, + sha256: digest(await readFile(artifact)), + sourceCommit, + entries, + targetSha256, + services: ["nodedc-backend", "app"], + healthchecks: [ + "http://127.0.0.1:8080/", + "http://127.0.0.1:3001/health", + ], + mcpVersion: "0.7.0", + graphContract: "semantic-revision-cas-v1", + actorPlanes: ["external_codex_mcp", "ai_workspace", "engine_ui"], + untouched: [ + "L2 graph data", + "credentials", + "databases", + "n8n runtime", + "AI Workspace connector", + ], + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +function assertSourceCommit() { + const actual = run("git", ["-C", engineRoot, "rev-parse", "HEAD"]).stdout.trim(); + if (actual !== sourceCommit) { + throw new Error( + `engine_l2_closed_loop_source_commit_mismatch:expected=${sourceCommit}:actual=${actual}`, + ); + } +} + +async function assertExactSources() { + const expectedPaths = Object.keys(targetSha256).sort(); + const actualPaths = []; + for (const entry of entries) { + const source = join(engineRoot, entry); + const info = await lstat(source); + if (info.isSymbolicLink() || (!info.isFile() && !info.isDirectory())) { + throw new Error(`engine_l2_closed_loop_source_unsafe:${entry}`); + } + if (info.isDirectory()) { + const result = run("find", [source, "-type", "f", "-print"]); + for (const path of result.stdout.split("\n").filter(Boolean)) { + actualPaths.push(path.slice(engineRoot.length + 1)); + } + } else { + actualPaths.push(entry); + } + } + actualPaths.sort(); + if (JSON.stringify(actualPaths) !== JSON.stringify(expectedPaths)) { + throw new Error( + `engine_l2_closed_loop_source_set_mismatch:` + + `expected=${expectedPaths.join(",")}:actual=${actualPaths.join(",")}`, + ); + } + for (const relativePath of expectedPaths) { + const actual = digest(await readFile(join(engineRoot, relativePath))); + if (actual !== targetSha256[relativePath]) { + throw new Error( + `engine_l2_closed_loop_target_mismatch:${relativePath}:` + + `expected=${targetSha256[relativePath]}:actual=${actual}`, + ); + } + } +} + +async function assertFresh(path) { + try { + await lstat(path); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + throw new Error("artifact_already_exists"); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'xb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +function digest(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function run(command, args) { + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + } + return result; +} diff --git a/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py b/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py new file mode 100644 index 0000000..59a2d78 --- /dev/null +++ b/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py @@ -0,0 +1,139 @@ +import hashlib +import importlib.machinery +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +BUILDER_PATH = SCRIPT_DIR / "build-engine-l2-closed-loop-artifact.mjs" +ENGINE_ROOT = SCRIPT_DIR.parent.parent.parent / "NODEDC_ENGINE_INFRA" +PATCH_ID = "engine-l2-closed-loop-20991231-999" +SOURCE_COMMIT = "dda405cff27977622af0c218abb4d4420c408536" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_engine_l2_closed_loop", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class EngineL2ClosedLoopArtifactTest(unittest.TestCase): + def require_target_commit(self): + current = subprocess.run( + ["git", "-C", str(ENGINE_ROOT), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if current != SOURCE_COMMIT: + self.skipTest("historical Engine L2 closed-loop source has advanced") + + def build(self, artifact_dir): + self.require_target_commit() + environment = os.environ.copy() + environment["NODEDC_ENGINE_SOURCE_ROOT"] = str(ENGINE_ROOT) + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + completed = subprocess.run( + ["node", str(BUILDER_PATH), PATCH_ID], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(completed.stdout) + + def test_builder_is_commit_bound_and_byte_deterministic(self): + with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-closed-loop-") as directory: + root = Path(directory) + first = self.build(root / "first") + second = self.build(root / "second") + first_artifact = Path(first["artifact"]) + second_artifact = Path(second["artifact"]) + + self.assertEqual(first["sourceCommit"], SOURCE_COMMIT) + self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes()) + self.assertEqual( + first["sha256"], + hashlib.sha256(first_artifact.read_bytes()).hexdigest(), + ) + self.assertEqual(first["services"], ["nodedc-backend", "app"]) + self.assertEqual(first["mcpVersion"], "0.7.0") + + def test_artifact_is_a_safe_exact_engine_overlay(self): + with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-overlay-") as directory: + root = Path(directory) + built = self.build(root / "artifact") + artifact = Path(built["artifact"]) + + with tarfile.open(artifact, "r:gz") as archive: + members = archive.getmembers() + for member in members: + self.assertIn(member.type, (tarfile.REGTYPE, tarfile.DIRTYPE)) + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + self.assertEqual(member.mtime, 0) + self.assertNotIn(".DS_Store", member.name) + self.assertNotIn("__MACOSX", member.name) + archive.extractall(root / "extract", filter="data") + + manifest, entries, payload = RUNNER.load_artifact( + artifact, + root / "loaded", + ) + self.assertEqual(manifest["component"], "engine") + self.assertEqual(manifest["type"], "app-overlay") + self.assertEqual(tuple(entries), tuple(built["entries"])) + self.assertEqual( + RUNNER.component_services("engine", entries), + ("nodedc-backend", "app"), + ) + self.assertEqual( + RUNNER.component_healthchecks( + "engine", + entries, + ("nodedc-backend", "app"), + ), + ( + "http://127.0.0.1:8080/", + "http://127.0.0.1:3001/health", + ), + ) + self.assertTrue(RUNNER.component_publish_dist("engine", entries)) + self.assertEqual(RUNNER.component_builds("engine", entries), ()) + + payload_files = { + path.relative_to(payload).as_posix() + for path in payload.rglob("*") + if path.is_file() + } + self.assertEqual(payload_files, set(built["targetSha256"])) + for relative_path, expected in built["targetSha256"].items(): + self.assertEqual( + hashlib.sha256((payload / relative_path).read_bytes()).hexdigest(), + expected, + ) + + self.assertTrue(all("/tests/" not in path for path in payload_files)) + self.assertTrue(all("/data/" not in path for path in payload_files)) + self.assertTrue(all("/storage/" not in path for path in payload_files)) + self.assertTrue(all("/logs/" not in path for path in payload_files)) + self.assertTrue(all(not path.endswith(".env") for path in payload_files)) + + +if __name__ == "__main__": + unittest.main(verbosity=2)