Add reproducible Engine L2 deployment artifact
This commit is contained in:
Codex
parent
2a30e1e991
commit
ad3760e767
|
|
@ -0,0 +1,231 @@
|
|||
#!/usr/bin/env node
|
||||
import { createHash } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
copyFile,
|
||||
cp,
|
||||
lstat,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const platformRoot = resolve(here, "../..");
|
||||
const engineRoot = resolve(
|
||||
process.env.NODEDC_ENGINE_SOURCE_ROOT || resolve(platformRoot, "../NODEDC_ENGINE_INFRA"),
|
||||
);
|
||||
const artifactRoot = resolve(
|
||||
process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(here, "../deploy-artifacts"),
|
||||
);
|
||||
const [patchId = "", ...extra] = process.argv.slice(2);
|
||||
if (extra.length || !/^engine-l2-closed-loop-\d{8}-\d{3}$/.test(patchId)) {
|
||||
throw new Error(
|
||||
"usage: build-engine-l2-closed-loop-artifact.mjs " +
|
||||
"<engine-l2-closed-loop-YYYYMMDD-NNN>",
|
||||
);
|
||||
}
|
||||
|
||||
const sourceCommit = "dda405cff27977622af0c218abb4d4420c408536";
|
||||
const entries = Object.freeze([
|
||||
"nodedc-source/server/index.js",
|
||||
"nodedc-source/server/l2/graphRepository.js",
|
||||
"nodedc-source/server/realtime/ws.js",
|
||||
"nodedc-source/server/routes/engineAgentGateway.js",
|
||||
"nodedc-source/server/routes/n8n.js",
|
||||
"nodedc-source/server/routes/ndcAgentMcp.js",
|
||||
"nodedc-source/src/App.tsx",
|
||||
"nodedc-source/src/n8n/N8nSubworkflowHost.tsx",
|
||||
"nodedc-source/src/realtime/useMultiplayer.ts",
|
||||
"nodedc-source/src/utils/n8nApi.ts",
|
||||
"nodedc-source/dist/index.html",
|
||||
"nodedc-source/dist/assets",
|
||||
]);
|
||||
const targetSha256 = Object.freeze({
|
||||
"nodedc-source/server/index.js":
|
||||
"1896e1cade61579863c50ff3f52f2e81ff27f2a511a9d362db81925ee21cefad",
|
||||
"nodedc-source/server/l2/graphRepository.js":
|
||||
"94ad08e1bb7e7be04854f1f911e06ee1acd6e09631f33f4c01e42e230665ac33",
|
||||
"nodedc-source/server/realtime/ws.js":
|
||||
"82cfa833e05c2fc6d6049dd4564af06164b5c784fdfb82c243ca67519f4509c2",
|
||||
"nodedc-source/server/routes/engineAgentGateway.js":
|
||||
"4f600a2781be9118bec891fa2a6f20d7f55e0892ef2f06af24059193cebd28f4",
|
||||
"nodedc-source/server/routes/n8n.js":
|
||||
"752cb1524160adc1c95163c34e139b615c063d2ce8980f2a63879bddbd0f1e08",
|
||||
"nodedc-source/server/routes/ndcAgentMcp.js":
|
||||
"353a10291ceb93c3641ece60ec6e809a394be86f5ceb97cb44755178940409dd",
|
||||
"nodedc-source/src/App.tsx":
|
||||
"b3e61a89330b774f309660208d05143d299ab582f18765c1c14e2122a62c4d5e",
|
||||
"nodedc-source/src/n8n/N8nSubworkflowHost.tsx":
|
||||
"683aa44ba92aeac4879889e2bdb5319282976d7680d620701578830ce9677087",
|
||||
"nodedc-source/src/realtime/useMultiplayer.ts":
|
||||
"a4d001d9914098e50a78d8abe0a66344d23680b7a19b3573aa7b6f48c8bb9c35",
|
||||
"nodedc-source/src/utils/n8nApi.ts":
|
||||
"6f16d4992c51ffcc1e71a7bd172fd9ceb440d6e1a74d69ef1db62e0ac4230b3c",
|
||||
"nodedc-source/dist/index.html":
|
||||
"90d72f8790dc8cf951066b1210447c84d640373117bae23f3a4cb3227fb96d6d",
|
||||
"nodedc-source/dist/assets/index-Bim2pv1P.css":
|
||||
"18322addd45c126a7b8396f36b005f3085fddba3e9b346dd2c910f6fa6987ebf",
|
||||
"nodedc-source/dist/assets/index-CqvJfRRS.js":
|
||||
"06e6c23b03ea2ba8a4890e1f1714fd08ebaf18d735834200af6ab430af360ca8",
|
||||
});
|
||||
const artifact = join(artifactRoot, `nodedc-${patchId}.tgz`);
|
||||
|
||||
await assertFresh(artifact);
|
||||
assertSourceCommit();
|
||||
await assertExactSources();
|
||||
|
||||
const stage = await mkdtemp(join(tmpdir(), "nodedc-engine-l2-closed-loop-"));
|
||||
try {
|
||||
const payload = join(stage, "payload");
|
||||
for (const relativePath of entries) {
|
||||
const source = join(engineRoot, relativePath);
|
||||
const destination = join(payload, relativePath);
|
||||
await mkdir(dirname(destination), { recursive: true });
|
||||
const info = await lstat(source);
|
||||
if (info.isSymbolicLink()) {
|
||||
throw new Error(`engine_l2_closed_loop_source_symlink:${relativePath}`);
|
||||
}
|
||||
if (info.isDirectory()) {
|
||||
await cp(source, destination, {
|
||||
recursive: true,
|
||||
force: false,
|
||||
errorOnExist: true,
|
||||
});
|
||||
} else if (info.isFile()) {
|
||||
await copyFile(source, destination);
|
||||
} else {
|
||||
throw new Error(`engine_l2_closed_loop_source_unsafe:${relativePath}`);
|
||||
}
|
||||
}
|
||||
await writeFile(
|
||||
join(stage, "manifest.env"),
|
||||
`id=${patchId}\ncomponent=engine\ntype=app-overlay\n`,
|
||||
{ encoding: "utf8", flag: "wx", mode: 0o644 },
|
||||
);
|
||||
await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "wx",
|
||||
mode: 0o644,
|
||||
});
|
||||
await mkdir(artifactRoot, { recursive: true });
|
||||
run("python3", ["-c", canonicalTarScript(), artifact, stage]);
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
patchId,
|
||||
artifact,
|
||||
sha256: digest(await readFile(artifact)),
|
||||
sourceCommit,
|
||||
entries,
|
||||
targetSha256,
|
||||
services: ["nodedc-backend", "app"],
|
||||
healthchecks: [
|
||||
"http://127.0.0.1:8080/",
|
||||
"http://127.0.0.1:3001/health",
|
||||
],
|
||||
mcpVersion: "0.7.0",
|
||||
graphContract: "semantic-revision-cas-v1",
|
||||
actorPlanes: ["external_codex_mcp", "ai_workspace", "engine_ui"],
|
||||
untouched: [
|
||||
"L2 graph data",
|
||||
"credentials",
|
||||
"databases",
|
||||
"n8n runtime",
|
||||
"AI Workspace connector",
|
||||
],
|
||||
}, null, 2));
|
||||
} finally {
|
||||
await rm(stage, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function assertSourceCommit() {
|
||||
const actual = run("git", ["-C", engineRoot, "rev-parse", "HEAD"]).stdout.trim();
|
||||
if (actual !== sourceCommit) {
|
||||
throw new Error(
|
||||
`engine_l2_closed_loop_source_commit_mismatch:expected=${sourceCommit}:actual=${actual}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertExactSources() {
|
||||
const expectedPaths = Object.keys(targetSha256).sort();
|
||||
const actualPaths = [];
|
||||
for (const entry of entries) {
|
||||
const source = join(engineRoot, entry);
|
||||
const info = await lstat(source);
|
||||
if (info.isSymbolicLink() || (!info.isFile() && !info.isDirectory())) {
|
||||
throw new Error(`engine_l2_closed_loop_source_unsafe:${entry}`);
|
||||
}
|
||||
if (info.isDirectory()) {
|
||||
const result = run("find", [source, "-type", "f", "-print"]);
|
||||
for (const path of result.stdout.split("\n").filter(Boolean)) {
|
||||
actualPaths.push(path.slice(engineRoot.length + 1));
|
||||
}
|
||||
} else {
|
||||
actualPaths.push(entry);
|
||||
}
|
||||
}
|
||||
actualPaths.sort();
|
||||
if (JSON.stringify(actualPaths) !== JSON.stringify(expectedPaths)) {
|
||||
throw new Error(
|
||||
`engine_l2_closed_loop_source_set_mismatch:` +
|
||||
`expected=${expectedPaths.join(",")}:actual=${actualPaths.join(",")}`,
|
||||
);
|
||||
}
|
||||
for (const relativePath of expectedPaths) {
|
||||
const actual = digest(await readFile(join(engineRoot, relativePath)));
|
||||
if (actual !== targetSha256[relativePath]) {
|
||||
throw new Error(
|
||||
`engine_l2_closed_loop_target_mismatch:${relativePath}:` +
|
||||
`expected=${targetSha256[relativePath]}:actual=${actual}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function assertFresh(path) {
|
||||
try {
|
||||
await lstat(path);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return;
|
||||
throw error;
|
||||
}
|
||||
throw new Error("artifact_already_exists");
|
||||
}
|
||||
|
||||
function canonicalTarScript() {
|
||||
return [
|
||||
"import gzip,io,pathlib,sys,tarfile",
|
||||
"root=pathlib.Path(sys.argv[2])",
|
||||
"with open(sys.argv[1],'xb') as out:",
|
||||
" with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:",
|
||||
" with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:",
|
||||
" for top in ('manifest.env','files.txt','payload'):",
|
||||
" p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])",
|
||||
" for x in paths:",
|
||||
" info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())",
|
||||
" info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644",
|
||||
" with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function digest(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
const result = spawnSync(command, args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`${command}_failed:${result.stderr || result.stdout}`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
import hashlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy"
|
||||
BUILDER_PATH = SCRIPT_DIR / "build-engine-l2-closed-loop-artifact.mjs"
|
||||
ENGINE_ROOT = SCRIPT_DIR.parent.parent.parent / "NODEDC_ENGINE_INFRA"
|
||||
PATCH_ID = "engine-l2-closed-loop-20991231-999"
|
||||
SOURCE_COMMIT = "dda405cff27977622af0c218abb4d4420c408536"
|
||||
|
||||
|
||||
def load_runner():
|
||||
loader = importlib.machinery.SourceFileLoader(
|
||||
"nodedc_engine_l2_closed_loop",
|
||||
str(RUNNER_PATH),
|
||||
)
|
||||
spec = importlib.util.spec_from_loader(loader.name, loader)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
RUNNER = load_runner()
|
||||
|
||||
|
||||
class EngineL2ClosedLoopArtifactTest(unittest.TestCase):
|
||||
def require_target_commit(self):
|
||||
current = subprocess.run(
|
||||
["git", "-C", str(ENGINE_ROOT), "rev-parse", "HEAD"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout.strip()
|
||||
if current != SOURCE_COMMIT:
|
||||
self.skipTest("historical Engine L2 closed-loop source has advanced")
|
||||
|
||||
def build(self, artifact_dir):
|
||||
self.require_target_commit()
|
||||
environment = os.environ.copy()
|
||||
environment["NODEDC_ENGINE_SOURCE_ROOT"] = str(ENGINE_ROOT)
|
||||
environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir)
|
||||
completed = subprocess.run(
|
||||
["node", str(BUILDER_PATH), PATCH_ID],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
def test_builder_is_commit_bound_and_byte_deterministic(self):
|
||||
with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-closed-loop-") as directory:
|
||||
root = Path(directory)
|
||||
first = self.build(root / "first")
|
||||
second = self.build(root / "second")
|
||||
first_artifact = Path(first["artifact"])
|
||||
second_artifact = Path(second["artifact"])
|
||||
|
||||
self.assertEqual(first["sourceCommit"], SOURCE_COMMIT)
|
||||
self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes())
|
||||
self.assertEqual(
|
||||
first["sha256"],
|
||||
hashlib.sha256(first_artifact.read_bytes()).hexdigest(),
|
||||
)
|
||||
self.assertEqual(first["services"], ["nodedc-backend", "app"])
|
||||
self.assertEqual(first["mcpVersion"], "0.7.0")
|
||||
|
||||
def test_artifact_is_a_safe_exact_engine_overlay(self):
|
||||
with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-overlay-") as directory:
|
||||
root = Path(directory)
|
||||
built = self.build(root / "artifact")
|
||||
artifact = Path(built["artifact"])
|
||||
|
||||
with tarfile.open(artifact, "r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
for member in members:
|
||||
self.assertIn(member.type, (tarfile.REGTYPE, tarfile.DIRTYPE))
|
||||
self.assertEqual(member.uid, 0)
|
||||
self.assertEqual(member.gid, 0)
|
||||
self.assertEqual(member.mtime, 0)
|
||||
self.assertNotIn(".DS_Store", member.name)
|
||||
self.assertNotIn("__MACOSX", member.name)
|
||||
archive.extractall(root / "extract", filter="data")
|
||||
|
||||
manifest, entries, payload = RUNNER.load_artifact(
|
||||
artifact,
|
||||
root / "loaded",
|
||||
)
|
||||
self.assertEqual(manifest["component"], "engine")
|
||||
self.assertEqual(manifest["type"], "app-overlay")
|
||||
self.assertEqual(tuple(entries), tuple(built["entries"]))
|
||||
self.assertEqual(
|
||||
RUNNER.component_services("engine", entries),
|
||||
("nodedc-backend", "app"),
|
||||
)
|
||||
self.assertEqual(
|
||||
RUNNER.component_healthchecks(
|
||||
"engine",
|
||||
entries,
|
||||
("nodedc-backend", "app"),
|
||||
),
|
||||
(
|
||||
"http://127.0.0.1:8080/",
|
||||
"http://127.0.0.1:3001/health",
|
||||
),
|
||||
)
|
||||
self.assertTrue(RUNNER.component_publish_dist("engine", entries))
|
||||
self.assertEqual(RUNNER.component_builds("engine", entries), ())
|
||||
|
||||
payload_files = {
|
||||
path.relative_to(payload).as_posix()
|
||||
for path in payload.rglob("*")
|
||||
if path.is_file()
|
||||
}
|
||||
self.assertEqual(payload_files, set(built["targetSha256"]))
|
||||
for relative_path, expected in built["targetSha256"].items():
|
||||
self.assertEqual(
|
||||
hashlib.sha256((payload / relative_path).read_bytes()).hexdigest(),
|
||||
expected,
|
||||
)
|
||||
|
||||
self.assertTrue(all("/tests/" not in path for path in payload_files))
|
||||
self.assertTrue(all("/data/" not in path for path in payload_files))
|
||||
self.assertTrue(all("/storage/" not in path for path in payload_files))
|
||||
self.assertTrue(all("/logs/" not in path for path in payload_files))
|
||||
self.assertTrue(all(not path.endswith(".env") for path in payload_files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Loading…
Reference in New Issue