diff --git a/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs b/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs index 2e66751..f8ad341 100644 --- a/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs +++ b/infra/deploy-runner/build-engine-l2-closed-loop-artifact.mjs @@ -23,6 +23,22 @@ const engineRoot = resolve( const artifactRoot = resolve( process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(here, "../deploy-artifacts"), ); +const baselineArtifact = resolve( + here, + "../deploy-artifacts/nodedc-engine-provider-authority-diagnostics-20260723-029.tgz", +); +const baselineArtifactSha256 = + "b4a0a1f707bf9814a4decc3f7f261b8afffe5727af2129c14bcdc76f81675e38"; +const descriptorRel = + "nodedc-source/services/node-intelligence/activation.json"; +const baselineDescriptorSha256 = + "84b0e15a10cedf334ad04d6f31c908da2dc155503c9974f0eeb75b01e20fb884"; +const predecessorGatewaySha256 = + "5331f6dc8dc306f641370a2968eb025ee3e278e27ae9a217e4cfc2901fec369c"; +const targetGatewaySha256 = + "4f600a2781be9118bec891fa2a6f20d7f55e0892ef2f06af24059193cebd28f4"; +const targetDescriptorSha256 = + "63e7619c6971d02102583bb5d80d33ece293b952f113200fa0b76f0e88c2dd32"; const [patchId = "", ...extra] = process.argv.slice(2); if (extra.length || !/^engine-l2-closed-loop-\d{8}-\d{3}$/.test(patchId)) { throw new Error( @@ -45,6 +61,7 @@ const entries = Object.freeze([ "nodedc-source/src/utils/n8nApi.ts", "nodedc-source/dist/index.html", "nodedc-source/dist/assets", + descriptorRel, ]); const targetSha256 = Object.freeze({ "nodedc-source/server/index.js": @@ -73,20 +90,30 @@ const targetSha256 = Object.freeze({ "18322addd45c126a7b8396f36b005f3085fddba3e9b346dd2c910f6fa6987ebf", "nodedc-source/dist/assets/index-CqvJfRRS.js": "06e6c23b03ea2ba8a4890e1f1714fd08ebaf18d735834200af6ab430af360ca8", + [descriptorRel]: targetDescriptorSha256, }); const artifact = join(artifactRoot, `nodedc-${patchId}.tgz`); await assertFresh(artifact); assertSourceCommit(); await assertExactSources(); +const targetDescriptor = await buildTargetDescriptor(); const stage = await mkdtemp(join(tmpdir(), "nodedc-engine-l2-closed-loop-")); try { const payload = join(stage, "payload"); for (const relativePath of entries) { - const source = join(engineRoot, relativePath); const destination = join(payload, relativePath); await mkdir(dirname(destination), { recursive: true }); + if (relativePath === descriptorRel) { + await writeFile(destination, targetDescriptor, { + encoding: "utf8", + flag: "wx", + mode: 0o644, + }); + continue; + } + const source = join(engineRoot, relativePath); const info = await lstat(source); if (info.isSymbolicLink()) { throw new Error(`engine_l2_closed_loop_source_symlink:${relativePath}`); @@ -130,6 +157,12 @@ try { ], mcpVersion: "0.7.0", graphContract: "semantic-revision-cas-v1", + transition: "failed-030-partial-source-reconciliation", + predecessorDescriptorSha256: baselineDescriptorSha256, + targetDescriptorSha256, + predecessorGatewaySha256, + targetGatewaySha256, + recoveryArtifact: "nodedc-engine-l2-closed-loop-20260723-030.tgz", actorPlanes: ["external_codex_mcp", "ai_workspace", "engine_ui"], untouched: [ "L2 graph data", @@ -153,9 +186,12 @@ function assertSourceCommit() { } async function assertExactSources() { - const expectedPaths = Object.keys(targetSha256).sort(); + const sourceEntries = entries.filter((entry) => entry !== descriptorRel); + const expectedPaths = Object.keys(targetSha256) + .filter((entry) => entry !== descriptorRel) + .sort(); const actualPaths = []; - for (const entry of entries) { + for (const entry of sourceEntries) { const source = join(engineRoot, entry); const info = await lstat(source); if (info.isSymbolicLink() || (!info.isFile() && !info.isDirectory())) { @@ -188,6 +224,39 @@ async function assertExactSources() { } } +async function buildTargetDescriptor() { + const baselineBytes = await readFile(baselineArtifact); + if (digest(baselineBytes) !== baselineArtifactSha256) { + throw new Error("engine_l2_closed_loop_baseline_artifact_mismatch"); + } + const baseline = run("tar", [ + "-xOf", + baselineArtifact, + `payload/${descriptorRel}`, + ]).stdout; + if (digest(Buffer.from(baseline, "utf8")) !== baselineDescriptorSha256) { + throw new Error("engine_l2_closed_loop_baseline_descriptor_mismatch"); + } + const descriptor = JSON.parse(baseline); + if ( + descriptor?.schemaVersion !== + "nodedc.engine-node-intelligence-transition/v1" || + descriptor?.action !== "activate" || + descriptor?.releaseId !== "2.33.2-974a9fb3492f" || + descriptor?.source?.gatewaySha256 !== predecessorGatewaySha256 || + descriptor?.source?.upstreamProjectionSha256 !== + "2dfa6b4f37d9bfa8b92a8109d4060d02dd2634ceb8f7924504b83ccf3fdd1523" + ) { + throw new Error("engine_l2_closed_loop_baseline_descriptor_contract_mismatch"); + } + descriptor.source.gatewaySha256 = targetGatewaySha256; + const rendered = `${JSON.stringify(descriptor, null, 2)}\n`; + if (digest(Buffer.from(rendered, "utf8")) !== targetDescriptorSha256) { + throw new Error("engine_l2_closed_loop_target_descriptor_mismatch"); + } + return rendered; +} + async function assertFresh(path) { try { await lstat(path); diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index 4f2d215..8a72c46 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -235,6 +235,76 @@ ENGINE_MCP_AUTONOMY_PROVIDER_V5_TARGET_SHA256 = { ENGINE_MCP_AUTONOMY_PROVIDER_V5_NEW_PATHS = ( "nodedc-source/server/assets/nodedc-engine-codex-agent-0.1.6.tgz", ) +ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL = ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL +ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES = ( + "nodedc-source/server/index.js", + "nodedc-source/server/l2/graphRepository.js", + "nodedc-source/server/realtime/ws.js", + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL, + "nodedc-source/server/routes/n8n.js", + "nodedc-source/server/routes/ndcAgentMcp.js", + "nodedc-source/src/App.tsx", + "nodedc-source/src/n8n/N8nSubworkflowHost.tsx", + "nodedc-source/src/realtime/useMultiplayer.ts", + "nodedc-source/src/utils/n8nApi.ts", + "nodedc-source/dist/index.html", + "nodedc-source/dist/assets", + ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL, +) +ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256 = { + "nodedc-source/server/index.js": "1896e1cade61579863c50ff3f52f2e81ff27f2a511a9d362db81925ee21cefad", + "nodedc-source/server/l2/graphRepository.js": "94ad08e1bb7e7be04854f1f911e06ee1acd6e09631f33f4c01e42e230665ac33", + "nodedc-source/server/realtime/ws.js": "82cfa833e05c2fc6d6049dd4564af06164b5c784fdfb82c243ca67519f4509c2", + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL: "4f600a2781be9118bec891fa2a6f20d7f55e0892ef2f06af24059193cebd28f4", + "nodedc-source/server/routes/n8n.js": "752cb1524160adc1c95163c34e139b615c063d2ce8980f2a63879bddbd0f1e08", + "nodedc-source/server/routes/ndcAgentMcp.js": "353a10291ceb93c3641ece60ec6e809a394be86f5ceb97cb44755178940409dd", + "nodedc-source/src/App.tsx": "b3e61a89330b774f309660208d05143d299ab582f18765c1c14e2122a62c4d5e", + "nodedc-source/src/n8n/N8nSubworkflowHost.tsx": "683aa44ba92aeac4879889e2bdb5319282976d7680d620701578830ce9677087", + "nodedc-source/src/realtime/useMultiplayer.ts": "a4d001d9914098e50a78d8abe0a66344d23680b7a19b3573aa7b6f48c8bb9c35", + "nodedc-source/src/utils/n8nApi.ts": "6f16d4992c51ffcc1e71a7bd172fd9ceb440d6e1a74d69ef1db62e0ac4230b3c", + "nodedc-source/dist/index.html": "90d72f8790dc8cf951066b1210447c84d640373117bae23f3a4cb3227fb96d6d", + "nodedc-source/dist/assets/index-Bim2pv1P.css": "18322addd45c126a7b8396f36b005f3085fddba3e9b346dd2c910f6fa6987ebf", + "nodedc-source/dist/assets/index-CqvJfRRS.js": "06e6c23b03ea2ba8a4890e1f1714fd08ebaf18d735834200af6ab430af360ca8", + ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL: "84b0e15a10cedf334ad04d6f31c908da2dc155503c9974f0eeb75b01e20fb884", +} +ENGINE_L2_CLOSED_LOOP_TARGET_SHA256 = { + **ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256, + ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL: "63e7619c6971d02102583bb5d80d33ece293b952f113200fa0b76f0e88c2dd32", +} +ENGINE_L2_CLOSED_LOOP_STABLE_SHA256 = { + "nodedc-source/server/index.js": "0ac408e0e9a7bc5c8e13a00afc957b982b065e2bc414919839e0b0a11aa05ba4", + "nodedc-source/server/realtime/ws.js": "340437ee2c6cc06c41b1e47f8b1ada24b68e175bb6fc4c361398bef00dd89686", + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL: "5331f6dc8dc306f641370a2968eb025ee3e278e27ae9a217e4cfc2901fec369c", + "nodedc-source/server/routes/n8n.js": "9bc3638e271102abec91bb80413329befb89d60c0e4dc0548f0dd11e93220d0a", + "nodedc-source/server/routes/ndcAgentMcp.js": "534e2c85e1faecc72a00da7ad32d0584ee09b6bd89eeec2221c3b23d80e1e962", + "nodedc-source/src/App.tsx": "d9bc0f23ceaaf4f5534ac9a9b1f97d84fd8eee4679c59b5999b5a75b8c77bb32", + "nodedc-source/src/n8n/N8nSubworkflowHost.tsx": "306694f329670a64322651ad249b057a74b34dace82d68d4071a365636d91d71", + "nodedc-source/src/realtime/useMultiplayer.ts": "3d8f150037d994dc8739fa9f6696acfe874c45128e567a6090cc4996b76f9e9d", + "nodedc-source/src/utils/n8nApi.ts": "2d10acda54d8c10637008612af9e3ad7625bd7fe17e9cfa38aa45c340eb171eb", + "nodedc-source/dist/index.html": "8894f62ad590168862e81266bc4602410279634b666af72cb14ad80aeb71ea74", + "nodedc-source/dist/assets/index-4TenBzkg.js": "a48fcacf3348c20a432cd6da2627b89c0c3e54bc989c727b620125dd3412538a", + "nodedc-source/dist/assets/index-BQ3RBHy9.js": "6e0f4c841afe3e0beb398943af915c8ee5d4a7df0b77b92e277e66faa3fa26a8", + "nodedc-source/dist/assets/index-Bim2pv1P.css": "18322addd45c126a7b8396f36b005f3085fddba3e9b346dd2c910f6fa6987ebf", + "nodedc-source/dist/assets/index-DJ1CMfu4.js": "a0aaf72d2ed390142ff2ea03dbcfdd534637e5faefd80d8aa7d705a804abe065", + ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL: "84b0e15a10cedf334ad04d6f31c908da2dc155503c9974f0eeb75b01e20fb884", +} +ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID = "engine-l2-closed-loop-20260723-030" +ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT = ( + "nodedc-engine-l2-closed-loop-20260723-030.tgz.20260723-123228" +) +ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256 = ( + "4c8401fcb1d318c52933cc3327015e9e4cbaaa5c38f1b45856acec2c72efdd2e" +) +ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_ID = ( + "engine-engine-l2-closed-loop-20260723-030-20260723-123228" +) +ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_SHA256 = { + "manifest.env": "7cf2a04c64f62547d408613307f7880bf350b9d305306af0e4eb8b81cfaa73bc", + "files.txt": "c4d2e0395b8ff076b648bd348283e4113feecf50b056ba6d1f495bafe4d09f27", + "source-before.tgz": "056d9400d52eeda7a51d225a803ab4d3465e06541d6646b4b728296a21c08ec5", + "existing-files.txt": "4c5de3d062d96b398dcaf44a6dde20c168ca0efd263893d51ce1fc545c59b8f8", + "missing-files.txt": "16d708945792cb35fcbe11d2c23a146e489fb183c4f498496ef40a97090c6008", +} ENGINE_PROVIDER_SECURITY_CATALOG_REL = ( "nodedc-source/server/assets/provider-packages/v1/catalog.json" ) @@ -3156,6 +3226,7 @@ def validate_engine_node_intelligence_transition(payload_dir, entries): "const ENGINE_AGENT_MCP_VERSION = '0.4.0'", "const ENGINE_AGENT_MCP_VERSION = '0.5.0'", "const ENGINE_AGENT_MCP_VERSION = '0.6.0'", + "const ENGINE_AGENT_MCP_VERSION = '0.7.0'", ) ): die("Engine node-intelligence gateway MCP version is not registered") @@ -4658,6 +4729,87 @@ def validate_engine_provider_authority_diagnostics_slice(payload_dir, entries): die("Engine private node validation reconciliation descriptor mismatch") +def validate_engine_l2_closed_loop_payload(payload_dir, entries): + if tuple(entries) != ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES: + die("Engine L2 closed-loop files.txt exact set/order mismatch") + actual = collect_exact_files( + payload_dir, + entries, + "Engine L2 closed-loop payload", + ) + if actual != ENGINE_L2_CLOSED_LOOP_TARGET_SHA256: + changed = sorted( + set(actual) ^ set(ENGINE_L2_CLOSED_LOOP_TARGET_SHA256) + | { + rel + for rel in set(actual) & set(ENGINE_L2_CLOSED_LOOP_TARGET_SHA256) + if actual[rel] != ENGINE_L2_CLOSED_LOOP_TARGET_SHA256[rel] + } + ) + detail = changed[0] if changed else "unknown" + die(f"Engine L2 closed-loop payload digest mismatch: {detail}") + + marker_contract = { + "nodedc-source/server/l2/graphRepository.js": ( + "expectedRevision", + "expectedContentRevision", + "writeAtomic", + "hydrateWorkflow", + ), + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL: ( + "ENGINE_AGENT_MCP_VERSION = '0.7.0'", + "engine_get_node_output_profile", + "expectedRevision", + ), + "nodedc-source/server/routes/n8n.js": ( + "toSafeNodeOutputProfile", + "expectedRevision", + "l2GraphRepository", + ), + "nodedc-source/server/routes/ndcAgentMcp.js": ( + "external_codex_mcp", + "l2GraphRepository", + "expectedRevision", + ), + "nodedc-source/server/realtime/ws.js": ( + "broadcastWorkflowEvent", + ), + } + for rel, markers in marker_contract.items(): + try: + source = (payload_dir / rel).read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + die(f"Engine L2 closed-loop source is unreadable: {rel}") + if any(marker not in source for marker in markers): + die(f"Engine L2 closed-loop source contract mismatch: {rel}") + + for rel in ( + "nodedc-source/server/l2/graphRepository.js", + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL, + "nodedc-source/server/routes/ndcAgentMcp.js", + ): + source = (payload_dir / rel).read_text(encoding="utf-8") + if re.search(r"\b(?:gelios|robot2b)\b", source, re.IGNORECASE): + die(f"Engine L2 closed-loop provider hardcode is forbidden: {rel}") + + descriptor = read_engine_node_intelligence_descriptor( + payload_dir / ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL, + "Engine L2 closed-loop node-intelligence descriptor", + ) + if ( + descriptor.get("action") != "activate" + or descriptor.get("releaseId") != ENGINE_NODE_INTELLIGENCE_RELEASE_ID + or descriptor.get("source", {}).get("gatewaySha256") + != ENGINE_L2_CLOSED_LOOP_TARGET_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ] + or descriptor.get("source", {}).get("upstreamProjectionSha256") + != "2dfa6b4f37d9bfa8b92a8109d4060d02dd2634ceb8f7924504b83ccf3fdd1523" + ): + die("Engine L2 closed-loop descriptor target mismatch") + return descriptor + + def validate_engine_depttrans_zone_authority_v1_slice(payload_dir, entries): if tuple(entries) != ENGINE_DEPTTRANS_ZONE_AUTHORITY_V1_ARTIFACT_ENTRIES: die("Engine Depttrans zone authority v1 files.txt exact set/order mismatch") @@ -4822,7 +4974,10 @@ def current_engine_node_intelligence_descriptor(): ) -def validate_installed_engine_node_intelligence_source(descriptor): +def validate_installed_engine_node_intelligence_source( + descriptor, + expected_gateway_sha256=None, +): if descriptor is None or descriptor.get("action") != "activate": die("active Engine node-intelligence descriptor is required") root = component_root("engine") @@ -4849,8 +5004,15 @@ def validate_installed_engine_node_intelligence_source(descriptor): path = source_root / rel if path.is_symlink() or sha256_file(path) != expected_sha256: die(f"installed Engine node-intelligence source drift detected: {rel}") + gateway_sha256 = ( + expected_gateway_sha256 + if expected_gateway_sha256 is not None + else descriptor["source"]["gatewaySha256"] + ) + if not re.fullmatch(r"[a-f0-9]{64}", str(gateway_sha256)): + die("installed Engine node-intelligence gateway expectation is invalid") gateway = root / ENGINE_NODE_INTELLIGENCE_GATEWAY_REL - if gateway.is_symlink() or sha256_file(gateway) != descriptor["source"]["gatewaySha256"]: + if gateway.is_symlink() or sha256_file(gateway) != gateway_sha256: die("installed Engine node-intelligence gateway drift detected") override = root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL try: @@ -4868,7 +5030,7 @@ def validate_installed_engine_node_intelligence_source(descriptor): readme = root / ENGINE_NODE_INTELLIGENCE_SERVICE_ROOT_REL / "README.md" if readme.is_symlink() or sha256_file(readme) != descriptor["source"]["readmeSha256"]: die("installed Engine node-intelligence README drift detected") - return descriptor["source"]["gatewaySha256"] + return gateway_sha256 def preflight_engine_node_intelligence_predecessor(payload_dir): @@ -4931,6 +5093,178 @@ def preflight_engine_node_intelligence_predecessor(payload_dir): } +def validate_engine_l2_closed_loop_recovery_evidence(): + backup_dir = BACKUPS_DIR / ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_ID + try: + backup_stat = backup_dir.lstat() + except FileNotFoundError: + die("Engine L2 closed-loop recovery backup is missing") + if stat.S_ISLNK(backup_stat.st_mode) or not stat.S_ISDIR(backup_stat.st_mode): + die("Engine L2 closed-loop recovery backup is unsafe") + backup_names = { + child.name + for child in backup_dir.iterdir() + } + if backup_names != set(ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_SHA256): + die("Engine L2 closed-loop recovery backup file set mismatch") + for name, expected in ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_SHA256.items(): + path = backup_dir / name + path_stat = path.lstat() + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected + ): + die(f"Engine L2 closed-loop recovery backup drift detected: {name}") + + failed_artifact = FAILED_DIR / ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT + try: + failed_stat = failed_artifact.lstat() + except FileNotFoundError: + die("Engine L2 closed-loop failed artifact is missing") + if ( + stat.S_ISLNK(failed_stat.st_mode) + or not stat.S_ISREG(failed_stat.st_mode) + or sha256_file(failed_artifact) + != ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256 + ): + die("Engine L2 closed-loop failed artifact evidence mismatch") + + try: + state_stat = FAILED_STATE_FILE.lstat() + state_lines = FAILED_STATE_FILE.read_text(encoding="utf-8").splitlines() + except (FileNotFoundError, OSError, UnicodeDecodeError): + die("Engine L2 closed-loop failed journal is unreadable") + if stat.S_ISLNK(state_stat.st_mode) or not stat.S_ISREG(state_stat.st_mode): + die("Engine L2 closed-loop failed journal is unsafe") + records = [] + for line in state_lines: + try: + value = json.loads(line) + except json.JSONDecodeError: + die("Engine L2 closed-loop failed journal contains invalid JSON") + if ( + isinstance(value, dict) + and value.get("id") == ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID + ): + records.append(value) + if len(records) != 1: + die("Engine L2 closed-loop failed journal evidence count mismatch") + record = records[0] + if ( + record.get("artifact") != ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT + or record.get("backup_id") != ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_ID + or record.get("component") != "engine" + or record.get("sha256") != ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256 + or record.get("started_apply") is not True + or record.get("rollback_status") != "not-required" + or record.get("status") != "failed" + or record.get("message") + != "installed Engine node-intelligence gateway drift detected" + ): + die("Engine L2 closed-loop failed journal evidence mismatch") + return backup_dir + + +def preflight_engine_l2_closed_loop_predecessor(payload_dir): + candidate = validate_engine_l2_closed_loop_payload( + payload_dir, + ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES, + ) + recovery_backup = validate_engine_l2_closed_loop_recovery_evidence() + root = component_root("engine") + actual = collect_exact_files( + root, + ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES, + "installed Engine L2 closed-loop partial state", + ) + if actual != ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256: + changed = sorted( + set(actual) ^ set(ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256) + | { + rel + for rel in set(actual) & set(ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256) + if actual[rel] != ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[rel] + } + ) + detail = changed[0] if changed else "unknown" + die(f"Engine L2 closed-loop partial predecessor drift detected: {detail}") + + nginx_entries = ("nginx-html/index.html", "nginx-html/assets") + nginx_actual = collect_exact_files( + root, + nginx_entries, + "installed Engine L2 closed-loop nginx publication", + ) + nginx_expected = { + "nginx-html/index.html": ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + "nodedc-source/dist/index.html" + ], + "nginx-html/assets/index-Bim2pv1P.css": + ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + "nodedc-source/dist/assets/index-Bim2pv1P.css" + ], + "nginx-html/assets/index-CqvJfRRS.js": + ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + "nodedc-source/dist/assets/index-CqvJfRRS.js" + ], + } + if nginx_actual != nginx_expected: + die("Engine L2 closed-loop nginx partial predecessor drift detected") + + installed = current_engine_node_intelligence_descriptor() + if ( + installed is None + or installed.get("action") != "activate" + or sha256_file(root / ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL) + != ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL + ] + ): + die("Engine L2 closed-loop partial descriptor mismatch") + expected_candidate = json.loads(json.dumps(installed)) + expected_candidate["source"]["gatewaySha256"] = ( + ENGINE_L2_CLOSED_LOOP_TARGET_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ] + ) + if candidate != expected_candidate: + die("Engine L2 closed-loop descriptor crosses node-intelligence boundary") + validate_installed_engine_node_intelligence_source( + installed, + expected_gateway_sha256=ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + backend = preflight_engine_credential_backend_runtime( + expected_node_intelligence_gateway_sha256= + ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + if backend["mode"] != "verified-derived-retry": + die("Engine L2 closed-loop requires the active immutable backend") + app_container_id = engine_compose_service_container_id_for_gateway( + "app", + ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + return { + "mode": "failed-030-partial-source-reconciliation", + "descriptor": candidate, + "recovery_backup": recovery_backup, + "predecessor_gateway_sha256": installed["source"]["gatewaySha256"], + "partial_gateway_sha256": ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + "target_gateway_sha256": candidate["source"]["gatewaySha256"], + "backend_mode": backend["mode"], + "backend_container_id": backend["container_id"], + "app_container_id": app_container_id, + } + + def validate_engine_n8n_staged_release(descriptor): release_rel = f"releases/n8n-nodes-ndc/{descriptor['releaseId']}" release_dir = N8N_PRIVATE_EXTENSION_RELEASES_ROOT / release_rel @@ -5009,6 +5343,7 @@ def load_artifact(artifact, work_dir): and not is_engine_mcp_control_plane_slice(manifest["component"], entries) and not is_engine_mcp_ontology_sdk_slice(manifest["component"], entries) and not is_engine_mcp_autonomy_provider_v5_slice(manifest["component"], entries) + and not is_engine_l2_closed_loop_slice(manifest["component"], entries) and not is_engine_provider_authority_diagnostics_slice( manifest["component"], entries, @@ -5023,6 +5358,8 @@ def load_artifact(artifact, work_dir): validate_engine_mcp_ontology_sdk_payload(payload_dir, entries) if is_engine_mcp_autonomy_provider_v5_slice(manifest["component"], entries): validate_engine_mcp_autonomy_provider_v5_payload(payload_dir, entries) + if is_engine_l2_closed_loop_slice(manifest["component"], entries): + validate_engine_l2_closed_loop_payload(payload_dir, entries) if is_engine_composite_provider_v4_slice(manifest["component"], entries): validate_engine_composite_provider_v4_slice(payload_dir, entries) if is_engine_provider_rotating_slot_slice(manifest["component"], entries): @@ -5089,6 +5426,17 @@ def state_has_patch_id(patch_id): return any(row.get("id") == patch_id for row in load_state(STATE_FILE)) +def reject_terminal_engine_l2_failed_artifact(manifest, sha256): + if ( + manifest.get("id") == ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID + or sha256 == ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256 + ): + die( + "Engine L2 closed-loop 030 is terminal failed; " + "use the exact registered reconciliation successor" + ) + + class DeployLock: def __enter__(self): try: @@ -5221,6 +5569,42 @@ def is_engine_agent_full_grant_migration_slice(component, entries): ) +def is_engine_l2_closed_loop_slice(component, entries): + return ( + component == "engine" + and entries is not None + and tuple(entries) == ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES + ) + + +def collect_exact_files(root, entries, label): + files = {} + for rel in entries: + path = root / rel + try: + path_stat = path.lstat() + except FileNotFoundError: + die(f"{label} path is missing: {rel}") + if stat.S_ISLNK(path_stat.st_mode): + die(f"{label} path is unsafe: {rel}") + if stat.S_ISREG(path_stat.st_mode): + files[rel] = sha256_file(path) + continue + if not stat.S_ISDIR(path_stat.st_mode): + die(f"{label} path is unsafe: {rel}") + for child in sorted(path.rglob("*")): + child_rel = child.relative_to(root).as_posix() + child_stat = child.lstat() + if stat.S_ISLNK(child_stat.st_mode): + die(f"{label} path is unsafe: {child_rel}") + if stat.S_ISDIR(child_stat.st_mode): + continue + if not stat.S_ISREG(child_stat.st_mode): + die(f"{label} path is unsafe: {child_rel}") + files[child_rel] = sha256_file(child) + return files + + def collect_engine_data_product_publish_grant_files(root, entries, label): files = {} for rel in entries: @@ -5658,6 +6042,12 @@ def is_platform_provider_catalog_only(entries): def component_services(component, entries=None): + if is_engine_l2_closed_loop_slice(component, entries): + # The failed 030 apply published both the backend source and the built + # UI before Compose rejected the descriptor/source mismatch. The exact + # reconciliation transition must therefore activate both generations. + return ("nodedc-backend", "app") + if ( is_engine_mcp_control_plane_slice(component, entries) or is_engine_mcp_ontology_sdk_slice(component, entries) @@ -5830,7 +6220,11 @@ def engine_backend_immutable_runtime_is_current(): ) -def component_compose_files(component, allow_prepared_engine_backend=False): +def component_compose_files( + component, + allow_prepared_engine_backend=False, + expected_node_intelligence_gateway_sha256=None, +): if component == "engine": root = component_root(component) files = [root / "docker-compose.yml"] @@ -5882,9 +6276,16 @@ def component_compose_files(component, allow_prepared_engine_backend=False): node_intelligence_descriptor is not None and node_intelligence_descriptor["action"] == "activate" ): - validate_installed_engine_node_intelligence_source( - node_intelligence_descriptor - ) + if expected_node_intelligence_gateway_sha256 is None: + validate_installed_engine_node_intelligence_source( + node_intelligence_descriptor + ) + else: + validate_installed_engine_node_intelligence_source( + node_intelligence_descriptor, + expected_gateway_sha256= + expected_node_intelligence_gateway_sha256, + ) files.append(root / ENGINE_NODE_INTELLIGENCE_OVERRIDE_REL) return tuple(files) files = COMPONENTS[component].get("compose_files", ()) @@ -6275,6 +6676,43 @@ def engine_backend_container_id(): return container_ids[0] +def engine_backend_container_id_for_gateway(expected_gateway_sha256): + return engine_compose_service_container_id_for_gateway( + "nodedc-backend", + expected_gateway_sha256, + ) + + +def engine_compose_service_container_id_for_gateway( + service, + expected_gateway_sha256, +): + if service not in ("nodedc-backend", "app"): + die("Engine L2 closed-loop service lookup is not registered") + result = subprocess.run( + [ + *compose_base_cmd( + "engine", + expected_node_intelligence_gateway_sha256= + expected_gateway_sha256, + ), + "ps", + "-q", + service, + ], + cwd=str(component_compose_root("engine")), + check=False, + capture_output=True, + text=True, + ) + container_ids = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if (result.returncode != 0 + or len(container_ids) != 1 + or not re.fullmatch(r"[a-f0-9]{12,64}", container_ids[0])): + die(f"Engine L2 closed-loop service topology mismatch: {service}") + return container_ids[0] + + def run_engine_backend_probe(arguments, label, container_id=None, image_ref=None): if (container_id is None) == (image_ref is None): die(f"{label} probe target mismatch") @@ -6438,6 +6876,7 @@ def validate_engine_backend_mounts_for_installed_runtime( def validate_engine_backend_node_intelligence_mounts_for_installed_runtime( container, node_modules_read_only, + expected_gateway_sha256=None, ): descriptor = current_engine_node_intelligence_descriptor() active = descriptor is not None and descriptor["action"] == "activate" @@ -6450,7 +6889,13 @@ def validate_engine_backend_node_intelligence_mounts_for_installed_runtime( node_modules_read_only, ) - validate_installed_engine_node_intelligence_source(descriptor) + if expected_gateway_sha256 is None: + validate_installed_engine_node_intelligence_source(descriptor) + else: + validate_installed_engine_node_intelligence_source( + descriptor, + expected_gateway_sha256=expected_gateway_sha256, + ) mounts = container.get("Mounts") if not isinstance(mounts, list): die("Engine backend mount inventory is missing") @@ -6882,7 +7327,9 @@ def ensure_engine_backend_release_image( return image -def preflight_engine_credential_backend_runtime(): +def preflight_engine_credential_backend_runtime( + expected_node_intelligence_gateway_sha256=None, +): base_image_id = inspect_local_image( ENGINE_CREDENTIAL_BACKEND_BASE_IMAGE, "Engine backend base image", @@ -6907,7 +7354,13 @@ def preflight_engine_credential_backend_runtime(): or node_modules_stat.st_mode & (stat.S_IWGRP | stat.S_IWOTH)): die("Engine backend node_modules host source is unsafe") - container_id = engine_backend_container_id() + container_id = ( + engine_backend_container_id() + if expected_node_intelligence_gateway_sha256 is None + else engine_backend_container_id_for_gateway( + expected_node_intelligence_gateway_sha256 + ) + ) containers = docker_json(["container", "inspect", container_id], "Engine backend container inspect") if not isinstance(containers, list) or len(containers) != 1 or not isinstance(containers[0], dict): die("Engine backend container inspect shape mismatch") @@ -6925,6 +7378,7 @@ def preflight_engine_credential_backend_runtime(): validate_engine_backend_node_intelligence_mounts_for_installed_runtime( container, node_modules_read_only=config_image == ENGINE_CREDENTIAL_BACKEND_IMAGE, + expected_gateway_sha256=expected_node_intelligence_gateway_sha256, ) if config_image == ENGINE_CREDENTIAL_BACKEND_BASE_IMAGE and current_image_id == base_image_id: runtime_presence = { @@ -7805,6 +8259,7 @@ def plan_artifact(artifact): mcp_control_plane_preflight = None mcp_ontology_sdk_preflight = None mcp_autonomy_provider_v5_preflight = None + l2_closed_loop_preflight = None composite_provider_v4_preflight = None provider_rotating_slot_preflight = None provider_authority_diagnostics_preflight = None @@ -7813,6 +8268,7 @@ def plan_artifact(artifact): provider_catalog_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) + reject_terminal_engine_l2_failed_artifact(manifest, sha) transition_descriptor = None transition_preflight = None if is_engine_n8n_transition(manifest["component"], entries): @@ -7843,6 +8299,10 @@ def plan_artifact(artifact): mcp_autonomy_provider_v5_preflight = ( preflight_engine_mcp_autonomy_provider_v5_predecessor(payload_dir) ) + if is_engine_l2_closed_loop_slice(manifest["component"], entries): + l2_closed_loop_preflight = preflight_engine_l2_closed_loop_predecessor( + payload_dir + ) if is_engine_composite_provider_v4_slice(manifest["component"], entries): composite_provider_v4_preflight = preflight_engine_composite_provider_v4_predecessor() if is_engine_provider_rotating_slot_slice(manifest["component"], entries): @@ -7961,6 +8421,65 @@ def plan_artifact(artifact): print(f"build_root={build_root}") print(f"build={' '.join((str(DOCKER),) + tuple(build_args))}") print(f"services={' '.join(services)}") + if l2_closed_loop_preflight: + descriptor = l2_closed_loop_preflight["descriptor"] + print( + "engine_l2_transition=" + f"{l2_closed_loop_preflight['mode']}" + ) + print( + "failed_patch=" + f"{ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID}" + ) + print( + "failed_artifact_sha256=" + f"{ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256}" + ) + print( + "recovery_backup=" + f"{l2_closed_loop_preflight['recovery_backup'].name}" + ) + print( + "predecessor_gateway_sha256=" + f"{l2_closed_loop_preflight['predecessor_gateway_sha256']}" + ) + print( + "partial_gateway_sha256=" + f"{l2_closed_loop_preflight['partial_gateway_sha256']}" + ) + print( + "target_gateway_sha256=" + f"{l2_closed_loop_preflight['target_gateway_sha256']}" + ) + print( + "predecessor_descriptor_sha256=" + f"{ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL]}" + ) + print( + "target_descriptor_sha256=" + f"{ENGINE_L2_CLOSED_LOOP_TARGET_SHA256[ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL]}" + ) + print(f"node_intelligence_release={descriptor['releaseId']}") + print( + "backend_current_barrier=" + f"{l2_closed_loop_preflight['backend_mode']}" + ) + print( + "backend_source_container_id=" + f"{l2_closed_loop_preflight['backend_container_id']}" + ) + print( + "app_source_container_id=" + f"{l2_closed_loop_preflight['app_container_id']}" + ) + print("node_intelligence_image=preserved") + print("backend_force_recreate=yes") + print("app_force_recreate=yes") + print("backend_pull=never") + print("n8n_l1_l2_data=preserved") + print("credentials=preserved") + print("databases=preserved") + print("ai_workspace=untouched") if node_intelligence_preflight: descriptor = node_intelligence_preflight["descriptor"] print(f"node_intelligence_transition={descriptor['action']}") @@ -8485,6 +9004,117 @@ def rollback_engine_apply(root, backup_dir, entries, current_stamp, runtime_star return f"source+runtime-restored:{restored_count}" +def validate_engine_l2_closed_loop_stable_source(root): + stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256) + actual = collect_exact_files( + root, + stable_entries, + "Engine L2 closed-loop stable rollback", + ) + if actual != ENGINE_L2_CLOSED_LOOP_STABLE_SHA256: + changed = sorted( + rel + for rel in ENGINE_L2_CLOSED_LOOP_STABLE_SHA256 + if actual.get(rel) != ENGINE_L2_CLOSED_LOOP_STABLE_SHA256[rel] + ) + detail = changed[0] if changed else "unknown" + die(f"Engine L2 closed-loop stable rollback drift detected: {detail}") + graph_repository = root / "nodedc-source/server/l2/graphRepository.js" + if graph_repository.exists() or graph_repository.is_symlink(): + die("Engine L2 closed-loop stable rollback retained candidate-only repository") + + nginx_actual = collect_exact_files( + root, + ("nginx-html/index.html", "nginx-html/assets"), + "Engine L2 closed-loop stable nginx rollback", + ) + nginx_expected = { + rel.replace("nodedc-source/dist/", "nginx-html/", 1): digest + for rel, digest in ENGINE_L2_CLOSED_LOOP_STABLE_SHA256.items() + if rel.startswith("nodedc-source/dist/") + } + if nginx_actual != nginx_expected: + die("Engine L2 closed-loop stable nginx rollback drift detected") + + descriptor = current_engine_node_intelligence_descriptor() + if descriptor is None or descriptor.get("action") != "activate": + die("Engine L2 closed-loop stable descriptor was not restored") + validate_installed_engine_node_intelligence_source(descriptor) + return descriptor + + +def rollback_engine_l2_closed_loop_reconciliation( + root, + candidate_backup_dir, + entries, + current_stamp, + runtime_started, + applied_services, +): + if tuple(entries) != ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES: + die("Engine L2 closed-loop rollback entry set mismatch") + if tuple(applied_services) != ("nodedc-backend", "app"): + die("Engine L2 closed-loop rollback service set mismatch") + + recovery_backup = validate_engine_l2_closed_loop_recovery_evidence() + candidate_restore_entries = [*entries, "nginx-html"] + validate_backup_partition( + candidate_restore_entries, + read_backup_path_list(candidate_backup_dir / "existing-files.txt"), + read_backup_path_list(candidate_backup_dir / "missing-files.txt"), + "Engine L2 closed-loop candidate rollback", + ) + candidate_restored = restore_platform_overlay( + root, + candidate_backup_dir, + candidate_restore_entries, + f"{current_stamp}-l2-candidate", + ) + + original_entries = [ + rel + for rel in ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES + if rel != ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL + ] + original_restore_entries = [*original_entries, "nginx-html"] + validate_backup_partition( + original_restore_entries, + read_backup_path_list(recovery_backup / "existing-files.txt"), + read_backup_path_list(recovery_backup / "missing-files.txt"), + "Engine L2 closed-loop original rollback", + ) + original_restored = restore_platform_overlay( + root, + recovery_backup, + original_restore_entries, + f"{current_stamp}-l2-original", + ) + validate_engine_l2_closed_loop_stable_source(root) + + total_restored = candidate_restored + original_restored + if not runtime_started: + return f"stable-source-restored-runtime-unchanged:{total_restored}" + + stable_entries = tuple(ENGINE_L2_CLOSED_LOOP_STABLE_SHA256) + run_component_runtime( + "engine", + stable_entries, + applied_services, + ) + for service in applied_services: + healthcheck_compose_service("engine", service) + for check in component_healthchecks( + "engine", + stable_entries, + applied_services, + ): + healthcheck_url(check) + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die("Engine L2 closed-loop stable backend rollback acceptance failed") + return f"stable-source+runtime-restored:{total_restored}" + + def inactive_engine_n8n_descriptor(descriptor): inactive = dict(descriptor) inactive["action"] = "rollback-inactive" @@ -8791,7 +9421,11 @@ def run_build(component, entries=None): subprocess.run(cmd, cwd=str(build_root), env=env, check=True) -def compose_base_cmd(component, allow_prepared_engine_backend=False): +def compose_base_cmd( + component, + allow_prepared_engine_backend=False, + expected_node_intelligence_gateway_sha256=None, +): cmd = [str(DOCKER), "compose"] compose_project = component_compose_project(component) if compose_project: @@ -8802,6 +9436,8 @@ def compose_base_cmd(component, allow_prepared_engine_backend=False): for compose_file in component_compose_files( component, allow_prepared_engine_backend=allow_prepared_engine_backend, + expected_node_intelligence_gateway_sha256= + expected_node_intelligence_gateway_sha256, ): cmd.extend(["-f", str(compose_file)]) return cmd @@ -9429,9 +10065,83 @@ process.stdout.write('store-v2-full-developer:'+digest); return result +def accept_engine_l2_closed_loop_runtime(): + root = component_root("engine") + actual = collect_exact_files( + root, + ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES, + "Engine L2 closed-loop installed target", + ) + if actual != ENGINE_L2_CLOSED_LOOP_TARGET_SHA256: + die("Engine L2 closed-loop installed target digest mismatch") + + nginx_actual = collect_exact_files( + root, + ("nginx-html/index.html", "nginx-html/assets"), + "Engine L2 closed-loop nginx target", + ) + nginx_expected = { + rel.replace("nodedc-source/dist/", "nginx-html/", 1): digest + for rel, digest in ENGINE_L2_CLOSED_LOOP_TARGET_SHA256.items() + if rel.startswith("nodedc-source/dist/") + } + if nginx_actual != nginx_expected: + die("Engine L2 closed-loop nginx target digest mismatch") + + descriptor = current_engine_node_intelligence_descriptor() + if descriptor is None or descriptor.get("action") != "activate": + die("Engine L2 closed-loop target descriptor is missing") + validate_installed_engine_node_intelligence_source(descriptor) + + script = """ +const gateway=await import('file:///app/server/routes/engineAgentGateway.js'); +const graph=await import('file:///app/server/l2/graphRepository.js'); +const n8n=await import('file:///app/server/routes/n8n.js'); +const embedded=await import('file:///app/server/routes/ndcAgentMcp.js'); +const required=['engine_plan_l2_patch','engine_apply_l2_patch','engine_deploy_and_run','engine_get_node_output_profile']; +const names=new Set(gateway.engineAgentTools.map((tool)=>tool.name)); +if(gateway.ENGINE_AGENT_MCP_VERSION!=='0.7.0'||required.some((name)=>!names.has(name)))throw new Error('mcp-catalog'); +if(typeof graph.createL2GraphRepository!=='function'||typeof graph.l2GraphRepository?.commit!=='function')throw new Error('graph-repository'); +const profile=n8n.toSafeNodeOutputProfile({id:'acceptance',data:{resultData:{runData:{Probe:[{data:{main:[[{json:{value:7,authorization:'must-not-escape'}}]]}}]}}}},'Probe'); +if(!profile?.found||profile.valuesIncluded!==false||JSON.stringify(profile).includes('must-not-escape'))throw new Error('safe-output-profile'); +const actor=embedded.graphMutationActor({demoAccess:{internal:true},body:{intent:'Deploy acceptance'},get:(name)=>name.toLowerCase()==='x-ndc-actor-plane'?'external_codex_mcp':''}); +if(actor.plane!=='external_codex_mcp'||actor.editedBy!=='engine-agent-mcp')throw new Error('external-actor-plane'); +process.stdout.write('engine-l2-closed-loop:0.7.0:cas+safe-profile+external-plane'); +""".strip() + result = run_engine_backend_probe( + ( + "node", + "--input-type=module", + "-e", + script, + ), + "Engine L2 closed-loop live contract", + container_id=engine_backend_container_id(), + ) + expected = ( + "engine-l2-closed-loop:0.7.0:" + "cas+safe-profile+external-plane" + ) + if result != expected: + die("Engine L2 closed-loop live contract acceptance mismatch") + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die("Engine L2 closed-loop immutable backend acceptance failed") + return result + + def run_healthchecks(component, entries=None, services=None): if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries): return + if is_engine_l2_closed_loop_slice(component, entries): + if tuple(services or ()) != ("nodedc-backend", "app"): + die("Engine L2 closed-loop acceptance service set mismatch") + for service in services: + healthcheck_compose_service("engine", service) + for check in component_healthchecks(component, entries, services): + healthcheck_url(check) + accept_engine_l2_closed_loop_runtime() + return if is_engine_n8n_transition(component, entries): descriptor = current_engine_n8n_transition_descriptor() if descriptor is None: @@ -9631,6 +10341,7 @@ def apply_artifact(artifact): services = None transition_descriptor = None node_intelligence_descriptor = None + l2_closed_loop_preflight = None node_intelligence_service_stopped = False apply_started = False engine_backend_recreated = False @@ -9643,6 +10354,7 @@ def apply_artifact(artifact): with tempfile.TemporaryDirectory(prefix=f"apply-{current_stamp}-", dir=TMP_DIR) as tmp: work = Path(tmp) manifest, entries, payload_dir = load_artifact(artifact, work) + reject_terminal_engine_l2_failed_artifact(manifest, sha) patch_id = manifest["id"] component = manifest["component"] root = component_root(component) @@ -9706,6 +10418,10 @@ def apply_artifact(artifact): preflight_engine_mcp_ontology_sdk_predecessor(payload_dir) if is_engine_mcp_autonomy_provider_v5_slice(component, entries): preflight_engine_mcp_autonomy_provider_v5_predecessor(payload_dir) + if is_engine_l2_closed_loop_slice(component, entries): + l2_closed_loop_preflight = ( + preflight_engine_l2_closed_loop_predecessor(payload_dir) + ) if is_engine_provider_security_catalog_slice(component, entries): preflight_engine_provider_security_catalog_predecessor() if not artifact_only and not compose_root.is_dir(): @@ -9713,7 +10429,14 @@ def apply_artifact(artifact): compose_root.mkdir(parents=True, exist_ok=True) else: die(f"component compose root not found: {compose_root}") - compose_files = component_compose_files(component) + compose_files = component_compose_files( + component, + expected_node_intelligence_gateway_sha256=( + l2_closed_loop_preflight["partial_gateway_sha256"] + if l2_closed_loop_preflight is not None + else None + ), + ) if not artifact_only and compose_files: for compose_file in compose_files: if not compose_file.is_file(): @@ -9803,6 +10526,18 @@ def apply_artifact(artifact): # candidate start is always eligible for its domain rollback. runtime_started = bool(services) run_component_runtime(component, entries, services) + if ( + l2_closed_loop_preflight is not None + and engine_backend_container_id() + == l2_closed_loop_preflight["backend_container_id"] + ): + die("Engine L2 closed-loop backend generation was not recreated") + if ( + l2_closed_loop_preflight is not None + and compose_service_container_id("engine", "app") + == l2_closed_loop_preflight["app_container_id"] + ): + die("Engine L2 closed-loop app generation was not recreated") run_healthchecks(component, entries, services) applied_path = move_artifact(artifact, APPLIED_DIR) @@ -9880,6 +10615,37 @@ def apply_artifact(artifact): except Exception as rollback_exc: rollback_status = f"failed:{type(rollback_exc).__name__}" print("engine-credential-automatic-rollback=failed", file=sys.stderr) + elif ( + l2_closed_loop_preflight is not None + and is_engine_l2_closed_loop_slice(component, entries) + and services is not None + ): + try: + restored_state = ( + rollback_engine_l2_closed_loop_reconciliation( + root, + backup_dir, + entries, + current_stamp, + runtime_started, + services, + ) + ) + rollback_status = ( + "ok:engine-l2-reconciliation:" + f"{restored_state}" + ) + print( + "engine-l2-automatic-rollback=" + f"{rollback_status}", + file=sys.stderr, + ) + except Exception as rollback_exc: + rollback_status = f"failed:{type(rollback_exc).__name__}" + print( + "engine-l2-automatic-rollback=failed", + file=sys.stderr, + ) elif ( ( is_engine_data_product_publish_grant_slice(component, entries) diff --git a/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py b/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py index 59a2d78..478ae52 100644 --- a/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py +++ b/infra/deploy-runner/test_engine_l2_closed_loop_artifact.py @@ -4,10 +4,12 @@ import importlib.util import json import os from pathlib import Path +import shutil import subprocess import tarfile import tempfile import unittest +from unittest import mock SCRIPT_DIR = Path(__file__).resolve().parent @@ -16,6 +18,11 @@ BUILDER_PATH = SCRIPT_DIR / "build-engine-l2-closed-loop-artifact.mjs" ENGINE_ROOT = SCRIPT_DIR.parent.parent.parent / "NODEDC_ENGINE_INFRA" PATCH_ID = "engine-l2-closed-loop-20991231-999" SOURCE_COMMIT = "dda405cff27977622af0c218abb4d4420c408536" +BASELINE_ARTIFACT = ( + SCRIPT_DIR.parent + / "deploy-artifacts" + / "nodedc-engine-provider-authority-diagnostics-20260723-029.tgz" +) def load_runner(): @@ -57,6 +64,37 @@ class EngineL2ClosedLoopArtifactTest(unittest.TestCase): ) return json.loads(completed.stdout) + def load_built_payload(self, root): + built = self.build(root / "artifact") + artifact = Path(built["artifact"]) + manifest, entries, payload = RUNNER.load_artifact( + artifact, + root / "loaded", + ) + return built, manifest, entries, payload + + def materialize_partial_predecessor(self, payload, entries, root): + for relative_path in entries: + source = payload / relative_path + destination = root / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, destination) + else: + shutil.copy2(source, destination) + with tarfile.open(BASELINE_ARTIFACT, "r:gz") as archive: + member = archive.extractfile( + "payload/nodedc-source/services/node-intelligence/activation.json" + ) + self.assertIsNotNone(member) + (root / RUNNER.ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL).write_bytes( + member.read() + ) + shutil.copytree( + root / "nodedc-source/dist", + root / "nginx-html", + ) + def test_builder_is_commit_bound_and_byte_deterministic(self): with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-closed-loop-") as directory: root = Path(directory) @@ -74,6 +112,30 @@ class EngineL2ClosedLoopArtifactTest(unittest.TestCase): self.assertEqual(first["services"], ["nodedc-backend", "app"]) self.assertEqual(first["mcpVersion"], "0.7.0") + def test_failed_030_identity_is_terminal_and_cannot_be_retried(self): + with self.assertRaisesRegex( + RUNNER.DeployError, + "terminal failed", + ): + RUNNER.reject_terminal_engine_l2_failed_artifact( + { + "id": RUNNER.ENGINE_L2_CLOSED_LOOP_FAILED_PATCH_ID, + }, + "0" * 64, + ) + with self.assertRaisesRegex( + RUNNER.DeployError, + "terminal failed", + ): + RUNNER.reject_terminal_engine_l2_failed_artifact( + {"id": "different-successor"}, + RUNNER.ENGINE_L2_CLOSED_LOOP_FAILED_ARTIFACT_SHA256, + ) + RUNNER.reject_terminal_engine_l2_failed_artifact( + {"id": "engine-l2-closed-loop-20260723-031"}, + "1" * 64, + ) + def test_artifact_is_a_safe_exact_engine_overlay(self): with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-overlay-") as directory: root = Path(directory) @@ -134,6 +196,192 @@ class EngineL2ClosedLoopArtifactTest(unittest.TestCase): self.assertTrue(all("/logs/" not in path for path in payload_files)) self.assertTrue(all(not path.endswith(".env") for path in payload_files)) + def test_preflight_accepts_only_the_exact_failed_030_partial_state(self): + with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-preflight-") as directory: + workspace = Path(directory) + _built, _manifest, entries, payload = self.load_built_payload( + workspace / "build" + ) + live = workspace / "live" + self.materialize_partial_predecessor(payload, entries, live) + recovery = workspace / RUNNER.ENGINE_L2_CLOSED_LOOP_RECOVERY_BACKUP_ID + + backend = { + "mode": "verified-derived-retry", + "container_id": "a" * 64, + } + with ( + mock.patch.object(RUNNER, "component_root", return_value=live), + mock.patch.object( + RUNNER, + "validate_engine_l2_closed_loop_recovery_evidence", + return_value=recovery, + ), + mock.patch.object( + RUNNER, + "validate_installed_engine_node_intelligence_source", + ) as validate_source, + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value=backend, + ) as validate_backend, + mock.patch.object( + RUNNER, + "engine_compose_service_container_id_for_gateway", + return_value="b" * 64, + ) as app_lookup, + ): + result = RUNNER.preflight_engine_l2_closed_loop_predecessor( + payload + ) + self.assertEqual( + result["mode"], + "failed-030-partial-source-reconciliation", + ) + self.assertEqual(result["backend_container_id"], "a" * 64) + self.assertEqual(result["app_container_id"], "b" * 64) + self.assertEqual( + result["target_gateway_sha256"], + RUNNER.ENGINE_L2_CLOSED_LOOP_TARGET_SHA256[ + RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + self.assertEqual( + validate_source.call_args.kwargs[ + "expected_gateway_sha256" + ], + RUNNER.ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + self.assertEqual( + validate_backend.call_args.kwargs[ + "expected_node_intelligence_gateway_sha256" + ], + RUNNER.ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + app_lookup.assert_called_once_with( + "app", + RUNNER.ENGINE_L2_CLOSED_LOOP_PARTIAL_SHA256[ + RUNNER.ENGINE_NODE_INTELLIGENCE_GATEWAY_REL + ], + ) + + index_path = live / "nodedc-source/dist/index.html" + index_path.write_bytes(index_path.read_bytes() + b"\n") + with ( + mock.patch.object(RUNNER, "component_root", return_value=live), + mock.patch.object( + RUNNER, + "validate_engine_l2_closed_loop_recovery_evidence", + return_value=recovery, + ), + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + ) as unreachable_backend, + ): + with self.assertRaisesRegex( + RUNNER.DeployError, + "partial predecessor drift", + ): + RUNNER.preflight_engine_l2_closed_loop_predecessor(payload) + unreachable_backend.assert_not_called() + + def test_reconciliation_rollback_uses_failed_030_backup_and_restores_runtime(self): + entries = RUNNER.ENGINE_L2_CLOSED_LOOP_ARTIFACT_ENTRIES + services = ("nodedc-backend", "app") + with tempfile.TemporaryDirectory(prefix="nodedc-engine-l2-rollback-") as directory: + root = Path(directory) + candidate = root / "candidate-backup" + recovery = root / "recovery-backup" + candidate_entries = [*entries, "nginx-html"] + original_entries = [ + rel + for rel in entries + if rel != RUNNER.ENGINE_L2_CLOSED_LOOP_DESCRIPTOR_REL + ] + original_entries.append("nginx-html") + + def backup_paths(path): + if path.parent == candidate: + return candidate_entries if path.name == "existing-files.txt" else [] + if path.parent == recovery: + if path.name == "existing-files.txt": + return [ + rel + for rel in original_entries + if rel != "nodedc-source/server/l2/graphRepository.js" + ] + return ["nodedc-source/server/l2/graphRepository.js"] + raise AssertionError(path) + + with ( + mock.patch.object( + RUNNER, + "validate_engine_l2_closed_loop_recovery_evidence", + return_value=recovery, + ), + mock.patch.object( + RUNNER, + "read_backup_path_list", + side_effect=backup_paths, + ), + mock.patch.object( + RUNNER, + "restore_platform_overlay", + side_effect=(len(candidate_entries), len(original_entries)), + ) as restore, + mock.patch.object( + RUNNER, + "validate_engine_l2_closed_loop_stable_source", + ) as validate_stable, + mock.patch.object(RUNNER, "run_component_runtime") as run_runtime, + mock.patch.object(RUNNER, "healthcheck_compose_service") as check_service, + mock.patch.object( + RUNNER, + "component_healthchecks", + return_value=( + "http://127.0.0.1:8080/", + "http://127.0.0.1:3001/health", + ), + ), + mock.patch.object(RUNNER, "healthcheck_url") as check_url, + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value={"mode": "verified-derived-retry"}, + ), + ): + result = RUNNER.rollback_engine_l2_closed_loop_reconciliation( + root, + candidate, + entries, + "20991231-235959", + True, + services, + ) + + self.assertTrue(result.startswith("stable-source+runtime-restored:")) + self.assertEqual( + [call.args[1] for call in restore.call_args_list], + [candidate, recovery], + ) + validate_stable.assert_called_once_with(root) + run_runtime.assert_called_once_with( + "engine", + tuple(RUNNER.ENGINE_L2_CLOSED_LOOP_STABLE_SHA256), + services, + ) + self.assertEqual( + [call.args for call in check_service.call_args_list], + [("engine", "nodedc-backend"), ("engine", "app")], + ) + self.assertEqual(check_url.call_count, 2) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/infra/deploy-runner/test_engine_provider_target_host_policy.py b/infra/deploy-runner/test_engine_provider_target_host_policy.py index 19f4c4c..b00cb2f 100644 --- a/infra/deploy-runner/test_engine_provider_target_host_policy.py +++ b/infra/deploy-runner/test_engine_provider_target_host_policy.py @@ -33,6 +33,20 @@ RUNNER = load_runner() class EngineProviderTargetHostPolicyTest(unittest.TestCase): + def require_historical_target_source(self): + for relative_path, expected in ( + RUNNER.ENGINE_PROVIDER_TARGET_HOST_POLICY_TARGET_SHA256.items() + ): + path = ENGINE_ROOT / relative_path + if ( + not path.is_file() + or hashlib.sha256(path.read_bytes()).hexdigest() != expected + ): + self.skipTest( + "historical target-host policy source has advanced " + "to its exact successor" + ) + def build(self, artifact_dir): environment = os.environ.copy() environment["NODEDC_ENGINE_SOURCE_ROOT"] = str(ENGINE_ROOT) @@ -47,6 +61,7 @@ class EngineProviderTargetHostPolicyTest(unittest.TestCase): return json.loads(completed.stdout) def test_builder_emits_exact_deterministic_one_file_slice(self): + self.require_historical_target_source() with tempfile.TemporaryDirectory(prefix="nodedc-provider-target-host-") as directory: root = Path(directory) first = self.build(root / "first") @@ -126,6 +141,7 @@ class EngineProviderTargetHostPolicyTest(unittest.TestCase): acceptance.assert_called_once_with() def test_live_acceptance_uses_exact_literal_host_contract(self): + self.require_historical_target_source() expected_live = "engine-provider-target-host-policy:exact-literal-host:v1" with ( mock.patch.object(RUNNER, "component_root", return_value=ENGINE_ROOT),