diff --git a/infra/deploy-runner/build-engine-mcp-execution-plan-telemetry-runtime-artifact.mjs b/infra/deploy-runner/build-engine-mcp-execution-plan-telemetry-runtime-artifact.mjs new file mode 100755 index 0000000..133abf6 --- /dev/null +++ b/infra/deploy-runner/build-engine-mcp-execution-plan-telemetry-runtime-artifact.mjs @@ -0,0 +1,224 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + copyFile, + lstat, + mkdir, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const workspaceRoot = resolve(here, "../../.."); +const engineRoot = resolve( + process.env.NODEDC_ENGINE_SOURCE_ROOT || join(workspaceRoot, "NODEDC_ENGINE_INFRA"), +); +const artifactRoot = resolve( + process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(here, "../deploy-artifacts"), +); +const descriptorPath = + "nodedc-source/server/deployTransitions/executionPlanTelemetryRuntimeV2.json"; +const targetSha256 = Object.freeze({ + "nodedc-source/server/l2ExecutionPlan/compiler.js": + "6b783ad15c26dc7de0645082c8a426002d943138c70bf31a240247b16a33b6a0", + "nodedc-source/server/assets/execution-plans/v1/catalog.json": + "5bdfc92284a7c836ff326e3a89b559110e376b32efd34b5f23f2bec272745781", + [descriptorPath]: + "68b275efb6303284336d8b637c966c24d891a4bd0b3fec4fb83247359929ef79", +}); +const entries = Object.freeze(Object.keys(targetSha256)); +const [patchId = "", ...extra] = process.argv.slice(2); +if ( + extra.length + || !/^engine-mcp-execution-plan-telemetry-runtime-\d{8}-\d{3}$/.test(patchId) +) { + throw new Error( + "usage: build-engine-mcp-execution-plan-telemetry-runtime-artifact.mjs " + + "", + ); +} +const artifact = join(artifactRoot, `nodedc-${patchId}.tgz`); + +await assertFresh(artifact); +await assertExactSources(); +const stage = await mkdtemp( + join(tmpdir(), "nodedc-engine-mcp-execution-plan-telemetry-runtime-"), +); +try { + const payload = join(stage, "payload"); + for (const relativePath of entries) { + const destination = join(payload, relativePath); + await mkdir(dirname(destination), { recursive: true }); + await copyFile(join(engineRoot, relativePath), destination); + } + await writeFile( + join(stage, "manifest.env"), + `id=${patchId}\ncomponent=engine\ntype=app-overlay\n`, + { encoding: "utf8", flag: "wx", mode: 0o644 }, + ); + await writeFile(join(stage, "files.txt"), `${entries.join("\n")}\n`, { + encoding: "utf8", + flag: "wx", + mode: 0o644, + }); + await mkdir(artifactRoot, { recursive: true }); + run("python3", ["-c", canonicalTarScript(), artifact, stage]); + console.log(JSON.stringify({ + ok: true, + patchId, + artifact, + sha256: digest(await readFile(artifact)), + entries, + targetSha256, + services: ["nodedc-backend"], + transition: "execution-plan-materialization-v1-to-telemetry-runtime-v2", + mcpVersion: "0.9.0", + mcpSurface: "external-codex", + compilerVersions: ["1.1.0", "1.2.0"], + legacyRuntimePreserved: true, + telemetryAuthority: [ + "trusted-telemetry-projection", + "visible-sensor-definition", + ], + unprojectedParameters: "discarded", + rawProviderPayloadAtPublish: "forbidden", + providerLogicAuthority: "trusted-provider-package", + untouched: [ + "live L2 graphs", + "n8n workflow data", + "L1", + "Engine UI", + "node-intelligence image and descriptor", + "databases", + "credentials", + "MCP Nginx", + "embedded AI Workspace", + ], + }, null, 2)); +} finally { + await rm(stage, { recursive: true, force: true }); +} + +async function assertExactSources() { + for (const [relativePath, expected] of Object.entries(targetSha256)) { + const sourcePath = join(engineRoot, relativePath); + const info = await lstat(sourcePath); + if (!info.isFile() || info.isSymbolicLink()) { + throw new Error( + `engine_mcp_execution_plan_telemetry_runtime_source_unsafe:${relativePath}`, + ); + } + const actual = digest(await readFile(sourcePath)); + if (actual !== expected) { + throw new Error( + `engine_mcp_execution_plan_telemetry_runtime_target_mismatch:${relativePath}:` + + `expected=${expected}:actual=${actual}`, + ); + } + } + + const compiler = await readFile( + join(engineRoot, "nodedc-source/server/l2ExecutionPlan/compiler.js"), + "utf8", + ); + for (const marker of [ + "if (compilerVersion === '1.1.0')", + "if (compilerVersion !== '1.2.0')", + '"msgParam", "msg_param"', + "descriptor.telemetryProjection", + '"message-param"', + "convertedSensorValue", + "visibleSensorDefinition", + ]) { + if (!compiler.includes(marker)) { + throw new Error( + `engine_mcp_execution_plan_telemetry_runtime_marker_missing:${marker}`, + ); + } + } + if (/gelios|robot2b/i.test(compiler)) { + throw new Error("engine_mcp_execution_plan_telemetry_runtime_provider_hardcode"); + } + + const catalog = JSON.parse(await readFile( + join( + engineRoot, + "nodedc-source/server/assets/execution-plans/v1/catalog.json", + ), + "utf8", + )); + if ( + JSON.stringify(catalog?.runtime?.compilerVersions) + !== JSON.stringify(["1.1.0", "1.2.0"]) + ) { + throw new Error("engine_mcp_execution_plan_telemetry_runtime_catalog_mismatch"); + } + + const descriptor = JSON.parse( + await readFile(join(engineRoot, descriptorPath), "utf8"), + ); + if ( + descriptor?.schemaVersion !== "nodedc.engine.deploy-transition/v1" + || descriptor?.id + !== "engine-mcp-l2-execution-plan-telemetry-runtime-v2" + || descriptor?.mcpVersion !== "0.9.0" + || JSON.stringify(descriptor?.compilerTransition?.supported) + !== JSON.stringify(["1.1.0", "1.2.0"]) + || descriptor?.compilerTransition?.legacyRuntimePreserved !== true + || descriptor?.providerLogicAuthority !== "trusted-provider-package" + || descriptor?.engineProviderHardcode !== false + || descriptor?.embeddedCodexChanged !== false + ) { + throw new Error( + "engine_mcp_execution_plan_telemetry_runtime_descriptor_contract_mismatch", + ); + } +} + +async function assertFresh(path) { + try { + await lstat(path); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + throw new Error("artifact_already_exists"); +} + +function canonicalTarScript() { + return [ + "import gzip,io,pathlib,sys,tarfile", + "root=pathlib.Path(sys.argv[2])", + "with open(sys.argv[1],'xb') as out:", + " with gzip.GzipFile(filename='',mode='wb',fileobj=out,compresslevel=9,mtime=0) as gz:", + " with tarfile.open(fileobj=gz,mode='w',format=tarfile.PAX_FORMAT) as tar:", + " for top in ('manifest.env','files.txt','payload'):", + " p=root/top; paths=[p]+(sorted(p.rglob('*')) if p.is_dir() else [])", + " for x in paths:", + " info=tar.gettarinfo(str(x),arcname=x.relative_to(root).as_posix())", + " info.uid=info.gid=0; info.uname=info.gname='root'; info.mtime=0; info.mode=0o755 if info.isdir() else 0o644", + " with (open(x,'rb') if info.isfile() else io.BytesIO()) as src: tar.addfile(info,src if info.isfile() else None)", + ].join("\n"); +} + +function digest(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function run(command, args) { + const result = spawnSync(command, args, { + encoding: "utf8", + maxBuffer: 128 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + throw new Error(`${command}_failed:${result.stderr || result.stdout}`); + } + return result; +} diff --git a/infra/deploy-runner/nodedc-deploy b/infra/deploy-runner/nodedc-deploy index c7b4d64..b209f20 100755 --- a/infra/deploy-runner/nodedc-deploy +++ b/infra/deploy-runner/nodedc-deploy @@ -612,6 +612,45 @@ ENGINE_MCP_EXECUTION_PLAN_MATERIALIZATION_NEW_PATHS = ( "nodedc-source/server/assets/execution-plans/v1/catalog.json", ENGINE_MCP_EXECUTION_PLAN_MATERIALIZATION_DESCRIPTOR_REL, ) +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_DESCRIPTOR_REL = ( + "nodedc-source/server/deployTransitions/executionPlanTelemetryRuntimeV2.json" +) +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES = ( + "nodedc-source/server/l2ExecutionPlan/compiler.js", + "nodedc-source/server/assets/execution-plans/v1/catalog.json", + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_DESCRIPTOR_REL, +) +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_PREDECESSOR_SHA256 = { + "nodedc-source/server/l2ExecutionPlan/compiler.js": + "beb3f664073f9a372432643936a04d0cb0028cd695f759c68bd053e9cd892fe4", + "nodedc-source/server/assets/execution-plans/v1/catalog.json": + "a153e040e0a592bad4375b98d9a1d83d148923aa0f0fd47d225b92eda281c4e9", +} +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256 = { + "nodedc-source/server/routes/n8n.js": + "391fc81228fdacd6efc6c0868991a3485f71869708e49ac15819db6ccce1bfce", + "nodedc-source/server/routes/engineAgentGateway.js": + "9020cf49a3558c0497fed4ecfd81367cbc11b5b249881804c29fe437900c71f8", + "nodedc-source/server/l2ExecutionPlan/catalog.js": + "c35b4c7ad9319aabbf1366a11ff52a6e99d961893bafdfcb4e84fc5f24fc04be", + "nodedc-source/server/l2ExecutionPlan/materializer.js": + "ee3bcfd06b3a5fa46800df974a2dedfdd55eeaf662329f837486c011a9bd713e", + ENGINE_MCP_EXECUTION_PLAN_MATERIALIZATION_DESCRIPTOR_REL: + "52c0152cff49c251be5581a9209d2e63ba710c16e815bdd6ece24f7c9dd7e480", + ENGINE_NODE_INTELLIGENCE_DESCRIPTOR_REL: + "03b2ba120c3929e9cf99940ddc927082de376ceff762895a84103144202aef42", +} +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256 = { + "nodedc-source/server/l2ExecutionPlan/compiler.js": + "6b783ad15c26dc7de0645082c8a426002d943138c70bf31a240247b16a33b6a0", + "nodedc-source/server/assets/execution-plans/v1/catalog.json": + "5bdfc92284a7c836ff326e3a89b559110e376b32efd34b5f23f2bec272745781", + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_DESCRIPTOR_REL: + "68b275efb6303284336d8b637c966c24d891a4bd0b3fec4fb83247359929ef79", +} +ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS = ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_DESCRIPTOR_REL, +) ENGINE_AGENT_FULL_GRANT_MIGRATION_ARTIFACT_ENTRIES = ( ENGINE_AGENT_FULL_GRANT_MIGRATION_STORE_REL, ) @@ -4506,6 +4545,44 @@ process.stdout.write('engine-mcp-execution-plan-materialization:0.9.0:provider-p } +def accept_engine_mcp_execution_plan_telemetry_runtime(): + root = component_root("engine") + validate_engine_mcp_execution_plan_telemetry_runtime_slice( + root, + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES, + ) + live = run_engine_backend_probe( + ( + "node", + "--input-type=module", + "-e", + """ +const fs=await import('node:fs/promises'); +const catalogModule=await import('file:///app/server/l2ExecutionPlan/catalog.js'); +const catalog=await catalogModule.loadExecutionPlanCatalog(); +const compiler=await fs.readFile('/app/server/l2ExecutionPlan/compiler.js','utf8'); +const versions=catalog.runtime?.compilerVersions||[]; +if(versions.join(',')!=='1.1.0,1.2.0'||!compiler.includes("if (compilerVersion === '1.1.0')")||!compiler.includes("if (compilerVersion !== '1.2.0')")||!compiler.includes('"msgParam", "msg_param"')||!compiler.includes('descriptor.telemetryProjection')||!compiler.includes('"message-param"')||!compiler.includes('convertedSensorValue')||!compiler.includes('visibleSensorDefinition')||/gelios|robot2b/i.test(compiler))process.exit(2); +process.stdout.write('engine-mcp-execution-plan-telemetry-runtime:0.9.0:compiler-1.2.0:declared-projected:v2'); +""".strip(), + ), + "Engine MCP execution plan telemetry runtime", + container_id=engine_backend_container_id(), + ) + expected = ( + "engine-mcp-execution-plan-telemetry-runtime:" + "0.9.0:compiler-1.2.0:declared-projected:v2" + ) + if live != expected: + die("Engine MCP execution plan telemetry runtime live acceptance mismatch") + return { + "target_sha256": dict( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256 + ), + "live": live, + } + + def validate_no_lifecycle_scripts(payload_dir, label): forbidden = ("preinstall", "install", "postinstall", "prepare", "prepack", "postpack") for package_path in payload_dir.rglob("package.json"): @@ -5391,6 +5468,107 @@ def validate_engine_mcp_execution_plan_materialization_slice(payload_dir, entrie ) +def validate_engine_mcp_execution_plan_telemetry_runtime_slice( + payload_dir, + entries, +): + if ( + tuple(entries) + != ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES + ): + die( + "Engine MCP execution plan telemetry runtime files.txt exact " + "set/order mismatch" + ) + for rel, expected_sha256 in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256.items() + ): + path = payload_dir / rel + try: + path_stat = path.lstat() + except FileNotFoundError: + die( + "Engine MCP execution plan telemetry runtime target is " + f"missing: {rel}" + ) + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Engine MCP execution plan telemetry runtime target sha256 " + f"mismatch: {rel}" + ) + descriptor = read_strict_json( + payload_dir + / ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_DESCRIPTOR_REL, + "Engine MCP execution plan telemetry runtime descriptor", + max_bytes=32 * 1024, + ) + expected_descriptor = { + "schemaVersion": "nodedc.engine.deploy-transition/v1", + "id": "engine-mcp-l2-execution-plan-telemetry-runtime-v2", + "component": "engine", + "scope": "external-mcp-l2-authoring-runtime", + "mcpVersion": "0.9.0", + "sourcePaths": [ + "nodedc-source/server/l2ExecutionPlan/compiler.js", + "nodedc-source/server/assets/execution-plans/v1/catalog.json", + ], + "compilerTransition": { + "predecessor": "1.1.0", + "supported": ["1.1.0", "1.2.0"], + "legacyRuntimePreserved": True, + }, + "telemetryInputBoundary": { + "sources": [ + "declared-sensor-values", + "declared-message-parameters", + ], + "authorization": [ + "trusted-telemetry-projection", + "visible-sensor-definition", + ], + "unprojectedParameters": "discarded", + "rawProviderPayloadAtPublish": "forbidden", + "conversion": "bounded-declared-conversion-table", + }, + "providerLogicAuthority": "trusted-provider-package", + "engineProviderHardcode": False, + "embeddedCodexChanged": False, + } + if descriptor != expected_descriptor: + die( + "Engine MCP execution plan telemetry runtime descriptor " + "contract mismatch" + ) + compiler_source = ( + payload_dir / "nodedc-source/server/l2ExecutionPlan/compiler.js" + ).read_text(encoding="utf-8") + catalog = read_strict_json( + payload_dir + / "nodedc-source/server/assets/execution-plans/v1/catalog.json", + "Engine execution plan catalog", + max_bytes=512 * 1024, + ) + required_markers = ( + "if (compilerVersion === '1.1.0')", + "if (compilerVersion !== '1.2.0')", + '"msgParam", "msg_param"', + "descriptor.telemetryProjection", + '"message-param"', + "convertedSensorValue", + "visibleSensorDefinition", + ) + if any(marker not in compiler_source for marker in required_markers): + die("Engine MCP execution plan telemetry runtime contract mismatch") + if catalog.get("runtime", {}).get("compilerVersions") != ["1.1.0", "1.2.0"]: + die("Engine MCP execution plan telemetry runtime catalog mismatch") + if re.search(r"gelios|robot2b", compiler_source, re.IGNORECASE): + die("Engine MCP execution plan telemetry runtime contains provider hardcode") + + def validate_engine_agent_full_grant_migration_slice(payload_dir, entries): if tuple(entries) != ENGINE_AGENT_FULL_GRANT_MIGRATION_ARTIFACT_ENTRIES: die("Engine agent full grant migration files.txt exact set/order mismatch") @@ -5859,6 +6037,14 @@ def load_artifact(artifact, work_dir): payload_dir, entries, ) + if is_engine_mcp_execution_plan_telemetry_runtime_slice( + manifest["component"], + entries, + ): + validate_engine_mcp_execution_plan_telemetry_runtime_slice( + payload_dir, + entries, + ) if is_engine_provider_security_catalog_slice(manifest["component"], entries): validate_engine_provider_security_catalog_payload(payload_dir, entries) if touches_engine_credential_sink(manifest["component"], entries): @@ -5884,6 +6070,10 @@ def load_artifact(artifact, work_dir): manifest["component"], entries, ) + and not is_engine_mcp_execution_plan_telemetry_runtime_slice( + manifest["component"], + entries, + ) and ( touches_engine_data_product_publish_grant(entries) or ENGINE_DATA_PRODUCT_PUBLISH_GRANT_OVERRIDE_REL in entries @@ -6087,6 +6277,15 @@ def is_engine_mcp_execution_plan_materialization_slice(component, entries): ) +def is_engine_mcp_execution_plan_telemetry_runtime_slice(component, entries): + return ( + component == "engine" + and entries is not None + and tuple(entries) + == ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES + ) + + def is_engine_agent_full_grant_migration_slice(component, entries): return ( component == "engine" @@ -6689,6 +6888,82 @@ def preflight_engine_mcp_execution_plan_materialization_predecessor(): } +def preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor(): + root = component_root("engine") + actual = {} + for rel, expected_sha256 in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_PREDECESSOR_SHA256.items() + ): + path = root / rel + try: + path_stat = path.lstat() + except FileNotFoundError: + die( + "Engine MCP execution plan telemetry runtime predecessor is " + f"missing: {rel}" + ) + if stat.S_ISLNK(path_stat.st_mode) or not stat.S_ISREG(path_stat.st_mode): + die( + "Engine MCP execution plan telemetry runtime predecessor is " + f"unsafe: {rel}" + ) + actual_sha256 = sha256_file(path) + if actual_sha256 != expected_sha256: + die( + "Engine MCP execution plan telemetry runtime predecessor drift " + f"detected: path={rel} expected={expected_sha256} " + f"actual={actual_sha256}" + ) + actual[rel] = actual_sha256 + foundation = {} + for rel, expected_sha256 in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256.items() + ): + path = root / rel + try: + path_stat = path.lstat() + except FileNotFoundError: + die( + "Engine MCP execution plan telemetry runtime foundation is " + f"missing: {rel}" + ) + if ( + stat.S_ISLNK(path_stat.st_mode) + or not stat.S_ISREG(path_stat.st_mode) + or sha256_file(path) != expected_sha256 + ): + die( + "Engine MCP execution plan telemetry runtime foundation drift " + f"detected: {rel}" + ) + foundation[rel] = expected_sha256 + for rel in ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS: + path = root / rel + if path.exists() or path.is_symlink(): + die( + "Engine MCP execution plan telemetry runtime new path already " + f"exists: {rel}" + ) + backend = preflight_engine_credential_backend_runtime() + if backend["mode"] != "verified-derived-retry": + die( + "Engine MCP execution plan telemetry runtime requires the active " + "immutable backend" + ) + return { + "mode": "execution-plan-materialization-v1-to-telemetry-runtime-v2", + "predecessor_sha256": actual, + "foundation_sha256": foundation, + "target_sha256": dict( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256 + ), + "new_paths": tuple( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS + ), + "backend_mode": backend["mode"], + } + + def preflight_engine_agent_full_grant_migration_predecessor(): root = component_root("engine") store_path = root / ENGINE_AGENT_FULL_GRANT_MIGRATION_STORE_REL @@ -6745,6 +7020,7 @@ def component_services(component, entries=None): or is_engine_mcp_execution_profile_decoder_slice(component, entries) or is_engine_mcp_telemetry_catalog_slice(component, entries) or is_engine_mcp_execution_plan_materialization_slice(component, entries) + or is_engine_mcp_execution_plan_telemetry_runtime_slice(component, entries) or is_engine_provider_security_catalog_slice(component, entries) ): # This slice updates only the existing Engine backend control plane. @@ -8956,6 +9232,7 @@ def plan_artifact(artifact): mcp_execution_profile_decoder_preflight = None mcp_telemetry_catalog_preflight = None mcp_execution_plan_materialization_preflight = None + mcp_execution_plan_telemetry_runtime_preflight = None provider_catalog_preflight = None with tempfile.TemporaryDirectory(prefix="plan-", dir=TMP_DIR) as tmp: manifest, entries, payload_dir = load_artifact(artifact, Path(tmp)) @@ -9028,6 +9305,13 @@ def plan_artifact(artifact): mcp_execution_plan_materialization_preflight = ( preflight_engine_mcp_execution_plan_materialization_predecessor() ) + if is_engine_mcp_execution_plan_telemetry_runtime_slice( + manifest["component"], + entries, + ): + mcp_execution_plan_telemetry_runtime_preflight = ( + preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor() + ) if is_engine_provider_security_catalog_slice(manifest["component"], entries): provider_catalog_preflight = preflight_engine_provider_security_catalog_predecessor() @@ -9076,6 +9360,12 @@ def plan_artifact(artifact): entries, ) ) + touches_mcp_execution_plan_telemetry_runtime = ( + is_engine_mcp_execution_plan_telemetry_runtime_slice( + component, + entries, + ) + ) touches_agent_grant_migration = is_engine_agent_full_grant_migration_slice( component, entries, @@ -9094,6 +9384,7 @@ def plan_artifact(artifact): or touches_mcp_execution_profile_decoder or touches_mcp_telemetry_catalog or touches_mcp_execution_plan_materialization + or touches_mcp_execution_plan_telemetry_runtime or touches_agent_grant_migration ): credential_backend_preflight = preflight_engine_credential_backend_runtime() @@ -9152,6 +9443,17 @@ def plan_artifact(artifact): "Engine MCP execution plan materialization requires the active " "immutable credential backend" ) + if touches_mcp_execution_plan_telemetry_runtime: + if mcp_execution_plan_telemetry_runtime_preflight is None: + die( + "Engine MCP execution plan telemetry runtime preflight is " + "missing" + ) + if credential_backend_preflight["mode"] != "verified-derived-retry": + die( + "Engine MCP execution plan telemetry runtime requires the " + "active immutable credential backend" + ) if touches_agent_grant_migration: agent_grant_migration_predecessor_sha256 = ( preflight_engine_agent_full_grant_migration_predecessor() @@ -9594,6 +9896,78 @@ def plan_artifact(artifact): print("credentials=preserved") print("mcp_nginx=untouched") print("embedded_ai_workspace=untouched") + if mcp_execution_plan_telemetry_runtime_preflight: + print( + "engine_mcp_authoring_runtime_transition=" + f"{mcp_execution_plan_telemetry_runtime_preflight['mode']}" + ) + print("engine_mcp_version=0.9.0") + print("engine_mcp_surface=external-codex") + print("engine_mcp_compiler_predecessor=1.1.0") + print("engine_mcp_compiler_supported=1.1.0,1.2.0") + print("engine_mcp_legacy_runtime_preserved=yes") + print("engine_mcp_provider_logic_authority=trusted-provider-package") + print( + "engine_mcp_telemetry_authority=" + "trusted-projection+visible-declared-sensor" + ) + print("engine_mcp_unprojected_parameters=discarded") + print("engine_mcp_raw_provider_payload_at_publish=forbidden") + for foundation_path, foundation_sha256 in ( + mcp_execution_plan_telemetry_runtime_preflight[ + "foundation_sha256" + ].items() + ): + print( + "engine_mcp_execution_plan_telemetry_runtime_foundation_sha256" + f"[{foundation_path}]={foundation_sha256}" + ) + for changed_path in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES + ): + print( + "engine_mcp_execution_plan_telemetry_runtime_changed_path=" + f"{changed_path}" + ) + if changed_path in mcp_execution_plan_telemetry_runtime_preflight[ + "predecessor_sha256" + ]: + print( + "engine_mcp_execution_plan_telemetry_runtime_predecessor_sha256" + f"[{changed_path}]=" + f"{mcp_execution_plan_telemetry_runtime_preflight['predecessor_sha256'][changed_path]}" + ) + elif changed_path in mcp_execution_plan_telemetry_runtime_preflight[ + "new_paths" + ]: + print( + "engine_mcp_execution_plan_telemetry_runtime_predecessor_state" + f"[{changed_path}]=absent" + ) + else: + die( + "Engine MCP execution plan telemetry runtime plan has no " + f"predecessor state: {changed_path}" + ) + print( + "engine_mcp_execution_plan_telemetry_runtime_target_sha256" + f"[{changed_path}]=" + f"{mcp_execution_plan_telemetry_runtime_preflight['target_sha256'][changed_path]}" + ) + print( + "backend_current_barrier=" + f"{mcp_execution_plan_telemetry_runtime_preflight['backend_mode']}" + ) + print("backend_force_recreate=yes") + print("backend_pull=never") + print("l2_graph=untouched") + print("n8n_l1=untouched") + print("engine_ui=untouched") + print("engine_databases=untouched") + print("credentials=preserved") + print("node_intelligence_image=preserved") + print("mcp_nginx=untouched") + print("embedded_ai_workspace=untouched") if transition_descriptor: print(f"n8n_transition={transition_descriptor['action']}") print(f"n8n_version={transition_descriptor['n8nVersion']}") @@ -10716,6 +11090,10 @@ def component_healthchecks(component, entries=None, services=None): component, entries, ) + or is_engine_mcp_execution_plan_telemetry_runtime_slice( + component, + entries, + ) or is_engine_agent_full_grant_migration_slice(component, entries) or is_engine_mcp_control_plane_slice(component, entries) or is_engine_mcp_ontology_sdk_slice(component, entries) @@ -11139,6 +11517,12 @@ def run_healthchecks(component, entries=None, services=None): entries, ) ) + touches_mcp_execution_plan_telemetry_runtime = ( + is_engine_mcp_execution_plan_telemetry_runtime_slice( + component, + entries, + ) + ) touches_agent_grant_migration = is_engine_agent_full_grant_migration_slice( component, entries, @@ -11158,6 +11542,7 @@ def run_healthchecks(component, entries=None, services=None): or touches_mcp_execution_profile_decoder or touches_mcp_telemetry_catalog or touches_mcp_execution_plan_materialization + or touches_mcp_execution_plan_telemetry_runtime or touches_agent_grant_migration or touches_mcp_control_plane or touches_mcp_ontology_sdk @@ -11181,6 +11566,7 @@ def run_healthchecks(component, entries=None, services=None): or touches_mcp_execution_profile_decoder or touches_mcp_telemetry_catalog or touches_mcp_execution_plan_materialization + or touches_mcp_execution_plan_telemetry_runtime or touches_agent_grant_migration or touches_mcp_control_plane or touches_mcp_ontology_sdk @@ -11368,6 +11754,48 @@ def run_healthchecks(component, entries=None, services=None): "Engine MCP execution plan materialization installed state is " "neither target nor rollback predecessor" ) + if touches_mcp_execution_plan_telemetry_runtime: + root = component_root("engine") + target_state = all( + (root / rel).is_file() + and not (root / rel).is_symlink() + and sha256_file(root / rel) == expected + for rel, expected in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256.items() + ) + ) and all( + (root / rel).is_file() + and not (root / rel).is_symlink() + and sha256_file(root / rel) == expected + for rel, expected in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256.items() + ) + ) + predecessor_state = all( + (root / rel).is_file() + and not (root / rel).is_symlink() + and sha256_file(root / rel) == expected + for rel, expected in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_PREDECESSOR_SHA256.items() + ) + ) and all( + (root / rel).is_file() + and not (root / rel).is_symlink() + and sha256_file(root / rel) == expected + for rel, expected in ( + ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256.items() + ) + ) and all( + not (root / rel).exists() and not (root / rel).is_symlink() + for rel in ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS + ) + if target_state: + accept_engine_mcp_execution_plan_telemetry_runtime() + elif not predecessor_state: + die( + "Engine MCP execution plan telemetry runtime installed state " + "is neither target nor rollback predecessor" + ) container_name = COMPONENTS[component].get("health_container") if container_name: healthcheck_container(container_name) @@ -11501,6 +11929,22 @@ def apply_artifact(artifact): "Engine MCP execution plan materialization requires " "the active immutable credential backend" ) + if is_engine_mcp_execution_plan_telemetry_runtime_slice( + component, + entries, + ): + preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor() + execution_plan_telemetry_runtime_backend_preflight = ( + preflight_engine_credential_backend_runtime() + ) + if ( + execution_plan_telemetry_runtime_backend_preflight["mode"] + != "verified-derived-retry" + ): + die( + "Engine MCP execution plan telemetry runtime " + "requires the active immutable credential backend" + ) if is_engine_agent_full_grant_migration_slice(component, entries): preflight_engine_agent_full_grant_migration_predecessor() migration_backend_preflight = preflight_engine_credential_backend_runtime() @@ -11762,6 +12206,10 @@ def apply_artifact(artifact): component, entries, ) + or is_engine_mcp_execution_plan_telemetry_runtime_slice( + component, + entries, + ) or is_engine_agent_full_grant_migration_slice(component, entries) or is_engine_mcp_control_plane_slice(component, entries) or is_engine_mcp_ontology_sdk_slice(component, entries) diff --git a/infra/deploy-runner/test_engine_mcp_execution_plan_telemetry_runtime.py b/infra/deploy-runner/test_engine_mcp_execution_plan_telemetry_runtime.py new file mode 100644 index 0000000..79cd36b --- /dev/null +++ b/infra/deploy-runner/test_engine_mcp_execution_plan_telemetry_runtime.py @@ -0,0 +1,352 @@ +import hashlib +import importlib.machinery +import importlib.util +import io +import json +import os +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from contextlib import redirect_stdout +from unittest import mock + + +SCRIPT_DIR = Path(__file__).resolve().parent +RUNNER_PATH = SCRIPT_DIR / "nodedc-deploy" +BUILDER_PATH = ( + SCRIPT_DIR + / "build-engine-mcp-execution-plan-telemetry-runtime-artifact.mjs" +) +ENGINE_ROOT = SCRIPT_DIR.parent.parent.parent / "NODEDC_ENGINE_INFRA" +PATCH_ID = "engine-mcp-execution-plan-telemetry-runtime-20991231-999" + + +def load_runner(): + loader = importlib.machinery.SourceFileLoader( + "nodedc_engine_mcp_execution_plan_telemetry_runtime", + str(RUNNER_PATH), + ) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +RUNNER = load_runner() + + +class EngineMcpExecutionPlanTelemetryRuntimeTest(unittest.TestCase): + def require_current_target_source(self): + for relative_path, expected in ( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256 + .items() + ): + path = ENGINE_ROOT / relative_path + if ( + not path.is_file() + or hashlib.sha256(path.read_bytes()).hexdigest() != expected + ): + self.skipTest( + "execution plan telemetry runtime source has advanced" + ) + + def build(self, artifact_dir): + environment = os.environ.copy() + environment["NODEDC_ENGINE_SOURCE_ROOT"] = str(ENGINE_ROOT) + environment["NODEDC_DEPLOY_ARTIFACT_DIR"] = str(artifact_dir) + completed = subprocess.run( + ["node", str(BUILDER_PATH), PATCH_ID], + check=True, + capture_output=True, + text=True, + env=environment, + ) + return json.loads(completed.stdout) + + def test_builder_emits_exact_deterministic_backend_only_successor(self): + self.require_current_target_source() + with tempfile.TemporaryDirectory( + prefix="nodedc-engine-mcp-execution-plan-telemetry-runtime-" + ) as directory: + root = Path(directory) + first = self.build(root / "first") + second = self.build(root / "second") + first_artifact = Path(first["artifact"]) + second_artifact = Path(second["artifact"]) + self.assertEqual(first_artifact.read_bytes(), second_artifact.read_bytes()) + self.assertEqual(first["services"], ["nodedc-backend"]) + self.assertEqual(first["mcpVersion"], "0.9.0") + self.assertEqual(first["compilerVersions"], ["1.1.0", "1.2.0"]) + self.assertTrue(first["legacyRuntimePreserved"]) + self.assertEqual(first["unprojectedParameters"], "discarded") + self.assertEqual( + tuple(first["entries"]), + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES, + ) + + extract = root / "extract" + with tarfile.open(first_artifact, "r:gz") as archive: + archive.extractall(extract, filter="data") + entries = tuple( + (extract / "files.txt").read_text(encoding="utf-8").splitlines() + ) + payload = extract / "payload" + self.assertTrue( + RUNNER.is_engine_mcp_execution_plan_telemetry_runtime_slice( + "engine", + entries, + ) + ) + self.assertFalse( + RUNNER.is_engine_mcp_execution_plan_materialization_slice( + "engine", + entries, + ) + ) + self.assertEqual( + RUNNER.component_services("engine", entries), + ("nodedc-backend",), + ) + RUNNER.validate_engine_mcp_execution_plan_telemetry_runtime_slice( + payload, + entries, + ) + with tempfile.TemporaryDirectory( + prefix="nodedc-engine-mcp-execution-plan-telemetry-load-" + ) as load_directory: + loaded = RUNNER.load_artifact( + first_artifact, + Path(load_directory), + ) + self.assertEqual(tuple(loaded[1]), entries) + + def test_preflight_requires_exact_036_foundation_and_absent_descriptor(self): + with tempfile.TemporaryDirectory( + prefix="nodedc-engine-mcp-execution-plan-telemetry-preflight-" + ) as directory: + root = Path(directory) + hashes = {} + for mapping in ( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_PREDECESSOR_SHA256, + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256, + ): + for relative_path, expected in mapping.items(): + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("installed\n", encoding="utf-8") + hashes[path] = expected + with ( + mock.patch.object(RUNNER, "component_root", return_value=root), + mock.patch.object( + RUNNER, + "sha256_file", + side_effect=lambda path: hashes[Path(path)], + ), + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value={"mode": "verified-derived-retry"}, + ), + ): + result = ( + RUNNER + .preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor() + ) + self.assertEqual( + result["mode"], + "execution-plan-materialization-v1-to-telemetry-runtime-v2", + ) + self.assertEqual( + result["foundation_sha256"], + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256, + ) + + new_path = ( + root + / RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS[0] + ) + new_path.parent.mkdir(parents=True, exist_ok=True) + new_path.write_text("{}\n", encoding="utf-8") + with ( + mock.patch.object(RUNNER, "component_root", return_value=root), + mock.patch.object( + RUNNER, + "sha256_file", + side_effect=lambda path: hashes[Path(path)], + ), + ): + with self.assertRaises(RUNNER.DeployError): + ( + RUNNER + .preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor() + ) + + def test_plan_renders_closed_provider_neutral_telemetry_boundary(self): + self.require_current_target_source() + with tempfile.TemporaryDirectory( + prefix="nodedc-engine-mcp-execution-plan-telemetry-plan-" + ) as directory: + root = Path(directory) + artifact = Path(self.build(root / "artifacts")["artifact"]) + live_root = root / "live" + live_root.mkdir() + preflight = { + "mode": "execution-plan-materialization-v1-to-telemetry-runtime-v2", + "predecessor_sha256": dict( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_PREDECESSOR_SHA256 + ), + "foundation_sha256": dict( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256 + ), + "target_sha256": dict( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256 + ), + "new_paths": tuple( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_NEW_PATHS + ), + "backend_mode": "verified-derived-retry", + } + output = io.StringIO() + with ( + mock.patch.object(RUNNER, "validate_artifact_location"), + mock.patch.object(RUNNER, "ensure_layout"), + mock.patch.object(RUNNER, "TMP_DIR", root), + mock.patch.object(RUNNER, "component_root", return_value=live_root), + mock.patch.object( + RUNNER, + "component_compose_root", + return_value=live_root, + ), + mock.patch.object( + RUNNER, + "preflight_engine_mcp_execution_plan_telemetry_runtime_predecessor", + return_value=preflight, + ), + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value={"mode": "verified-derived-retry"}, + ), + mock.patch.object(RUNNER, "state_has_sha", return_value=False), + mock.patch.object(RUNNER, "state_has_patch_id", return_value=False), + redirect_stdout(output), + ): + RUNNER.plan_artifact(artifact) + plan = output.getvalue() + self.assertIn("services=nodedc-backend", plan) + self.assertIn("engine_mcp_version=0.9.0", plan) + self.assertIn( + "engine_mcp_compiler_supported=1.1.0,1.2.0", + plan, + ) + self.assertIn("engine_mcp_legacy_runtime_preserved=yes", plan) + self.assertIn( + "engine_mcp_telemetry_authority=" + "trusted-projection+visible-declared-sensor", + plan, + ) + self.assertIn("engine_mcp_unprojected_parameters=discarded", plan) + self.assertIn( + "engine_mcp_raw_provider_payload_at_publish=forbidden", + plan, + ) + self.assertIn("l2_graph=untouched", plan) + self.assertIn("embedded_ai_workspace=untouched", plan) + self.assertIn("state=new", plan) + + def test_healthchecks_dispatch_versioned_runtime_acceptance(self): + entries = ( + RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_ARTIFACT_ENTRIES + ) + with tempfile.TemporaryDirectory( + prefix="nodedc-engine-mcp-execution-plan-telemetry-health-" + ) as directory: + root = Path(directory) + all_hashes = { + **RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_TARGET_SHA256, + **RUNNER + .ENGINE_MCP_EXECUTION_PLAN_TELEMETRY_RUNTIME_FOUNDATION_SHA256, + } + for relative_path in all_hashes: + path = root / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("target\n", encoding="utf-8") + + def target_hash(path): + relative_path = Path(path).relative_to(root).as_posix() + return all_hashes[relative_path] + + with ( + mock.patch.object(RUNNER, "component_root", return_value=root), + mock.patch.object(RUNNER, "sha256_file", side_effect=target_hash), + mock.patch.object(RUNNER, "healthcheck_compose_service"), + mock.patch.object(RUNNER, "component_healthchecks", return_value=()), + mock.patch.object( + RUNNER, + "preflight_engine_credential_backend_runtime", + return_value={"mode": "verified-derived-retry"}, + ), + mock.patch.object( + RUNNER, + "accept_engine_mcp_execution_plan_telemetry_runtime", + return_value={"live": "accepted"}, + ) as acceptance, + mock.patch.object(RUNNER, "healthcheck_container"), + ): + RUNNER.run_healthchecks( + "engine", + entries, + ("nodedc-backend",), + ) + acceptance.assert_called_once_with() + + def test_live_acceptance_probes_version_and_provider_neutral_boundary(self): + expected_live = ( + "engine-mcp-execution-plan-telemetry-runtime:" + "0.9.0:compiler-1.2.0:declared-projected:v2" + ) + with ( + mock.patch.object(RUNNER, "component_root", return_value=ENGINE_ROOT), + mock.patch.object( + RUNNER, + "validate_engine_mcp_execution_plan_telemetry_runtime_slice", + ), + mock.patch.object( + RUNNER, + "engine_backend_container_id", + return_value="backend", + ), + mock.patch.object( + RUNNER, + "run_engine_backend_probe", + return_value=expected_live, + ) as backend_probe, + ): + result = ( + RUNNER + .accept_engine_mcp_execution_plan_telemetry_runtime() + ) + self.assertEqual(result["live"], expected_live) + probe_source = backend_probe.call_args.args[0][-1] + self.assertIn("1.1.0,1.2.0", probe_source) + self.assertIn("msgParam", probe_source) + self.assertIn("descriptor.telemetryProjection", probe_source) + self.assertIn("gelios|robot2b", probe_source) + + +if __name__ == "__main__": + unittest.main(verbosity=2)