SILVER
/
NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(n8n): make replacement replay idempotent

This commit is contained in:
Codex 2026-07-21 15:06:59 +03:00
parent bc23c550db
commit 358b259aac
10 changed files with 117 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
infra/deploy-runner/build-engine-n8n-private-extension-artifact.mjs
View File

@ -13,22 +13,22 @@ const engineRoot = resolve(platformRoot, "../NODEDC_ENGINE_INFRA");
const artifactRoot = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || join(here, "../deploy-artifacts"));
const stageArtifact = resolve(
process.env.NODEDC_N8N_EXTENSION_STAGE_ARTIFACT
|| join(artifactRoot, "nodedc-n8n-private-extension-n8n-nodes-ndc-geozone-replace-20260720-017.tgz"),
|| join(artifactRoot, "nodedc-n8n-private-extension-n8n-nodes-ndc-replace-replay-v1-20260721-009.tgz"),
);
const predecessorArtifact = resolve(
process.env.NODEDC_N8N_EXTENSION_PREDECESSOR_ARTIFACT
|| join(artifactRoot, "nodedc-engine-n8n-private-extension-20260718-008.tgz"),
|| join(artifactRoot, "nodedc-engine-n8n-private-extension-20260721-005.tgz"),
);
const stageArtifactSha256 = "0c680d2436917eb6f2416c5c855f2e01dd59ed50bedb1e36456a7405f3f595af";
const predecessorArtifactSha256 = "5ef46404dfd7753b4ffd14c5e9b9666f96d2c9875c8fe470714cdea1c6decdce";
const releaseId = "0.1.5-3c8ae53f010d7c88";
const packageVersion = "0.1.5";
const packageSha256 = "3c8ae53f010d7c88b6e90cb8fc4929e7d0880089ad8c3a9edb876ce034786743";
const stageArtifactSha256 = "4984d4937e2c8de1ecbf5b836cf7fafac131cc0dc64d49c06034507d0cb94577";
const predecessorArtifactSha256 = "037d4790b03c0b245b543fe2a841d28f3cfc7822b9a12f2bff9324bf1328b395";
const releaseId = "0.1.6-25cc2d7a52a0d0ee";
const packageVersion = "0.1.6";
const packageSha256 = "25cc2d7a52a0d0ee288b7a4b3de9f436489c010f234498a94bcccd114d7355af";
const n8nVersion = "2.3.2";
const baseImage = "docker.n8n.io/n8nio/n8n:2.3.2";
const architecture = "amd64";
const generatedAt = "2026-07-21T10:30:00.000Z";
const generatedAt = "2026-07-21T12:00:00.000Z";
const previouslyIssuedTransitionIds = new Set([
"20260715-002",
"20260716-003",
@ -37,6 +37,8 @@ const previouslyIssuedTransitionIds = new Set([
"20260718-006",
"20260718-007",
"20260718-008",
"20260721-005",
"20260721-008",
]);
const transitionId = readTransitionId(process.argv.slice(2), process.env.NODEDC_N8N_TRANSITION_ID);
const activationId = `engine-n8n-private-extension-${transitionId}`;
@ -354,15 +356,15 @@ function assertPackage(value) {
function assertPredecessorCatalogs(descriptorValue, nodes, credentials, meta) {
if (descriptorValue?.action !== "activate"
|| descriptorValue?.releaseId !== "0.1.4-59dc9f7882721d6a"
|| descriptorValue?.packageVersion !== "0.1.4"
|| descriptorValue?.packageSha256 !== "59dc9f7882721d6a5e2ae84ffa6aa8cd9bc5432f5904d24e70e363573ed2757f") {
|| descriptorValue?.releaseId !== "0.1.5-3c8ae53f010d7c88"
|| descriptorValue?.packageVersion !== "0.1.5"
|| descriptorValue?.packageSha256 !== "3c8ae53f010d7c88b6e90cb8fc4929e7d0880089ad8c3a9edb876ce034786743") {
throw new Error("predecessor_descriptor_mismatch");
}
if (!Array.isArray(nodes) || nodes.length !== 437
|| !Array.isArray(credentials) || credentials.length !== 389
|| meta?.n8nVersion !== n8nVersion
|| meta?.source !== "n8n-core+n8n-nodes-ndc@0.1.4"
|| meta?.source !== "n8n-core+n8n-nodes-ndc@0.1.5"
|| meta?.nodeCount !== 437
|| meta?.credentialCount !== 389) {
throw new Error("predecessor_catalog_mismatch");

4
infra/deploy-runner/build-n8n-private-extension-artifact.mjs
View File

@ -12,8 +12,8 @@ const platformRoot = resolve(scriptDir, "../..");
const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc");
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
const requireModule = createRequire(import.meta.url);
const expectedPackageVersion = "0.1.5";
const [patchId = "n8n-nodes-ndc-geozone-replace-20260720-017", ...extra] = process.argv.slice(2);
const expectedPackageVersion = "0.1.6";
const [patchId = "n8n-nodes-ndc-replace-replay-v1-20260721-009", ...extra] = process.argv.slice(2);
const expectedRuntimeNodes = [
{
file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",

6
infra/deploy-runner/nodedc-deploy
View File

@ -291,6 +291,11 @@ ENGINE_N8N_RELEASE_CATALOG_JSON_SHA256 = {
"credentials": "af8ada839070a4c24b9b10981a751c2b2db651dced1392333d7a4ef13de4564e",
"meta": "b324592c38d518af8320d8540c7061e4684056ae88a676cf5e50c6eef03c0dd7",
},
"0.1.6-25cc2d7a52a0d0ee": {
"nodes": "fb2ef4d38b9c071fae117a27381714658a1a19334b82fc79d64d13f27555fe5a",
"credentials": "af8ada839070a4c24b9b10981a751c2b2db651dced1392333d7a4ef13de4564e",
"meta": "58adcb182e5f231a4798636a99887a1f214e3c6e1eb4c0dc2d9b2f3aa80a99ef",
},
}
ENGINE_N8N_INACTIVE_NODES_CATALOG_JSON_SHA256 = "b70d9d8130d498c55de46a5d0758c844f1242b70803457b2291ab3a9de8056f2"
ENGINE_N8N_INACTIVE_CREDENTIALS_CATALOG_JSON_SHA256 = "680e9f52aac791efbd38e3bd99bd51ef5cded9756d867897c6c2755850e87b50"
@ -337,6 +342,7 @@ ENGINE_N8N_CREDENTIAL_TYPES_BY_RELEASE = {
"0.1.3-3354149b5245e39a": ENGINE_N8N_CREDENTIAL_TYPES[:3],
"0.1.4-59dc9f7882721d6a": ENGINE_N8N_CREDENTIAL_TYPES,
"0.1.5-3c8ae53f010d7c88": ENGINE_N8N_CREDENTIAL_TYPES,
"0.1.6-25cc2d7a52a0d0ee": ENGINE_N8N_CREDENTIAL_TYPES,
}
ENGINE_CREDENTIAL_SINK_ARTIFACT_ENTRIES = (
"nodedc-source/server/credentialPolicies/ndcPrivateNode.js",

20
infra/deploy-runner/test_engine_n8n_private_extension.py
View File

@ -23,17 +23,17 @@ BUILDER_PATH = SCRIPT_DIR / "build-engine-n8n-private-extension-artifact.mjs"
STAGE_ARTIFACT = (
PLATFORM_ROOT
/ "infra/deploy-artifacts"
/ "nodedc-n8n-private-extension-n8n-nodes-ndc-geozone-replace-20260720-017.tgz"
/ "nodedc-n8n-private-extension-n8n-nodes-ndc-replace-replay-v1-20260721-009.tgz"
)
PREDECESSOR_ARTIFACT = (
PLATFORM_ROOT
/ "infra/deploy-artifacts"
/ "nodedc-engine-n8n-private-extension-20260718-008.tgz"
/ "nodedc-engine-n8n-private-extension-20260721-005.tgz"
)
TRANSITION_ID = "20260721-005"
SEALED_RELEASE_ID = "0.1.5-3c8ae53f010d7c88"
SEALED_PACKAGE_SHA256 = "3c8ae53f010d7c88b6e90cb8fc4929e7d0880089ad8c3a9edb876ce034786743"
PREDECESSOR_RELEASE_ID = "0.1.4-59dc9f7882721d6a"
TRANSITION_ID = "20260721-010"
SEALED_RELEASE_ID = "0.1.6-25cc2d7a52a0d0ee"
SEALED_PACKAGE_SHA256 = "25cc2d7a52a0d0ee288b7a4b3de9f436489c010f234498a94bcccd114d7355af"
PREDECESSOR_RELEASE_ID = "0.1.5-3c8ae53f010d7c88"
BUILDER_ENGINE_PATHS = (
"nodedc-source/server/assets/n8n/schema/v2.3.2/nodes.catalog.json",
"nodedc-source/server/assets/n8n/schema/v2.3.2/credentials.catalog.json",
@ -177,7 +177,9 @@ class EngineN8nPrivateExtensionTest(unittest.TestCase):
([], "transition_id_required"),
(["20260718-007"], "transition_id_already_issued"),
(["20260718-008"], "transition_id_already_issued"),
(["engine-n8n-private-extension-20260721-005"], "transition_id_invalid"),
(["20260721-005"], "transition_id_already_issued"),
(["20260721-008"], "transition_id_already_issued"),
(["engine-n8n-private-extension-20260721-010"], "transition_id_invalid"),
(["20260230-004"], "transition_id_invalid"),
(["20260717-000"], "transition_id_invalid"),
([TRANSITION_ID, "unexpected-second-id"], "transition_id_argument_count_invalid"),
@ -288,7 +290,7 @@ class EngineN8nPrivateExtensionTest(unittest.TestCase):
self.assertEqual(override, RUNNER.expected_engine_n8n_compose_override(descriptor))
self.assertEqual(
hashlib.sha256(override.encode("utf-8")).hexdigest(),
"304377edf7d0e184fa4156475df74b7a807a47e4472c8d8e13102639eb9e9d0b",
"7fc118de152b7f5b76a7a502aea0dfc8d7dd191f6bfd304d6de9a4bf06f8fe1e",
)
self.assertIn("pull_policy: never", override)
self.assertIn("N8N_USER_FOLDER: /home/node", override)
@ -580,7 +582,7 @@ class EngineN8nPrivateExtensionTest(unittest.TestCase):
def test_sealed_release_exact_set_includes_implicit_directories(self):
release_relative = Path(
"payload/releases/n8n-nodes-ndc/0.1.5-3c8ae53f010d7c88"
"payload/releases/n8n-nodes-ndc/0.1.6-25cc2d7a52a0d0ee"
)
with tempfile.TemporaryDirectory() as directory:
work = Path(directory)

8
infra/deploy-runner/test_n8n_private_extension.py
View File

@ -19,7 +19,7 @@ RUNNER_PATH = Path(
os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy")
).resolve()
BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs"
PATCH_ID = "n8n-nodes-ndc-geozone-replace-20260720-017"
PATCH_ID = "n8n-nodes-ndc-replace-replay-v1-20260721-009"
EXPECTED_NODES = [
"dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
"dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js",
@ -110,7 +110,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory))
self.assertEqual(manifest["component"], "n8n-private-extension")
self.assertEqual(len(entries), 1)
self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.5-[a-f0-9]{16}$")
self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.6-[a-f0-9]{16}$")
with tarfile.open(self.artifacts[0], "r:gz") as archive:
names = archive.getnames()
@ -122,7 +122,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive:
package_json_member = archive.getmember("package/package.json")
package_json = json.loads(archive.extractfile(package_json_member).read())
self.assertEqual(package_json["version"], "0.1.5")
self.assertEqual(package_json["version"], "0.1.6")
self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES)
self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS)
for node_path in EXPECTED_NODES:
@ -158,7 +158,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
"requiresPreActivationVerification": True,
}
self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2")
self.assertEqual(release["package"]["version"], "0.1.5")
self.assertEqual(release["package"]["version"], "0.1.6")
self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy)
self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2")
self.assertEqual(rollback["baselinePolicy"], expected_policy)

6
packages/n8n-nodes-ndc/README.md
View File

@ -98,7 +98,11 @@ Each incoming item must be either a canonical fact or `{ "fact": <fact> }`:
The node emits `nodedc.data-product.publish/v1`, enforces the 5000-fact and
64-KiB-per-attributes limits, rejects secret-like material in attributes, and
derives stable run/idempotency identifiers from workflow execution context.
derives upsert run/idempotency identifiers from workflow execution context.
Complete-snapshot replacement instead derives one canonical identity from the
product, generation timestamp, sequence and normalized fact set. Replaying the
same immutable generation therefore reaches the Data Plane as the same request,
while changed content under an already-used generation remains a conflict.
## Build and verification

42
packages/n8n-nodes-ndc/nodes/shared/contracts.ts
View File

@ -109,10 +109,6 @@ export function buildPublishPayload(
): NdcPublishPayload {
if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid');
requireIdentifier(dataProductId, 'dataProductId');
const runId = `run-${sha256(`${workflowId}|${executionId}`).slice(0, 48)}`;
const idempotencyKey = `publish-${sha256(
`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`,
)}`;
const facts = factsFromItems(items, publishMode === 'replace');
const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts)
? items[0].json.generationAt
@ -120,6 +116,15 @@ export function buildPublishPayload(
const generationAt = publishMode === 'replace'
? replacementGenerationAt(facts, explicitGenerationAt)
: undefined;
const replacementDigest = generationAt
? replacementReplayDigest(dataProductId, sequence, generationAt, facts)
: undefined;
const runId = replacementDigest
? `run-${replacementDigest.slice(0, 48)}`
: `run-${sha256(`${workflowId}|${executionId}`).slice(0, 48)}`;
const idempotencyKey = replacementDigest
? `publish-${replacementDigest}`
: `publish-${sha256(`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`)}`;
return {
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
batch: {
@ -332,6 +337,35 @@ function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string {
return timestamps[0];
}
function replacementReplayDigest(
dataProductId: string,
sequence: number,
generationAt: string,
facts: NdcFact[],
): string {
const canonicalFacts = [...facts].sort((left, right) => {
const leftKey = `${left.sourceId}\u0000${left.semanticType}`;
const rightKey = `${right.sourceId}\u0000${right.semanticType}`;
return leftKey.localeCompare(rightKey);
});
return sha256(stableJson({
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
dataProductId,
mode: 'replace',
generationAt,
sequence,
facts: canonicalFacts,
}));
}
function stableJson(value: unknown): string {
if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
if (isObject(value)) {
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(',')}}`;
}
return JSON.stringify(value);
}
function requireIsoTimestamp(value: unknown, field: string): string {
const normalized = String(value ?? '').trim();
if (!normalized || Number.isNaN(Date.parse(normalized))) throw new Error(`${field}_invalid`);

4
packages/n8n-nodes-ndc/package-lock.json generated
View File

@ -1,12 +1,12 @@
{
"name": "n8n-nodes-ndc",
"version": "0.1.5",
"version": "0.1.6",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "n8n-nodes-ndc",
"version": "0.1.5",
"version": "0.1.6",
"license": "UNLICENSED",
"devDependencies": {
"@n8n/node-cli": "0.39.3",

2
packages/n8n-nodes-ndc/package.json
View File

@ -1,6 +1,6 @@
{
"name": "n8n-nodes-ndc",
"version": "0.1.5",
"version": "0.1.6",
"description": "Private NODE.DC nodes for scoped data products and Foundry bindings.",
"private": true,
"license": "UNLICENSED",

34
packages/n8n-nodes-ndc/test/package-policy.test.cjs
View File

@ -170,6 +170,39 @@ async function main() {
assert.equal(externalContract.validateDataProductPublish(replacement).ok, true);
assert.equal(replacement.batch.mode, 'replace');
assert.equal(replacement.batch.generationAt, zoneItems[0].json.observedAt);
const replacementReplay = contracts.buildPublishPayload(
zoneItems,
'execution-replay',
'workflow-other',
'node-other',
'map.zones.current.v1',
0,
'replace',
);
assert.equal(
replacementReplay.batch.runId,
replacement.batch.runId,
'the same complete generation must keep one run identity across executions',
);
assert.equal(
replacementReplay.batch.idempotencyKey,
replacement.batch.idempotencyKey,
'the same complete generation must replay through the data plane idempotently',
);
const changedReplacement = contracts.buildPublishPayload(
[{ json: { ...zoneItems[0].json, attributes: { display_name: 'Zone 42 changed' } } }],
'execution-replay',
'workflow-other',
'node-other',
'map.zones.current.v1',
0,
'replace',
);
assert.notEqual(
changedReplacement.batch.idempotencyKey,
replacement.batch.idempotencyKey,
'different content at one declared generation must not alias an accepted replay',
);
const replacementBatch = contracts.buildPublishPayload(
[{ json: { generationAt: zoneItems[0].json.observedAt, facts: zoneItems.map((item) => item.json) } }],
'execution-44',
@ -181,6 +214,7 @@ async function main() {
);
assert.deepEqual(replacementBatch.facts, replacement.facts);
assert.equal(replacementBatch.batch.generationAt, zoneItems[0].json.observedAt);
assert.equal(replacementBatch.batch.idempotencyKey, replacement.batch.idempotencyKey);
const emptyReplacement = contracts.buildPublishPayload(
[{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: [] } }],
'execution-45',