SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): add managed data product history plane

Browse Source
This commit is contained in:
Codex
2026-07-18 14:38:06 +03:00
parent 02816c4352
commit 3c5d8f6cef
34 changed files with 2091 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/n8n-nodes-ndc/README.md
+13 -6
View File
@@ -9,8 +9,8 @@ qualified:
- `n8n-nodes-ndc.ndcDataProductPublish` — publishes canonical facts into an
approved Data Product.
- `n8n-nodes-ndc.ndcDataProductRead` — reads the current snapshot of an
approved Data Product.
- `n8n-nodes-ndc.ndcDataProductRead` — reads the current snapshot or a bounded,
cursor-paged history window of an approved Data Product.
- `n8n-nodes-ndc.ndcFoundryBinding` — creates or updates a declarative Foundry
page-slot binding. This is a control-plane operation, not a runtime data
transport.
@@ -42,6 +42,7 @@ never accepts them from a workflow.
- `GET /internal/data-plane/v1/reader/data-products`
- `POST /internal/data-plane/v1/data-products/:dataProductId/publish`
- `GET /internal/data-plane/v1/data-products/:dataProductId/snapshot`
- `GET /internal/data-plane/v1/data-products/:dataProductId/history`
- `GET /internal/foundry/v1/data-products`
- `POST /internal/foundry/v1/data-product-bindings`
@@ -58,14 +59,20 @@ materializes actor, owner and exact application/page/binding/product scope from
that grant; none of those authorization claims are accepted from headers or
workflow data.
`NDC Data Product Read` supports only the bounded
`nodedc.data-product.snapshot/v1` contract. Its `Page Size` parameter is a
`NDC Data Product Read` supports the bounded
`nodedc.data-product.snapshot/v1` contract. Its snapshot `Page Size` parameter is a
safety ceiling, not pagination: the complete scoped current projection must fit
within 5000 entity keys. A larger product returns
`data_product_snapshot_limit_exceeded`; the node must not assemble independent
pages or start a patch stream from an incomplete snapshot. Large products need
stable partition Data Products or a separately versioned query contract with a
single snapshot barrier.
stable partition Data Products.
History mode uses `nodedc.data-product.history/v1` and requires a closed
`from`/`to` interval. It supports provider-neutral `sourceIds`, a resolution
that is a multiple of the Data Product native sampling interval, a limit up to
5000 points and an opaque keyset continuation cursor. The response stays as one
envelope item so L2 logic can preserve `query` and `nextCursor` while iterating.
The workflow never selects a database, provider, tenant or connection.
## Publish input
packages/n8n-nodes-ndc/nodes/NdcDataProductRead/NdcDataProductRead.node.ts
+115 -4
View File
@@ -9,6 +9,7 @@ import { NodeConnectionTypes, NodeOperationError } from 'n8n-workflow';
import {
DATA_PRODUCT_BASE_PATH,
DATA_PRODUCT_HISTORY_SUFFIX,
DATA_PRODUCT_READER_CATALOG_PATH,
DATA_PRODUCT_SNAPSHOT_SUFFIX,
DATA_PLANE_DEFAULT_BASE_URL,
@@ -16,7 +17,12 @@ import {
NDC_DATA_PRODUCT_READER_CREDENTIAL,
NDC_NODE_ICON,
} from '../shared/constants';
import { normalizeSnapshotFacts, snapshotReadParameters } from '../shared/contracts';
import {
historyReadParameters,
normalizeHistoryResponse,
normalizeSnapshotFacts,
snapshotReadParameters,
} from '../shared/contracts';
import { safeHttpError, safeInputError } from '../shared/errors';
import { loadDataProductOptions, ndcRequest, serviceBaseUrl } from '../shared/http';
@@ -26,14 +32,25 @@ export class NdcDataProductRead implements INodeType {
name: 'ndcDataProductRead',
icon: NDC_NODE_ICON,
group: ['input'],
version: 1,
subtitle: '={{$parameter["dataProductId"]}}',
description: 'Read a scoped NDC Data Product snapshot',
version: [1, 2],
subtitle: '={{$parameter["mode"] || "snapshot"}} · {{$parameter["dataProductId"]}}',
description: 'Read a scoped NDC Data Product snapshot or bounded history',
defaults: { name: 'NDC Data Product Read' },
inputs: [NodeConnectionTypes.Main],
outputs: [NodeConnectionTypes.Main],
credentials: [{ name: NDC_DATA_PRODUCT_READER_CREDENTIAL, required: true }],
properties: [
{
displayName: 'Mode',
name: 'mode',
type: 'options',
options: [
{ name: 'Current Snapshot', value: 'snapshot' },
{ name: 'History', value: 'history' },
],
default: 'snapshot',
required: true,
},
{
displayName: 'Data Product Name or ID',
name: 'dataProductId',
@@ -50,6 +67,59 @@ export class NdcDataProductRead implements INodeType {
default: 1000,
required: true,
description: 'Maximum number of snapshot facts to return',
displayOptions: { show: { mode: ['snapshot'] } },
},
{
displayName: 'From',
name: 'from',
type: 'dateTime',
default: '',
required: true,
displayOptions: { show: { mode: ['history'] } },
},
{
displayName: 'To',
name: 'to',
type: 'dateTime',
default: '',
required: true,
displayOptions: { show: { mode: ['history'] } },
},
{
displayName: 'Resolution (MS)',
name: 'resolutionMs',
type: 'number',
typeOptions: { minValue: 1000, maxValue: 86400000, numberStepSize: 1000 },
default: 60000,
required: true,
description: 'Provider-neutral history bucket size; must be a multiple of the Data Product native history interval',
displayOptions: { show: { mode: ['history'] } },
},
{
displayName: 'Source IDs',
name: 'sourceIds',
type: 'string',
default: '',
description: 'Optional comma-separated entity IDs; empty reads all entities in the scoped Data Product',
displayOptions: { show: { mode: ['history'] } },
},
{
displayName: 'Limit',
name: 'limit',
type: 'number',
typeOptions: { minValue: 1, maxValue: 5000, numberStepSize: 1 },
default: 50,
required: true,
description: 'Max number of results to return',
displayOptions: { show: { mode: ['history'] } },
},
{
displayName: 'Cursor',
name: 'cursor',
type: 'string',
default: '',
description: 'Opaque continuation cursor returned by the previous history page',
displayOptions: { show: { mode: ['history'] } },
},
],
};
@@ -69,7 +139,48 @@ export class NdcDataProductRead implements INodeType {
};
async execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]> {
const mode = this.getNodeParameter('mode', 0, 'snapshot') as string;
const dataProductId = this.getNodeParameter('dataProductId', 0) as string;
if (mode === 'history') {
let parameters;
try {
parameters = historyReadParameters({
dataProductId,
from: this.getNodeParameter('from', 0),
to: this.getNodeParameter('to', 0),
resolutionMs: this.getNodeParameter('resolutionMs', 0, 60000),
sourceIds: this.getNodeParameter('sourceIds', 0, ''),
limit: this.getNodeParameter('limit', 0, 1000),
cursor: this.getNodeParameter('cursor', 0, ''),
});
} catch (error) {
throw new NodeOperationError(this.getNode(), safeInputError(error, 'ndc_data_product_history_input_invalid'));
}
try {
const baseUrl = serviceBaseUrl(DATA_PLANE_BASE_URL_ENV, DATA_PLANE_DEFAULT_BASE_URL);
const response = await ndcRequest(this, NDC_DATA_PRODUCT_READER_CREDENTIAL, {
method: 'GET',
url: `${baseUrl}${DATA_PRODUCT_BASE_PATH}/${parameters.encodedProductId}${DATA_PRODUCT_HISTORY_SUFFIX}`,
qs: {
from: parameters.from,
to: parameters.to,
resolutionMs: parameters.resolutionMs,
...(parameters.sourceIds.length ? { sourceIds: parameters.sourceIds.join(',') } : {}),
limit: parameters.limit,
...(parameters.cursor ? { cursor: parameters.cursor } : {}),
},
});
return [[{ json: normalizeHistoryResponse(response), pairedItem: { item: 0 } }]];
} catch (error) {
const safe = error instanceof Error && error.message === 'data_product_history_invalid'
? safeInputError(error, 'ndc_data_product_history_invalid')
: safeHttpError(error, 'ndc_data_product_history_read_failed');
throw new NodeOperationError(this.getNode(), safe);
}
}
if (mode !== 'snapshot') {
throw new NodeOperationError(this.getNode(), safeInputError(new Error('read_mode_invalid'), 'ndc_data_product_read_input_invalid'));
}
const pageSize = this.getNodeParameter('pageSize', 0, 1000) as number;
let parameters;
try {
packages/n8n-nodes-ndc/nodes/shared/constants.ts
+1
View File
@@ -5,6 +5,7 @@ export const DATA_PRODUCT_WRITER_CATALOG_PATH = '/internal/data-plane/v1/writer/
export const DATA_PRODUCT_READER_CATALOG_PATH = '/internal/data-plane/v1/reader/data-products';
export const DATA_PRODUCT_BASE_PATH = '/internal/data-plane/v1/data-products';
export const DATA_PRODUCT_SNAPSHOT_SUFFIX = '/snapshot';
export const DATA_PRODUCT_HISTORY_SUFFIX = '/history';
export const FOUNDRY_BINDING_PATH = '/internal/foundry/v1/data-product-bindings';
export const FOUNDRY_CATALOG_PATH = '/internal/foundry/v1/data-products';
packages/n8n-nodes-ndc/nodes/shared/contracts.ts
+59
View File
@@ -118,6 +118,19 @@ export function normalizeSnapshotFacts(value: unknown): IDataObject[] {
});
}
export function normalizeHistoryResponse(value: unknown): IDataObject {
if (!isObject(value) || value.schemaVersion !== 'nodedc.data-product.history/v1'
|| !isObject(value.dataProduct) || !isObject(value.query) || !Array.isArray(value.facts)) {
throw new Error('data_product_history_invalid');
}
value.facts.forEach((fact, index) => {
if (!isObject(fact) || Number.isNaN(Date.parse(String(fact.bucketStart ?? '')))) {
throw new Error(`history_fact_${index}_invalid`);
}
});
return value;
}
export function snapshotReadParameters(
dataProductId: unknown,
pageSize: unknown,
@@ -132,6 +145,52 @@ export function snapshotReadParameters(
};
}
export function historyReadParameters(value: {
dataProductId: unknown;
from: unknown;
to: unknown;
resolutionMs: unknown;
sourceIds: unknown;
limit: unknown;
cursor: unknown;
}): {
encodedProductId: string;
from: string;
to: string;
resolutionMs: number;
sourceIds: string[];
limit: number;
cursor: string;
} {
const from = requireIsoTimestamp(value.from, 'from');
const to = requireIsoTimestamp(value.to, 'to');
if (Date.parse(from) >= Date.parse(to)) throw new Error('history_range_invalid');
const resolutionMs = Number(value.resolutionMs);
if (!Number.isInteger(resolutionMs) || resolutionMs < 1000 || resolutionMs > 24 * 60 * 60 * 1000) {
throw new Error('history_resolution_invalid');
}
const limit = Number(value.limit);
if (!Number.isInteger(limit) || limit < 1 || limit > 5000) throw new Error('history_limit_invalid');
const sourceIds = [...new Set((Array.isArray(value.sourceIds)
? value.sourceIds
: String(value.sourceIds ?? '').split(','))
.map((item) => String(item ?? '').trim())
.filter(Boolean)
.map((item) => requireIdentifier(item, 'sourceIds')))];
if (sourceIds.length > 1000) throw new Error('history_source_ids_limit_exceeded');
const cursor = String(value.cursor ?? '').trim();
if (cursor && (cursor.length > 1024 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new Error('history_cursor_invalid');
return {
encodedProductId: encodeIdentifierPath(value.dataProductId, 'dataProductId'),
from,
to,
resolutionMs,
sourceIds,
limit,
cursor,
};
}
export function dataProductOptions(value: unknown): INodeListSearchItems[] {
if (!isObject(value)) throw new Error('data_product_catalog_invalid');
const rows = Array.isArray(value.dataProducts)
packages/n8n-nodes-ndc/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "n8n-nodes-ndc",
"version": "0.1.2",
"version": "0.1.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "n8n-nodes-ndc",
"version": "0.1.2",
"version": "0.1.3",
"license": "UNLICENSED",
"devDependencies": {
"@n8n/node-cli": "0.39.3",
packages/n8n-nodes-ndc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "n8n-nodes-ndc",
"version": "0.1.2",
"version": "0.1.3",
"description": "Private NODE.DC nodes for scoped data products and Foundry bindings.",
"private": true,
"license": "UNLICENSED",
packages/n8n-nodes-ndc/test/package-policy.test.cjs
+48 -2
View File
@@ -172,8 +172,9 @@ async function main() {
restoreEnvironment('NDC_DATA_PLANE_BASE_URL', previousDefaultProbe);
const readProperties = nodes.get('NdcDataProductRead').description.properties.map((property) => property.name);
assert.equal(readProperties.includes('cursor'), false, 'snapshot endpoint has no cursor input');
assert.equal(readProperties.includes('history'), false, 'history must stay hidden until its endpoint exists');
assert.equal(readProperties.includes('mode'), true, 'read mode must expose snapshot/history explicitly');
assert.equal(readProperties.includes('cursor'), true, 'history endpoint must expose its opaque continuation cursor');
assert.equal(readProperties.includes('sourceIds'), true, 'history filtering remains provider-neutral');
const pageSizeProperty = nodes.get('NdcDataProductRead').description.properties.find((property) => property.name === 'pageSize');
assert.equal(pageSizeProperty.typeOptions.minValue, 1);
assert.equal(pageSizeProperty.typeOptions.maxValue, 5000, 'bounded snapshot v1 must not expose more than 5000 facts');
@@ -255,6 +256,51 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) {
assert.deepEqual(readCalls[0].options.qs, { limit: 1000 });
assert.equal(readResult[0][0].json.sourceId, 'fleet.unit.42');
const historyCalls = [];
const historyResponse = {
schemaVersion: 'nodedc.data-product.history/v1',
dataProduct: { id: 'fleet.positions.current.v1', version: '1.0.0' },
generatedAt: '2026-07-15T12:05:00.000Z',
query: {
from: '2026-07-15T12:00:00.000Z',
to: '2026-07-15T12:05:00.000Z',
resolutionMs: 60000,
sourceIds: ['fleet.unit.42'],
order: 'asc',
},
facts: [{ ...canonicalFact(), receivedAt: '2026-07-15T12:00:00.100Z', bucketStart: '2026-07-15T12:00:00.000Z' }],
nextCursor: 'opaque_cursor',
};
const historyContext = executionContext({
parameters: {
mode: 'history',
dataProductId: 'fleet.positions.current.v1',
from: '2026-07-15T12:00:00.000Z',
to: '2026-07-15T12:05:00.000Z',
resolutionMs: 60000,
sourceIds: 'fleet.unit.42',
limit: 1000,
cursor: '',
},
response: historyResponse,
calls: historyCalls,
});
const historyResult = await nodes.get('NdcDataProductRead').execute.call(historyContext);
assert.equal(historyCalls[0].credentialName, 'ndcDataProductReaderApi');
assert.equal(historyCalls[0].options.method, 'GET');
assert.equal(
historyCalls[0].options.url,
'http://data-plane.test/internal/data-plane/v1/data-products/fleet.positions.current.v1/history',
);
assert.deepEqual(historyCalls[0].options.qs, {
from: '2026-07-15T12:00:00.000Z',
to: '2026-07-15T12:05:00.000Z',
resolutionMs: 60000,
sourceIds: 'fleet.unit.42',
limit: 1000,
});
assert.deepEqual(historyResult[0][0].json, historyResponse);
const foundryCalls = [];
const foundryContext = executionContext({
parameters: {