import { createHash } from 'node:crypto'; import type { IDataObject, INodeExecutionData, INodeListSearchItems } from 'n8n-workflow'; import { DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION, } from './constants'; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const MAX_FACTS = 5000; const MAX_ATTRIBUTES_BYTES = 64 * 1024; const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; const SECRET_LIKE_VALUE = /(?:ndc_edp(?:wb|rb)_[A-Za-z0-9_-]*|(?:bearer|basic)\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/i; export interface NdcFact extends IDataObject { sourceId: string; semanticType: string; observedAt: string; attributes?: IDataObject; geometry?: NdcGeometry; } export type NdcGeometry = | { type: 'Point'; coordinates: [number, number] } | { type: 'LineString'; coordinates: [number, number][] } | { type: 'Polygon'; coordinates: [number, number][][] } | { type: 'MultiPolygon'; coordinates: [number, number][][][] }; export interface NdcPublishPayload extends IDataObject { schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION; batch: { runId: string; sequence: number; idempotencyKey: string; mode?: 'replace'; generationAt?: string; }; facts: NdcFact[]; } export interface NdcFoundryBindingInput extends IDataObject { applicationId: string; pageId: string; binding: { id: string; dataProductId: string; slotId: string; semanticTypes: string[]; fieldProjection: string[]; }; } export function normalizeBaseUrl(value: unknown): string { const normalized = String(value ?? '').trim().replace(/\/+$/, ''); if (!/^https?:\/\/[^\s]+$/i.test(normalized)) throw new Error('ndc_internal_api_base_url_invalid'); return normalized; } export function requireIdentifier(value: unknown, field: string): string { const normalized = String(value ?? '').trim(); if (!IDENTIFIER.test(normalized)) throw new Error(`${field}_invalid`); return normalized; } export function encodeIdentifierPath(value: unknown, field: string): string { return encodeURIComponent(requireIdentifier(value, field)); } export function factsFromItems(items: INodeExecutionData[], allowEmpty = false): NdcFact[] { const batchFacts = items.length === 1 && Array.isArray(items[0]?.json?.facts) ? items[0].json.facts : null; const sources = batchFacts ?? items.map((item) => (isObject(item.json.fact) ? item.json.fact : item.json)); if (!sources.length && !allowEmpty) throw new Error('facts_required'); if (sources.length > MAX_FACTS) throw new Error('facts_limit_exceeded'); const entityKeys = new Set(); return sources.map((source, index) => { if (!isObject(source)) throw new Error(`facts_${index}_invalid`); const fact: NdcFact = { sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`), semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`), observedAt: requireIsoTimestamp(source.observedAt, `facts_${index}_observedAt`), }; const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; if (entityKeys.has(entityKey)) throw new Error('facts_duplicate_entity_key'); entityKeys.add(entityKey); if (source.attributes !== undefined) { if (!isObject(source.attributes)) throw new Error(`facts_${index}_attributes_invalid`); if (Buffer.byteLength(JSON.stringify(source.attributes)) > MAX_ATTRIBUTES_BYTES) { throw new Error(`facts_${index}_attributes_size_exceeded`); } if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`); fact.attributes = source.attributes; } if (source.geometry !== undefined) fact.geometry = normalizeGeometry(source.geometry, index); return fact; }); } export function buildPublishPayload( items: INodeExecutionData[], executionId: string, workflowId: string, nodeId: string, dataProductId: string, sequence: number, publishMode: 'upsert' | 'replace' = 'upsert', ): NdcPublishPayload { if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid'); requireIdentifier(dataProductId, 'dataProductId'); const facts = factsFromItems(items, publishMode === 'replace'); const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts) ? items[0].json.generationAt : undefined; const generationAt = publishMode === 'replace' ? replacementGenerationAt(facts, explicitGenerationAt) : undefined; const replacementDigest = generationAt ? replacementReplayDigest(dataProductId, sequence, generationAt, facts) : undefined; const runId = replacementDigest ? `run-${replacementDigest.slice(0, 48)}` : `run-${sha256(`${workflowId}|${executionId}`).slice(0, 48)}`; const idempotencyKey = replacementDigest ? `publish-${replacementDigest}` : `publish-${sha256(`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`)}`; return { schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, batch: { runId, sequence, idempotencyKey, ...(generationAt ? { mode: 'replace' as const, generationAt } : {}), }, facts, }; } export function normalizeSnapshotFacts(value: unknown): IDataObject[] { if (!isObject(value) || !Array.isArray(value.facts)) throw new Error('data_product_snapshot_invalid'); return value.facts.map((fact, index) => { if (!isObject(fact)) throw new Error(`snapshot_fact_${index}_invalid`); return fact; }); } export function normalizeHistoryResponse(value: unknown): IDataObject { if (!isObject(value) || value.schemaVersion !== 'nodedc.data-product.history/v1' || !isObject(value.dataProduct) || !isObject(value.query) || !Array.isArray(value.facts)) { throw new Error('data_product_history_invalid'); } value.facts.forEach((fact, index) => { if (!isObject(fact) || Number.isNaN(Date.parse(String(fact.bucketStart ?? '')))) { throw new Error(`history_fact_${index}_invalid`); } }); return value; } export function snapshotReadParameters( dataProductId: unknown, pageSize: unknown, ): { encodedProductId: string; pageSize: number } { const normalizedPageSize = Number(pageSize); if (!Number.isInteger(normalizedPageSize) || normalizedPageSize < 1 || normalizedPageSize > 5000) { throw new Error('page_size_invalid'); } return { encodedProductId: encodeIdentifierPath(dataProductId, 'dataProductId'), pageSize: normalizedPageSize, }; } export function historyReadParameters(value: { dataProductId: unknown; from: unknown; to: unknown; resolutionMs: unknown; sourceIds: unknown; limit: unknown; cursor: unknown; }): { encodedProductId: string; from: string; to: string; resolutionMs: number; sourceIds: string[]; limit: number; cursor: string; } { const from = requireIsoTimestamp(value.from, 'from'); const to = requireIsoTimestamp(value.to, 'to'); if (Date.parse(from) >= Date.parse(to)) throw new Error('history_range_invalid'); const resolutionMs = Number(value.resolutionMs); if (!Number.isInteger(resolutionMs) || resolutionMs < 1000 || resolutionMs > 24 * 60 * 60 * 1000) { throw new Error('history_resolution_invalid'); } const limit = Number(value.limit); if (!Number.isInteger(limit) || limit < 1 || limit > 5000) throw new Error('history_limit_invalid'); const sourceIds = [...new Set((Array.isArray(value.sourceIds) ? value.sourceIds : String(value.sourceIds ?? '').split(',')) .map((item) => String(item ?? '').trim()) .filter(Boolean) .map((item) => requireIdentifier(item, 'sourceIds')))]; if (sourceIds.length > 1000) throw new Error('history_source_ids_limit_exceeded'); const cursor = String(value.cursor ?? '').trim(); if (cursor && (cursor.length > 1024 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new Error('history_cursor_invalid'); return { encodedProductId: encodeIdentifierPath(value.dataProductId, 'dataProductId'), from, to, resolutionMs, sourceIds, limit, cursor, }; } export function dataProductOptions(value: unknown): INodeListSearchItems[] { if (!isObject(value)) throw new Error('data_product_catalog_invalid'); const rows = Array.isArray(value.dataProducts) ? value.dataProducts : Array.isArray(value.products) ? value.products : []; return rows.flatMap((row) => { if (typeof row === 'string') { return IDENTIFIER.test(row) ? [{ name: row, value: row }] : []; } if (!isObject(row)) return []; const id = String(row.id ?? row.dataProductId ?? '').trim(); if (!IDENTIFIER.test(id)) return []; const version = String(row.version ?? '').trim(); const label = String(row.label ?? row.name ?? id).trim() || id; return [{ name: version ? `${label} ยท ${version}` : label, value: id }]; }); } export function buildFoundryBindingPayload(input: NdcFoundryBindingInput): IDataObject { const applicationId = requireFoundryApplicationId(input.applicationId); const pageId = requireFoundryPageId(input.pageId); const bindingId = requireIdentifier(input.binding.id, 'binding_id'); const dataProductId = requireIdentifier(input.binding.dataProductId, 'dataProductId'); const slotId = requireFoundrySlotId(input.binding.slotId); const semanticTypes = uniqueIdentifiers(input.binding.semanticTypes, 'semanticTypes'); if (!semanticTypes.length) throw new Error('semanticTypes_required'); const fieldProjection = uniqueIdentifiers(input.binding.fieldProjection, 'fieldProjection', true); const binding = { id: bindingId, dataProductId, slotId, semanticTypes, fieldProjection }; const digest = createHash('sha256') .update(JSON.stringify({ applicationId, pageId, binding })) .digest('hex') .slice(0, 32); return { schemaVersion: FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION, applicationId, pageId, idempotencyKey: `foundry-binding-${digest}`, binding, }; } function requireFoundryApplicationId(value: unknown): string { const normalized = String(value ?? '').trim(); if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) { throw new Error('applicationId_invalid'); } return normalized; } function requireFoundryPageId(value: unknown): string { const normalized = String(value ?? '').trim(); if (!/^[a-z0-9][a-z0-9-]{0,79}$/.test(normalized)) throw new Error('pageId_invalid'); return normalized; } function requireFoundrySlotId(value: unknown): string { const normalized = String(value ?? '').trim(); if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,79}$/.test(normalized)) throw new Error('slotId_invalid'); return normalized; } function normalizeGeometry(value: unknown, index: number): NdcGeometry { if (!isObject(value) || !['Point', 'LineString', 'Polygon', 'MultiPolygon'].includes(String(value.type)) || !Array.isArray(value.coordinates)) { throw new Error(`facts_${index}_geometry_invalid`); } if (Buffer.byteLength(JSON.stringify(value)) > 192 * 1024) throw new Error(`facts_${index}_geometry_size_exceeded`); let vertices = 0; const position = (candidate: unknown, path: string): [number, number] => { if (!Array.isArray(candidate) || candidate.length !== 2 || !candidate.every(Number.isFinite)) { throw new Error(`${path}_position_invalid`); } const [longitude, latitude] = candidate as [number, number]; if (longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) { throw new Error(`${path}_position_out_of_range`); } vertices += 1; if (vertices > 10_000) throw new Error(`facts_${index}_geometry_vertex_limit_exceeded`); return [longitude, latitude]; }; const line = (candidate: unknown, path: string, minimum: number): [number, number][] => { if (!Array.isArray(candidate) || candidate.length < minimum) throw new Error(`${path}_line_invalid`); return candidate.map((entry, positionIndex) => position(entry, `${path}_${positionIndex}`)); }; const ring = (candidate: unknown, path: string): [number, number][] => { const result = line(candidate, path, 4); const first = result[0]; const last = result.at(-1); if (!last || first[0] !== last[0] || first[1] !== last[1]) throw new Error(`${path}_ring_not_closed`); return result; }; const polygon = (candidate: unknown, path: string): [number, number][][] => { if (!Array.isArray(candidate) || !candidate.length) throw new Error(`${path}_polygon_invalid`); return candidate.map((entry, ringIndex) => ring(entry, `${path}_${ringIndex}`)); }; if (value.type === 'Point') return { type: 'Point', coordinates: position(value.coordinates, `facts_${index}_geometry`) }; if (value.type === 'LineString') return { type: 'LineString', coordinates: line(value.coordinates, `facts_${index}_geometry`, 2) }; if (value.type === 'Polygon') return { type: 'Polygon', coordinates: polygon(value.coordinates, `facts_${index}_geometry`) }; const coordinates = value.coordinates.map((entry, polygonIndex) => polygon(entry, `facts_${index}_geometry_${polygonIndex}`)); if (!coordinates.length) throw new Error(`facts_${index}_geometry_multipolygon_invalid`); return { type: 'MultiPolygon', coordinates }; } function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string { const declared = explicit === undefined ? undefined : requireIsoTimestamp(explicit, 'replace_generationAt'); const timestamps = [...new Set(facts.map((fact) => new Date(fact.observedAt).toISOString()))]; if (!timestamps.length) { if (!declared) throw new Error('replace_generationAt_required_for_empty_generation'); return new Date(declared).toISOString(); } if (timestamps.length !== 1) throw new Error('replace_generation_requires_one_observedAt'); if (declared && new Date(declared).toISOString() !== timestamps[0]) { throw new Error('replace_generationAt_observedAt_mismatch'); } return timestamps[0]; } function replacementReplayDigest( dataProductId: string, sequence: number, generationAt: string, facts: NdcFact[], ): string { const canonicalFacts = [...facts].sort((left, right) => { const leftKey = `${left.sourceId}\u0000${left.semanticType}`; const rightKey = `${right.sourceId}\u0000${right.semanticType}`; return leftKey.localeCompare(rightKey); }); return sha256(stableJson({ schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, dataProductId, mode: 'replace', generationAt, sequence, facts: canonicalFacts, })); } function stableJson(value: unknown): string { if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`; if (isObject(value)) { return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(',')}}`; } return JSON.stringify(value); } function requireIsoTimestamp(value: unknown, field: string): string { const normalized = String(value ?? '').trim(); if (!normalized || Number.isNaN(Date.parse(normalized))) throw new Error(`${field}_invalid`); return normalized; } function uniqueIdentifiers(value: unknown, field: string, allowEmpty = false): string[] { if (!Array.isArray(value)) throw new Error(`${field}_invalid`); const normalized = [...new Set(value.map((item) => requireIdentifier(item, field)))]; if (!allowEmpty && !normalized.length) throw new Error(`${field}_required`); return normalized; } function sha256(value: string): string { return createHash('sha256').update(value).digest('hex'); } function containsSecretLikeMaterial(value: unknown): boolean { if (typeof value === 'string') return SECRET_LIKE_VALUE.test(value); if (Array.isArray(value)) return value.some(containsSecretLikeMaterial); if (!isObject(value)) return false; return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child)); } function isObject(value: unknown): value is IDataObject { return Boolean(value) && typeof value === 'object' && !Array.isArray(value); }