362 lines
15 KiB
TypeScript
362 lines
15 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
|
|
import type { IDataObject, INodeExecutionData, INodeListSearchItems } from 'n8n-workflow';
|
|
|
|
import {
|
|
DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
|
FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION,
|
|
} from './constants';
|
|
|
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
|
const MAX_FACTS = 5000;
|
|
const MAX_ATTRIBUTES_BYTES = 64 * 1024;
|
|
const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
|
|
const SECRET_LIKE_VALUE = /(?:ndc_edp(?:wb|rb)_[A-Za-z0-9_-]*|(?:bearer|basic)\s+\S+|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/i;
|
|
|
|
export interface NdcFact extends IDataObject {
|
|
sourceId: string;
|
|
semanticType: string;
|
|
observedAt: string;
|
|
attributes?: IDataObject;
|
|
geometry?: NdcGeometry;
|
|
}
|
|
|
|
export type NdcGeometry =
|
|
| { type: 'Point'; coordinates: [number, number] }
|
|
| { type: 'LineString'; coordinates: [number, number][] }
|
|
| { type: 'Polygon'; coordinates: [number, number][][] }
|
|
| { type: 'MultiPolygon'; coordinates: [number, number][][][] };
|
|
|
|
export interface NdcPublishPayload extends IDataObject {
|
|
schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION;
|
|
batch: {
|
|
runId: string;
|
|
sequence: number;
|
|
idempotencyKey: string;
|
|
mode?: 'replace';
|
|
generationAt?: string;
|
|
};
|
|
facts: NdcFact[];
|
|
}
|
|
|
|
export interface NdcFoundryBindingInput extends IDataObject {
|
|
applicationId: string;
|
|
pageId: string;
|
|
binding: {
|
|
id: string;
|
|
dataProductId: string;
|
|
slotId: string;
|
|
semanticTypes: string[];
|
|
fieldProjection: string[];
|
|
};
|
|
}
|
|
|
|
export function normalizeBaseUrl(value: unknown): string {
|
|
const normalized = String(value ?? '').trim().replace(/\/+$/, '');
|
|
if (!/^https?:\/\/[^\s]+$/i.test(normalized)) throw new Error('ndc_internal_api_base_url_invalid');
|
|
return normalized;
|
|
}
|
|
|
|
export function requireIdentifier(value: unknown, field: string): string {
|
|
const normalized = String(value ?? '').trim();
|
|
if (!IDENTIFIER.test(normalized)) throw new Error(`${field}_invalid`);
|
|
return normalized;
|
|
}
|
|
|
|
export function encodeIdentifierPath(value: unknown, field: string): string {
|
|
return encodeURIComponent(requireIdentifier(value, field));
|
|
}
|
|
|
|
export function factsFromItems(items: INodeExecutionData[], allowEmpty = false): NdcFact[] {
|
|
const batchFacts = items.length === 1 && Array.isArray(items[0]?.json?.facts)
|
|
? items[0].json.facts
|
|
: null;
|
|
const sources = batchFacts ?? items.map((item) => (isObject(item.json.fact) ? item.json.fact : item.json));
|
|
if (!sources.length && !allowEmpty) throw new Error('facts_required');
|
|
if (sources.length > MAX_FACTS) throw new Error('facts_limit_exceeded');
|
|
const entityKeys = new Set<string>();
|
|
return sources.map((source, index) => {
|
|
if (!isObject(source)) throw new Error(`facts_${index}_invalid`);
|
|
const fact: NdcFact = {
|
|
sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`),
|
|
semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`),
|
|
observedAt: requireIsoTimestamp(source.observedAt, `facts_${index}_observedAt`),
|
|
};
|
|
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
|
if (entityKeys.has(entityKey)) throw new Error('facts_duplicate_entity_key');
|
|
entityKeys.add(entityKey);
|
|
if (source.attributes !== undefined) {
|
|
if (!isObject(source.attributes)) throw new Error(`facts_${index}_attributes_invalid`);
|
|
if (Buffer.byteLength(JSON.stringify(source.attributes)) > MAX_ATTRIBUTES_BYTES) {
|
|
throw new Error(`facts_${index}_attributes_size_exceeded`);
|
|
}
|
|
if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`);
|
|
fact.attributes = source.attributes;
|
|
}
|
|
if (source.geometry !== undefined) fact.geometry = normalizeGeometry(source.geometry, index);
|
|
return fact;
|
|
});
|
|
}
|
|
|
|
export function buildPublishPayload(
|
|
items: INodeExecutionData[],
|
|
executionId: string,
|
|
workflowId: string,
|
|
nodeId: string,
|
|
dataProductId: string,
|
|
sequence: number,
|
|
publishMode: 'upsert' | 'replace' = 'upsert',
|
|
): NdcPublishPayload {
|
|
if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid');
|
|
requireIdentifier(dataProductId, 'dataProductId');
|
|
const runId = `run-${sha256(`${workflowId}|${executionId}`).slice(0, 48)}`;
|
|
const idempotencyKey = `publish-${sha256(
|
|
`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`,
|
|
)}`;
|
|
const facts = factsFromItems(items, publishMode === 'replace');
|
|
const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts)
|
|
? items[0].json.generationAt
|
|
: undefined;
|
|
const generationAt = publishMode === 'replace'
|
|
? replacementGenerationAt(facts, explicitGenerationAt)
|
|
: undefined;
|
|
return {
|
|
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
|
batch: {
|
|
runId,
|
|
sequence,
|
|
idempotencyKey,
|
|
...(generationAt ? { mode: 'replace' as const, generationAt } : {}),
|
|
},
|
|
facts,
|
|
};
|
|
}
|
|
|
|
export function normalizeSnapshotFacts(value: unknown): IDataObject[] {
|
|
if (!isObject(value) || !Array.isArray(value.facts)) throw new Error('data_product_snapshot_invalid');
|
|
return value.facts.map((fact, index) => {
|
|
if (!isObject(fact)) throw new Error(`snapshot_fact_${index}_invalid`);
|
|
return fact;
|
|
});
|
|
}
|
|
|
|
export function normalizeHistoryResponse(value: unknown): IDataObject {
|
|
if (!isObject(value) || value.schemaVersion !== 'nodedc.data-product.history/v1'
|
|
|| !isObject(value.dataProduct) || !isObject(value.query) || !Array.isArray(value.facts)) {
|
|
throw new Error('data_product_history_invalid');
|
|
}
|
|
value.facts.forEach((fact, index) => {
|
|
if (!isObject(fact) || Number.isNaN(Date.parse(String(fact.bucketStart ?? '')))) {
|
|
throw new Error(`history_fact_${index}_invalid`);
|
|
}
|
|
});
|
|
return value;
|
|
}
|
|
|
|
export function snapshotReadParameters(
|
|
dataProductId: unknown,
|
|
pageSize: unknown,
|
|
): { encodedProductId: string; pageSize: number } {
|
|
const normalizedPageSize = Number(pageSize);
|
|
if (!Number.isInteger(normalizedPageSize) || normalizedPageSize < 1 || normalizedPageSize > 5000) {
|
|
throw new Error('page_size_invalid');
|
|
}
|
|
return {
|
|
encodedProductId: encodeIdentifierPath(dataProductId, 'dataProductId'),
|
|
pageSize: normalizedPageSize,
|
|
};
|
|
}
|
|
|
|
export function historyReadParameters(value: {
|
|
dataProductId: unknown;
|
|
from: unknown;
|
|
to: unknown;
|
|
resolutionMs: unknown;
|
|
sourceIds: unknown;
|
|
limit: unknown;
|
|
cursor: unknown;
|
|
}): {
|
|
encodedProductId: string;
|
|
from: string;
|
|
to: string;
|
|
resolutionMs: number;
|
|
sourceIds: string[];
|
|
limit: number;
|
|
cursor: string;
|
|
} {
|
|
const from = requireIsoTimestamp(value.from, 'from');
|
|
const to = requireIsoTimestamp(value.to, 'to');
|
|
if (Date.parse(from) >= Date.parse(to)) throw new Error('history_range_invalid');
|
|
const resolutionMs = Number(value.resolutionMs);
|
|
if (!Number.isInteger(resolutionMs) || resolutionMs < 1000 || resolutionMs > 24 * 60 * 60 * 1000) {
|
|
throw new Error('history_resolution_invalid');
|
|
}
|
|
const limit = Number(value.limit);
|
|
if (!Number.isInteger(limit) || limit < 1 || limit > 5000) throw new Error('history_limit_invalid');
|
|
const sourceIds = [...new Set((Array.isArray(value.sourceIds)
|
|
? value.sourceIds
|
|
: String(value.sourceIds ?? '').split(','))
|
|
.map((item) => String(item ?? '').trim())
|
|
.filter(Boolean)
|
|
.map((item) => requireIdentifier(item, 'sourceIds')))];
|
|
if (sourceIds.length > 1000) throw new Error('history_source_ids_limit_exceeded');
|
|
const cursor = String(value.cursor ?? '').trim();
|
|
if (cursor && (cursor.length > 1024 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new Error('history_cursor_invalid');
|
|
return {
|
|
encodedProductId: encodeIdentifierPath(value.dataProductId, 'dataProductId'),
|
|
from,
|
|
to,
|
|
resolutionMs,
|
|
sourceIds,
|
|
limit,
|
|
cursor,
|
|
};
|
|
}
|
|
|
|
export function dataProductOptions(value: unknown): INodeListSearchItems[] {
|
|
if (!isObject(value)) throw new Error('data_product_catalog_invalid');
|
|
const rows = Array.isArray(value.dataProducts)
|
|
? value.dataProducts
|
|
: Array.isArray(value.products)
|
|
? value.products
|
|
: [];
|
|
return rows.flatMap((row) => {
|
|
if (typeof row === 'string') {
|
|
return IDENTIFIER.test(row) ? [{ name: row, value: row }] : [];
|
|
}
|
|
if (!isObject(row)) return [];
|
|
const id = String(row.id ?? row.dataProductId ?? '').trim();
|
|
if (!IDENTIFIER.test(id)) return [];
|
|
const version = String(row.version ?? '').trim();
|
|
const label = String(row.label ?? row.name ?? id).trim() || id;
|
|
return [{ name: version ? `${label} · ${version}` : label, value: id }];
|
|
});
|
|
}
|
|
|
|
export function buildFoundryBindingPayload(input: NdcFoundryBindingInput): IDataObject {
|
|
const applicationId = requireFoundryApplicationId(input.applicationId);
|
|
const pageId = requireFoundryPageId(input.pageId);
|
|
const bindingId = requireIdentifier(input.binding.id, 'binding_id');
|
|
const dataProductId = requireIdentifier(input.binding.dataProductId, 'dataProductId');
|
|
const slotId = requireFoundrySlotId(input.binding.slotId);
|
|
const semanticTypes = uniqueIdentifiers(input.binding.semanticTypes, 'semanticTypes');
|
|
if (!semanticTypes.length) throw new Error('semanticTypes_required');
|
|
const fieldProjection = uniqueIdentifiers(input.binding.fieldProjection, 'fieldProjection', true);
|
|
const binding = { id: bindingId, dataProductId, slotId, semanticTypes, fieldProjection };
|
|
const digest = createHash('sha256')
|
|
.update(JSON.stringify({ applicationId, pageId, binding }))
|
|
.digest('hex')
|
|
.slice(0, 32);
|
|
return {
|
|
schemaVersion: FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION,
|
|
applicationId,
|
|
pageId,
|
|
idempotencyKey: `foundry-binding-${digest}`,
|
|
binding,
|
|
};
|
|
}
|
|
|
|
function requireFoundryApplicationId(value: unknown): string {
|
|
const normalized = String(value ?? '').trim();
|
|
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) {
|
|
throw new Error('applicationId_invalid');
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function requireFoundryPageId(value: unknown): string {
|
|
const normalized = String(value ?? '').trim();
|
|
if (!/^[a-z0-9][a-z0-9-]{0,79}$/.test(normalized)) throw new Error('pageId_invalid');
|
|
return normalized;
|
|
}
|
|
|
|
function requireFoundrySlotId(value: unknown): string {
|
|
const normalized = String(value ?? '').trim();
|
|
if (!/^[A-Za-z0-9][A-Za-z0-9-]{0,79}$/.test(normalized)) throw new Error('slotId_invalid');
|
|
return normalized;
|
|
}
|
|
|
|
function normalizeGeometry(value: unknown, index: number): NdcGeometry {
|
|
if (!isObject(value) || !['Point', 'LineString', 'Polygon', 'MultiPolygon'].includes(String(value.type))
|
|
|| !Array.isArray(value.coordinates)) {
|
|
throw new Error(`facts_${index}_geometry_invalid`);
|
|
}
|
|
if (Buffer.byteLength(JSON.stringify(value)) > 192 * 1024) throw new Error(`facts_${index}_geometry_size_exceeded`);
|
|
let vertices = 0;
|
|
const position = (candidate: unknown, path: string): [number, number] => {
|
|
if (!Array.isArray(candidate) || candidate.length !== 2 || !candidate.every(Number.isFinite)) {
|
|
throw new Error(`${path}_position_invalid`);
|
|
}
|
|
const [longitude, latitude] = candidate as [number, number];
|
|
if (longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) {
|
|
throw new Error(`${path}_position_out_of_range`);
|
|
}
|
|
vertices += 1;
|
|
if (vertices > 10_000) throw new Error(`facts_${index}_geometry_vertex_limit_exceeded`);
|
|
return [longitude, latitude];
|
|
};
|
|
const line = (candidate: unknown, path: string, minimum: number): [number, number][] => {
|
|
if (!Array.isArray(candidate) || candidate.length < minimum) throw new Error(`${path}_line_invalid`);
|
|
return candidate.map((entry, positionIndex) => position(entry, `${path}_${positionIndex}`));
|
|
};
|
|
const ring = (candidate: unknown, path: string): [number, number][] => {
|
|
const result = line(candidate, path, 4);
|
|
const first = result[0];
|
|
const last = result.at(-1);
|
|
if (!last || first[0] !== last[0] || first[1] !== last[1]) throw new Error(`${path}_ring_not_closed`);
|
|
return result;
|
|
};
|
|
const polygon = (candidate: unknown, path: string): [number, number][][] => {
|
|
if (!Array.isArray(candidate) || !candidate.length) throw new Error(`${path}_polygon_invalid`);
|
|
return candidate.map((entry, ringIndex) => ring(entry, `${path}_${ringIndex}`));
|
|
};
|
|
if (value.type === 'Point') return { type: 'Point', coordinates: position(value.coordinates, `facts_${index}_geometry`) };
|
|
if (value.type === 'LineString') return { type: 'LineString', coordinates: line(value.coordinates, `facts_${index}_geometry`, 2) };
|
|
if (value.type === 'Polygon') return { type: 'Polygon', coordinates: polygon(value.coordinates, `facts_${index}_geometry`) };
|
|
const coordinates = value.coordinates.map((entry, polygonIndex) => polygon(entry, `facts_${index}_geometry_${polygonIndex}`));
|
|
if (!coordinates.length) throw new Error(`facts_${index}_geometry_multipolygon_invalid`);
|
|
return { type: 'MultiPolygon', coordinates };
|
|
}
|
|
|
|
function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string {
|
|
const declared = explicit === undefined ? undefined : requireIsoTimestamp(explicit, 'replace_generationAt');
|
|
const timestamps = [...new Set(facts.map((fact) => new Date(fact.observedAt).toISOString()))];
|
|
if (!timestamps.length) {
|
|
if (!declared) throw new Error('replace_generationAt_required_for_empty_generation');
|
|
return new Date(declared).toISOString();
|
|
}
|
|
if (timestamps.length !== 1) throw new Error('replace_generation_requires_one_observedAt');
|
|
if (declared && new Date(declared).toISOString() !== timestamps[0]) {
|
|
throw new Error('replace_generationAt_observedAt_mismatch');
|
|
}
|
|
return timestamps[0];
|
|
}
|
|
|
|
function requireIsoTimestamp(value: unknown, field: string): string {
|
|
const normalized = String(value ?? '').trim();
|
|
if (!normalized || Number.isNaN(Date.parse(normalized))) throw new Error(`${field}_invalid`);
|
|
return normalized;
|
|
}
|
|
|
|
function uniqueIdentifiers(value: unknown, field: string, allowEmpty = false): string[] {
|
|
if (!Array.isArray(value)) throw new Error(`${field}_invalid`);
|
|
const normalized = [...new Set(value.map((item) => requireIdentifier(item, field)))];
|
|
if (!allowEmpty && !normalized.length) throw new Error(`${field}_required`);
|
|
return normalized;
|
|
}
|
|
|
|
function sha256(value: string): string {
|
|
return createHash('sha256').update(value).digest('hex');
|
|
}
|
|
|
|
function containsSecretLikeMaterial(value: unknown): boolean {
|
|
if (typeof value === 'string') return SECRET_LIKE_VALUE.test(value);
|
|
if (Array.isArray(value)) return value.some(containsSecretLikeMaterial);
|
|
if (!isObject(value)) return false;
|
|
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeMaterial(child));
|
|
}
|
|
|
|
function isObject(value: unknown): value is IDataObject {
|
|
return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
|
|
}
|