SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): complete the Gelios external data loop

Browse Source
This commit is contained in:
Codex
2026-07-20 20:45:05 +03:00
parent def9a24e0d
commit 8a7465cf0e
66 changed files with 5730 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/external-provider-contract/src/provider-package.mjs
+187 -35
View File
@@ -20,6 +20,7 @@ const TOKEN_REFRESH_MODES = new Set(["not_applicable", "operator_managed", "runt
const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]);
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
const L2_STEP_KINDS = new Set([
"collection_trigger",
"provider_request",
@@ -115,8 +116,10 @@ const DATA_PRODUCT_KEYS = new Set([
"deliveryMode",
"semanticTypes",
"fields",
"fieldContracts",
"history",
]);
const FIELD_CONTRACT_KEYS = new Set(["type", "required", "enum", "minimum", "maximum"]);
const HISTORY_KEYS = new Set(["mode", "intervalMs", "strategy", "retentionDays"]);
const MAPPING_KEYS = new Set([
"schemaVersion",
@@ -130,7 +133,10 @@ const MAPPING_KEYS = new Set([
]);
const MAPPING_TARGET_KEYS = new Set(["dataProductId", "version", "ontologyRevision", "semanticType"]);
const FACT_KEYS = new Set(["sourceId", "semanticType", "observedAt", "geometry", "attributes"]);
const EXPRESSION_KEYS = new Set(["strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive", "omitIfMissing"]);
const EXPRESSION_KEYS = new Set([
"strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive",
"omitIfMissing", "omitIfInvalid", "minimum", "maximum",
]);
const GEOMETRY_KEYS = new Set(["type", "longitude", "latitude", "omitIfInvalid"]);
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
const TEMPLATE_KEYS = new Set([
@@ -226,9 +232,6 @@ export function validateProviderPackage(value) {
}
for (const profile of collectionProfiles) {
const selectedCapabilityIds = Array.isArray(profile.capabilityIds) ? profile.capabilityIds : [];
if (selectedCapabilityIds.length !== 1) {
errors.push(`collectionProfile.${profile.id}.capabilityIds_must_select_one_v1_read`);
}
selectedCapabilityIds.forEach((id) => {
assertReference(id, capabilityIds, `collectionProfile.${profile.id}.capabilityIds`, errors);
const capability = capabilities.find((item) => item.id === id);
@@ -247,10 +250,6 @@ export function validateProviderPackage(value) {
if (!selectedCapabilityIds.includes(mapping.sourceCapabilityId)) {
errors.push(`collectionProfile.${profile.id}.mapping_source_capability_must_be_selected`);
}
if (selectedCapabilityIds.length !== 1
|| selectedCapabilityIds[0] !== mapping.sourceCapabilityId) {
errors.push(`collectionProfile.${profile.id}.capability_chain_must_be_exact_singleton`);
}
if (mapping.fieldPolicyId !== profile.fieldPolicyId) errors.push(`collectionProfile.${profile.id}.mapping_field_policy_mismatch`);
if (mapping.target?.dataProductId !== profile.dataProductId) errors.push(`collectionProfile.${profile.id}.mapping_data_product_mismatch`);
}
@@ -258,12 +257,19 @@ export function validateProviderPackage(value) {
errors.push(`collectionProfile.${profile.id}.field_policy_data_product_mismatch`);
}
if (template) {
const [, requestStep, extractStep, mappingStep, publishStep] = Array.isArray(template.steps) ? template.steps : [];
if (!selectedCapabilityIds.includes(requestStep?.capabilityId) || extractStep?.capabilityId !== requestStep?.capabilityId) {
const steps = Array.isArray(template.steps) ? template.steps : [];
const requestSteps = steps.filter((step) => step?.kind === "provider_request");
const extractSteps = steps.filter((step) => step?.kind === "extract_items");
const mappingSteps = steps.filter((step) => step?.kind === "semantic_mapping");
const publishSteps = steps.filter((step) => step?.kind === "data_product_publish");
const executedCapabilityIds = requestSteps.map((step) => step.capabilityId);
if (!sameSet(selectedCapabilityIds, executedCapabilityIds)
|| extractSteps.length !== 1
|| extractSteps[0]?.capabilityId !== mapping?.sourceCapabilityId) {
errors.push(`collectionProfile.${profile.id}.template_capability_chain_mismatch`);
}
if (mappingStep?.mappingContractId !== profile.mappingContractId) errors.push(`collectionProfile.${profile.id}.template_mapping_mismatch`);
if (publishStep?.dataProductId !== profile.dataProductId) errors.push(`collectionProfile.${profile.id}.template_data_product_mismatch`);
if (mappingSteps.length !== 1 || mappingSteps[0]?.mappingContractId !== profile.mappingContractId) errors.push(`collectionProfile.${profile.id}.template_mapping_mismatch`);
if (publishSteps.length !== 1 || publishSteps[0]?.dataProductId !== profile.dataProductId) errors.push(`collectionProfile.${profile.id}.template_data_product_mismatch`);
}
}
for (const mapping of mappingContracts) {
@@ -315,13 +321,15 @@ export function validateProviderPackage(value) {
const providerBinding = Array.isArray(template.credentialBindings)
? template.credentialBindings.find((binding) => binding?.role === "provider")
: undefined;
const providerRequest = Array.isArray(template.steps)
? template.steps.find((step) => step?.kind === "provider_request")
: undefined;
const requestCapability = capabilities.find((capability) => capability.id === providerRequest?.capabilityId);
if (providerBinding && requestCapability
&& providerBinding.authModeId !== requestCapability.authModeId) {
errors.push(`l2Template.${template.id}.provider_credential_auth_mode_mismatch`);
const providerRequests = Array.isArray(template.steps)
? template.steps.filter((step) => step?.kind === "provider_request")
: [];
for (const providerRequest of providerRequests) {
const requestCapability = capabilities.find((capability) => capability.id === providerRequest?.capabilityId);
if (providerBinding && requestCapability
&& providerBinding.authModeId !== requestCapability.authModeId) {
errors.push(`l2Template.${template.id}.provider_credential_auth_mode_mismatch`);
}
}
}
@@ -557,6 +565,7 @@ function validateDataProductDefinition(value, path, errors) {
if (Array.isArray(value.fields) && value.fields.some((field) => SECRET_FIELD_NAME.test(String(field)))) {
errors.push(`${path}.fields_must_not_contain_secret_fields`);
}
validateFieldContracts(value.fieldContracts, value.fields, `${path}.fieldContracts`, errors);
if (!isPlainObject(value.history)) {
errors.push(`${path}.history_must_be_object`);
} else {
@@ -688,8 +697,8 @@ function validateL2Template(value, path, errors) {
});
if (!roles.has("provider") || !roles.has("publisher")) errors.push(`${path}.credentialBindings_roles_invalid`);
}
if (!Array.isArray(value.steps) || value.steps.length !== 5) {
errors.push(`${path}.steps_must_define_five_boundary_steps`);
if (!Array.isArray(value.steps) || value.steps.length < 5) {
errors.push(`${path}.steps_must_define_complete_boundary`);
} else {
const stepIds = new Set();
value.steps.forEach((step, index) => {
@@ -701,26 +710,41 @@ function validateL2Template(value, path, errors) {
stepIds.add(step.id);
if (!L2_STEP_KINDS.has(step.kind)) errors.push(`${stepPath}.kind_invalid`);
});
const expectedKinds = [
"collection_trigger",
"provider_request",
"extract_items",
"semantic_mapping",
"data_product_publish",
];
if (!value.steps.every((step, index) => step?.kind === expectedKinds[index])) errors.push(`${path}.steps_sequence_invalid`);
const requestSteps = value.steps.filter((step) => step?.kind === "provider_request");
const extractSteps = value.steps.filter((step) => step?.kind === "extract_items");
const mappingSteps = value.steps.filter((step) => step?.kind === "semantic_mapping");
const publishSteps = value.steps.filter((step) => step?.kind === "data_product_publish");
const requestStart = 1;
const extractIndex = value.steps.findIndex((step) => step?.kind === "extract_items");
const expectedSequence = (
value.steps[0]?.kind === "collection_trigger"
&& requestSteps.length >= 1
&& value.steps.slice(requestStart, extractIndex).every((step) => step?.kind === "provider_request")
&& extractSteps.length === 1
&& mappingSteps.length === 1
&& publishSteps.length === 1
&& value.steps.at(-3)?.kind === "extract_items"
&& value.steps.at(-2)?.kind === "semantic_mapping"
&& value.steps.at(-1)?.kind === "data_product_publish"
);
if (!expectedSequence) errors.push(`${path}.steps_sequence_invalid`);
if (value.steps[0]?.collectionProfileDriven !== true) errors.push(`${path}.collection_trigger_must_be_profile_driven`);
if (!value.steps[1]?.capabilityId || value.steps[1]?.mappingContractId !== undefined || value.steps[1]?.dataProductId !== undefined) {
errors.push(`${path}.provider_request_shape_invalid`);
for (const step of requestSteps) {
if (!step?.capabilityId || step?.mappingContractId !== undefined || step?.dataProductId !== undefined) {
errors.push(`${path}.provider_request_shape_invalid`);
}
}
if (!value.steps[2]?.capabilityId || value.steps[2]?.mappingContractId !== undefined || value.steps[2]?.dataProductId !== undefined) {
const extractStep = extractSteps[0];
if (!extractStep?.capabilityId || extractStep?.mappingContractId !== undefined || extractStep?.dataProductId !== undefined) {
errors.push(`${path}.extract_items_shape_invalid`);
}
if (value.steps[1]?.capabilityId !== value.steps[2]?.capabilityId) errors.push(`${path}.request_extract_capability_mismatch`);
if (!value.steps[3]?.mappingContractId || value.steps[3]?.capabilityId !== undefined || value.steps[3]?.dataProductId !== undefined) {
if (!requestSteps.some((step) => step.capabilityId === extractStep?.capabilityId)) errors.push(`${path}.request_extract_capability_mismatch`);
const mappingStep = mappingSteps[0];
if (!mappingStep?.mappingContractId || mappingStep?.capabilityId !== undefined || mappingStep?.dataProductId !== undefined) {
errors.push(`${path}.semantic_mapping_shape_invalid`);
}
if (!value.steps[4]?.dataProductId || value.steps[4]?.nodeType !== "n8n-nodes-ndc.ndcDataProductPublish") {
const publishStep = publishSteps[0];
if (!publishStep?.dataProductId || publishStep?.nodeType !== "n8n-nodes-ndc.ndcDataProductPublish") {
errors.push(`${path}.data_product_publish_shape_invalid`);
}
}
@@ -745,6 +769,122 @@ function validateMappingAgainstProduct(mapping, product, errors) {
}
const mappedFields = [...Object.keys(mapping.fact?.attributes || {}), ...(mapping.fact?.geometry ? ["geometry"] : [])];
if (!sameSet(mappedFields, product.fields)) errors.push(`${path}.fact_fields_must_match_data_product_fields`);
const contracts = isPlainObject(product.fieldContracts) ? product.fieldContracts : {};
for (const [field, contract] of Object.entries(contracts)) {
if (!isPlainObject(contract)) continue;
if (field === "geometry") {
if (contract.type !== "point") errors.push(`${path}.fact.geometry_contract_must_be_point`);
if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) {
errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`);
}
continue;
}
const expression = mapping.fact?.attributes?.[field];
if (!isPlainObject(expression)) continue;
validateExpressionAgainstFieldContract(expression, contract, mapping.derivations, `${path}.fact.attributes.${field}`, errors);
}
}
function validateFieldContracts(value, fields, path, errors) {
if (value === undefined) return;
if (!isPlainObject(value)) return errors.push(`${path}_must_be_object`);
const names = Object.keys(value);
if (names.length && !sameSet(names, fields)) errors.push(`${path}_must_exactly_match_fields`);
for (const [field, contract] of Object.entries(value)) {
requiredIdentifier(field, `${path}.${field}`, errors);
if (SECRET_FIELD_NAME.test(field)) errors.push(`${path}.${field}_secret_field_not_allowed`);
validateFieldContract(contract, `${path}.${field}`, errors);
}
}
function validateFieldContract(value, path, errors) {
if (!isPlainObject(value)) return errors.push(`${path}_must_be_object`);
rejectUnknownKeys(value, FIELD_CONTRACT_KEYS, path, errors);
if (!FIELD_CONTRACT_TYPES.has(value.type)) errors.push(`${path}.type_invalid`);
if (typeof value.required !== "boolean") errors.push(`${path}.required_must_be_boolean`);
if (value.enum !== undefined) {
if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) {
errors.push(`${path}.enum_must_be_nonempty_unique_array`);
} else if (new Set(["point", "string_array"]).has(value.type)
|| value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) {
errors.push(`${path}.enum_value_type_invalid`);
}
}
if (value.minimum !== undefined || value.maximum !== undefined) {
if (value.type !== "number") errors.push(`${path}.range_only_allowed_for_number`);
if (value.minimum !== undefined && !Number.isFinite(value.minimum)) errors.push(`${path}.minimum_invalid`);
if (value.maximum !== undefined && !Number.isFinite(value.maximum)) errors.push(`${path}.maximum_invalid`);
if (Number.isFinite(value.minimum) && Number.isFinite(value.maximum) && value.minimum > value.maximum) {
errors.push(`${path}.range_invalid`);
}
if (Array.isArray(value.enum) && value.enum.some((item) => (
typeof item === "number"
&& ((Number.isFinite(value.minimum) && item < value.minimum)
|| (Number.isFinite(value.maximum) && item > value.maximum))
))) errors.push(`${path}.enum_value_out_of_range`);
}
}
function validateExpressionAgainstFieldContract(expression, contract, derivations, path, errors) {
if (contract.required === true && expression.omitIfMissing === true) {
errors.push(`${path}_required_but_mapping_can_omit`);
}
if (contract.required === true && expression.omitIfInvalid === true) {
errors.push(`${path}_required_but_mapping_can_omit_invalid`);
}
if (expression.paths !== undefined) {
const expectedCoerce = new Map([["string", "string"], ["number", "number"], ["boolean", "boolean"]]).get(contract.type);
if (expectedCoerce && expression.coerce !== expectedCoerce) errors.push(`${path}.coerce_must_match_field_contract`);
if (Array.isArray(contract.enum)) errors.push(`${path}.enum_field_must_use_constant_or_derivation`);
if (contract.type === "number") {
if (!Object.is(expression.minimum, contract.minimum)) errors.push(`${path}.minimum_must_match_field_contract`);
if (!Object.is(expression.maximum, contract.maximum)) errors.push(`${path}.maximum_must_match_field_contract`);
if ((contract.minimum !== undefined || contract.maximum !== undefined) && expression.omitIfInvalid !== true) {
errors.push(`${path}.bounded_number_must_omit_invalid`);
}
}
}
if (expression.constant !== undefined) {
validateMappedValueAgainstFieldContract(expression.constant, contract, `${path}.constant`, errors);
}
if (typeof expression.derive === "string") {
const derivation = isPlainObject(derivations) ? derivations[expression.derive] : undefined;
if (!isPlainObject(derivation)) return;
validateMappedValueAgainstFieldContract(derivation.default, contract, `${path}.derivation.default`, errors);
if (Array.isArray(contract.enum) && Array.isArray(derivation.rules)) {
for (const [index, rule] of derivation.rules.entries()) {
const output = typeof rule === "string" ? rule.slice(rule.lastIndexOf(".") + 1) : undefined;
validateMappedValueAgainstFieldContract(output, contract, `${path}.derivation.rules[${index}]`, errors);
}
}
}
}
function validateMappedValueAgainstFieldContract(value, contract, path, errors) {
if (!fieldContractValueMatchesType(value, contract.type)) errors.push(`${path}_type_mismatch`);
if (Array.isArray(contract.enum) && !contract.enum.some((allowed) => Object.is(allowed, value))) {
errors.push(`${path}_not_in_field_contract_enum`);
}
if (typeof value === "number") {
if (Number.isFinite(contract.minimum) && value < contract.minimum) errors.push(`${path}_below_field_contract_minimum`);
if (Number.isFinite(contract.maximum) && value > contract.maximum) errors.push(`${path}_above_field_contract_maximum`);
}
}
function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") {
return isPlainObject(value)
&& value.type === "Point"
&& Array.isArray(value.coordinates)
&& value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite);
}
return typeof value === type && (type !== "number" || Number.isFinite(value));
}
function stableLiteral(value) {
return `${typeof value}:${JSON.stringify(value)}`;
}
function validateExpression(value, path, errors) {
@@ -769,6 +909,18 @@ function validateExpression(value, path, errors) {
if (value.fallback !== undefined && value.fallback !== "collection_received_at") errors.push(`${path}.fallback_invalid`);
if (value.derive !== undefined) requiredIdentifier(value.derive, `${path}.derive`, errors);
if (value.omitIfMissing !== undefined && typeof value.omitIfMissing !== "boolean") errors.push(`${path}.omitIfMissing_must_be_boolean`);
if (value.omitIfInvalid !== undefined && typeof value.omitIfInvalid !== "boolean") errors.push(`${path}.omitIfInvalid_must_be_boolean`);
if (value.minimum !== undefined || value.maximum !== undefined) {
if (value.coerce !== "number") errors.push(`${path}.range_requires_number_coercion`);
if (value.minimum !== undefined && !Number.isFinite(value.minimum)) errors.push(`${path}.minimum_invalid`);
if (value.maximum !== undefined && !Number.isFinite(value.maximum)) errors.push(`${path}.maximum_invalid`);
if (Number.isFinite(value.minimum) && Number.isFinite(value.maximum) && value.minimum > value.maximum) {
errors.push(`${path}.range_invalid`);
}
if (value.omitIfInvalid !== true) errors.push(`${path}.range_requires_omitIfInvalid`);
} else if (value.omitIfInvalid !== undefined) {
errors.push(`${path}.omitIfInvalid_requires_range`);
}
}
function validateDerivation(value, path, errors) {