diff --git a/infra/deploy-runner/generate-engine-gelios-v11-catalogs.mjs b/infra/deploy-runner/generate-engine-gelios-v11-catalogs.mjs new file mode 100644 index 0000000..662a9de --- /dev/null +++ b/infra/deploy-runner/generate-engine-gelios-v11-catalogs.mjs @@ -0,0 +1,181 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + compileL2ExecutionPlan, + instantiateL2Connection, +} from "../../packages/external-provider-contract/src/index.mjs"; +import { + geliosProviderPackageV11, + geliosTelemetryFieldRegistryV4, +} from "../../packages/external-provider-contract/providers/gelios/v11/index.mjs"; + +const engineRoot = resolve( + process.env.NODEDC_ENGINE_SOURCE_ROOT || "../NODEDC_ENGINE_INFRA", +); +const executionCatalogPath = resolve( + engineRoot, + "nodedc-source/server/assets/execution-plans/v1/catalog.json", +); +const securityCatalogPath = resolve( + engineRoot, + "nodedc-source/server/assets/provider-packages/v1/catalog.json", +); + +const executionCatalog = JSON.parse( + await readFile(executionCatalogPath, "utf8"), +); +executionCatalog.runtime.compilerVersions = [ + ...new Set([...executionCatalog.runtime.compilerVersions, "1.4.0"]), +].sort(); +executionCatalog.runtime.derivationKinds = [ + ...new Set([ + ...executionCatalog.runtime.derivationKinds, + "bounded_named_values", + "bounded_string_list", + ]), +].sort(); +executionCatalog.runtime.ruleIds = [ + ...new Set([ + ...executionCatalog.runtime.ruleIds, + "array.named_values", + "array.string_values", + ]), +].sort(); +executionCatalog.packages = executionCatalog.packages.filter( + (providerPackage) => providerPackage.id !== geliosProviderPackageV11.id, +); +executionCatalog.packages.push(executionCatalogEntry()); +await writeFile(executionCatalogPath, `${JSON.stringify(executionCatalog, null, 2)}\n`); + +const securityCatalog = JSON.parse( + await readFile(securityCatalogPath, "utf8"), +); +securityCatalog.packages = securityCatalog.packages.filter( + (providerPackage) => providerPackage.id !== geliosProviderPackageV11.id, +); +securityCatalog.packages.push(securityCatalogEntry()); +await writeFile(securityCatalogPath, `${JSON.stringify(securityCatalog, null, 2)}\n`); + +console.log(JSON.stringify({ + ok: true, + providerPackage: geliosProviderPackageV11.id, + executionCatalogPath, + securityCatalogPath, +}, null, 2)); + +function executionCatalogEntry() { + const profiles = geliosProviderPackageV11.collectionProfiles.map((profile) => { + const connection = instantiateL2Connection(geliosProviderPackageV11, { + tenantId: "tenant-catalog-build", + connectionId: `catalog-${profile.id.replaceAll(".", "-")}`, + collectionProfileId: profile.id, + providerCredentialRef: "ndc-credref:catalog-build-provider-v11", + }); + const plan = compileL2ExecutionPlan( + geliosProviderPackageV11, + connection, + profile.dataProductId === "fleet.positions.current.v5" + ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV4 } + : {}, + ); + const { packageDigest: _packageDigest, ...artifacts } = plan.artifacts; + return { + id: profile.id, + dataProductId: profile.dataProductId, + capabilityIds: [...profile.capabilityIds], + stepSignatures: plan.steps.map((step) => ({ + id: step.id, + kind: step.kind, + ...(step.config.capabilityId ? { capabilityId: step.config.capabilityId } : {}), + ...(step.config.mappingContractId ? { + mappingContractId: step.config.mappingContractId, + } : {}), + ...(step.config.dataProductId ? { dataProductId: step.config.dataProductId } : {}), + ...(step.config.nodeType ? { nodeType: step.config.nodeType } : {}), + })), + artifacts, + }; + }); + return { + id: geliosProviderPackageV11.id, + providerId: geliosProviderPackageV11.providerId, + version: geliosProviderPackageV11.version, + contractDigest: canonicalDigest(geliosProviderPackageV11), + providerCredential: { + authModeId: "gelios.rest-rotating-bearer.v3", + credentialType: "ndcProviderRotatingAccessApi", + }, + publisher: { + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + credentialType: "ndcDataProductWriterApi", + }, + capabilities: geliosProviderPackageV11.capabilities.map((capability) => ({ + id: capability.id, + contractDigest: canonicalDigest(capability), + requestDigest: canonicalDigest(capability.request), + method: capability.request.method, + url: requestUrl(capability.request), + })), + profiles, + }; +} + +function securityCatalogEntry() { + const productsByCapability = new Map( + geliosProviderPackageV11.capabilities.map((capability) => [capability.id, new Set()]), + ); + for (const profile of geliosProviderPackageV11.collectionProfiles) { + if (profile.dataProductId !== "fleet.units.identity.current.v1") continue; + for (const capabilityId of profile.capabilityIds) { + productsByCapability.get(capabilityId)?.add(profile.dataProductId); + } + } + return { + id: geliosProviderPackageV11.id, + version: geliosProviderPackageV11.version, + providerId: geliosProviderPackageV11.providerId, + providerCredential: { + authModeId: "gelios.rest-rotating-bearer.v3", + credentialType: "ndcProviderRotatingAccessApi", + }, + capabilities: geliosProviderPackageV11.capabilities + .filter((capability) => productsByCapability.get(capability.id)?.size) + .map((capability) => ({ + id: capability.id, + classification: capability.classification, + status: capability.status, + request: { + method: capability.request.method, + url: requestUrl(capability.request), + }, + dataProductIds: [...productsByCapability.get(capability.id)].sort(), + })), + publisher: { + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + credentialType: "ndcDataProductWriterApi", + }, + }; +} + +function requestUrl(request) { + const url = new URL(request.path, request.baseUrl); + for (const [name, value] of Object.entries(request.query || {})) { + url.searchParams.append(name, String(value)); + } + return url.toString(); +} + +function canonicalDigest(value) { + return `sha256:${createHash("sha256").update(JSON.stringify(stableValue(value)), "utf8").digest("hex")}`; +} + +function stableValue(value) { + if (Array.isArray(value)) return value.map(stableValue); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.keys(value).sort().map((key) => [key, stableValue(value[key])]), + ); +} diff --git a/packages/external-provider-contract/providers/gelios/capability-catalog-v3.mjs b/packages/external-provider-contract/providers/gelios/capability-catalog-v3.mjs new file mode 100644 index 0000000..4be1f64 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/capability-catalog-v3.mjs @@ -0,0 +1,44 @@ +import { geliosCapabilityCatalogV2 } from "./capability-catalog-v2.mjs"; + +const IDENTITY_PRODUCT_ID = "fleet.units.identity.current.v1"; +const IMPLEMENTED_FAMILIES = new Set([ + "gelios.unit.access_metadata", + "gelios.unit.asset_identifiers", + "gelios.unit.asset_profile", + "gelios.unit.custom_fields", + "gelios.unit.device_identifiers", + "gelios.unit.driver", + "gelios.unit.equipment", + "gelios.unit.identity", +]); + +export const geliosCapabilityCatalogV3 = deepFreeze(updateFamilies({ + ...structuredClone(geliosCapabilityCatalogV2), + version: "3.0.0", + authority: { + ...geliosCapabilityCatalogV2.authority, + checkedAt: "2026-07-24T08:00:00Z", + }, +})); + +function updateFamilies(value) { + value.dataFamilies = value.dataFamilies.map((family) => { + if (!IMPLEMENTED_FAMILIES.has(family.id)) return family; + return { + ...family, + disposition: "profile", + status: "implemented", + dataProductIds: [ + ...new Set([...(family.dataProductIds || []), IDENTITY_PRODUCT_ID]), + ], + }; + }); + return value; +} + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/providers/gelios/index.mjs b/packages/external-provider-contract/providers/gelios/index.mjs index fa43e40..b4ef90d 100644 --- a/packages/external-provider-contract/providers/gelios/index.mjs +++ b/packages/external-provider-contract/providers/gelios/index.mjs @@ -1,5 +1,8 @@ export { geliosCapabilityCatalogV1 } from "./capability-catalog-v1.mjs"; export { geliosCapabilityCatalogV2 } from "./capability-catalog-v2.mjs"; +export { geliosCapabilityCatalogV3 } from "./capability-catalog-v3.mjs"; export { geliosProviderPackageV7 } from "./v7/index.mjs"; export * from "./v8/index.mjs"; export * from "./v9/index.mjs"; +export { geliosProviderPackageV10 } from "./v10/index.mjs"; +export { geliosProviderPackageV11 } from "./v11/index.mjs"; diff --git a/packages/external-provider-contract/providers/gelios/v11/index.mjs b/packages/external-provider-contract/providers/gelios/v11/index.mjs new file mode 100644 index 0000000..ee7bf49 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v11/index.mjs @@ -0,0 +1,10 @@ +export { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, + GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + GELIOS_UNIT_IDENTITY_DATA_PRODUCT_VERSION, + GELIOS_UNIT_IDENTITY_ONTOLOGY_REVISION, + geliosProviderPackageV11, +} from "./package.mjs"; +export { geliosTelemetryFieldRegistryV4 } from "./telemetry-field-registry.mjs"; +export { geliosCapabilityCatalogV3 } from "../capability-catalog-v3.mjs"; diff --git a/packages/external-provider-contract/providers/gelios/v11/package.mjs b/packages/external-provider-contract/providers/gelios/v11/package.mjs new file mode 100644 index 0000000..dc45bae --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v11/package.mjs @@ -0,0 +1,309 @@ +import { geliosProviderPackageV10 } from "../v10/package.mjs"; + +export const GELIOS_PROVIDER_PACKAGE_ID = "gelios.provider.v11"; +export const GELIOS_PROVIDER_PACKAGE_VERSION = "11.0.0"; +export const GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID = "fleet.units.identity.current.v1"; +export const GELIOS_UNIT_IDENTITY_DATA_PRODUCT_VERSION = "1.0.0"; +export const GELIOS_UNIT_IDENTITY_ONTOLOGY_REVISION = "ontology.map.moving_object.v3"; + +const CAPABILITY_ID = "gelios.units.identity.read"; +const FIELD_POLICY_ID = "gelios.units.identity.fields.v1"; +const COLLECTION_PROFILE_ID = "gelios.units.identity.warm.v1"; +const MAPPING_ID = "gelios.units.to.fleet.units.identity.current.v1"; +const TEMPLATE_ID = "gelios.units.identity.l2.v1"; + +const identityFields = Object.freeze([ + "asset_brand", + "asset_color", + "asset_model", + "asset_number_plate", + "asset_vin", + "asset_year", + "assigned_driver_name", + "assigned_driver_phone", + "available_user_ids", + "available_user_logins", + "custom_fields", + "device_imei", + "device_phone_primary", + "device_phone_secondary", + "display_name", + "hardware_manufacturer_id", + "hardware_manufacturer_name", + "hardware_port", + "hardware_type_class", + "hardware_type_id", + "hardware_type_name", + "provider_creator_id", + "provider_creator_login", + "provider_unit_id", + "unit_type_class", + "unit_type_id", + "unit_type_name", +]); + +const value = structuredClone(geliosProviderPackageV10); +const baseUnitsCapability = value.capabilities.find( + (capability) => capability.id === "gelios.units.current.read", +); +if (!baseUnitsCapability) throw new Error("gelios_v11_units_capability_missing"); +const profileTemplate = value.l2Templates.find( + (template) => template.id === "gelios.units.profile.l2.v1", +); +if (!profileTemplate) throw new Error("gelios_v11_profile_template_missing"); + +value.id = GELIOS_PROVIDER_PACKAGE_ID; +value.version = GELIOS_PROVIDER_PACKAGE_VERSION; +value.manifest = { + ...value.manifest, + id: "gelios.provider.manifest.v11", + version: GELIOS_PROVIDER_PACKAGE_VERSION, + capabilityIds: [...value.manifest.capabilityIds, CAPABILITY_ID], + fieldPolicyIds: [...value.manifest.fieldPolicyIds, FIELD_POLICY_ID], + collectionProfileIds: [...value.manifest.collectionProfileIds, COLLECTION_PROFILE_ID], + dataProductIds: [...value.manifest.dataProductIds, GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID], + mappingContractIds: [...value.manifest.mappingContractIds, MAPPING_ID], + l2TemplateIds: [...value.manifest.l2TemplateIds, TEMPLATE_ID], +}; + +value.capabilities.push({ + ...structuredClone(baseUnitsCapability), + id: CAPABILITY_ID, + request: { + ...structuredClone(baseUnitsCapability.request), + query: { + inclextra: "true", + inclcstmflds: "true", + incldrvrs: "true", + inclusersav: "true", + }, + }, +}); + +value.fieldPolicies.push({ + id: FIELD_POLICY_ID, + version: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_VERSION, + dataProductId: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + targetFields: [...identityFields], + unknownSourceFields: "drop", + dynamicSourceFields: "drop_until_classified", + restrictedSourcePaths: [ + "commands", + "currentUserAccess", + "extraInfo.autocompleteParams", + "hwDecryptKey", + "lastMsg", + "lastSensorsVal", + "maintenancePlan", + "preSetCommandGroup", + "sensors", + "stationaryLat", + "stationaryLon", + "storageInfo", + ], +}); + +value.collectionProfiles.push({ + id: COLLECTION_PROFILE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + mode: "realtime", + schedule: { intervalMs: 5 * 60 * 1000 }, + capabilityIds: [CAPABILITY_ID], + dataProductId: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + mappingContractId: MAPPING_ID, + fieldPolicyId: FIELD_POLICY_ID, + l2TemplateId: TEMPLATE_ID, + entityScope: { + mode: "all_visible_to_credential", + refresh: "each_collection_run", + businessEntityFilter: "forbidden", + }, + batching: { maxFacts: 5000 }, + cardinality: { + maxCurrentEntities: 5000, + onExceed: "require_partitioned_data_product", + }, + retry: { maxAttempts: 4, backoff: "fixed_delay", delayMs: 2000 }, +}); + +value.dataProducts.push({ + id: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + version: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_UNIT_IDENTITY_ONTOLOGY_REVISION, + deliveryMode: "snapshot+patch", + semanticTypes: ["map.moving_object"], + fields: [...identityFields], + fieldContracts: { + asset_brand: optional("string"), + asset_color: optional("string"), + asset_model: optional("string"), + asset_number_plate: optional("string"), + asset_vin: optional("string"), + asset_year: nonNegative(), + assigned_driver_name: optional("string"), + assigned_driver_phone: optional("string"), + available_user_ids: optional("string_array"), + available_user_logins: optional("string_array"), + custom_fields: optional("string_array"), + device_imei: optional("string"), + device_phone_primary: optional("string"), + device_phone_secondary: optional("string"), + display_name: { type: "string", required: true }, + hardware_manufacturer_id: optional("string"), + hardware_manufacturer_name: optional("string"), + hardware_port: nonNegative(), + hardware_type_class: optional("string"), + hardware_type_id: optional("string"), + hardware_type_name: optional("string"), + provider_creator_id: optional("string"), + provider_creator_login: optional("string"), + provider_unit_id: { type: "string", required: true }, + unit_type_class: optional("string"), + unit_type_id: optional("string"), + unit_type_name: optional("string"), + }, + history: { mode: "none", retentionDays: 1 }, +}); + +value.mappingContracts.push({ + schemaVersion: "nodedc.semantic-mapping/v1", + id: MAPPING_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + sourceCapabilityId: CAPABILITY_ID, + fieldPolicyId: FIELD_POLICY_ID, + target: { + dataProductId: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + version: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_UNIT_IDENTITY_ONTOLOGY_REVISION, + semanticType: "map.moving_object", + }, + derivations: { + available_user_ids: boundedStringList("availableToUsers", "id", 256), + available_user_logins: boundedStringList("availableToUsers", "login", 256), + custom_fields: { + kind: "bounded_named_values", + rules: ["array.named_values"], + default: [], + parameters: { + arrayPath: "customFields", + namePath: "name", + valuePath: "value", + maxItems: 256, + maxItemLength: 512, + }, + }, + }, + fact: { + sourceId: { + strategy: "first_non_empty", + paths: ["id", "unit_id", "unitId"], + coerce: "string", + prefix: "gelios-unit-", + }, + semanticType: { constant: "map.moving_object" }, + observedAt: { + strategy: "first_non_empty", + paths: ["updatedAt"], + coerce: "unix_or_iso_timestamp", + fallback: "collection_received_at", + }, + attributes: { + asset_brand: stringField(["extraInfo.brand"]), + asset_color: stringField(["extraInfo.color"]), + asset_model: stringField(["extraInfo.model"]), + asset_number_plate: stringField(["extraInfo.numberPlate"]), + asset_vin: stringField(["extraInfo.vin"]), + asset_year: nonNegativeField(["extraInfo.year"]), + assigned_driver_name: stringField(["driver.name"]), + assigned_driver_phone: stringField(["driver.phone"]), + available_user_ids: { derive: "available_user_ids" }, + available_user_logins: { derive: "available_user_logins" }, + custom_fields: { derive: "custom_fields" }, + device_imei: stringField(["imei"]), + device_phone_primary: stringField(["phone"]), + device_phone_secondary: stringField(["phone2"]), + display_name: stringField(["name", "unit_name", "title", "label"], false), + hardware_manufacturer_id: stringField(["hwManufacturer.id"]), + hardware_manufacturer_name: stringField(["hwManufacturer.name"]), + hardware_port: nonNegativeField(["hwType.port"]), + hardware_type_class: stringField(["hwType.type"]), + hardware_type_id: stringField(["hwType.id"]), + hardware_type_name: stringField(["hwType.name"]), + provider_creator_id: stringField(["creator.id"]), + provider_creator_login: stringField(["creator.login"]), + provider_unit_id: stringField(["id", "unit_id", "unitId"], false), + unit_type_class: stringField(["unitType.type"]), + unit_type_id: stringField(["unitType.id"]), + unit_type_name: stringField(["unitType.name"]), + }, + }, +}); + +value.l2Templates.push({ + ...structuredClone(profileTemplate), + id: TEMPLATE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + steps: [ + { id: "collection.trigger", kind: "collection_trigger", collectionProfileDriven: true }, + { id: "provider.fetch-identity", kind: "provider_request", capabilityId: CAPABILITY_ID }, + { id: "provider.extract-units", kind: "extract_items", capabilityId: CAPABILITY_ID }, + { id: "ontology.map", kind: "semantic_mapping", mappingContractId: MAPPING_ID }, + { + id: "data-product.publish", + kind: "data_product_publish", + dataProductId: GELIOS_UNIT_IDENTITY_DATA_PRODUCT_ID, + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + }, + ], +}); + +export const geliosProviderPackageV11 = deepFreeze(value); + +function optional(type) { + return { type, required: false }; +} + +function nonNegative() { + return { type: "number", required: false, minimum: 0 }; +} + +function stringField(paths, omitIfMissing = true) { + return { + strategy: "first_non_empty", + paths, + coerce: "string", + ...(omitIfMissing ? { omitIfMissing: true } : {}), + }; +} + +function nonNegativeField(paths) { + return { + strategy: "first_non_empty", + paths, + coerce: "number", + minimum: 0, + omitIfMissing: true, + omitIfInvalid: true, + }; +} + +function boundedStringList(arrayPath, valuePath, maxItems) { + return { + kind: "bounded_string_list", + rules: ["array.string_values"], + default: [], + parameters: { + arrayPath, + valuePath, + maxItems, + maxItemLength: 512, + }, + }; +} + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/providers/gelios/v11/telemetry-field-registry.mjs b/packages/external-provider-contract/providers/gelios/v11/telemetry-field-registry.mjs new file mode 100644 index 0000000..062c90e --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v11/telemetry-field-registry.mjs @@ -0,0 +1,20 @@ +import { geliosTelemetryFieldRegistryV3 } from "../v10/telemetry-field-registry.mjs"; +import { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, +} from "./package.mjs"; + +const value = structuredClone(geliosTelemetryFieldRegistryV3); +value.providerPackage = { + id: GELIOS_PROVIDER_PACKAGE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, +}; + +export const geliosTelemetryFieldRegistryV4 = deepFreeze(value); + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/src/l2-execution-plan.mjs b/packages/external-provider-contract/src/l2-execution-plan.mjs index 19233e0..1ee45a8 100644 --- a/packages/external-provider-contract/src/l2-execution-plan.mjs +++ b/packages/external-provider-contract/src/l2-execution-plan.mjs @@ -9,10 +9,11 @@ export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1"; export const L2_GRAPH_BLUEPRINT_SCHEMA_VERSION = "nodedc.l2-graph-blueprint/v1"; export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1"; export const L2_MAPPING_RUNTIME_SCHEMA_VERSION = "nodedc.semantic-mapping-runtime/v1"; -export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.3.0"; +export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.4.0"; export const L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS = Object.freeze([ "1.1.0", "1.2.0", + "1.3.0", L2_EXECUTION_PLAN_COMPILER_VERSION, ]); diff --git a/packages/external-provider-contract/src/provider-package.mjs b/packages/external-provider-contract/src/provider-package.mjs index 590c915..235e779 100644 --- a/packages/external-provider-contract/src/provider-package.mjs +++ b/packages/external-provider-contract/src/provider-package.mjs @@ -295,7 +295,10 @@ export function validateProviderPackage(value) { const restricted = Array.isArray(fieldPolicy.restrictedSourcePaths) ? fieldPolicy.restrictedSourcePaths.filter(isCanonicalSourcePath) : []; - if (mappingSourcePaths(mapping.fact).some((path) => ( + if (mappingSourcePaths({ + fact: mapping.fact, + derivations: mapping.derivations, + }).some((path) => ( restricted.some((restrictedPath) => ( path === restrictedPath || path.startsWith(`${restrictedPath}.`) @@ -1041,7 +1044,14 @@ function validateExpression(value, path, errors) { function validateDerivation(value, path, errors) { if (!isPlainObject(value)) return errors.push(`${path}_must_be_object`); rejectUnknownKeys(value, DERIVATION_KEYS, path, errors); - if (!new Set(["boolean_rule", "ordered_rules", "flag_set", "bounded_readings"]).has(value.kind)) errors.push(`${path}.kind_invalid`); + if (!new Set([ + "boolean_rule", + "ordered_rules", + "flag_set", + "bounded_readings", + "bounded_string_list", + "bounded_named_values", + ]).has(value.kind)) errors.push(`${path}.kind_invalid`); requiredUniqueIdentifierArray(value.rules, `${path}.rules`, errors); if (value.default === undefined) { errors.push(`${path}.default_required`); @@ -1059,6 +1069,70 @@ function validateDerivation(value, path, errors) { } } } + if (value.kind === "bounded_string_list") { + validateBoundedStringListDerivation(value, path, errors); + } + if (value.kind === "bounded_named_values") { + validateBoundedNamedValuesDerivation(value, path, errors); + } +} + +function validateBoundedStringListDerivation(value, path, errors) { + const parameters = isPlainObject(value.parameters) ? value.parameters : {}; + const allowed = new Set(["arrayPath", "valuePath", "maxItems", "maxItemLength"]); + if (Object.keys(parameters).some((key) => !allowed.has(key))) { + errors.push(`${path}.parameters_shape_invalid`); + } + for (const key of ["arrayPath", "valuePath"]) { + if (!isCanonicalSourcePath(parameters[key])) { + errors.push(`${path}.parameters.${key}_must_be_canonical_dot_path`); + } + } + if (!Number.isInteger(parameters.maxItems) || parameters.maxItems < 1 || parameters.maxItems > 256) { + errors.push(`${path}.parameters.maxItems_invalid`); + } + if ( + !Number.isInteger(parameters.maxItemLength) + || parameters.maxItemLength < 1 + || parameters.maxItemLength > 512 + ) { + errors.push(`${path}.parameters.maxItemLength_invalid`); + } + if (!Array.isArray(value.default) || value.default.length !== 0) { + errors.push(`${path}.default_must_be_empty_array`); + } +} + +function validateBoundedNamedValuesDerivation(value, path, errors) { + const parameters = isPlainObject(value.parameters) ? value.parameters : {}; + const allowed = new Set([ + "arrayPath", + "namePath", + "valuePath", + "maxItems", + "maxItemLength", + ]); + if (Object.keys(parameters).some((key) => !allowed.has(key))) { + errors.push(`${path}.parameters_shape_invalid`); + } + for (const key of ["arrayPath", "namePath", "valuePath"]) { + if (!isCanonicalSourcePath(parameters[key])) { + errors.push(`${path}.parameters.${key}_must_be_canonical_dot_path`); + } + } + if (!Number.isInteger(parameters.maxItems) || parameters.maxItems < 1 || parameters.maxItems > 256) { + errors.push(`${path}.parameters.maxItems_invalid`); + } + if ( + !Number.isInteger(parameters.maxItemLength) + || parameters.maxItemLength < 1 + || parameters.maxItemLength > 512 + ) { + errors.push(`${path}.parameters.maxItemLength_invalid`); + } + if (!Array.isArray(value.default) || value.default.length !== 0) { + errors.push(`${path}.default_must_be_empty_array`); + } } function mappingDerivationReferences(value, found = new Set()) { @@ -1087,7 +1161,11 @@ function mappingSourcePaths(value, found = []) { if (isCanonicalSourcePath(value[key])) found.push(value[key]); } Object.entries(value).forEach(([key, item]) => { - if (key !== "paths" && !key.endsWith("Path")) mappingSourcePaths(item, found); + if (key !== "paths" && key.endsWith("Path") && isCanonicalSourcePath(item)) { + found.push(item); + } else if (key !== "paths" && !key.endsWith("Path")) { + mappingSourcePaths(item, found); + } }); return found; } diff --git a/packages/external-provider-contract/test/l2-execution-plan.test.mjs b/packages/external-provider-contract/test/l2-execution-plan.test.mjs index bbfb639..403b67d 100644 --- a/packages/external-provider-contract/test/l2-execution-plan.test.mjs +++ b/packages/external-provider-contract/test/l2-execution-plan.test.mjs @@ -14,26 +14,26 @@ import { validateL2ExecutionPlan, } from "../src/index.mjs"; import { - geliosProviderPackageV10, - geliosTelemetryFieldRegistryV3, -} from "../providers/gelios/v10/index.mjs"; + geliosProviderPackageV11, + geliosTelemetryFieldRegistryV4, +} from "../providers/gelios/v11/index.mjs"; -const positionsConnection = instantiateL2Connection(geliosProviderPackageV10, { +const positionsConnection = instantiateL2Connection(geliosProviderPackageV11, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-compiler-fixture", collectionProfileId: "gelios.positions.current.realtime.v7", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", }); const positionsPlan = compileL2ExecutionPlan( - geliosProviderPackageV10, + geliosProviderPackageV11, positionsConnection, - { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 }, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV4 }, ); assert.equal(positionsPlan.schemaVersion, L2_EXECUTION_PLAN_SCHEMA_VERSION); -assert.equal(L2_EXECUTION_PLAN_COMPILER_VERSION, "1.3.0"); -assert.deepEqual(L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS, ["1.1.0", "1.2.0", "1.3.0"]); -assert.equal(positionsPlan.compilerVersion, "1.3.0"); +assert.equal(L2_EXECUTION_PLAN_COMPILER_VERSION, "1.4.0"); +assert.deepEqual(L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS, ["1.1.0", "1.2.0", "1.3.0", "1.4.0"]); +assert.equal(positionsPlan.compilerVersion, "1.4.0"); assert.deepEqual(validateL2ExecutionPlan(positionsPlan), { ok: true, errors: [] }); assert.match(positionsPlan.executionPlanDigest, /^sha256:[a-f0-9]{64}$/); assert.match(positionsPlan.artifacts.packageDigest, /^sha256:[a-f0-9]{64}$/); @@ -123,9 +123,9 @@ assert.equal(Object.isFrozen(positionsPlan.steps), true); assert.equal(Object.isFrozen(positionsPlan.graphBlueprint), true); const clonedPlan = compileL2ExecutionPlan( - structuredClone(geliosProviderPackageV10), + structuredClone(geliosProviderPackageV11), structuredClone(positionsConnection), - { telemetryFieldRegistry: structuredClone(geliosTelemetryFieldRegistryV3) }, + { telemetryFieldRegistry: structuredClone(geliosTelemetryFieldRegistryV4) }, ); assert.equal(clonedPlan.executionPlanDigest, positionsPlan.executionPlanDigest); @@ -151,34 +151,34 @@ assert.equal( ); assert.throws( - () => compileL2ExecutionPlan(geliosProviderPackageV10, positionsConnection), + () => compileL2ExecutionPlan(geliosProviderPackageV11, positionsConnection), /telemetry_registry_required/, ); -const profileConnection = instantiateL2Connection(geliosProviderPackageV10, { +const profileConnection = instantiateL2Connection(geliosProviderPackageV11, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-profile-compiler-fixture", collectionProfileId: "gelios.units.profile.cold.v1", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", }); -const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV10, profileConnection); +const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV11, profileConnection); assert.deepEqual(validateL2ExecutionPlan(profilePlan), { ok: true, errors: [] }); assert.equal(profilePlan.artifacts.telemetryRegistryDigest, undefined); assert.notEqual(profilePlan.executionPlanDigest, positionsPlan.executionPlanDigest); assert.deepEqual(profilePlan.graphBlueprint.runtime.mappingRuntime.derivationKinds, []); -for (const collectionProfile of geliosProviderPackageV10.collectionProfiles) { - const connection = instantiateL2Connection(geliosProviderPackageV10, { +for (const collectionProfile of geliosProviderPackageV11.collectionProfiles) { + const connection = instantiateL2Connection(geliosProviderPackageV11, { tenantId: "tenant-all-profiles-fixture", connectionId: `connection-${collectionProfile.id.replaceAll(".", "-")}`, collectionProfileId: collectionProfile.id, providerCredentialRef: "ndc-credref:provider-all-profiles-fixture", }); const plan = compileL2ExecutionPlan( - geliosProviderPackageV10, + geliosProviderPackageV11, connection, collectionProfile.dataProductId === "fleet.positions.current.v5" - ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 } + ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV4 } : {}, ); assert.deepEqual(validateL2ExecutionPlan(plan), { ok: true, errors: [] }); @@ -188,16 +188,16 @@ for (const collectionProfile of geliosProviderPackageV10.collectionProfiles) { assert.equal(plan.graphBlueprint.edges.length, plan.steps.length + expectedEntrypoints - 2); } -const secondConnection = instantiateL2Connection(geliosProviderPackageV10, { +const secondConnection = instantiateL2Connection(geliosProviderPackageV11, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-compiler-fixture-two", collectionProfileId: "gelios.positions.current.realtime.v7", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0002", }); const secondPlan = compileL2ExecutionPlan( - geliosProviderPackageV10, + geliosProviderPackageV11, secondConnection, - { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 }, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV4 }, ); assert.notEqual(secondPlan.executionPlanDigest, positionsPlan.executionPlanDigest); diff --git a/packages/external-provider-contract/test/provider-capability-catalog.test.mjs b/packages/external-provider-contract/test/provider-capability-catalog.test.mjs index 8601c88..8d25049 100644 --- a/packages/external-provider-contract/test/provider-capability-catalog.test.mjs +++ b/packages/external-provider-contract/test/provider-capability-catalog.test.mjs @@ -3,11 +3,13 @@ import { validateProviderCapabilityCatalog } from "../src/index.mjs"; import { geliosCapabilityCatalogV1, geliosCapabilityCatalogV2, + geliosCapabilityCatalogV3, geliosProviderPackageV8, } from "../providers/gelios/index.mjs"; assert.deepEqual(validateProviderCapabilityCatalog(geliosCapabilityCatalogV1), { ok: true, errors: [] }); assert.deepEqual(validateProviderCapabilityCatalog(geliosCapabilityCatalogV2), { ok: true, errors: [] }); +assert.deepEqual(validateProviderCapabilityCatalog(geliosCapabilityCatalogV3), { ok: true, errors: [] }); const expectedUnitRoots = [ "activatedAt", "availableToUsers", "blockTime", "commands", "counters", "createdAt", @@ -54,6 +56,16 @@ assert.equal( geliosCapabilityCatalogV2.dataFamilies.find(({ id }) => id === "gelios.unit.sensor_definitions").status, "planned", ); +for (const familyId of [ + "gelios.unit.device_identifiers", + "gelios.unit.driver", + "gelios.unit.asset_identifiers", + "gelios.unit.custom_fields", +]) { + const family = geliosCapabilityCatalogV3.dataFamilies.find(({ id }) => id === familyId); + assert.equal(family.status, "implemented"); + assert.equal(family.dataProductIds.includes("fleet.units.identity.current.v1"), true); +} const restrictedPublish = structuredClone(geliosCapabilityCatalogV1); restrictedPublish.dataFamilies.find(({ id }) => id === "gelios.unit.device_identifiers").disposition = "publish"; diff --git a/packages/external-provider-contract/test/provider-package.test.mjs b/packages/external-provider-contract/test/provider-package.test.mjs index 87cd67a..2b0440e 100644 --- a/packages/external-provider-contract/test/provider-package.test.mjs +++ b/packages/external-provider-contract/test/provider-package.test.mjs @@ -23,6 +23,7 @@ import { geliosProviderPackageV7 } from "../providers/gelios/v7/index.mjs"; import { geliosProviderPackageV8 } from "../providers/gelios/v8/index.mjs"; import { geliosProviderPackageV9 } from "../providers/gelios/v9/index.mjs"; import { geliosProviderPackageV10 } from "../providers/gelios/v10/index.mjs"; +import { geliosProviderPackageV11 } from "../providers/gelios/v11/index.mjs"; import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs"; const expectedFields = [ @@ -51,10 +52,13 @@ assert.deepEqual(validateProviderPackage(geliosProviderPackageV7), { ok: true, e assert.deepEqual(validateProviderPackage(geliosProviderPackageV8), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV9), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV10), { ok: true, errors: [] }); +assert.deepEqual(validateProviderPackage(geliosProviderPackageV11), { ok: true, errors: [] }); assert.equal(geliosProviderPackageV9.id, "gelios.provider.v9"); assert.equal(geliosProviderPackageV9.version, "9.0.0"); assert.equal(geliosProviderPackageV10.id, "gelios.provider.v10"); assert.equal(geliosProviderPackageV10.version, "10.0.0"); +assert.equal(geliosProviderPackageV11.id, "gelios.provider.v11"); +assert.equal(geliosProviderPackageV11.version, "11.0.0"); assert.equal(geliosProviderPackageV8.id, "gelios.provider.v8"); const profileProduct = geliosProviderPackageV8.dataProducts.find((product) => product.id === "fleet.units.profile.current.v1"); @@ -97,6 +101,34 @@ assert.equal(contactsMapping.fact.attributes.device_phone_primary.guardPath, "ph assert.equal(contactsMapping.fact.attributes.device_phone_secondary.guardPath, "phone2IsVisible"); assert.equal(contactsMapping.fact.attributes.provider_creator_login.paths[0], "creator.login"); +const identityProduct = geliosProviderPackageV11.dataProducts.find( + (product) => product.id === "fleet.units.identity.current.v1", +); +const registeredIdentityProduct = JSON.parse(await readFile(new URL( + "../../../services/external-data-plane/definitions/fleet.units.identity.current.v1.json", + import.meta.url, +), "utf8")); +assert.deepEqual(identityProduct, registeredIdentityProduct); +assert.equal(identityProduct.fieldContracts.available_user_logins.type, "string_array"); +assert.equal(identityProduct.fieldContracts.custom_fields.type, "string_array"); +const identityCapability = geliosProviderPackageV11.capabilities.find( + (capability) => capability.id === "gelios.units.identity.read", +); +assert.deepEqual(identityCapability.request.query, { + inclextra: "true", + inclcstmflds: "true", + incldrvrs: "true", + inclusersav: "true", +}); +const identityMapping = geliosProviderPackageV11.mappingContracts.find( + (mapping) => mapping.id === "gelios.units.to.fleet.units.identity.current.v1", +); +assert.equal(identityMapping.fact.attributes.device_imei.guardPath, undefined); +assert.equal(identityMapping.fact.attributes.device_phone_primary.guardPath, undefined); +assert.equal(identityMapping.fact.attributes.assigned_driver_name.paths[0], "driver.name"); +assert.equal(identityMapping.derivations.available_user_logins.kind, "bounded_string_list"); +assert.equal(identityMapping.derivations.custom_fields.kind, "bounded_named_values"); + const incompleteGuard = structuredClone(geliosProviderPackageV10); delete incompleteGuard.mappingContracts.at(-1).fact.attributes.device_imei.guardEquals; assert.equal( diff --git a/services/external-data-plane/definitions/fleet.units.identity.current.v1.json b/services/external-data-plane/definitions/fleet.units.identity.current.v1.json new file mode 100644 index 0000000..cd7f49a --- /dev/null +++ b/services/external-data-plane/definitions/fleet.units.identity.current.v1.json @@ -0,0 +1,71 @@ +{ + "id": "fleet.units.identity.current.v1", + "version": "1.0.0", + "ontologyRevision": "ontology.map.moving_object.v3", + "deliveryMode": "snapshot+patch", + "semanticTypes": [ + "map.moving_object" + ], + "fields": [ + "asset_brand", + "asset_color", + "asset_model", + "asset_number_plate", + "asset_vin", + "asset_year", + "assigned_driver_name", + "assigned_driver_phone", + "available_user_ids", + "available_user_logins", + "custom_fields", + "device_imei", + "device_phone_primary", + "device_phone_secondary", + "display_name", + "hardware_manufacturer_id", + "hardware_manufacturer_name", + "hardware_port", + "hardware_type_class", + "hardware_type_id", + "hardware_type_name", + "provider_creator_id", + "provider_creator_login", + "provider_unit_id", + "unit_type_class", + "unit_type_id", + "unit_type_name" + ], + "fieldContracts": { + "asset_brand": { "type": "string", "required": false }, + "asset_color": { "type": "string", "required": false }, + "asset_model": { "type": "string", "required": false }, + "asset_number_plate": { "type": "string", "required": false }, + "asset_vin": { "type": "string", "required": false }, + "asset_year": { "type": "number", "required": false, "minimum": 0 }, + "assigned_driver_name": { "type": "string", "required": false }, + "assigned_driver_phone": { "type": "string", "required": false }, + "available_user_ids": { "type": "string_array", "required": false }, + "available_user_logins": { "type": "string_array", "required": false }, + "custom_fields": { "type": "string_array", "required": false }, + "device_imei": { "type": "string", "required": false }, + "device_phone_primary": { "type": "string", "required": false }, + "device_phone_secondary": { "type": "string", "required": false }, + "display_name": { "type": "string", "required": true }, + "hardware_manufacturer_id": { "type": "string", "required": false }, + "hardware_manufacturer_name": { "type": "string", "required": false }, + "hardware_port": { "type": "number", "required": false, "minimum": 0 }, + "hardware_type_class": { "type": "string", "required": false }, + "hardware_type_id": { "type": "string", "required": false }, + "hardware_type_name": { "type": "string", "required": false }, + "provider_creator_id": { "type": "string", "required": false }, + "provider_creator_login": { "type": "string", "required": false }, + "provider_unit_id": { "type": "string", "required": true }, + "unit_type_class": { "type": "string", "required": false }, + "unit_type_id": { "type": "string", "required": false }, + "unit_type_name": { "type": "string", "required": false } + }, + "history": { + "mode": "none", + "retentionDays": 1 + } +} diff --git a/services/external-data-plane/test/definitions.test.mjs b/services/external-data-plane/test/definitions.test.mjs index 99d8c43..cfabe70 100644 --- a/services/external-data-plane/test/definitions.test.mjs +++ b/services/external-data-plane/test/definitions.test.mjs @@ -13,6 +13,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [ "fleet.positions.current.v4", "fleet.positions.current.v5", "fleet.units.contacts.current.v1", + "fleet.units.identity.current.v1", "fleet.units.profile.current.v1", "map.zones.current.v1", "map.zones.current.v2", @@ -62,23 +63,30 @@ assert.equal(bundled[6].version, "1.0.0"); assert.equal(bundled[6].ontologyRevision, "ontology.map.moving_object.v3"); assert.deepEqual(bundled[6].semanticTypes, ["map.moving_object"]); assert.equal(bundled[6].history.mode, "none"); -assert.equal(bundled[6].fieldContracts.hardware_port.type, "number"); +assert.equal(bundled[6].fieldContracts.available_user_logins.type, "string_array"); +assert.equal(bundled[6].fieldContracts.device_imei.type, "string"); assert.equal(bundled[7].version, "1.0.0"); -assert.equal(bundled[7].ontologyRevision, "ontology.map.zone.v1"); -assert.deepEqual(bundled[7].semanticTypes, ["map.zone"]); -assert.deepEqual(bundled[7].fieldContracts.geometry, { type: "geometry", required: true }); -assert.equal(bundled[7].fields.includes("dataset_authority"), false); -assert.equal(bundled[8].version, "2.0.0"); +assert.equal(bundled[7].ontologyRevision, "ontology.map.moving_object.v3"); +assert.deepEqual(bundled[7].semanticTypes, ["map.moving_object"]); +assert.equal(bundled[7].history.mode, "none"); +assert.equal(bundled[7].fieldContracts.hardware_port.type, "number"); +assert.equal(bundled[8].version, "1.0.0"); assert.equal(bundled[8].ontologyRevision, "ontology.map.zone.v1"); assert.deepEqual(bundled[8].semanticTypes, ["map.zone"]); assert.deepEqual(bundled[8].fieldContracts.geometry, { type: "geometry", required: true }); -assert.deepEqual(bundled[8].fieldContracts.dataset_authority.enum, ["moscow-department-of-transport"]); +assert.equal(bundled[8].fields.includes("dataset_authority"), false); +assert.equal(bundled[9].version, "2.0.0"); +assert.equal(bundled[9].ontologyRevision, "ontology.map.zone.v1"); +assert.deepEqual(bundled[9].semanticTypes, ["map.zone"]); +assert.deepEqual(bundled[9].fieldContracts.geometry, { type: "geometry", required: true }); +assert.deepEqual(bundled[9].fieldContracts.dataset_authority.enum, ["moscow-department-of-transport"]); assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length); assert.equal(Object.keys(bundled[4].fieldContracts).length, bundled[4].fields.length); assert.equal(Object.keys(bundled[5].fieldContracts).length, bundled[5].fields.length); assert.equal(Object.keys(bundled[6].fieldContracts).length, bundled[6].fields.length); assert.equal(Object.keys(bundled[7].fieldContracts).length, bundled[7].fields.length); assert.equal(Object.keys(bundled[8].fieldContracts).length, bundled[8].fields.length); +assert.equal(Object.keys(bundled[9].fieldContracts).length, bundled[9].fields.length); const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-")); try {