diff --git a/infra/deploy-runner/generate-engine-gelios-v10-catalogs.mjs b/infra/deploy-runner/generate-engine-gelios-v10-catalogs.mjs new file mode 100644 index 0000000..ba06dc3 --- /dev/null +++ b/infra/deploy-runner/generate-engine-gelios-v10-catalogs.mjs @@ -0,0 +1,167 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import { + compileL2ExecutionPlan, + instantiateL2Connection, +} from "../../packages/external-provider-contract/src/index.mjs"; +import { + geliosProviderPackageV10, + geliosTelemetryFieldRegistryV3, +} from "../../packages/external-provider-contract/providers/gelios/v10/index.mjs"; + +const engineRoot = resolve( + process.env.NODEDC_ENGINE_SOURCE_ROOT || "../NODEDC_ENGINE_INFRA", +); +const executionCatalogPath = resolve( + engineRoot, + "nodedc-source/server/assets/execution-plans/v1/catalog.json", +); +const securityCatalogPath = resolve( + engineRoot, + "nodedc-source/server/assets/provider-packages/v1/catalog.json", +); + +const executionCatalog = JSON.parse( + await readFile(executionCatalogPath, "utf8"), +); +executionCatalog.runtime.compilerVersions = [ + ...new Set([...executionCatalog.runtime.compilerVersions, "1.3.0"]), +].sort(); +executionCatalog.packages = executionCatalog.packages.filter( + (providerPackage) => providerPackage.id !== geliosProviderPackageV10.id, +); +executionCatalog.packages.push(executionCatalogEntry()); +await writeFile(executionCatalogPath, `${JSON.stringify(executionCatalog, null, 2)}\n`); + +const securityCatalog = JSON.parse( + await readFile(securityCatalogPath, "utf8"), +); +securityCatalog.packages = securityCatalog.packages.filter( + (providerPackage) => providerPackage.id !== geliosProviderPackageV10.id, +); +securityCatalog.packages.push(securityCatalogEntry()); +await writeFile(securityCatalogPath, `${JSON.stringify(securityCatalog, null, 2)}\n`); + +console.log(JSON.stringify({ + ok: true, + providerPackage: geliosProviderPackageV10.id, + executionCatalogPath, + securityCatalogPath, +}, null, 2)); + +function executionCatalogEntry() { + const profiles = geliosProviderPackageV10.collectionProfiles.map((profile) => { + const connection = instantiateL2Connection(geliosProviderPackageV10, { + tenantId: "tenant-catalog-build", + connectionId: `catalog-${profile.id.replaceAll(".", "-")}`, + collectionProfileId: profile.id, + providerCredentialRef: "ndc-credref:catalog-build-provider-v10", + }); + const plan = compileL2ExecutionPlan( + geliosProviderPackageV10, + connection, + profile.dataProductId === "fleet.positions.current.v5" + ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 } + : {}, + ); + const { packageDigest: _packageDigest, ...artifacts } = plan.artifacts; + return { + id: profile.id, + dataProductId: profile.dataProductId, + capabilityIds: [...profile.capabilityIds], + stepSignatures: plan.steps.map((step) => ({ + id: step.id, + kind: step.kind, + ...(step.config.capabilityId ? { capabilityId: step.config.capabilityId } : {}), + ...(step.config.mappingContractId ? { + mappingContractId: step.config.mappingContractId, + } : {}), + ...(step.config.dataProductId ? { dataProductId: step.config.dataProductId } : {}), + ...(step.config.nodeType ? { nodeType: step.config.nodeType } : {}), + })), + artifacts, + }; + }); + return { + id: geliosProviderPackageV10.id, + providerId: geliosProviderPackageV10.providerId, + version: geliosProviderPackageV10.version, + contractDigest: canonicalDigest(geliosProviderPackageV10), + providerCredential: { + authModeId: "gelios.rest-rotating-bearer.v3", + credentialType: "ndcProviderRotatingAccessApi", + }, + publisher: { + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + credentialType: "ndcDataProductWriterApi", + }, + capabilities: geliosProviderPackageV10.capabilities.map((capability) => ({ + id: capability.id, + contractDigest: canonicalDigest(capability), + requestDigest: canonicalDigest(capability.request), + method: capability.request.method, + url: requestUrl(capability.request), + })), + profiles, + }; +} + +function securityCatalogEntry() { + const productsByCapability = new Map( + geliosProviderPackageV10.capabilities.map((capability) => [capability.id, new Set()]), + ); + for (const profile of geliosProviderPackageV10.collectionProfiles) { + if (profile.dataProductId !== "fleet.units.contacts.current.v1") continue; + for (const capabilityId of profile.capabilityIds) { + productsByCapability.get(capabilityId)?.add(profile.dataProductId); + } + } + return { + id: geliosProviderPackageV10.id, + version: geliosProviderPackageV10.version, + providerId: geliosProviderPackageV10.providerId, + providerCredential: { + authModeId: "gelios.rest-rotating-bearer.v3", + credentialType: "ndcProviderRotatingAccessApi", + }, + capabilities: geliosProviderPackageV10.capabilities + .filter((capability) => productsByCapability.get(capability.id)?.size) + .map((capability) => ({ + id: capability.id, + classification: capability.classification, + status: capability.status, + request: { + method: capability.request.method, + url: requestUrl(capability.request), + }, + dataProductIds: [...productsByCapability.get(capability.id)].sort(), + })), + publisher: { + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + credentialType: "ndcDataProductWriterApi", + }, + }; +} + +function requestUrl(request) { + const url = new URL(request.path, request.baseUrl); + for (const [name, value] of Object.entries(request.query || {})) { + url.searchParams.append(name, String(value)); + } + return url.toString(); +} + +function canonicalDigest(value) { + return `sha256:${createHash("sha256").update(JSON.stringify(stableValue(value)), "utf8").digest("hex")}`; +} + +function stableValue(value) { + if (Array.isArray(value)) return value.map(stableValue); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.keys(value).sort().map((key) => [key, stableValue(value[key])]), + ); +} diff --git a/packages/external-provider-contract/providers/gelios/v10/index.mjs b/packages/external-provider-contract/providers/gelios/v10/index.mjs new file mode 100644 index 0000000..a40be3c --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v10/index.mjs @@ -0,0 +1,9 @@ +export { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, + GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + GELIOS_UNIT_CONTACTS_DATA_PRODUCT_VERSION, + GELIOS_UNIT_CONTACTS_ONTOLOGY_REVISION, + geliosProviderPackageV10, +} from "./package.mjs"; +export { geliosTelemetryFieldRegistryV3 } from "./telemetry-field-registry.mjs"; diff --git a/packages/external-provider-contract/providers/gelios/v10/package.mjs b/packages/external-provider-contract/providers/gelios/v10/package.mjs new file mode 100644 index 0000000..127a894 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v10/package.mjs @@ -0,0 +1,188 @@ +import { geliosProviderPackageV9 } from "../v9/package.mjs"; + +export const GELIOS_PROVIDER_PACKAGE_ID = "gelios.provider.v10"; +export const GELIOS_PROVIDER_PACKAGE_VERSION = "10.0.0"; +export const GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID = "fleet.units.contacts.current.v1"; +export const GELIOS_UNIT_CONTACTS_DATA_PRODUCT_VERSION = "1.0.0"; +export const GELIOS_UNIT_CONTACTS_ONTOLOGY_REVISION = "ontology.map.moving_object.v3"; + +const FIELD_POLICY_ID = "gelios.units.contacts.fields.v1"; +const COLLECTION_PROFILE_ID = "gelios.units.contacts.cold.v1"; +const MAPPING_ID = "gelios.units.to.fleet.units.contacts.current.v1"; +const TEMPLATE_ID = "gelios.units.contacts.l2.v1"; + +const contactFields = Object.freeze([ + "device_imei", + "device_phone_primary", + "device_phone_secondary", + "display_name", + "provider_creator_login", +]); + +const value = structuredClone(geliosProviderPackageV9); +value.id = GELIOS_PROVIDER_PACKAGE_ID; +value.version = GELIOS_PROVIDER_PACKAGE_VERSION; +value.manifest = { + ...value.manifest, + id: "gelios.provider.manifest.v10", + version: GELIOS_PROVIDER_PACKAGE_VERSION, + fieldPolicyIds: [...value.manifest.fieldPolicyIds, FIELD_POLICY_ID], + collectionProfileIds: [...value.manifest.collectionProfileIds, COLLECTION_PROFILE_ID], + dataProductIds: [...value.manifest.dataProductIds, GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID], + mappingContractIds: [...value.manifest.mappingContractIds, MAPPING_ID], + l2TemplateIds: [...value.manifest.l2TemplateIds, TEMPLATE_ID], +}; + +value.fieldPolicies.push({ + id: FIELD_POLICY_ID, + version: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_VERSION, + dataProductId: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + targetFields: [...contactFields], + unknownSourceFields: "drop", + dynamicSourceFields: "drop_until_classified", + restrictedSourcePaths: [ + "availableToUsers", + "commands", + "currentUserAccess", + "customFields", + "driver", + "extraInfo", + "hwDecryptKey", + "lastMsg.address", + "lastMsg.params", + "lastSensorsVal", + "preSetCommandGroup", + "sensors", + "stationaryLat", + "stationaryLon", + ], +}); + +value.collectionProfiles.push({ + id: COLLECTION_PROFILE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + mode: "realtime", + schedule: { intervalMs: 6 * 60 * 60 * 1000 }, + capabilityIds: ["gelios.units.current.read"], + dataProductId: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + mappingContractId: MAPPING_ID, + fieldPolicyId: FIELD_POLICY_ID, + l2TemplateId: TEMPLATE_ID, + entityScope: { + mode: "all_visible_to_credential", + refresh: "each_collection_run", + businessEntityFilter: "forbidden", + }, + batching: { maxFacts: 5000 }, + cardinality: { + maxCurrentEntities: 5000, + onExceed: "require_partitioned_data_product", + }, + retry: { maxAttempts: 4, backoff: "fixed_delay", delayMs: 2000 }, +}); + +value.dataProducts.push({ + id: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + version: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_UNIT_CONTACTS_ONTOLOGY_REVISION, + deliveryMode: "snapshot+patch", + semanticTypes: ["map.moving_object"], + fields: [...contactFields], + fieldContracts: { + device_imei: optionalString(), + device_phone_primary: optionalString(), + device_phone_secondary: optionalString(), + display_name: { type: "string", required: true }, + provider_creator_login: optionalString(), + }, + history: { mode: "none", retentionDays: 1 }, +}); + +value.mappingContracts.push({ + schemaVersion: "nodedc.semantic-mapping/v1", + id: MAPPING_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + sourceCapabilityId: "gelios.units.current.read", + fieldPolicyId: FIELD_POLICY_ID, + target: { + dataProductId: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + version: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_UNIT_CONTACTS_ONTOLOGY_REVISION, + semanticType: "map.moving_object", + }, + derivations: {}, + fact: { + sourceId: { + strategy: "first_non_empty", + paths: ["id", "unit_id", "unitId"], + coerce: "string", + prefix: "gelios-unit-", + }, + semanticType: { constant: "map.moving_object" }, + observedAt: { + strategy: "first_non_empty", + paths: ["updatedAt"], + coerce: "unix_or_iso_timestamp", + fallback: "collection_received_at", + }, + attributes: { + device_imei: guardedStringField(["imei"], "imeiIsVisible"), + device_phone_primary: guardedStringField(["phone"], "phoneIsVisible"), + device_phone_secondary: guardedStringField(["phone2"], "phone2IsVisible"), + display_name: stringField(["name", "unit_name", "title", "label"], false), + provider_creator_login: stringField(["creator.login"]), + }, + }, +}); + +const profileTemplate = value.l2Templates.find( + (template) => template.id === "gelios.units.profile.l2.v1", +); +if (!profileTemplate) throw new Error("gelios_v10_profile_template_missing"); +value.l2Templates.push({ + ...structuredClone(profileTemplate), + id: TEMPLATE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + steps: [ + { id: "collection.trigger", kind: "collection_trigger", collectionProfileDriven: true }, + { id: "provider.fetch-units", kind: "provider_request", capabilityId: "gelios.units.current.read" }, + { id: "provider.extract-units", kind: "extract_items", capabilityId: "gelios.units.current.read" }, + { id: "ontology.map", kind: "semantic_mapping", mappingContractId: MAPPING_ID }, + { + id: "data-product.publish", + kind: "data_product_publish", + dataProductId: GELIOS_UNIT_CONTACTS_DATA_PRODUCT_ID, + nodeType: "n8n-nodes-ndc.ndcDataProductPublish", + }, + ], +}); + +export const geliosProviderPackageV10 = deepFreeze(value); + +function optionalString() { + return { type: "string", required: false }; +} + +function stringField(paths, omitIfMissing = true) { + return { + strategy: "first_non_empty", + paths, + coerce: "string", + ...(omitIfMissing ? { omitIfMissing: true } : {}), + }; +} + +function guardedStringField(paths, guardPath) { + return { + ...stringField(paths), + guardPath, + guardEquals: true, + }; +} + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/providers/gelios/v10/telemetry-field-registry.mjs b/packages/external-provider-contract/providers/gelios/v10/telemetry-field-registry.mjs new file mode 100644 index 0000000..ca44c43 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v10/telemetry-field-registry.mjs @@ -0,0 +1,20 @@ +import { geliosTelemetryFieldRegistryV2 } from "../v9/telemetry-field-registry.mjs"; +import { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, +} from "./package.mjs"; + +const value = structuredClone(geliosTelemetryFieldRegistryV2); +value.providerPackage = { + id: GELIOS_PROVIDER_PACKAGE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, +}; + +export const geliosTelemetryFieldRegistryV3 = deepFreeze(value); + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/src/l2-execution-plan.mjs b/packages/external-provider-contract/src/l2-execution-plan.mjs index bb0dc4a..19233e0 100644 --- a/packages/external-provider-contract/src/l2-execution-plan.mjs +++ b/packages/external-provider-contract/src/l2-execution-plan.mjs @@ -9,9 +9,10 @@ export const L2_EXECUTION_PLAN_SCHEMA_VERSION = "nodedc.l2-execution-plan/v1"; export const L2_GRAPH_BLUEPRINT_SCHEMA_VERSION = "nodedc.l2-graph-blueprint/v1"; export const L2_MATERIALIZATION_RECEIPT_SCHEMA_VERSION = "nodedc.l2-materialization-receipt/v1"; export const L2_MAPPING_RUNTIME_SCHEMA_VERSION = "nodedc.semantic-mapping-runtime/v1"; -export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.2.0"; +export const L2_EXECUTION_PLAN_COMPILER_VERSION = "1.3.0"; export const L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS = Object.freeze([ "1.1.0", + "1.2.0", L2_EXECUTION_PLAN_COMPILER_VERSION, ]); @@ -86,7 +87,10 @@ export function compileL2ExecutionPlan(providerPackage, connectionInstance, opti mappingContractDigest: canonicalDigest(mapping), fieldPolicyDigest: canonicalDigest(fieldPolicy), dataProductDigest: canonicalDigest(dataProduct), - ...(telemetryProjection ? { telemetryRegistryDigest: telemetryProjection.registry.digest } : {}), + ...(telemetryProjection ? { + telemetryRegistryDigest: telemetryProjection.registry.digest, + telemetryProjectionDigest: canonicalDigest(telemetryProjection), + } : {}), }; const steps = template.steps.map((step) => compileStep({ diff --git a/packages/external-provider-contract/src/provider-package.mjs b/packages/external-provider-contract/src/provider-package.mjs index cd82c94..590c915 100644 --- a/packages/external-provider-contract/src/provider-package.mjs +++ b/packages/external-provider-contract/src/provider-package.mjs @@ -140,7 +140,7 @@ const MAPPING_TARGET_KEYS = new Set(["dataProductId", "version", "ontologyRevisi const FACT_KEYS = new Set(["sourceId", "semanticType", "observedAt", "geometry", "attributes"]); const EXPRESSION_KEYS = new Set([ "strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive", - "omitIfMissing", "omitIfInvalid", "minimum", "maximum", + "omitIfMissing", "omitIfInvalid", "minimum", "maximum", "guardPath", "guardEquals", ]); const GEOMETRY_KEYS = new Set([ "type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid", @@ -1013,6 +1013,18 @@ function validateExpression(value, path, errors) { if (value.derive !== undefined) requiredIdentifier(value.derive, `${path}.derive`, errors); if (value.omitIfMissing !== undefined && typeof value.omitIfMissing !== "boolean") errors.push(`${path}.omitIfMissing_must_be_boolean`); if (value.omitIfInvalid !== undefined && typeof value.omitIfInvalid !== "boolean") errors.push(`${path}.omitIfInvalid_must_be_boolean`); + const hasGuardPath = value.guardPath !== undefined; + const hasGuardEquals = value.guardEquals !== undefined; + if (hasGuardPath !== hasGuardEquals) errors.push(`${path}.guard_requires_path_and_equals`); + if (hasGuardPath) { + if (!isCanonicalSourcePath(value.guardPath)) { + errors.push(`${path}.guardPath_must_be_canonical_dot_path`); + } + if (!["string", "number", "boolean"].includes(typeof value.guardEquals)) { + errors.push(`${path}.guardEquals_must_be_scalar`); + } + if (value.omitIfMissing !== true) errors.push(`${path}.guard_requires_omitIfMissing`); + } if (value.minimum !== undefined || value.maximum !== undefined) { if (value.coerce !== "number") errors.push(`${path}.range_requires_number_coercion`); if (value.minimum !== undefined && !Number.isFinite(value.minimum)) errors.push(`${path}.minimum_invalid`); diff --git a/packages/external-provider-contract/test/l2-execution-plan.test.mjs b/packages/external-provider-contract/test/l2-execution-plan.test.mjs index 2da9354..bbfb639 100644 --- a/packages/external-provider-contract/test/l2-execution-plan.test.mjs +++ b/packages/external-provider-contract/test/l2-execution-plan.test.mjs @@ -14,26 +14,26 @@ import { validateL2ExecutionPlan, } from "../src/index.mjs"; import { - geliosProviderPackageV9, - geliosTelemetryFieldRegistryV2, -} from "../providers/gelios/v9/index.mjs"; + geliosProviderPackageV10, + geliosTelemetryFieldRegistryV3, +} from "../providers/gelios/v10/index.mjs"; -const positionsConnection = instantiateL2Connection(geliosProviderPackageV9, { +const positionsConnection = instantiateL2Connection(geliosProviderPackageV10, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-compiler-fixture", collectionProfileId: "gelios.positions.current.realtime.v7", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", }); const positionsPlan = compileL2ExecutionPlan( - geliosProviderPackageV9, + geliosProviderPackageV10, positionsConnection, - { telemetryFieldRegistry: geliosTelemetryFieldRegistryV2 }, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 }, ); assert.equal(positionsPlan.schemaVersion, L2_EXECUTION_PLAN_SCHEMA_VERSION); -assert.equal(L2_EXECUTION_PLAN_COMPILER_VERSION, "1.2.0"); -assert.deepEqual(L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS, ["1.1.0", "1.2.0"]); -assert.equal(positionsPlan.compilerVersion, "1.2.0"); +assert.equal(L2_EXECUTION_PLAN_COMPILER_VERSION, "1.3.0"); +assert.deepEqual(L2_EXECUTION_PLAN_SUPPORTED_COMPILER_VERSIONS, ["1.1.0", "1.2.0", "1.3.0"]); +assert.equal(positionsPlan.compilerVersion, "1.3.0"); assert.deepEqual(validateL2ExecutionPlan(positionsPlan), { ok: true, errors: [] }); assert.match(positionsPlan.executionPlanDigest, /^sha256:[a-f0-9]{64}$/); assert.match(positionsPlan.artifacts.packageDigest, /^sha256:[a-f0-9]{64}$/); @@ -123,9 +123,9 @@ assert.equal(Object.isFrozen(positionsPlan.steps), true); assert.equal(Object.isFrozen(positionsPlan.graphBlueprint), true); const clonedPlan = compileL2ExecutionPlan( - structuredClone(geliosProviderPackageV9), + structuredClone(geliosProviderPackageV10), structuredClone(positionsConnection), - { telemetryFieldRegistry: structuredClone(geliosTelemetryFieldRegistryV2) }, + { telemetryFieldRegistry: structuredClone(geliosTelemetryFieldRegistryV3) }, ); assert.equal(clonedPlan.executionPlanDigest, positionsPlan.executionPlanDigest); @@ -151,34 +151,34 @@ assert.equal( ); assert.throws( - () => compileL2ExecutionPlan(geliosProviderPackageV9, positionsConnection), + () => compileL2ExecutionPlan(geliosProviderPackageV10, positionsConnection), /telemetry_registry_required/, ); -const profileConnection = instantiateL2Connection(geliosProviderPackageV9, { +const profileConnection = instantiateL2Connection(geliosProviderPackageV10, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-profile-compiler-fixture", collectionProfileId: "gelios.units.profile.cold.v1", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0001", }); -const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV9, profileConnection); +const profilePlan = compileL2ExecutionPlan(geliosProviderPackageV10, profileConnection); assert.deepEqual(validateL2ExecutionPlan(profilePlan), { ok: true, errors: [] }); assert.equal(profilePlan.artifacts.telemetryRegistryDigest, undefined); assert.notEqual(profilePlan.executionPlanDigest, positionsPlan.executionPlanDigest); assert.deepEqual(profilePlan.graphBlueprint.runtime.mappingRuntime.derivationKinds, []); -for (const collectionProfile of geliosProviderPackageV9.collectionProfiles) { - const connection = instantiateL2Connection(geliosProviderPackageV9, { +for (const collectionProfile of geliosProviderPackageV10.collectionProfiles) { + const connection = instantiateL2Connection(geliosProviderPackageV10, { tenantId: "tenant-all-profiles-fixture", connectionId: `connection-${collectionProfile.id.replaceAll(".", "-")}`, collectionProfileId: collectionProfile.id, providerCredentialRef: "ndc-credref:provider-all-profiles-fixture", }); const plan = compileL2ExecutionPlan( - geliosProviderPackageV9, + geliosProviderPackageV10, connection, collectionProfile.dataProductId === "fleet.positions.current.v5" - ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV2 } + ? { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 } : {}, ); assert.deepEqual(validateL2ExecutionPlan(plan), { ok: true, errors: [] }); @@ -188,16 +188,16 @@ for (const collectionProfile of geliosProviderPackageV9.collectionProfiles) { assert.equal(plan.graphBlueprint.edges.length, plan.steps.length + expectedEntrypoints - 2); } -const secondConnection = instantiateL2Connection(geliosProviderPackageV9, { +const secondConnection = instantiateL2Connection(geliosProviderPackageV10, { tenantId: "tenant-compiler-fixture", connectionId: "gelios-compiler-fixture-two", collectionProfileId: "gelios.positions.current.realtime.v7", providerCredentialRef: "ndc-credref:provider-compiler-fixture-0002", }); const secondPlan = compileL2ExecutionPlan( - geliosProviderPackageV9, + geliosProviderPackageV10, secondConnection, - { telemetryFieldRegistry: geliosTelemetryFieldRegistryV2 }, + { telemetryFieldRegistry: geliosTelemetryFieldRegistryV3 }, ); assert.notEqual(secondPlan.executionPlanDigest, positionsPlan.executionPlanDigest); diff --git a/packages/external-provider-contract/test/provider-package.test.mjs b/packages/external-provider-contract/test/provider-package.test.mjs index f1284a3..87cd67a 100644 --- a/packages/external-provider-contract/test/provider-package.test.mjs +++ b/packages/external-provider-contract/test/provider-package.test.mjs @@ -22,6 +22,7 @@ import { geliosProviderPackageV6 } from "../providers/gelios/v6/index.mjs"; import { geliosProviderPackageV7 } from "../providers/gelios/v7/index.mjs"; import { geliosProviderPackageV8 } from "../providers/gelios/v8/index.mjs"; import { geliosProviderPackageV9 } from "../providers/gelios/v9/index.mjs"; +import { geliosProviderPackageV10 } from "../providers/gelios/v10/index.mjs"; import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs"; const expectedFields = [ @@ -49,8 +50,11 @@ assert.deepEqual(validateProviderPackage(geliosProviderPackageV6), { ok: true, e assert.deepEqual(validateProviderPackage(geliosProviderPackageV7), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV8), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV9), { ok: true, errors: [] }); +assert.deepEqual(validateProviderPackage(geliosProviderPackageV10), { ok: true, errors: [] }); assert.equal(geliosProviderPackageV9.id, "gelios.provider.v9"); assert.equal(geliosProviderPackageV9.version, "9.0.0"); +assert.equal(geliosProviderPackageV10.id, "gelios.provider.v10"); +assert.equal(geliosProviderPackageV10.version, "10.0.0"); assert.equal(geliosProviderPackageV8.id, "gelios.provider.v8"); const profileProduct = geliosProviderPackageV8.dataProducts.find((product) => product.id === "fleet.units.profile.current.v1"); @@ -70,6 +74,38 @@ assert.equal(profileMapping.fact.attributes.hardware_type_name.paths[0], "hwType assert.equal(profileMapping.fact.attributes.trip_detection_type.paths[0], "tripParams.tripDetectionType.name"); assert.equal(profileMapping.fact.attributes.use_odometer.coerce, "boolean"); +const contactsProduct = geliosProviderPackageV10.dataProducts.find( + (product) => product.id === "fleet.units.contacts.current.v1", +); +const registeredContactsProduct = JSON.parse(await readFile(new URL( + "../../../services/external-data-plane/definitions/fleet.units.contacts.current.v1.json", + import.meta.url, +), "utf8")); +assert.deepEqual(contactsProduct, registeredContactsProduct); +const contactsMapping = geliosProviderPackageV10.mappingContracts.find( + (mapping) => mapping.id === "gelios.units.to.fleet.units.contacts.current.v1", +); +assert.deepEqual(contactsMapping.fact.attributes.device_imei, { + strategy: "first_non_empty", + paths: ["imei"], + coerce: "string", + omitIfMissing: true, + guardPath: "imeiIsVisible", + guardEquals: true, +}); +assert.equal(contactsMapping.fact.attributes.device_phone_primary.guardPath, "phoneIsVisible"); +assert.equal(contactsMapping.fact.attributes.device_phone_secondary.guardPath, "phone2IsVisible"); +assert.equal(contactsMapping.fact.attributes.provider_creator_login.paths[0], "creator.login"); + +const incompleteGuard = structuredClone(geliosProviderPackageV10); +delete incompleteGuard.mappingContracts.at(-1).fact.attributes.device_imei.guardEquals; +assert.equal( + validateProviderPackage(incompleteGuard).errors.some( + (error) => error.endsWith("guard_requires_path_and_equals"), + ), + true, +); + const telemetryProduct = geliosProviderPackageV7.dataProducts.find((product) => product.id === "fleet.positions.current.v5"); const registeredTelemetryProduct = JSON.parse(await readFile(new URL( "../../../services/external-data-plane/definitions/fleet.positions.current.v5.json", diff --git a/services/external-data-plane/definitions/fleet.units.contacts.current.v1.json b/services/external-data-plane/definitions/fleet.units.contacts.current.v1.json new file mode 100644 index 0000000..7140997 --- /dev/null +++ b/services/external-data-plane/definitions/fleet.units.contacts.current.v1.json @@ -0,0 +1,27 @@ +{ + "id": "fleet.units.contacts.current.v1", + "version": "1.0.0", + "ontologyRevision": "ontology.map.moving_object.v3", + "deliveryMode": "snapshot+patch", + "semanticTypes": [ + "map.moving_object" + ], + "fields": [ + "device_imei", + "device_phone_primary", + "device_phone_secondary", + "display_name", + "provider_creator_login" + ], + "fieldContracts": { + "device_imei": { "type": "string", "required": false }, + "device_phone_primary": { "type": "string", "required": false }, + "device_phone_secondary": { "type": "string", "required": false }, + "display_name": { "type": "string", "required": true }, + "provider_creator_login": { "type": "string", "required": false } + }, + "history": { + "mode": "none", + "retentionDays": 1 + } +} diff --git a/services/external-data-plane/test/definitions.test.mjs b/services/external-data-plane/test/definitions.test.mjs index b56edd5..99d8c43 100644 --- a/services/external-data-plane/test/definitions.test.mjs +++ b/services/external-data-plane/test/definitions.test.mjs @@ -12,6 +12,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [ "fleet.positions.current.v3", "fleet.positions.current.v4", "fleet.positions.current.v5", + "fleet.units.contacts.current.v1", "fleet.units.profile.current.v1", "map.zones.current.v1", "map.zones.current.v2", @@ -56,22 +57,28 @@ assert.equal(bundled[5].version, "1.0.0"); assert.equal(bundled[5].ontologyRevision, "ontology.map.moving_object.v3"); assert.deepEqual(bundled[5].semanticTypes, ["map.moving_object"]); assert.equal(bundled[5].history.mode, "none"); -assert.equal(bundled[5].fieldContracts.hardware_port.type, "number"); +assert.equal(bundled[5].fieldContracts.device_phone_primary.type, "string"); assert.equal(bundled[6].version, "1.0.0"); -assert.equal(bundled[6].ontologyRevision, "ontology.map.zone.v1"); -assert.deepEqual(bundled[6].semanticTypes, ["map.zone"]); -assert.deepEqual(bundled[6].fieldContracts.geometry, { type: "geometry", required: true }); -assert.equal(bundled[6].fields.includes("dataset_authority"), false); -assert.equal(bundled[7].version, "2.0.0"); +assert.equal(bundled[6].ontologyRevision, "ontology.map.moving_object.v3"); +assert.deepEqual(bundled[6].semanticTypes, ["map.moving_object"]); +assert.equal(bundled[6].history.mode, "none"); +assert.equal(bundled[6].fieldContracts.hardware_port.type, "number"); +assert.equal(bundled[7].version, "1.0.0"); assert.equal(bundled[7].ontologyRevision, "ontology.map.zone.v1"); assert.deepEqual(bundled[7].semanticTypes, ["map.zone"]); assert.deepEqual(bundled[7].fieldContracts.geometry, { type: "geometry", required: true }); -assert.deepEqual(bundled[7].fieldContracts.dataset_authority.enum, ["moscow-department-of-transport"]); +assert.equal(bundled[7].fields.includes("dataset_authority"), false); +assert.equal(bundled[8].version, "2.0.0"); +assert.equal(bundled[8].ontologyRevision, "ontology.map.zone.v1"); +assert.deepEqual(bundled[8].semanticTypes, ["map.zone"]); +assert.deepEqual(bundled[8].fieldContracts.geometry, { type: "geometry", required: true }); +assert.deepEqual(bundled[8].fieldContracts.dataset_authority.enum, ["moscow-department-of-transport"]); assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length); assert.equal(Object.keys(bundled[4].fieldContracts).length, bundled[4].fields.length); assert.equal(Object.keys(bundled[5].fieldContracts).length, bundled[5].fields.length); assert.equal(Object.keys(bundled[6].fieldContracts).length, bundled[6].fields.length); assert.equal(Object.keys(bundled[7].fieldContracts).length, bundled[7].fields.length); +assert.equal(Object.keys(bundled[8].fieldContracts).length, bundled[8].fields.length); const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-")); try {