diff --git a/infra/deploy-runner/README.md b/infra/deploy-runner/README.md index 1a6060f..144f802 100644 --- a/infra/deploy-runner/README.md +++ b/infra/deploy-runner/README.md @@ -90,11 +90,11 @@ node descriptions used `usableAsTool`, so n8n 2.3.2 exposed six NDC runtime types instead of the required three. It must not be activated, overwritten or deleted. -The active predecessor is package version `0.1.2` at immutable release -`0.1.2-05e4b38b14b4a019`. History Read is package version `0.1.3`, built with -patch id `n8n-nodes-ndc-history-read-20260717-004` and immutable release -`0.1.3-3354149b5245e39a`. Staging remains inert; only the separately reviewed -Engine-owned transition may select it after exact MCP schema acceptance. +The active predecessor is package version `0.1.4` at immutable release +`0.1.4-59dc9f7882721d6a`. Package `0.1.5` adds the provider-neutral complete +snapshot replace mode used by `map.zones.current.v1`; its Platform staging +remains inert until a separately reviewed Engine-owned transition selects the +exact digest and accepts the updated MCP node schema. The paired Engine activation is built by `build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not diff --git a/infra/deploy-runner/build-external-data-plane-artifact.mjs b/infra/deploy-runner/build-external-data-plane-artifact.mjs index 701c225..714d00c 100644 --- a/infra/deploy-runner/build-external-data-plane-artifact.mjs +++ b/infra/deploy-runner/build-external-data-plane-artifact.mjs @@ -22,10 +22,12 @@ const files = [ ["packages/external-provider-contract/src/contract-version.mjs", "platform/packages/external-provider-contract/src/contract-version.mjs"], ["packages/external-provider-contract/src/data-plane.mjs", "platform/packages/external-provider-contract/src/data-plane.mjs"], ["packages/external-provider-contract/src/data-product.mjs", "platform/packages/external-provider-contract/src/data-product.mjs"], + ["packages/external-provider-contract/src/geometry.mjs", "platform/packages/external-provider-contract/src/geometry.mjs"], ["packages/external-provider-contract/src/intake-batch.mjs", "platform/packages/external-provider-contract/src/intake-batch.mjs"], ["packages/external-provider-contract/src/index.mjs", "platform/packages/external-provider-contract/src/index.mjs"], ["packages/external-provider-contract/src/provider-package.mjs", "platform/packages/external-provider-contract/src/provider-package.mjs"], ["packages/external-provider-contract/src/sensitive-field-policy.mjs", "platform/packages/external-provider-contract/src/sensitive-field-policy.mjs"], + ["packages/external-provider-contract/src/zone-source.mjs", "platform/packages/external-provider-contract/src/zone-source.mjs"], ["packages/external-provider-contract/providers/gelios", "platform/packages/external-provider-contract/providers/gelios"], ]; const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); diff --git a/infra/deploy-runner/build-n8n-private-extension-artifact.mjs b/infra/deploy-runner/build-n8n-private-extension-artifact.mjs index 6e0eb10..af200da 100644 --- a/infra/deploy-runner/build-n8n-private-extension-artifact.mjs +++ b/infra/deploy-runner/build-n8n-private-extension-artifact.mjs @@ -12,8 +12,8 @@ const platformRoot = resolve(scriptDir, "../.."); const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc"); const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts")); const requireModule = createRequire(import.meta.url); -const expectedPackageVersion = "0.1.4"; -const [patchId = "n8n-nodes-ndc-provider-rotating-access-20260718-006", ...extra] = process.argv.slice(2); +const expectedPackageVersion = "0.1.5"; +const [patchId = "n8n-nodes-ndc-geozone-replace-20260720-017", ...extra] = process.argv.slice(2); const expectedRuntimeNodes = [ { file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", diff --git a/infra/deploy-runner/test_n8n_private_extension.py b/infra/deploy-runner/test_n8n_private_extension.py index dbaef80..33c7b0d 100644 --- a/infra/deploy-runner/test_n8n_private_extension.py +++ b/infra/deploy-runner/test_n8n_private_extension.py @@ -19,7 +19,7 @@ RUNNER_PATH = Path( os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy") ).resolve() BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs" -PATCH_ID = "n8n-nodes-ndc-provider-rotating-access-20260718-006" +PATCH_ID = "n8n-nodes-ndc-geozone-replace-20260720-017" EXPECTED_NODES = [ "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", "dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", @@ -110,7 +110,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase): manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory)) self.assertEqual(manifest["component"], "n8n-private-extension") self.assertEqual(len(entries), 1) - self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.4-[a-f0-9]{16}$") + self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.5-[a-f0-9]{16}$") with tarfile.open(self.artifacts[0], "r:gz") as archive: names = archive.getnames() @@ -122,7 +122,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase): with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive: package_json_member = archive.getmember("package/package.json") package_json = json.loads(archive.extractfile(package_json_member).read()) - self.assertEqual(package_json["version"], "0.1.4") + self.assertEqual(package_json["version"], "0.1.5") self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES) self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS) for node_path in EXPECTED_NODES: @@ -158,7 +158,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase): "requiresPreActivationVerification": True, } self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2") - self.assertEqual(release["package"]["version"], "0.1.4") + self.assertEqual(release["package"]["version"], "0.1.5") self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy) self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2") self.assertEqual(rollback["baselinePolicy"], expected_policy) diff --git a/packages/external-provider-contract/package.json b/packages/external-provider-contract/package.json index 7a665dc..21b6c97 100644 --- a/packages/external-provider-contract/package.json +++ b/packages/external-provider-contract/package.json @@ -9,6 +9,6 @@ "./providers/*": "./providers/*/index.mjs" }, "scripts": { - "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/engine-private-extension.test.mjs" + "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/zone-source.test.mjs && node test/engine-private-extension.test.mjs" } } diff --git a/packages/external-provider-contract/providers/gelios/v6/README.md b/packages/external-provider-contract/providers/gelios/v6/README.md new file mode 100644 index 0000000..0a1200f --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v6/README.md @@ -0,0 +1,13 @@ +# Gelios provider package v6 + +Adds the bounded, offset-paged `gelios.geozone` read capability and maps a +complete accepted source generation into the provider-neutral +`map.zones.current.v1` Data Product. The runtime publisher must use replace +mode: a partial provider page set never reaches Publish, while a complete newer +generation atomically upserts present zones and emits remove patches for zones +that disappeared. + +`gelios-rest-v1` and `mmap-snapshot-v1` are source adapters, not Foundry +bindings. Both must preserve the native Gelios zone id (or use an explicit +crosswalk) and produce the same `map.zone` facts. Foundry only consumes the +provider-neutral product. diff --git a/packages/external-provider-contract/providers/gelios/v6/index.mjs b/packages/external-provider-contract/providers/gelios/v6/index.mjs new file mode 100644 index 0000000..7781a60 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v6/index.mjs @@ -0,0 +1,9 @@ +export { + GELIOS_PROVIDER_PACKAGE_ID, + GELIOS_PROVIDER_PACKAGE_VERSION, + GELIOS_ZONES_DATA_PRODUCT_ID, + GELIOS_ZONES_DATA_PRODUCT_VERSION, + GELIOS_ZONES_ONTOLOGY_REVISION, + GELIOS_ZONE_SOURCE_ID_PREFIX, + geliosProviderPackageV6, +} from "./package.mjs"; diff --git a/packages/external-provider-contract/providers/gelios/v6/package.mjs b/packages/external-provider-contract/providers/gelios/v6/package.mjs new file mode 100644 index 0000000..41b7329 --- /dev/null +++ b/packages/external-provider-contract/providers/gelios/v6/package.mjs @@ -0,0 +1,192 @@ +import { geliosProviderPackageV5 } from "../v5/package.mjs"; + +export const GELIOS_PROVIDER_PACKAGE_ID = "gelios.provider.v6"; +export const GELIOS_PROVIDER_PACKAGE_VERSION = "6.0.0"; +export const GELIOS_ZONES_DATA_PRODUCT_ID = "map.zones.current.v1"; +export const GELIOS_ZONES_DATA_PRODUCT_VERSION = "1.0.0"; +export const GELIOS_ZONES_ONTOLOGY_REVISION = "ontology.map.zone.v1"; +export const GELIOS_ZONE_SOURCE_ID_PREFIX = "gelios-zone-"; + +const ZONES_CAPABILITY_ID = "gelios.geozones.current.read"; +const ZONES_FIELD_POLICY_ID = "gelios.geozones.current.fields.v1"; +const ZONES_PROFILE_ID = "gelios.geozones.current.realtime.v1"; +const ZONES_MAPPING_ID = "gelios.geozones.to.map.zones.current.v1"; +const ZONES_TEMPLATE_ID = "gelios.geozones.current.l2.v1"; + +const value = structuredClone(geliosProviderPackageV5); +const authModeId = value.authModes[0].id; +const zonesFields = [ + "area_square_meters", + "description", + "display_name", + "geometry", + "geometry_kind", + "max_speed_kph", + "perimeter_meters", + "source_kind", + "source_revision", + "style_color", +]; + +value.id = GELIOS_PROVIDER_PACKAGE_ID; +value.version = GELIOS_PROVIDER_PACKAGE_VERSION; +value.manifest = { + ...value.manifest, + id: "gelios.provider.manifest.v6", + version: GELIOS_PROVIDER_PACKAGE_VERSION, + capabilityIds: [...value.manifest.capabilityIds, ZONES_CAPABILITY_ID], + fieldPolicyIds: [...value.manifest.fieldPolicyIds, ZONES_FIELD_POLICY_ID], + collectionProfileIds: [...value.manifest.collectionProfileIds, ZONES_PROFILE_ID], + dataProductIds: [...value.manifest.dataProductIds, GELIOS_ZONES_DATA_PRODUCT_ID], + mappingContractIds: [...value.manifest.mappingContractIds, ZONES_MAPPING_ID], + l2TemplateIds: [...value.manifest.l2TemplateIds, ZONES_TEMPLATE_ID], +}; +value.capabilities.push({ + id: ZONES_CAPABILITY_ID, + classification: "read", + status: "implemented", + authModeId, + request: { + method: "GET", + baseUrl: "https://api.geliospro.com", + path: "/api/v1/geozones", + query: { pl: 100, po: 0 }, + response: { + collectionPaths: ["items"], + pagination: { + mode: "offset", + limitParameter: "pl", + offsetParameter: "po", + pageSize: 100, + itemsPath: "items", + totalPath: "paginationMetadata.totalCount", + maxPages: 50, + maxItems: 5000, + maxResponseBytes: 16 * 1024 * 1024, + }, + }, + }, + entityScope: { + mode: "all_visible_to_credential", + refresh: "each_collection_run", + businessEntityFilter: "forbidden", + }, +}); +value.fieldPolicies.push({ + id: ZONES_FIELD_POLICY_ID, + version: GELIOS_ZONES_DATA_PRODUCT_VERSION, + dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, + targetFields: [...zonesFields], + unknownSourceFields: "drop", + dynamicSourceFields: "drop_until_classified", + restrictedSourcePaths: ["accessToken", "authorization", "refreshToken", "userToken"], +}); +value.collectionProfiles.push({ + id: ZONES_PROFILE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + mode: "realtime", + schedule: { intervalMs: 3_600_000 }, + capabilityIds: [ZONES_CAPABILITY_ID], + dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, + mappingContractId: ZONES_MAPPING_ID, + fieldPolicyId: ZONES_FIELD_POLICY_ID, + l2TemplateId: ZONES_TEMPLATE_ID, + entityScope: { + mode: "all_visible_to_credential", + refresh: "each_collection_run", + businessEntityFilter: "forbidden", + }, + batching: { maxFacts: 5000 }, + cardinality: { maxCurrentEntities: 5000, onExceed: "require_partitioned_data_product" }, + retry: { maxAttempts: 4, backoff: "exponential_with_jitter" }, +}); +value.dataProducts.push({ + id: GELIOS_ZONES_DATA_PRODUCT_ID, + version: GELIOS_ZONES_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION, + deliveryMode: "snapshot+patch", + semanticTypes: ["map.zone"], + fields: [...zonesFields], + fieldContracts: { + area_square_meters: { type: "number", required: false, minimum: 0 }, + description: { type: "string", required: false }, + display_name: { type: "string", required: true }, + geometry: { type: "geometry", required: true }, + geometry_kind: { type: "string", required: true, enum: ["circle", "corridor", "polygon"] }, + max_speed_kph: { type: "number", required: false, minimum: 0 }, + perimeter_meters: { type: "number", required: false, minimum: 0 }, + source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] }, + source_revision: { type: "string", required: true }, + style_color: { type: "string", required: false }, + }, + history: { mode: "none", retentionDays: 1 }, +}); +value.mappingContracts.push({ + schemaVersion: "nodedc.semantic-mapping/v1", + id: ZONES_MAPPING_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + sourceCapabilityId: ZONES_CAPABILITY_ID, + fieldPolicyId: ZONES_FIELD_POLICY_ID, + target: { + dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, + version: GELIOS_ZONES_DATA_PRODUCT_VERSION, + ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION, + semanticType: "map.zone", + }, + derivations: { + geometry_kind: { + kind: "ordered_rules", + rules: ["type_circle.circle", "type_line.corridor", "otherwise.polygon"], + default: "polygon", + }, + }, + fact: { + sourceId: { strategy: "first_non_empty", paths: ["id"], coerce: "string", prefix: GELIOS_ZONE_SOURCE_ID_PREFIX }, + semanticType: { constant: "map.zone" }, + observedAt: { strategy: "first_non_empty", paths: ["updatedAt"], coerce: "unix_or_iso_timestamp", fallback: "collection_received_at" }, + geometry: { + type: "GeoJSON", + strategy: "gelios_geozone_v1", + paths: ["type", "points", "line", "radius"], + allowedTypes: ["Polygon", "MultiPolygon"], + omitIfInvalid: false, + }, + attributes: { + area_square_meters: { strategy: "first_non_empty", paths: ["surfaceArea", "surface_area"], coerce: "number", minimum: 0, omitIfInvalid: true }, + description: { strategy: "first_non_empty", paths: ["description", "descr"], coerce: "string", omitIfMissing: true }, + display_name: { strategy: "first_non_empty", paths: ["name"], coerce: "string" }, + geometry_kind: { derive: "geometry_kind" }, + max_speed_kph: { strategy: "first_non_empty", paths: ["maxPermissibleSpeed", "max_permissible_speed"], coerce: "number", minimum: 0, omitIfInvalid: true }, + perimeter_meters: { strategy: "first_non_empty", paths: ["perimeter"], coerce: "number", minimum: 0, omitIfInvalid: true }, + source_kind: { constant: "live_api" }, + source_revision: { constant: "gelios-rest-v1" }, + style_color: { strategy: "first_non_empty", paths: ["color"], coerce: "string", omitIfMissing: true }, + }, + }, +}); +value.l2Templates.push({ + schemaVersion: "nodedc.l2-template/v1", + id: ZONES_TEMPLATE_ID, + version: GELIOS_PROVIDER_PACKAGE_VERSION, + runtime: "ndc_l2", + instanceMode: "one_connection_per_workflow", + connectionParameters: ["tenant_id", "connection_id", "collection_profile_id", "provider_credential_ref"], + credentialBindings: structuredClone(value.l2Templates[0].credentialBindings), + steps: [ + { id: "collection.trigger", kind: "collection_trigger", collectionProfileDriven: true }, + { id: "provider.fetch-geozones", kind: "provider_request", capabilityId: ZONES_CAPABILITY_ID }, + { id: "provider.extract-geozones", kind: "extract_items", capabilityId: ZONES_CAPABILITY_ID }, + { id: "ontology.map", kind: "semantic_mapping", mappingContractId: ZONES_MAPPING_ID }, + { id: "data-product.publish", kind: "data_product_publish", dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, nodeType: "n8n-nodes-ndc.ndcDataProductPublish" }, + ], + invariants: structuredClone(value.l2Templates[0].invariants), +}); + +export const geliosProviderPackageV6 = deepFreeze(value); + +function deepFreeze(input) { + if (!input || typeof input !== "object" || Object.isFrozen(input)) return input; + Object.freeze(input); + for (const child of Object.values(input)) deepFreeze(child); + return input; +} diff --git a/packages/external-provider-contract/src/data-plane.mjs b/packages/external-provider-contract/src/data-plane.mjs index b48ba78..15d2ee3 100644 --- a/packages/external-provider-contract/src/data-plane.mjs +++ b/packages/external-provider-contract/src/data-plane.mjs @@ -1,5 +1,12 @@ export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs"; export { validateIntakeBatch } from "./intake-batch.mjs"; +export { + DATA_PRODUCT_GEOMETRY_TYPES, + DATA_PRODUCT_MAX_GEOMETRY_BYTES, + DATA_PRODUCT_MAX_GEOMETRY_VERTICES, + isBoundedGeoJsonGeometry, + validateGeoJsonGeometry, +} from "./geometry.mjs"; export { DATA_PRODUCT_HISTORY_SCHEMA_VERSION, DATA_PRODUCT_PATCH_SCHEMA_VERSION, diff --git a/packages/external-provider-contract/src/data-product.mjs b/packages/external-provider-contract/src/data-product.mjs index 32b0fdc..eabbd71 100644 --- a/packages/external-provider-contract/src/data-product.mjs +++ b/packages/external-provider-contract/src/data-product.mjs @@ -4,6 +4,7 @@ export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1"; export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1"; import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs"; +import { validateGeoJsonGeometry } from "./geometry.mjs"; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; @@ -30,22 +31,33 @@ export function validateDataProductPublish(value, { maxFacts = 5000, maxAttribut if (!isPlainObject(value.batch)) { errors.push("batch_must_be_object"); } else { - rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey"]), "batch", errors); + rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey", "mode", "generationAt"]), "batch", errors); requiredIdentifier(value.batch.runId, "batch.runId", errors); requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors); if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) { errors.push("batch.sequence_must_be_integer_0_to_2147483647"); } + const mode = value.batch.mode === undefined ? "upsert" : value.batch.mode; + if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid"); + if (mode === "replace") requiredIsoTimestamp(value.batch.generationAt, "batch.generationAt", errors); + if (mode === "upsert" && value.batch.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert"); } - if (!Array.isArray(value.facts) || value.facts.length === 0) { - errors.push("facts_must_be_nonempty_array"); + if (!Array.isArray(value.facts)) { + errors.push("facts_must_be_array"); + } else if (value.facts.length === 0 && value.batch?.mode !== "replace") { + errors.push("facts_must_be_nonempty_array_for_upsert"); } else if (value.facts.length > maxFacts) { errors.push("facts_limit_exceeded"); } else { const entityKeys = new Set(); value.facts.forEach((fact, index) => { validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling }); + if (value.batch?.mode === "replace" + && !Number.isNaN(Date.parse(fact?.observedAt)) + && Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) { + errors.push(`facts[${index}].observedAt_must_equal_generationAt`); + } if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return; const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key"); @@ -143,12 +155,18 @@ export function validateDataProductPatch(value) { errors.push("operations_must_be_nonempty_array"); } else { value.operations.forEach((operation, index) => { - if (!isPlainObject(operation) || operation.op !== "upsert") { - errors.push(`operations[${index}].op_must_be_upsert`); + if (!isPlainObject(operation) || !new Set(["upsert", "remove"]).has(operation.op)) { + errors.push(`operations[${index}].op_invalid`); return; } - rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors); - validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors); + if (operation.op === "upsert") { + rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors); + validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors); + } else { + rejectUnknownKeys(operation, new Set(["op", "sourceId", "semanticType"]), `operations[${index}]`, errors); + requiredIdentifier(operation.sourceId, `operations[${index}].sourceId`, errors); + requiredIdentifier(operation.semanticType, `operations[${index}].semanticType`, errors); + } }); } if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material"); @@ -189,7 +207,7 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal errors.push(`${path}.attributes_size_exceeded`); } } - if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); + if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors); } function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) { @@ -198,21 +216,6 @@ function validateCanonicalFact(value, path, errors, { allowBucketStart = false } requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors); } -function validatePointGeometry(value, path, errors) { - if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { - errors.push(`${path}_must_be_geojson_point`); - return; - } - rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors); - if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) { - errors.push(`${path}_coordinates_must_be_finite_numbers`); - return; - } - const [longitude, latitude] = value.coordinates; - if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`); - if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`); -} - function rejectUnknownKeys(value, allowed, path, errors) { if (!isPlainObject(value)) return; for (const key of Object.keys(value)) { diff --git a/packages/external-provider-contract/src/geometry.mjs b/packages/external-provider-contract/src/geometry.mjs new file mode 100644 index 0000000..8094fe1 --- /dev/null +++ b/packages/external-provider-contract/src/geometry.mjs @@ -0,0 +1,103 @@ +const GEOMETRY_TYPES = new Set(["Point", "LineString", "Polygon", "MultiPolygon"]); +const MAX_GEOMETRY_BYTES = 192 * 1024; +const MAX_GEOMETRY_VERTICES = 10_000; + +/** + * Validate the bounded GeoJSON geometry subset accepted by provider-neutral + * Data Products. GeometryCollections and arbitrary feature properties are + * deliberately excluded: a fact owns exactly one geometry and its attributes + * stay under the fact field policy. + */ +export function validateGeoJsonGeometry(value, path, errors, { allowedTypes = GEOMETRY_TYPES } = {}) { + if (!isPlainObject(value) || !allowedTypes.has(value.type) || !Array.isArray(value.coordinates)) { + errors.push(`${path}_must_be_bounded_geojson_geometry`); + return; + } + rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors); + if (serializedByteLength(value) > MAX_GEOMETRY_BYTES) { + errors.push(`${path}_size_exceeded`); + return; + } + + const vertices = []; + if (value.type === "Point") { + validatePosition(value.coordinates, path, errors, vertices); + } else if (value.type === "LineString") { + validateLineString(value.coordinates, path, errors, vertices); + } else if (value.type === "Polygon") { + validatePolygon(value.coordinates, path, errors, vertices); + } else if (value.type === "MultiPolygon") { + if (!value.coordinates.length) errors.push(`${path}.coordinates_must_be_nonempty`); + value.coordinates.forEach((polygon, index) => validatePolygon(polygon, `${path}.coordinates[${index}]`, errors, vertices)); + } + if (vertices.length > MAX_GEOMETRY_VERTICES) errors.push(`${path}_vertex_limit_exceeded`); +} + +export function isBoundedGeoJsonGeometry(value, allowedTypes = GEOMETRY_TYPES) { + const errors = []; + validateGeoJsonGeometry(value, "geometry", errors, { allowedTypes }); + return errors.length === 0; +} + +function validatePolygon(value, path, errors, vertices) { + if (!Array.isArray(value) || !value.length) { + errors.push(`${path}_polygon_rings_required`); + return; + } + value.forEach((ring, index) => { + const ringPath = `${path}[${index}]`; + if (!Array.isArray(ring) || ring.length < 4) { + errors.push(`${ringPath}_linear_ring_requires_four_positions`); + return; + } + ring.forEach((position, positionIndex) => validatePosition(position, `${ringPath}[${positionIndex}]`, errors, vertices)); + if (!samePosition(ring[0], ring.at(-1))) errors.push(`${ringPath}_linear_ring_must_be_closed`); + }); +} + +function validateLineString(value, path, errors, vertices) { + if (!Array.isArray(value) || value.length < 2) { + errors.push(`${path}_line_string_requires_two_positions`); + return; + } + value.forEach((position, index) => validatePosition(position, `${path}[${index}]`, errors, vertices)); +} + +function validatePosition(value, path, errors, vertices) { + if (!Array.isArray(value) || value.length !== 2 || !value.every(Number.isFinite)) { + errors.push(`${path}_position_invalid`); + return; + } + const [longitude, latitude] = value; + if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`); + if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`); + vertices.push(value); +} + +function samePosition(left, right) { + return Array.isArray(left) && Array.isArray(right) + && left.length === 2 && right.length === 2 + && Object.is(left[0], right[0]) && Object.is(left[1], right[1]); +} + +function rejectUnknownKeys(value, allowed, path, errors) { + for (const key of Object.keys(value)) { + if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`); + } +} + +function serializedByteLength(value) { + try { + return Buffer.byteLength(JSON.stringify(value)); + } catch { + return Number.POSITIVE_INFINITY; + } +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +export const DATA_PRODUCT_GEOMETRY_TYPES = Object.freeze([...GEOMETRY_TYPES]); +export const DATA_PRODUCT_MAX_GEOMETRY_BYTES = MAX_GEOMETRY_BYTES; +export const DATA_PRODUCT_MAX_GEOMETRY_VERTICES = MAX_GEOMETRY_VERTICES; diff --git a/packages/external-provider-contract/src/index.mjs b/packages/external-provider-contract/src/index.mjs index e0dd018..2656aaf 100644 --- a/packages/external-provider-contract/src/index.mjs +++ b/packages/external-provider-contract/src/index.mjs @@ -2,6 +2,18 @@ import { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs"; export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs"; export { validateIntakeBatch } from "./intake-batch.mjs"; +export { + DATA_PRODUCT_GEOMETRY_TYPES, + DATA_PRODUCT_MAX_GEOMETRY_BYTES, + DATA_PRODUCT_MAX_GEOMETRY_VERTICES, + isBoundedGeoJsonGeometry, + validateGeoJsonGeometry, +} from "./geometry.mjs"; +export { + ZONE_SOURCE_ADAPTERS, + ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + normalizeZoneSourceGeneration, +} from "./zone-source.mjs"; export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1"; import { diff --git a/packages/external-provider-contract/src/intake-batch.mjs b/packages/external-provider-contract/src/intake-batch.mjs index 9f12683..ac2ad52 100644 --- a/packages/external-provider-contract/src/intake-batch.mjs +++ b/packages/external-provider-contract/src/intake-batch.mjs @@ -4,6 +4,7 @@ import { SECRET_LIKE_REFERENCE, SECRET_LIKE_VALUE, } from "./sensitive-field-policy.mjs"; +import { validateGeoJsonGeometry } from "./geometry.mjs"; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; @@ -16,7 +17,7 @@ export function validateIntakeBatch(value) { rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors); rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors); rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors); - rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt"]), "batch", errors); + rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt", "mode", "generationAt"]), "batch", errors); requiredIdentifier(value?.source?.providerId, "source.providerId", errors); requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors); requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors); @@ -32,11 +33,24 @@ export function validateIntakeBatch(value) { errors.push("batch.sequence_must_be_integer_0_to_2147483647"); } requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors); + const mode = value?.batch?.mode === undefined ? "upsert" : value.batch.mode; + if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid"); + if (mode === "replace") requiredIsoTimestamp(value?.batch?.generationAt, "batch.generationAt", errors); + if (mode === "upsert" && value?.batch?.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert"); - if (!Array.isArray(value?.facts) || value.facts.length === 0) { - errors.push("facts_must_be_nonempty_array"); + if (!Array.isArray(value?.facts)) { + errors.push("facts_must_be_array"); + } else if (value.facts.length === 0 && mode !== "replace") { + errors.push("facts_must_be_nonempty_array_for_upsert"); } else { - value.facts.forEach((fact, index) => validateFact(fact, `facts[${index}]`, errors)); + value.facts.forEach((fact, index) => { + validateFact(fact, `facts[${index}]`, errors); + if (mode === "replace" + && !Number.isNaN(Date.parse(fact?.observedAt)) + && Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) { + errors.push(`facts[${index}].observedAt_must_equal_generationAt`); + } + }); } if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors); @@ -80,22 +94,7 @@ function validateFact(value, path, errors) { errors.push(`${path}.attributes_size_exceeded`); } } - if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); -} - -function validatePointGeometry(value, path, errors) { - if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { - errors.push(`${path}_must_be_geojson_point`); - return; - } - rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors); - if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) { - errors.push(`${path}_coordinates_must_be_finite_numbers`); - return; - } - const [longitude, latitude] = value.coordinates; - if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`); - if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`); + if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors); } function validateRawEnvelope(value, errors) { diff --git a/packages/external-provider-contract/src/provider-package.mjs b/packages/external-provider-contract/src/provider-package.mjs index 6b0b68c..bd85fab 100644 --- a/packages/external-provider-contract/src/provider-package.mjs +++ b/packages/external-provider-contract/src/provider-package.mjs @@ -20,7 +20,7 @@ const TOKEN_REFRESH_MODES = new Set(["not_applicable", "operator_managed", "runt const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]); const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); const HISTORY_MODES = new Set(["none", "all", "sampled"]); -const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]); +const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]); const L2_STEP_KINDS = new Set([ "collection_trigger", "provider_request", @@ -84,6 +84,10 @@ const CAPABILITY_KEYS = new Set([ ]); const REQUEST_KEYS = new Set(["method", "baseUrl", "path", "query", "response"]); const RESPONSE_KEYS = new Set(["collectionPaths", "pagination"]); +const OFFSET_PAGINATION_KEYS = new Set([ + "mode", "limitParameter", "offsetParameter", "pageSize", "itemsPath", "totalPath", + "maxPages", "maxItems", "maxResponseBytes", +]); const ENTITY_SCOPE_KEYS = new Set(["mode", "refresh", "businessEntityFilter"]); const FIELD_POLICY_KEYS = new Set([ "id", @@ -137,7 +141,9 @@ const EXPRESSION_KEYS = new Set([ "strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive", "omitIfMissing", "omitIfInvalid", "minimum", "maximum", ]); -const GEOMETRY_KEYS = new Set(["type", "longitude", "latitude", "omitIfInvalid"]); +const GEOMETRY_KEYS = new Set([ + "type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid", +]); const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]); const TEMPLATE_KEYS = new Set([ "schemaVersion", @@ -496,12 +502,50 @@ function validateCapability(value, path, errors) { } else { rejectUnknownKeys(value.request.response, RESPONSE_KEYS, `${path}.request.response`, errors); requiredCollectionPathArray(value.request.response.collectionPaths, `${path}.request.response.collectionPaths`, errors); - if (value.request.response.pagination !== "single_bounded_response") errors.push(`${path}.request.response.pagination_invalid`); + validateResponsePagination(value.request.response.pagination, value.request.query, `${path}.request.response.pagination`, errors); } } validateCredentialVisibleScope(value.entityScope, `${path}.entityScope`, errors); } +function validateResponsePagination(value, query, path, errors) { + if (value === "single_bounded_response") return; + if (!isPlainObject(value)) { + errors.push(`${path}_invalid`); + return; + } + rejectUnknownKeys(value, OFFSET_PAGINATION_KEYS, path, errors); + if (value.mode !== "offset") errors.push(`${path}.mode_must_be_offset`); + for (const field of ["limitParameter", "offsetParameter"]) { + if (typeof value[field] !== "string" || !/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(value[field])) { + errors.push(`${path}.${field}_invalid`); + } + } + if (!Number.isInteger(value.pageSize) || value.pageSize < 1 || value.pageSize > 500) { + errors.push(`${path}.pageSize_invalid`); + } + if (!Number.isInteger(value.maxPages) || value.maxPages < 1 || value.maxPages > 1000) { + errors.push(`${path}.maxPages_invalid`); + } + if (!Number.isInteger(value.maxItems) || value.maxItems < 1 || value.maxItems > 5000) { + errors.push(`${path}.maxItems_invalid`); + } + if (!Number.isInteger(value.maxResponseBytes) || value.maxResponseBytes < 1024 || value.maxResponseBytes > 32 * 1024 * 1024) { + errors.push(`${path}.maxResponseBytes_invalid`); + } + for (const field of ["itemsPath", "totalPath"]) { + if (!isCanonicalSourcePath(value[field])) errors.push(`${path}.${field}_invalid`); + } + if (isPlainObject(query)) { + if (query[value.limitParameter] !== value.pageSize) errors.push(`${path}.pageSize_must_match_static_query_limit`); + if (query[value.offsetParameter] !== 0) errors.push(`${path}.offset_query_must_start_at_zero`); + } + if (Number.isInteger(value.maxPages) && Number.isInteger(value.pageSize) + && Number.isInteger(value.maxItems) && value.maxPages * value.pageSize < value.maxItems) { + errors.push(`${path}.maxPages_cannot_cover_maxItems`); + } +} + function validateFieldPolicy(value, path, errors) { rejectUnknownKeys(value, FIELD_POLICY_KEYS, path, errors); requiredIdentifier(value.id, `${path}.id`, errors); @@ -619,10 +663,26 @@ function validateMappingContract(value, path, errors) { errors.push(`${path}.fact.geometry_must_be_object`); } else if (isPlainObject(value.fact.geometry)) { rejectUnknownKeys(value.fact.geometry, GEOMETRY_KEYS, `${path}.fact.geometry`, errors); - if (value.fact.geometry.type !== "Point") errors.push(`${path}.fact.geometry.type_must_be_Point`); - validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors); - validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors); - if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`); + if (value.fact.geometry.type === "Point") { + validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors); + validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors); + if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`); + if (value.fact.geometry.strategy !== undefined || value.fact.geometry.paths !== undefined || value.fact.geometry.allowedTypes !== undefined) { + errors.push(`${path}.fact.geometry.point_shape_invalid`); + } + } else if (value.fact.geometry.type === "GeoJSON") { + if (value.fact.geometry.strategy !== "gelios_geozone_v1") errors.push(`${path}.fact.geometry.strategy_invalid`); + requiredSourcePathArray(value.fact.geometry.paths, `${path}.fact.geometry.paths`, errors); + if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) { + errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`); + } + if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`); + if (value.fact.geometry.longitude !== undefined || value.fact.geometry.latitude !== undefined) { + errors.push(`${path}.fact.geometry.geojson_shape_invalid`); + } + } else { + errors.push(`${path}.fact.geometry.type_invalid`); + } } if (value.fact.attributes !== undefined && !isPlainObject(value.fact.attributes)) { errors.push(`${path}.fact.attributes_must_be_object`); @@ -773,7 +833,13 @@ function validateMappingAgainstProduct(mapping, product, errors) { for (const [field, contract] of Object.entries(contracts)) { if (!isPlainObject(contract)) continue; if (field === "geometry") { - if (contract.type !== "point") errors.push(`${path}.fact.geometry_contract_must_be_point`); + if (contract.type === "point" && mapping.fact?.geometry?.type !== "Point") { + errors.push(`${path}.fact.geometry_contract_must_be_point`); + } + if (contract.type === "geometry" && mapping.fact?.geometry?.type !== "GeoJSON") { + errors.push(`${path}.fact.geometry_contract_must_be_geojson`); + } + if (!new Set(["point", "geometry"]).has(contract.type)) errors.push(`${path}.fact.geometry_contract_invalid`); if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) { errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`); } @@ -805,7 +871,7 @@ function validateFieldContract(value, path, errors) { if (value.enum !== undefined) { if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) { errors.push(`${path}.enum_must_be_nonempty_unique_array`); - } else if (new Set(["point", "string_array"]).has(value.type) + } else if (new Set(["point", "geometry", "string_array"]).has(value.type) || value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) { errors.push(`${path}.enum_value_type_invalid`); } @@ -880,6 +946,11 @@ function fieldContractValueMatchesType(value, type) { && value.coordinates.length === 2 && value.coordinates.every(Number.isFinite); } + if (type === "geometry") { + return isPlainObject(value) + && new Set(["Point", "LineString", "Polygon", "MultiPolygon"]).has(value.type) + && Array.isArray(value.coordinates); + } return typeof value === type && (type !== "number" || Number.isFinite(value)); } diff --git a/packages/external-provider-contract/src/zone-source.mjs b/packages/external-provider-contract/src/zone-source.mjs new file mode 100644 index 0000000..f291abb --- /dev/null +++ b/packages/external-provider-contract/src/zone-source.mjs @@ -0,0 +1,285 @@ +import { createHash } from "node:crypto"; + +import { isBoundedGeoJsonGeometry } from "./geometry.mjs"; + +export const ZONE_SOURCE_GENERATION_SCHEMA_VERSION = "nodedc.zone-source-generation/v1"; +export const ZONE_SOURCE_ADAPTERS = Object.freeze({ + "gelios-rest-v1": Object.freeze({ sourceKind: "live_api", sourceRevision: "gelios-rest-v1" }), + "mmap-snapshot-v1": Object.freeze({ sourceKind: "versioned_snapshot", sourceRevision: "mmap-snapshot-v1" }), +}); + +/** + * Convert one already-complete source generation into map.zone facts. Fetching, + * credentials and page iteration stay in Engine; this function is the common + * adapter boundary used by both live REST and immutable snapshot sources. + */ +export function normalizeZoneSourceGeneration(value, { + identityCrosswalk = {}, + maxZones = 5000, + maxBytes = 16 * 1024 * 1024, + circleSegments = 64, +} = {}) { + if (!isPlainObject(value) || value.schemaVersion !== ZONE_SOURCE_GENERATION_SCHEMA_VERSION) { + throw zoneSourceError("zone_source_generation_schema_invalid"); + } + const adapter = ZONE_SOURCE_ADAPTERS[value.adapterId]; + if (!adapter) throw zoneSourceError("zone_source_adapter_unsupported"); + if (!isPlainObject(identityCrosswalk)) throw zoneSourceError("zone_source_identity_crosswalk_invalid"); + if (value.complete !== true) throw zoneSourceError("zone_source_generation_incomplete"); + const generatedAt = iso(value.generatedAt, "zone_source_generated_at_invalid"); + if (!Array.isArray(value.zones) || value.zones.length > maxZones) { + throw zoneSourceError("zone_source_zone_limit_exceeded"); + } + if (byteLength(value) > maxBytes) throw zoneSourceError("zone_source_generation_bytes_exceeded"); + if (!Number.isInteger(circleSegments) || circleSegments < 16 || circleSegments > 256) { + throw zoneSourceError("zone_source_circle_segments_invalid"); + } + + const facts = []; + const sourceIds = new Set(); + for (const [index, rawZone] of value.zones.entries()) { + const zone = normalizeRawZone(rawZone, index, identityCrosswalk, circleSegments, value.adapterId); + if (sourceIds.has(zone.sourceId)) throw zoneSourceError("zone_source_identity_duplicate"); + sourceIds.add(zone.sourceId); + facts.push({ + sourceId: zone.sourceId, + semanticType: "map.zone", + observedAt: generatedAt, + geometry: zone.geometry, + attributes: compact({ + area_square_meters: finiteNonNegative(first(rawZone.surfaceArea, rawZone.surface_area)), + description: optionalString(first(rawZone.description, rawZone.descr)), + display_name: requiredString(first(rawZone.name, rawZone.display_name), "zone_source_name_required"), + geometry_kind: zone.geometryKind, + max_speed_kph: finiteNonNegative(first(rawZone.maxPermissibleSpeed, rawZone.max_permissible_speed)), + perimeter_meters: finiteNonNegative(rawZone.perimeter), + source_kind: adapter.sourceKind, + source_revision: revision(value, adapter), + style_color: optionalString(first(rawZone.color, rawZone.style_color)), + }), + }); + } + facts.sort((left, right) => left.sourceId.localeCompare(right.sourceId)); + const digest = createHash("sha256").update(stableJson({ + adapterId: value.adapterId, + generatedAt, + revision: revision(value, adapter), + facts, + })).digest("hex"); + return Object.freeze({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: value.adapterId, + sourceKind: adapter.sourceKind, + sourceRevision: revision(value, adapter), + generatedAt, + complete: true, + digest, + facts: Object.freeze(facts.map(deepFreeze)), + }); +} + +function normalizeRawZone(value, index, identityCrosswalk, circleSegments, adapterId) { + if (!isPlainObject(value)) throw zoneSourceError(`zone_source_zone_${index}_invalid`); + const nativeId = resolveNativeId(value, identityCrosswalk, adapterId); + const sourceId = nativeId.startsWith("gelios-zone-") ? nativeId : `gelios-zone-${nativeId}`; + if (!/^[a-z][a-z0-9._:-]{2,127}$/.test(sourceId)) throw zoneSourceError("zone_source_identity_invalid"); + const rawType = String(first(value.type, value.geometry_kind, "polygon")).trim().toLowerCase(); + const geometryKind = rawType === "circle" ? "circle" : rawType === "line" || rawType === "corridor" ? "corridor" : "polygon"; + const geometry = normalizeGeometry(value, geometryKind, circleSegments); + if (!isBoundedGeoJsonGeometry(geometry, new Set(["Polygon", "MultiPolygon"]))) { + throw zoneSourceError(`zone_source_zone_${index}_geometry_invalid`); + } + return { sourceId, geometryKind, geometry }; +} + +function resolveNativeId(zone, identityCrosswalk, adapterId) { + if (adapterId === "mmap-snapshot-v1") { + const snapshotKey = optionalString(first(zone.snapshotKey, zone.snapshot_key)); + if (!snapshotKey || !Object.hasOwn(identityCrosswalk, snapshotKey)) { + throw zoneSourceError("zone_source_identity_crosswalk_required"); + } + const mapped = identityCrosswalk[snapshotKey]; + if (mapped === undefined || mapped === null || !String(mapped).trim()) { + throw zoneSourceError("zone_source_identity_crosswalk_required"); + } + return String(mapped).trim(); + } + const direct = first(zone.id, zone.geozoneId, zone.zone_id, zone.sourceId); + if (direct !== undefined && direct !== null && String(direct).trim()) return String(direct).trim(); + throw zoneSourceError("zone_source_native_identity_required"); +} + +function normalizeGeometry(zone, geometryKind, circleSegments) { + if (isBoundedGeoJsonGeometry(zone.geometry, new Set(["Polygon", "MultiPolygon"]))) { + return structuredClone(zone.geometry); + } + const points = parseCoordinates(first(zone.points, zone.coords, zone.coordinates)); + if (geometryKind === "circle") { + if (!points.length) throw zoneSourceError("zone_source_circle_center_required"); + const radius = finiteNonNegative(zone.radius); + if (!radius || radius <= 0) throw zoneSourceError("zone_source_circle_radius_required"); + return circlePolygon(points[0], radius, circleSegments); + } + if (geometryKind === "corridor") { + const linePoints = parseCoordinates(first(zone.line, zone.points, zone.coords, zone.coordinates)); + const width = finiteNonNegative(first(zone.lineWidthMeters, zone.line_width_meters, zone.width, zone.radius)); + if (linePoints.length < 2 || !width || width <= 0) { + throw zoneSourceError("zone_source_corridor_path_and_width_required"); + } + return corridorPolygon(linePoints, width); + } + if (points.length < 3) throw zoneSourceError("zone_source_polygon_points_required"); + const ring = [...points]; + if (!samePosition(ring[0], ring.at(-1))) ring.push([...ring[0]]); + return { type: "Polygon", coordinates: [ring] }; +} + +function parseCoordinates(value) { + if (typeof value === "string") { + return value.split(";").map((entry) => entry.trim()).filter(Boolean).map((entry) => { + const [latitude, longitude] = entry.split(",").map(Number); + return position(longitude, latitude); + }); + } + if (!Array.isArray(value)) return []; + return value.map((entry) => { + if (Array.isArray(entry)) { + const [latitude, longitude] = entry.map(Number); + return position(longitude, latitude); + } + if (!isPlainObject(entry)) throw zoneSourceError("zone_source_coordinate_invalid"); + return position( + firstNumber(entry.longitude, entry.lon, entry.lng, entry.x), + firstNumber(entry.latitude, entry.lat, entry.y), + ); + }); +} + +function circlePolygon([longitude, latitude], radiusMeters, segments) { + const earthRadius = 6_371_008.8; + const angularDistance = radiusMeters / earthRadius; + const latitudeRad = radians(latitude); + const longitudeRad = radians(longitude); + const ring = []; + for (let index = 0; index <= segments; index += 1) { + const bearing = 2 * Math.PI * index / segments; + const targetLatitude = Math.asin( + Math.sin(latitudeRad) * Math.cos(angularDistance) + + Math.cos(latitudeRad) * Math.sin(angularDistance) * Math.cos(bearing), + ); + const targetLongitude = longitudeRad + Math.atan2( + Math.sin(bearing) * Math.sin(angularDistance) * Math.cos(latitudeRad), + Math.cos(angularDistance) - Math.sin(latitudeRad) * Math.sin(targetLatitude), + ); + ring.push([degrees(targetLongitude), degrees(targetLatitude)]); + } + ring[ring.length - 1] = [...ring[0]]; + return { type: "Polygon", coordinates: [ring] }; +} + +function corridorPolygon(points, widthMeters) { + const halfWidth = widthMeters / 2; + const left = []; + const right = []; + for (let index = 0; index < points.length; index += 1) { + const previous = points[Math.max(0, index - 1)]; + const next = points[Math.min(points.length - 1, index + 1)]; + const meanLatitude = radians((previous[1] + next[1]) / 2); + const dx = (next[0] - previous[0]) * Math.cos(meanLatitude); + const dy = next[1] - previous[1]; + const length = Math.hypot(dx, dy); + if (!length) throw zoneSourceError("zone_source_corridor_zero_length_segment"); + const metersPerDegreeLatitude = 111_320; + const longitudeOffset = (-dy / length) * halfWidth / (metersPerDegreeLatitude * Math.max(0.01, Math.cos(radians(points[index][1])))); + const latitudeOffset = (dx / length) * halfWidth / metersPerDegreeLatitude; + left.push([points[index][0] + longitudeOffset, points[index][1] + latitudeOffset]); + right.push([points[index][0] - longitudeOffset, points[index][1] - latitudeOffset]); + } + const ring = [...left, ...right.reverse(), [...left[0]]]; + return { type: "Polygon", coordinates: [ring] }; +} + +function revision(value, adapter) { + const explicit = optionalString(value.sourceRevision); + if (value.adapterId === "mmap-snapshot-v1") { + const digest = optionalString(value.snapshotDigest); + if (!digest || !/^[a-f0-9]{64}$/.test(digest)) throw zoneSourceError("zone_source_snapshot_digest_required"); + return `${explicit || adapter.sourceRevision}@sha256:${digest}`; + } + return explicit || adapter.sourceRevision; +} + +function position(longitude, latitude) { + if (![longitude, latitude].every(Number.isFinite) + || longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) { + throw zoneSourceError("zone_source_coordinate_invalid"); + } + return [longitude, latitude]; +} + +function first(...values) { + return values.find((value) => value !== undefined && value !== null && value !== ""); +} + +function firstNumber(...values) { + const value = first(...values); + const number = typeof value === "number" ? value : Number(String(value ?? "").trim()); + return Number.isFinite(number) ? number : Number.NaN; +} + +function finiteNonNegative(value) { + const number = firstNumber(value); + return Number.isFinite(number) && number >= 0 ? number : undefined; +} + +function requiredString(value, code) { + const result = optionalString(value); + if (!result) throw zoneSourceError(code); + return result; +} + +function optionalString(value) { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function iso(value, code) { + const date = new Date(String(value ?? "")); + if (Number.isNaN(date.getTime())) throw zoneSourceError(code); + return date.toISOString(); +} + +function compact(value) { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined)); +} + +function samePosition(left, right) { + return Array.isArray(left) && Array.isArray(right) && left[0] === right[0] && left[1] === right[1]; +} + +function stableJson(value) { + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (isPlainObject(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`; + return JSON.stringify(value); +} + +function byteLength(value) { + try { return Buffer.byteLength(JSON.stringify(value)); } catch { return Number.POSITIVE_INFINITY; } +} + +function deepFreeze(value) { + if (!value || typeof value !== "object" || Object.isFrozen(value)) return value; + Object.freeze(value); + Object.values(value).forEach(deepFreeze); + return value; +} + +function isPlainObject(value) { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function radians(value) { return value * Math.PI / 180; } +function degrees(value) { return value * 180 / Math.PI; } + +function zoneSourceError(code) { + return Object.assign(new Error(code), { code }); +} diff --git a/packages/external-provider-contract/test/data-product.test.mjs b/packages/external-provider-contract/test/data-product.test.mjs index 2ff5ddd..67c2bd4 100644 --- a/packages/external-provider-contract/test/data-product.test.mjs +++ b/packages/external-provider-contract/test/data-product.test.mjs @@ -72,6 +72,32 @@ assert.equal(validateDataProductPublish({ ...publish, facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }], }).errors.includes("facts[0].geometry.latitude_out_of_range"), true); +const zoneGenerationAt = "2026-07-15T10:10:00.000Z"; +const zoneFact = { + sourceId: "gelios-zone-42", + semanticType: "map.zone", + observedAt: zoneGenerationAt, + attributes: { display_name: "Zone 42" }, + geometry: { + type: "Polygon", + coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]], + }, +}; +const replacement = { + ...publish, + batch: { ...publish.batch, mode: "replace", generationAt: zoneGenerationAt }, + facts: [zoneFact], +}; +assert.equal(validateDataProductPublish(replacement).ok, true); +assert.equal(validateDataProductPublish({ ...replacement, facts: [] }).ok, true); +assert.equal(validateDataProductPublish({ + ...replacement, + facts: [{ ...zoneFact, observedAt: "2026-07-15T10:09:59.000Z" }], +}).errors.includes("facts[0].observedAt_must_equal_generationAt"), true); +assert.equal(validateDataProductPublish({ + ...replacement, + facts: [{ ...zoneFact, geometry: { ...zoneFact.geometry, coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.61, 55.75]]] } }], +}).errors.includes("facts[0].geometry[0]_linear_ring_must_be_closed"), true); const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" }; const snapshot = { @@ -147,7 +173,11 @@ assert.equal(validateDataProductPatch(patch).ok, true); assert.equal(validateDataProductPatch({ ...patch, operations: [{ op: "delete", fact: canonicalFact }], -}).errors.includes("operations[0].op_must_be_upsert"), true); +}).errors.includes("operations[0].op_invalid"), true); +assert.equal(validateDataProductPatch({ + ...patch, + operations: [{ op: "remove", sourceId: "gelios-zone-42", semanticType: "map.zone" }], +}).ok, true); assert.equal(validateDataProductPatch({ ...patch, operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }], diff --git a/packages/external-provider-contract/test/provider-package.test.mjs b/packages/external-provider-contract/test/provider-package.test.mjs index 277280c..bff9357 100644 --- a/packages/external-provider-contract/test/provider-package.test.mjs +++ b/packages/external-provider-contract/test/provider-package.test.mjs @@ -18,6 +18,7 @@ import { geliosProviderPackageV2 } from "../providers/gelios/v2/index.mjs"; import { geliosProviderPackageV3 } from "../providers/gelios/v3/index.mjs"; import { geliosProviderPackageV4 } from "../providers/gelios/v4/index.mjs"; import { geliosProviderPackageV5 } from "../providers/gelios/v5/index.mjs"; +import { geliosProviderPackageV6 } from "../providers/gelios/v6/index.mjs"; import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs"; const expectedFields = [ @@ -41,6 +42,24 @@ assert.deepEqual(validateProviderPackage(geliosProviderPackageV2), { ok: true, e assert.deepEqual(validateProviderPackage(geliosProviderPackageV3), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV4), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV5), { ok: true, errors: [] }); +assert.deepEqual(validateProviderPackage(geliosProviderPackageV6), { ok: true, errors: [] }); + +const zonesProduct = geliosProviderPackageV6.dataProducts.find((product) => product.id === "map.zones.current.v1"); +const registeredZonesProduct = JSON.parse(await readFile(new URL( + "../../../services/external-data-plane/definitions/map.zones.current.v1.json", + import.meta.url, +), "utf8")); +assert.deepEqual(zonesProduct, registeredZonesProduct); +assert.deepEqual(zonesProduct.semanticTypes, ["map.zone"]); +assert.equal(zonesProduct.fieldContracts.geometry.type, "geometry"); +const zonesCapability = geliosProviderPackageV6.capabilities.find((capability) => capability.id === "gelios.geozones.current.read"); +assert.equal(zonesCapability.request.query.pl, 100); +assert.equal(zonesCapability.request.query.po, 0); +assert.equal(zonesCapability.request.response.pagination.maxItems, 5000); +assert.equal(zonesCapability.request.response.pagination.totalPath, "paginationMetadata.totalCount"); +const zonesMapping = geliosProviderPackageV6.mappingContracts.find((mapping) => mapping.id === "gelios.geozones.to.map.zones.current.v1"); +assert.deepEqual(zonesMapping.fact.observedAt.paths, ["updatedAt"]); +assert.equal(zonesMapping.fact.observedAt.fallback, "collection_received_at"); const strictProduct = geliosProviderPackageV5.dataProducts[0]; const registeredStrictProduct = JSON.parse(await readFile(new URL( diff --git a/packages/external-provider-contract/test/zone-source.test.mjs b/packages/external-provider-contract/test/zone-source.test.mjs new file mode 100644 index 0000000..f57a36a --- /dev/null +++ b/packages/external-provider-contract/test/zone-source.test.mjs @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; + +import { + ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + normalizeZoneSourceGeneration, + validateDataProductPublish, +} from "../src/index.mjs"; + +const generatedAt = "2026-07-20T18:00:00.000Z"; +const live = normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "gelios-rest-v1", + sourceRevision: "gelios-rest-v1", + generatedAt, + complete: true, + zones: [{ + id: 42, + name: "Polygon zone", + type: "polygon", + points: [ + { latitude: 55.74, longitude: 37.60 }, + { latitude: 55.74, longitude: 37.62 }, + { latitude: 55.76, longitude: 37.62 }, + ], + color: "#ff00ff", + }, { + id: 43, + name: "Circle zone", + type: "circle", + points: [{ latitude: 55.75, longitude: 37.61 }], + radius: 150, + }], +}); +assert.equal(live.complete, true); +assert.equal(live.sourceKind, "live_api"); +assert.equal(live.facts.length, 2); +assert.deepEqual(live.facts.map((fact) => fact.sourceId), ["gelios-zone-42", "gelios-zone-43"]); +assert.equal(live.facts.every((fact) => fact.observedAt === generatedAt), true); +assert.equal(live.facts.every((fact) => fact.geometry.type === "Polygon"), true); +assert.equal(validateDataProductPublish({ + schemaVersion: "nodedc.data-product.publish/v1", + batch: { runId: "zones-live-1", sequence: 0, idempotencyKey: "zones-live-1.batch-0", mode: "replace", generationAt: generatedAt }, + facts: live.facts, +}).ok, true); + +const snapshotDigest = "a".repeat(64); +const snapshot = normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "mmap-snapshot-v1", + snapshotDigest, + generatedAt, + complete: true, + zones: [{ + snapshotKey: "legacy-zone-a", + name: "Snapshot zone", + type: "polygon", + points: "55.74,37.60;55.74,37.62;55.76,37.62", + }], +}, { identityCrosswalk: { "legacy-zone-a": 42 } }); +assert.equal(snapshot.sourceKind, "versioned_snapshot"); +assert.equal(snapshot.facts[0].sourceId, live.facts[0].sourceId); +assert.match(snapshot.sourceRevision, new RegExp(`@sha256:${snapshotDigest}$`)); + +assert.throws(() => normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "mmap-snapshot-v1", + snapshotDigest, + generatedAt, + complete: true, + zones: [{ snapshotKey: "unknown", name: "Unknown", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }], +}), /zone_source_identity_crosswalk_required/); +assert.throws(() => normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "mmap-snapshot-v1", + snapshotDigest, + generatedAt, + complete: true, + zones: [{ id: 42, name: "Uncrossed", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }], +}), /zone_source_identity_crosswalk_required/); +assert.throws(() => normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "gelios-rest-v1", + generatedAt, + complete: true, + zones: [{ name: "No native id", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }], +}), /zone_source_native_identity_required/); +assert.throws(() => normalizeZoneSourceGeneration({ + schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION, + adapterId: "gelios-rest-v1", + generatedAt, + complete: false, + zones: [], +}), /zone_source_generation_incomplete/); + +console.log("external-provider zone source: ok"); diff --git a/packages/n8n-nodes-ndc/nodes/NdcDataProductPublish/NdcDataProductPublish.node.ts b/packages/n8n-nodes-ndc/nodes/NdcDataProductPublish/NdcDataProductPublish.node.ts index 6663c0f..2f315a9 100644 --- a/packages/n8n-nodes-ndc/nodes/NdcDataProductPublish/NdcDataProductPublish.node.ts +++ b/packages/n8n-nodes-ndc/nodes/NdcDataProductPublish/NdcDataProductPublish.node.ts @@ -25,7 +25,7 @@ export class NdcDataProductPublish implements INodeType { name: 'ndcDataProductPublish', icon: NDC_NODE_ICON, group: ['output'], - version: 1, + version: [1, 2], subtitle: '={{$parameter["dataProductId"]}}', description: 'Publish canonical facts through a scoped NDC Data Product grant', defaults: { name: 'NDC Data Product Publish' }, @@ -41,6 +41,19 @@ export class NdcDataProductPublish implements INodeType { default: '', required: true, }, + { + displayName: 'Publish Mode', + name: 'publishMode', + type: 'options', + options: [ + { name: 'Create or Update', value: 'upsert' }, + { name: 'Replace Complete Snapshot', value: 'replace' }, + ], + default: 'upsert', + required: true, + displayOptions: { show: { '@version': [2] } }, + description: 'Replace commits one complete generation atomically and removes keys absent from it', + }, { displayName: 'Batch Sequence', name: 'sequence', @@ -71,6 +84,9 @@ export class NdcDataProductPublish implements INodeType { const inputItems = this.getInputData(); const dataProductId = this.getNodeParameter('dataProductId', 0) as string; const sequence = this.getNodeParameter('sequence', 0, 0) as number; + const publishMode = this.getNode().typeVersion >= 2 + ? this.getNodeParameter('publishMode', 0, 'upsert') as 'upsert' | 'replace' + : 'upsert'; let body; let encodedProductId; try { @@ -81,6 +97,7 @@ export class NdcDataProductPublish implements INodeType { this.getNode().id, dataProductId, sequence, + publishMode, ); encodedProductId = encodeIdentifierPath(dataProductId, 'dataProductId'); } catch (error) { diff --git a/packages/n8n-nodes-ndc/nodes/shared/contracts.ts b/packages/n8n-nodes-ndc/nodes/shared/contracts.ts index dc88a7e..72315f2 100644 --- a/packages/n8n-nodes-ndc/nodes/shared/contracts.ts +++ b/packages/n8n-nodes-ndc/nodes/shared/contracts.ts @@ -18,18 +18,23 @@ export interface NdcFact extends IDataObject { semanticType: string; observedAt: string; attributes?: IDataObject; - geometry?: { - type: 'Point'; - coordinates: [number, number]; - }; + geometry?: NdcGeometry; } +export type NdcGeometry = + | { type: 'Point'; coordinates: [number, number] } + | { type: 'LineString'; coordinates: [number, number][] } + | { type: 'Polygon'; coordinates: [number, number][][] } + | { type: 'MultiPolygon'; coordinates: [number, number][][][] }; + export interface NdcPublishPayload extends IDataObject { schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION; batch: { runId: string; sequence: number; idempotencyKey: string; + mode?: 'replace'; + generationAt?: string; }; facts: NdcFact[]; } @@ -62,12 +67,16 @@ export function encodeIdentifierPath(value: unknown, field: string): string { return encodeURIComponent(requireIdentifier(value, field)); } -export function factsFromItems(items: INodeExecutionData[]): NdcFact[] { - if (!items.length) throw new Error('facts_required'); - if (items.length > MAX_FACTS) throw new Error('facts_limit_exceeded'); +export function factsFromItems(items: INodeExecutionData[], allowEmpty = false): NdcFact[] { + const batchFacts = items.length === 1 && Array.isArray(items[0]?.json?.facts) + ? items[0].json.facts + : null; + const sources = batchFacts ?? items.map((item) => (isObject(item.json.fact) ? item.json.fact : item.json)); + if (!sources.length && !allowEmpty) throw new Error('facts_required'); + if (sources.length > MAX_FACTS) throw new Error('facts_limit_exceeded'); const entityKeys = new Set(); - return items.map((item, index) => { - const source = isObject(item.json.fact) ? item.json.fact : item.json; + return sources.map((source, index) => { + if (!isObject(source)) throw new Error(`facts_${index}_invalid`); const fact: NdcFact = { sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`), semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`), @@ -84,7 +93,7 @@ export function factsFromItems(items: INodeExecutionData[]): NdcFact[] { if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`); fact.attributes = source.attributes; } - if (source.geometry !== undefined) fact.geometry = normalizePoint(source.geometry, index); + if (source.geometry !== undefined) fact.geometry = normalizeGeometry(source.geometry, index); return fact; }); } @@ -96,6 +105,7 @@ export function buildPublishPayload( nodeId: string, dataProductId: string, sequence: number, + publishMode: 'upsert' | 'replace' = 'upsert', ): NdcPublishPayload { if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid'); requireIdentifier(dataProductId, 'dataProductId'); @@ -103,10 +113,22 @@ export function buildPublishPayload( const idempotencyKey = `publish-${sha256( `${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`, )}`; + const facts = factsFromItems(items, publishMode === 'replace'); + const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts) + ? items[0].json.generationAt + : undefined; + const generationAt = publishMode === 'replace' + ? replacementGenerationAt(facts, explicitGenerationAt) + : undefined; return { schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, - batch: { runId, sequence, idempotencyKey }, - facts: factsFromItems(items), + batch: { + runId, + sequence, + idempotencyKey, + ...(generationAt ? { mode: 'replace' as const, generationAt } : {}), + }, + facts, }; } @@ -254,18 +276,60 @@ function requireFoundrySlotId(value: unknown): string { return normalized; } -function normalizePoint(value: unknown, index: number): NdcFact['geometry'] { - if (!isObject(value) || value.type !== 'Point' || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { +function normalizeGeometry(value: unknown, index: number): NdcGeometry { + if (!isObject(value) || !['Point', 'LineString', 'Polygon', 'MultiPolygon'].includes(String(value.type)) + || !Array.isArray(value.coordinates)) { throw new Error(`facts_${index}_geometry_invalid`); } - const [longitude, latitude] = value.coordinates; - if (![longitude, latitude].every((coordinate) => typeof coordinate === 'number' && Number.isFinite(coordinate))) { - throw new Error(`facts_${index}_geometry_coordinates_invalid`); + if (Buffer.byteLength(JSON.stringify(value)) > 192 * 1024) throw new Error(`facts_${index}_geometry_size_exceeded`); + let vertices = 0; + const position = (candidate: unknown, path: string): [number, number] => { + if (!Array.isArray(candidate) || candidate.length !== 2 || !candidate.every(Number.isFinite)) { + throw new Error(`${path}_position_invalid`); + } + const [longitude, latitude] = candidate as [number, number]; + if (longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) { + throw new Error(`${path}_position_out_of_range`); + } + vertices += 1; + if (vertices > 10_000) throw new Error(`facts_${index}_geometry_vertex_limit_exceeded`); + return [longitude, latitude]; + }; + const line = (candidate: unknown, path: string, minimum: number): [number, number][] => { + if (!Array.isArray(candidate) || candidate.length < minimum) throw new Error(`${path}_line_invalid`); + return candidate.map((entry, positionIndex) => position(entry, `${path}_${positionIndex}`)); + }; + const ring = (candidate: unknown, path: string): [number, number][] => { + const result = line(candidate, path, 4); + const first = result[0]; + const last = result.at(-1); + if (!last || first[0] !== last[0] || first[1] !== last[1]) throw new Error(`${path}_ring_not_closed`); + return result; + }; + const polygon = (candidate: unknown, path: string): [number, number][][] => { + if (!Array.isArray(candidate) || !candidate.length) throw new Error(`${path}_polygon_invalid`); + return candidate.map((entry, ringIndex) => ring(entry, `${path}_${ringIndex}`)); + }; + if (value.type === 'Point') return { type: 'Point', coordinates: position(value.coordinates, `facts_${index}_geometry`) }; + if (value.type === 'LineString') return { type: 'LineString', coordinates: line(value.coordinates, `facts_${index}_geometry`, 2) }; + if (value.type === 'Polygon') return { type: 'Polygon', coordinates: polygon(value.coordinates, `facts_${index}_geometry`) }; + const coordinates = value.coordinates.map((entry, polygonIndex) => polygon(entry, `facts_${index}_geometry_${polygonIndex}`)); + if (!coordinates.length) throw new Error(`facts_${index}_geometry_multipolygon_invalid`); + return { type: 'MultiPolygon', coordinates }; +} + +function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string { + const declared = explicit === undefined ? undefined : requireIsoTimestamp(explicit, 'replace_generationAt'); + const timestamps = [...new Set(facts.map((fact) => new Date(fact.observedAt).toISOString()))]; + if (!timestamps.length) { + if (!declared) throw new Error('replace_generationAt_required_for_empty_generation'); + return new Date(declared).toISOString(); } - if ((longitude as number) < -180 || (longitude as number) > 180 || (latitude as number) < -90 || (latitude as number) > 90) { - throw new Error(`facts_${index}_geometry_coordinates_out_of_range`); + if (timestamps.length !== 1) throw new Error('replace_generation_requires_one_observedAt'); + if (declared && new Date(declared).toISOString() !== timestamps[0]) { + throw new Error('replace_generationAt_observedAt_mismatch'); } - return { type: 'Point', coordinates: [longitude as number, latitude as number] }; + return timestamps[0]; } function requireIsoTimestamp(value: unknown, field: string): string { diff --git a/packages/n8n-nodes-ndc/package-lock.json b/packages/n8n-nodes-ndc/package-lock.json index 1c0428e..75ab3b8 100644 --- a/packages/n8n-nodes-ndc/package-lock.json +++ b/packages/n8n-nodes-ndc/package-lock.json @@ -1,12 +1,12 @@ { "name": "n8n-nodes-ndc", - "version": "0.1.4", + "version": "0.1.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "n8n-nodes-ndc", - "version": "0.1.4", + "version": "0.1.5", "license": "UNLICENSED", "devDependencies": { "@n8n/node-cli": "0.39.3", diff --git a/packages/n8n-nodes-ndc/package.json b/packages/n8n-nodes-ndc/package.json index 7a450e0..6625e35 100644 --- a/packages/n8n-nodes-ndc/package.json +++ b/packages/n8n-nodes-ndc/package.json @@ -1,6 +1,6 @@ { "name": "n8n-nodes-ndc", - "version": "0.1.4", + "version": "0.1.5", "description": "Private NODE.DC nodes for scoped data products and Foundry bindings.", "private": true, "license": "UNLICENSED", diff --git a/packages/n8n-nodes-ndc/test/package-policy.test.cjs b/packages/n8n-nodes-ndc/test/package-policy.test.cjs index c30ef06..0081b44 100644 --- a/packages/n8n-nodes-ndc/test/package-policy.test.cjs +++ b/packages/n8n-nodes-ndc/test/package-policy.test.cjs @@ -146,6 +146,64 @@ async function main() { publish.batch.idempotencyKey, contracts.buildPublishPayload(items, 'execution-42', 'workflow-7', 'node-3', 'fleet.positions.current.v2', 0).batch.idempotencyKey, ); + const zoneItems = [{ + json: { + sourceId: 'gelios-zone-42', + semanticType: 'map.zone', + observedAt: '2026-07-15T12:10:00.000Z', + attributes: { display_name: 'Zone 42' }, + geometry: { + type: 'Polygon', + coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]], + }, + }, + }]; + const replacement = contracts.buildPublishPayload( + zoneItems, + 'execution-43', + 'workflow-7', + 'node-zones', + 'map.zones.current.v1', + 0, + 'replace', + ); + assert.equal(externalContract.validateDataProductPublish(replacement).ok, true); + assert.equal(replacement.batch.mode, 'replace'); + assert.equal(replacement.batch.generationAt, zoneItems[0].json.observedAt); + const replacementBatch = contracts.buildPublishPayload( + [{ json: { generationAt: zoneItems[0].json.observedAt, facts: zoneItems.map((item) => item.json) } }], + 'execution-44', + 'workflow-7', + 'node-zones', + 'map.zones.current.v1', + 0, + 'replace', + ); + assert.deepEqual(replacementBatch.facts, replacement.facts); + assert.equal(replacementBatch.batch.generationAt, zoneItems[0].json.observedAt); + const emptyReplacement = contracts.buildPublishPayload( + [{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: [] } }], + 'execution-45', + 'workflow-7', + 'node-zones', + 'map.zones.current.v1', + 0, + 'replace', + ); + assert.deepEqual(emptyReplacement.facts, []); + assert.equal(externalContract.validateDataProductPublish(emptyReplacement).ok, true); + assert.throws( + () => contracts.buildPublishPayload( + [{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: zoneItems.map((item) => item.json) } }], + 'execution-46', + 'workflow-7', + 'node-zones', + 'map.zones.current.v1', + 0, + 'replace', + ), + /replace_generationAt_observedAt_mismatch/, + ); assert.throws( () => contracts.buildPublishPayload( [{ json: { ...items[0].json, attributes: { accessToken: 'forbidden' } } }], @@ -303,6 +361,29 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) { assert.equal(externalContract.validateDataProductPublish(publishCalls[0].options.body).ok, true); assert.equal(publishResult[0][0].json.publishedFactCount, 1); + const replacementCalls = []; + const replacementContext = executionContext({ + typeVersion: 2, + parameters: { dataProductId: 'map.zones.current.v1', publishMode: 'replace', sequence: 0 }, + input: [{ + json: { + generationAt: '2026-07-15T12:10:00.000Z', + facts: [{ + sourceId: 'gelios-zone-42', + semanticType: 'map.zone', + observedAt: '2026-07-15T12:10:00.000Z', + attributes: { display_name: 'Zone 42' }, + geometry: { type: 'Polygon', coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]] }, + }], + }, + }], + response: { ok: true, publishedFactCount: 1, currentRemovedCount: 0 }, + calls: replacementCalls, + }); + await nodes.get('NdcDataProductPublish').execute.call(replacementContext); + assert.equal(replacementCalls[0].options.body.batch.mode, 'replace'); + assert.equal(externalContract.validateDataProductPublish(replacementCalls[0].options.body).ok, true); + const readCalls = []; const readContext = executionContext({ parameters: { dataProductId: 'fleet.positions.current.v1', pageSize: 1000 }, @@ -411,7 +492,7 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) { } } -function executionContext({ parameters, input = [{ json: {} }], response, calls }) { +function executionContext({ parameters, input = [{ json: {} }], response, calls, typeVersion = 1 }) { return { getNodeParameter(name, _index, defaultValue) { return Object.prototype.hasOwnProperty.call(parameters, name) ? parameters[name] : defaultValue; @@ -419,7 +500,7 @@ function executionContext({ parameters, input = [{ json: {} }], response, calls getInputData() { return input; }, getExecutionId() { return 'execution-42'; }, getWorkflow() { return { id: 'workflow-7' }; }, - getNode() { return { id: 'node-3' }; }, + getNode() { return { id: 'node-3', typeVersion }; }, helpers: { async httpRequestWithAuthentication(credentialName, options) { calls.push({ credentialName, options }); diff --git a/services/external-data-plane/Dockerfile b/services/external-data-plane/Dockerfile index 53805af..0ae7369 100644 --- a/services/external-data-plane/Dockerfile +++ b/services/external-data-plane/Dockerfile @@ -6,6 +6,7 @@ COPY packages/external-provider-contract/package.json ./packages/external-provid COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs +COPY packages/external-provider-contract/src/geometry.mjs ./packages/external-provider-contract/src/geometry.mjs COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/ diff --git a/services/external-data-plane/definitions/map.zones.current.v1.json b/services/external-data-plane/definitions/map.zones.current.v1.json new file mode 100644 index 0000000..0bff86b --- /dev/null +++ b/services/external-data-plane/definitions/map.zones.current.v1.json @@ -0,0 +1,32 @@ +{ + "id": "map.zones.current.v1", + "version": "1.0.0", + "ontologyRevision": "ontology.map.zone.v1", + "deliveryMode": "snapshot+patch", + "semanticTypes": ["map.zone"], + "fields": [ + "area_square_meters", + "description", + "display_name", + "geometry", + "geometry_kind", + "max_speed_kph", + "perimeter_meters", + "source_kind", + "source_revision", + "style_color" + ], + "fieldContracts": { + "area_square_meters": { "type": "number", "required": false, "minimum": 0 }, + "description": { "type": "string", "required": false }, + "display_name": { "type": "string", "required": true }, + "geometry": { "type": "geometry", "required": true }, + "geometry_kind": { "type": "string", "required": true, "enum": ["circle", "corridor", "polygon"] }, + "max_speed_kph": { "type": "number", "required": false, "minimum": 0 }, + "perimeter_meters": { "type": "number", "required": false, "minimum": 0 }, + "source_kind": { "type": "string", "required": true, "enum": ["live_api", "versioned_snapshot"] }, + "source_revision": { "type": "string", "required": true }, + "style_color": { "type": "string", "required": false } + }, + "history": { "mode": "none", "retentionDays": 1 } +} diff --git a/services/external-data-plane/src/data-product-delivery.mjs b/services/external-data-plane/src/data-product-delivery.mjs index b478312..1f749d5 100644 --- a/services/external-data-plane/src/data-product-delivery.mjs +++ b/services/external-data-plane/src/data-product-delivery.mjs @@ -3,6 +3,7 @@ import { DATA_PRODUCT_HISTORY_SCHEMA_VERSION, DATA_PRODUCT_PATCH_SCHEMA_VERSION, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, + isBoundedGeoJsonGeometry, } from "@nodedc/external-provider-contract/data-plane"; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; @@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, { const existing = await client.query( `select id as "batchId", fact_count as "publishedFactCount", current_updated_count as "currentUpdatedCount", + current_removed_count as "currentRemovedCount", history_inserted_count as "historyInsertedCount", patch_operation_count as "patchOperationCount", delivery_cursor::text as cursor, received_at as "acceptedAt", @@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, { return { ...normalizeReceipt(existing.rows[0]), idempotent: true }; } + if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch); + const records = batch.facts.map((fact) => ({ source_id: fact.sourceId, semantic_type: fact.semanticType, @@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, { select $1, $2, $3, $4, item.source_id, item.semantic_type, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), case when item.geometry is null then null - else ST_SetSRID(ST_MakePoint( - (item.geometry->'coordinates'->>0)::double precision, - (item.geometry->'coordinates'->>1)::double precision - ), 4326)::geography end, + else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end, item.fingerprint, now() from jsonb_to_recordset($5::jsonb) as item( source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz, @@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, { returning source_id as "sourceId", semantic_type as "semanticType", observed_at as "observedAt", received_at as "receivedAt", attributes, case when geometry is null then null - else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( - ST_X(geometry::geometry), ST_Y(geometry::geometry) - )) end as geometry, + else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry, fingerprint`, [ batch.source.tenantId, batch.source.connectionId, batch.source.providerId, @@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, { ], ); + const removed = batch.batch.mode === "replace" + ? await removeMissingCurrent(client, batch, records) + : []; const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records); - const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })); + const operations = [ + ...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })), + ...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })), + ]; const chunks = definition.deliveryMode === "snapshot+patch" ? chunkOperations(operations, maxPatchOperations, maxPatchBytes) : []; const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch); + if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch); await client.query( `update external_data_plane_batches set current_updated_count = $2, - history_inserted_count = $3, - patch_operation_count = $4, - delivery_cursor = $5 + current_removed_count = $3, + history_inserted_count = $4, + patch_operation_count = $5, + delivery_cursor = $6 where id = $1`, - [batchId, updated.rowCount, historyInsertedCount, operations.length, cursor], + [batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor], ); await client.query("commit"); return normalizeReceipt({ @@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, { idempotent: false, publishedFactCount: batch.facts.length, currentUpdatedCount: updated.rowCount, + currentRemovedCount: removed.length, historyInsertedCount, patchOperationCount: operations.length, cursor: String(cursor), @@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit `select source_id as "sourceId", semantic_type as "semanticType", observed_at as "observedAt", received_at as "receivedAt", attributes, case when geometry is null then null - else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( - ST_X(geometry::geometry), ST_Y(geometry::geometry) - )) end as geometry + else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry from external_data_plane_current where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4 order by source_id asc, semantic_type asc @@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options) select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType", observed_at as "observedAt", received_at as "receivedAt", attributes, case when geometry is null then null - else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( - ST_X(geometry::geometry), ST_Y(geometry::geometry) - )) end as geometry + else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry from ranked where sample_rank = 1 and ( @@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) { return result.rowCount; } +async function lockReplacementGeneration(client, batch) { + const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId]; + await client.query( + `insert into external_data_plane_delivery_state ( + tenant_id, connection_id, provider_id, data_product_id, current_cursor + ) values ($1, $2, $3, $4, 0) + on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`, + scope, + ); + const state = await client.query( + `select current_generation_at as "currentGenerationAt" + from external_data_plane_delivery_state + where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4 + for update`, + scope, + ); + const generationAt = new Date(batch.batch.generationAt); + const currentGenerationAt = state.rows[0]?.currentGenerationAt + ? new Date(state.rows[0].currentGenerationAt) + : null; + if (currentGenerationAt && generationAt <= currentGenerationAt) { + throw deliveryError("data_product_generation_not_newer", 409); + } +} + +async function removeMissingCurrent(client, batch, records) { + const result = await client.query( + `delete from external_data_plane_current as target + where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3 + and target.data_product_id = $4 + and not exists ( + select 1 + from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text) + where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type + ) + returning source_id as "sourceId", semantic_type as "semanticType"`, + [ + batch.source.tenantId, + batch.source.connectionId, + batch.source.providerId, + batch.contract.dataProductId, + JSON.stringify(records), + ], + ); + return result.rows; +} + +async function persistReplacementGeneration(client, batch) { + await client.query( + `update external_data_plane_delivery_state + set current_generation_at = $5, current_generation_id = $6, updated_at = now() + where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`, + [ + batch.source.tenantId, + batch.source.connectionId, + batch.source.providerId, + batch.contract.dataProductId, + batch.batch.generationAt, + batch.batch.runId, + ], + ); +} + async function persistHistory(client, batch, policy, facts) { if (!facts.length || policy?.mode === "none") return 0; const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1; @@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) { select $1, $2, $3, $4, item.source_id, item.semantic_type, item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), case when item.geometry is null then null - else ST_SetSRID(ST_MakePoint( - (item.geometry->'coordinates'->>0)::double precision, - (item.geometry->'coordinates'->>1)::double precision - ), 4326)::geography end, + else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end, item.fingerprint, now() from jsonb_to_recordset($5::jsonb) as item( source_id text, semantic_type text, bucket_start timestamptz, @@ -570,6 +634,7 @@ function normalizeReceipt(value) { idempotent: value.idempotent === true, publishedFactCount: Number(value.publishedFactCount || 0), currentUpdatedCount: Number(value.currentUpdatedCount || 0), + currentRemovedCount: Number(value.currentRemovedCount || 0), historyInsertedCount: Number(value.historyInsertedCount || 0), patchOperationCount: Number(value.patchOperationCount || 0), cursor: String(value.cursor || "0"), @@ -652,6 +717,8 @@ function publishFingerprint(batch) { runId: batch.batch.runId, sequence: batch.batch.sequence, idempotencyKey: batch.batch.idempotencyKey, + mode: batch.batch.mode || "upsert", + ...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}), }, facts: batch.facts, }); @@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) { function fieldContractValueMatchesType(value, type) { if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string"); - if (type === "point") { - return value?.type === "Point" - && Array.isArray(value.coordinates) - && value.coordinates.length === 2 - && value.coordinates.every(Number.isFinite) - && value.coordinates[0] >= -180 - && value.coordinates[0] <= 180 - && value.coordinates[1] >= -90 - && value.coordinates[1] <= 90; - } + if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"])); + if (type === "geometry") return isBoundedGeoJsonGeometry(value); return typeof value === type && (type !== "number" || Number.isFinite(value)); } diff --git a/services/external-data-plane/src/data-product-policy.mjs b/services/external-data-plane/src/data-product-policy.mjs index 9c13a34..ee21dee 100644 --- a/services/external-data-plane/src/data-product-policy.mjs +++ b/services/external-data-plane/src/data-product-policy.mjs @@ -1,8 +1,10 @@ +import { isBoundedGeoJsonGeometry } from "@nodedc/external-provider-contract/data-plane"; + const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); const HISTORY_MODES = new Set(["none", "all", "sampled"]); -const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]); +const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]); const DEFINITION_KEYS = new Set([ "id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history", ]); @@ -63,7 +65,7 @@ function normalizeFieldContract(value) { if (value.enum !== undefined) { if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length - || new Set(["point", "string_array"]).has(type) + || new Set(["point", "geometry", "string_array"]).has(type) || value.enum.some((item) => !fieldContractValueMatchesType(item, type))) { throw policyError("data_product_field_contract_enum_invalid"); } @@ -171,6 +173,9 @@ function hasOnlyKeys(value, allowed) { } function fieldContractValueMatchesType(value, type) { + if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string"); + if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"])); + if (type === "geometry") return isBoundedGeoJsonGeometry(value); return typeof value === type && (type !== "number" || Number.isFinite(value)); } diff --git a/services/external-data-plane/src/schema.mjs b/services/external-data-plane/src/schema.mjs index ffb3d66..77fe408 100644 --- a/services/external-data-plane/src/schema.mjs +++ b/services/external-data-plane/src/schema.mjs @@ -61,6 +61,7 @@ export async function migrate(pool) { `); await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)"); await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0"); + await pool.query("alter table external_data_plane_batches add column if not exists current_removed_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint"); @@ -245,7 +246,7 @@ export async function migrate(pool) { observed_at timestamptz not null, received_at timestamptz not null, attributes jsonb not null default '{}'::jsonb, - geometry geography(Point, 4326), + geometry geography(Geometry, 4326), fingerprint text not null, primary key (id, observed_at), unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint) @@ -266,12 +267,30 @@ export async function migrate(pool) { observed_at timestamptz not null, received_at timestamptz not null, attributes jsonb not null default '{}'::jsonb, - geometry geography(Point, 4326), + geometry geography(Geometry, 4326), fingerprint text not null, updated_at timestamptz not null default now(), primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type) ) `); + await pool.query(` + do $$ + begin + if exists ( + select 1 + from pg_attribute + where attrelid = 'external_data_plane_current'::regclass + and attname = 'geometry' + and not attisdropped + and format_type(atttypid, atttypmod) = 'geography(Point,4326)' + ) then + alter table external_data_plane_current + alter column geometry type geography(Geometry, 4326) + using geometry::geometry::geography; + end if; + end + $$ + `); await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)"); await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)"); @@ -282,10 +301,14 @@ export async function migrate(pool) { provider_id text not null, data_product_id text not null, current_cursor bigint not null default 0, + current_generation_at timestamptz, + current_generation_id text, updated_at timestamptz not null default now(), primary key (tenant_id, connection_id, provider_id, data_product_id) ) `); + await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_at timestamptz"); + await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_id text"); await pool.query(` create table if not exists external_data_plane_patch_outbox ( @@ -316,7 +339,7 @@ export async function migrate(pool) { observed_at timestamptz not null, received_at timestamptz not null, attributes jsonb not null default '{}'::jsonb, - geometry geography(Point, 4326), + geometry geography(Geometry, 4326), fingerprint text not null, updated_at timestamptz not null default now(), primary key ( diff --git a/services/external-data-plane/src/writer-binding.mjs b/services/external-data-plane/src/writer-binding.mjs index 55fa97d..285df40 100644 --- a/services/external-data-plane/src/writer-binding.mjs +++ b/services/external-data-plane/src/writer-binding.mjs @@ -208,6 +208,10 @@ export function materializeDataProductPublish(value, binding, definition, dataPr sequence: value.batch?.sequence, idempotencyKey: value.batch?.idempotencyKey, receivedAt: now.toISOString(), + ...(value.batch?.mode === "replace" ? { + mode: "replace", + generationAt: value.batch?.generationAt, + } : {}), }, facts: value.facts, }; diff --git a/services/external-data-plane/test/data-product-delivery.integration.test.mjs b/services/external-data-plane/test/data-product-delivery.integration.test.mjs index a131f76..09df3c8 100644 --- a/services/external-data-plane/test/data-product-delivery.integration.test.mjs +++ b/services/external-data-plane/test/data-product-delivery.integration.test.mjs @@ -205,6 +205,54 @@ try { (error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid", ); + const zoneDefinition = normalizeDataProductDefinition({ + id: "test.zones.current.v1", + version: "1.0.0", + ontologyRevision: "ontology.map.zone.v1", + deliveryMode: "snapshot+patch", + semanticTypes: ["map.zone"], + fields: ["display_name", "geometry", "geometry_kind", "source_kind", "source_revision"], + fieldContracts: { + display_name: { type: "string", required: true }, + geometry: { type: "geometry", required: true }, + geometry_kind: { type: "string", required: true, enum: ["polygon"] }, + source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] }, + source_revision: { type: "string", required: true }, + }, + history: { mode: "none", retentionDays: 1 }, + }); + await persistDataProductDefinition(pool, zoneDefinition); + const storedZoneDefinition = await loadDataProductDefinition(pool, zoneDefinition.id); + const zoneBinding = { ...binding, allowedDataProductIds: [zoneDefinition.id] }; + const firstZoneGenerationAt = "2026-07-15T10:10:00.000Z"; + const firstZoneGeneration = await persistDataProductPublish( + pool, + zoneBatch("zones-01", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt), zoneFact("zone-02", firstZoneGenerationAt)]), + storedZoneDefinition, + ); + assert.equal(firstZoneGeneration.currentUpdatedCount, 2); + assert.equal(firstZoneGeneration.currentRemovedCount, 0); + const secondZoneGenerationAt = "2026-07-15T10:11:00.000Z"; + const secondZoneGeneration = await persistDataProductPublish( + pool, + zoneBatch("zones-02", secondZoneGenerationAt, [zoneFact("zone-01", secondZoneGenerationAt)]), + storedZoneDefinition, + ); + assert.equal(secondZoneGeneration.currentUpdatedCount, 1); + assert.equal(secondZoneGeneration.currentRemovedCount, 1); + assert.equal(secondZoneGeneration.patchOperationCount, 2); + const zoneSnapshot = await readDataProductSnapshot(pool, zoneBinding, storedZoneDefinition); + assert.equal(validateDataProductSnapshot(zoneSnapshot).ok, true); + assert.deepEqual(zoneSnapshot.facts.map((value) => value.sourceId), ["zone-01"]); + assert.equal(zoneSnapshot.facts[0].geometry.type, "Polygon"); + const zonePatches = await readPatchEvents(pool, zoneBinding, storedZoneDefinition, 0n); + assert.equal(zonePatches.every((value) => validateDataProductPatch(value).ok), true); + assert.equal(zonePatches.flatMap((value) => value.operations).some((value) => value.op === "remove" && value.sourceId === "zone-02"), true); + await assert.rejects( + persistDataProductPublish(pool, zoneBatch("zones-stale", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt)]), storedZoneDefinition), + (error) => error?.status === 409 && error?.code === "data_product_generation_not_newer", + ); + console.log("external-data-plane delivery integration: ok"); } finally { await pool.end(); @@ -233,3 +281,38 @@ function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10 geometry: { type: "Point", coordinates: [longitude, latitude] }, }; } + +function zoneBatch(runId, generationAt, facts) { + return { + schemaVersion: "nodedc.external-provider-contract/v1", + source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" }, + contract: { dataProductId: "test.zones.current.v1", ontologyRevision: "ontology.map.zone.v1", version: "1.0.0" }, + batch: { + runId, + sequence: 0, + idempotencyKey: `${runId}.chunk-0`, + receivedAt: generationAt, + mode: "replace", + generationAt, + }, + facts, + }; +} + +function zoneFact(sourceId, observedAt) { + return { + sourceId, + semanticType: "map.zone", + observedAt, + attributes: { + display_name: sourceId, + geometry_kind: "polygon", + source_kind: "live_api", + source_revision: "gelios-rest-v1", + }, + geometry: { + type: "Polygon", + coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]], + }, + }; +} diff --git a/services/external-data-plane/test/definitions.test.mjs b/services/external-data-plane/test/definitions.test.mjs index e0b0874..dd7f1b9 100644 --- a/services/external-data-plane/test/definitions.test.mjs +++ b/services/external-data-plane/test/definitions.test.mjs @@ -11,6 +11,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [ "fleet.positions.current.v2", "fleet.positions.current.v3", "fleet.positions.current.v4", + "map.zones.current.v1", ]); assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]); assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1"); @@ -45,6 +46,10 @@ assert.deepEqual(bundled[3].fieldContracts.movement_state, { required: true, enum: ["moving", "stopped"], }); +assert.equal(bundled[4].version, "1.0.0"); +assert.equal(bundled[4].ontologyRevision, "ontology.map.zone.v1"); +assert.deepEqual(bundled[4].semanticTypes, ["map.zone"]); +assert.deepEqual(bundled[4].fieldContracts.geometry, { type: "geometry", required: true }); assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length); const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));