feat(platform): add replaceable geozone data layer

This commit is contained in:
Codex 2026-07-20 21:48:27 +03:00
parent 8a7465cf0e
commit 3446f3edc2
31 changed files with 1342 additions and 128 deletions

View File

@ -90,11 +90,11 @@ node descriptions used `usableAsTool`, so n8n 2.3.2 exposed six NDC runtime
types instead of the required three. It must not be activated, overwritten or types instead of the required three. It must not be activated, overwritten or
deleted. deleted.
The active predecessor is package version `0.1.2` at immutable release The active predecessor is package version `0.1.4` at immutable release
`0.1.2-05e4b38b14b4a019`. History Read is package version `0.1.3`, built with `0.1.4-59dc9f7882721d6a`. Package `0.1.5` adds the provider-neutral complete
patch id `n8n-nodes-ndc-history-read-20260717-004` and immutable release snapshot replace mode used by `map.zones.current.v1`; its Platform staging
`0.1.3-3354149b5245e39a`. Staging remains inert; only the separately reviewed remains inert until a separately reviewed Engine-owned transition selects the
Engine-owned transition may select it after exact MCP schema acceptance. exact digest and accepts the updated MCP node schema.
The paired Engine activation is built by The paired Engine activation is built by
`build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not `build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not

View File

@ -22,10 +22,12 @@ const files = [
["packages/external-provider-contract/src/contract-version.mjs", "platform/packages/external-provider-contract/src/contract-version.mjs"], ["packages/external-provider-contract/src/contract-version.mjs", "platform/packages/external-provider-contract/src/contract-version.mjs"],
["packages/external-provider-contract/src/data-plane.mjs", "platform/packages/external-provider-contract/src/data-plane.mjs"], ["packages/external-provider-contract/src/data-plane.mjs", "platform/packages/external-provider-contract/src/data-plane.mjs"],
["packages/external-provider-contract/src/data-product.mjs", "platform/packages/external-provider-contract/src/data-product.mjs"], ["packages/external-provider-contract/src/data-product.mjs", "platform/packages/external-provider-contract/src/data-product.mjs"],
["packages/external-provider-contract/src/geometry.mjs", "platform/packages/external-provider-contract/src/geometry.mjs"],
["packages/external-provider-contract/src/intake-batch.mjs", "platform/packages/external-provider-contract/src/intake-batch.mjs"], ["packages/external-provider-contract/src/intake-batch.mjs", "platform/packages/external-provider-contract/src/intake-batch.mjs"],
["packages/external-provider-contract/src/index.mjs", "platform/packages/external-provider-contract/src/index.mjs"], ["packages/external-provider-contract/src/index.mjs", "platform/packages/external-provider-contract/src/index.mjs"],
["packages/external-provider-contract/src/provider-package.mjs", "platform/packages/external-provider-contract/src/provider-package.mjs"], ["packages/external-provider-contract/src/provider-package.mjs", "platform/packages/external-provider-contract/src/provider-package.mjs"],
["packages/external-provider-contract/src/sensitive-field-policy.mjs", "platform/packages/external-provider-contract/src/sensitive-field-policy.mjs"], ["packages/external-provider-contract/src/sensitive-field-policy.mjs", "platform/packages/external-provider-contract/src/sensitive-field-policy.mjs"],
["packages/external-provider-contract/src/zone-source.mjs", "platform/packages/external-provider-contract/src/zone-source.mjs"],
["packages/external-provider-contract/providers/gelios", "platform/packages/external-provider-contract/providers/gelios"], ["packages/external-provider-contract/providers/gelios", "platform/packages/external-provider-contract/providers/gelios"],
]; ];
const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]); const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]);

View File

@ -12,8 +12,8 @@ const platformRoot = resolve(scriptDir, "../..");
const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc"); const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc");
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts")); const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
const requireModule = createRequire(import.meta.url); const requireModule = createRequire(import.meta.url);
const expectedPackageVersion = "0.1.4"; const expectedPackageVersion = "0.1.5";
const [patchId = "n8n-nodes-ndc-provider-rotating-access-20260718-006", ...extra] = process.argv.slice(2); const [patchId = "n8n-nodes-ndc-geozone-replace-20260720-017", ...extra] = process.argv.slice(2);
const expectedRuntimeNodes = [ const expectedRuntimeNodes = [
{ {
file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",

View File

@ -19,7 +19,7 @@ RUNNER_PATH = Path(
os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy") os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy")
).resolve() ).resolve()
BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs" BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs"
PATCH_ID = "n8n-nodes-ndc-provider-rotating-access-20260718-006" PATCH_ID = "n8n-nodes-ndc-geozone-replace-20260720-017"
EXPECTED_NODES = [ EXPECTED_NODES = [
"dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js", "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
"dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js", "dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js",
@ -110,7 +110,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory)) manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory))
self.assertEqual(manifest["component"], "n8n-private-extension") self.assertEqual(manifest["component"], "n8n-private-extension")
self.assertEqual(len(entries), 1) self.assertEqual(len(entries), 1)
self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.4-[a-f0-9]{16}$") self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.5-[a-f0-9]{16}$")
with tarfile.open(self.artifacts[0], "r:gz") as archive: with tarfile.open(self.artifacts[0], "r:gz") as archive:
names = archive.getnames() names = archive.getnames()
@ -122,7 +122,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive: with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive:
package_json_member = archive.getmember("package/package.json") package_json_member = archive.getmember("package/package.json")
package_json = json.loads(archive.extractfile(package_json_member).read()) package_json = json.loads(archive.extractfile(package_json_member).read())
self.assertEqual(package_json["version"], "0.1.4") self.assertEqual(package_json["version"], "0.1.5")
self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES) self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES)
self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS) self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS)
for node_path in EXPECTED_NODES: for node_path in EXPECTED_NODES:
@ -158,7 +158,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
"requiresPreActivationVerification": True, "requiresPreActivationVerification": True,
} }
self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2") self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2")
self.assertEqual(release["package"]["version"], "0.1.4") self.assertEqual(release["package"]["version"], "0.1.5")
self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy) self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy)
self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2") self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2")
self.assertEqual(rollback["baselinePolicy"], expected_policy) self.assertEqual(rollback["baselinePolicy"], expected_policy)

View File

@ -9,6 +9,6 @@
"./providers/*": "./providers/*/index.mjs" "./providers/*": "./providers/*/index.mjs"
}, },
"scripts": { "scripts": {
"check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/engine-private-extension.test.mjs" "check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/zone-source.test.mjs && node test/engine-private-extension.test.mjs"
} }
} }

View File

@ -0,0 +1,13 @@
# Gelios provider package v6
Adds the bounded, offset-paged `gelios.geozone` read capability and maps a
complete accepted source generation into the provider-neutral
`map.zones.current.v1` Data Product. The runtime publisher must use replace
mode: a partial provider page set never reaches Publish, while a complete newer
generation atomically upserts present zones and emits remove patches for zones
that disappeared.
`gelios-rest-v1` and `mmap-snapshot-v1` are source adapters, not Foundry
bindings. Both must preserve the native Gelios zone id (or use an explicit
crosswalk) and produce the same `map.zone` facts. Foundry only consumes the
provider-neutral product.

View File

@ -0,0 +1,9 @@
export {
GELIOS_PROVIDER_PACKAGE_ID,
GELIOS_PROVIDER_PACKAGE_VERSION,
GELIOS_ZONES_DATA_PRODUCT_ID,
GELIOS_ZONES_DATA_PRODUCT_VERSION,
GELIOS_ZONES_ONTOLOGY_REVISION,
GELIOS_ZONE_SOURCE_ID_PREFIX,
geliosProviderPackageV6,
} from "./package.mjs";

View File

@ -0,0 +1,192 @@
import { geliosProviderPackageV5 } from "../v5/package.mjs";
export const GELIOS_PROVIDER_PACKAGE_ID = "gelios.provider.v6";
export const GELIOS_PROVIDER_PACKAGE_VERSION = "6.0.0";
export const GELIOS_ZONES_DATA_PRODUCT_ID = "map.zones.current.v1";
export const GELIOS_ZONES_DATA_PRODUCT_VERSION = "1.0.0";
export const GELIOS_ZONES_ONTOLOGY_REVISION = "ontology.map.zone.v1";
export const GELIOS_ZONE_SOURCE_ID_PREFIX = "gelios-zone-";
const ZONES_CAPABILITY_ID = "gelios.geozones.current.read";
const ZONES_FIELD_POLICY_ID = "gelios.geozones.current.fields.v1";
const ZONES_PROFILE_ID = "gelios.geozones.current.realtime.v1";
const ZONES_MAPPING_ID = "gelios.geozones.to.map.zones.current.v1";
const ZONES_TEMPLATE_ID = "gelios.geozones.current.l2.v1";
const value = structuredClone(geliosProviderPackageV5);
const authModeId = value.authModes[0].id;
const zonesFields = [
"area_square_meters",
"description",
"display_name",
"geometry",
"geometry_kind",
"max_speed_kph",
"perimeter_meters",
"source_kind",
"source_revision",
"style_color",
];
value.id = GELIOS_PROVIDER_PACKAGE_ID;
value.version = GELIOS_PROVIDER_PACKAGE_VERSION;
value.manifest = {
...value.manifest,
id: "gelios.provider.manifest.v6",
version: GELIOS_PROVIDER_PACKAGE_VERSION,
capabilityIds: [...value.manifest.capabilityIds, ZONES_CAPABILITY_ID],
fieldPolicyIds: [...value.manifest.fieldPolicyIds, ZONES_FIELD_POLICY_ID],
collectionProfileIds: [...value.manifest.collectionProfileIds, ZONES_PROFILE_ID],
dataProductIds: [...value.manifest.dataProductIds, GELIOS_ZONES_DATA_PRODUCT_ID],
mappingContractIds: [...value.manifest.mappingContractIds, ZONES_MAPPING_ID],
l2TemplateIds: [...value.manifest.l2TemplateIds, ZONES_TEMPLATE_ID],
};
value.capabilities.push({
id: ZONES_CAPABILITY_ID,
classification: "read",
status: "implemented",
authModeId,
request: {
method: "GET",
baseUrl: "https://api.geliospro.com",
path: "/api/v1/geozones",
query: { pl: 100, po: 0 },
response: {
collectionPaths: ["items"],
pagination: {
mode: "offset",
limitParameter: "pl",
offsetParameter: "po",
pageSize: 100,
itemsPath: "items",
totalPath: "paginationMetadata.totalCount",
maxPages: 50,
maxItems: 5000,
maxResponseBytes: 16 * 1024 * 1024,
},
},
},
entityScope: {
mode: "all_visible_to_credential",
refresh: "each_collection_run",
businessEntityFilter: "forbidden",
},
});
value.fieldPolicies.push({
id: ZONES_FIELD_POLICY_ID,
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
targetFields: [...zonesFields],
unknownSourceFields: "drop",
dynamicSourceFields: "drop_until_classified",
restrictedSourcePaths: ["accessToken", "authorization", "refreshToken", "userToken"],
});
value.collectionProfiles.push({
id: ZONES_PROFILE_ID,
version: GELIOS_PROVIDER_PACKAGE_VERSION,
mode: "realtime",
schedule: { intervalMs: 3_600_000 },
capabilityIds: [ZONES_CAPABILITY_ID],
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
mappingContractId: ZONES_MAPPING_ID,
fieldPolicyId: ZONES_FIELD_POLICY_ID,
l2TemplateId: ZONES_TEMPLATE_ID,
entityScope: {
mode: "all_visible_to_credential",
refresh: "each_collection_run",
businessEntityFilter: "forbidden",
},
batching: { maxFacts: 5000 },
cardinality: { maxCurrentEntities: 5000, onExceed: "require_partitioned_data_product" },
retry: { maxAttempts: 4, backoff: "exponential_with_jitter" },
});
value.dataProducts.push({
id: GELIOS_ZONES_DATA_PRODUCT_ID,
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION,
deliveryMode: "snapshot+patch",
semanticTypes: ["map.zone"],
fields: [...zonesFields],
fieldContracts: {
area_square_meters: { type: "number", required: false, minimum: 0 },
description: { type: "string", required: false },
display_name: { type: "string", required: true },
geometry: { type: "geometry", required: true },
geometry_kind: { type: "string", required: true, enum: ["circle", "corridor", "polygon"] },
max_speed_kph: { type: "number", required: false, minimum: 0 },
perimeter_meters: { type: "number", required: false, minimum: 0 },
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
source_revision: { type: "string", required: true },
style_color: { type: "string", required: false },
},
history: { mode: "none", retentionDays: 1 },
});
value.mappingContracts.push({
schemaVersion: "nodedc.semantic-mapping/v1",
id: ZONES_MAPPING_ID,
version: GELIOS_PROVIDER_PACKAGE_VERSION,
sourceCapabilityId: ZONES_CAPABILITY_ID,
fieldPolicyId: ZONES_FIELD_POLICY_ID,
target: {
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION,
semanticType: "map.zone",
},
derivations: {
geometry_kind: {
kind: "ordered_rules",
rules: ["type_circle.circle", "type_line.corridor", "otherwise.polygon"],
default: "polygon",
},
},
fact: {
sourceId: { strategy: "first_non_empty", paths: ["id"], coerce: "string", prefix: GELIOS_ZONE_SOURCE_ID_PREFIX },
semanticType: { constant: "map.zone" },
observedAt: { strategy: "first_non_empty", paths: ["updatedAt"], coerce: "unix_or_iso_timestamp", fallback: "collection_received_at" },
geometry: {
type: "GeoJSON",
strategy: "gelios_geozone_v1",
paths: ["type", "points", "line", "radius"],
allowedTypes: ["Polygon", "MultiPolygon"],
omitIfInvalid: false,
},
attributes: {
area_square_meters: { strategy: "first_non_empty", paths: ["surfaceArea", "surface_area"], coerce: "number", minimum: 0, omitIfInvalid: true },
description: { strategy: "first_non_empty", paths: ["description", "descr"], coerce: "string", omitIfMissing: true },
display_name: { strategy: "first_non_empty", paths: ["name"], coerce: "string" },
geometry_kind: { derive: "geometry_kind" },
max_speed_kph: { strategy: "first_non_empty", paths: ["maxPermissibleSpeed", "max_permissible_speed"], coerce: "number", minimum: 0, omitIfInvalid: true },
perimeter_meters: { strategy: "first_non_empty", paths: ["perimeter"], coerce: "number", minimum: 0, omitIfInvalid: true },
source_kind: { constant: "live_api" },
source_revision: { constant: "gelios-rest-v1" },
style_color: { strategy: "first_non_empty", paths: ["color"], coerce: "string", omitIfMissing: true },
},
},
});
value.l2Templates.push({
schemaVersion: "nodedc.l2-template/v1",
id: ZONES_TEMPLATE_ID,
version: GELIOS_PROVIDER_PACKAGE_VERSION,
runtime: "ndc_l2",
instanceMode: "one_connection_per_workflow",
connectionParameters: ["tenant_id", "connection_id", "collection_profile_id", "provider_credential_ref"],
credentialBindings: structuredClone(value.l2Templates[0].credentialBindings),
steps: [
{ id: "collection.trigger", kind: "collection_trigger", collectionProfileDriven: true },
{ id: "provider.fetch-geozones", kind: "provider_request", capabilityId: ZONES_CAPABILITY_ID },
{ id: "provider.extract-geozones", kind: "extract_items", capabilityId: ZONES_CAPABILITY_ID },
{ id: "ontology.map", kind: "semantic_mapping", mappingContractId: ZONES_MAPPING_ID },
{ id: "data-product.publish", kind: "data_product_publish", dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, nodeType: "n8n-nodes-ndc.ndcDataProductPublish" },
],
invariants: structuredClone(value.l2Templates[0].invariants),
});
export const geliosProviderPackageV6 = deepFreeze(value);
function deepFreeze(input) {
if (!input || typeof input !== "object" || Object.isFrozen(input)) return input;
Object.freeze(input);
for (const child of Object.values(input)) deepFreeze(child);
return input;
}

View File

@ -1,5 +1,12 @@
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs"; export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
export { validateIntakeBatch } from "./intake-batch.mjs"; export { validateIntakeBatch } from "./intake-batch.mjs";
export {
DATA_PRODUCT_GEOMETRY_TYPES,
DATA_PRODUCT_MAX_GEOMETRY_BYTES,
DATA_PRODUCT_MAX_GEOMETRY_VERTICES,
isBoundedGeoJsonGeometry,
validateGeoJsonGeometry,
} from "./geometry.mjs";
export { export {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION, DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION, DATA_PRODUCT_PATCH_SCHEMA_VERSION,

View File

@ -4,6 +4,7 @@ export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1"; export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs"; import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
import { validateGeoJsonGeometry } from "./geometry.mjs";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
@ -30,22 +31,33 @@ export function validateDataProductPublish(value, { maxFacts = 5000, maxAttribut
if (!isPlainObject(value.batch)) { if (!isPlainObject(value.batch)) {
errors.push("batch_must_be_object"); errors.push("batch_must_be_object");
} else { } else {
rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey"]), "batch", errors); rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey", "mode", "generationAt"]), "batch", errors);
requiredIdentifier(value.batch.runId, "batch.runId", errors); requiredIdentifier(value.batch.runId, "batch.runId", errors);
requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors); requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors);
if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) { if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) {
errors.push("batch.sequence_must_be_integer_0_to_2147483647"); errors.push("batch.sequence_must_be_integer_0_to_2147483647");
} }
const mode = value.batch.mode === undefined ? "upsert" : value.batch.mode;
if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid");
if (mode === "replace") requiredIsoTimestamp(value.batch.generationAt, "batch.generationAt", errors);
if (mode === "upsert" && value.batch.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert");
} }
if (!Array.isArray(value.facts) || value.facts.length === 0) { if (!Array.isArray(value.facts)) {
errors.push("facts_must_be_nonempty_array"); errors.push("facts_must_be_array");
} else if (value.facts.length === 0 && value.batch?.mode !== "replace") {
errors.push("facts_must_be_nonempty_array_for_upsert");
} else if (value.facts.length > maxFacts) { } else if (value.facts.length > maxFacts) {
errors.push("facts_limit_exceeded"); errors.push("facts_limit_exceeded");
} else { } else {
const entityKeys = new Set(); const entityKeys = new Set();
value.facts.forEach((fact, index) => { value.facts.forEach((fact, index) => {
validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling }); validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling });
if (value.batch?.mode === "replace"
&& !Number.isNaN(Date.parse(fact?.observedAt))
&& Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) {
errors.push(`facts[${index}].observedAt_must_equal_generationAt`);
}
if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return; if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return;
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`; const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key"); if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key");
@ -143,12 +155,18 @@ export function validateDataProductPatch(value) {
errors.push("operations_must_be_nonempty_array"); errors.push("operations_must_be_nonempty_array");
} else { } else {
value.operations.forEach((operation, index) => { value.operations.forEach((operation, index) => {
if (!isPlainObject(operation) || operation.op !== "upsert") { if (!isPlainObject(operation) || !new Set(["upsert", "remove"]).has(operation.op)) {
errors.push(`operations[${index}].op_must_be_upsert`); errors.push(`operations[${index}].op_invalid`);
return; return;
} }
rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors); if (operation.op === "upsert") {
validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors); rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors);
validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors);
} else {
rejectUnknownKeys(operation, new Set(["op", "sourceId", "semanticType"]), `operations[${index}]`, errors);
requiredIdentifier(operation.sourceId, `operations[${index}].sourceId`, errors);
requiredIdentifier(operation.semanticType, `operations[${index}].semanticType`, errors);
}
}); });
} }
if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material"); if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material");
@ -189,7 +207,7 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal
errors.push(`${path}.attributes_size_exceeded`); errors.push(`${path}.attributes_size_exceeded`);
} }
} }
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors);
} }
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) { function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
@ -198,21 +216,6 @@ function validateCanonicalFact(value, path, errors, { allowBucketStart = false }
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors); requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
} }
function validatePointGeometry(value, path, errors) {
if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) {
errors.push(`${path}_must_be_geojson_point`);
return;
}
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) {
errors.push(`${path}_coordinates_must_be_finite_numbers`);
return;
}
const [longitude, latitude] = value.coordinates;
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
}
function rejectUnknownKeys(value, allowed, path, errors) { function rejectUnknownKeys(value, allowed, path, errors) {
if (!isPlainObject(value)) return; if (!isPlainObject(value)) return;
for (const key of Object.keys(value)) { for (const key of Object.keys(value)) {

View File

@ -0,0 +1,103 @@
const GEOMETRY_TYPES = new Set(["Point", "LineString", "Polygon", "MultiPolygon"]);
const MAX_GEOMETRY_BYTES = 192 * 1024;
const MAX_GEOMETRY_VERTICES = 10_000;
/**
* Validate the bounded GeoJSON geometry subset accepted by provider-neutral
* Data Products. GeometryCollections and arbitrary feature properties are
* deliberately excluded: a fact owns exactly one geometry and its attributes
* stay under the fact field policy.
*/
export function validateGeoJsonGeometry(value, path, errors, { allowedTypes = GEOMETRY_TYPES } = {}) {
if (!isPlainObject(value) || !allowedTypes.has(value.type) || !Array.isArray(value.coordinates)) {
errors.push(`${path}_must_be_bounded_geojson_geometry`);
return;
}
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
if (serializedByteLength(value) > MAX_GEOMETRY_BYTES) {
errors.push(`${path}_size_exceeded`);
return;
}
const vertices = [];
if (value.type === "Point") {
validatePosition(value.coordinates, path, errors, vertices);
} else if (value.type === "LineString") {
validateLineString(value.coordinates, path, errors, vertices);
} else if (value.type === "Polygon") {
validatePolygon(value.coordinates, path, errors, vertices);
} else if (value.type === "MultiPolygon") {
if (!value.coordinates.length) errors.push(`${path}.coordinates_must_be_nonempty`);
value.coordinates.forEach((polygon, index) => validatePolygon(polygon, `${path}.coordinates[${index}]`, errors, vertices));
}
if (vertices.length > MAX_GEOMETRY_VERTICES) errors.push(`${path}_vertex_limit_exceeded`);
}
export function isBoundedGeoJsonGeometry(value, allowedTypes = GEOMETRY_TYPES) {
const errors = [];
validateGeoJsonGeometry(value, "geometry", errors, { allowedTypes });
return errors.length === 0;
}
function validatePolygon(value, path, errors, vertices) {
if (!Array.isArray(value) || !value.length) {
errors.push(`${path}_polygon_rings_required`);
return;
}
value.forEach((ring, index) => {
const ringPath = `${path}[${index}]`;
if (!Array.isArray(ring) || ring.length < 4) {
errors.push(`${ringPath}_linear_ring_requires_four_positions`);
return;
}
ring.forEach((position, positionIndex) => validatePosition(position, `${ringPath}[${positionIndex}]`, errors, vertices));
if (!samePosition(ring[0], ring.at(-1))) errors.push(`${ringPath}_linear_ring_must_be_closed`);
});
}
function validateLineString(value, path, errors, vertices) {
if (!Array.isArray(value) || value.length < 2) {
errors.push(`${path}_line_string_requires_two_positions`);
return;
}
value.forEach((position, index) => validatePosition(position, `${path}[${index}]`, errors, vertices));
}
function validatePosition(value, path, errors, vertices) {
if (!Array.isArray(value) || value.length !== 2 || !value.every(Number.isFinite)) {
errors.push(`${path}_position_invalid`);
return;
}
const [longitude, latitude] = value;
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
vertices.push(value);
}
function samePosition(left, right) {
return Array.isArray(left) && Array.isArray(right)
&& left.length === 2 && right.length === 2
&& Object.is(left[0], right[0]) && Object.is(left[1], right[1]);
}
function rejectUnknownKeys(value, allowed, path, errors) {
for (const key of Object.keys(value)) {
if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`);
}
}
function serializedByteLength(value) {
try {
return Buffer.byteLength(JSON.stringify(value));
} catch {
return Number.POSITIVE_INFINITY;
}
}
function isPlainObject(value) {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
export const DATA_PRODUCT_GEOMETRY_TYPES = Object.freeze([...GEOMETRY_TYPES]);
export const DATA_PRODUCT_MAX_GEOMETRY_BYTES = MAX_GEOMETRY_BYTES;
export const DATA_PRODUCT_MAX_GEOMETRY_VERTICES = MAX_GEOMETRY_VERTICES;

View File

@ -2,6 +2,18 @@ import { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs"; export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
export { validateIntakeBatch } from "./intake-batch.mjs"; export { validateIntakeBatch } from "./intake-batch.mjs";
export {
DATA_PRODUCT_GEOMETRY_TYPES,
DATA_PRODUCT_MAX_GEOMETRY_BYTES,
DATA_PRODUCT_MAX_GEOMETRY_VERTICES,
isBoundedGeoJsonGeometry,
validateGeoJsonGeometry,
} from "./geometry.mjs";
export {
ZONE_SOURCE_ADAPTERS,
ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
normalizeZoneSourceGeneration,
} from "./zone-source.mjs";
export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1"; export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1";
import { import {

View File

@ -4,6 +4,7 @@ import {
SECRET_LIKE_REFERENCE, SECRET_LIKE_REFERENCE,
SECRET_LIKE_VALUE, SECRET_LIKE_VALUE,
} from "./sensitive-field-policy.mjs"; } from "./sensitive-field-policy.mjs";
import { validateGeoJsonGeometry } from "./geometry.mjs";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
@ -16,7 +17,7 @@ export function validateIntakeBatch(value) {
rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors); rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors);
rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors); rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors);
rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors); rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors);
rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt"]), "batch", errors); rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt", "mode", "generationAt"]), "batch", errors);
requiredIdentifier(value?.source?.providerId, "source.providerId", errors); requiredIdentifier(value?.source?.providerId, "source.providerId", errors);
requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors); requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors);
requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors); requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors);
@ -32,11 +33,24 @@ export function validateIntakeBatch(value) {
errors.push("batch.sequence_must_be_integer_0_to_2147483647"); errors.push("batch.sequence_must_be_integer_0_to_2147483647");
} }
requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors); requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors);
const mode = value?.batch?.mode === undefined ? "upsert" : value.batch.mode;
if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid");
if (mode === "replace") requiredIsoTimestamp(value?.batch?.generationAt, "batch.generationAt", errors);
if (mode === "upsert" && value?.batch?.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert");
if (!Array.isArray(value?.facts) || value.facts.length === 0) { if (!Array.isArray(value?.facts)) {
errors.push("facts_must_be_nonempty_array"); errors.push("facts_must_be_array");
} else if (value.facts.length === 0 && mode !== "replace") {
errors.push("facts_must_be_nonempty_array_for_upsert");
} else { } else {
value.facts.forEach((fact, index) => validateFact(fact, `facts[${index}]`, errors)); value.facts.forEach((fact, index) => {
validateFact(fact, `facts[${index}]`, errors);
if (mode === "replace"
&& !Number.isNaN(Date.parse(fact?.observedAt))
&& Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) {
errors.push(`facts[${index}].observedAt_must_equal_generationAt`);
}
});
} }
if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors); if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors);
@ -80,22 +94,7 @@ function validateFact(value, path, errors) {
errors.push(`${path}.attributes_size_exceeded`); errors.push(`${path}.attributes_size_exceeded`);
} }
} }
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors); if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors);
}
function validatePointGeometry(value, path, errors) {
if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) {
errors.push(`${path}_must_be_geojson_point`);
return;
}
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) {
errors.push(`${path}_coordinates_must_be_finite_numbers`);
return;
}
const [longitude, latitude] = value.coordinates;
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
} }
function validateRawEnvelope(value, errors) { function validateRawEnvelope(value, errors) {

View File

@ -20,7 +20,7 @@ const TOKEN_REFRESH_MODES = new Set(["not_applicable", "operator_managed", "runt
const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]); const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]);
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
const HISTORY_MODES = new Set(["none", "all", "sampled"]); const HISTORY_MODES = new Set(["none", "all", "sampled"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]); const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
const L2_STEP_KINDS = new Set([ const L2_STEP_KINDS = new Set([
"collection_trigger", "collection_trigger",
"provider_request", "provider_request",
@ -84,6 +84,10 @@ const CAPABILITY_KEYS = new Set([
]); ]);
const REQUEST_KEYS = new Set(["method", "baseUrl", "path", "query", "response"]); const REQUEST_KEYS = new Set(["method", "baseUrl", "path", "query", "response"]);
const RESPONSE_KEYS = new Set(["collectionPaths", "pagination"]); const RESPONSE_KEYS = new Set(["collectionPaths", "pagination"]);
const OFFSET_PAGINATION_KEYS = new Set([
"mode", "limitParameter", "offsetParameter", "pageSize", "itemsPath", "totalPath",
"maxPages", "maxItems", "maxResponseBytes",
]);
const ENTITY_SCOPE_KEYS = new Set(["mode", "refresh", "businessEntityFilter"]); const ENTITY_SCOPE_KEYS = new Set(["mode", "refresh", "businessEntityFilter"]);
const FIELD_POLICY_KEYS = new Set([ const FIELD_POLICY_KEYS = new Set([
"id", "id",
@ -137,7 +141,9 @@ const EXPRESSION_KEYS = new Set([
"strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive", "strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive",
"omitIfMissing", "omitIfInvalid", "minimum", "maximum", "omitIfMissing", "omitIfInvalid", "minimum", "maximum",
]); ]);
const GEOMETRY_KEYS = new Set(["type", "longitude", "latitude", "omitIfInvalid"]); const GEOMETRY_KEYS = new Set([
"type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid",
]);
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]); const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
const TEMPLATE_KEYS = new Set([ const TEMPLATE_KEYS = new Set([
"schemaVersion", "schemaVersion",
@ -496,12 +502,50 @@ function validateCapability(value, path, errors) {
} else { } else {
rejectUnknownKeys(value.request.response, RESPONSE_KEYS, `${path}.request.response`, errors); rejectUnknownKeys(value.request.response, RESPONSE_KEYS, `${path}.request.response`, errors);
requiredCollectionPathArray(value.request.response.collectionPaths, `${path}.request.response.collectionPaths`, errors); requiredCollectionPathArray(value.request.response.collectionPaths, `${path}.request.response.collectionPaths`, errors);
if (value.request.response.pagination !== "single_bounded_response") errors.push(`${path}.request.response.pagination_invalid`); validateResponsePagination(value.request.response.pagination, value.request.query, `${path}.request.response.pagination`, errors);
} }
} }
validateCredentialVisibleScope(value.entityScope, `${path}.entityScope`, errors); validateCredentialVisibleScope(value.entityScope, `${path}.entityScope`, errors);
} }
function validateResponsePagination(value, query, path, errors) {
if (value === "single_bounded_response") return;
if (!isPlainObject(value)) {
errors.push(`${path}_invalid`);
return;
}
rejectUnknownKeys(value, OFFSET_PAGINATION_KEYS, path, errors);
if (value.mode !== "offset") errors.push(`${path}.mode_must_be_offset`);
for (const field of ["limitParameter", "offsetParameter"]) {
if (typeof value[field] !== "string" || !/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(value[field])) {
errors.push(`${path}.${field}_invalid`);
}
}
if (!Number.isInteger(value.pageSize) || value.pageSize < 1 || value.pageSize > 500) {
errors.push(`${path}.pageSize_invalid`);
}
if (!Number.isInteger(value.maxPages) || value.maxPages < 1 || value.maxPages > 1000) {
errors.push(`${path}.maxPages_invalid`);
}
if (!Number.isInteger(value.maxItems) || value.maxItems < 1 || value.maxItems > 5000) {
errors.push(`${path}.maxItems_invalid`);
}
if (!Number.isInteger(value.maxResponseBytes) || value.maxResponseBytes < 1024 || value.maxResponseBytes > 32 * 1024 * 1024) {
errors.push(`${path}.maxResponseBytes_invalid`);
}
for (const field of ["itemsPath", "totalPath"]) {
if (!isCanonicalSourcePath(value[field])) errors.push(`${path}.${field}_invalid`);
}
if (isPlainObject(query)) {
if (query[value.limitParameter] !== value.pageSize) errors.push(`${path}.pageSize_must_match_static_query_limit`);
if (query[value.offsetParameter] !== 0) errors.push(`${path}.offset_query_must_start_at_zero`);
}
if (Number.isInteger(value.maxPages) && Number.isInteger(value.pageSize)
&& Number.isInteger(value.maxItems) && value.maxPages * value.pageSize < value.maxItems) {
errors.push(`${path}.maxPages_cannot_cover_maxItems`);
}
}
function validateFieldPolicy(value, path, errors) { function validateFieldPolicy(value, path, errors) {
rejectUnknownKeys(value, FIELD_POLICY_KEYS, path, errors); rejectUnknownKeys(value, FIELD_POLICY_KEYS, path, errors);
requiredIdentifier(value.id, `${path}.id`, errors); requiredIdentifier(value.id, `${path}.id`, errors);
@ -619,10 +663,26 @@ function validateMappingContract(value, path, errors) {
errors.push(`${path}.fact.geometry_must_be_object`); errors.push(`${path}.fact.geometry_must_be_object`);
} else if (isPlainObject(value.fact.geometry)) { } else if (isPlainObject(value.fact.geometry)) {
rejectUnknownKeys(value.fact.geometry, GEOMETRY_KEYS, `${path}.fact.geometry`, errors); rejectUnknownKeys(value.fact.geometry, GEOMETRY_KEYS, `${path}.fact.geometry`, errors);
if (value.fact.geometry.type !== "Point") errors.push(`${path}.fact.geometry.type_must_be_Point`); if (value.fact.geometry.type === "Point") {
validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors); validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors);
validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors); validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors);
if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`); if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`);
if (value.fact.geometry.strategy !== undefined || value.fact.geometry.paths !== undefined || value.fact.geometry.allowedTypes !== undefined) {
errors.push(`${path}.fact.geometry.point_shape_invalid`);
}
} else if (value.fact.geometry.type === "GeoJSON") {
if (value.fact.geometry.strategy !== "gelios_geozone_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
requiredSourcePathArray(value.fact.geometry.paths, `${path}.fact.geometry.paths`, errors);
if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) {
errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`);
}
if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`);
if (value.fact.geometry.longitude !== undefined || value.fact.geometry.latitude !== undefined) {
errors.push(`${path}.fact.geometry.geojson_shape_invalid`);
}
} else {
errors.push(`${path}.fact.geometry.type_invalid`);
}
} }
if (value.fact.attributes !== undefined && !isPlainObject(value.fact.attributes)) { if (value.fact.attributes !== undefined && !isPlainObject(value.fact.attributes)) {
errors.push(`${path}.fact.attributes_must_be_object`); errors.push(`${path}.fact.attributes_must_be_object`);
@ -773,7 +833,13 @@ function validateMappingAgainstProduct(mapping, product, errors) {
for (const [field, contract] of Object.entries(contracts)) { for (const [field, contract] of Object.entries(contracts)) {
if (!isPlainObject(contract)) continue; if (!isPlainObject(contract)) continue;
if (field === "geometry") { if (field === "geometry") {
if (contract.type !== "point") errors.push(`${path}.fact.geometry_contract_must_be_point`); if (contract.type === "point" && mapping.fact?.geometry?.type !== "Point") {
errors.push(`${path}.fact.geometry_contract_must_be_point`);
}
if (contract.type === "geometry" && mapping.fact?.geometry?.type !== "GeoJSON") {
errors.push(`${path}.fact.geometry_contract_must_be_geojson`);
}
if (!new Set(["point", "geometry"]).has(contract.type)) errors.push(`${path}.fact.geometry_contract_invalid`);
if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) { if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) {
errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`); errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`);
} }
@ -805,7 +871,7 @@ function validateFieldContract(value, path, errors) {
if (value.enum !== undefined) { if (value.enum !== undefined) {
if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) { if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) {
errors.push(`${path}.enum_must_be_nonempty_unique_array`); errors.push(`${path}.enum_must_be_nonempty_unique_array`);
} else if (new Set(["point", "string_array"]).has(value.type) } else if (new Set(["point", "geometry", "string_array"]).has(value.type)
|| value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) { || value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) {
errors.push(`${path}.enum_value_type_invalid`); errors.push(`${path}.enum_value_type_invalid`);
} }
@ -880,6 +946,11 @@ function fieldContractValueMatchesType(value, type) {
&& value.coordinates.length === 2 && value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite); && value.coordinates.every(Number.isFinite);
} }
if (type === "geometry") {
return isPlainObject(value)
&& new Set(["Point", "LineString", "Polygon", "MultiPolygon"]).has(value.type)
&& Array.isArray(value.coordinates);
}
return typeof value === type && (type !== "number" || Number.isFinite(value)); return typeof value === type && (type !== "number" || Number.isFinite(value));
} }

View File

@ -0,0 +1,285 @@
import { createHash } from "node:crypto";
import { isBoundedGeoJsonGeometry } from "./geometry.mjs";
export const ZONE_SOURCE_GENERATION_SCHEMA_VERSION = "nodedc.zone-source-generation/v1";
export const ZONE_SOURCE_ADAPTERS = Object.freeze({
"gelios-rest-v1": Object.freeze({ sourceKind: "live_api", sourceRevision: "gelios-rest-v1" }),
"mmap-snapshot-v1": Object.freeze({ sourceKind: "versioned_snapshot", sourceRevision: "mmap-snapshot-v1" }),
});
/**
* Convert one already-complete source generation into map.zone facts. Fetching,
* credentials and page iteration stay in Engine; this function is the common
* adapter boundary used by both live REST and immutable snapshot sources.
*/
export function normalizeZoneSourceGeneration(value, {
identityCrosswalk = {},
maxZones = 5000,
maxBytes = 16 * 1024 * 1024,
circleSegments = 64,
} = {}) {
if (!isPlainObject(value) || value.schemaVersion !== ZONE_SOURCE_GENERATION_SCHEMA_VERSION) {
throw zoneSourceError("zone_source_generation_schema_invalid");
}
const adapter = ZONE_SOURCE_ADAPTERS[value.adapterId];
if (!adapter) throw zoneSourceError("zone_source_adapter_unsupported");
if (!isPlainObject(identityCrosswalk)) throw zoneSourceError("zone_source_identity_crosswalk_invalid");
if (value.complete !== true) throw zoneSourceError("zone_source_generation_incomplete");
const generatedAt = iso(value.generatedAt, "zone_source_generated_at_invalid");
if (!Array.isArray(value.zones) || value.zones.length > maxZones) {
throw zoneSourceError("zone_source_zone_limit_exceeded");
}
if (byteLength(value) > maxBytes) throw zoneSourceError("zone_source_generation_bytes_exceeded");
if (!Number.isInteger(circleSegments) || circleSegments < 16 || circleSegments > 256) {
throw zoneSourceError("zone_source_circle_segments_invalid");
}
const facts = [];
const sourceIds = new Set();
for (const [index, rawZone] of value.zones.entries()) {
const zone = normalizeRawZone(rawZone, index, identityCrosswalk, circleSegments, value.adapterId);
if (sourceIds.has(zone.sourceId)) throw zoneSourceError("zone_source_identity_duplicate");
sourceIds.add(zone.sourceId);
facts.push({
sourceId: zone.sourceId,
semanticType: "map.zone",
observedAt: generatedAt,
geometry: zone.geometry,
attributes: compact({
area_square_meters: finiteNonNegative(first(rawZone.surfaceArea, rawZone.surface_area)),
description: optionalString(first(rawZone.description, rawZone.descr)),
display_name: requiredString(first(rawZone.name, rawZone.display_name), "zone_source_name_required"),
geometry_kind: zone.geometryKind,
max_speed_kph: finiteNonNegative(first(rawZone.maxPermissibleSpeed, rawZone.max_permissible_speed)),
perimeter_meters: finiteNonNegative(rawZone.perimeter),
source_kind: adapter.sourceKind,
source_revision: revision(value, adapter),
style_color: optionalString(first(rawZone.color, rawZone.style_color)),
}),
});
}
facts.sort((left, right) => left.sourceId.localeCompare(right.sourceId));
const digest = createHash("sha256").update(stableJson({
adapterId: value.adapterId,
generatedAt,
revision: revision(value, adapter),
facts,
})).digest("hex");
return Object.freeze({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: value.adapterId,
sourceKind: adapter.sourceKind,
sourceRevision: revision(value, adapter),
generatedAt,
complete: true,
digest,
facts: Object.freeze(facts.map(deepFreeze)),
});
}
function normalizeRawZone(value, index, identityCrosswalk, circleSegments, adapterId) {
if (!isPlainObject(value)) throw zoneSourceError(`zone_source_zone_${index}_invalid`);
const nativeId = resolveNativeId(value, identityCrosswalk, adapterId);
const sourceId = nativeId.startsWith("gelios-zone-") ? nativeId : `gelios-zone-${nativeId}`;
if (!/^[a-z][a-z0-9._:-]{2,127}$/.test(sourceId)) throw zoneSourceError("zone_source_identity_invalid");
const rawType = String(first(value.type, value.geometry_kind, "polygon")).trim().toLowerCase();
const geometryKind = rawType === "circle" ? "circle" : rawType === "line" || rawType === "corridor" ? "corridor" : "polygon";
const geometry = normalizeGeometry(value, geometryKind, circleSegments);
if (!isBoundedGeoJsonGeometry(geometry, new Set(["Polygon", "MultiPolygon"]))) {
throw zoneSourceError(`zone_source_zone_${index}_geometry_invalid`);
}
return { sourceId, geometryKind, geometry };
}
function resolveNativeId(zone, identityCrosswalk, adapterId) {
if (adapterId === "mmap-snapshot-v1") {
const snapshotKey = optionalString(first(zone.snapshotKey, zone.snapshot_key));
if (!snapshotKey || !Object.hasOwn(identityCrosswalk, snapshotKey)) {
throw zoneSourceError("zone_source_identity_crosswalk_required");
}
const mapped = identityCrosswalk[snapshotKey];
if (mapped === undefined || mapped === null || !String(mapped).trim()) {
throw zoneSourceError("zone_source_identity_crosswalk_required");
}
return String(mapped).trim();
}
const direct = first(zone.id, zone.geozoneId, zone.zone_id, zone.sourceId);
if (direct !== undefined && direct !== null && String(direct).trim()) return String(direct).trim();
throw zoneSourceError("zone_source_native_identity_required");
}
function normalizeGeometry(zone, geometryKind, circleSegments) {
if (isBoundedGeoJsonGeometry(zone.geometry, new Set(["Polygon", "MultiPolygon"]))) {
return structuredClone(zone.geometry);
}
const points = parseCoordinates(first(zone.points, zone.coords, zone.coordinates));
if (geometryKind === "circle") {
if (!points.length) throw zoneSourceError("zone_source_circle_center_required");
const radius = finiteNonNegative(zone.radius);
if (!radius || radius <= 0) throw zoneSourceError("zone_source_circle_radius_required");
return circlePolygon(points[0], radius, circleSegments);
}
if (geometryKind === "corridor") {
const linePoints = parseCoordinates(first(zone.line, zone.points, zone.coords, zone.coordinates));
const width = finiteNonNegative(first(zone.lineWidthMeters, zone.line_width_meters, zone.width, zone.radius));
if (linePoints.length < 2 || !width || width <= 0) {
throw zoneSourceError("zone_source_corridor_path_and_width_required");
}
return corridorPolygon(linePoints, width);
}
if (points.length < 3) throw zoneSourceError("zone_source_polygon_points_required");
const ring = [...points];
if (!samePosition(ring[0], ring.at(-1))) ring.push([...ring[0]]);
return { type: "Polygon", coordinates: [ring] };
}
function parseCoordinates(value) {
if (typeof value === "string") {
return value.split(";").map((entry) => entry.trim()).filter(Boolean).map((entry) => {
const [latitude, longitude] = entry.split(",").map(Number);
return position(longitude, latitude);
});
}
if (!Array.isArray(value)) return [];
return value.map((entry) => {
if (Array.isArray(entry)) {
const [latitude, longitude] = entry.map(Number);
return position(longitude, latitude);
}
if (!isPlainObject(entry)) throw zoneSourceError("zone_source_coordinate_invalid");
return position(
firstNumber(entry.longitude, entry.lon, entry.lng, entry.x),
firstNumber(entry.latitude, entry.lat, entry.y),
);
});
}
function circlePolygon([longitude, latitude], radiusMeters, segments) {
const earthRadius = 6_371_008.8;
const angularDistance = radiusMeters / earthRadius;
const latitudeRad = radians(latitude);
const longitudeRad = radians(longitude);
const ring = [];
for (let index = 0; index <= segments; index += 1) {
const bearing = 2 * Math.PI * index / segments;
const targetLatitude = Math.asin(
Math.sin(latitudeRad) * Math.cos(angularDistance)
+ Math.cos(latitudeRad) * Math.sin(angularDistance) * Math.cos(bearing),
);
const targetLongitude = longitudeRad + Math.atan2(
Math.sin(bearing) * Math.sin(angularDistance) * Math.cos(latitudeRad),
Math.cos(angularDistance) - Math.sin(latitudeRad) * Math.sin(targetLatitude),
);
ring.push([degrees(targetLongitude), degrees(targetLatitude)]);
}
ring[ring.length - 1] = [...ring[0]];
return { type: "Polygon", coordinates: [ring] };
}
function corridorPolygon(points, widthMeters) {
const halfWidth = widthMeters / 2;
const left = [];
const right = [];
for (let index = 0; index < points.length; index += 1) {
const previous = points[Math.max(0, index - 1)];
const next = points[Math.min(points.length - 1, index + 1)];
const meanLatitude = radians((previous[1] + next[1]) / 2);
const dx = (next[0] - previous[0]) * Math.cos(meanLatitude);
const dy = next[1] - previous[1];
const length = Math.hypot(dx, dy);
if (!length) throw zoneSourceError("zone_source_corridor_zero_length_segment");
const metersPerDegreeLatitude = 111_320;
const longitudeOffset = (-dy / length) * halfWidth / (metersPerDegreeLatitude * Math.max(0.01, Math.cos(radians(points[index][1]))));
const latitudeOffset = (dx / length) * halfWidth / metersPerDegreeLatitude;
left.push([points[index][0] + longitudeOffset, points[index][1] + latitudeOffset]);
right.push([points[index][0] - longitudeOffset, points[index][1] - latitudeOffset]);
}
const ring = [...left, ...right.reverse(), [...left[0]]];
return { type: "Polygon", coordinates: [ring] };
}
function revision(value, adapter) {
const explicit = optionalString(value.sourceRevision);
if (value.adapterId === "mmap-snapshot-v1") {
const digest = optionalString(value.snapshotDigest);
if (!digest || !/^[a-f0-9]{64}$/.test(digest)) throw zoneSourceError("zone_source_snapshot_digest_required");
return `${explicit || adapter.sourceRevision}@sha256:${digest}`;
}
return explicit || adapter.sourceRevision;
}
function position(longitude, latitude) {
if (![longitude, latitude].every(Number.isFinite)
|| longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) {
throw zoneSourceError("zone_source_coordinate_invalid");
}
return [longitude, latitude];
}
function first(...values) {
return values.find((value) => value !== undefined && value !== null && value !== "");
}
function firstNumber(...values) {
const value = first(...values);
const number = typeof value === "number" ? value : Number(String(value ?? "").trim());
return Number.isFinite(number) ? number : Number.NaN;
}
function finiteNonNegative(value) {
const number = firstNumber(value);
return Number.isFinite(number) && number >= 0 ? number : undefined;
}
function requiredString(value, code) {
const result = optionalString(value);
if (!result) throw zoneSourceError(code);
return result;
}
function optionalString(value) {
return typeof value === "string" && value.trim() ? value.trim() : undefined;
}
function iso(value, code) {
const date = new Date(String(value ?? ""));
if (Number.isNaN(date.getTime())) throw zoneSourceError(code);
return date.toISOString();
}
function compact(value) {
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
}
function samePosition(left, right) {
return Array.isArray(left) && Array.isArray(right) && left[0] === right[0] && left[1] === right[1];
}
function stableJson(value) {
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
if (isPlainObject(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`;
return JSON.stringify(value);
}
function byteLength(value) {
try { return Buffer.byteLength(JSON.stringify(value)); } catch { return Number.POSITIVE_INFINITY; }
}
function deepFreeze(value) {
if (!value || typeof value !== "object" || Object.isFrozen(value)) return value;
Object.freeze(value);
Object.values(value).forEach(deepFreeze);
return value;
}
function isPlainObject(value) {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function radians(value) { return value * Math.PI / 180; }
function degrees(value) { return value * 180 / Math.PI; }
function zoneSourceError(code) {
return Object.assign(new Error(code), { code });
}

View File

@ -72,6 +72,32 @@ assert.equal(validateDataProductPublish({
...publish, ...publish,
facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }], facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }],
}).errors.includes("facts[0].geometry.latitude_out_of_range"), true); }).errors.includes("facts[0].geometry.latitude_out_of_range"), true);
const zoneGenerationAt = "2026-07-15T10:10:00.000Z";
const zoneFact = {
sourceId: "gelios-zone-42",
semanticType: "map.zone",
observedAt: zoneGenerationAt,
attributes: { display_name: "Zone 42" },
geometry: {
type: "Polygon",
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
},
};
const replacement = {
...publish,
batch: { ...publish.batch, mode: "replace", generationAt: zoneGenerationAt },
facts: [zoneFact],
};
assert.equal(validateDataProductPublish(replacement).ok, true);
assert.equal(validateDataProductPublish({ ...replacement, facts: [] }).ok, true);
assert.equal(validateDataProductPublish({
...replacement,
facts: [{ ...zoneFact, observedAt: "2026-07-15T10:09:59.000Z" }],
}).errors.includes("facts[0].observedAt_must_equal_generationAt"), true);
assert.equal(validateDataProductPublish({
...replacement,
facts: [{ ...zoneFact, geometry: { ...zoneFact.geometry, coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.61, 55.75]]] } }],
}).errors.includes("facts[0].geometry[0]_linear_ring_must_be_closed"), true);
const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" }; const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" };
const snapshot = { const snapshot = {
@ -147,7 +173,11 @@ assert.equal(validateDataProductPatch(patch).ok, true);
assert.equal(validateDataProductPatch({ assert.equal(validateDataProductPatch({
...patch, ...patch,
operations: [{ op: "delete", fact: canonicalFact }], operations: [{ op: "delete", fact: canonicalFact }],
}).errors.includes("operations[0].op_must_be_upsert"), true); }).errors.includes("operations[0].op_invalid"), true);
assert.equal(validateDataProductPatch({
...patch,
operations: [{ op: "remove", sourceId: "gelios-zone-42", semanticType: "map.zone" }],
}).ok, true);
assert.equal(validateDataProductPatch({ assert.equal(validateDataProductPatch({
...patch, ...patch,
operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }], operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }],

View File

@ -18,6 +18,7 @@ import { geliosProviderPackageV2 } from "../providers/gelios/v2/index.mjs";
import { geliosProviderPackageV3 } from "../providers/gelios/v3/index.mjs"; import { geliosProviderPackageV3 } from "../providers/gelios/v3/index.mjs";
import { geliosProviderPackageV4 } from "../providers/gelios/v4/index.mjs"; import { geliosProviderPackageV4 } from "../providers/gelios/v4/index.mjs";
import { geliosProviderPackageV5 } from "../providers/gelios/v5/index.mjs"; import { geliosProviderPackageV5 } from "../providers/gelios/v5/index.mjs";
import { geliosProviderPackageV6 } from "../providers/gelios/v6/index.mjs";
import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs"; import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs";
const expectedFields = [ const expectedFields = [
@ -41,6 +42,24 @@ assert.deepEqual(validateProviderPackage(geliosProviderPackageV2), { ok: true, e
assert.deepEqual(validateProviderPackage(geliosProviderPackageV3), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV3), { ok: true, errors: [] });
assert.deepEqual(validateProviderPackage(geliosProviderPackageV4), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV4), { ok: true, errors: [] });
assert.deepEqual(validateProviderPackage(geliosProviderPackageV5), { ok: true, errors: [] }); assert.deepEqual(validateProviderPackage(geliosProviderPackageV5), { ok: true, errors: [] });
assert.deepEqual(validateProviderPackage(geliosProviderPackageV6), { ok: true, errors: [] });
const zonesProduct = geliosProviderPackageV6.dataProducts.find((product) => product.id === "map.zones.current.v1");
const registeredZonesProduct = JSON.parse(await readFile(new URL(
"../../../services/external-data-plane/definitions/map.zones.current.v1.json",
import.meta.url,
), "utf8"));
assert.deepEqual(zonesProduct, registeredZonesProduct);
assert.deepEqual(zonesProduct.semanticTypes, ["map.zone"]);
assert.equal(zonesProduct.fieldContracts.geometry.type, "geometry");
const zonesCapability = geliosProviderPackageV6.capabilities.find((capability) => capability.id === "gelios.geozones.current.read");
assert.equal(zonesCapability.request.query.pl, 100);
assert.equal(zonesCapability.request.query.po, 0);
assert.equal(zonesCapability.request.response.pagination.maxItems, 5000);
assert.equal(zonesCapability.request.response.pagination.totalPath, "paginationMetadata.totalCount");
const zonesMapping = geliosProviderPackageV6.mappingContracts.find((mapping) => mapping.id === "gelios.geozones.to.map.zones.current.v1");
assert.deepEqual(zonesMapping.fact.observedAt.paths, ["updatedAt"]);
assert.equal(zonesMapping.fact.observedAt.fallback, "collection_received_at");
const strictProduct = geliosProviderPackageV5.dataProducts[0]; const strictProduct = geliosProviderPackageV5.dataProducts[0];
const registeredStrictProduct = JSON.parse(await readFile(new URL( const registeredStrictProduct = JSON.parse(await readFile(new URL(

View File

@ -0,0 +1,95 @@
import assert from "node:assert/strict";
import {
ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
normalizeZoneSourceGeneration,
validateDataProductPublish,
} from "../src/index.mjs";
const generatedAt = "2026-07-20T18:00:00.000Z";
const live = normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "gelios-rest-v1",
sourceRevision: "gelios-rest-v1",
generatedAt,
complete: true,
zones: [{
id: 42,
name: "Polygon zone",
type: "polygon",
points: [
{ latitude: 55.74, longitude: 37.60 },
{ latitude: 55.74, longitude: 37.62 },
{ latitude: 55.76, longitude: 37.62 },
],
color: "#ff00ff",
}, {
id: 43,
name: "Circle zone",
type: "circle",
points: [{ latitude: 55.75, longitude: 37.61 }],
radius: 150,
}],
});
assert.equal(live.complete, true);
assert.equal(live.sourceKind, "live_api");
assert.equal(live.facts.length, 2);
assert.deepEqual(live.facts.map((fact) => fact.sourceId), ["gelios-zone-42", "gelios-zone-43"]);
assert.equal(live.facts.every((fact) => fact.observedAt === generatedAt), true);
assert.equal(live.facts.every((fact) => fact.geometry.type === "Polygon"), true);
assert.equal(validateDataProductPublish({
schemaVersion: "nodedc.data-product.publish/v1",
batch: { runId: "zones-live-1", sequence: 0, idempotencyKey: "zones-live-1.batch-0", mode: "replace", generationAt: generatedAt },
facts: live.facts,
}).ok, true);
const snapshotDigest = "a".repeat(64);
const snapshot = normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "mmap-snapshot-v1",
snapshotDigest,
generatedAt,
complete: true,
zones: [{
snapshotKey: "legacy-zone-a",
name: "Snapshot zone",
type: "polygon",
points: "55.74,37.60;55.74,37.62;55.76,37.62",
}],
}, { identityCrosswalk: { "legacy-zone-a": 42 } });
assert.equal(snapshot.sourceKind, "versioned_snapshot");
assert.equal(snapshot.facts[0].sourceId, live.facts[0].sourceId);
assert.match(snapshot.sourceRevision, new RegExp(`@sha256:${snapshotDigest}$`));
assert.throws(() => normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "mmap-snapshot-v1",
snapshotDigest,
generatedAt,
complete: true,
zones: [{ snapshotKey: "unknown", name: "Unknown", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
}), /zone_source_identity_crosswalk_required/);
assert.throws(() => normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "mmap-snapshot-v1",
snapshotDigest,
generatedAt,
complete: true,
zones: [{ id: 42, name: "Uncrossed", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
}), /zone_source_identity_crosswalk_required/);
assert.throws(() => normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "gelios-rest-v1",
generatedAt,
complete: true,
zones: [{ name: "No native id", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
}), /zone_source_native_identity_required/);
assert.throws(() => normalizeZoneSourceGeneration({
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
adapterId: "gelios-rest-v1",
generatedAt,
complete: false,
zones: [],
}), /zone_source_generation_incomplete/);
console.log("external-provider zone source: ok");

View File

@ -25,7 +25,7 @@ export class NdcDataProductPublish implements INodeType {
name: 'ndcDataProductPublish', name: 'ndcDataProductPublish',
icon: NDC_NODE_ICON, icon: NDC_NODE_ICON,
group: ['output'], group: ['output'],
version: 1, version: [1, 2],
subtitle: '={{$parameter["dataProductId"]}}', subtitle: '={{$parameter["dataProductId"]}}',
description: 'Publish canonical facts through a scoped NDC Data Product grant', description: 'Publish canonical facts through a scoped NDC Data Product grant',
defaults: { name: 'NDC Data Product Publish' }, defaults: { name: 'NDC Data Product Publish' },
@ -41,6 +41,19 @@ export class NdcDataProductPublish implements INodeType {
default: '', default: '',
required: true, required: true,
}, },
{
displayName: 'Publish Mode',
name: 'publishMode',
type: 'options',
options: [
{ name: 'Create or Update', value: 'upsert' },
{ name: 'Replace Complete Snapshot', value: 'replace' },
],
default: 'upsert',
required: true,
displayOptions: { show: { '@version': [2] } },
description: 'Replace commits one complete generation atomically and removes keys absent from it',
},
{ {
displayName: 'Batch Sequence', displayName: 'Batch Sequence',
name: 'sequence', name: 'sequence',
@ -71,6 +84,9 @@ export class NdcDataProductPublish implements INodeType {
const inputItems = this.getInputData(); const inputItems = this.getInputData();
const dataProductId = this.getNodeParameter('dataProductId', 0) as string; const dataProductId = this.getNodeParameter('dataProductId', 0) as string;
const sequence = this.getNodeParameter('sequence', 0, 0) as number; const sequence = this.getNodeParameter('sequence', 0, 0) as number;
const publishMode = this.getNode().typeVersion >= 2
? this.getNodeParameter('publishMode', 0, 'upsert') as 'upsert' | 'replace'
: 'upsert';
let body; let body;
let encodedProductId; let encodedProductId;
try { try {
@ -81,6 +97,7 @@ export class NdcDataProductPublish implements INodeType {
this.getNode().id, this.getNode().id,
dataProductId, dataProductId,
sequence, sequence,
publishMode,
); );
encodedProductId = encodeIdentifierPath(dataProductId, 'dataProductId'); encodedProductId = encodeIdentifierPath(dataProductId, 'dataProductId');
} catch (error) { } catch (error) {

View File

@ -18,18 +18,23 @@ export interface NdcFact extends IDataObject {
semanticType: string; semanticType: string;
observedAt: string; observedAt: string;
attributes?: IDataObject; attributes?: IDataObject;
geometry?: { geometry?: NdcGeometry;
type: 'Point';
coordinates: [number, number];
};
} }
export type NdcGeometry =
| { type: 'Point'; coordinates: [number, number] }
| { type: 'LineString'; coordinates: [number, number][] }
| { type: 'Polygon'; coordinates: [number, number][][] }
| { type: 'MultiPolygon'; coordinates: [number, number][][][] };
export interface NdcPublishPayload extends IDataObject { export interface NdcPublishPayload extends IDataObject {
schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION; schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION;
batch: { batch: {
runId: string; runId: string;
sequence: number; sequence: number;
idempotencyKey: string; idempotencyKey: string;
mode?: 'replace';
generationAt?: string;
}; };
facts: NdcFact[]; facts: NdcFact[];
} }
@ -62,12 +67,16 @@ export function encodeIdentifierPath(value: unknown, field: string): string {
return encodeURIComponent(requireIdentifier(value, field)); return encodeURIComponent(requireIdentifier(value, field));
} }
export function factsFromItems(items: INodeExecutionData[]): NdcFact[] { export function factsFromItems(items: INodeExecutionData[], allowEmpty = false): NdcFact[] {
if (!items.length) throw new Error('facts_required'); const batchFacts = items.length === 1 && Array.isArray(items[0]?.json?.facts)
if (items.length > MAX_FACTS) throw new Error('facts_limit_exceeded'); ? items[0].json.facts
: null;
const sources = batchFacts ?? items.map((item) => (isObject(item.json.fact) ? item.json.fact : item.json));
if (!sources.length && !allowEmpty) throw new Error('facts_required');
if (sources.length > MAX_FACTS) throw new Error('facts_limit_exceeded');
const entityKeys = new Set<string>(); const entityKeys = new Set<string>();
return items.map((item, index) => { return sources.map((source, index) => {
const source = isObject(item.json.fact) ? item.json.fact : item.json; if (!isObject(source)) throw new Error(`facts_${index}_invalid`);
const fact: NdcFact = { const fact: NdcFact = {
sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`), sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`),
semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`), semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`),
@ -84,7 +93,7 @@ export function factsFromItems(items: INodeExecutionData[]): NdcFact[] {
if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`); if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`);
fact.attributes = source.attributes; fact.attributes = source.attributes;
} }
if (source.geometry !== undefined) fact.geometry = normalizePoint(source.geometry, index); if (source.geometry !== undefined) fact.geometry = normalizeGeometry(source.geometry, index);
return fact; return fact;
}); });
} }
@ -96,6 +105,7 @@ export function buildPublishPayload(
nodeId: string, nodeId: string,
dataProductId: string, dataProductId: string,
sequence: number, sequence: number,
publishMode: 'upsert' | 'replace' = 'upsert',
): NdcPublishPayload { ): NdcPublishPayload {
if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid'); if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid');
requireIdentifier(dataProductId, 'dataProductId'); requireIdentifier(dataProductId, 'dataProductId');
@ -103,10 +113,22 @@ export function buildPublishPayload(
const idempotencyKey = `publish-${sha256( const idempotencyKey = `publish-${sha256(
`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`, `${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`,
)}`; )}`;
const facts = factsFromItems(items, publishMode === 'replace');
const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts)
? items[0].json.generationAt
: undefined;
const generationAt = publishMode === 'replace'
? replacementGenerationAt(facts, explicitGenerationAt)
: undefined;
return { return {
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION, schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
batch: { runId, sequence, idempotencyKey }, batch: {
facts: factsFromItems(items), runId,
sequence,
idempotencyKey,
...(generationAt ? { mode: 'replace' as const, generationAt } : {}),
},
facts,
}; };
} }
@ -254,18 +276,60 @@ function requireFoundrySlotId(value: unknown): string {
return normalized; return normalized;
} }
function normalizePoint(value: unknown, index: number): NdcFact['geometry'] { function normalizeGeometry(value: unknown, index: number): NdcGeometry {
if (!isObject(value) || value.type !== 'Point' || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) { if (!isObject(value) || !['Point', 'LineString', 'Polygon', 'MultiPolygon'].includes(String(value.type))
|| !Array.isArray(value.coordinates)) {
throw new Error(`facts_${index}_geometry_invalid`); throw new Error(`facts_${index}_geometry_invalid`);
} }
const [longitude, latitude] = value.coordinates; if (Buffer.byteLength(JSON.stringify(value)) > 192 * 1024) throw new Error(`facts_${index}_geometry_size_exceeded`);
if (![longitude, latitude].every((coordinate) => typeof coordinate === 'number' && Number.isFinite(coordinate))) { let vertices = 0;
throw new Error(`facts_${index}_geometry_coordinates_invalid`); const position = (candidate: unknown, path: string): [number, number] => {
if (!Array.isArray(candidate) || candidate.length !== 2 || !candidate.every(Number.isFinite)) {
throw new Error(`${path}_position_invalid`);
}
const [longitude, latitude] = candidate as [number, number];
if (longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) {
throw new Error(`${path}_position_out_of_range`);
}
vertices += 1;
if (vertices > 10_000) throw new Error(`facts_${index}_geometry_vertex_limit_exceeded`);
return [longitude, latitude];
};
const line = (candidate: unknown, path: string, minimum: number): [number, number][] => {
if (!Array.isArray(candidate) || candidate.length < minimum) throw new Error(`${path}_line_invalid`);
return candidate.map((entry, positionIndex) => position(entry, `${path}_${positionIndex}`));
};
const ring = (candidate: unknown, path: string): [number, number][] => {
const result = line(candidate, path, 4);
const first = result[0];
const last = result.at(-1);
if (!last || first[0] !== last[0] || first[1] !== last[1]) throw new Error(`${path}_ring_not_closed`);
return result;
};
const polygon = (candidate: unknown, path: string): [number, number][][] => {
if (!Array.isArray(candidate) || !candidate.length) throw new Error(`${path}_polygon_invalid`);
return candidate.map((entry, ringIndex) => ring(entry, `${path}_${ringIndex}`));
};
if (value.type === 'Point') return { type: 'Point', coordinates: position(value.coordinates, `facts_${index}_geometry`) };
if (value.type === 'LineString') return { type: 'LineString', coordinates: line(value.coordinates, `facts_${index}_geometry`, 2) };
if (value.type === 'Polygon') return { type: 'Polygon', coordinates: polygon(value.coordinates, `facts_${index}_geometry`) };
const coordinates = value.coordinates.map((entry, polygonIndex) => polygon(entry, `facts_${index}_geometry_${polygonIndex}`));
if (!coordinates.length) throw new Error(`facts_${index}_geometry_multipolygon_invalid`);
return { type: 'MultiPolygon', coordinates };
}
function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string {
const declared = explicit === undefined ? undefined : requireIsoTimestamp(explicit, 'replace_generationAt');
const timestamps = [...new Set(facts.map((fact) => new Date(fact.observedAt).toISOString()))];
if (!timestamps.length) {
if (!declared) throw new Error('replace_generationAt_required_for_empty_generation');
return new Date(declared).toISOString();
} }
if ((longitude as number) < -180 || (longitude as number) > 180 || (latitude as number) < -90 || (latitude as number) > 90) { if (timestamps.length !== 1) throw new Error('replace_generation_requires_one_observedAt');
throw new Error(`facts_${index}_geometry_coordinates_out_of_range`); if (declared && new Date(declared).toISOString() !== timestamps[0]) {
throw new Error('replace_generationAt_observedAt_mismatch');
} }
return { type: 'Point', coordinates: [longitude as number, latitude as number] }; return timestamps[0];
} }
function requireIsoTimestamp(value: unknown, field: string): string { function requireIsoTimestamp(value: unknown, field: string): string {

View File

@ -1,12 +1,12 @@
{ {
"name": "n8n-nodes-ndc", "name": "n8n-nodes-ndc",
"version": "0.1.4", "version": "0.1.5",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "n8n-nodes-ndc", "name": "n8n-nodes-ndc",
"version": "0.1.4", "version": "0.1.5",
"license": "UNLICENSED", "license": "UNLICENSED",
"devDependencies": { "devDependencies": {
"@n8n/node-cli": "0.39.3", "@n8n/node-cli": "0.39.3",

View File

@ -1,6 +1,6 @@
{ {
"name": "n8n-nodes-ndc", "name": "n8n-nodes-ndc",
"version": "0.1.4", "version": "0.1.5",
"description": "Private NODE.DC nodes for scoped data products and Foundry bindings.", "description": "Private NODE.DC nodes for scoped data products and Foundry bindings.",
"private": true, "private": true,
"license": "UNLICENSED", "license": "UNLICENSED",

View File

@ -146,6 +146,64 @@ async function main() {
publish.batch.idempotencyKey, publish.batch.idempotencyKey,
contracts.buildPublishPayload(items, 'execution-42', 'workflow-7', 'node-3', 'fleet.positions.current.v2', 0).batch.idempotencyKey, contracts.buildPublishPayload(items, 'execution-42', 'workflow-7', 'node-3', 'fleet.positions.current.v2', 0).batch.idempotencyKey,
); );
const zoneItems = [{
json: {
sourceId: 'gelios-zone-42',
semanticType: 'map.zone',
observedAt: '2026-07-15T12:10:00.000Z',
attributes: { display_name: 'Zone 42' },
geometry: {
type: 'Polygon',
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
},
},
}];
const replacement = contracts.buildPublishPayload(
zoneItems,
'execution-43',
'workflow-7',
'node-zones',
'map.zones.current.v1',
0,
'replace',
);
assert.equal(externalContract.validateDataProductPublish(replacement).ok, true);
assert.equal(replacement.batch.mode, 'replace');
assert.equal(replacement.batch.generationAt, zoneItems[0].json.observedAt);
const replacementBatch = contracts.buildPublishPayload(
[{ json: { generationAt: zoneItems[0].json.observedAt, facts: zoneItems.map((item) => item.json) } }],
'execution-44',
'workflow-7',
'node-zones',
'map.zones.current.v1',
0,
'replace',
);
assert.deepEqual(replacementBatch.facts, replacement.facts);
assert.equal(replacementBatch.batch.generationAt, zoneItems[0].json.observedAt);
const emptyReplacement = contracts.buildPublishPayload(
[{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: [] } }],
'execution-45',
'workflow-7',
'node-zones',
'map.zones.current.v1',
0,
'replace',
);
assert.deepEqual(emptyReplacement.facts, []);
assert.equal(externalContract.validateDataProductPublish(emptyReplacement).ok, true);
assert.throws(
() => contracts.buildPublishPayload(
[{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: zoneItems.map((item) => item.json) } }],
'execution-46',
'workflow-7',
'node-zones',
'map.zones.current.v1',
0,
'replace',
),
/replace_generationAt_observedAt_mismatch/,
);
assert.throws( assert.throws(
() => contracts.buildPublishPayload( () => contracts.buildPublishPayload(
[{ json: { ...items[0].json, attributes: { accessToken: 'forbidden' } } }], [{ json: { ...items[0].json, attributes: { accessToken: 'forbidden' } } }],
@ -303,6 +361,29 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) {
assert.equal(externalContract.validateDataProductPublish(publishCalls[0].options.body).ok, true); assert.equal(externalContract.validateDataProductPublish(publishCalls[0].options.body).ok, true);
assert.equal(publishResult[0][0].json.publishedFactCount, 1); assert.equal(publishResult[0][0].json.publishedFactCount, 1);
const replacementCalls = [];
const replacementContext = executionContext({
typeVersion: 2,
parameters: { dataProductId: 'map.zones.current.v1', publishMode: 'replace', sequence: 0 },
input: [{
json: {
generationAt: '2026-07-15T12:10:00.000Z',
facts: [{
sourceId: 'gelios-zone-42',
semanticType: 'map.zone',
observedAt: '2026-07-15T12:10:00.000Z',
attributes: { display_name: 'Zone 42' },
geometry: { type: 'Polygon', coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]] },
}],
},
}],
response: { ok: true, publishedFactCount: 1, currentRemovedCount: 0 },
calls: replacementCalls,
});
await nodes.get('NdcDataProductPublish').execute.call(replacementContext);
assert.equal(replacementCalls[0].options.body.batch.mode, 'replace');
assert.equal(externalContract.validateDataProductPublish(replacementCalls[0].options.body).ok, true);
const readCalls = []; const readCalls = [];
const readContext = executionContext({ const readContext = executionContext({
parameters: { dataProductId: 'fleet.positions.current.v1', pageSize: 1000 }, parameters: { dataProductId: 'fleet.positions.current.v1', pageSize: 1000 },
@ -411,7 +492,7 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) {
} }
} }
function executionContext({ parameters, input = [{ json: {} }], response, calls }) { function executionContext({ parameters, input = [{ json: {} }], response, calls, typeVersion = 1 }) {
return { return {
getNodeParameter(name, _index, defaultValue) { getNodeParameter(name, _index, defaultValue) {
return Object.prototype.hasOwnProperty.call(parameters, name) ? parameters[name] : defaultValue; return Object.prototype.hasOwnProperty.call(parameters, name) ? parameters[name] : defaultValue;
@ -419,7 +500,7 @@ function executionContext({ parameters, input = [{ json: {} }], response, calls
getInputData() { return input; }, getInputData() { return input; },
getExecutionId() { return 'execution-42'; }, getExecutionId() { return 'execution-42'; },
getWorkflow() { return { id: 'workflow-7' }; }, getWorkflow() { return { id: 'workflow-7' }; },
getNode() { return { id: 'node-3' }; }, getNode() { return { id: 'node-3', typeVersion }; },
helpers: { helpers: {
async httpRequestWithAuthentication(credentialName, options) { async httpRequestWithAuthentication(credentialName, options) {
calls.push({ credentialName, options }); calls.push({ credentialName, options });

View File

@ -6,6 +6,7 @@ COPY packages/external-provider-contract/package.json ./packages/external-provid
COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs
COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs
COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs
COPY packages/external-provider-contract/src/geometry.mjs ./packages/external-provider-contract/src/geometry.mjs
COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs
COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs
COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/ COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/

View File

@ -0,0 +1,32 @@
{
"id": "map.zones.current.v1",
"version": "1.0.0",
"ontologyRevision": "ontology.map.zone.v1",
"deliveryMode": "snapshot+patch",
"semanticTypes": ["map.zone"],
"fields": [
"area_square_meters",
"description",
"display_name",
"geometry",
"geometry_kind",
"max_speed_kph",
"perimeter_meters",
"source_kind",
"source_revision",
"style_color"
],
"fieldContracts": {
"area_square_meters": { "type": "number", "required": false, "minimum": 0 },
"description": { "type": "string", "required": false },
"display_name": { "type": "string", "required": true },
"geometry": { "type": "geometry", "required": true },
"geometry_kind": { "type": "string", "required": true, "enum": ["circle", "corridor", "polygon"] },
"max_speed_kph": { "type": "number", "required": false, "minimum": 0 },
"perimeter_meters": { "type": "number", "required": false, "minimum": 0 },
"source_kind": { "type": "string", "required": true, "enum": ["live_api", "versioned_snapshot"] },
"source_revision": { "type": "string", "required": true },
"style_color": { "type": "string", "required": false }
},
"history": { "mode": "none", "retentionDays": 1 }
}

View File

@ -3,6 +3,7 @@ import {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION, DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION, DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
isBoundedGeoJsonGeometry,
} from "@nodedc/external-provider-contract/data-plane"; } from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
const existing = await client.query( const existing = await client.query(
`select id as "batchId", fact_count as "publishedFactCount", `select id as "batchId", fact_count as "publishedFactCount",
current_updated_count as "currentUpdatedCount", current_updated_count as "currentUpdatedCount",
current_removed_count as "currentRemovedCount",
history_inserted_count as "historyInsertedCount", history_inserted_count as "historyInsertedCount",
patch_operation_count as "patchOperationCount", patch_operation_count as "patchOperationCount",
delivery_cursor::text as cursor, received_at as "acceptedAt", delivery_cursor::text as cursor, received_at as "acceptedAt",
@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, {
return { ...normalizeReceipt(existing.rows[0]), idempotent: true }; return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
} }
if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch);
const records = batch.facts.map((fact) => ({ const records = batch.facts.map((fact) => ({
source_id: fact.sourceId, source_id: fact.sourceId,
semantic_type: fact.semanticType, semantic_type: fact.semanticType,
@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
select $1, $2, $3, $4, item.source_id, item.semantic_type, select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint( else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
item.fingerprint, now() item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item( from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz, source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
returning source_id as "sourceId", semantic_type as "semanticType", returning source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes, observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry,
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry,
fingerprint`, fingerprint`,
[ [
batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, {
], ],
); );
const removed = batch.batch.mode === "replace"
? await removeMissingCurrent(client, batch, records)
: [];
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records); const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })); const operations = [
...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })),
...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })),
];
const chunks = definition.deliveryMode === "snapshot+patch" const chunks = definition.deliveryMode === "snapshot+patch"
? chunkOperations(operations, maxPatchOperations, maxPatchBytes) ? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
: []; : [];
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch); const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch);
await client.query( await client.query(
`update external_data_plane_batches set `update external_data_plane_batches set
current_updated_count = $2, current_updated_count = $2,
history_inserted_count = $3, current_removed_count = $3,
patch_operation_count = $4, history_inserted_count = $4,
delivery_cursor = $5 patch_operation_count = $5,
delivery_cursor = $6
where id = $1`, where id = $1`,
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor], [batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor],
); );
await client.query("commit"); await client.query("commit");
return normalizeReceipt({ return normalizeReceipt({
@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
idempotent: false, idempotent: false,
publishedFactCount: batch.facts.length, publishedFactCount: batch.facts.length,
currentUpdatedCount: updated.rowCount, currentUpdatedCount: updated.rowCount,
currentRemovedCount: removed.length,
historyInsertedCount, historyInsertedCount,
patchOperationCount: operations.length, patchOperationCount: operations.length,
cursor: String(cursor), cursor: String(cursor),
@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
`select source_id as "sourceId", semantic_type as "semanticType", `select source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes, observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
from external_data_plane_current from external_data_plane_current
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4 where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
order by source_id asc, semantic_type asc order by source_id asc, semantic_type asc
@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options)
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType", select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes, observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array( else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
from ranked from ranked
where sample_rank = 1 where sample_rank = 1
and ( and (
@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
return result.rowCount; return result.rowCount;
} }
async function lockReplacementGeneration(client, batch) {
const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId];
await client.query(
`insert into external_data_plane_delivery_state (
tenant_id, connection_id, provider_id, data_product_id, current_cursor
) values ($1, $2, $3, $4, 0)
on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`,
scope,
);
const state = await client.query(
`select current_generation_at as "currentGenerationAt"
from external_data_plane_delivery_state
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
for update`,
scope,
);
const generationAt = new Date(batch.batch.generationAt);
const currentGenerationAt = state.rows[0]?.currentGenerationAt
? new Date(state.rows[0].currentGenerationAt)
: null;
if (currentGenerationAt && generationAt <= currentGenerationAt) {
throw deliveryError("data_product_generation_not_newer", 409);
}
}
async function removeMissingCurrent(client, batch, records) {
const result = await client.query(
`delete from external_data_plane_current as target
where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3
and target.data_product_id = $4
and not exists (
select 1
from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text)
where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type
)
returning source_id as "sourceId", semantic_type as "semanticType"`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
JSON.stringify(records),
],
);
return result.rows;
}
async function persistReplacementGeneration(client, batch) {
await client.query(
`update external_data_plane_delivery_state
set current_generation_at = $5, current_generation_id = $6, updated_at = now()
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
batch.batch.generationAt,
batch.batch.runId,
],
);
}
async function persistHistory(client, batch, policy, facts) { async function persistHistory(client, batch, policy, facts) {
if (!facts.length || policy?.mode === "none") return 0; if (!facts.length || policy?.mode === "none") return 0;
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1; const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) {
select $1, $2, $3, $4, item.source_id, item.semantic_type, select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb), item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint( else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
item.fingerprint, now() item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item( from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, bucket_start timestamptz, source_id text, semantic_type text, bucket_start timestamptz,
@ -570,6 +634,7 @@ function normalizeReceipt(value) {
idempotent: value.idempotent === true, idempotent: value.idempotent === true,
publishedFactCount: Number(value.publishedFactCount || 0), publishedFactCount: Number(value.publishedFactCount || 0),
currentUpdatedCount: Number(value.currentUpdatedCount || 0), currentUpdatedCount: Number(value.currentUpdatedCount || 0),
currentRemovedCount: Number(value.currentRemovedCount || 0),
historyInsertedCount: Number(value.historyInsertedCount || 0), historyInsertedCount: Number(value.historyInsertedCount || 0),
patchOperationCount: Number(value.patchOperationCount || 0), patchOperationCount: Number(value.patchOperationCount || 0),
cursor: String(value.cursor || "0"), cursor: String(value.cursor || "0"),
@ -652,6 +717,8 @@ function publishFingerprint(batch) {
runId: batch.batch.runId, runId: batch.batch.runId,
sequence: batch.batch.sequence, sequence: batch.batch.sequence,
idempotencyKey: batch.batch.idempotencyKey, idempotencyKey: batch.batch.idempotencyKey,
mode: batch.batch.mode || "upsert",
...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}),
}, },
facts: batch.facts, facts: batch.facts,
}); });
@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) {
function fieldContractValueMatchesType(value, type) { function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string"); if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") { if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
return value?.type === "Point" if (type === "geometry") return isBoundedGeoJsonGeometry(value);
&& Array.isArray(value.coordinates)
&& value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite)
&& value.coordinates[0] >= -180
&& value.coordinates[0] <= 180
&& value.coordinates[1] >= -90
&& value.coordinates[1] <= 90;
}
return typeof value === type && (type !== "number" || Number.isFinite(value)); return typeof value === type && (type !== "number" || Number.isFinite(value));
} }

View File

@ -1,8 +1,10 @@
import { isBoundedGeoJsonGeometry } from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i; const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]); const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
const HISTORY_MODES = new Set(["none", "all", "sampled"]); const HISTORY_MODES = new Set(["none", "all", "sampled"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]); const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
const DEFINITION_KEYS = new Set([ const DEFINITION_KEYS = new Set([
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history", "id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
]); ]);
@ -63,7 +65,7 @@ function normalizeFieldContract(value) {
if (value.enum !== undefined) { if (value.enum !== undefined) {
if (!Array.isArray(value.enum) || value.enum.length === 0 if (!Array.isArray(value.enum) || value.enum.length === 0
|| new Set(value.enum.map(stableLiteral)).size !== value.enum.length || new Set(value.enum.map(stableLiteral)).size !== value.enum.length
|| new Set(["point", "string_array"]).has(type) || new Set(["point", "geometry", "string_array"]).has(type)
|| value.enum.some((item) => !fieldContractValueMatchesType(item, type))) { || value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
throw policyError("data_product_field_contract_enum_invalid"); throw policyError("data_product_field_contract_enum_invalid");
} }
@ -171,6 +173,9 @@ function hasOnlyKeys(value, allowed) {
} }
function fieldContractValueMatchesType(value, type) { function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
return typeof value === type && (type !== "number" || Number.isFinite(value)); return typeof value === type && (type !== "number" || Number.isFinite(value));
} }

View File

@ -61,6 +61,7 @@ export async function migrate(pool) {
`); `);
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)"); await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists current_removed_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0"); await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint"); await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
@ -245,7 +246,7 @@ export async function migrate(pool) {
observed_at timestamptz not null, observed_at timestamptz not null,
received_at timestamptz not null, received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb, attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326), geometry geography(Geometry, 4326),
fingerprint text not null, fingerprint text not null,
primary key (id, observed_at), primary key (id, observed_at),
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint) unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
@ -266,12 +267,30 @@ export async function migrate(pool) {
observed_at timestamptz not null, observed_at timestamptz not null,
received_at timestamptz not null, received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb, attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326), geometry geography(Geometry, 4326),
fingerprint text not null, fingerprint text not null,
updated_at timestamptz not null default now(), updated_at timestamptz not null default now(),
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type) primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
) )
`); `);
await pool.query(`
do $$
begin
if exists (
select 1
from pg_attribute
where attrelid = 'external_data_plane_current'::regclass
and attname = 'geometry'
and not attisdropped
and format_type(atttypid, atttypmod) = 'geography(Point,4326)'
) then
alter table external_data_plane_current
alter column geometry type geography(Geometry, 4326)
using geometry::geometry::geography;
end if;
end
$$
`);
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)"); await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)"); await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
@ -282,10 +301,14 @@ export async function migrate(pool) {
provider_id text not null, provider_id text not null,
data_product_id text not null, data_product_id text not null,
current_cursor bigint not null default 0, current_cursor bigint not null default 0,
current_generation_at timestamptz,
current_generation_id text,
updated_at timestamptz not null default now(), updated_at timestamptz not null default now(),
primary key (tenant_id, connection_id, provider_id, data_product_id) primary key (tenant_id, connection_id, provider_id, data_product_id)
) )
`); `);
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_at timestamptz");
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_id text");
await pool.query(` await pool.query(`
create table if not exists external_data_plane_patch_outbox ( create table if not exists external_data_plane_patch_outbox (
@ -316,7 +339,7 @@ export async function migrate(pool) {
observed_at timestamptz not null, observed_at timestamptz not null,
received_at timestamptz not null, received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb, attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326), geometry geography(Geometry, 4326),
fingerprint text not null, fingerprint text not null,
updated_at timestamptz not null default now(), updated_at timestamptz not null default now(),
primary key ( primary key (

View File

@ -208,6 +208,10 @@ export function materializeDataProductPublish(value, binding, definition, dataPr
sequence: value.batch?.sequence, sequence: value.batch?.sequence,
idempotencyKey: value.batch?.idempotencyKey, idempotencyKey: value.batch?.idempotencyKey,
receivedAt: now.toISOString(), receivedAt: now.toISOString(),
...(value.batch?.mode === "replace" ? {
mode: "replace",
generationAt: value.batch?.generationAt,
} : {}),
}, },
facts: value.facts, facts: value.facts,
}; };

View File

@ -205,6 +205,54 @@ try {
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid", (error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
); );
const zoneDefinition = normalizeDataProductDefinition({
id: "test.zones.current.v1",
version: "1.0.0",
ontologyRevision: "ontology.map.zone.v1",
deliveryMode: "snapshot+patch",
semanticTypes: ["map.zone"],
fields: ["display_name", "geometry", "geometry_kind", "source_kind", "source_revision"],
fieldContracts: {
display_name: { type: "string", required: true },
geometry: { type: "geometry", required: true },
geometry_kind: { type: "string", required: true, enum: ["polygon"] },
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
source_revision: { type: "string", required: true },
},
history: { mode: "none", retentionDays: 1 },
});
await persistDataProductDefinition(pool, zoneDefinition);
const storedZoneDefinition = await loadDataProductDefinition(pool, zoneDefinition.id);
const zoneBinding = { ...binding, allowedDataProductIds: [zoneDefinition.id] };
const firstZoneGenerationAt = "2026-07-15T10:10:00.000Z";
const firstZoneGeneration = await persistDataProductPublish(
pool,
zoneBatch("zones-01", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt), zoneFact("zone-02", firstZoneGenerationAt)]),
storedZoneDefinition,
);
assert.equal(firstZoneGeneration.currentUpdatedCount, 2);
assert.equal(firstZoneGeneration.currentRemovedCount, 0);
const secondZoneGenerationAt = "2026-07-15T10:11:00.000Z";
const secondZoneGeneration = await persistDataProductPublish(
pool,
zoneBatch("zones-02", secondZoneGenerationAt, [zoneFact("zone-01", secondZoneGenerationAt)]),
storedZoneDefinition,
);
assert.equal(secondZoneGeneration.currentUpdatedCount, 1);
assert.equal(secondZoneGeneration.currentRemovedCount, 1);
assert.equal(secondZoneGeneration.patchOperationCount, 2);
const zoneSnapshot = await readDataProductSnapshot(pool, zoneBinding, storedZoneDefinition);
assert.equal(validateDataProductSnapshot(zoneSnapshot).ok, true);
assert.deepEqual(zoneSnapshot.facts.map((value) => value.sourceId), ["zone-01"]);
assert.equal(zoneSnapshot.facts[0].geometry.type, "Polygon");
const zonePatches = await readPatchEvents(pool, zoneBinding, storedZoneDefinition, 0n);
assert.equal(zonePatches.every((value) => validateDataProductPatch(value).ok), true);
assert.equal(zonePatches.flatMap((value) => value.operations).some((value) => value.op === "remove" && value.sourceId === "zone-02"), true);
await assert.rejects(
persistDataProductPublish(pool, zoneBatch("zones-stale", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt)]), storedZoneDefinition),
(error) => error?.status === 409 && error?.code === "data_product_generation_not_newer",
);
console.log("external-data-plane delivery integration: ok"); console.log("external-data-plane delivery integration: ok");
} finally { } finally {
await pool.end(); await pool.end();
@ -233,3 +281,38 @@ function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10
geometry: { type: "Point", coordinates: [longitude, latitude] }, geometry: { type: "Point", coordinates: [longitude, latitude] },
}; };
} }
function zoneBatch(runId, generationAt, facts) {
return {
schemaVersion: "nodedc.external-provider-contract/v1",
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
contract: { dataProductId: "test.zones.current.v1", ontologyRevision: "ontology.map.zone.v1", version: "1.0.0" },
batch: {
runId,
sequence: 0,
idempotencyKey: `${runId}.chunk-0`,
receivedAt: generationAt,
mode: "replace",
generationAt,
},
facts,
};
}
function zoneFact(sourceId, observedAt) {
return {
sourceId,
semanticType: "map.zone",
observedAt,
attributes: {
display_name: sourceId,
geometry_kind: "polygon",
source_kind: "live_api",
source_revision: "gelios-rest-v1",
},
geometry: {
type: "Polygon",
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
},
};
}

View File

@ -11,6 +11,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [
"fleet.positions.current.v2", "fleet.positions.current.v2",
"fleet.positions.current.v3", "fleet.positions.current.v3",
"fleet.positions.current.v4", "fleet.positions.current.v4",
"map.zones.current.v1",
]); ]);
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]); assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1"); assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
@ -45,6 +46,10 @@ assert.deepEqual(bundled[3].fieldContracts.movement_state, {
required: true, required: true,
enum: ["moving", "stopped"], enum: ["moving", "stopped"],
}); });
assert.equal(bundled[4].version, "1.0.0");
assert.equal(bundled[4].ontologyRevision, "ontology.map.zone.v1");
assert.deepEqual(bundled[4].semanticTypes, ["map.zone"]);
assert.deepEqual(bundled[4].fieldContracts.geometry, { type: "geometry", required: true });
assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length); assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length);
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-")); const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));