feat(platform): add replaceable geozone data layer
This commit is contained in:
parent
8a7465cf0e
commit
3446f3edc2
|
|
@ -90,11 +90,11 @@ node descriptions used `usableAsTool`, so n8n 2.3.2 exposed six NDC runtime
|
||||||
types instead of the required three. It must not be activated, overwritten or
|
types instead of the required three. It must not be activated, overwritten or
|
||||||
deleted.
|
deleted.
|
||||||
|
|
||||||
The active predecessor is package version `0.1.2` at immutable release
|
The active predecessor is package version `0.1.4` at immutable release
|
||||||
`0.1.2-05e4b38b14b4a019`. History Read is package version `0.1.3`, built with
|
`0.1.4-59dc9f7882721d6a`. Package `0.1.5` adds the provider-neutral complete
|
||||||
patch id `n8n-nodes-ndc-history-read-20260717-004` and immutable release
|
snapshot replace mode used by `map.zones.current.v1`; its Platform staging
|
||||||
`0.1.3-3354149b5245e39a`. Staging remains inert; only the separately reviewed
|
remains inert until a separately reviewed Engine-owned transition selects the
|
||||||
Engine-owned transition may select it after exact MCP schema acceptance.
|
exact digest and accepts the updated MCP node schema.
|
||||||
|
|
||||||
The paired Engine activation is built by
|
The paired Engine activation is built by
|
||||||
`build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not
|
`build-engine-n8n-private-extension-artifact.mjs`. It deliberately does not
|
||||||
|
|
|
||||||
|
|
@ -22,10 +22,12 @@ const files = [
|
||||||
["packages/external-provider-contract/src/contract-version.mjs", "platform/packages/external-provider-contract/src/contract-version.mjs"],
|
["packages/external-provider-contract/src/contract-version.mjs", "platform/packages/external-provider-contract/src/contract-version.mjs"],
|
||||||
["packages/external-provider-contract/src/data-plane.mjs", "platform/packages/external-provider-contract/src/data-plane.mjs"],
|
["packages/external-provider-contract/src/data-plane.mjs", "platform/packages/external-provider-contract/src/data-plane.mjs"],
|
||||||
["packages/external-provider-contract/src/data-product.mjs", "platform/packages/external-provider-contract/src/data-product.mjs"],
|
["packages/external-provider-contract/src/data-product.mjs", "platform/packages/external-provider-contract/src/data-product.mjs"],
|
||||||
|
["packages/external-provider-contract/src/geometry.mjs", "platform/packages/external-provider-contract/src/geometry.mjs"],
|
||||||
["packages/external-provider-contract/src/intake-batch.mjs", "platform/packages/external-provider-contract/src/intake-batch.mjs"],
|
["packages/external-provider-contract/src/intake-batch.mjs", "platform/packages/external-provider-contract/src/intake-batch.mjs"],
|
||||||
["packages/external-provider-contract/src/index.mjs", "platform/packages/external-provider-contract/src/index.mjs"],
|
["packages/external-provider-contract/src/index.mjs", "platform/packages/external-provider-contract/src/index.mjs"],
|
||||||
["packages/external-provider-contract/src/provider-package.mjs", "platform/packages/external-provider-contract/src/provider-package.mjs"],
|
["packages/external-provider-contract/src/provider-package.mjs", "platform/packages/external-provider-contract/src/provider-package.mjs"],
|
||||||
["packages/external-provider-contract/src/sensitive-field-policy.mjs", "platform/packages/external-provider-contract/src/sensitive-field-policy.mjs"],
|
["packages/external-provider-contract/src/sensitive-field-policy.mjs", "platform/packages/external-provider-contract/src/sensitive-field-policy.mjs"],
|
||||||
|
["packages/external-provider-contract/src/zone-source.mjs", "platform/packages/external-provider-contract/src/zone-source.mjs"],
|
||||||
["packages/external-provider-contract/providers/gelios", "platform/packages/external-provider-contract/providers/gelios"],
|
["packages/external-provider-contract/providers/gelios", "platform/packages/external-provider-contract/providers/gelios"],
|
||||||
];
|
];
|
||||||
const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]);
|
const ignoredBasenames = new Set([".DS_Store", ".git", "node_modules"]);
|
||||||
|
|
|
||||||
|
|
@ -12,8 +12,8 @@ const platformRoot = resolve(scriptDir, "../..");
|
||||||
const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc");
|
const packageRoot = resolve(platformRoot, "packages/n8n-nodes-ndc");
|
||||||
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
|
const artifactDir = resolve(process.env.NODEDC_DEPLOY_ARTIFACT_DIR || resolve(scriptDir, "../deploy-artifacts"));
|
||||||
const requireModule = createRequire(import.meta.url);
|
const requireModule = createRequire(import.meta.url);
|
||||||
const expectedPackageVersion = "0.1.4";
|
const expectedPackageVersion = "0.1.5";
|
||||||
const [patchId = "n8n-nodes-ndc-provider-rotating-access-20260718-006", ...extra] = process.argv.slice(2);
|
const [patchId = "n8n-nodes-ndc-geozone-replace-20260720-017", ...extra] = process.argv.slice(2);
|
||||||
const expectedRuntimeNodes = [
|
const expectedRuntimeNodes = [
|
||||||
{
|
{
|
||||||
file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
|
file: "dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
|
||||||
|
|
|
||||||
|
|
@ -19,7 +19,7 @@ RUNNER_PATH = Path(
|
||||||
os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy")
|
os.environ.get("NODEDC_DEPLOY_RUNNER_UNDER_TEST", SCRIPT_DIR / "nodedc-deploy")
|
||||||
).resolve()
|
).resolve()
|
||||||
BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs"
|
BUILDER_PATH = SCRIPT_DIR / "build-n8n-private-extension-artifact.mjs"
|
||||||
PATCH_ID = "n8n-nodes-ndc-provider-rotating-access-20260718-006"
|
PATCH_ID = "n8n-nodes-ndc-geozone-replace-20260720-017"
|
||||||
EXPECTED_NODES = [
|
EXPECTED_NODES = [
|
||||||
"dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
|
"dist/nodes/NdcDataProductPublish/NdcDataProductPublish.node.js",
|
||||||
"dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js",
|
"dist/nodes/NdcDataProductRead/NdcDataProductRead.node.js",
|
||||||
|
|
@ -110,7 +110,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
|
||||||
manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory))
|
manifest, entries, _payload = RUNNER.load_artifact(self.artifacts[0], Path(directory))
|
||||||
self.assertEqual(manifest["component"], "n8n-private-extension")
|
self.assertEqual(manifest["component"], "n8n-private-extension")
|
||||||
self.assertEqual(len(entries), 1)
|
self.assertEqual(len(entries), 1)
|
||||||
self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.4-[a-f0-9]{16}$")
|
self.assertRegex(entries[0], r"^releases/n8n-nodes-ndc/0\.1\.5-[a-f0-9]{16}$")
|
||||||
|
|
||||||
with tarfile.open(self.artifacts[0], "r:gz") as archive:
|
with tarfile.open(self.artifacts[0], "r:gz") as archive:
|
||||||
names = archive.getnames()
|
names = archive.getnames()
|
||||||
|
|
@ -122,7 +122,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
|
||||||
with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive:
|
with tarfile.open(fileobj=io.BytesIO(package), mode="r:gz") as archive:
|
||||||
package_json_member = archive.getmember("package/package.json")
|
package_json_member = archive.getmember("package/package.json")
|
||||||
package_json = json.loads(archive.extractfile(package_json_member).read())
|
package_json = json.loads(archive.extractfile(package_json_member).read())
|
||||||
self.assertEqual(package_json["version"], "0.1.4")
|
self.assertEqual(package_json["version"], "0.1.5")
|
||||||
self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES)
|
self.assertEqual(package_json["n8n"]["nodes"], EXPECTED_NODES)
|
||||||
self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS)
|
self.assertEqual(package_json["n8n"]["credentials"], EXPECTED_CREDENTIALS)
|
||||||
for node_path in EXPECTED_NODES:
|
for node_path in EXPECTED_NODES:
|
||||||
|
|
@ -158,7 +158,7 @@ class N8nPrivateExtensionPolicyTest(unittest.TestCase):
|
||||||
"requiresPreActivationVerification": True,
|
"requiresPreActivationVerification": True,
|
||||||
}
|
}
|
||||||
self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2")
|
self.assertEqual(release["schemaVersion"], "nodedc.n8n-private-extension-release/v2")
|
||||||
self.assertEqual(release["package"]["version"], "0.1.4")
|
self.assertEqual(release["package"]["version"], "0.1.5")
|
||||||
self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy)
|
self.assertEqual(release["activation"]["rollbackBaselinePolicy"], expected_policy)
|
||||||
self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2")
|
self.assertEqual(rollback["schemaVersion"], "nodedc.n8n-private-extension-rollback/v2")
|
||||||
self.assertEqual(rollback["baselinePolicy"], expected_policy)
|
self.assertEqual(rollback["baselinePolicy"], expected_policy)
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,6 @@
|
||||||
"./providers/*": "./providers/*/index.mjs"
|
"./providers/*": "./providers/*/index.mjs"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/engine-private-extension.test.mjs"
|
"check": "node test/contract.test.mjs && node test/data-product.test.mjs && node test/provider-package.test.mjs && node test/zone-source.test.mjs && node test/engine-private-extension.test.mjs"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,13 @@
|
||||||
|
# Gelios provider package v6
|
||||||
|
|
||||||
|
Adds the bounded, offset-paged `gelios.geozone` read capability and maps a
|
||||||
|
complete accepted source generation into the provider-neutral
|
||||||
|
`map.zones.current.v1` Data Product. The runtime publisher must use replace
|
||||||
|
mode: a partial provider page set never reaches Publish, while a complete newer
|
||||||
|
generation atomically upserts present zones and emits remove patches for zones
|
||||||
|
that disappeared.
|
||||||
|
|
||||||
|
`gelios-rest-v1` and `mmap-snapshot-v1` are source adapters, not Foundry
|
||||||
|
bindings. Both must preserve the native Gelios zone id (or use an explicit
|
||||||
|
crosswalk) and produce the same `map.zone` facts. Foundry only consumes the
|
||||||
|
provider-neutral product.
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
export {
|
||||||
|
GELIOS_PROVIDER_PACKAGE_ID,
|
||||||
|
GELIOS_PROVIDER_PACKAGE_VERSION,
|
||||||
|
GELIOS_ZONES_DATA_PRODUCT_ID,
|
||||||
|
GELIOS_ZONES_DATA_PRODUCT_VERSION,
|
||||||
|
GELIOS_ZONES_ONTOLOGY_REVISION,
|
||||||
|
GELIOS_ZONE_SOURCE_ID_PREFIX,
|
||||||
|
geliosProviderPackageV6,
|
||||||
|
} from "./package.mjs";
|
||||||
|
|
@ -0,0 +1,192 @@
|
||||||
|
import { geliosProviderPackageV5 } from "../v5/package.mjs";
|
||||||
|
|
||||||
|
export const GELIOS_PROVIDER_PACKAGE_ID = "gelios.provider.v6";
|
||||||
|
export const GELIOS_PROVIDER_PACKAGE_VERSION = "6.0.0";
|
||||||
|
export const GELIOS_ZONES_DATA_PRODUCT_ID = "map.zones.current.v1";
|
||||||
|
export const GELIOS_ZONES_DATA_PRODUCT_VERSION = "1.0.0";
|
||||||
|
export const GELIOS_ZONES_ONTOLOGY_REVISION = "ontology.map.zone.v1";
|
||||||
|
export const GELIOS_ZONE_SOURCE_ID_PREFIX = "gelios-zone-";
|
||||||
|
|
||||||
|
const ZONES_CAPABILITY_ID = "gelios.geozones.current.read";
|
||||||
|
const ZONES_FIELD_POLICY_ID = "gelios.geozones.current.fields.v1";
|
||||||
|
const ZONES_PROFILE_ID = "gelios.geozones.current.realtime.v1";
|
||||||
|
const ZONES_MAPPING_ID = "gelios.geozones.to.map.zones.current.v1";
|
||||||
|
const ZONES_TEMPLATE_ID = "gelios.geozones.current.l2.v1";
|
||||||
|
|
||||||
|
const value = structuredClone(geliosProviderPackageV5);
|
||||||
|
const authModeId = value.authModes[0].id;
|
||||||
|
const zonesFields = [
|
||||||
|
"area_square_meters",
|
||||||
|
"description",
|
||||||
|
"display_name",
|
||||||
|
"geometry",
|
||||||
|
"geometry_kind",
|
||||||
|
"max_speed_kph",
|
||||||
|
"perimeter_meters",
|
||||||
|
"source_kind",
|
||||||
|
"source_revision",
|
||||||
|
"style_color",
|
||||||
|
];
|
||||||
|
|
||||||
|
value.id = GELIOS_PROVIDER_PACKAGE_ID;
|
||||||
|
value.version = GELIOS_PROVIDER_PACKAGE_VERSION;
|
||||||
|
value.manifest = {
|
||||||
|
...value.manifest,
|
||||||
|
id: "gelios.provider.manifest.v6",
|
||||||
|
version: GELIOS_PROVIDER_PACKAGE_VERSION,
|
||||||
|
capabilityIds: [...value.manifest.capabilityIds, ZONES_CAPABILITY_ID],
|
||||||
|
fieldPolicyIds: [...value.manifest.fieldPolicyIds, ZONES_FIELD_POLICY_ID],
|
||||||
|
collectionProfileIds: [...value.manifest.collectionProfileIds, ZONES_PROFILE_ID],
|
||||||
|
dataProductIds: [...value.manifest.dataProductIds, GELIOS_ZONES_DATA_PRODUCT_ID],
|
||||||
|
mappingContractIds: [...value.manifest.mappingContractIds, ZONES_MAPPING_ID],
|
||||||
|
l2TemplateIds: [...value.manifest.l2TemplateIds, ZONES_TEMPLATE_ID],
|
||||||
|
};
|
||||||
|
value.capabilities.push({
|
||||||
|
id: ZONES_CAPABILITY_ID,
|
||||||
|
classification: "read",
|
||||||
|
status: "implemented",
|
||||||
|
authModeId,
|
||||||
|
request: {
|
||||||
|
method: "GET",
|
||||||
|
baseUrl: "https://api.geliospro.com",
|
||||||
|
path: "/api/v1/geozones",
|
||||||
|
query: { pl: 100, po: 0 },
|
||||||
|
response: {
|
||||||
|
collectionPaths: ["items"],
|
||||||
|
pagination: {
|
||||||
|
mode: "offset",
|
||||||
|
limitParameter: "pl",
|
||||||
|
offsetParameter: "po",
|
||||||
|
pageSize: 100,
|
||||||
|
itemsPath: "items",
|
||||||
|
totalPath: "paginationMetadata.totalCount",
|
||||||
|
maxPages: 50,
|
||||||
|
maxItems: 5000,
|
||||||
|
maxResponseBytes: 16 * 1024 * 1024,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
entityScope: {
|
||||||
|
mode: "all_visible_to_credential",
|
||||||
|
refresh: "each_collection_run",
|
||||||
|
businessEntityFilter: "forbidden",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
value.fieldPolicies.push({
|
||||||
|
id: ZONES_FIELD_POLICY_ID,
|
||||||
|
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
|
||||||
|
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
|
||||||
|
targetFields: [...zonesFields],
|
||||||
|
unknownSourceFields: "drop",
|
||||||
|
dynamicSourceFields: "drop_until_classified",
|
||||||
|
restrictedSourcePaths: ["accessToken", "authorization", "refreshToken", "userToken"],
|
||||||
|
});
|
||||||
|
value.collectionProfiles.push({
|
||||||
|
id: ZONES_PROFILE_ID,
|
||||||
|
version: GELIOS_PROVIDER_PACKAGE_VERSION,
|
||||||
|
mode: "realtime",
|
||||||
|
schedule: { intervalMs: 3_600_000 },
|
||||||
|
capabilityIds: [ZONES_CAPABILITY_ID],
|
||||||
|
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
|
||||||
|
mappingContractId: ZONES_MAPPING_ID,
|
||||||
|
fieldPolicyId: ZONES_FIELD_POLICY_ID,
|
||||||
|
l2TemplateId: ZONES_TEMPLATE_ID,
|
||||||
|
entityScope: {
|
||||||
|
mode: "all_visible_to_credential",
|
||||||
|
refresh: "each_collection_run",
|
||||||
|
businessEntityFilter: "forbidden",
|
||||||
|
},
|
||||||
|
batching: { maxFacts: 5000 },
|
||||||
|
cardinality: { maxCurrentEntities: 5000, onExceed: "require_partitioned_data_product" },
|
||||||
|
retry: { maxAttempts: 4, backoff: "exponential_with_jitter" },
|
||||||
|
});
|
||||||
|
value.dataProducts.push({
|
||||||
|
id: GELIOS_ZONES_DATA_PRODUCT_ID,
|
||||||
|
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
|
||||||
|
ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION,
|
||||||
|
deliveryMode: "snapshot+patch",
|
||||||
|
semanticTypes: ["map.zone"],
|
||||||
|
fields: [...zonesFields],
|
||||||
|
fieldContracts: {
|
||||||
|
area_square_meters: { type: "number", required: false, minimum: 0 },
|
||||||
|
description: { type: "string", required: false },
|
||||||
|
display_name: { type: "string", required: true },
|
||||||
|
geometry: { type: "geometry", required: true },
|
||||||
|
geometry_kind: { type: "string", required: true, enum: ["circle", "corridor", "polygon"] },
|
||||||
|
max_speed_kph: { type: "number", required: false, minimum: 0 },
|
||||||
|
perimeter_meters: { type: "number", required: false, minimum: 0 },
|
||||||
|
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
|
||||||
|
source_revision: { type: "string", required: true },
|
||||||
|
style_color: { type: "string", required: false },
|
||||||
|
},
|
||||||
|
history: { mode: "none", retentionDays: 1 },
|
||||||
|
});
|
||||||
|
value.mappingContracts.push({
|
||||||
|
schemaVersion: "nodedc.semantic-mapping/v1",
|
||||||
|
id: ZONES_MAPPING_ID,
|
||||||
|
version: GELIOS_PROVIDER_PACKAGE_VERSION,
|
||||||
|
sourceCapabilityId: ZONES_CAPABILITY_ID,
|
||||||
|
fieldPolicyId: ZONES_FIELD_POLICY_ID,
|
||||||
|
target: {
|
||||||
|
dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID,
|
||||||
|
version: GELIOS_ZONES_DATA_PRODUCT_VERSION,
|
||||||
|
ontologyRevision: GELIOS_ZONES_ONTOLOGY_REVISION,
|
||||||
|
semanticType: "map.zone",
|
||||||
|
},
|
||||||
|
derivations: {
|
||||||
|
geometry_kind: {
|
||||||
|
kind: "ordered_rules",
|
||||||
|
rules: ["type_circle.circle", "type_line.corridor", "otherwise.polygon"],
|
||||||
|
default: "polygon",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fact: {
|
||||||
|
sourceId: { strategy: "first_non_empty", paths: ["id"], coerce: "string", prefix: GELIOS_ZONE_SOURCE_ID_PREFIX },
|
||||||
|
semanticType: { constant: "map.zone" },
|
||||||
|
observedAt: { strategy: "first_non_empty", paths: ["updatedAt"], coerce: "unix_or_iso_timestamp", fallback: "collection_received_at" },
|
||||||
|
geometry: {
|
||||||
|
type: "GeoJSON",
|
||||||
|
strategy: "gelios_geozone_v1",
|
||||||
|
paths: ["type", "points", "line", "radius"],
|
||||||
|
allowedTypes: ["Polygon", "MultiPolygon"],
|
||||||
|
omitIfInvalid: false,
|
||||||
|
},
|
||||||
|
attributes: {
|
||||||
|
area_square_meters: { strategy: "first_non_empty", paths: ["surfaceArea", "surface_area"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
||||||
|
description: { strategy: "first_non_empty", paths: ["description", "descr"], coerce: "string", omitIfMissing: true },
|
||||||
|
display_name: { strategy: "first_non_empty", paths: ["name"], coerce: "string" },
|
||||||
|
geometry_kind: { derive: "geometry_kind" },
|
||||||
|
max_speed_kph: { strategy: "first_non_empty", paths: ["maxPermissibleSpeed", "max_permissible_speed"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
||||||
|
perimeter_meters: { strategy: "first_non_empty", paths: ["perimeter"], coerce: "number", minimum: 0, omitIfInvalid: true },
|
||||||
|
source_kind: { constant: "live_api" },
|
||||||
|
source_revision: { constant: "gelios-rest-v1" },
|
||||||
|
style_color: { strategy: "first_non_empty", paths: ["color"], coerce: "string", omitIfMissing: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
value.l2Templates.push({
|
||||||
|
schemaVersion: "nodedc.l2-template/v1",
|
||||||
|
id: ZONES_TEMPLATE_ID,
|
||||||
|
version: GELIOS_PROVIDER_PACKAGE_VERSION,
|
||||||
|
runtime: "ndc_l2",
|
||||||
|
instanceMode: "one_connection_per_workflow",
|
||||||
|
connectionParameters: ["tenant_id", "connection_id", "collection_profile_id", "provider_credential_ref"],
|
||||||
|
credentialBindings: structuredClone(value.l2Templates[0].credentialBindings),
|
||||||
|
steps: [
|
||||||
|
{ id: "collection.trigger", kind: "collection_trigger", collectionProfileDriven: true },
|
||||||
|
{ id: "provider.fetch-geozones", kind: "provider_request", capabilityId: ZONES_CAPABILITY_ID },
|
||||||
|
{ id: "provider.extract-geozones", kind: "extract_items", capabilityId: ZONES_CAPABILITY_ID },
|
||||||
|
{ id: "ontology.map", kind: "semantic_mapping", mappingContractId: ZONES_MAPPING_ID },
|
||||||
|
{ id: "data-product.publish", kind: "data_product_publish", dataProductId: GELIOS_ZONES_DATA_PRODUCT_ID, nodeType: "n8n-nodes-ndc.ndcDataProductPublish" },
|
||||||
|
],
|
||||||
|
invariants: structuredClone(value.l2Templates[0].invariants),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const geliosProviderPackageV6 = deepFreeze(value);
|
||||||
|
|
||||||
|
function deepFreeze(input) {
|
||||||
|
if (!input || typeof input !== "object" || Object.isFrozen(input)) return input;
|
||||||
|
Object.freeze(input);
|
||||||
|
for (const child of Object.values(input)) deepFreeze(child);
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
@ -1,5 +1,12 @@
|
||||||
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
||||||
export { validateIntakeBatch } from "./intake-batch.mjs";
|
export { validateIntakeBatch } from "./intake-batch.mjs";
|
||||||
|
export {
|
||||||
|
DATA_PRODUCT_GEOMETRY_TYPES,
|
||||||
|
DATA_PRODUCT_MAX_GEOMETRY_BYTES,
|
||||||
|
DATA_PRODUCT_MAX_GEOMETRY_VERTICES,
|
||||||
|
isBoundedGeoJsonGeometry,
|
||||||
|
validateGeoJsonGeometry,
|
||||||
|
} from "./geometry.mjs";
|
||||||
export {
|
export {
|
||||||
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ export const DATA_PRODUCT_PATCH_SCHEMA_VERSION = "nodedc.data-product.patch/v1";
|
||||||
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
|
export const DATA_PRODUCT_HISTORY_SCHEMA_VERSION = "nodedc.data-product.history/v1";
|
||||||
|
|
||||||
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
|
import { SECRET_LIKE_KEY, SECRET_LIKE_VALUE } from "./sensitive-field-policy.mjs";
|
||||||
|
import { validateGeoJsonGeometry } from "./geometry.mjs";
|
||||||
|
|
||||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||||
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
||||||
|
|
@ -30,22 +31,33 @@ export function validateDataProductPublish(value, { maxFacts = 5000, maxAttribut
|
||||||
if (!isPlainObject(value.batch)) {
|
if (!isPlainObject(value.batch)) {
|
||||||
errors.push("batch_must_be_object");
|
errors.push("batch_must_be_object");
|
||||||
} else {
|
} else {
|
||||||
rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey"]), "batch", errors);
|
rejectUnknownKeys(value.batch, new Set(["runId", "sequence", "idempotencyKey", "mode", "generationAt"]), "batch", errors);
|
||||||
requiredIdentifier(value.batch.runId, "batch.runId", errors);
|
requiredIdentifier(value.batch.runId, "batch.runId", errors);
|
||||||
requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors);
|
requiredIdentifier(value.batch.idempotencyKey, "batch.idempotencyKey", errors);
|
||||||
if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) {
|
if (!Number.isInteger(value.batch.sequence) || value.batch.sequence < 0 || value.batch.sequence > MAX_BATCH_SEQUENCE) {
|
||||||
errors.push("batch.sequence_must_be_integer_0_to_2147483647");
|
errors.push("batch.sequence_must_be_integer_0_to_2147483647");
|
||||||
}
|
}
|
||||||
|
const mode = value.batch.mode === undefined ? "upsert" : value.batch.mode;
|
||||||
|
if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid");
|
||||||
|
if (mode === "replace") requiredIsoTimestamp(value.batch.generationAt, "batch.generationAt", errors);
|
||||||
|
if (mode === "upsert" && value.batch.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Array.isArray(value.facts) || value.facts.length === 0) {
|
if (!Array.isArray(value.facts)) {
|
||||||
errors.push("facts_must_be_nonempty_array");
|
errors.push("facts_must_be_array");
|
||||||
|
} else if (value.facts.length === 0 && value.batch?.mode !== "replace") {
|
||||||
|
errors.push("facts_must_be_nonempty_array_for_upsert");
|
||||||
} else if (value.facts.length > maxFacts) {
|
} else if (value.facts.length > maxFacts) {
|
||||||
errors.push("facts_limit_exceeded");
|
errors.push("facts_limit_exceeded");
|
||||||
} else {
|
} else {
|
||||||
const entityKeys = new Set();
|
const entityKeys = new Set();
|
||||||
value.facts.forEach((fact, index) => {
|
value.facts.forEach((fact, index) => {
|
||||||
validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling });
|
validateFact(fact, `facts[${index}]`, errors, { maxAttributesBytes: attributesCeiling });
|
||||||
|
if (value.batch?.mode === "replace"
|
||||||
|
&& !Number.isNaN(Date.parse(fact?.observedAt))
|
||||||
|
&& Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) {
|
||||||
|
errors.push(`facts[${index}].observedAt_must_equal_generationAt`);
|
||||||
|
}
|
||||||
if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return;
|
if (!isPlainObject(fact) || typeof fact.sourceId !== "string" || typeof fact.semanticType !== "string") return;
|
||||||
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
||||||
if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key");
|
if (entityKeys.has(entityKey)) errors.push("facts_duplicate_entity_key");
|
||||||
|
|
@ -143,12 +155,18 @@ export function validateDataProductPatch(value) {
|
||||||
errors.push("operations_must_be_nonempty_array");
|
errors.push("operations_must_be_nonempty_array");
|
||||||
} else {
|
} else {
|
||||||
value.operations.forEach((operation, index) => {
|
value.operations.forEach((operation, index) => {
|
||||||
if (!isPlainObject(operation) || operation.op !== "upsert") {
|
if (!isPlainObject(operation) || !new Set(["upsert", "remove"]).has(operation.op)) {
|
||||||
errors.push(`operations[${index}].op_must_be_upsert`);
|
errors.push(`operations[${index}].op_invalid`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors);
|
if (operation.op === "upsert") {
|
||||||
validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors);
|
rejectUnknownKeys(operation, new Set(["op", "fact"]), `operations[${index}]`, errors);
|
||||||
|
validateCanonicalFact(operation.fact, `operations[${index}].fact`, errors);
|
||||||
|
} else {
|
||||||
|
rejectUnknownKeys(operation, new Set(["op", "sourceId", "semanticType"]), `operations[${index}]`, errors);
|
||||||
|
requiredIdentifier(operation.sourceId, `operations[${index}].sourceId`, errors);
|
||||||
|
requiredIdentifier(operation.semanticType, `operations[${index}].semanticType`, errors);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material");
|
if (containsSecretLikeMaterial(value)) errors.push("patch_must_not_contain_secret_material");
|
||||||
|
|
@ -189,7 +207,7 @@ function validateFact(value, path, errors, { maxAttributesBytes, canonical = fal
|
||||||
errors.push(`${path}.attributes_size_exceeded`);
|
errors.push(`${path}.attributes_size_exceeded`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors);
|
if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
|
function validateCanonicalFact(value, path, errors, { allowBucketStart = false } = {}) {
|
||||||
|
|
@ -198,21 +216,6 @@ function validateCanonicalFact(value, path, errors, { allowBucketStart = false }
|
||||||
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
|
requiredIsoTimestamp(value.receivedAt, `${path}.receivedAt`, errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
function validatePointGeometry(value, path, errors) {
|
|
||||||
if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) {
|
|
||||||
errors.push(`${path}_must_be_geojson_point`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
|
|
||||||
if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) {
|
|
||||||
errors.push(`${path}_coordinates_must_be_finite_numbers`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const [longitude, latitude] = value.coordinates;
|
|
||||||
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
|
|
||||||
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
|
|
||||||
}
|
|
||||||
|
|
||||||
function rejectUnknownKeys(value, allowed, path, errors) {
|
function rejectUnknownKeys(value, allowed, path, errors) {
|
||||||
if (!isPlainObject(value)) return;
|
if (!isPlainObject(value)) return;
|
||||||
for (const key of Object.keys(value)) {
|
for (const key of Object.keys(value)) {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,103 @@
|
||||||
|
const GEOMETRY_TYPES = new Set(["Point", "LineString", "Polygon", "MultiPolygon"]);
|
||||||
|
const MAX_GEOMETRY_BYTES = 192 * 1024;
|
||||||
|
const MAX_GEOMETRY_VERTICES = 10_000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate the bounded GeoJSON geometry subset accepted by provider-neutral
|
||||||
|
* Data Products. GeometryCollections and arbitrary feature properties are
|
||||||
|
* deliberately excluded: a fact owns exactly one geometry and its attributes
|
||||||
|
* stay under the fact field policy.
|
||||||
|
*/
|
||||||
|
export function validateGeoJsonGeometry(value, path, errors, { allowedTypes = GEOMETRY_TYPES } = {}) {
|
||||||
|
if (!isPlainObject(value) || !allowedTypes.has(value.type) || !Array.isArray(value.coordinates)) {
|
||||||
|
errors.push(`${path}_must_be_bounded_geojson_geometry`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
|
||||||
|
if (serializedByteLength(value) > MAX_GEOMETRY_BYTES) {
|
||||||
|
errors.push(`${path}_size_exceeded`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const vertices = [];
|
||||||
|
if (value.type === "Point") {
|
||||||
|
validatePosition(value.coordinates, path, errors, vertices);
|
||||||
|
} else if (value.type === "LineString") {
|
||||||
|
validateLineString(value.coordinates, path, errors, vertices);
|
||||||
|
} else if (value.type === "Polygon") {
|
||||||
|
validatePolygon(value.coordinates, path, errors, vertices);
|
||||||
|
} else if (value.type === "MultiPolygon") {
|
||||||
|
if (!value.coordinates.length) errors.push(`${path}.coordinates_must_be_nonempty`);
|
||||||
|
value.coordinates.forEach((polygon, index) => validatePolygon(polygon, `${path}.coordinates[${index}]`, errors, vertices));
|
||||||
|
}
|
||||||
|
if (vertices.length > MAX_GEOMETRY_VERTICES) errors.push(`${path}_vertex_limit_exceeded`);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isBoundedGeoJsonGeometry(value, allowedTypes = GEOMETRY_TYPES) {
|
||||||
|
const errors = [];
|
||||||
|
validateGeoJsonGeometry(value, "geometry", errors, { allowedTypes });
|
||||||
|
return errors.length === 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
function validatePolygon(value, path, errors, vertices) {
|
||||||
|
if (!Array.isArray(value) || !value.length) {
|
||||||
|
errors.push(`${path}_polygon_rings_required`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
value.forEach((ring, index) => {
|
||||||
|
const ringPath = `${path}[${index}]`;
|
||||||
|
if (!Array.isArray(ring) || ring.length < 4) {
|
||||||
|
errors.push(`${ringPath}_linear_ring_requires_four_positions`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
ring.forEach((position, positionIndex) => validatePosition(position, `${ringPath}[${positionIndex}]`, errors, vertices));
|
||||||
|
if (!samePosition(ring[0], ring.at(-1))) errors.push(`${ringPath}_linear_ring_must_be_closed`);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateLineString(value, path, errors, vertices) {
|
||||||
|
if (!Array.isArray(value) || value.length < 2) {
|
||||||
|
errors.push(`${path}_line_string_requires_two_positions`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
value.forEach((position, index) => validatePosition(position, `${path}[${index}]`, errors, vertices));
|
||||||
|
}
|
||||||
|
|
||||||
|
function validatePosition(value, path, errors, vertices) {
|
||||||
|
if (!Array.isArray(value) || value.length !== 2 || !value.every(Number.isFinite)) {
|
||||||
|
errors.push(`${path}_position_invalid`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const [longitude, latitude] = value;
|
||||||
|
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
|
||||||
|
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
|
||||||
|
vertices.push(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function samePosition(left, right) {
|
||||||
|
return Array.isArray(left) && Array.isArray(right)
|
||||||
|
&& left.length === 2 && right.length === 2
|
||||||
|
&& Object.is(left[0], right[0]) && Object.is(left[1], right[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
function rejectUnknownKeys(value, allowed, path, errors) {
|
||||||
|
for (const key of Object.keys(value)) {
|
||||||
|
if (!allowed.has(key)) errors.push(`${path}.${key}_not_allowed`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function serializedByteLength(value) {
|
||||||
|
try {
|
||||||
|
return Buffer.byteLength(JSON.stringify(value));
|
||||||
|
} catch {
|
||||||
|
return Number.POSITIVE_INFINITY;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlainObject(value) {
|
||||||
|
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DATA_PRODUCT_GEOMETRY_TYPES = Object.freeze([...GEOMETRY_TYPES]);
|
||||||
|
export const DATA_PRODUCT_MAX_GEOMETRY_BYTES = MAX_GEOMETRY_BYTES;
|
||||||
|
export const DATA_PRODUCT_MAX_GEOMETRY_VERTICES = MAX_GEOMETRY_VERTICES;
|
||||||
|
|
@ -2,6 +2,18 @@ import { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
||||||
|
|
||||||
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
export { EXTERNAL_PROVIDER_CONTRACT_VERSION } from "./contract-version.mjs";
|
||||||
export { validateIntakeBatch } from "./intake-batch.mjs";
|
export { validateIntakeBatch } from "./intake-batch.mjs";
|
||||||
|
export {
|
||||||
|
DATA_PRODUCT_GEOMETRY_TYPES,
|
||||||
|
DATA_PRODUCT_MAX_GEOMETRY_BYTES,
|
||||||
|
DATA_PRODUCT_MAX_GEOMETRY_VERTICES,
|
||||||
|
isBoundedGeoJsonGeometry,
|
||||||
|
validateGeoJsonGeometry,
|
||||||
|
} from "./geometry.mjs";
|
||||||
|
export {
|
||||||
|
ZONE_SOURCE_ADAPTERS,
|
||||||
|
ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
normalizeZoneSourceGeneration,
|
||||||
|
} from "./zone-source.mjs";
|
||||||
export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1";
|
export const FOUNDRY_BINDING_UPSERT_SCHEMA_VERSION = "nodedc.foundry.binding-upsert/v1";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import {
|
||||||
SECRET_LIKE_REFERENCE,
|
SECRET_LIKE_REFERENCE,
|
||||||
SECRET_LIKE_VALUE,
|
SECRET_LIKE_VALUE,
|
||||||
} from "./sensitive-field-policy.mjs";
|
} from "./sensitive-field-policy.mjs";
|
||||||
|
import { validateGeoJsonGeometry } from "./geometry.mjs";
|
||||||
|
|
||||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||||
const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
const CONTRACT_VERSION = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
||||||
|
|
@ -16,7 +17,7 @@ export function validateIntakeBatch(value) {
|
||||||
rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors);
|
rejectUnknownKeys(value, new Set(["schemaVersion", "source", "contract", "batch", "raw", "facts"]), "intakeBatch", errors);
|
||||||
rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors);
|
rejectUnknownKeys(value?.source, new Set(["providerId", "tenantId", "connectionId"]), "source", errors);
|
||||||
rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors);
|
rejectUnknownKeys(value?.contract, new Set(["dataProductId", "ontologyRevision", "version"]), "contract", errors);
|
||||||
rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt"]), "batch", errors);
|
rejectUnknownKeys(value?.batch, new Set(["runId", "sequence", "idempotencyKey", "receivedAt", "mode", "generationAt"]), "batch", errors);
|
||||||
requiredIdentifier(value?.source?.providerId, "source.providerId", errors);
|
requiredIdentifier(value?.source?.providerId, "source.providerId", errors);
|
||||||
requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors);
|
requiredIdentifier(value?.source?.tenantId, "source.tenantId", errors);
|
||||||
requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors);
|
requiredIdentifier(value?.source?.connectionId, "source.connectionId", errors);
|
||||||
|
|
@ -32,11 +33,24 @@ export function validateIntakeBatch(value) {
|
||||||
errors.push("batch.sequence_must_be_integer_0_to_2147483647");
|
errors.push("batch.sequence_must_be_integer_0_to_2147483647");
|
||||||
}
|
}
|
||||||
requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors);
|
requiredIsoTimestamp(value?.batch?.receivedAt, "batch.receivedAt", errors);
|
||||||
|
const mode = value?.batch?.mode === undefined ? "upsert" : value.batch.mode;
|
||||||
|
if (!new Set(["upsert", "replace"]).has(mode)) errors.push("batch.mode_invalid");
|
||||||
|
if (mode === "replace") requiredIsoTimestamp(value?.batch?.generationAt, "batch.generationAt", errors);
|
||||||
|
if (mode === "upsert" && value?.batch?.generationAt !== undefined) errors.push("batch.generationAt_forbidden_for_upsert");
|
||||||
|
|
||||||
if (!Array.isArray(value?.facts) || value.facts.length === 0) {
|
if (!Array.isArray(value?.facts)) {
|
||||||
errors.push("facts_must_be_nonempty_array");
|
errors.push("facts_must_be_array");
|
||||||
|
} else if (value.facts.length === 0 && mode !== "replace") {
|
||||||
|
errors.push("facts_must_be_nonempty_array_for_upsert");
|
||||||
} else {
|
} else {
|
||||||
value.facts.forEach((fact, index) => validateFact(fact, `facts[${index}]`, errors));
|
value.facts.forEach((fact, index) => {
|
||||||
|
validateFact(fact, `facts[${index}]`, errors);
|
||||||
|
if (mode === "replace"
|
||||||
|
&& !Number.isNaN(Date.parse(fact?.observedAt))
|
||||||
|
&& Date.parse(fact.observedAt) !== Date.parse(value.batch.generationAt)) {
|
||||||
|
errors.push(`facts[${index}].observedAt_must_equal_generationAt`);
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors);
|
if (value?.raw !== undefined) validateRawEnvelope(value.raw, errors);
|
||||||
|
|
@ -80,22 +94,7 @@ function validateFact(value, path, errors) {
|
||||||
errors.push(`${path}.attributes_size_exceeded`);
|
errors.push(`${path}.attributes_size_exceeded`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (value.geometry !== undefined) validatePointGeometry(value.geometry, `${path}.geometry`, errors);
|
if (value.geometry !== undefined) validateGeoJsonGeometry(value.geometry, `${path}.geometry`, errors);
|
||||||
}
|
|
||||||
|
|
||||||
function validatePointGeometry(value, path, errors) {
|
|
||||||
if (!isPlainObject(value) || value.type !== "Point" || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) {
|
|
||||||
errors.push(`${path}_must_be_geojson_point`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
rejectUnknownKeys(value, new Set(["type", "coordinates"]), path, errors);
|
|
||||||
if (!value.coordinates.every((coordinate) => typeof coordinate === "number" && Number.isFinite(coordinate))) {
|
|
||||||
errors.push(`${path}_coordinates_must_be_finite_numbers`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const [longitude, latitude] = value.coordinates;
|
|
||||||
if (longitude < -180 || longitude > 180) errors.push(`${path}.longitude_out_of_range`);
|
|
||||||
if (latitude < -90 || latitude > 90) errors.push(`${path}.latitude_out_of_range`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function validateRawEnvelope(value, errors) {
|
function validateRawEnvelope(value, errors) {
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ const TOKEN_REFRESH_MODES = new Set(["not_applicable", "operator_managed", "runt
|
||||||
const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]);
|
const COLLECTION_MODES = new Set(["realtime", "manual", "history", "weekly"]);
|
||||||
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
||||||
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
||||||
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
|
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
|
||||||
const L2_STEP_KINDS = new Set([
|
const L2_STEP_KINDS = new Set([
|
||||||
"collection_trigger",
|
"collection_trigger",
|
||||||
"provider_request",
|
"provider_request",
|
||||||
|
|
@ -84,6 +84,10 @@ const CAPABILITY_KEYS = new Set([
|
||||||
]);
|
]);
|
||||||
const REQUEST_KEYS = new Set(["method", "baseUrl", "path", "query", "response"]);
|
const REQUEST_KEYS = new Set(["method", "baseUrl", "path", "query", "response"]);
|
||||||
const RESPONSE_KEYS = new Set(["collectionPaths", "pagination"]);
|
const RESPONSE_KEYS = new Set(["collectionPaths", "pagination"]);
|
||||||
|
const OFFSET_PAGINATION_KEYS = new Set([
|
||||||
|
"mode", "limitParameter", "offsetParameter", "pageSize", "itemsPath", "totalPath",
|
||||||
|
"maxPages", "maxItems", "maxResponseBytes",
|
||||||
|
]);
|
||||||
const ENTITY_SCOPE_KEYS = new Set(["mode", "refresh", "businessEntityFilter"]);
|
const ENTITY_SCOPE_KEYS = new Set(["mode", "refresh", "businessEntityFilter"]);
|
||||||
const FIELD_POLICY_KEYS = new Set([
|
const FIELD_POLICY_KEYS = new Set([
|
||||||
"id",
|
"id",
|
||||||
|
|
@ -137,7 +141,9 @@ const EXPRESSION_KEYS = new Set([
|
||||||
"strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive",
|
"strategy", "paths", "coerce", "prefix", "fallback", "constant", "derive",
|
||||||
"omitIfMissing", "omitIfInvalid", "minimum", "maximum",
|
"omitIfMissing", "omitIfInvalid", "minimum", "maximum",
|
||||||
]);
|
]);
|
||||||
const GEOMETRY_KEYS = new Set(["type", "longitude", "latitude", "omitIfInvalid"]);
|
const GEOMETRY_KEYS = new Set([
|
||||||
|
"type", "longitude", "latitude", "strategy", "paths", "allowedTypes", "omitIfInvalid",
|
||||||
|
]);
|
||||||
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
|
const DERIVATION_KEYS = new Set(["kind", "rules", "default", "parameters"]);
|
||||||
const TEMPLATE_KEYS = new Set([
|
const TEMPLATE_KEYS = new Set([
|
||||||
"schemaVersion",
|
"schemaVersion",
|
||||||
|
|
@ -496,12 +502,50 @@ function validateCapability(value, path, errors) {
|
||||||
} else {
|
} else {
|
||||||
rejectUnknownKeys(value.request.response, RESPONSE_KEYS, `${path}.request.response`, errors);
|
rejectUnknownKeys(value.request.response, RESPONSE_KEYS, `${path}.request.response`, errors);
|
||||||
requiredCollectionPathArray(value.request.response.collectionPaths, `${path}.request.response.collectionPaths`, errors);
|
requiredCollectionPathArray(value.request.response.collectionPaths, `${path}.request.response.collectionPaths`, errors);
|
||||||
if (value.request.response.pagination !== "single_bounded_response") errors.push(`${path}.request.response.pagination_invalid`);
|
validateResponsePagination(value.request.response.pagination, value.request.query, `${path}.request.response.pagination`, errors);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
validateCredentialVisibleScope(value.entityScope, `${path}.entityScope`, errors);
|
validateCredentialVisibleScope(value.entityScope, `${path}.entityScope`, errors);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validateResponsePagination(value, query, path, errors) {
|
||||||
|
if (value === "single_bounded_response") return;
|
||||||
|
if (!isPlainObject(value)) {
|
||||||
|
errors.push(`${path}_invalid`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
rejectUnknownKeys(value, OFFSET_PAGINATION_KEYS, path, errors);
|
||||||
|
if (value.mode !== "offset") errors.push(`${path}.mode_must_be_offset`);
|
||||||
|
for (const field of ["limitParameter", "offsetParameter"]) {
|
||||||
|
if (typeof value[field] !== "string" || !/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(value[field])) {
|
||||||
|
errors.push(`${path}.${field}_invalid`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(value.pageSize) || value.pageSize < 1 || value.pageSize > 500) {
|
||||||
|
errors.push(`${path}.pageSize_invalid`);
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(value.maxPages) || value.maxPages < 1 || value.maxPages > 1000) {
|
||||||
|
errors.push(`${path}.maxPages_invalid`);
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(value.maxItems) || value.maxItems < 1 || value.maxItems > 5000) {
|
||||||
|
errors.push(`${path}.maxItems_invalid`);
|
||||||
|
}
|
||||||
|
if (!Number.isInteger(value.maxResponseBytes) || value.maxResponseBytes < 1024 || value.maxResponseBytes > 32 * 1024 * 1024) {
|
||||||
|
errors.push(`${path}.maxResponseBytes_invalid`);
|
||||||
|
}
|
||||||
|
for (const field of ["itemsPath", "totalPath"]) {
|
||||||
|
if (!isCanonicalSourcePath(value[field])) errors.push(`${path}.${field}_invalid`);
|
||||||
|
}
|
||||||
|
if (isPlainObject(query)) {
|
||||||
|
if (query[value.limitParameter] !== value.pageSize) errors.push(`${path}.pageSize_must_match_static_query_limit`);
|
||||||
|
if (query[value.offsetParameter] !== 0) errors.push(`${path}.offset_query_must_start_at_zero`);
|
||||||
|
}
|
||||||
|
if (Number.isInteger(value.maxPages) && Number.isInteger(value.pageSize)
|
||||||
|
&& Number.isInteger(value.maxItems) && value.maxPages * value.pageSize < value.maxItems) {
|
||||||
|
errors.push(`${path}.maxPages_cannot_cover_maxItems`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function validateFieldPolicy(value, path, errors) {
|
function validateFieldPolicy(value, path, errors) {
|
||||||
rejectUnknownKeys(value, FIELD_POLICY_KEYS, path, errors);
|
rejectUnknownKeys(value, FIELD_POLICY_KEYS, path, errors);
|
||||||
requiredIdentifier(value.id, `${path}.id`, errors);
|
requiredIdentifier(value.id, `${path}.id`, errors);
|
||||||
|
|
@ -619,10 +663,26 @@ function validateMappingContract(value, path, errors) {
|
||||||
errors.push(`${path}.fact.geometry_must_be_object`);
|
errors.push(`${path}.fact.geometry_must_be_object`);
|
||||||
} else if (isPlainObject(value.fact.geometry)) {
|
} else if (isPlainObject(value.fact.geometry)) {
|
||||||
rejectUnknownKeys(value.fact.geometry, GEOMETRY_KEYS, `${path}.fact.geometry`, errors);
|
rejectUnknownKeys(value.fact.geometry, GEOMETRY_KEYS, `${path}.fact.geometry`, errors);
|
||||||
if (value.fact.geometry.type !== "Point") errors.push(`${path}.fact.geometry.type_must_be_Point`);
|
if (value.fact.geometry.type === "Point") {
|
||||||
validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors);
|
validateExpression(value.fact.geometry.longitude, `${path}.fact.geometry.longitude`, errors);
|
||||||
validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors);
|
validateExpression(value.fact.geometry.latitude, `${path}.fact.geometry.latitude`, errors);
|
||||||
if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`);
|
if (value.fact.geometry.omitIfInvalid !== true) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_true`);
|
||||||
|
if (value.fact.geometry.strategy !== undefined || value.fact.geometry.paths !== undefined || value.fact.geometry.allowedTypes !== undefined) {
|
||||||
|
errors.push(`${path}.fact.geometry.point_shape_invalid`);
|
||||||
|
}
|
||||||
|
} else if (value.fact.geometry.type === "GeoJSON") {
|
||||||
|
if (value.fact.geometry.strategy !== "gelios_geozone_v1") errors.push(`${path}.fact.geometry.strategy_invalid`);
|
||||||
|
requiredSourcePathArray(value.fact.geometry.paths, `${path}.fact.geometry.paths`, errors);
|
||||||
|
if (!sameSet(value.fact.geometry.allowedTypes, ["Polygon", "MultiPolygon"])) {
|
||||||
|
errors.push(`${path}.fact.geometry.allowedTypes_must_be_polygon_or_multipolygon`);
|
||||||
|
}
|
||||||
|
if (value.fact.geometry.omitIfInvalid !== false) errors.push(`${path}.fact.geometry.omitIfInvalid_must_be_false`);
|
||||||
|
if (value.fact.geometry.longitude !== undefined || value.fact.geometry.latitude !== undefined) {
|
||||||
|
errors.push(`${path}.fact.geometry.geojson_shape_invalid`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
errors.push(`${path}.fact.geometry.type_invalid`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (value.fact.attributes !== undefined && !isPlainObject(value.fact.attributes)) {
|
if (value.fact.attributes !== undefined && !isPlainObject(value.fact.attributes)) {
|
||||||
errors.push(`${path}.fact.attributes_must_be_object`);
|
errors.push(`${path}.fact.attributes_must_be_object`);
|
||||||
|
|
@ -773,7 +833,13 @@ function validateMappingAgainstProduct(mapping, product, errors) {
|
||||||
for (const [field, contract] of Object.entries(contracts)) {
|
for (const [field, contract] of Object.entries(contracts)) {
|
||||||
if (!isPlainObject(contract)) continue;
|
if (!isPlainObject(contract)) continue;
|
||||||
if (field === "geometry") {
|
if (field === "geometry") {
|
||||||
if (contract.type !== "point") errors.push(`${path}.fact.geometry_contract_must_be_point`);
|
if (contract.type === "point" && mapping.fact?.geometry?.type !== "Point") {
|
||||||
|
errors.push(`${path}.fact.geometry_contract_must_be_point`);
|
||||||
|
}
|
||||||
|
if (contract.type === "geometry" && mapping.fact?.geometry?.type !== "GeoJSON") {
|
||||||
|
errors.push(`${path}.fact.geometry_contract_must_be_geojson`);
|
||||||
|
}
|
||||||
|
if (!new Set(["point", "geometry"]).has(contract.type)) errors.push(`${path}.fact.geometry_contract_invalid`);
|
||||||
if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) {
|
if (contract.required === true && mapping.fact?.geometry?.omitIfInvalid === true) {
|
||||||
errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`);
|
errors.push(`${path}.fact.geometry_required_but_mapping_can_omit`);
|
||||||
}
|
}
|
||||||
|
|
@ -805,7 +871,7 @@ function validateFieldContract(value, path, errors) {
|
||||||
if (value.enum !== undefined) {
|
if (value.enum !== undefined) {
|
||||||
if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) {
|
if (!Array.isArray(value.enum) || value.enum.length === 0 || new Set(value.enum.map(stableLiteral)).size !== value.enum.length) {
|
||||||
errors.push(`${path}.enum_must_be_nonempty_unique_array`);
|
errors.push(`${path}.enum_must_be_nonempty_unique_array`);
|
||||||
} else if (new Set(["point", "string_array"]).has(value.type)
|
} else if (new Set(["point", "geometry", "string_array"]).has(value.type)
|
||||||
|| value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) {
|
|| value.enum.some((item) => !fieldContractValueMatchesType(item, value.type))) {
|
||||||
errors.push(`${path}.enum_value_type_invalid`);
|
errors.push(`${path}.enum_value_type_invalid`);
|
||||||
}
|
}
|
||||||
|
|
@ -880,6 +946,11 @@ function fieldContractValueMatchesType(value, type) {
|
||||||
&& value.coordinates.length === 2
|
&& value.coordinates.length === 2
|
||||||
&& value.coordinates.every(Number.isFinite);
|
&& value.coordinates.every(Number.isFinite);
|
||||||
}
|
}
|
||||||
|
if (type === "geometry") {
|
||||||
|
return isPlainObject(value)
|
||||||
|
&& new Set(["Point", "LineString", "Polygon", "MultiPolygon"]).has(value.type)
|
||||||
|
&& Array.isArray(value.coordinates);
|
||||||
|
}
|
||||||
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,285 @@
|
||||||
|
import { createHash } from "node:crypto";
|
||||||
|
|
||||||
|
import { isBoundedGeoJsonGeometry } from "./geometry.mjs";
|
||||||
|
|
||||||
|
export const ZONE_SOURCE_GENERATION_SCHEMA_VERSION = "nodedc.zone-source-generation/v1";
|
||||||
|
export const ZONE_SOURCE_ADAPTERS = Object.freeze({
|
||||||
|
"gelios-rest-v1": Object.freeze({ sourceKind: "live_api", sourceRevision: "gelios-rest-v1" }),
|
||||||
|
"mmap-snapshot-v1": Object.freeze({ sourceKind: "versioned_snapshot", sourceRevision: "mmap-snapshot-v1" }),
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert one already-complete source generation into map.zone facts. Fetching,
|
||||||
|
* credentials and page iteration stay in Engine; this function is the common
|
||||||
|
* adapter boundary used by both live REST and immutable snapshot sources.
|
||||||
|
*/
|
||||||
|
export function normalizeZoneSourceGeneration(value, {
|
||||||
|
identityCrosswalk = {},
|
||||||
|
maxZones = 5000,
|
||||||
|
maxBytes = 16 * 1024 * 1024,
|
||||||
|
circleSegments = 64,
|
||||||
|
} = {}) {
|
||||||
|
if (!isPlainObject(value) || value.schemaVersion !== ZONE_SOURCE_GENERATION_SCHEMA_VERSION) {
|
||||||
|
throw zoneSourceError("zone_source_generation_schema_invalid");
|
||||||
|
}
|
||||||
|
const adapter = ZONE_SOURCE_ADAPTERS[value.adapterId];
|
||||||
|
if (!adapter) throw zoneSourceError("zone_source_adapter_unsupported");
|
||||||
|
if (!isPlainObject(identityCrosswalk)) throw zoneSourceError("zone_source_identity_crosswalk_invalid");
|
||||||
|
if (value.complete !== true) throw zoneSourceError("zone_source_generation_incomplete");
|
||||||
|
const generatedAt = iso(value.generatedAt, "zone_source_generated_at_invalid");
|
||||||
|
if (!Array.isArray(value.zones) || value.zones.length > maxZones) {
|
||||||
|
throw zoneSourceError("zone_source_zone_limit_exceeded");
|
||||||
|
}
|
||||||
|
if (byteLength(value) > maxBytes) throw zoneSourceError("zone_source_generation_bytes_exceeded");
|
||||||
|
if (!Number.isInteger(circleSegments) || circleSegments < 16 || circleSegments > 256) {
|
||||||
|
throw zoneSourceError("zone_source_circle_segments_invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
const facts = [];
|
||||||
|
const sourceIds = new Set();
|
||||||
|
for (const [index, rawZone] of value.zones.entries()) {
|
||||||
|
const zone = normalizeRawZone(rawZone, index, identityCrosswalk, circleSegments, value.adapterId);
|
||||||
|
if (sourceIds.has(zone.sourceId)) throw zoneSourceError("zone_source_identity_duplicate");
|
||||||
|
sourceIds.add(zone.sourceId);
|
||||||
|
facts.push({
|
||||||
|
sourceId: zone.sourceId,
|
||||||
|
semanticType: "map.zone",
|
||||||
|
observedAt: generatedAt,
|
||||||
|
geometry: zone.geometry,
|
||||||
|
attributes: compact({
|
||||||
|
area_square_meters: finiteNonNegative(first(rawZone.surfaceArea, rawZone.surface_area)),
|
||||||
|
description: optionalString(first(rawZone.description, rawZone.descr)),
|
||||||
|
display_name: requiredString(first(rawZone.name, rawZone.display_name), "zone_source_name_required"),
|
||||||
|
geometry_kind: zone.geometryKind,
|
||||||
|
max_speed_kph: finiteNonNegative(first(rawZone.maxPermissibleSpeed, rawZone.max_permissible_speed)),
|
||||||
|
perimeter_meters: finiteNonNegative(rawZone.perimeter),
|
||||||
|
source_kind: adapter.sourceKind,
|
||||||
|
source_revision: revision(value, adapter),
|
||||||
|
style_color: optionalString(first(rawZone.color, rawZone.style_color)),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
facts.sort((left, right) => left.sourceId.localeCompare(right.sourceId));
|
||||||
|
const digest = createHash("sha256").update(stableJson({
|
||||||
|
adapterId: value.adapterId,
|
||||||
|
generatedAt,
|
||||||
|
revision: revision(value, adapter),
|
||||||
|
facts,
|
||||||
|
})).digest("hex");
|
||||||
|
return Object.freeze({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: value.adapterId,
|
||||||
|
sourceKind: adapter.sourceKind,
|
||||||
|
sourceRevision: revision(value, adapter),
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
digest,
|
||||||
|
facts: Object.freeze(facts.map(deepFreeze)),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRawZone(value, index, identityCrosswalk, circleSegments, adapterId) {
|
||||||
|
if (!isPlainObject(value)) throw zoneSourceError(`zone_source_zone_${index}_invalid`);
|
||||||
|
const nativeId = resolveNativeId(value, identityCrosswalk, adapterId);
|
||||||
|
const sourceId = nativeId.startsWith("gelios-zone-") ? nativeId : `gelios-zone-${nativeId}`;
|
||||||
|
if (!/^[a-z][a-z0-9._:-]{2,127}$/.test(sourceId)) throw zoneSourceError("zone_source_identity_invalid");
|
||||||
|
const rawType = String(first(value.type, value.geometry_kind, "polygon")).trim().toLowerCase();
|
||||||
|
const geometryKind = rawType === "circle" ? "circle" : rawType === "line" || rawType === "corridor" ? "corridor" : "polygon";
|
||||||
|
const geometry = normalizeGeometry(value, geometryKind, circleSegments);
|
||||||
|
if (!isBoundedGeoJsonGeometry(geometry, new Set(["Polygon", "MultiPolygon"]))) {
|
||||||
|
throw zoneSourceError(`zone_source_zone_${index}_geometry_invalid`);
|
||||||
|
}
|
||||||
|
return { sourceId, geometryKind, geometry };
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveNativeId(zone, identityCrosswalk, adapterId) {
|
||||||
|
if (adapterId === "mmap-snapshot-v1") {
|
||||||
|
const snapshotKey = optionalString(first(zone.snapshotKey, zone.snapshot_key));
|
||||||
|
if (!snapshotKey || !Object.hasOwn(identityCrosswalk, snapshotKey)) {
|
||||||
|
throw zoneSourceError("zone_source_identity_crosswalk_required");
|
||||||
|
}
|
||||||
|
const mapped = identityCrosswalk[snapshotKey];
|
||||||
|
if (mapped === undefined || mapped === null || !String(mapped).trim()) {
|
||||||
|
throw zoneSourceError("zone_source_identity_crosswalk_required");
|
||||||
|
}
|
||||||
|
return String(mapped).trim();
|
||||||
|
}
|
||||||
|
const direct = first(zone.id, zone.geozoneId, zone.zone_id, zone.sourceId);
|
||||||
|
if (direct !== undefined && direct !== null && String(direct).trim()) return String(direct).trim();
|
||||||
|
throw zoneSourceError("zone_source_native_identity_required");
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeGeometry(zone, geometryKind, circleSegments) {
|
||||||
|
if (isBoundedGeoJsonGeometry(zone.geometry, new Set(["Polygon", "MultiPolygon"]))) {
|
||||||
|
return structuredClone(zone.geometry);
|
||||||
|
}
|
||||||
|
const points = parseCoordinates(first(zone.points, zone.coords, zone.coordinates));
|
||||||
|
if (geometryKind === "circle") {
|
||||||
|
if (!points.length) throw zoneSourceError("zone_source_circle_center_required");
|
||||||
|
const radius = finiteNonNegative(zone.radius);
|
||||||
|
if (!radius || radius <= 0) throw zoneSourceError("zone_source_circle_radius_required");
|
||||||
|
return circlePolygon(points[0], radius, circleSegments);
|
||||||
|
}
|
||||||
|
if (geometryKind === "corridor") {
|
||||||
|
const linePoints = parseCoordinates(first(zone.line, zone.points, zone.coords, zone.coordinates));
|
||||||
|
const width = finiteNonNegative(first(zone.lineWidthMeters, zone.line_width_meters, zone.width, zone.radius));
|
||||||
|
if (linePoints.length < 2 || !width || width <= 0) {
|
||||||
|
throw zoneSourceError("zone_source_corridor_path_and_width_required");
|
||||||
|
}
|
||||||
|
return corridorPolygon(linePoints, width);
|
||||||
|
}
|
||||||
|
if (points.length < 3) throw zoneSourceError("zone_source_polygon_points_required");
|
||||||
|
const ring = [...points];
|
||||||
|
if (!samePosition(ring[0], ring.at(-1))) ring.push([...ring[0]]);
|
||||||
|
return { type: "Polygon", coordinates: [ring] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseCoordinates(value) {
|
||||||
|
if (typeof value === "string") {
|
||||||
|
return value.split(";").map((entry) => entry.trim()).filter(Boolean).map((entry) => {
|
||||||
|
const [latitude, longitude] = entry.split(",").map(Number);
|
||||||
|
return position(longitude, latitude);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (!Array.isArray(value)) return [];
|
||||||
|
return value.map((entry) => {
|
||||||
|
if (Array.isArray(entry)) {
|
||||||
|
const [latitude, longitude] = entry.map(Number);
|
||||||
|
return position(longitude, latitude);
|
||||||
|
}
|
||||||
|
if (!isPlainObject(entry)) throw zoneSourceError("zone_source_coordinate_invalid");
|
||||||
|
return position(
|
||||||
|
firstNumber(entry.longitude, entry.lon, entry.lng, entry.x),
|
||||||
|
firstNumber(entry.latitude, entry.lat, entry.y),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function circlePolygon([longitude, latitude], radiusMeters, segments) {
|
||||||
|
const earthRadius = 6_371_008.8;
|
||||||
|
const angularDistance = radiusMeters / earthRadius;
|
||||||
|
const latitudeRad = radians(latitude);
|
||||||
|
const longitudeRad = radians(longitude);
|
||||||
|
const ring = [];
|
||||||
|
for (let index = 0; index <= segments; index += 1) {
|
||||||
|
const bearing = 2 * Math.PI * index / segments;
|
||||||
|
const targetLatitude = Math.asin(
|
||||||
|
Math.sin(latitudeRad) * Math.cos(angularDistance)
|
||||||
|
+ Math.cos(latitudeRad) * Math.sin(angularDistance) * Math.cos(bearing),
|
||||||
|
);
|
||||||
|
const targetLongitude = longitudeRad + Math.atan2(
|
||||||
|
Math.sin(bearing) * Math.sin(angularDistance) * Math.cos(latitudeRad),
|
||||||
|
Math.cos(angularDistance) - Math.sin(latitudeRad) * Math.sin(targetLatitude),
|
||||||
|
);
|
||||||
|
ring.push([degrees(targetLongitude), degrees(targetLatitude)]);
|
||||||
|
}
|
||||||
|
ring[ring.length - 1] = [...ring[0]];
|
||||||
|
return { type: "Polygon", coordinates: [ring] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function corridorPolygon(points, widthMeters) {
|
||||||
|
const halfWidth = widthMeters / 2;
|
||||||
|
const left = [];
|
||||||
|
const right = [];
|
||||||
|
for (let index = 0; index < points.length; index += 1) {
|
||||||
|
const previous = points[Math.max(0, index - 1)];
|
||||||
|
const next = points[Math.min(points.length - 1, index + 1)];
|
||||||
|
const meanLatitude = radians((previous[1] + next[1]) / 2);
|
||||||
|
const dx = (next[0] - previous[0]) * Math.cos(meanLatitude);
|
||||||
|
const dy = next[1] - previous[1];
|
||||||
|
const length = Math.hypot(dx, dy);
|
||||||
|
if (!length) throw zoneSourceError("zone_source_corridor_zero_length_segment");
|
||||||
|
const metersPerDegreeLatitude = 111_320;
|
||||||
|
const longitudeOffset = (-dy / length) * halfWidth / (metersPerDegreeLatitude * Math.max(0.01, Math.cos(radians(points[index][1]))));
|
||||||
|
const latitudeOffset = (dx / length) * halfWidth / metersPerDegreeLatitude;
|
||||||
|
left.push([points[index][0] + longitudeOffset, points[index][1] + latitudeOffset]);
|
||||||
|
right.push([points[index][0] - longitudeOffset, points[index][1] - latitudeOffset]);
|
||||||
|
}
|
||||||
|
const ring = [...left, ...right.reverse(), [...left[0]]];
|
||||||
|
return { type: "Polygon", coordinates: [ring] };
|
||||||
|
}
|
||||||
|
|
||||||
|
function revision(value, adapter) {
|
||||||
|
const explicit = optionalString(value.sourceRevision);
|
||||||
|
if (value.adapterId === "mmap-snapshot-v1") {
|
||||||
|
const digest = optionalString(value.snapshotDigest);
|
||||||
|
if (!digest || !/^[a-f0-9]{64}$/.test(digest)) throw zoneSourceError("zone_source_snapshot_digest_required");
|
||||||
|
return `${explicit || adapter.sourceRevision}@sha256:${digest}`;
|
||||||
|
}
|
||||||
|
return explicit || adapter.sourceRevision;
|
||||||
|
}
|
||||||
|
|
||||||
|
function position(longitude, latitude) {
|
||||||
|
if (![longitude, latitude].every(Number.isFinite)
|
||||||
|
|| longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) {
|
||||||
|
throw zoneSourceError("zone_source_coordinate_invalid");
|
||||||
|
}
|
||||||
|
return [longitude, latitude];
|
||||||
|
}
|
||||||
|
|
||||||
|
function first(...values) {
|
||||||
|
return values.find((value) => value !== undefined && value !== null && value !== "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstNumber(...values) {
|
||||||
|
const value = first(...values);
|
||||||
|
const number = typeof value === "number" ? value : Number(String(value ?? "").trim());
|
||||||
|
return Number.isFinite(number) ? number : Number.NaN;
|
||||||
|
}
|
||||||
|
|
||||||
|
function finiteNonNegative(value) {
|
||||||
|
const number = firstNumber(value);
|
||||||
|
return Number.isFinite(number) && number >= 0 ? number : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function requiredString(value, code) {
|
||||||
|
const result = optionalString(value);
|
||||||
|
if (!result) throw zoneSourceError(code);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function optionalString(value) {
|
||||||
|
return typeof value === "string" && value.trim() ? value.trim() : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function iso(value, code) {
|
||||||
|
const date = new Date(String(value ?? ""));
|
||||||
|
if (Number.isNaN(date.getTime())) throw zoneSourceError(code);
|
||||||
|
return date.toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function compact(value) {
|
||||||
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
|
||||||
|
}
|
||||||
|
|
||||||
|
function samePosition(left, right) {
|
||||||
|
return Array.isArray(left) && Array.isArray(right) && left[0] === right[0] && left[1] === right[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
function stableJson(value) {
|
||||||
|
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
||||||
|
if (isPlainObject(value)) return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`;
|
||||||
|
return JSON.stringify(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function byteLength(value) {
|
||||||
|
try { return Buffer.byteLength(JSON.stringify(value)); } catch { return Number.POSITIVE_INFINITY; }
|
||||||
|
}
|
||||||
|
|
||||||
|
function deepFreeze(value) {
|
||||||
|
if (!value || typeof value !== "object" || Object.isFrozen(value)) return value;
|
||||||
|
Object.freeze(value);
|
||||||
|
Object.values(value).forEach(deepFreeze);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPlainObject(value) {
|
||||||
|
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function radians(value) { return value * Math.PI / 180; }
|
||||||
|
function degrees(value) { return value * 180 / Math.PI; }
|
||||||
|
|
||||||
|
function zoneSourceError(code) {
|
||||||
|
return Object.assign(new Error(code), { code });
|
||||||
|
}
|
||||||
|
|
@ -72,6 +72,32 @@ assert.equal(validateDataProductPublish({
|
||||||
...publish,
|
...publish,
|
||||||
facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }],
|
facts: [{ ...fact, geometry: { type: "Point", coordinates: [37.6173, 90.0001] } }],
|
||||||
}).errors.includes("facts[0].geometry.latitude_out_of_range"), true);
|
}).errors.includes("facts[0].geometry.latitude_out_of_range"), true);
|
||||||
|
const zoneGenerationAt = "2026-07-15T10:10:00.000Z";
|
||||||
|
const zoneFact = {
|
||||||
|
sourceId: "gelios-zone-42",
|
||||||
|
semanticType: "map.zone",
|
||||||
|
observedAt: zoneGenerationAt,
|
||||||
|
attributes: { display_name: "Zone 42" },
|
||||||
|
geometry: {
|
||||||
|
type: "Polygon",
|
||||||
|
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const replacement = {
|
||||||
|
...publish,
|
||||||
|
batch: { ...publish.batch, mode: "replace", generationAt: zoneGenerationAt },
|
||||||
|
facts: [zoneFact],
|
||||||
|
};
|
||||||
|
assert.equal(validateDataProductPublish(replacement).ok, true);
|
||||||
|
assert.equal(validateDataProductPublish({ ...replacement, facts: [] }).ok, true);
|
||||||
|
assert.equal(validateDataProductPublish({
|
||||||
|
...replacement,
|
||||||
|
facts: [{ ...zoneFact, observedAt: "2026-07-15T10:09:59.000Z" }],
|
||||||
|
}).errors.includes("facts[0].observedAt_must_equal_generationAt"), true);
|
||||||
|
assert.equal(validateDataProductPublish({
|
||||||
|
...replacement,
|
||||||
|
facts: [{ ...zoneFact, geometry: { ...zoneFact.geometry, coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.61, 55.75]]] } }],
|
||||||
|
}).errors.includes("facts[0].geometry[0]_linear_ring_must_be_closed"), true);
|
||||||
|
|
||||||
const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" };
|
const canonicalFact = { ...fact, receivedAt: "2026-07-15T10:00:01.000Z" };
|
||||||
const snapshot = {
|
const snapshot = {
|
||||||
|
|
@ -147,7 +173,11 @@ assert.equal(validateDataProductPatch(patch).ok, true);
|
||||||
assert.equal(validateDataProductPatch({
|
assert.equal(validateDataProductPatch({
|
||||||
...patch,
|
...patch,
|
||||||
operations: [{ op: "delete", fact: canonicalFact }],
|
operations: [{ op: "delete", fact: canonicalFact }],
|
||||||
}).errors.includes("operations[0].op_must_be_upsert"), true);
|
}).errors.includes("operations[0].op_invalid"), true);
|
||||||
|
assert.equal(validateDataProductPatch({
|
||||||
|
...patch,
|
||||||
|
operations: [{ op: "remove", sourceId: "gelios-zone-42", semanticType: "map.zone" }],
|
||||||
|
}).ok, true);
|
||||||
assert.equal(validateDataProductPatch({
|
assert.equal(validateDataProductPatch({
|
||||||
...patch,
|
...patch,
|
||||||
operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }],
|
operations: [{ op: "upsert", fact: { ...canonicalFact, attributes: attributesWithSerializedSize((64 * 1024) + 1) } }],
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ import { geliosProviderPackageV2 } from "../providers/gelios/v2/index.mjs";
|
||||||
import { geliosProviderPackageV3 } from "../providers/gelios/v3/index.mjs";
|
import { geliosProviderPackageV3 } from "../providers/gelios/v3/index.mjs";
|
||||||
import { geliosProviderPackageV4 } from "../providers/gelios/v4/index.mjs";
|
import { geliosProviderPackageV4 } from "../providers/gelios/v4/index.mjs";
|
||||||
import { geliosProviderPackageV5 } from "../providers/gelios/v5/index.mjs";
|
import { geliosProviderPackageV5 } from "../providers/gelios/v5/index.mjs";
|
||||||
|
import { geliosProviderPackageV6 } from "../providers/gelios/v6/index.mjs";
|
||||||
import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs";
|
import { normalizeDataProductDefinition } from "../../../services/external-data-plane/src/data-product-policy.mjs";
|
||||||
|
|
||||||
const expectedFields = [
|
const expectedFields = [
|
||||||
|
|
@ -41,6 +42,24 @@ assert.deepEqual(validateProviderPackage(geliosProviderPackageV2), { ok: true, e
|
||||||
assert.deepEqual(validateProviderPackage(geliosProviderPackageV3), { ok: true, errors: [] });
|
assert.deepEqual(validateProviderPackage(geliosProviderPackageV3), { ok: true, errors: [] });
|
||||||
assert.deepEqual(validateProviderPackage(geliosProviderPackageV4), { ok: true, errors: [] });
|
assert.deepEqual(validateProviderPackage(geliosProviderPackageV4), { ok: true, errors: [] });
|
||||||
assert.deepEqual(validateProviderPackage(geliosProviderPackageV5), { ok: true, errors: [] });
|
assert.deepEqual(validateProviderPackage(geliosProviderPackageV5), { ok: true, errors: [] });
|
||||||
|
assert.deepEqual(validateProviderPackage(geliosProviderPackageV6), { ok: true, errors: [] });
|
||||||
|
|
||||||
|
const zonesProduct = geliosProviderPackageV6.dataProducts.find((product) => product.id === "map.zones.current.v1");
|
||||||
|
const registeredZonesProduct = JSON.parse(await readFile(new URL(
|
||||||
|
"../../../services/external-data-plane/definitions/map.zones.current.v1.json",
|
||||||
|
import.meta.url,
|
||||||
|
), "utf8"));
|
||||||
|
assert.deepEqual(zonesProduct, registeredZonesProduct);
|
||||||
|
assert.deepEqual(zonesProduct.semanticTypes, ["map.zone"]);
|
||||||
|
assert.equal(zonesProduct.fieldContracts.geometry.type, "geometry");
|
||||||
|
const zonesCapability = geliosProviderPackageV6.capabilities.find((capability) => capability.id === "gelios.geozones.current.read");
|
||||||
|
assert.equal(zonesCapability.request.query.pl, 100);
|
||||||
|
assert.equal(zonesCapability.request.query.po, 0);
|
||||||
|
assert.equal(zonesCapability.request.response.pagination.maxItems, 5000);
|
||||||
|
assert.equal(zonesCapability.request.response.pagination.totalPath, "paginationMetadata.totalCount");
|
||||||
|
const zonesMapping = geliosProviderPackageV6.mappingContracts.find((mapping) => mapping.id === "gelios.geozones.to.map.zones.current.v1");
|
||||||
|
assert.deepEqual(zonesMapping.fact.observedAt.paths, ["updatedAt"]);
|
||||||
|
assert.equal(zonesMapping.fact.observedAt.fallback, "collection_received_at");
|
||||||
|
|
||||||
const strictProduct = geliosProviderPackageV5.dataProducts[0];
|
const strictProduct = geliosProviderPackageV5.dataProducts[0];
|
||||||
const registeredStrictProduct = JSON.parse(await readFile(new URL(
|
const registeredStrictProduct = JSON.parse(await readFile(new URL(
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,95 @@
|
||||||
|
import assert from "node:assert/strict";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
normalizeZoneSourceGeneration,
|
||||||
|
validateDataProductPublish,
|
||||||
|
} from "../src/index.mjs";
|
||||||
|
|
||||||
|
const generatedAt = "2026-07-20T18:00:00.000Z";
|
||||||
|
const live = normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "gelios-rest-v1",
|
||||||
|
sourceRevision: "gelios-rest-v1",
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
zones: [{
|
||||||
|
id: 42,
|
||||||
|
name: "Polygon zone",
|
||||||
|
type: "polygon",
|
||||||
|
points: [
|
||||||
|
{ latitude: 55.74, longitude: 37.60 },
|
||||||
|
{ latitude: 55.74, longitude: 37.62 },
|
||||||
|
{ latitude: 55.76, longitude: 37.62 },
|
||||||
|
],
|
||||||
|
color: "#ff00ff",
|
||||||
|
}, {
|
||||||
|
id: 43,
|
||||||
|
name: "Circle zone",
|
||||||
|
type: "circle",
|
||||||
|
points: [{ latitude: 55.75, longitude: 37.61 }],
|
||||||
|
radius: 150,
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
assert.equal(live.complete, true);
|
||||||
|
assert.equal(live.sourceKind, "live_api");
|
||||||
|
assert.equal(live.facts.length, 2);
|
||||||
|
assert.deepEqual(live.facts.map((fact) => fact.sourceId), ["gelios-zone-42", "gelios-zone-43"]);
|
||||||
|
assert.equal(live.facts.every((fact) => fact.observedAt === generatedAt), true);
|
||||||
|
assert.equal(live.facts.every((fact) => fact.geometry.type === "Polygon"), true);
|
||||||
|
assert.equal(validateDataProductPublish({
|
||||||
|
schemaVersion: "nodedc.data-product.publish/v1",
|
||||||
|
batch: { runId: "zones-live-1", sequence: 0, idempotencyKey: "zones-live-1.batch-0", mode: "replace", generationAt: generatedAt },
|
||||||
|
facts: live.facts,
|
||||||
|
}).ok, true);
|
||||||
|
|
||||||
|
const snapshotDigest = "a".repeat(64);
|
||||||
|
const snapshot = normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "mmap-snapshot-v1",
|
||||||
|
snapshotDigest,
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
zones: [{
|
||||||
|
snapshotKey: "legacy-zone-a",
|
||||||
|
name: "Snapshot zone",
|
||||||
|
type: "polygon",
|
||||||
|
points: "55.74,37.60;55.74,37.62;55.76,37.62",
|
||||||
|
}],
|
||||||
|
}, { identityCrosswalk: { "legacy-zone-a": 42 } });
|
||||||
|
assert.equal(snapshot.sourceKind, "versioned_snapshot");
|
||||||
|
assert.equal(snapshot.facts[0].sourceId, live.facts[0].sourceId);
|
||||||
|
assert.match(snapshot.sourceRevision, new RegExp(`@sha256:${snapshotDigest}$`));
|
||||||
|
|
||||||
|
assert.throws(() => normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "mmap-snapshot-v1",
|
||||||
|
snapshotDigest,
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
zones: [{ snapshotKey: "unknown", name: "Unknown", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
|
||||||
|
}), /zone_source_identity_crosswalk_required/);
|
||||||
|
assert.throws(() => normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "mmap-snapshot-v1",
|
||||||
|
snapshotDigest,
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
zones: [{ id: 42, name: "Uncrossed", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
|
||||||
|
}), /zone_source_identity_crosswalk_required/);
|
||||||
|
assert.throws(() => normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "gelios-rest-v1",
|
||||||
|
generatedAt,
|
||||||
|
complete: true,
|
||||||
|
zones: [{ name: "No native id", type: "polygon", points: "55.74,37.60;55.74,37.62;55.76,37.62" }],
|
||||||
|
}), /zone_source_native_identity_required/);
|
||||||
|
assert.throws(() => normalizeZoneSourceGeneration({
|
||||||
|
schemaVersion: ZONE_SOURCE_GENERATION_SCHEMA_VERSION,
|
||||||
|
adapterId: "gelios-rest-v1",
|
||||||
|
generatedAt,
|
||||||
|
complete: false,
|
||||||
|
zones: [],
|
||||||
|
}), /zone_source_generation_incomplete/);
|
||||||
|
|
||||||
|
console.log("external-provider zone source: ok");
|
||||||
|
|
@ -25,7 +25,7 @@ export class NdcDataProductPublish implements INodeType {
|
||||||
name: 'ndcDataProductPublish',
|
name: 'ndcDataProductPublish',
|
||||||
icon: NDC_NODE_ICON,
|
icon: NDC_NODE_ICON,
|
||||||
group: ['output'],
|
group: ['output'],
|
||||||
version: 1,
|
version: [1, 2],
|
||||||
subtitle: '={{$parameter["dataProductId"]}}',
|
subtitle: '={{$parameter["dataProductId"]}}',
|
||||||
description: 'Publish canonical facts through a scoped NDC Data Product grant',
|
description: 'Publish canonical facts through a scoped NDC Data Product grant',
|
||||||
defaults: { name: 'NDC Data Product Publish' },
|
defaults: { name: 'NDC Data Product Publish' },
|
||||||
|
|
@ -41,6 +41,19 @@ export class NdcDataProductPublish implements INodeType {
|
||||||
default: '',
|
default: '',
|
||||||
required: true,
|
required: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
displayName: 'Publish Mode',
|
||||||
|
name: 'publishMode',
|
||||||
|
type: 'options',
|
||||||
|
options: [
|
||||||
|
{ name: 'Create or Update', value: 'upsert' },
|
||||||
|
{ name: 'Replace Complete Snapshot', value: 'replace' },
|
||||||
|
],
|
||||||
|
default: 'upsert',
|
||||||
|
required: true,
|
||||||
|
displayOptions: { show: { '@version': [2] } },
|
||||||
|
description: 'Replace commits one complete generation atomically and removes keys absent from it',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
displayName: 'Batch Sequence',
|
displayName: 'Batch Sequence',
|
||||||
name: 'sequence',
|
name: 'sequence',
|
||||||
|
|
@ -71,6 +84,9 @@ export class NdcDataProductPublish implements INodeType {
|
||||||
const inputItems = this.getInputData();
|
const inputItems = this.getInputData();
|
||||||
const dataProductId = this.getNodeParameter('dataProductId', 0) as string;
|
const dataProductId = this.getNodeParameter('dataProductId', 0) as string;
|
||||||
const sequence = this.getNodeParameter('sequence', 0, 0) as number;
|
const sequence = this.getNodeParameter('sequence', 0, 0) as number;
|
||||||
|
const publishMode = this.getNode().typeVersion >= 2
|
||||||
|
? this.getNodeParameter('publishMode', 0, 'upsert') as 'upsert' | 'replace'
|
||||||
|
: 'upsert';
|
||||||
let body;
|
let body;
|
||||||
let encodedProductId;
|
let encodedProductId;
|
||||||
try {
|
try {
|
||||||
|
|
@ -81,6 +97,7 @@ export class NdcDataProductPublish implements INodeType {
|
||||||
this.getNode().id,
|
this.getNode().id,
|
||||||
dataProductId,
|
dataProductId,
|
||||||
sequence,
|
sequence,
|
||||||
|
publishMode,
|
||||||
);
|
);
|
||||||
encodedProductId = encodeIdentifierPath(dataProductId, 'dataProductId');
|
encodedProductId = encodeIdentifierPath(dataProductId, 'dataProductId');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
|
||||||
|
|
@ -18,18 +18,23 @@ export interface NdcFact extends IDataObject {
|
||||||
semanticType: string;
|
semanticType: string;
|
||||||
observedAt: string;
|
observedAt: string;
|
||||||
attributes?: IDataObject;
|
attributes?: IDataObject;
|
||||||
geometry?: {
|
geometry?: NdcGeometry;
|
||||||
type: 'Point';
|
|
||||||
coordinates: [number, number];
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type NdcGeometry =
|
||||||
|
| { type: 'Point'; coordinates: [number, number] }
|
||||||
|
| { type: 'LineString'; coordinates: [number, number][] }
|
||||||
|
| { type: 'Polygon'; coordinates: [number, number][][] }
|
||||||
|
| { type: 'MultiPolygon'; coordinates: [number, number][][][] };
|
||||||
|
|
||||||
export interface NdcPublishPayload extends IDataObject {
|
export interface NdcPublishPayload extends IDataObject {
|
||||||
schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION;
|
schemaVersion: typeof DATA_PRODUCT_PUBLISH_SCHEMA_VERSION;
|
||||||
batch: {
|
batch: {
|
||||||
runId: string;
|
runId: string;
|
||||||
sequence: number;
|
sequence: number;
|
||||||
idempotencyKey: string;
|
idempotencyKey: string;
|
||||||
|
mode?: 'replace';
|
||||||
|
generationAt?: string;
|
||||||
};
|
};
|
||||||
facts: NdcFact[];
|
facts: NdcFact[];
|
||||||
}
|
}
|
||||||
|
|
@ -62,12 +67,16 @@ export function encodeIdentifierPath(value: unknown, field: string): string {
|
||||||
return encodeURIComponent(requireIdentifier(value, field));
|
return encodeURIComponent(requireIdentifier(value, field));
|
||||||
}
|
}
|
||||||
|
|
||||||
export function factsFromItems(items: INodeExecutionData[]): NdcFact[] {
|
export function factsFromItems(items: INodeExecutionData[], allowEmpty = false): NdcFact[] {
|
||||||
if (!items.length) throw new Error('facts_required');
|
const batchFacts = items.length === 1 && Array.isArray(items[0]?.json?.facts)
|
||||||
if (items.length > MAX_FACTS) throw new Error('facts_limit_exceeded');
|
? items[0].json.facts
|
||||||
|
: null;
|
||||||
|
const sources = batchFacts ?? items.map((item) => (isObject(item.json.fact) ? item.json.fact : item.json));
|
||||||
|
if (!sources.length && !allowEmpty) throw new Error('facts_required');
|
||||||
|
if (sources.length > MAX_FACTS) throw new Error('facts_limit_exceeded');
|
||||||
const entityKeys = new Set<string>();
|
const entityKeys = new Set<string>();
|
||||||
return items.map((item, index) => {
|
return sources.map((source, index) => {
|
||||||
const source = isObject(item.json.fact) ? item.json.fact : item.json;
|
if (!isObject(source)) throw new Error(`facts_${index}_invalid`);
|
||||||
const fact: NdcFact = {
|
const fact: NdcFact = {
|
||||||
sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`),
|
sourceId: requireIdentifier(source.sourceId, `facts_${index}_sourceId`),
|
||||||
semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`),
|
semanticType: requireIdentifier(source.semanticType, `facts_${index}_semanticType`),
|
||||||
|
|
@ -84,7 +93,7 @@ export function factsFromItems(items: INodeExecutionData[]): NdcFact[] {
|
||||||
if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`);
|
if (containsSecretLikeMaterial(source.attributes)) throw new Error(`facts_${index}_attributes_secret_material_forbidden`);
|
||||||
fact.attributes = source.attributes;
|
fact.attributes = source.attributes;
|
||||||
}
|
}
|
||||||
if (source.geometry !== undefined) fact.geometry = normalizePoint(source.geometry, index);
|
if (source.geometry !== undefined) fact.geometry = normalizeGeometry(source.geometry, index);
|
||||||
return fact;
|
return fact;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -96,6 +105,7 @@ export function buildPublishPayload(
|
||||||
nodeId: string,
|
nodeId: string,
|
||||||
dataProductId: string,
|
dataProductId: string,
|
||||||
sequence: number,
|
sequence: number,
|
||||||
|
publishMode: 'upsert' | 'replace' = 'upsert',
|
||||||
): NdcPublishPayload {
|
): NdcPublishPayload {
|
||||||
if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid');
|
if (!Number.isInteger(sequence) || sequence < 0 || sequence > 2_147_483_647) throw new Error('batch_sequence_invalid');
|
||||||
requireIdentifier(dataProductId, 'dataProductId');
|
requireIdentifier(dataProductId, 'dataProductId');
|
||||||
|
|
@ -103,10 +113,22 @@ export function buildPublishPayload(
|
||||||
const idempotencyKey = `publish-${sha256(
|
const idempotencyKey = `publish-${sha256(
|
||||||
`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`,
|
`${workflowId}|${executionId}|${nodeId}|${dataProductId}|${sequence}`,
|
||||||
)}`;
|
)}`;
|
||||||
|
const facts = factsFromItems(items, publishMode === 'replace');
|
||||||
|
const explicitGenerationAt = items.length === 1 && Array.isArray(items[0]?.json?.facts)
|
||||||
|
? items[0].json.generationAt
|
||||||
|
: undefined;
|
||||||
|
const generationAt = publishMode === 'replace'
|
||||||
|
? replacementGenerationAt(facts, explicitGenerationAt)
|
||||||
|
: undefined;
|
||||||
return {
|
return {
|
||||||
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
schemaVersion: DATA_PRODUCT_PUBLISH_SCHEMA_VERSION,
|
||||||
batch: { runId, sequence, idempotencyKey },
|
batch: {
|
||||||
facts: factsFromItems(items),
|
runId,
|
||||||
|
sequence,
|
||||||
|
idempotencyKey,
|
||||||
|
...(generationAt ? { mode: 'replace' as const, generationAt } : {}),
|
||||||
|
},
|
||||||
|
facts,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -254,18 +276,60 @@ function requireFoundrySlotId(value: unknown): string {
|
||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizePoint(value: unknown, index: number): NdcFact['geometry'] {
|
function normalizeGeometry(value: unknown, index: number): NdcGeometry {
|
||||||
if (!isObject(value) || value.type !== 'Point' || !Array.isArray(value.coordinates) || value.coordinates.length !== 2) {
|
if (!isObject(value) || !['Point', 'LineString', 'Polygon', 'MultiPolygon'].includes(String(value.type))
|
||||||
|
|| !Array.isArray(value.coordinates)) {
|
||||||
throw new Error(`facts_${index}_geometry_invalid`);
|
throw new Error(`facts_${index}_geometry_invalid`);
|
||||||
}
|
}
|
||||||
const [longitude, latitude] = value.coordinates;
|
if (Buffer.byteLength(JSON.stringify(value)) > 192 * 1024) throw new Error(`facts_${index}_geometry_size_exceeded`);
|
||||||
if (![longitude, latitude].every((coordinate) => typeof coordinate === 'number' && Number.isFinite(coordinate))) {
|
let vertices = 0;
|
||||||
throw new Error(`facts_${index}_geometry_coordinates_invalid`);
|
const position = (candidate: unknown, path: string): [number, number] => {
|
||||||
|
if (!Array.isArray(candidate) || candidate.length !== 2 || !candidate.every(Number.isFinite)) {
|
||||||
|
throw new Error(`${path}_position_invalid`);
|
||||||
|
}
|
||||||
|
const [longitude, latitude] = candidate as [number, number];
|
||||||
|
if (longitude < -180 || longitude > 180 || latitude < -90 || latitude > 90) {
|
||||||
|
throw new Error(`${path}_position_out_of_range`);
|
||||||
|
}
|
||||||
|
vertices += 1;
|
||||||
|
if (vertices > 10_000) throw new Error(`facts_${index}_geometry_vertex_limit_exceeded`);
|
||||||
|
return [longitude, latitude];
|
||||||
|
};
|
||||||
|
const line = (candidate: unknown, path: string, minimum: number): [number, number][] => {
|
||||||
|
if (!Array.isArray(candidate) || candidate.length < minimum) throw new Error(`${path}_line_invalid`);
|
||||||
|
return candidate.map((entry, positionIndex) => position(entry, `${path}_${positionIndex}`));
|
||||||
|
};
|
||||||
|
const ring = (candidate: unknown, path: string): [number, number][] => {
|
||||||
|
const result = line(candidate, path, 4);
|
||||||
|
const first = result[0];
|
||||||
|
const last = result.at(-1);
|
||||||
|
if (!last || first[0] !== last[0] || first[1] !== last[1]) throw new Error(`${path}_ring_not_closed`);
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
const polygon = (candidate: unknown, path: string): [number, number][][] => {
|
||||||
|
if (!Array.isArray(candidate) || !candidate.length) throw new Error(`${path}_polygon_invalid`);
|
||||||
|
return candidate.map((entry, ringIndex) => ring(entry, `${path}_${ringIndex}`));
|
||||||
|
};
|
||||||
|
if (value.type === 'Point') return { type: 'Point', coordinates: position(value.coordinates, `facts_${index}_geometry`) };
|
||||||
|
if (value.type === 'LineString') return { type: 'LineString', coordinates: line(value.coordinates, `facts_${index}_geometry`, 2) };
|
||||||
|
if (value.type === 'Polygon') return { type: 'Polygon', coordinates: polygon(value.coordinates, `facts_${index}_geometry`) };
|
||||||
|
const coordinates = value.coordinates.map((entry, polygonIndex) => polygon(entry, `facts_${index}_geometry_${polygonIndex}`));
|
||||||
|
if (!coordinates.length) throw new Error(`facts_${index}_geometry_multipolygon_invalid`);
|
||||||
|
return { type: 'MultiPolygon', coordinates };
|
||||||
|
}
|
||||||
|
|
||||||
|
function replacementGenerationAt(facts: NdcFact[], explicit: unknown): string {
|
||||||
|
const declared = explicit === undefined ? undefined : requireIsoTimestamp(explicit, 'replace_generationAt');
|
||||||
|
const timestamps = [...new Set(facts.map((fact) => new Date(fact.observedAt).toISOString()))];
|
||||||
|
if (!timestamps.length) {
|
||||||
|
if (!declared) throw new Error('replace_generationAt_required_for_empty_generation');
|
||||||
|
return new Date(declared).toISOString();
|
||||||
}
|
}
|
||||||
if ((longitude as number) < -180 || (longitude as number) > 180 || (latitude as number) < -90 || (latitude as number) > 90) {
|
if (timestamps.length !== 1) throw new Error('replace_generation_requires_one_observedAt');
|
||||||
throw new Error(`facts_${index}_geometry_coordinates_out_of_range`);
|
if (declared && new Date(declared).toISOString() !== timestamps[0]) {
|
||||||
|
throw new Error('replace_generationAt_observedAt_mismatch');
|
||||||
}
|
}
|
||||||
return { type: 'Point', coordinates: [longitude as number, latitude as number] };
|
return timestamps[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
function requireIsoTimestamp(value: unknown, field: string): string {
|
function requireIsoTimestamp(value: unknown, field: string): string {
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
{
|
{
|
||||||
"name": "n8n-nodes-ndc",
|
"name": "n8n-nodes-ndc",
|
||||||
"version": "0.1.4",
|
"version": "0.1.5",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "n8n-nodes-ndc",
|
"name": "n8n-nodes-ndc",
|
||||||
"version": "0.1.4",
|
"version": "0.1.5",
|
||||||
"license": "UNLICENSED",
|
"license": "UNLICENSED",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@n8n/node-cli": "0.39.3",
|
"@n8n/node-cli": "0.39.3",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
{
|
{
|
||||||
"name": "n8n-nodes-ndc",
|
"name": "n8n-nodes-ndc",
|
||||||
"version": "0.1.4",
|
"version": "0.1.5",
|
||||||
"description": "Private NODE.DC nodes for scoped data products and Foundry bindings.",
|
"description": "Private NODE.DC nodes for scoped data products and Foundry bindings.",
|
||||||
"private": true,
|
"private": true,
|
||||||
"license": "UNLICENSED",
|
"license": "UNLICENSED",
|
||||||
|
|
|
||||||
|
|
@ -146,6 +146,64 @@ async function main() {
|
||||||
publish.batch.idempotencyKey,
|
publish.batch.idempotencyKey,
|
||||||
contracts.buildPublishPayload(items, 'execution-42', 'workflow-7', 'node-3', 'fleet.positions.current.v2', 0).batch.idempotencyKey,
|
contracts.buildPublishPayload(items, 'execution-42', 'workflow-7', 'node-3', 'fleet.positions.current.v2', 0).batch.idempotencyKey,
|
||||||
);
|
);
|
||||||
|
const zoneItems = [{
|
||||||
|
json: {
|
||||||
|
sourceId: 'gelios-zone-42',
|
||||||
|
semanticType: 'map.zone',
|
||||||
|
observedAt: '2026-07-15T12:10:00.000Z',
|
||||||
|
attributes: { display_name: 'Zone 42' },
|
||||||
|
geometry: {
|
||||||
|
type: 'Polygon',
|
||||||
|
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}];
|
||||||
|
const replacement = contracts.buildPublishPayload(
|
||||||
|
zoneItems,
|
||||||
|
'execution-43',
|
||||||
|
'workflow-7',
|
||||||
|
'node-zones',
|
||||||
|
'map.zones.current.v1',
|
||||||
|
0,
|
||||||
|
'replace',
|
||||||
|
);
|
||||||
|
assert.equal(externalContract.validateDataProductPublish(replacement).ok, true);
|
||||||
|
assert.equal(replacement.batch.mode, 'replace');
|
||||||
|
assert.equal(replacement.batch.generationAt, zoneItems[0].json.observedAt);
|
||||||
|
const replacementBatch = contracts.buildPublishPayload(
|
||||||
|
[{ json: { generationAt: zoneItems[0].json.observedAt, facts: zoneItems.map((item) => item.json) } }],
|
||||||
|
'execution-44',
|
||||||
|
'workflow-7',
|
||||||
|
'node-zones',
|
||||||
|
'map.zones.current.v1',
|
||||||
|
0,
|
||||||
|
'replace',
|
||||||
|
);
|
||||||
|
assert.deepEqual(replacementBatch.facts, replacement.facts);
|
||||||
|
assert.equal(replacementBatch.batch.generationAt, zoneItems[0].json.observedAt);
|
||||||
|
const emptyReplacement = contracts.buildPublishPayload(
|
||||||
|
[{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: [] } }],
|
||||||
|
'execution-45',
|
||||||
|
'workflow-7',
|
||||||
|
'node-zones',
|
||||||
|
'map.zones.current.v1',
|
||||||
|
0,
|
||||||
|
'replace',
|
||||||
|
);
|
||||||
|
assert.deepEqual(emptyReplacement.facts, []);
|
||||||
|
assert.equal(externalContract.validateDataProductPublish(emptyReplacement).ok, true);
|
||||||
|
assert.throws(
|
||||||
|
() => contracts.buildPublishPayload(
|
||||||
|
[{ json: { generationAt: '2026-07-15T13:00:00.000Z', facts: zoneItems.map((item) => item.json) } }],
|
||||||
|
'execution-46',
|
||||||
|
'workflow-7',
|
||||||
|
'node-zones',
|
||||||
|
'map.zones.current.v1',
|
||||||
|
0,
|
||||||
|
'replace',
|
||||||
|
),
|
||||||
|
/replace_generationAt_observedAt_mismatch/,
|
||||||
|
);
|
||||||
assert.throws(
|
assert.throws(
|
||||||
() => contracts.buildPublishPayload(
|
() => contracts.buildPublishPayload(
|
||||||
[{ json: { ...items[0].json, attributes: { accessToken: 'forbidden' } } }],
|
[{ json: { ...items[0].json, attributes: { accessToken: 'forbidden' } } }],
|
||||||
|
|
@ -303,6 +361,29 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) {
|
||||||
assert.equal(externalContract.validateDataProductPublish(publishCalls[0].options.body).ok, true);
|
assert.equal(externalContract.validateDataProductPublish(publishCalls[0].options.body).ok, true);
|
||||||
assert.equal(publishResult[0][0].json.publishedFactCount, 1);
|
assert.equal(publishResult[0][0].json.publishedFactCount, 1);
|
||||||
|
|
||||||
|
const replacementCalls = [];
|
||||||
|
const replacementContext = executionContext({
|
||||||
|
typeVersion: 2,
|
||||||
|
parameters: { dataProductId: 'map.zones.current.v1', publishMode: 'replace', sequence: 0 },
|
||||||
|
input: [{
|
||||||
|
json: {
|
||||||
|
generationAt: '2026-07-15T12:10:00.000Z',
|
||||||
|
facts: [{
|
||||||
|
sourceId: 'gelios-zone-42',
|
||||||
|
semanticType: 'map.zone',
|
||||||
|
observedAt: '2026-07-15T12:10:00.000Z',
|
||||||
|
attributes: { display_name: 'Zone 42' },
|
||||||
|
geometry: { type: 'Polygon', coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]] },
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
response: { ok: true, publishedFactCount: 1, currentRemovedCount: 0 },
|
||||||
|
calls: replacementCalls,
|
||||||
|
});
|
||||||
|
await nodes.get('NdcDataProductPublish').execute.call(replacementContext);
|
||||||
|
assert.equal(replacementCalls[0].options.body.batch.mode, 'replace');
|
||||||
|
assert.equal(externalContract.validateDataProductPublish(replacementCalls[0].options.body).ok, true);
|
||||||
|
|
||||||
const readCalls = [];
|
const readCalls = [];
|
||||||
const readContext = executionContext({
|
const readContext = executionContext({
|
||||||
parameters: { dataProductId: 'fleet.positions.current.v1', pageSize: 1000 },
|
parameters: { dataProductId: 'fleet.positions.current.v1', pageSize: 1000 },
|
||||||
|
|
@ -411,7 +492,7 @@ async function assertNodeHttpContracts(nodes, externalContract, constants) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function executionContext({ parameters, input = [{ json: {} }], response, calls }) {
|
function executionContext({ parameters, input = [{ json: {} }], response, calls, typeVersion = 1 }) {
|
||||||
return {
|
return {
|
||||||
getNodeParameter(name, _index, defaultValue) {
|
getNodeParameter(name, _index, defaultValue) {
|
||||||
return Object.prototype.hasOwnProperty.call(parameters, name) ? parameters[name] : defaultValue;
|
return Object.prototype.hasOwnProperty.call(parameters, name) ? parameters[name] : defaultValue;
|
||||||
|
|
@ -419,7 +500,7 @@ function executionContext({ parameters, input = [{ json: {} }], response, calls
|
||||||
getInputData() { return input; },
|
getInputData() { return input; },
|
||||||
getExecutionId() { return 'execution-42'; },
|
getExecutionId() { return 'execution-42'; },
|
||||||
getWorkflow() { return { id: 'workflow-7' }; },
|
getWorkflow() { return { id: 'workflow-7' }; },
|
||||||
getNode() { return { id: 'node-3' }; },
|
getNode() { return { id: 'node-3', typeVersion }; },
|
||||||
helpers: {
|
helpers: {
|
||||||
async httpRequestWithAuthentication(credentialName, options) {
|
async httpRequestWithAuthentication(credentialName, options) {
|
||||||
calls.push({ credentialName, options });
|
calls.push({ credentialName, options });
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ COPY packages/external-provider-contract/package.json ./packages/external-provid
|
||||||
COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs
|
COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs
|
||||||
COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs
|
COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs
|
||||||
COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs
|
COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs
|
||||||
|
COPY packages/external-provider-contract/src/geometry.mjs ./packages/external-provider-contract/src/geometry.mjs
|
||||||
COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs
|
COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs
|
||||||
COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs
|
COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs
|
||||||
COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/
|
COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,32 @@
|
||||||
|
{
|
||||||
|
"id": "map.zones.current.v1",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"ontologyRevision": "ontology.map.zone.v1",
|
||||||
|
"deliveryMode": "snapshot+patch",
|
||||||
|
"semanticTypes": ["map.zone"],
|
||||||
|
"fields": [
|
||||||
|
"area_square_meters",
|
||||||
|
"description",
|
||||||
|
"display_name",
|
||||||
|
"geometry",
|
||||||
|
"geometry_kind",
|
||||||
|
"max_speed_kph",
|
||||||
|
"perimeter_meters",
|
||||||
|
"source_kind",
|
||||||
|
"source_revision",
|
||||||
|
"style_color"
|
||||||
|
],
|
||||||
|
"fieldContracts": {
|
||||||
|
"area_square_meters": { "type": "number", "required": false, "minimum": 0 },
|
||||||
|
"description": { "type": "string", "required": false },
|
||||||
|
"display_name": { "type": "string", "required": true },
|
||||||
|
"geometry": { "type": "geometry", "required": true },
|
||||||
|
"geometry_kind": { "type": "string", "required": true, "enum": ["circle", "corridor", "polygon"] },
|
||||||
|
"max_speed_kph": { "type": "number", "required": false, "minimum": 0 },
|
||||||
|
"perimeter_meters": { "type": "number", "required": false, "minimum": 0 },
|
||||||
|
"source_kind": { "type": "string", "required": true, "enum": ["live_api", "versioned_snapshot"] },
|
||||||
|
"source_revision": { "type": "string", "required": true },
|
||||||
|
"style_color": { "type": "string", "required": false }
|
||||||
|
},
|
||||||
|
"history": { "mode": "none", "retentionDays": 1 }
|
||||||
|
}
|
||||||
|
|
@ -3,6 +3,7 @@ import {
|
||||||
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
|
||||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||||
|
isBoundedGeoJsonGeometry,
|
||||||
} from "@nodedc/external-provider-contract/data-plane";
|
} from "@nodedc/external-provider-contract/data-plane";
|
||||||
|
|
||||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||||
|
|
@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
const existing = await client.query(
|
const existing = await client.query(
|
||||||
`select id as "batchId", fact_count as "publishedFactCount",
|
`select id as "batchId", fact_count as "publishedFactCount",
|
||||||
current_updated_count as "currentUpdatedCount",
|
current_updated_count as "currentUpdatedCount",
|
||||||
|
current_removed_count as "currentRemovedCount",
|
||||||
history_inserted_count as "historyInsertedCount",
|
history_inserted_count as "historyInsertedCount",
|
||||||
patch_operation_count as "patchOperationCount",
|
patch_operation_count as "patchOperationCount",
|
||||||
delivery_cursor::text as cursor, received_at as "acceptedAt",
|
delivery_cursor::text as cursor, received_at as "acceptedAt",
|
||||||
|
|
@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
|
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch);
|
||||||
|
|
||||||
const records = batch.facts.map((fact) => ({
|
const records = batch.facts.map((fact) => ({
|
||||||
source_id: fact.sourceId,
|
source_id: fact.sourceId,
|
||||||
semantic_type: fact.semanticType,
|
semantic_type: fact.semanticType,
|
||||||
|
|
@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||||
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||||
case when item.geometry is null then null
|
case when item.geometry is null then null
|
||||||
else ST_SetSRID(ST_MakePoint(
|
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
|
||||||
(item.geometry->'coordinates'->>0)::double precision,
|
|
||||||
(item.geometry->'coordinates'->>1)::double precision
|
|
||||||
), 4326)::geography end,
|
|
||||||
item.fingerprint, now()
|
item.fingerprint, now()
|
||||||
from jsonb_to_recordset($5::jsonb) as item(
|
from jsonb_to_recordset($5::jsonb) as item(
|
||||||
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
|
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
|
||||||
|
|
@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
returning source_id as "sourceId", semantic_type as "semanticType",
|
returning source_id as "sourceId", semantic_type as "semanticType",
|
||||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||||
case when geometry is null then null
|
case when geometry is null then null
|
||||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry,
|
||||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
|
||||||
)) end as geometry,
|
|
||||||
fingerprint`,
|
fingerprint`,
|
||||||
[
|
[
|
||||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||||
|
|
@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const removed = batch.batch.mode === "replace"
|
||||||
|
? await removeMissingCurrent(client, batch, records)
|
||||||
|
: [];
|
||||||
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
|
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
|
||||||
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
|
const operations = [
|
||||||
|
...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })),
|
||||||
|
...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })),
|
||||||
|
];
|
||||||
const chunks = definition.deliveryMode === "snapshot+patch"
|
const chunks = definition.deliveryMode === "snapshot+patch"
|
||||||
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
|
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
|
||||||
: [];
|
: [];
|
||||||
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
|
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
|
||||||
|
if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch);
|
||||||
|
|
||||||
await client.query(
|
await client.query(
|
||||||
`update external_data_plane_batches set
|
`update external_data_plane_batches set
|
||||||
current_updated_count = $2,
|
current_updated_count = $2,
|
||||||
history_inserted_count = $3,
|
current_removed_count = $3,
|
||||||
patch_operation_count = $4,
|
history_inserted_count = $4,
|
||||||
delivery_cursor = $5
|
patch_operation_count = $5,
|
||||||
|
delivery_cursor = $6
|
||||||
where id = $1`,
|
where id = $1`,
|
||||||
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
|
[batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor],
|
||||||
);
|
);
|
||||||
await client.query("commit");
|
await client.query("commit");
|
||||||
return normalizeReceipt({
|
return normalizeReceipt({
|
||||||
|
|
@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
|
||||||
idempotent: false,
|
idempotent: false,
|
||||||
publishedFactCount: batch.facts.length,
|
publishedFactCount: batch.facts.length,
|
||||||
currentUpdatedCount: updated.rowCount,
|
currentUpdatedCount: updated.rowCount,
|
||||||
|
currentRemovedCount: removed.length,
|
||||||
historyInsertedCount,
|
historyInsertedCount,
|
||||||
patchOperationCount: operations.length,
|
patchOperationCount: operations.length,
|
||||||
cursor: String(cursor),
|
cursor: String(cursor),
|
||||||
|
|
@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
|
||||||
`select source_id as "sourceId", semantic_type as "semanticType",
|
`select source_id as "sourceId", semantic_type as "semanticType",
|
||||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||||
case when geometry is null then null
|
case when geometry is null then null
|
||||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
|
||||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
|
||||||
)) end as geometry
|
|
||||||
from external_data_plane_current
|
from external_data_plane_current
|
||||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
|
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
|
||||||
order by source_id asc, semantic_type asc
|
order by source_id asc, semantic_type asc
|
||||||
|
|
@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options)
|
||||||
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
|
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
|
||||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||||
case when geometry is null then null
|
case when geometry is null then null
|
||||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
|
||||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
|
||||||
)) end as geometry
|
|
||||||
from ranked
|
from ranked
|
||||||
where sample_rank = 1
|
where sample_rank = 1
|
||||||
and (
|
and (
|
||||||
|
|
@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
|
||||||
return result.rowCount;
|
return result.rowCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function lockReplacementGeneration(client, batch) {
|
||||||
|
const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId];
|
||||||
|
await client.query(
|
||||||
|
`insert into external_data_plane_delivery_state (
|
||||||
|
tenant_id, connection_id, provider_id, data_product_id, current_cursor
|
||||||
|
) values ($1, $2, $3, $4, 0)
|
||||||
|
on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`,
|
||||||
|
scope,
|
||||||
|
);
|
||||||
|
const state = await client.query(
|
||||||
|
`select current_generation_at as "currentGenerationAt"
|
||||||
|
from external_data_plane_delivery_state
|
||||||
|
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
|
||||||
|
for update`,
|
||||||
|
scope,
|
||||||
|
);
|
||||||
|
const generationAt = new Date(batch.batch.generationAt);
|
||||||
|
const currentGenerationAt = state.rows[0]?.currentGenerationAt
|
||||||
|
? new Date(state.rows[0].currentGenerationAt)
|
||||||
|
: null;
|
||||||
|
if (currentGenerationAt && generationAt <= currentGenerationAt) {
|
||||||
|
throw deliveryError("data_product_generation_not_newer", 409);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function removeMissingCurrent(client, batch, records) {
|
||||||
|
const result = await client.query(
|
||||||
|
`delete from external_data_plane_current as target
|
||||||
|
where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3
|
||||||
|
and target.data_product_id = $4
|
||||||
|
and not exists (
|
||||||
|
select 1
|
||||||
|
from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text)
|
||||||
|
where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type
|
||||||
|
)
|
||||||
|
returning source_id as "sourceId", semantic_type as "semanticType"`,
|
||||||
|
[
|
||||||
|
batch.source.tenantId,
|
||||||
|
batch.source.connectionId,
|
||||||
|
batch.source.providerId,
|
||||||
|
batch.contract.dataProductId,
|
||||||
|
JSON.stringify(records),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
return result.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function persistReplacementGeneration(client, batch) {
|
||||||
|
await client.query(
|
||||||
|
`update external_data_plane_delivery_state
|
||||||
|
set current_generation_at = $5, current_generation_id = $6, updated_at = now()
|
||||||
|
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
|
||||||
|
[
|
||||||
|
batch.source.tenantId,
|
||||||
|
batch.source.connectionId,
|
||||||
|
batch.source.providerId,
|
||||||
|
batch.contract.dataProductId,
|
||||||
|
batch.batch.generationAt,
|
||||||
|
batch.batch.runId,
|
||||||
|
],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
async function persistHistory(client, batch, policy, facts) {
|
async function persistHistory(client, batch, policy, facts) {
|
||||||
if (!facts.length || policy?.mode === "none") return 0;
|
if (!facts.length || policy?.mode === "none") return 0;
|
||||||
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
|
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
|
||||||
|
|
@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) {
|
||||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||||
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||||
case when item.geometry is null then null
|
case when item.geometry is null then null
|
||||||
else ST_SetSRID(ST_MakePoint(
|
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
|
||||||
(item.geometry->'coordinates'->>0)::double precision,
|
|
||||||
(item.geometry->'coordinates'->>1)::double precision
|
|
||||||
), 4326)::geography end,
|
|
||||||
item.fingerprint, now()
|
item.fingerprint, now()
|
||||||
from jsonb_to_recordset($5::jsonb) as item(
|
from jsonb_to_recordset($5::jsonb) as item(
|
||||||
source_id text, semantic_type text, bucket_start timestamptz,
|
source_id text, semantic_type text, bucket_start timestamptz,
|
||||||
|
|
@ -570,6 +634,7 @@ function normalizeReceipt(value) {
|
||||||
idempotent: value.idempotent === true,
|
idempotent: value.idempotent === true,
|
||||||
publishedFactCount: Number(value.publishedFactCount || 0),
|
publishedFactCount: Number(value.publishedFactCount || 0),
|
||||||
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
|
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
|
||||||
|
currentRemovedCount: Number(value.currentRemovedCount || 0),
|
||||||
historyInsertedCount: Number(value.historyInsertedCount || 0),
|
historyInsertedCount: Number(value.historyInsertedCount || 0),
|
||||||
patchOperationCount: Number(value.patchOperationCount || 0),
|
patchOperationCount: Number(value.patchOperationCount || 0),
|
||||||
cursor: String(value.cursor || "0"),
|
cursor: String(value.cursor || "0"),
|
||||||
|
|
@ -652,6 +717,8 @@ function publishFingerprint(batch) {
|
||||||
runId: batch.batch.runId,
|
runId: batch.batch.runId,
|
||||||
sequence: batch.batch.sequence,
|
sequence: batch.batch.sequence,
|
||||||
idempotencyKey: batch.batch.idempotencyKey,
|
idempotencyKey: batch.batch.idempotencyKey,
|
||||||
|
mode: batch.batch.mode || "upsert",
|
||||||
|
...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}),
|
||||||
},
|
},
|
||||||
facts: batch.facts,
|
facts: batch.facts,
|
||||||
});
|
});
|
||||||
|
|
@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) {
|
||||||
|
|
||||||
function fieldContractValueMatchesType(value, type) {
|
function fieldContractValueMatchesType(value, type) {
|
||||||
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
|
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
|
||||||
if (type === "point") {
|
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
|
||||||
return value?.type === "Point"
|
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
|
||||||
&& Array.isArray(value.coordinates)
|
|
||||||
&& value.coordinates.length === 2
|
|
||||||
&& value.coordinates.every(Number.isFinite)
|
|
||||||
&& value.coordinates[0] >= -180
|
|
||||||
&& value.coordinates[0] <= 180
|
|
||||||
&& value.coordinates[1] >= -90
|
|
||||||
&& value.coordinates[1] <= 90;
|
|
||||||
}
|
|
||||||
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,10 @@
|
||||||
|
import { isBoundedGeoJsonGeometry } from "@nodedc/external-provider-contract/data-plane";
|
||||||
|
|
||||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||||
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
||||||
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
||||||
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
||||||
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
|
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
|
||||||
const DEFINITION_KEYS = new Set([
|
const DEFINITION_KEYS = new Set([
|
||||||
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
|
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
|
||||||
]);
|
]);
|
||||||
|
|
@ -63,7 +65,7 @@ function normalizeFieldContract(value) {
|
||||||
if (value.enum !== undefined) {
|
if (value.enum !== undefined) {
|
||||||
if (!Array.isArray(value.enum) || value.enum.length === 0
|
if (!Array.isArray(value.enum) || value.enum.length === 0
|
||||||
|| new Set(value.enum.map(stableLiteral)).size !== value.enum.length
|
|| new Set(value.enum.map(stableLiteral)).size !== value.enum.length
|
||||||
|| new Set(["point", "string_array"]).has(type)
|
|| new Set(["point", "geometry", "string_array"]).has(type)
|
||||||
|| value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
|
|| value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
|
||||||
throw policyError("data_product_field_contract_enum_invalid");
|
throw policyError("data_product_field_contract_enum_invalid");
|
||||||
}
|
}
|
||||||
|
|
@ -171,6 +173,9 @@ function hasOnlyKeys(value, allowed) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function fieldContractValueMatchesType(value, type) {
|
function fieldContractValueMatchesType(value, type) {
|
||||||
|
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
|
||||||
|
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
|
||||||
|
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
|
||||||
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
return typeof value === type && (type !== "number" || Number.isFinite(value));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -61,6 +61,7 @@ export async function migrate(pool) {
|
||||||
`);
|
`);
|
||||||
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
|
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
|
||||||
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
|
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
|
||||||
|
await pool.query("alter table external_data_plane_batches add column if not exists current_removed_count integer not null default 0");
|
||||||
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
|
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
|
||||||
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
|
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
|
||||||
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
|
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
|
||||||
|
|
@ -245,7 +246,7 @@ export async function migrate(pool) {
|
||||||
observed_at timestamptz not null,
|
observed_at timestamptz not null,
|
||||||
received_at timestamptz not null,
|
received_at timestamptz not null,
|
||||||
attributes jsonb not null default '{}'::jsonb,
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
geometry geography(Point, 4326),
|
geometry geography(Geometry, 4326),
|
||||||
fingerprint text not null,
|
fingerprint text not null,
|
||||||
primary key (id, observed_at),
|
primary key (id, observed_at),
|
||||||
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
|
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
|
||||||
|
|
@ -266,12 +267,30 @@ export async function migrate(pool) {
|
||||||
observed_at timestamptz not null,
|
observed_at timestamptz not null,
|
||||||
received_at timestamptz not null,
|
received_at timestamptz not null,
|
||||||
attributes jsonb not null default '{}'::jsonb,
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
geometry geography(Point, 4326),
|
geometry geography(Geometry, 4326),
|
||||||
fingerprint text not null,
|
fingerprint text not null,
|
||||||
updated_at timestamptz not null default now(),
|
updated_at timestamptz not null default now(),
|
||||||
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
|
await pool.query(`
|
||||||
|
do $$
|
||||||
|
begin
|
||||||
|
if exists (
|
||||||
|
select 1
|
||||||
|
from pg_attribute
|
||||||
|
where attrelid = 'external_data_plane_current'::regclass
|
||||||
|
and attname = 'geometry'
|
||||||
|
and not attisdropped
|
||||||
|
and format_type(atttypid, atttypmod) = 'geography(Point,4326)'
|
||||||
|
) then
|
||||||
|
alter table external_data_plane_current
|
||||||
|
alter column geometry type geography(Geometry, 4326)
|
||||||
|
using geometry::geometry::geography;
|
||||||
|
end if;
|
||||||
|
end
|
||||||
|
$$
|
||||||
|
`);
|
||||||
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
|
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
|
||||||
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
|
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
|
||||||
|
|
||||||
|
|
@ -282,10 +301,14 @@ export async function migrate(pool) {
|
||||||
provider_id text not null,
|
provider_id text not null,
|
||||||
data_product_id text not null,
|
data_product_id text not null,
|
||||||
current_cursor bigint not null default 0,
|
current_cursor bigint not null default 0,
|
||||||
|
current_generation_at timestamptz,
|
||||||
|
current_generation_id text,
|
||||||
updated_at timestamptz not null default now(),
|
updated_at timestamptz not null default now(),
|
||||||
primary key (tenant_id, connection_id, provider_id, data_product_id)
|
primary key (tenant_id, connection_id, provider_id, data_product_id)
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
|
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_at timestamptz");
|
||||||
|
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_id text");
|
||||||
|
|
||||||
await pool.query(`
|
await pool.query(`
|
||||||
create table if not exists external_data_plane_patch_outbox (
|
create table if not exists external_data_plane_patch_outbox (
|
||||||
|
|
@ -316,7 +339,7 @@ export async function migrate(pool) {
|
||||||
observed_at timestamptz not null,
|
observed_at timestamptz not null,
|
||||||
received_at timestamptz not null,
|
received_at timestamptz not null,
|
||||||
attributes jsonb not null default '{}'::jsonb,
|
attributes jsonb not null default '{}'::jsonb,
|
||||||
geometry geography(Point, 4326),
|
geometry geography(Geometry, 4326),
|
||||||
fingerprint text not null,
|
fingerprint text not null,
|
||||||
updated_at timestamptz not null default now(),
|
updated_at timestamptz not null default now(),
|
||||||
primary key (
|
primary key (
|
||||||
|
|
|
||||||
|
|
@ -208,6 +208,10 @@ export function materializeDataProductPublish(value, binding, definition, dataPr
|
||||||
sequence: value.batch?.sequence,
|
sequence: value.batch?.sequence,
|
||||||
idempotencyKey: value.batch?.idempotencyKey,
|
idempotencyKey: value.batch?.idempotencyKey,
|
||||||
receivedAt: now.toISOString(),
|
receivedAt: now.toISOString(),
|
||||||
|
...(value.batch?.mode === "replace" ? {
|
||||||
|
mode: "replace",
|
||||||
|
generationAt: value.batch?.generationAt,
|
||||||
|
} : {}),
|
||||||
},
|
},
|
||||||
facts: value.facts,
|
facts: value.facts,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -205,6 +205,54 @@ try {
|
||||||
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
|
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const zoneDefinition = normalizeDataProductDefinition({
|
||||||
|
id: "test.zones.current.v1",
|
||||||
|
version: "1.0.0",
|
||||||
|
ontologyRevision: "ontology.map.zone.v1",
|
||||||
|
deliveryMode: "snapshot+patch",
|
||||||
|
semanticTypes: ["map.zone"],
|
||||||
|
fields: ["display_name", "geometry", "geometry_kind", "source_kind", "source_revision"],
|
||||||
|
fieldContracts: {
|
||||||
|
display_name: { type: "string", required: true },
|
||||||
|
geometry: { type: "geometry", required: true },
|
||||||
|
geometry_kind: { type: "string", required: true, enum: ["polygon"] },
|
||||||
|
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
|
||||||
|
source_revision: { type: "string", required: true },
|
||||||
|
},
|
||||||
|
history: { mode: "none", retentionDays: 1 },
|
||||||
|
});
|
||||||
|
await persistDataProductDefinition(pool, zoneDefinition);
|
||||||
|
const storedZoneDefinition = await loadDataProductDefinition(pool, zoneDefinition.id);
|
||||||
|
const zoneBinding = { ...binding, allowedDataProductIds: [zoneDefinition.id] };
|
||||||
|
const firstZoneGenerationAt = "2026-07-15T10:10:00.000Z";
|
||||||
|
const firstZoneGeneration = await persistDataProductPublish(
|
||||||
|
pool,
|
||||||
|
zoneBatch("zones-01", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt), zoneFact("zone-02", firstZoneGenerationAt)]),
|
||||||
|
storedZoneDefinition,
|
||||||
|
);
|
||||||
|
assert.equal(firstZoneGeneration.currentUpdatedCount, 2);
|
||||||
|
assert.equal(firstZoneGeneration.currentRemovedCount, 0);
|
||||||
|
const secondZoneGenerationAt = "2026-07-15T10:11:00.000Z";
|
||||||
|
const secondZoneGeneration = await persistDataProductPublish(
|
||||||
|
pool,
|
||||||
|
zoneBatch("zones-02", secondZoneGenerationAt, [zoneFact("zone-01", secondZoneGenerationAt)]),
|
||||||
|
storedZoneDefinition,
|
||||||
|
);
|
||||||
|
assert.equal(secondZoneGeneration.currentUpdatedCount, 1);
|
||||||
|
assert.equal(secondZoneGeneration.currentRemovedCount, 1);
|
||||||
|
assert.equal(secondZoneGeneration.patchOperationCount, 2);
|
||||||
|
const zoneSnapshot = await readDataProductSnapshot(pool, zoneBinding, storedZoneDefinition);
|
||||||
|
assert.equal(validateDataProductSnapshot(zoneSnapshot).ok, true);
|
||||||
|
assert.deepEqual(zoneSnapshot.facts.map((value) => value.sourceId), ["zone-01"]);
|
||||||
|
assert.equal(zoneSnapshot.facts[0].geometry.type, "Polygon");
|
||||||
|
const zonePatches = await readPatchEvents(pool, zoneBinding, storedZoneDefinition, 0n);
|
||||||
|
assert.equal(zonePatches.every((value) => validateDataProductPatch(value).ok), true);
|
||||||
|
assert.equal(zonePatches.flatMap((value) => value.operations).some((value) => value.op === "remove" && value.sourceId === "zone-02"), true);
|
||||||
|
await assert.rejects(
|
||||||
|
persistDataProductPublish(pool, zoneBatch("zones-stale", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt)]), storedZoneDefinition),
|
||||||
|
(error) => error?.status === 409 && error?.code === "data_product_generation_not_newer",
|
||||||
|
);
|
||||||
|
|
||||||
console.log("external-data-plane delivery integration: ok");
|
console.log("external-data-plane delivery integration: ok");
|
||||||
} finally {
|
} finally {
|
||||||
await pool.end();
|
await pool.end();
|
||||||
|
|
@ -233,3 +281,38 @@ function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10
|
||||||
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
geometry: { type: "Point", coordinates: [longitude, latitude] },
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function zoneBatch(runId, generationAt, facts) {
|
||||||
|
return {
|
||||||
|
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||||
|
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
|
||||||
|
contract: { dataProductId: "test.zones.current.v1", ontologyRevision: "ontology.map.zone.v1", version: "1.0.0" },
|
||||||
|
batch: {
|
||||||
|
runId,
|
||||||
|
sequence: 0,
|
||||||
|
idempotencyKey: `${runId}.chunk-0`,
|
||||||
|
receivedAt: generationAt,
|
||||||
|
mode: "replace",
|
||||||
|
generationAt,
|
||||||
|
},
|
||||||
|
facts,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function zoneFact(sourceId, observedAt) {
|
||||||
|
return {
|
||||||
|
sourceId,
|
||||||
|
semanticType: "map.zone",
|
||||||
|
observedAt,
|
||||||
|
attributes: {
|
||||||
|
display_name: sourceId,
|
||||||
|
geometry_kind: "polygon",
|
||||||
|
source_kind: "live_api",
|
||||||
|
source_revision: "gelios-rest-v1",
|
||||||
|
},
|
||||||
|
geometry: {
|
||||||
|
type: "Polygon",
|
||||||
|
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [
|
||||||
"fleet.positions.current.v2",
|
"fleet.positions.current.v2",
|
||||||
"fleet.positions.current.v3",
|
"fleet.positions.current.v3",
|
||||||
"fleet.positions.current.v4",
|
"fleet.positions.current.v4",
|
||||||
|
"map.zones.current.v1",
|
||||||
]);
|
]);
|
||||||
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
|
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
|
||||||
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
|
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
|
||||||
|
|
@ -45,6 +46,10 @@ assert.deepEqual(bundled[3].fieldContracts.movement_state, {
|
||||||
required: true,
|
required: true,
|
||||||
enum: ["moving", "stopped"],
|
enum: ["moving", "stopped"],
|
||||||
});
|
});
|
||||||
|
assert.equal(bundled[4].version, "1.0.0");
|
||||||
|
assert.equal(bundled[4].ontologyRevision, "ontology.map.zone.v1");
|
||||||
|
assert.deepEqual(bundled[4].semanticTypes, ["map.zone"]);
|
||||||
|
assert.deepEqual(bundled[4].fieldContracts.geometry, { type: "geometry", required: true });
|
||||||
assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length);
|
assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length);
|
||||||
|
|
||||||
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));
|
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue