feat(platform): add replaceable geozone data layer

This commit is contained in:
Codex
2026-07-20 21:48:27 +03:00
parent 8a7465cf0e
commit 3446f3edc2
31 changed files with 1342 additions and 128 deletions
+1
View File
@@ -6,6 +6,7 @@ COPY packages/external-provider-contract/package.json ./packages/external-provid
COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs
COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs
COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs
COPY packages/external-provider-contract/src/geometry.mjs ./packages/external-provider-contract/src/geometry.mjs
COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs
COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs
COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/
@@ -0,0 +1,32 @@
{
"id": "map.zones.current.v1",
"version": "1.0.0",
"ontologyRevision": "ontology.map.zone.v1",
"deliveryMode": "snapshot+patch",
"semanticTypes": ["map.zone"],
"fields": [
"area_square_meters",
"description",
"display_name",
"geometry",
"geometry_kind",
"max_speed_kph",
"perimeter_meters",
"source_kind",
"source_revision",
"style_color"
],
"fieldContracts": {
"area_square_meters": { "type": "number", "required": false, "minimum": 0 },
"description": { "type": "string", "required": false },
"display_name": { "type": "string", "required": true },
"geometry": { "type": "geometry", "required": true },
"geometry_kind": { "type": "string", "required": true, "enum": ["circle", "corridor", "polygon"] },
"max_speed_kph": { "type": "number", "required": false, "minimum": 0 },
"perimeter_meters": { "type": "number", "required": false, "minimum": 0 },
"source_kind": { "type": "string", "required": true, "enum": ["live_api", "versioned_snapshot"] },
"source_revision": { "type": "string", "required": true },
"style_color": { "type": "string", "required": false }
},
"history": { "mode": "none", "retentionDays": 1 }
}
@@ -3,6 +3,7 @@ import {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
isBoundedGeoJsonGeometry,
} from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
@@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
const existing = await client.query(
`select id as "batchId", fact_count as "publishedFactCount",
current_updated_count as "currentUpdatedCount",
current_removed_count as "currentRemovedCount",
history_inserted_count as "historyInsertedCount",
patch_operation_count as "patchOperationCount",
delivery_cursor::text as cursor, received_at as "acceptedAt",
@@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, {
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
}
if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch);
const records = batch.facts.map((fact) => ({
source_id: fact.sourceId,
semantic_type: fact.semanticType,
@@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint(
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
@@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
returning source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry,
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry,
fingerprint`,
[
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
@@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, {
],
);
const removed = batch.batch.mode === "replace"
? await removeMissingCurrent(client, batch, records)
: [];
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
const operations = [
...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })),
...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })),
];
const chunks = definition.deliveryMode === "snapshot+patch"
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
: [];
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch);
await client.query(
`update external_data_plane_batches set
current_updated_count = $2,
history_inserted_count = $3,
patch_operation_count = $4,
delivery_cursor = $5
current_removed_count = $3,
history_inserted_count = $4,
patch_operation_count = $5,
delivery_cursor = $6
where id = $1`,
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
[batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor],
);
await client.query("commit");
return normalizeReceipt({
@@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
idempotent: false,
publishedFactCount: batch.facts.length,
currentUpdatedCount: updated.rowCount,
currentRemovedCount: removed.length,
historyInsertedCount,
patchOperationCount: operations.length,
cursor: String(cursor),
@@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
`select source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
from external_data_plane_current
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
order by source_id asc, semantic_type asc
@@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options)
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
from ranked
where sample_rank = 1
and (
@@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
return result.rowCount;
}
async function lockReplacementGeneration(client, batch) {
const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId];
await client.query(
`insert into external_data_plane_delivery_state (
tenant_id, connection_id, provider_id, data_product_id, current_cursor
) values ($1, $2, $3, $4, 0)
on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`,
scope,
);
const state = await client.query(
`select current_generation_at as "currentGenerationAt"
from external_data_plane_delivery_state
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
for update`,
scope,
);
const generationAt = new Date(batch.batch.generationAt);
const currentGenerationAt = state.rows[0]?.currentGenerationAt
? new Date(state.rows[0].currentGenerationAt)
: null;
if (currentGenerationAt && generationAt <= currentGenerationAt) {
throw deliveryError("data_product_generation_not_newer", 409);
}
}
async function removeMissingCurrent(client, batch, records) {
const result = await client.query(
`delete from external_data_plane_current as target
where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3
and target.data_product_id = $4
and not exists (
select 1
from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text)
where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type
)
returning source_id as "sourceId", semantic_type as "semanticType"`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
JSON.stringify(records),
],
);
return result.rows;
}
async function persistReplacementGeneration(client, batch) {
await client.query(
`update external_data_plane_delivery_state
set current_generation_at = $5, current_generation_id = $6, updated_at = now()
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
batch.batch.generationAt,
batch.batch.runId,
],
);
}
async function persistHistory(client, batch, policy, facts) {
if (!facts.length || policy?.mode === "none") return 0;
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
@@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) {
select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint(
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, bucket_start timestamptz,
@@ -570,6 +634,7 @@ function normalizeReceipt(value) {
idempotent: value.idempotent === true,
publishedFactCount: Number(value.publishedFactCount || 0),
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
currentRemovedCount: Number(value.currentRemovedCount || 0),
historyInsertedCount: Number(value.historyInsertedCount || 0),
patchOperationCount: Number(value.patchOperationCount || 0),
cursor: String(value.cursor || "0"),
@@ -652,6 +717,8 @@ function publishFingerprint(batch) {
runId: batch.batch.runId,
sequence: batch.batch.sequence,
idempotencyKey: batch.batch.idempotencyKey,
mode: batch.batch.mode || "upsert",
...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}),
},
facts: batch.facts,
});
@@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) {
function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") {
return value?.type === "Point"
&& Array.isArray(value.coordinates)
&& value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite)
&& value.coordinates[0] >= -180
&& value.coordinates[0] <= 180
&& value.coordinates[1] >= -90
&& value.coordinates[1] <= 90;
}
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
return typeof value === type && (type !== "number" || Number.isFinite(value));
}
@@ -1,8 +1,10 @@
import { isBoundedGeoJsonGeometry } from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
const DEFINITION_KEYS = new Set([
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
]);
@@ -63,7 +65,7 @@ function normalizeFieldContract(value) {
if (value.enum !== undefined) {
if (!Array.isArray(value.enum) || value.enum.length === 0
|| new Set(value.enum.map(stableLiteral)).size !== value.enum.length
|| new Set(["point", "string_array"]).has(type)
|| new Set(["point", "geometry", "string_array"]).has(type)
|| value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
throw policyError("data_product_field_contract_enum_invalid");
}
@@ -171,6 +173,9 @@ function hasOnlyKeys(value, allowed) {
}
function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
return typeof value === type && (type !== "number" || Number.isFinite(value));
}
+26 -3
View File
@@ -61,6 +61,7 @@ export async function migrate(pool) {
`);
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists current_removed_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
@@ -245,7 +246,7 @@ export async function migrate(pool) {
observed_at timestamptz not null,
received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326),
geometry geography(Geometry, 4326),
fingerprint text not null,
primary key (id, observed_at),
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
@@ -266,12 +267,30 @@ export async function migrate(pool) {
observed_at timestamptz not null,
received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326),
geometry geography(Geometry, 4326),
fingerprint text not null,
updated_at timestamptz not null default now(),
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
)
`);
await pool.query(`
do $$
begin
if exists (
select 1
from pg_attribute
where attrelid = 'external_data_plane_current'::regclass
and attname = 'geometry'
and not attisdropped
and format_type(atttypid, atttypmod) = 'geography(Point,4326)'
) then
alter table external_data_plane_current
alter column geometry type geography(Geometry, 4326)
using geometry::geometry::geography;
end if;
end
$$
`);
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
@@ -282,10 +301,14 @@ export async function migrate(pool) {
provider_id text not null,
data_product_id text not null,
current_cursor bigint not null default 0,
current_generation_at timestamptz,
current_generation_id text,
updated_at timestamptz not null default now(),
primary key (tenant_id, connection_id, provider_id, data_product_id)
)
`);
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_at timestamptz");
await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_id text");
await pool.query(`
create table if not exists external_data_plane_patch_outbox (
@@ -316,7 +339,7 @@ export async function migrate(pool) {
observed_at timestamptz not null,
received_at timestamptz not null,
attributes jsonb not null default '{}'::jsonb,
geometry geography(Point, 4326),
geometry geography(Geometry, 4326),
fingerprint text not null,
updated_at timestamptz not null default now(),
primary key (
@@ -208,6 +208,10 @@ export function materializeDataProductPublish(value, binding, definition, dataPr
sequence: value.batch?.sequence,
idempotencyKey: value.batch?.idempotencyKey,
receivedAt: now.toISOString(),
...(value.batch?.mode === "replace" ? {
mode: "replace",
generationAt: value.batch?.generationAt,
} : {}),
},
facts: value.facts,
};
@@ -205,6 +205,54 @@ try {
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
);
const zoneDefinition = normalizeDataProductDefinition({
id: "test.zones.current.v1",
version: "1.0.0",
ontologyRevision: "ontology.map.zone.v1",
deliveryMode: "snapshot+patch",
semanticTypes: ["map.zone"],
fields: ["display_name", "geometry", "geometry_kind", "source_kind", "source_revision"],
fieldContracts: {
display_name: { type: "string", required: true },
geometry: { type: "geometry", required: true },
geometry_kind: { type: "string", required: true, enum: ["polygon"] },
source_kind: { type: "string", required: true, enum: ["live_api", "versioned_snapshot"] },
source_revision: { type: "string", required: true },
},
history: { mode: "none", retentionDays: 1 },
});
await persistDataProductDefinition(pool, zoneDefinition);
const storedZoneDefinition = await loadDataProductDefinition(pool, zoneDefinition.id);
const zoneBinding = { ...binding, allowedDataProductIds: [zoneDefinition.id] };
const firstZoneGenerationAt = "2026-07-15T10:10:00.000Z";
const firstZoneGeneration = await persistDataProductPublish(
pool,
zoneBatch("zones-01", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt), zoneFact("zone-02", firstZoneGenerationAt)]),
storedZoneDefinition,
);
assert.equal(firstZoneGeneration.currentUpdatedCount, 2);
assert.equal(firstZoneGeneration.currentRemovedCount, 0);
const secondZoneGenerationAt = "2026-07-15T10:11:00.000Z";
const secondZoneGeneration = await persistDataProductPublish(
pool,
zoneBatch("zones-02", secondZoneGenerationAt, [zoneFact("zone-01", secondZoneGenerationAt)]),
storedZoneDefinition,
);
assert.equal(secondZoneGeneration.currentUpdatedCount, 1);
assert.equal(secondZoneGeneration.currentRemovedCount, 1);
assert.equal(secondZoneGeneration.patchOperationCount, 2);
const zoneSnapshot = await readDataProductSnapshot(pool, zoneBinding, storedZoneDefinition);
assert.equal(validateDataProductSnapshot(zoneSnapshot).ok, true);
assert.deepEqual(zoneSnapshot.facts.map((value) => value.sourceId), ["zone-01"]);
assert.equal(zoneSnapshot.facts[0].geometry.type, "Polygon");
const zonePatches = await readPatchEvents(pool, zoneBinding, storedZoneDefinition, 0n);
assert.equal(zonePatches.every((value) => validateDataProductPatch(value).ok), true);
assert.equal(zonePatches.flatMap((value) => value.operations).some((value) => value.op === "remove" && value.sourceId === "zone-02"), true);
await assert.rejects(
persistDataProductPublish(pool, zoneBatch("zones-stale", firstZoneGenerationAt, [zoneFact("zone-01", firstZoneGenerationAt)]), storedZoneDefinition),
(error) => error?.status === 409 && error?.code === "data_product_generation_not_newer",
);
console.log("external-data-plane delivery integration: ok");
} finally {
await pool.end();
@@ -233,3 +281,38 @@ function fact(sourceId, longitude, latitude, status, observedAt = "2026-07-15T10
geometry: { type: "Point", coordinates: [longitude, latitude] },
};
}
function zoneBatch(runId, generationAt, facts) {
return {
schemaVersion: "nodedc.external-provider-contract/v1",
source: { tenantId: "tenant-test", connectionId: "connection-test", providerId: "provider-test" },
contract: { dataProductId: "test.zones.current.v1", ontologyRevision: "ontology.map.zone.v1", version: "1.0.0" },
batch: {
runId,
sequence: 0,
idempotencyKey: `${runId}.chunk-0`,
receivedAt: generationAt,
mode: "replace",
generationAt,
},
facts,
};
}
function zoneFact(sourceId, observedAt) {
return {
sourceId,
semanticType: "map.zone",
observedAt,
attributes: {
display_name: sourceId,
geometry_kind: "polygon",
source_kind: "live_api",
source_revision: "gelios-rest-v1",
},
geometry: {
type: "Polygon",
coordinates: [[[37.60, 55.74], [37.62, 55.74], [37.62, 55.76], [37.60, 55.74]]],
},
};
}
@@ -11,6 +11,7 @@ assert.deepEqual(bundled.map((definition) => definition.id), [
"fleet.positions.current.v2",
"fleet.positions.current.v3",
"fleet.positions.current.v4",
"map.zones.current.v1",
]);
assert.deepEqual(bundled[0].semanticTypes, ["map.moving_object"]);
assert.equal(bundled[0].ontologyRevision, "ontology.map.moving_object.v1");
@@ -45,6 +46,10 @@ assert.deepEqual(bundled[3].fieldContracts.movement_state, {
required: true,
enum: ["moving", "stopped"],
});
assert.equal(bundled[4].version, "1.0.0");
assert.equal(bundled[4].ontologyRevision, "ontology.map.zone.v1");
assert.deepEqual(bundled[4].semanticTypes, ["map.zone"]);
assert.deepEqual(bundled[4].fieldContracts.geometry, { type: "geometry", required: true });
assert.equal(Object.keys(bundled[3].fieldContracts).length, bundled[3].fields.length);
const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-definitions-"));