feat(platform): add replaceable geozone data layer

2026-07-20 21:48:27 +03:00
parent 8a7465cf0e
commit 3446f3edc2
31 changed files with 1342 additions and 128 deletions
@@ -3,6 +3,7 @@ import {
  DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
  DATA_PRODUCT_PATCH_SCHEMA_VERSION,
  DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
+  isBoundedGeoJsonGeometry,
 } from "@nodedc/external-provider-contract/data-plane";

 const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
@@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
      const existing = await client.query(
        `select id as "batchId", fact_count as "publishedFactCount",
           current_updated_count as "currentUpdatedCount",
+           current_removed_count as "currentRemovedCount",
           history_inserted_count as "historyInsertedCount",
           patch_operation_count as "patchOperationCount",
           delivery_cursor::text as cursor, received_at as "acceptedAt",
@@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, {
      return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
    }

+    if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch);
+
    const records = batch.facts.map((fact) => ({
      source_id: fact.sourceId,
      semantic_type: fact.semanticType,
@@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
      select $1, $2, $3, $4, item.source_id, item.semantic_type,
             item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
             case when item.geometry is null then null
-                  else ST_SetSRID(ST_MakePoint(
-                    (item.geometry->'coordinates'->>0)::double precision,
-                    (item.geometry->'coordinates'->>1)::double precision
-                  ), 4326)::geography end,
+                  else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
             item.fingerprint, now()
      from jsonb_to_recordset($5::jsonb) as item(
        source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
@@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
      returning source_id as "sourceId", semantic_type as "semanticType",
        observed_at as "observedAt", received_at as "receivedAt", attributes,
        case when geometry is null then null
-             else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
-               ST_X(geometry::geometry), ST_Y(geometry::geometry)
-             )) end as geometry,
+             else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry,
        fingerprint`,
      [
        batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
@@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, {
      ],
    );

+    const removed = batch.batch.mode === "replace"
+      ? await removeMissingCurrent(client, batch, records)
+      : [];
    const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
-    const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
+    const operations = [
+      ...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })),
+      ...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })),
+    ];
    const chunks = definition.deliveryMode === "snapshot+patch"
      ? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
      : [];
    const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
+    if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch);

    await client.query(
      `update external_data_plane_batches set
         current_updated_count = $2,
-         history_inserted_count = $3,
-         patch_operation_count = $4,
-         delivery_cursor = $5
+         current_removed_count = $3,
+         history_inserted_count = $4,
+         patch_operation_count = $5,
+         delivery_cursor = $6
       where id = $1`,
-      [batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
+      [batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor],
    );
    await client.query("commit");
    return normalizeReceipt({
@@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
      idempotent: false,
      publishedFactCount: batch.facts.length,
      currentUpdatedCount: updated.rowCount,
+      currentRemovedCount: removed.length,
      historyInsertedCount,
      patchOperationCount: operations.length,
      cursor: String(cursor),
@@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
      `select source_id as "sourceId", semantic_type as "semanticType",
         observed_at as "observedAt", received_at as "receivedAt", attributes,
         case when geometry is null then null
-              else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
-                ST_X(geometry::geometry), ST_Y(geometry::geometry)
-              )) end as geometry
+              else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
       from external_data_plane_current
       where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
       order by source_id asc, semantic_type asc
@@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options)
     select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
       observed_at as "observedAt", received_at as "receivedAt", attributes,
       case when geometry is null then null
-            else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
-              ST_X(geometry::geometry), ST_Y(geometry::geometry)
-            )) end as geometry
+            else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
     from ranked
     where sample_rank = 1
       and (
@@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
  return result.rowCount;
 }

+async function lockReplacementGeneration(client, batch) {
+  const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId];
+  await client.query(
+    `insert into external_data_plane_delivery_state (
+       tenant_id, connection_id, provider_id, data_product_id, current_cursor
+     ) values ($1, $2, $3, $4, 0)
+     on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`,
+    scope,
+  );
+  const state = await client.query(
+    `select current_generation_at as "currentGenerationAt"
+     from external_data_plane_delivery_state
+     where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
+     for update`,
+    scope,
+  );
+  const generationAt = new Date(batch.batch.generationAt);
+  const currentGenerationAt = state.rows[0]?.currentGenerationAt
+    ? new Date(state.rows[0].currentGenerationAt)
+    : null;
+  if (currentGenerationAt && generationAt <= currentGenerationAt) {
+    throw deliveryError("data_product_generation_not_newer", 409);
+  }
+}
+
+async function removeMissingCurrent(client, batch, records) {
+  const result = await client.query(
+    `delete from external_data_plane_current as target
+     where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3
+       and target.data_product_id = $4
+       and not exists (
+         select 1
+         from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text)
+         where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type
+       )
+     returning source_id as "sourceId", semantic_type as "semanticType"`,
+    [
+      batch.source.tenantId,
+      batch.source.connectionId,
+      batch.source.providerId,
+      batch.contract.dataProductId,
+      JSON.stringify(records),
+    ],
+  );
+  return result.rows;
+}
+
+async function persistReplacementGeneration(client, batch) {
+  await client.query(
+    `update external_data_plane_delivery_state
+     set current_generation_at = $5, current_generation_id = $6, updated_at = now()
+     where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
+    [
+      batch.source.tenantId,
+      batch.source.connectionId,
+      batch.source.providerId,
+      batch.contract.dataProductId,
+      batch.batch.generationAt,
+      batch.batch.runId,
+    ],
+  );
+}
+
 async function persistHistory(client, batch, policy, facts) {
  if (!facts.length || policy?.mode === "none") return 0;
  const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
@@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) {
    select $1, $2, $3, $4, item.source_id, item.semantic_type,
      item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
      case when item.geometry is null then null
-           else ST_SetSRID(ST_MakePoint(
-             (item.geometry->'coordinates'->>0)::double precision,
-             (item.geometry->'coordinates'->>1)::double precision
-           ), 4326)::geography end,
+           else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
      item.fingerprint, now()
    from jsonb_to_recordset($5::jsonb) as item(
      source_id text, semantic_type text, bucket_start timestamptz,
@@ -570,6 +634,7 @@ function normalizeReceipt(value) {
    idempotent: value.idempotent === true,
    publishedFactCount: Number(value.publishedFactCount || 0),
    currentUpdatedCount: Number(value.currentUpdatedCount || 0),
+    currentRemovedCount: Number(value.currentRemovedCount || 0),
    historyInsertedCount: Number(value.historyInsertedCount || 0),
    patchOperationCount: Number(value.patchOperationCount || 0),
    cursor: String(value.cursor || "0"),
@@ -652,6 +717,8 @@ function publishFingerprint(batch) {
      runId: batch.batch.runId,
      sequence: batch.batch.sequence,
      idempotencyKey: batch.batch.idempotencyKey,
+      mode: batch.batch.mode || "upsert",
+      ...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}),
    },
    facts: batch.facts,
  });
@@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) {

 function fieldContractValueMatchesType(value, type) {
  if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
-  if (type === "point") {
-    return value?.type === "Point"
-      && Array.isArray(value.coordinates)
-      && value.coordinates.length === 2
-      && value.coordinates.every(Number.isFinite)
-      && value.coordinates[0] >= -180
-      && value.coordinates[0] <= 180
-      && value.coordinates[1] >= -90
-      && value.coordinates[1] <= 90;
-  }
+  if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
+  if (type === "geometry") return isBoundedGeoJsonGeometry(value);
  return typeof value === type && (type !== "number" || Number.isFinite(value));
 }

@@ -1,8 +1,10 @@
+import { isBoundedGeoJsonGeometry } from "@nodedc/external-provider-contract/data-plane";
+
 const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
 const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
 const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
 const HISTORY_MODES = new Set(["none", "all", "sampled"]);
-const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point"]);
+const FIELD_CONTRACT_TYPES = new Set(["string", "number", "boolean", "string_array", "point", "geometry"]);
 const DEFINITION_KEYS = new Set([
  "id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "fieldContracts", "history",
 ]);
@@ -63,7 +65,7 @@ function normalizeFieldContract(value) {
  if (value.enum !== undefined) {
    if (!Array.isArray(value.enum) || value.enum.length === 0
      || new Set(value.enum.map(stableLiteral)).size !== value.enum.length
-      || new Set(["point", "string_array"]).has(type)
+      || new Set(["point", "geometry", "string_array"]).has(type)
      || value.enum.some((item) => !fieldContractValueMatchesType(item, type))) {
      throw policyError("data_product_field_contract_enum_invalid");
    }
@@ -171,6 +173,9 @@ function hasOnlyKeys(value, allowed) {
 }

 function fieldContractValueMatchesType(value, type) {
+  if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
+  if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
+  if (type === "geometry") return isBoundedGeoJsonGeometry(value);
  return typeof value === type && (type !== "number" || Number.isFinite(value));
 }

@@ -61,6 +61,7 @@ export async function migrate(pool) {
  `);
  await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
  await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
+  await pool.query("alter table external_data_plane_batches add column if not exists current_removed_count integer not null default 0");
  await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
  await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
  await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
@@ -245,7 +246,7 @@ export async function migrate(pool) {
      observed_at timestamptz not null,
      received_at timestamptz not null,
      attributes jsonb not null default '{}'::jsonb,
-      geometry geography(Point, 4326),
+      geometry geography(Geometry, 4326),
      fingerprint text not null,
      primary key (id, observed_at),
      unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
@@ -266,12 +267,30 @@ export async function migrate(pool) {
      observed_at timestamptz not null,
      received_at timestamptz not null,
      attributes jsonb not null default '{}'::jsonb,
-      geometry geography(Point, 4326),
+      geometry geography(Geometry, 4326),
      fingerprint text not null,
      updated_at timestamptz not null default now(),
      primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
    )
  `);
+  await pool.query(`
+    do $$
+    begin
+      if exists (
+        select 1
+        from pg_attribute
+        where attrelid = 'external_data_plane_current'::regclass
+          and attname = 'geometry'
+          and not attisdropped
+          and format_type(atttypid, atttypmod) = 'geography(Point,4326)'
+      ) then
+        alter table external_data_plane_current
+          alter column geometry type geography(Geometry, 4326)
+          using geometry::geometry::geography;
+      end if;
+    end
+    $$
+  `);
  await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
  await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");

@@ -282,10 +301,14 @@ export async function migrate(pool) {
      provider_id text not null,
      data_product_id text not null,
      current_cursor bigint not null default 0,
+      current_generation_at timestamptz,
+      current_generation_id text,
      updated_at timestamptz not null default now(),
      primary key (tenant_id, connection_id, provider_id, data_product_id)
    )
  `);
+  await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_at timestamptz");
+  await pool.query("alter table external_data_plane_delivery_state add column if not exists current_generation_id text");

  await pool.query(`
    create table if not exists external_data_plane_patch_outbox (
@@ -316,7 +339,7 @@ export async function migrate(pool) {
      observed_at timestamptz not null,
      received_at timestamptz not null,
      attributes jsonb not null default '{}'::jsonb,
-      geometry geography(Point, 4326),
+      geometry geography(Geometry, 4326),
      fingerprint text not null,
      updated_at timestamptz not null default now(),
      primary key (
@@ -208,6 +208,10 @@ export function materializeDataProductPublish(value, binding, definition, dataPr
      sequence: value.batch?.sequence,
      idempotencyKey: value.batch?.idempotencyKey,
      receivedAt: now.toISOString(),
+      ...(value.batch?.mode === "replace" ? {
+        mode: "replace",
+        generationAt: value.batch?.generationAt,
+      } : {}),
    },
    facts: value.facts,
  };