SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): add replaceable geozone data layer

Browse Source
This commit is contained in:
Codex
2026-07-20 21:48:27 +03:00
parent 8a7465cf0e
commit 3446f3edc2
31 changed files with 1342 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/external-data-plane/src/data-product-delivery.mjs
+91 -32
View File
@@ -3,6 +3,7 @@ import {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
isBoundedGeoJsonGeometry,
} from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
@@ -86,6 +87,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
const existing = await client.query(
`select id as "batchId", fact_count as "publishedFactCount",
current_updated_count as "currentUpdatedCount",
current_removed_count as "currentRemovedCount",
history_inserted_count as "historyInsertedCount",
patch_operation_count as "patchOperationCount",
delivery_cursor::text as cursor, received_at as "acceptedAt",
@@ -102,6 +104,8 @@ export async function persistDataProductPublish(pool, batch, definition, {
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
}
if (batch.batch.mode === "replace") await lockReplacementGeneration(client, batch);
const records = batch.facts.map((fact) => ({
source_id: fact.sourceId,
semantic_type: fact.semanticType,
@@ -120,10 +124,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint(
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
@@ -143,9 +144,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
returning source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry,
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry,
fingerprint`,
[
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
@@ -153,21 +152,29 @@ export async function persistDataProductPublish(pool, batch, definition, {
],
);
const removed = batch.batch.mode === "replace"
? await removeMissingCurrent(client, batch, records)
: [];
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
const operations = [
...updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) })),
...removed.map((fact) => ({ op: "remove", sourceId: fact.sourceId, semanticType: fact.semanticType })),
];
const chunks = definition.deliveryMode === "snapshot+patch"
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
: [];
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
if (batch.batch.mode === "replace") await persistReplacementGeneration(client, batch);
await client.query(
`update external_data_plane_batches set
current_updated_count = $2,
history_inserted_count = $3,
patch_operation_count = $4,
delivery_cursor = $5
current_removed_count = $3,
history_inserted_count = $4,
patch_operation_count = $5,
delivery_cursor = $6
where id = $1`,
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
[batchId, updated.rowCount, removed.length, historyInsertedCount, operations.length, cursor],
);
await client.query("commit");
return normalizeReceipt({
@@ -175,6 +182,7 @@ export async function persistDataProductPublish(pool, batch, definition, {
idempotent: false,
publishedFactCount: batch.facts.length,
currentUpdatedCount: updated.rowCount,
currentRemovedCount: removed.length,
historyInsertedCount,
patchOperationCount: operations.length,
cursor: String(cursor),
@@ -198,9 +206,7 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
`select source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
from external_data_plane_current
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
order by source_id asc, semantic_type asc
@@ -294,9 +300,7 @@ export async function readDataProductHistory(pool, binding, definition, options)
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
else ST_AsGeoJSON(geometry::geometry)::jsonb end as geometry
from ranked
where sample_rank = 1
and (
@@ -444,6 +448,69 @@ export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
return result.rowCount;
}
async function lockReplacementGeneration(client, batch) {
const scope = [batch.source.tenantId, batch.source.connectionId, batch.source.providerId, batch.contract.dataProductId];
await client.query(
`insert into external_data_plane_delivery_state (
tenant_id, connection_id, provider_id, data_product_id, current_cursor
) values ($1, $2, $3, $4, 0)
on conflict (tenant_id, connection_id, provider_id, data_product_id) do nothing`,
scope,
);
const state = await client.query(
`select current_generation_at as "currentGenerationAt"
from external_data_plane_delivery_state
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
for update`,
scope,
);
const generationAt = new Date(batch.batch.generationAt);
const currentGenerationAt = state.rows[0]?.currentGenerationAt
? new Date(state.rows[0].currentGenerationAt)
: null;
if (currentGenerationAt && generationAt <= currentGenerationAt) {
throw deliveryError("data_product_generation_not_newer", 409);
}
}
async function removeMissingCurrent(client, batch, records) {
const result = await client.query(
`delete from external_data_plane_current as target
where target.tenant_id = $1 and target.connection_id = $2 and target.provider_id = $3
and target.data_product_id = $4
and not exists (
select 1
from jsonb_to_recordset($5::jsonb) as incoming(source_id text, semantic_type text)
where incoming.source_id = target.source_id and incoming.semantic_type = target.semantic_type
)
returning source_id as "sourceId", semantic_type as "semanticType"`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
JSON.stringify(records),
],
);
return result.rows;
}
async function persistReplacementGeneration(client, batch) {
await client.query(
`update external_data_plane_delivery_state
set current_generation_at = $5, current_generation_id = $6, updated_at = now()
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
[
batch.source.tenantId,
batch.source.connectionId,
batch.source.providerId,
batch.contract.dataProductId,
batch.batch.generationAt,
batch.batch.runId,
],
);
}
async function persistHistory(client, batch, policy, facts) {
if (!facts.length || policy?.mode === "none") return 0;
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
@@ -459,10 +526,7 @@ async function persistHistory(client, batch, policy, facts) {
select $1, $2, $3, $4, item.source_id, item.semantic_type,
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
case when item.geometry is null then null
else ST_SetSRID(ST_MakePoint(
(item.geometry->'coordinates'->>0)::double precision,
(item.geometry->'coordinates'->>1)::double precision
), 4326)::geography end,
else ST_SetSRID(ST_GeomFromGeoJSON(item.geometry), 4326)::geography end,
item.fingerprint, now()
from jsonb_to_recordset($5::jsonb) as item(
source_id text, semantic_type text, bucket_start timestamptz,
@@ -570,6 +634,7 @@ function normalizeReceipt(value) {
idempotent: value.idempotent === true,
publishedFactCount: Number(value.publishedFactCount || 0),
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
currentRemovedCount: Number(value.currentRemovedCount || 0),
historyInsertedCount: Number(value.historyInsertedCount || 0),
patchOperationCount: Number(value.patchOperationCount || 0),
cursor: String(value.cursor || "0"),
@@ -652,6 +717,8 @@ function publishFingerprint(batch) {
runId: batch.batch.runId,
sequence: batch.batch.sequence,
idempotencyKey: batch.batch.idempotencyKey,
mode: batch.batch.mode || "upsert",
...(batch.batch.generationAt ? { generationAt: batch.batch.generationAt } : {}),
},
facts: batch.facts,
});
@@ -712,16 +779,8 @@ function assertFieldContractValue(value, contract) {
function fieldContractValueMatchesType(value, type) {
if (type === "string_array") return Array.isArray(value) && value.every((item) => typeof item === "string");
if (type === "point") {
return value?.type === "Point"
&& Array.isArray(value.coordinates)
&& value.coordinates.length === 2
&& value.coordinates.every(Number.isFinite)
&& value.coordinates[0] >= -180
&& value.coordinates[0] <= 180
&& value.coordinates[1] >= -90
&& value.coordinates[1] <= 90;
}
if (type === "point") return isBoundedGeoJsonGeometry(value, new Set(["Point"]));
if (type === "geometry") return isBoundedGeoJsonGeometry(value);
return typeof value === type && (type !== "number" || Number.isFinite(value));
}