feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,75 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
|
||||
export function readConfig(env = process.env) {
|
||||
const provisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED, false);
|
||||
const config = {
|
||||
port: integer(env.PORT, 18106, 1, 65535),
|
||||
databaseUrl: required(env.EXTERNAL_DATA_PLANE_DATABASE_URL, "EXTERNAL_DATA_PLANE_DATABASE_URL"),
|
||||
databasePoolSize: integer(env.EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE, 10, 1, 50),
|
||||
internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN),
|
||||
provisionerApiEnabled,
|
||||
provisionerAccessToken: provisionerApiEnabled ? secretFile(env.EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE) : "",
|
||||
rawRetentionDays: integer(env.EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS, 14, 1, 3650),
|
||||
maxBatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES, 5 * 1024 * 1024, 1024, 50 * 1024 * 1024),
|
||||
maxFactsPerPublish: integer(env.EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH, 5000, 1, 100_000),
|
||||
maxAttributesBytesPerFact: integer(env.EXTERNAL_DATA_PLANE_MAX_ATTRIBUTES_BYTES_PER_FACT, 64 * 1024, 256, 1024 * 1024),
|
||||
maxPatchOperations: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_OPERATIONS, 500, 1, 5000),
|
||||
maxPatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_PATCH_BYTES, 256 * 1024, 16 * 1024, 5 * 1024 * 1024),
|
||||
patchRetentionMs: integer(env.EXTERNAL_DATA_PLANE_PATCH_RETENTION_MS, 60 * 60 * 1000, 60 * 1000, 7 * 24 * 60 * 60 * 1000),
|
||||
receiptRetentionMs: integer(env.EXTERNAL_DATA_PLANE_RECEIPT_RETENTION_MS, 7 * 24 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, 90 * 24 * 60 * 60 * 1000),
|
||||
retentionDeleteLimit: integer(env.EXTERNAL_DATA_PLANE_RETENTION_DELETE_LIMIT, 10_000, 100, 100_000),
|
||||
streamHeartbeatMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS, 20_000, 5_000, 60_000),
|
||||
streamPollMs: integer(env.EXTERNAL_DATA_PLANE_STREAM_POLL_MS, 1_000, 250, 10_000),
|
||||
maxReaderStreams: integer(env.EXTERNAL_DATA_PLANE_MAX_READER_STREAMS, 10, 1, 1000),
|
||||
writerBindingMaxTtlDays: integer(env.EXTERNAL_DATA_PLANE_WRITER_BINDING_MAX_TTL_DAYS, 90, 1, 365),
|
||||
maxFutureSkewSeconds: integer(env.EXTERNAL_DATA_PLANE_MAX_FUTURE_SKEW_SECONDS, 300, 0, 86400),
|
||||
retentionSweepMs: integer(env.EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS, 60 * 60 * 1000, 60 * 1000, 24 * 60 * 60 * 1000),
|
||||
legacyIntakeEnabled: boolean(env.EXTERNAL_DATA_PLANE_LEGACY_INTAKE_ENABLED, false),
|
||||
};
|
||||
if (config.internalAccessToken && config.provisionerAccessToken && config.internalAccessToken === config.provisionerAccessToken) {
|
||||
throw new Error("provisioner_token_must_differ_from_internal_token");
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
function required(value, name) {
|
||||
const normalized = optional(value);
|
||||
if (!normalized) throw new Error(`${name}_required`);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function optional(value) {
|
||||
const normalized = String(value ?? "").trim();
|
||||
return normalized || "";
|
||||
}
|
||||
|
||||
function integer(value, fallback, min, max) {
|
||||
const candidate = optional(value);
|
||||
if (!candidate) return fallback;
|
||||
const number = Number.parseInt(candidate, 10);
|
||||
if (!Number.isInteger(number) || number < min || number > max) throw new Error(`invalid_integer:${candidate}`);
|
||||
return number;
|
||||
}
|
||||
|
||||
function boolean(value, fallback) {
|
||||
const normalized = optional(value).toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
if (normalized === "true") return true;
|
||||
if (normalized === "false") return false;
|
||||
throw new Error(`invalid_boolean:${normalized}`);
|
||||
}
|
||||
|
||||
function secretFile(value) {
|
||||
const path = optional(value);
|
||||
if (!path) return "";
|
||||
let secret = "";
|
||||
try {
|
||||
secret = readFileSync(path, "utf8").trim();
|
||||
} catch {
|
||||
throw new Error("external_data_plane_provisioner_token_file_unreadable");
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{48,256}$/.test(secret)) {
|
||||
throw new Error("external_data_plane_provisioner_token_file_invalid");
|
||||
}
|
||||
return secret;
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
} from "@nodedc/external-provider-contract";
|
||||
|
||||
export async function loadDataProductDefinition(db, dataProductId, { activeOnly = true } = {}) {
|
||||
const result = await db.query(
|
||||
`select id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"
|
||||
from external_data_plane_products
|
||||
where id = $1 ${activeOnly ? "and active = true" : ""}`,
|
||||
[dataProductId],
|
||||
);
|
||||
return result.rows[0] || null;
|
||||
}
|
||||
|
||||
export async function persistDataProductDefinition(db, definition) {
|
||||
const result = await db.query(
|
||||
`insert into external_data_plane_products (
|
||||
id, version, ontology_revision, delivery_mode, semantic_types, fields, history_policy
|
||||
) values ($1, $2, $3, $4, $5::jsonb, $6::jsonb, $7::jsonb)
|
||||
on conflict (id) do update set
|
||||
active = true,
|
||||
updated_at = now()
|
||||
where external_data_plane_products.version = excluded.version
|
||||
and external_data_plane_products.ontology_revision = excluded.ontology_revision
|
||||
and external_data_plane_products.delivery_mode = excluded.delivery_mode
|
||||
and external_data_plane_products.semantic_types = excluded.semantic_types
|
||||
and external_data_plane_products.fields = excluded.fields
|
||||
and external_data_plane_products.history_policy = excluded.history_policy
|
||||
returning id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"`,
|
||||
[
|
||||
definition.id,
|
||||
definition.version,
|
||||
definition.ontologyRevision,
|
||||
definition.deliveryMode,
|
||||
JSON.stringify(definition.semanticTypes),
|
||||
JSON.stringify(definition.fields),
|
||||
JSON.stringify(definition.history),
|
||||
],
|
||||
);
|
||||
if (!result.rowCount) throw deliveryError("data_product_version_is_immutable", 409);
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
export async function persistDataProductPublish(pool, batch, definition, {
|
||||
maxPatchOperations = 500,
|
||||
maxPatchBytes = 256 * 1024,
|
||||
} = {}) {
|
||||
assertPublishMatchesDefinition(batch, definition);
|
||||
const requestFingerprint = publishFingerprint(batch);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const batchId = randomUUID();
|
||||
const insertedBatch = await client.query(
|
||||
`insert into external_data_plane_batches (
|
||||
id, tenant_id, connection_id, provider_id, data_product_id, contract_version,
|
||||
ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count,
|
||||
request_fingerprint
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, idempotency_key)
|
||||
do nothing
|
||||
returning id`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision,
|
||||
batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey,
|
||||
batch.batch.receivedAt, batch.facts.length, requestFingerprint,
|
||||
],
|
||||
);
|
||||
if (!insertedBatch.rowCount) {
|
||||
const existing = await client.query(
|
||||
`select id as "batchId", fact_count as "publishedFactCount",
|
||||
current_updated_count as "currentUpdatedCount",
|
||||
history_inserted_count as "historyInsertedCount",
|
||||
patch_operation_count as "patchOperationCount",
|
||||
delivery_cursor::text as cursor, received_at as "acceptedAt",
|
||||
request_fingerprint as "requestFingerprint"
|
||||
from external_data_plane_batches
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and idempotency_key = $5`,
|
||||
scopeValues(batch, batch.batch.idempotencyKey),
|
||||
);
|
||||
if (!existing.rowCount || existing.rows[0].requestFingerprint !== requestFingerprint) {
|
||||
throw deliveryError("idempotency_key_reused", 409);
|
||||
}
|
||||
await client.query("commit");
|
||||
return { ...normalizeReceipt(existing.rows[0]), idempotent: true };
|
||||
}
|
||||
|
||||
const records = batch.facts.map((fact) => ({
|
||||
source_id: fact.sourceId,
|
||||
semantic_type: fact.semanticType,
|
||||
observed_at: fact.observedAt,
|
||||
received_at: batch.batch.receivedAt,
|
||||
attributes: fact.attributes || {},
|
||||
geometry: fact.geometry || null,
|
||||
fingerprint: fingerprint(fact),
|
||||
}));
|
||||
|
||||
const updated = await client.query(
|
||||
`insert into external_data_plane_current (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||
item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint, now()
|
||||
from jsonb_to_recordset($5::jsonb) as item(
|
||||
source_id text, semantic_type text, observed_at timestamptz, received_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at > external_data_plane_current.observed_at
|
||||
or (excluded.observed_at = external_data_plane_current.observed_at
|
||||
and excluded.fingerprint <> external_data_plane_current.fingerprint)
|
||||
returning source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry,
|
||||
fingerprint`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
|
||||
const historyInsertedCount = await persistHistory(client, batch, definition.historyPolicy, records);
|
||||
const operations = updated.rows.map((fact) => ({ op: "upsert", fact: publicFact(fact) }));
|
||||
const chunks = definition.deliveryMode === "snapshot+patch"
|
||||
? chunkOperations(operations, maxPatchOperations, maxPatchBytes)
|
||||
: [];
|
||||
const cursor = chunks.length ? await persistPatchChunks(client, batch, definition, batchId, chunks) : await currentCursor(client, batch);
|
||||
|
||||
await client.query(
|
||||
`update external_data_plane_batches set
|
||||
current_updated_count = $2,
|
||||
history_inserted_count = $3,
|
||||
patch_operation_count = $4,
|
||||
delivery_cursor = $5
|
||||
where id = $1`,
|
||||
[batchId, updated.rowCount, historyInsertedCount, operations.length, cursor],
|
||||
);
|
||||
await client.query("commit");
|
||||
return normalizeReceipt({
|
||||
batchId,
|
||||
idempotent: false,
|
||||
publishedFactCount: batch.facts.length,
|
||||
currentUpdatedCount: updated.rowCount,
|
||||
historyInsertedCount,
|
||||
patchOperationCount: operations.length,
|
||||
cursor: String(cursor),
|
||||
acceptedAt: batch.batch.receivedAt,
|
||||
});
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readDataProductSnapshot(pool, binding, definition, { limit = 5000 } = {}) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin isolation level repeatable read read only");
|
||||
const cursor = await currentCursor(client, binding, definition.id);
|
||||
const rows = await client.query(
|
||||
`select source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry
|
||||
from external_data_plane_current
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
|
||||
order by source_id asc, semantic_type asc
|
||||
limit $5`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, definition.id, limit + 1],
|
||||
);
|
||||
if (rows.rowCount > limit) throw deliveryError("data_product_snapshot_limit_exceeded", 413);
|
||||
await client.query("commit");
|
||||
return {
|
||||
schemaVersion: DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
|
||||
dataProduct: { id: definition.id, version: definition.version },
|
||||
generatedAt: new Date().toISOString(),
|
||||
cursor: String(cursor),
|
||||
facts: rows.rows.map(publicFact),
|
||||
};
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readPatchEvents(pool, binding, definition, after, { limit = 100 } = {}) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin isolation level repeatable read read only");
|
||||
const current = await currentCursor(client, binding, definition.id);
|
||||
if (after > current) throw deliveryError("resync_required", 409);
|
||||
const floor = await patchFloor(client, binding, definition.id);
|
||||
if (after < floor - 1n) throw deliveryError("resync_required", 409);
|
||||
const rows = await client.query(
|
||||
`select cursor::text, previous_cursor::text as "previousCursor",
|
||||
operations, emitted_at as "emittedAt"
|
||||
from external_data_plane_patch_outbox
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and cursor > $5
|
||||
order by cursor asc limit $6`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, definition.id, after.toString(), limit],
|
||||
);
|
||||
await client.query("commit");
|
||||
return rows.rows.map((row) => ({
|
||||
schemaVersion: DATA_PRODUCT_PATCH_SCHEMA_VERSION,
|
||||
dataProduct: { id: definition.id, version: definition.version },
|
||||
previousCursor: row.previousCursor,
|
||||
cursor: row.cursor,
|
||||
emittedAt: new Date(row.emittedAt).toISOString(),
|
||||
operations: row.operations,
|
||||
}));
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function prunePatchOutbox(db, { retentionMs, limit = 10_000 }) {
|
||||
const cutoff = new Date(Date.now() - retentionMs);
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select tenant_id, connection_id, provider_id, data_product_id, cursor
|
||||
from external_data_plane_patch_outbox
|
||||
where emitted_at < $1
|
||||
order by emitted_at asc
|
||||
limit $2
|
||||
)
|
||||
delete from external_data_plane_patch_outbox as target
|
||||
using expired
|
||||
where target.tenant_id = expired.tenant_id
|
||||
and target.connection_id = expired.connection_id
|
||||
and target.provider_id = expired.provider_id
|
||||
and target.data_product_id = expired.data_product_id
|
||||
and target.cursor = expired.cursor`,
|
||||
[cutoff, limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
export async function pruneDataProductHistory(db, { limit = 10_000 } = {}) {
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select history.tenant_id, history.connection_id, history.provider_id,
|
||||
history.data_product_id, history.source_id, history.semantic_type, history.bucket_start
|
||||
from external_data_plane_history as history
|
||||
join external_data_plane_products as product on product.id = history.data_product_id
|
||||
where history.bucket_start < now() - (
|
||||
greatest(1, coalesce((product.history_policy->>'retentionDays')::integer, 1)) * interval '1 day'
|
||||
)
|
||||
order by history.bucket_start asc
|
||||
limit $1
|
||||
)
|
||||
delete from external_data_plane_history as target
|
||||
using expired
|
||||
where target.tenant_id = expired.tenant_id
|
||||
and target.connection_id = expired.connection_id
|
||||
and target.provider_id = expired.provider_id
|
||||
and target.data_product_id = expired.data_product_id
|
||||
and target.source_id = expired.source_id
|
||||
and target.semantic_type = expired.semantic_type
|
||||
and target.bucket_start = expired.bucket_start`,
|
||||
[limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
export async function pruneBatchReceipts(db, { retentionMs, limit = 10_000 }) {
|
||||
const cutoff = new Date(Date.now() - retentionMs);
|
||||
const result = await db.query(
|
||||
`with expired as (
|
||||
select batch.id
|
||||
from external_data_plane_batches as batch
|
||||
where batch.created_at < $1
|
||||
and not exists (select 1 from external_data_plane_patch_outbox as patch where patch.batch_id = batch.id)
|
||||
and not exists (select 1 from external_data_plane_raw_envelopes as raw where raw.batch_id = batch.id)
|
||||
order by batch.created_at asc
|
||||
limit $2
|
||||
)
|
||||
delete from external_data_plane_batches as target
|
||||
using expired
|
||||
where target.id = expired.id`,
|
||||
[cutoff, limit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function persistHistory(client, batch, policy, facts) {
|
||||
if (!facts.length || policy?.mode === "none") return 0;
|
||||
const intervalMs = policy?.mode === "sampled" ? Number(policy.intervalMs) : 1;
|
||||
const records = facts.map((fact) => ({
|
||||
...fact,
|
||||
bucket_start: new Date(Math.floor(new Date(fact.observed_at).getTime() / intervalMs) * intervalMs).toISOString(),
|
||||
}));
|
||||
const result = await client.query(
|
||||
`insert into external_data_plane_history (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
bucket_start, observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select $1, $2, $3, $4, item.source_id, item.semantic_type,
|
||||
item.bucket_start, item.observed_at, item.received_at, coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint, now()
|
||||
from jsonb_to_recordset($5::jsonb) as item(
|
||||
source_id text, semantic_type text, bucket_start timestamptz,
|
||||
observed_at timestamptz, received_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, bucket_start)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= external_data_plane_history.observed_at`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId,
|
||||
JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function persistPatchChunks(client, batch, definition, batchId, chunks) {
|
||||
const endCursorResult = await client.query(
|
||||
`insert into external_data_plane_delivery_state (
|
||||
tenant_id, connection_id, provider_id, data_product_id, current_cursor
|
||||
) values ($1, $2, $3, $4, $5)
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id)
|
||||
do update set
|
||||
current_cursor = external_data_plane_delivery_state.current_cursor + excluded.current_cursor,
|
||||
updated_at = now()
|
||||
returning current_cursor`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
definition.id, chunks.length,
|
||||
],
|
||||
);
|
||||
const end = BigInt(endCursorResult.rows[0].current_cursor);
|
||||
const start = end - BigInt(chunks.length) + 1n;
|
||||
for (let index = 0; index < chunks.length; index += 1) {
|
||||
const cursor = start + BigInt(index);
|
||||
await client.query(
|
||||
`insert into external_data_plane_patch_outbox (
|
||||
tenant_id, connection_id, provider_id, data_product_id,
|
||||
cursor, previous_cursor, batch_id, operations
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
definition.id, cursor.toString(), (cursor - 1n).toString(), batchId,
|
||||
JSON.stringify(chunks[index]),
|
||||
],
|
||||
);
|
||||
}
|
||||
return end;
|
||||
}
|
||||
|
||||
async function currentCursor(db, scope, explicitProductId) {
|
||||
const dataProductId = explicitProductId || scope.contract.dataProductId;
|
||||
const result = await db.query(
|
||||
`select current_cursor from external_data_plane_delivery_state
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
|
||||
[scope.tenantId || scope.source.tenantId, scope.connectionId || scope.source.connectionId,
|
||||
scope.providerId || scope.source.providerId, dataProductId],
|
||||
);
|
||||
return result.rowCount ? BigInt(result.rows[0].current_cursor) : 0n;
|
||||
}
|
||||
|
||||
async function patchFloor(db, binding, dataProductId) {
|
||||
const result = await db.query(
|
||||
`select min(cursor) as floor from external_data_plane_patch_outbox
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4`,
|
||||
[binding.tenantId, binding.connectionId, binding.providerId, dataProductId],
|
||||
);
|
||||
if (result.rows[0]?.floor !== null && result.rows[0]?.floor !== undefined) return BigInt(result.rows[0].floor);
|
||||
const current = await currentCursor(db, binding, dataProductId);
|
||||
return current + 1n;
|
||||
}
|
||||
|
||||
function scopeValues(batch, tail) {
|
||||
return [
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, tail,
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeReceipt(value) {
|
||||
return {
|
||||
batchId: value.batchId,
|
||||
idempotent: value.idempotent === true,
|
||||
publishedFactCount: Number(value.publishedFactCount || 0),
|
||||
currentUpdatedCount: Number(value.currentUpdatedCount || 0),
|
||||
historyInsertedCount: Number(value.historyInsertedCount || 0),
|
||||
patchOperationCount: Number(value.patchOperationCount || 0),
|
||||
cursor: String(value.cursor || "0"),
|
||||
acceptedAt: new Date(value.acceptedAt).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function publicFact(fact) {
|
||||
const value = {
|
||||
sourceId: fact.sourceId,
|
||||
semanticType: fact.semanticType,
|
||||
observedAt: new Date(fact.observedAt).toISOString(),
|
||||
receivedAt: new Date(fact.receivedAt).toISOString(),
|
||||
attributes: fact.attributes || {},
|
||||
};
|
||||
if (fact.geometry) value.geometry = fact.geometry;
|
||||
return value;
|
||||
}
|
||||
|
||||
function chunkOperations(values, maxOperations, maxBytes) {
|
||||
const result = [];
|
||||
let current = [];
|
||||
let currentBytes = 2;
|
||||
for (const value of values) {
|
||||
const serializedBytes = Buffer.byteLength(JSON.stringify(value));
|
||||
if (serializedBytes + 2 > maxBytes) throw deliveryError("data_product_patch_operation_size_exceeded", 413);
|
||||
const separatorBytes = current.length ? 1 : 0;
|
||||
if (current.length && (current.length >= maxOperations || currentBytes + separatorBytes + serializedBytes > maxBytes)) {
|
||||
result.push(current);
|
||||
current = [];
|
||||
currentBytes = 2;
|
||||
}
|
||||
current.push(value);
|
||||
currentBytes += (current.length > 1 ? 1 : 0) + serializedBytes;
|
||||
}
|
||||
if (current.length) result.push(current);
|
||||
return result;
|
||||
}
|
||||
|
||||
function fingerprint(value) {
|
||||
return createHash("sha256").update(stableJson(value)).digest("hex");
|
||||
}
|
||||
|
||||
function publishFingerprint(batch) {
|
||||
return fingerprint({
|
||||
schemaVersion: batch.schemaVersion,
|
||||
source: batch.source,
|
||||
contract: batch.contract,
|
||||
batch: {
|
||||
runId: batch.batch.runId,
|
||||
sequence: batch.batch.sequence,
|
||||
idempotencyKey: batch.batch.idempotencyKey,
|
||||
},
|
||||
facts: batch.facts,
|
||||
});
|
||||
}
|
||||
|
||||
export function assertPublishMatchesDefinition(batch, definition) {
|
||||
const semanticTypes = new Set(Array.isArray(definition?.semanticTypes) ? definition.semanticTypes : []);
|
||||
const fields = new Set(Array.isArray(definition?.fields) ? definition.fields : []);
|
||||
const entityKeys = new Set();
|
||||
for (const fact of batch?.facts || []) {
|
||||
if (!semanticTypes.has(fact.semanticType)) throw deliveryError("data_product_semantic_type_forbidden", 422);
|
||||
const entityKey = `${fact.sourceId}\u0000${fact.semanticType}`;
|
||||
if (entityKeys.has(entityKey)) throw deliveryError("data_product_duplicate_entity_key", 422);
|
||||
entityKeys.add(entityKey);
|
||||
for (const attribute of Object.keys(fact.attributes || {})) {
|
||||
if (!fields.has(attribute) && !fields.has(`attributes.${attribute}`)) {
|
||||
throw deliveryError("data_product_field_forbidden", 422);
|
||||
}
|
||||
}
|
||||
if (fact.geometry !== undefined && !geometryDeclared(fields)) {
|
||||
throw deliveryError("data_product_geometry_forbidden", 422);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function geometryDeclared(fields) {
|
||||
return fields.has("geometry")
|
||||
|| fields.has("coordinates")
|
||||
|| (fields.has("longitude") && fields.has("latitude"));
|
||||
}
|
||||
|
||||
function stableJson(value) {
|
||||
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
||||
if (value && typeof value === "object") {
|
||||
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function deliveryError(code, status) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const SEMVER = /^\d+\.\d+\.\d+(?:[-+][a-z0-9.-]+)?$/i;
|
||||
const DELIVERY_MODES = new Set(["snapshot", "snapshot+patch", "query"]);
|
||||
const HISTORY_MODES = new Set(["none", "all", "sampled"]);
|
||||
const DEFINITION_KEYS = new Set([
|
||||
"id", "version", "ontologyRevision", "deliveryMode", "semanticTypes", "fields", "history",
|
||||
]);
|
||||
const HISTORY_KEYS = new Set(["mode", "intervalMs", "retentionDays", "strategy"]);
|
||||
|
||||
export function normalizeDataProductDefinition(value) {
|
||||
if (!isPlainObject(value) || !hasOnlyKeys(value, DEFINITION_KEYS)) {
|
||||
throw policyError("data_product_definition_invalid");
|
||||
}
|
||||
const id = identifier(value.id);
|
||||
const version = string(value.version);
|
||||
const ontologyRevision = identifier(value.ontologyRevision);
|
||||
const deliveryMode = string(value.deliveryMode);
|
||||
const semanticTypes = identifierSet(
|
||||
value.semanticTypes,
|
||||
"data_product_definition_semantic_types_invalid",
|
||||
"data_product_definition_semantic_types_duplicate",
|
||||
);
|
||||
const fields = identifierSet(
|
||||
value.fields,
|
||||
"data_product_definition_fields_invalid",
|
||||
"data_product_definition_fields_duplicate",
|
||||
);
|
||||
if (!id || !SEMVER.test(version) || !ontologyRevision || !DELIVERY_MODES.has(deliveryMode)) {
|
||||
throw policyError("data_product_definition_identity_invalid");
|
||||
}
|
||||
if (!semanticTypes.length || !fields.length) throw policyError("data_product_definition_shape_invalid");
|
||||
|
||||
const history = normalizeHistoryPolicy(value.history);
|
||||
return Object.freeze({ id, version, ontologyRevision, deliveryMode, semanticTypes, fields, history });
|
||||
}
|
||||
|
||||
export function normalizeHistoryPolicy(value = { mode: "none" }) {
|
||||
if (!isPlainObject(value) || !hasOnlyKeys(value, HISTORY_KEYS)) throw policyError("history_policy_invalid");
|
||||
const mode = string(value.mode);
|
||||
if (!HISTORY_MODES.has(mode)) throw policyError("history_policy_mode_invalid");
|
||||
const retentionDays = integer(value.retentionDays, mode === "none" ? 1 : 90, 1, 3650);
|
||||
if (mode === "none") {
|
||||
if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_none_has_sampling_fields");
|
||||
return Object.freeze({ mode, retentionDays });
|
||||
}
|
||||
if (mode === "all") {
|
||||
if (value.intervalMs !== undefined || value.strategy !== undefined) throw policyError("history_policy_all_has_sampling_fields");
|
||||
return Object.freeze({ mode, retentionDays });
|
||||
}
|
||||
|
||||
const intervalMs = integer(value.intervalMs, 60_000, 1_000, 24 * 60 * 60 * 1000);
|
||||
const strategy = string(value.strategy || "latest-per-entity-per-bucket");
|
||||
if (strategy !== "latest-per-entity-per-bucket") throw policyError("history_policy_strategy_invalid");
|
||||
return Object.freeze({ mode, intervalMs, strategy, retentionDays });
|
||||
}
|
||||
|
||||
export function safeDataProductDefinition(row) {
|
||||
return {
|
||||
id: row.id,
|
||||
version: row.version,
|
||||
ontologyRevision: row.ontologyRevision,
|
||||
deliveryMode: row.deliveryMode,
|
||||
semanticTypes: array(row.semanticTypes),
|
||||
fields: array(row.fields),
|
||||
history: isPlainObject(row.historyPolicy) ? row.historyPolicy : {},
|
||||
active: row.active === true,
|
||||
createdAt: iso(row.createdAt),
|
||||
updatedAt: iso(row.updatedAt),
|
||||
};
|
||||
}
|
||||
|
||||
function policyError(code) {
|
||||
return Object.assign(new Error(code), { status: 400, code });
|
||||
}
|
||||
|
||||
function identifier(value) {
|
||||
const normalized = string(value);
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function identifierSet(value, invalidCode, duplicateCode) {
|
||||
if (!Array.isArray(value)) throw policyError(invalidCode);
|
||||
|
||||
const normalized = value.map((entry) => {
|
||||
const result = identifier(entry);
|
||||
if (!result) throw policyError(invalidCode);
|
||||
return result;
|
||||
});
|
||||
if (new Set(normalized).size !== normalized.length) throw policyError(duplicateCode);
|
||||
|
||||
return Object.freeze(normalized.sort());
|
||||
}
|
||||
|
||||
function integer(value, fallback, min, max) {
|
||||
const number = value === undefined ? fallback : Number(value);
|
||||
if (!Number.isInteger(number) || number < min || number > max) throw policyError("history_policy_number_invalid");
|
||||
return number;
|
||||
}
|
||||
|
||||
function string(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function array(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function iso(value) {
|
||||
return value ? new Date(value).toISOString() : undefined;
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowed) {
|
||||
return Object.keys(value).every((key) => allowed.has(key));
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { lstat, readFile, readdir } from "node:fs/promises";
|
||||
import { basename, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { normalizeDataProductDefinition } from "./data-product-policy.mjs";
|
||||
import { persistDataProductDefinition } from "./data-product-delivery.mjs";
|
||||
|
||||
const bundledDefinitionsDir = fileURLToPath(new URL("../definitions/", import.meta.url));
|
||||
|
||||
export async function loadDataProductDefinitions(directory = bundledDefinitionsDir) {
|
||||
const root = resolve(directory);
|
||||
const entries = (await readdir(root, { withFileTypes: true }))
|
||||
.filter((entry) => entry.name.endsWith(".json"))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
const definitions = [];
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || entry.isSymbolicLink()) throw new Error("data_product_definition_file_invalid");
|
||||
const path = join(root, entry.name);
|
||||
const metadata = await lstat(path);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size < 2 || metadata.size > 256 * 1024) {
|
||||
throw new Error("data_product_definition_file_invalid");
|
||||
}
|
||||
let value;
|
||||
try {
|
||||
value = JSON.parse(await readFile(path, "utf8"));
|
||||
} catch {
|
||||
throw new Error("data_product_definition_json_invalid");
|
||||
}
|
||||
const definition = normalizeDataProductDefinition(value);
|
||||
if (`${definition.id}.json` !== basename(path)) throw new Error("data_product_definition_filename_mismatch");
|
||||
definitions.push(definition);
|
||||
}
|
||||
if (new Set(definitions.map((definition) => definition.id)).size !== definitions.length) {
|
||||
throw new Error("data_product_definition_duplicate");
|
||||
}
|
||||
return Object.freeze(definitions);
|
||||
}
|
||||
|
||||
export async function reconcileDataProductDefinitions(db, directory = bundledDefinitionsDir) {
|
||||
const definitions = await loadDataProductDefinitions(directory);
|
||||
for (const definition of definitions) await persistDataProductDefinition(db, definition);
|
||||
return definitions;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
export function assertBatchTimeBounds(batch, { now = new Date(), maxFutureSkewSeconds = 300 } = {}) {
|
||||
const acceptedAt = validDate(now, "intake_policy_clock_invalid");
|
||||
const maxFutureAt = new Date(acceptedAt.getTime() + Number(maxFutureSkewSeconds) * 1000);
|
||||
if (!Number.isInteger(maxFutureSkewSeconds) || maxFutureSkewSeconds < 0) {
|
||||
throw intakePolicyError("intake_policy_future_skew_invalid");
|
||||
}
|
||||
|
||||
const receivedAt = validDate(batch?.batch?.receivedAt, "batch_received_at_invalid");
|
||||
if (receivedAt > maxFutureAt) throw intakePolicyError("batch_received_at_too_far_in_future");
|
||||
|
||||
for (const fact of batch?.facts || []) {
|
||||
const observedAt = validDate(fact?.observedAt, "fact_observed_at_invalid");
|
||||
if (observedAt > maxFutureAt) throw intakePolicyError("fact_observed_at_too_far_in_future");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retention is based on the server's acceptance time, never on provider or L2
|
||||
* timestamps supplied in a batch.
|
||||
*/
|
||||
export function rawRetentionExpiry({ now = new Date(), rawRetentionDays }) {
|
||||
const acceptedAt = validDate(now, "intake_policy_clock_invalid");
|
||||
if (!Number.isInteger(rawRetentionDays) || rawRetentionDays < 1) {
|
||||
throw intakePolicyError("raw_retention_days_invalid");
|
||||
}
|
||||
return new Date(acceptedAt.getTime() + rawRetentionDays * 24 * 60 * 60 * 1000);
|
||||
}
|
||||
|
||||
export function intakePolicyError(code) {
|
||||
return Object.assign(new Error(code), { status: 422, code });
|
||||
}
|
||||
|
||||
function validDate(value, code) {
|
||||
const parsed = value instanceof Date ? new Date(value.getTime()) : new Date(String(value ?? ""));
|
||||
if (Number.isNaN(parsed.getTime())) throw intakePolicyError(code);
|
||||
return parsed;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const TOKEN_PREFIX = "ndc_edprb_";
|
||||
const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
|
||||
const REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]);
|
||||
const SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
|
||||
|
||||
export function createReaderToken() {
|
||||
return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function hashReaderToken(token) {
|
||||
return createHash("sha256").update(String(token), "utf8").digest("hex");
|
||||
}
|
||||
|
||||
export function normalizeReaderBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source)) throw readerError("reader_binding_request_invalid");
|
||||
if (containsSecretLikeKey(value)) throw readerError("reader_binding_request_secret_material_forbidden");
|
||||
if (!hasOnlyKeys(value, REQUEST_KEYS) || !hasOnlyKeys(value.source, SOURCE_KEYS)) {
|
||||
throw readerError("reader_binding_request_fields_invalid");
|
||||
}
|
||||
const tenantId = identifier(value.source.tenantId);
|
||||
const connectionId = identifier(value.source.connectionId);
|
||||
const providerId = identifier(value.source.providerId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
|
||||
if (!tenantId || !connectionId || !providerId || !allowedDataProductIds.length) {
|
||||
throw readerError("reader_binding_scope_invalid");
|
||||
}
|
||||
const expiresAt = new Date(String(value.expiresAt || ""));
|
||||
const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000);
|
||||
if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) {
|
||||
throw readerError("reader_binding_expiry_invalid");
|
||||
}
|
||||
return Object.freeze({ tenantId, connectionId, providerId, allowedDataProductIds, expiresAt: expiresAt.toISOString() });
|
||||
}
|
||||
|
||||
export function assertReaderProduct(binding, dataProductId, now = new Date()) {
|
||||
if (!isPlainObject(binding) || binding.active !== true || new Date(binding.expiresAt) <= now) {
|
||||
throw readerError("reader_binding_inactive", 401);
|
||||
}
|
||||
const normalized = identifier(dataProductId);
|
||||
if (!normalized || !uniqueIdentifiers(binding.allowedDataProductIds).includes(normalized)) {
|
||||
throw readerError("reader_binding_data_product_forbidden", 403);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function safeReaderBinding(binding) {
|
||||
return {
|
||||
id: binding.id,
|
||||
tenantId: binding.tenantId,
|
||||
connectionId: binding.connectionId,
|
||||
providerId: binding.providerId,
|
||||
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
|
||||
active: binding.active === true,
|
||||
expiresAt: new Date(binding.expiresAt).toISOString(),
|
||||
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
|
||||
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
|
||||
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
function readerError(code, status = 400) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
|
||||
function identifier(value) {
|
||||
const normalized = typeof value === "string" ? value.trim() : "";
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function uniqueIdentifiers(value) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return [...new Set(value.map(identifier).filter(Boolean))];
|
||||
}
|
||||
|
||||
function containsSecretLikeKey(value) {
|
||||
if (Array.isArray(value)) return value.some(containsSecretLikeKey);
|
||||
if (!isPlainObject(value)) return false;
|
||||
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child));
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowed) {
|
||||
return Object.keys(value).every((key) => allowed.has(key));
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
export async function migrate(pool) {
|
||||
await pool.query("create extension if not exists timescaledb");
|
||||
await pool.query("create extension if not exists postgis");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_products (
|
||||
id text primary key,
|
||||
version text not null,
|
||||
ontology_revision text not null,
|
||||
delivery_mode text not null,
|
||||
semantic_types jsonb not null,
|
||||
fields jsonb not null,
|
||||
history_policy jsonb not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
updated_at timestamptz not null default now(),
|
||||
check (jsonb_typeof(semantic_types) = 'array' and jsonb_array_length(semantic_types) > 0),
|
||||
check (jsonb_typeof(fields) = 'array' and jsonb_array_length(fields) > 0)
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_batches (
|
||||
id uuid primary key,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
contract_version text not null,
|
||||
ontology_revision text not null,
|
||||
run_id text not null,
|
||||
sequence integer not null,
|
||||
idempotency_key text not null,
|
||||
received_at timestamptz not null,
|
||||
fact_count integer not null default 0,
|
||||
inserted_fact_count integer not null default 0,
|
||||
created_at timestamptz not null default now(),
|
||||
unique (tenant_id, connection_id, provider_id, data_product_id, idempotency_key)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_batches_scope_idx on external_data_plane_batches (tenant_id, connection_id, data_product_id, received_at desc)");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists current_updated_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists history_inserted_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists patch_operation_count integer not null default 0");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists delivery_cursor bigint");
|
||||
await pool.query("alter table external_data_plane_batches add column if not exists request_fingerprint text");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_raw_envelopes (
|
||||
id uuid primary key,
|
||||
batch_id uuid not null unique references external_data_plane_batches(id) on delete cascade,
|
||||
payload_hash text not null,
|
||||
content_type text not null,
|
||||
payload jsonb,
|
||||
payload_ref text,
|
||||
payload_bytes integer,
|
||||
received_at timestamptz not null,
|
||||
expires_at timestamptz not null,
|
||||
check (payload is not null or payload_ref is not null)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_raw_expiry_idx on external_data_plane_raw_envelopes (expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_writer_bindings (
|
||||
id uuid primary key,
|
||||
token_hash text not null unique,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
allowed_data_product_ids jsonb not null,
|
||||
expires_at timestamptz not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
rotated_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
check (
|
||||
case when jsonb_typeof(allowed_data_product_ids) = 'array'
|
||||
then jsonb_array_length(allowed_data_product_ids) > 0
|
||||
else false
|
||||
end
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_writer_bindings_active_idx on external_data_plane_writer_bindings (active, expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_reader_bindings (
|
||||
id uuid primary key,
|
||||
token_hash text not null unique,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
allowed_data_product_ids jsonb not null,
|
||||
expires_at timestamptz not null,
|
||||
active boolean not null default true,
|
||||
created_at timestamptz not null default now(),
|
||||
rotated_at timestamptz,
|
||||
revoked_at timestamptz,
|
||||
check (
|
||||
case when jsonb_typeof(allowed_data_product_ids) = 'array'
|
||||
then jsonb_array_length(allowed_data_product_ids) > 0
|
||||
else false
|
||||
end
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_reader_bindings_active_idx on external_data_plane_reader_bindings (active, expires_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_facts (
|
||||
id uuid not null,
|
||||
batch_id uuid not null references external_data_plane_batches(id) on delete cascade,
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
primary key (id, observed_at),
|
||||
unique (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type, observed_at, fingerprint)
|
||||
)
|
||||
`);
|
||||
await pool.query("select create_hypertable('external_data_plane_facts', 'observed_at', if_not_exists => true, migrate_data => true)");
|
||||
await pool.query("create index if not exists external_data_plane_facts_source_time_idx on external_data_plane_facts (tenant_id, connection_id, data_product_id, source_id, observed_at desc)");
|
||||
await pool.query("create index if not exists external_data_plane_facts_geometry_idx on external_data_plane_facts using gist (geometry)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_current (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_current_scope_idx on external_data_plane_current (tenant_id, connection_id, data_product_id, observed_at desc)");
|
||||
await pool.query("create index if not exists external_data_plane_current_geometry_idx on external_data_plane_current using gist (geometry)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_delivery_state (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
current_cursor bigint not null default 0,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id)
|
||||
)
|
||||
`);
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_patch_outbox (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
cursor bigint not null,
|
||||
previous_cursor bigint not null,
|
||||
batch_id uuid not null references external_data_plane_batches(id) on delete cascade,
|
||||
operations jsonb not null,
|
||||
emitted_at timestamptz not null default now(),
|
||||
primary key (tenant_id, connection_id, provider_id, data_product_id, cursor),
|
||||
check (jsonb_typeof(operations) = 'array' and jsonb_array_length(operations) > 0)
|
||||
)
|
||||
`);
|
||||
await pool.query("create index if not exists external_data_plane_patch_outbox_retention_idx on external_data_plane_patch_outbox (emitted_at)");
|
||||
|
||||
await pool.query(`
|
||||
create table if not exists external_data_plane_history (
|
||||
tenant_id text not null,
|
||||
connection_id text not null,
|
||||
provider_id text not null,
|
||||
data_product_id text not null,
|
||||
source_id text not null,
|
||||
semantic_type text not null,
|
||||
bucket_start timestamptz not null,
|
||||
observed_at timestamptz not null,
|
||||
received_at timestamptz not null,
|
||||
attributes jsonb not null default '{}'::jsonb,
|
||||
geometry geography(Point, 4326),
|
||||
fingerprint text not null,
|
||||
updated_at timestamptz not null default now(),
|
||||
primary key (
|
||||
tenant_id, connection_id, provider_id, data_product_id,
|
||||
source_id, semantic_type, bucket_start
|
||||
)
|
||||
)
|
||||
`);
|
||||
await pool.query("select create_hypertable('external_data_plane_history', 'bucket_start', if_not_exists => true, migrate_data => true)");
|
||||
await pool.query("create index if not exists external_data_plane_history_source_time_idx on external_data_plane_history (tenant_id, connection_id, data_product_id, source_id, bucket_start desc)");
|
||||
await pool.query("create index if not exists external_data_plane_history_geometry_idx on external_data_plane_history using gist (geometry)");
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
import express from "express";
|
||||
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract";
|
||||
import { readConfig } from "./config.mjs";
|
||||
import {
|
||||
loadDataProductDefinition,
|
||||
assertPublishMatchesDefinition,
|
||||
persistDataProductDefinition,
|
||||
persistDataProductPublish,
|
||||
pruneBatchReceipts,
|
||||
pruneDataProductHistory,
|
||||
prunePatchOutbox,
|
||||
readDataProductSnapshot,
|
||||
readPatchEvents,
|
||||
} from "./data-product-delivery.mjs";
|
||||
import { normalizeDataProductDefinition, safeDataProductDefinition } from "./data-product-policy.mjs";
|
||||
import { reconcileDataProductDefinitions } from "./definitions.mjs";
|
||||
import { assertBatchTimeBounds, rawRetentionExpiry } from "./intake-policy.mjs";
|
||||
import {
|
||||
assertReaderProduct,
|
||||
createReaderToken,
|
||||
hashReaderToken,
|
||||
normalizeReaderBindingRequest,
|
||||
safeReaderBinding,
|
||||
} from "./reader-binding.mjs";
|
||||
import { migrate } from "./schema.mjs";
|
||||
import {
|
||||
createWriterToken,
|
||||
hashWriterToken,
|
||||
materializeDataProductPublish,
|
||||
materializeWriterBoundBatch,
|
||||
normalizeWriterBindingRequest,
|
||||
safeWriterBinding,
|
||||
} from "./writer-binding.mjs";
|
||||
|
||||
const config = readConfig();
|
||||
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
|
||||
const app = express();
|
||||
const httpServer = createServer(app);
|
||||
let retentionSweepTimer = null;
|
||||
let lastRetentionSweepAt = null;
|
||||
const activeReaderStreams = new Map();
|
||||
const activeStreamResponses = new Set();
|
||||
let shuttingDown = false;
|
||||
let shutdownPromise = null;
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.use(express.json({ limit: config.maxBatchBytes }));
|
||||
|
||||
app.get("/healthz", asyncRoute(async (_req, res) => {
|
||||
await pool.query("select 1");
|
||||
res.json({
|
||||
ok: true,
|
||||
service: "nodedc-external-data-plane",
|
||||
database: "ready",
|
||||
internalApiConfigured: Boolean(config.internalAccessToken),
|
||||
providerLogic: "absent",
|
||||
commandTransport: "absent",
|
||||
writerBindings: "supported",
|
||||
readerBindings: "supported",
|
||||
dataProductDelivery: "snapshot+durable-patch",
|
||||
legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled",
|
||||
writerBindingProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled",
|
||||
rawRetentionSweep: {
|
||||
mode: "server-scheduled",
|
||||
lastSweepAt: lastRetentionSweepAt,
|
||||
},
|
||||
});
|
||||
}));
|
||||
|
||||
app.put("/internal/data-plane/v1/data-products/:dataProductId", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const definition = normalizeDataProductDefinition({ ...req.body, id: req.params.dataProductId });
|
||||
const saved = await persistDataProductDefinition(pool, definition);
|
||||
res.json({ ok: true, dataProduct: safeDataProductDefinition(saved) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/data-products/:dataProductId/publish", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_publish_scope_headers_forbidden");
|
||||
const validation = validateDataProductPublish(req.body, {
|
||||
maxFacts: config.maxFactsPerPublish,
|
||||
maxAttributesBytes: config.maxAttributesBytesPerFact,
|
||||
});
|
||||
if (!validation.ok) throw httpError(422, validation.errors[0] || "invalid_data_product_publish");
|
||||
const dataProductId = requireIdentifier(req.params.dataProductId, "data_product_id_invalid");
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const batch = materializeDataProductPublish(req.body, req.writerBinding, definition, dataProductId);
|
||||
const canonicalValidation = validateIntakeBatch(batch);
|
||||
if (!canonicalValidation.ok) throw httpError(422, "materialized_publish_invalid");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
const receipt = await persistDataProductPublish(pool, batch, definition, {
|
||||
maxPatchOperations: config.maxPatchOperations,
|
||||
maxPatchBytes: config.maxPatchBytes,
|
||||
});
|
||||
res.status(receipt.idempotent ? 200 : 201).json({ ok: true, ...receipt });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/writer/data-products", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.writerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/reader/data-products", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.readerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const validation = validateIntakeBatch(req.body);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(req.body, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(req.body);
|
||||
const scope = requireScope(req);
|
||||
if (scope.tenantId !== req.body.source.tenantId || scope.connectionId !== req.body.source.connectionId) {
|
||||
throw httpError(403, "scope_mismatch");
|
||||
}
|
||||
|
||||
const result = await persistBatch(req.body);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const batch = materializeWriterBoundBatch(req.body, req.writerBinding, {
|
||||
hasScopeHeaders: hasScopeHeaders(req),
|
||||
});
|
||||
const validation = validateIntakeBatch(batch);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(batch);
|
||||
|
||||
const result = await persistBatch(batch);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeWriterBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createWriterToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_writer_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashWriterToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
// The token is intentionally returned exactly once to a trusted provisioner.
|
||||
// It must be placed directly into an opaque Engine credential reference and
|
||||
// must never be logged, stored in L2 graph data or shown to a consumer.
|
||||
sendOneTimeCapability(res, 201, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const token = createWriterToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeReaderBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createReaderToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_reader_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashReaderToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
sendOneTimeCapability(res, 201, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const token = createReaderToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, readerBinding: safeReaderBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const limit = boundedLimit(req.query.limit, 5000, 1, 5000);
|
||||
const snapshot = await readDataProductSnapshot(pool, req.readerBinding, definition, { limit });
|
||||
res.json(snapshot);
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (shuttingDown) throw httpError(503, "service_shutting_down");
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
if (definition.deliveryMode !== "snapshot+patch") throw httpError(409, "data_product_stream_not_supported");
|
||||
const streamCount = activeReaderStreams.get(req.readerBinding.id) || 0;
|
||||
if (streamCount >= config.maxReaderStreams) throw httpError(429, "reader_stream_limit_exceeded");
|
||||
activeReaderStreams.set(req.readerBinding.id, streamCount + 1);
|
||||
activeStreamResponses.add(res);
|
||||
|
||||
let poll = null;
|
||||
let heartbeat = null;
|
||||
let polling = false;
|
||||
let heartbeatWriting = false;
|
||||
let cleanedUp = false;
|
||||
const cleanup = () => {
|
||||
if (cleanedUp) return;
|
||||
cleanedUp = true;
|
||||
if (poll) clearInterval(poll);
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
activeStreamResponses.delete(res);
|
||||
const remaining = Math.max(0, (activeReaderStreams.get(req.readerBinding.id) || 1) - 1);
|
||||
if (remaining) activeReaderStreams.set(req.readerBinding.id, remaining);
|
||||
else activeReaderStreams.delete(req.readerBinding.id);
|
||||
};
|
||||
req.once("close", cleanup);
|
||||
res.once("close", cleanup);
|
||||
|
||||
try {
|
||||
let cursor = parseCursor(req.get("last-event-id") || req.query.after || "0");
|
||||
const initialEvents = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
if (res.destroyed || cleanedUp) return;
|
||||
res.status(200);
|
||||
res.set({
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Content-Type": "text/event-stream",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
});
|
||||
res.flushHeaders();
|
||||
await writeSseFrame(res, ": nodedc-data-product-stream\n\n");
|
||||
await writeSseFrame(res, `event: nodedc.data-product.ready.v1\ndata: ${JSON.stringify({
|
||||
schemaVersion: "nodedc.data-product.ready/v1",
|
||||
dataProductId,
|
||||
cursor: cursor.toString(),
|
||||
emittedAt: new Date().toISOString(),
|
||||
})}\n\n`);
|
||||
for (const event of initialEvents) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
|
||||
const pump = async () => {
|
||||
if (polling || res.writableEnded || res.destroyed || shuttingDown) return;
|
||||
polling = true;
|
||||
try {
|
||||
await assertReaderStreamAccess(req.readerBinding, dataProductId);
|
||||
const events = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
for (const event of events) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
} catch (error) {
|
||||
const code = safeErrorCode(error);
|
||||
if (!res.writableEnded && !res.destroyed) {
|
||||
await writeSseFrame(res, `event: error\ndata: ${JSON.stringify({ ok: false, error: code })}\n\n`).catch(() => {});
|
||||
res.end();
|
||||
}
|
||||
} finally {
|
||||
polling = false;
|
||||
}
|
||||
};
|
||||
poll = setInterval(() => { void pump(); }, config.streamPollMs);
|
||||
poll.unref();
|
||||
heartbeat = setInterval(() => {
|
||||
if (polling || heartbeatWriting || res.writableEnded || res.destroyed) return;
|
||||
heartbeatWriting = true;
|
||||
void writeSseFrame(res, `: heartbeat ${Date.now()}\n\n`)
|
||||
.catch(() => { if (!res.writableEnded) res.end(); })
|
||||
.finally(() => { heartbeatWriting = false; });
|
||||
}, config.streamHeartbeatMs);
|
||||
heartbeat.unref();
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
if (res.headersSent) {
|
||||
if (!res.writableEnded) res.end();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/current", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const dataProductId = String(req.params.dataProductId || "");
|
||||
if (!isIdentifier(dataProductId)) throw httpError(400, "data_product_id_invalid");
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await pool.query(
|
||||
`select
|
||||
provider_id as "providerId", data_product_id as "dataProductId",
|
||||
source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry
|
||||
from external_data_plane_current
|
||||
where tenant_id = $1 and connection_id = $2 and data_product_id = $3
|
||||
order by observed_at desc, source_id asc
|
||||
limit $4`,
|
||||
[scope.tenantId, scope.connectionId, dataProductId, limit],
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
dataProductId,
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
facts: rows.rows,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/status", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const result = await pool.query(
|
||||
`select count(*)::integer as "currentFactCount", max(received_at) as "lastReceivedAt"
|
||||
from external_data_plane_current where tenant_id = $1 and connection_id = $2`,
|
||||
[scope.tenantId, scope.connectionId],
|
||||
);
|
||||
res.json({ ok: true, ...scope, ...result.rows[0], providerLogic: "absent", commandTransport: "absent" });
|
||||
}));
|
||||
|
||||
app.use((error, _req, res, _next) => {
|
||||
const status = Number(error?.status || 500);
|
||||
const publicStatus = status >= 400 && status < 600 ? status : 500;
|
||||
console.error(JSON.stringify({ event: "external_data_plane_error", error: safeErrorCode(error), status: publicStatus }));
|
||||
res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) });
|
||||
});
|
||||
|
||||
await migrate(pool);
|
||||
const bundledDataProducts = await reconcileDataProductDefinitions(pool);
|
||||
retentionSweepTimer = setInterval(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
}, config.retentionSweepMs);
|
||||
retentionSweepTimer.unref();
|
||||
|
||||
httpServer.listen(config.port, "0.0.0.0", () => {
|
||||
console.log(`NODE.DC External Data Plane listening on http://0.0.0.0:${config.port}`);
|
||||
console.log(JSON.stringify({ event: "external_data_plane_definitions_ready", count: bundledDataProducts.length }));
|
||||
setImmediate(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
async function persistBatch(batch) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const existing = await client.query(
|
||||
`select id, fact_count as "factCount", inserted_fact_count as "insertedFactCount"
|
||||
from external_data_plane_batches
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and idempotency_key = $5
|
||||
for update`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.idempotencyKey,
|
||||
],
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("commit");
|
||||
return { batchId: existing.rows[0].id, idempotent: true, ...existing.rows[0] };
|
||||
}
|
||||
|
||||
const batchId = randomUUID();
|
||||
await client.query(
|
||||
`insert into external_data_plane_batches (
|
||||
id, tenant_id, connection_id, provider_id, data_product_id, contract_version,
|
||||
ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision,
|
||||
batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey,
|
||||
batch.batch.receivedAt, batch.facts.length,
|
||||
],
|
||||
);
|
||||
|
||||
if (batch.raw) await persistRawEnvelope(client, batchId, batch);
|
||||
|
||||
const records = batch.facts.map((fact) => ({
|
||||
id: randomUUID(),
|
||||
source_id: fact.sourceId,
|
||||
semantic_type: fact.semanticType,
|
||||
observed_at: fact.observedAt,
|
||||
attributes: fact.attributes || {},
|
||||
geometry: fact.geometry || null,
|
||||
fingerprint: hash(fact),
|
||||
}));
|
||||
const inserted = await client.query(
|
||||
`insert into external_data_plane_facts (
|
||||
id, batch_id, tenant_id, connection_id, provider_id, data_product_id,
|
||||
source_id, semantic_type, observed_at, received_at, attributes, geometry, fingerprint
|
||||
)
|
||||
select item.id::uuid, $1, $2, $3, $4, $5,
|
||||
item.source_id, item.semantic_type, item.observed_at, $6,
|
||||
coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint
|
||||
from jsonb_to_recordset($7::jsonb) as item(
|
||||
id text, source_id text, semantic_type text, observed_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict do nothing
|
||||
returning id`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.receivedAt, JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`insert into external_data_plane_current (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, now()
|
||||
from external_data_plane_facts where batch_id = $1
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= external_data_plane_current.observed_at`,
|
||||
[batchId],
|
||||
);
|
||||
await client.query(
|
||||
"update external_data_plane_batches set inserted_fact_count = $2 where id = $1",
|
||||
[batchId, inserted.rowCount],
|
||||
);
|
||||
await client.query("commit");
|
||||
return { batchId, idempotent: false, factCount: batch.facts.length, insertedFactCount: inserted.rowCount };
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function persistRawEnvelope(client, batchId, batch) {
|
||||
const serialized = batch.raw.payload === undefined ? null : JSON.stringify(batch.raw.payload);
|
||||
const expiresAt = rawRetentionExpiry({ rawRetentionDays: config.rawRetentionDays });
|
||||
await client.query(
|
||||
`insert into external_data_plane_raw_envelopes (
|
||||
id, batch_id, payload_hash, content_type, payload, payload_ref, payload_bytes, received_at, expires_at
|
||||
) values ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9)`,
|
||||
[
|
||||
randomUUID(), batchId, batch.raw.hash || hash(batch.raw.payload), batch.raw.contentType, serialized, batch.raw.ref || null,
|
||||
serialized ? Buffer.byteLength(serialized) : null, batch.batch.receivedAt, expiresAt,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
function requireInternalApi(req, _res, next) {
|
||||
if (!config.internalAccessToken) return next(httpError(503, "internal_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.internalAccessToken)) return next(httpError(401, "unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireLegacyIntake(_req, _res, next) {
|
||||
if (!config.legacyIntakeEnabled) return next(httpError(410, "legacy_intake_disabled"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireProvisionerApi(req, _res, next) {
|
||||
if (!config.provisionerApiEnabled) return next(httpError(503, "provisioner_api_disabled"));
|
||||
if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.provisionerAccessToken)) return next(httpError(401, "provisioner_unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireWriterBinding(req, _res, next) {
|
||||
return resolveWriterBinding(req)
|
||||
.then((binding) => {
|
||||
req.writerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
function requireReaderBinding(req, _res, next) {
|
||||
return resolveReaderBinding(req)
|
||||
.then((binding) => {
|
||||
req.readerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
async function resolveWriterBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "writer_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_writer_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function resolveReaderBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "reader_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, token_hash as "tokenHash", tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_reader_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function assertReaderStreamAccess(binding, dataProductId) {
|
||||
const result = await pool.query(
|
||||
`select binding.id
|
||||
from external_data_plane_reader_bindings as binding
|
||||
join external_data_plane_products as product
|
||||
on product.id = $3 and product.active = true
|
||||
where binding.id = $1 and binding.token_hash = $2
|
||||
and binding.active = true and binding.expires_at > now()
|
||||
and binding.allowed_data_product_ids ? $3`,
|
||||
[binding.id, binding.tokenHash, dataProductId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_stream_access_revoked");
|
||||
}
|
||||
|
||||
async function assertLegacyBatchProduct(batch) {
|
||||
const definition = await loadDataProductDefinition(pool, batch.contract.dataProductId);
|
||||
if (!definition) throw httpError(422, "legacy_data_product_not_registered");
|
||||
if (definition.version !== batch.contract.version || definition.ontologyRevision !== batch.contract.ontologyRevision) {
|
||||
throw httpError(422, "legacy_data_product_contract_mismatch");
|
||||
}
|
||||
assertPublishMatchesDefinition(batch, definition);
|
||||
}
|
||||
|
||||
async function listGrantedProducts(binding) {
|
||||
const allowed = Array.isArray(binding.allowedDataProductIds) ? binding.allowedDataProductIds : [];
|
||||
if (!allowed.length) return [];
|
||||
const result = await pool.query(
|
||||
`select id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"
|
||||
from external_data_plane_products
|
||||
where active = true and id = any($1::text[])
|
||||
order by id asc`,
|
||||
[allowed],
|
||||
);
|
||||
return result.rows.map(safeDataProductDefinition);
|
||||
}
|
||||
|
||||
async function assertRegisteredProductIds(dataProductIds) {
|
||||
const result = await pool.query(
|
||||
"select id from external_data_plane_products where active = true and id = any($1::text[])",
|
||||
[dataProductIds],
|
||||
);
|
||||
const registered = new Set(result.rows.map((row) => row.id));
|
||||
if (dataProductIds.some((id) => !registered.has(id))) throw httpError(422, "binding_data_product_not_registered");
|
||||
}
|
||||
|
||||
function requireScope(req) {
|
||||
const tenantId = String(req.get("x-nodedc-tenant-id") || "");
|
||||
const connectionId = String(req.get("x-nodedc-connection-id") || "");
|
||||
if (!isIdentifier(tenantId) || !isIdentifier(connectionId)) throw httpError(400, "scope_headers_required");
|
||||
return { tenantId, connectionId };
|
||||
}
|
||||
|
||||
function hasScopeHeaders(req) {
|
||||
return Object.hasOwn(req.headers, "x-nodedc-tenant-id") || Object.hasOwn(req.headers, "x-nodedc-connection-id");
|
||||
}
|
||||
|
||||
function bearerToken(req) {
|
||||
return String(req.get("authorization") || "").replace(/^Bearer\s+/i, "");
|
||||
}
|
||||
|
||||
function requireUuid(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) {
|
||||
throw httpError(400, code);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireIdentifier(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!isIdentifier(normalized)) throw httpError(400, code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseCursor(value) {
|
||||
const normalized = String(value ?? "");
|
||||
if (!/^(?:0|[1-9]\d*)$/.test(normalized)) throw httpError(400, "data_product_cursor_invalid");
|
||||
const cursor = BigInt(normalized);
|
||||
if (cursor > 9_223_372_036_854_775_807n) throw httpError(400, "data_product_cursor_invalid");
|
||||
return cursor;
|
||||
}
|
||||
|
||||
function writePatchEvent(res, event) {
|
||||
return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
||||
function writeSseFrame(res, frame) {
|
||||
if (res.writableEnded || res.destroyed) return Promise.reject(httpError(499, "reader_stream_closed"));
|
||||
if (res.write(frame)) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
res.off("drain", onDrain);
|
||||
res.off("close", onClose);
|
||||
res.off("error", onError);
|
||||
};
|
||||
const onDrain = () => { cleanup(); resolve(); };
|
||||
const onClose = () => { cleanup(); reject(httpError(499, "reader_stream_closed")); };
|
||||
const onError = (error) => { cleanup(); reject(error); };
|
||||
res.once("drain", onDrain);
|
||||
res.once("close", onClose);
|
||||
res.once("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function sendOneTimeCapability(res, status, payload) {
|
||||
// Create/rotate is the only boundary where a capability exists in
|
||||
// plaintext. A trusted provisioner must consume the response in memory and
|
||||
// place it directly into its opaque destination; intermediaries must never
|
||||
// cache or persist it.
|
||||
res.set({
|
||||
"Cache-Control": "no-store, max-age=0",
|
||||
Pragma: "no-cache",
|
||||
Expires: "0",
|
||||
});
|
||||
return res.status(status).json(payload);
|
||||
}
|
||||
|
||||
function safeEqual(left, right) {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
||||
}
|
||||
|
||||
function boundedLimit(value, fallback, min, max) {
|
||||
const candidate = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(candidate)) return fallback;
|
||||
return Math.max(min, Math.min(max, candidate));
|
||||
}
|
||||
|
||||
function hash(value) {
|
||||
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
}
|
||||
|
||||
function isIdentifier(value) {
|
||||
return /^[a-z][a-z0-9._:-]{2,127}$/i.test(value);
|
||||
}
|
||||
|
||||
function asyncRoute(handler) {
|
||||
return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next);
|
||||
}
|
||||
|
||||
function httpError(status, code) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return String(error?.code || error?.message || "internal_error").replace(/[^a-z0-9_.:-]/gi, "_").slice(0, 120);
|
||||
}
|
||||
|
||||
async function sweepExpiredRawEnvelopes() {
|
||||
const result = await pool.query(
|
||||
`with expired as (
|
||||
select id from external_data_plane_raw_envelopes
|
||||
where expires_at < now()
|
||||
order by expires_at asc
|
||||
limit $1
|
||||
)
|
||||
delete from external_data_plane_raw_envelopes as target
|
||||
using expired where target.id = expired.id`,
|
||||
[config.retentionDeleteLimit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function sweepRetention() {
|
||||
const rawEnvelopeCount = await sweepExpiredRawEnvelopes();
|
||||
const patchEventCount = await prunePatchOutbox(pool, {
|
||||
retentionMs: config.patchRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
const historyFactCount = await pruneDataProductHistory(pool, { limit: config.retentionDeleteLimit });
|
||||
const batchReceiptCount = await pruneBatchReceipts(pool, {
|
||||
retentionMs: config.receiptRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
lastRetentionSweepAt = new Date().toISOString();
|
||||
return { rawEnvelopeCount, patchEventCount, historyFactCount, batchReceiptCount };
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
if (shutdownPromise) return shutdownPromise;
|
||||
shutdownPromise = (async () => {
|
||||
shuttingDown = true;
|
||||
if (retentionSweepTimer) clearInterval(retentionSweepTimer);
|
||||
for (const response of activeStreamResponses) {
|
||||
if (!response.writableEnded) response.end();
|
||||
}
|
||||
const closed = new Promise((resolve) => httpServer.close(resolve));
|
||||
httpServer.closeIdleConnections?.();
|
||||
const forceClose = setTimeout(() => httpServer.closeAllConnections?.(), 250);
|
||||
await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 2_000))]);
|
||||
clearTimeout(forceClose);
|
||||
httpServer.closeAllConnections?.();
|
||||
await pool.end();
|
||||
})();
|
||||
return shutdownPromise;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
|
||||
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
|
||||
const TOKEN_PREFIX = "ndc_edpwb_";
|
||||
const SECRET_LIKE_KEY = /(token|secret|password|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
|
||||
const BINDING_REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]);
|
||||
const BINDING_SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
|
||||
|
||||
/**
|
||||
* Produces an opaque, high-entropy capability. The plaintext is returned only
|
||||
* to the trusted provisioning caller; persistence uses its SHA-256 digest.
|
||||
*/
|
||||
export function createWriterToken() {
|
||||
return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`;
|
||||
}
|
||||
|
||||
export function hashWriterToken(token) {
|
||||
return createHash("sha256").update(String(token), "utf8").digest("hex");
|
||||
}
|
||||
|
||||
export function normalizeWriterBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source)) {
|
||||
throw writerBindingError("writer_binding_request_invalid");
|
||||
}
|
||||
if (containsSecretLikeKey(value)) {
|
||||
throw writerBindingError("writer_binding_request_secret_material_forbidden");
|
||||
}
|
||||
if (value.endpoint !== undefined || value.url !== undefined || value.host !== undefined) {
|
||||
throw writerBindingError("writer_binding_request_transport_forbidden");
|
||||
}
|
||||
if (!hasOnlyKeys(value, BINDING_REQUEST_KEYS) || !hasOnlyKeys(value.source, BINDING_SOURCE_KEYS)) {
|
||||
throw writerBindingError("writer_binding_request_fields_invalid");
|
||||
}
|
||||
|
||||
const tenantId = normalizeIdentifier(value.source.tenantId);
|
||||
const connectionId = normalizeIdentifier(value.source.connectionId);
|
||||
const providerId = normalizeIdentifier(value.source.providerId);
|
||||
if (!tenantId || !connectionId || !providerId) {
|
||||
throw writerBindingError("writer_binding_scope_invalid");
|
||||
}
|
||||
|
||||
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
|
||||
if (!allowedDataProductIds.length) {
|
||||
throw writerBindingError("writer_binding_data_products_invalid");
|
||||
}
|
||||
|
||||
const expiresAt = new Date(String(value.expiresAt || ""));
|
||||
const maxExpiresAt = new Date(now.getTime() + maxTtlDays * 24 * 60 * 60 * 1000);
|
||||
if (Number.isNaN(expiresAt.getTime()) || expiresAt <= now || expiresAt > maxExpiresAt) {
|
||||
throw writerBindingError("writer_binding_expiry_invalid");
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
tenantId,
|
||||
connectionId,
|
||||
providerId,
|
||||
allowedDataProductIds,
|
||||
expiresAt: expiresAt.toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a caller-provided, deliberately unscoped intake envelope into the
|
||||
* canonical scoped form. Caller scope is rejected, never trusted or merged.
|
||||
*/
|
||||
export function materializeWriterBoundBatch(value, binding, { hasScopeHeaders = false, now = new Date() } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(value.source) || !isPlainObject(binding)) {
|
||||
throw writerBindingError("writer_bound_intake_invalid");
|
||||
}
|
||||
if (hasScopeHeaders || value.source.tenantId !== undefined || value.source.connectionId !== undefined) {
|
||||
throw writerBindingError("writer_bound_scope_forbidden");
|
||||
}
|
||||
if (binding.active !== true || !bindingIsCurrent(binding, now)) {
|
||||
throw writerBindingError("writer_binding_inactive");
|
||||
}
|
||||
|
||||
const providerId = normalizeIdentifier(value.source.providerId);
|
||||
const tenantId = normalizeIdentifier(binding.tenantId);
|
||||
const connectionId = normalizeIdentifier(binding.connectionId);
|
||||
if (!providerId || !tenantId || !connectionId || providerId !== binding.providerId) {
|
||||
throw writerBindingError("writer_binding_provider_forbidden");
|
||||
}
|
||||
|
||||
const dataProductId = normalizeIdentifier(value.contract?.dataProductId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds);
|
||||
if (!dataProductId || !allowedDataProductIds.includes(dataProductId)) {
|
||||
throw writerBindingError("writer_binding_data_product_forbidden");
|
||||
}
|
||||
|
||||
return {
|
||||
...value,
|
||||
source: { providerId, tenantId, connectionId },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Materializes the provider-neutral Data Product publish wire form. Unlike the
|
||||
* legacy writer-bound intake, the caller cannot send provider identity,
|
||||
* contract metadata, receivedAt or any scope at all.
|
||||
*/
|
||||
export function materializeDataProductPublish(value, binding, definition, dataProductId, { now = new Date() } = {}) {
|
||||
if (!isPlainObject(value) || !isPlainObject(binding) || !isPlainObject(definition)) {
|
||||
throw writerBindingError("data_product_publish_invalid");
|
||||
}
|
||||
if (binding.active !== true || !bindingIsCurrent(binding, now)) {
|
||||
throw writerBindingError("writer_binding_inactive");
|
||||
}
|
||||
const normalizedProductId = normalizeIdentifier(dataProductId);
|
||||
const allowedDataProductIds = uniqueIdentifiers(binding.allowedDataProductIds);
|
||||
if (!normalizedProductId || normalizedProductId !== definition.id || !allowedDataProductIds.includes(normalizedProductId)) {
|
||||
throw writerBindingError("writer_binding_data_product_forbidden");
|
||||
}
|
||||
const tenantId = normalizeIdentifier(binding.tenantId);
|
||||
const connectionId = normalizeIdentifier(binding.connectionId);
|
||||
const providerId = normalizeIdentifier(binding.providerId);
|
||||
if (!tenantId || !connectionId || !providerId) throw writerBindingError("writer_binding_scope_invalid");
|
||||
|
||||
return {
|
||||
schemaVersion: "nodedc.external-provider-contract/v1",
|
||||
source: { tenantId, connectionId, providerId },
|
||||
contract: {
|
||||
dataProductId: normalizedProductId,
|
||||
ontologyRevision: definition.ontologyRevision,
|
||||
version: definition.version,
|
||||
},
|
||||
batch: {
|
||||
runId: value.batch?.runId,
|
||||
sequence: value.batch?.sequence,
|
||||
idempotencyKey: value.batch?.idempotencyKey,
|
||||
receivedAt: now.toISOString(),
|
||||
},
|
||||
facts: value.facts,
|
||||
};
|
||||
}
|
||||
|
||||
export function safeWriterBinding(binding) {
|
||||
return {
|
||||
id: binding.id,
|
||||
tenantId: binding.tenantId,
|
||||
connectionId: binding.connectionId,
|
||||
providerId: binding.providerId,
|
||||
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
|
||||
active: binding.active === true,
|
||||
expiresAt: new Date(binding.expiresAt).toISOString(),
|
||||
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
|
||||
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
|
||||
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
export function writerBindingError(code) {
|
||||
return Object.assign(new Error(code), { status: 400, code });
|
||||
}
|
||||
|
||||
function bindingIsCurrent(binding, now) {
|
||||
const expiresAt = new Date(binding.expiresAt);
|
||||
return !Number.isNaN(expiresAt.getTime()) && expiresAt > now;
|
||||
}
|
||||
|
||||
function normalizeIdentifier(value) {
|
||||
const normalized = typeof value === "string" ? value.trim() : "";
|
||||
return IDENTIFIER.test(normalized) ? normalized : "";
|
||||
}
|
||||
|
||||
function uniqueIdentifiers(value) {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return [...new Set(value.map(normalizeIdentifier).filter(Boolean))];
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function containsSecretLikeKey(value) {
|
||||
if (Array.isArray(value)) return value.some(containsSecretLikeKey);
|
||||
if (!isPlainObject(value)) return false;
|
||||
return Object.entries(value).some(([key, child]) => SECRET_LIKE_KEY.test(key) || containsSecretLikeKey(child));
|
||||
}
|
||||
|
||||
function hasOnlyKeys(value, allowedKeys) {
|
||||
return Object.keys(value).every((key) => allowedKeys.has(key));
|
||||
}
|
||||
Reference in New Issue
Block a user