feat(data-plane): add provider contracts and ontology delivery
This commit is contained in:
Codex
@@ -0,0 +1,822 @@
|
||||
import express from "express";
|
||||
import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
|
||||
import { createServer } from "node:http";
|
||||
import { Pool } from "pg";
|
||||
import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract";
|
||||
import { readConfig } from "./config.mjs";
|
||||
import {
|
||||
loadDataProductDefinition,
|
||||
assertPublishMatchesDefinition,
|
||||
persistDataProductDefinition,
|
||||
persistDataProductPublish,
|
||||
pruneBatchReceipts,
|
||||
pruneDataProductHistory,
|
||||
prunePatchOutbox,
|
||||
readDataProductSnapshot,
|
||||
readPatchEvents,
|
||||
} from "./data-product-delivery.mjs";
|
||||
import { normalizeDataProductDefinition, safeDataProductDefinition } from "./data-product-policy.mjs";
|
||||
import { reconcileDataProductDefinitions } from "./definitions.mjs";
|
||||
import { assertBatchTimeBounds, rawRetentionExpiry } from "./intake-policy.mjs";
|
||||
import {
|
||||
assertReaderProduct,
|
||||
createReaderToken,
|
||||
hashReaderToken,
|
||||
normalizeReaderBindingRequest,
|
||||
safeReaderBinding,
|
||||
} from "./reader-binding.mjs";
|
||||
import { migrate } from "./schema.mjs";
|
||||
import {
|
||||
createWriterToken,
|
||||
hashWriterToken,
|
||||
materializeDataProductPublish,
|
||||
materializeWriterBoundBatch,
|
||||
normalizeWriterBindingRequest,
|
||||
safeWriterBinding,
|
||||
} from "./writer-binding.mjs";
|
||||
|
||||
const config = readConfig();
|
||||
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
|
||||
const app = express();
|
||||
const httpServer = createServer(app);
|
||||
let retentionSweepTimer = null;
|
||||
let lastRetentionSweepAt = null;
|
||||
const activeReaderStreams = new Map();
|
||||
const activeStreamResponses = new Set();
|
||||
let shuttingDown = false;
|
||||
let shutdownPromise = null;
|
||||
|
||||
app.disable("x-powered-by");
|
||||
app.use(express.json({ limit: config.maxBatchBytes }));
|
||||
|
||||
app.get("/healthz", asyncRoute(async (_req, res) => {
|
||||
await pool.query("select 1");
|
||||
res.json({
|
||||
ok: true,
|
||||
service: "nodedc-external-data-plane",
|
||||
database: "ready",
|
||||
internalApiConfigured: Boolean(config.internalAccessToken),
|
||||
providerLogic: "absent",
|
||||
commandTransport: "absent",
|
||||
writerBindings: "supported",
|
||||
readerBindings: "supported",
|
||||
dataProductDelivery: "snapshot+durable-patch",
|
||||
legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled",
|
||||
writerBindingProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled",
|
||||
rawRetentionSweep: {
|
||||
mode: "server-scheduled",
|
||||
lastSweepAt: lastRetentionSweepAt,
|
||||
},
|
||||
});
|
||||
}));
|
||||
|
||||
app.put("/internal/data-plane/v1/data-products/:dataProductId", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const definition = normalizeDataProductDefinition({ ...req.body, id: req.params.dataProductId });
|
||||
const saved = await persistDataProductDefinition(pool, definition);
|
||||
res.json({ ok: true, dataProduct: safeDataProductDefinition(saved) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/data-products/:dataProductId/publish", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_publish_scope_headers_forbidden");
|
||||
const validation = validateDataProductPublish(req.body, {
|
||||
maxFacts: config.maxFactsPerPublish,
|
||||
maxAttributesBytes: config.maxAttributesBytesPerFact,
|
||||
});
|
||||
if (!validation.ok) throw httpError(422, validation.errors[0] || "invalid_data_product_publish");
|
||||
const dataProductId = requireIdentifier(req.params.dataProductId, "data_product_id_invalid");
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const batch = materializeDataProductPublish(req.body, req.writerBinding, definition, dataProductId);
|
||||
const canonicalValidation = validateIntakeBatch(batch);
|
||||
if (!canonicalValidation.ok) throw httpError(422, "materialized_publish_invalid");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
const receipt = await persistDataProductPublish(pool, batch, definition, {
|
||||
maxPatchOperations: config.maxPatchOperations,
|
||||
maxPatchBytes: config.maxPatchBytes,
|
||||
});
|
||||
res.status(receipt.idempotent ? 200 : 201).json({ ok: true, ...receipt });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/writer/data-products", requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.writerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/reader/data-products", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
const dataProducts = await listGrantedProducts(req.readerBinding);
|
||||
res.json({ ok: true, dataProducts });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const validation = validateIntakeBatch(req.body);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(req.body, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(req.body);
|
||||
const scope = requireScope(req);
|
||||
if (scope.tenantId !== req.body.source.tenantId || scope.connectionId !== req.body.source.connectionId) {
|
||||
throw httpError(403, "scope_mismatch");
|
||||
}
|
||||
|
||||
const result = await persistBatch(req.body);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, requireWriterBinding, asyncRoute(async (req, res) => {
|
||||
const batch = materializeWriterBoundBatch(req.body, req.writerBinding, {
|
||||
hasScopeHeaders: hasScopeHeaders(req),
|
||||
});
|
||||
const validation = validateIntakeBatch(batch);
|
||||
if (!validation.ok) throw httpError(422, "invalid_intake_batch");
|
||||
assertBatchTimeBounds(batch, { maxFutureSkewSeconds: config.maxFutureSkewSeconds });
|
||||
await assertLegacyBatchProduct(batch);
|
||||
|
||||
const result = await persistBatch(batch);
|
||||
res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeWriterBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createWriterToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_writer_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashWriterToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
// The token is intentionally returned exactly once to a trusted provisioner.
|
||||
// It must be placed directly into an opaque Engine credential reference and
|
||||
// must never be logged, stored in L2 graph data or shown to a consumer.
|
||||
sendOneTimeCapability(res, 201, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const token = createWriterToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, writerBinding: safeWriterBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "writer_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_writer_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "writer_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const policy = normalizeReaderBindingRequest(req.body, {
|
||||
maxTtlDays: config.writerBindingMaxTtlDays,
|
||||
});
|
||||
await assertRegisteredProductIds(policy.allowedDataProductIds);
|
||||
const token = createReaderToken();
|
||||
const bindingId = randomUUID();
|
||||
const result = await pool.query(
|
||||
`insert into external_data_plane_reader_bindings (
|
||||
id, token_hash, tenant_id, connection_id, provider_id,
|
||||
allowed_data_product_ids, expires_at
|
||||
) values ($1, $2, $3, $4, $5, $6::jsonb, $7)
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[
|
||||
bindingId,
|
||||
hashReaderToken(token),
|
||||
policy.tenantId,
|
||||
policy.connectionId,
|
||||
policy.providerId,
|
||||
JSON.stringify(policy.allowedDataProductIds),
|
||||
policy.expiresAt,
|
||||
],
|
||||
);
|
||||
sendOneTimeCapability(res, 201, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const token = createReaderToken();
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set token_hash = $2, rotated_at = now()
|
||||
where id = $1 and active = true and expires_at > now()
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId, hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
sendOneTimeCapability(res, 200, { ok: true, readerBinding: safeReaderBinding(result.rows[0]), token });
|
||||
}));
|
||||
|
||||
app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requireProvisionerApi, asyncRoute(async (req, res) => {
|
||||
const bindingId = requireUuid(req.params.bindingId, "reader_binding_id_invalid");
|
||||
const result = await pool.query(
|
||||
`update external_data_plane_reader_bindings
|
||||
set active = false, revoked_at = now()
|
||||
where id = $1 and active = true
|
||||
returning id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
|
||||
[bindingId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(404, "reader_binding_not_found_or_inactive");
|
||||
res.json({ ok: true, readerBinding: safeReaderBinding(result.rows[0]) });
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
const limit = boundedLimit(req.query.limit, 5000, 1, 5000);
|
||||
const snapshot = await readDataProductSnapshot(pool, req.readerBinding, definition, { limit });
|
||||
res.json(snapshot);
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => {
|
||||
if (shuttingDown) throw httpError(503, "service_shutting_down");
|
||||
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
|
||||
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
|
||||
const definition = await loadDataProductDefinition(pool, dataProductId);
|
||||
if (!definition) throw httpError(404, "data_product_not_found");
|
||||
if (definition.deliveryMode !== "snapshot+patch") throw httpError(409, "data_product_stream_not_supported");
|
||||
const streamCount = activeReaderStreams.get(req.readerBinding.id) || 0;
|
||||
if (streamCount >= config.maxReaderStreams) throw httpError(429, "reader_stream_limit_exceeded");
|
||||
activeReaderStreams.set(req.readerBinding.id, streamCount + 1);
|
||||
activeStreamResponses.add(res);
|
||||
|
||||
let poll = null;
|
||||
let heartbeat = null;
|
||||
let polling = false;
|
||||
let heartbeatWriting = false;
|
||||
let cleanedUp = false;
|
||||
const cleanup = () => {
|
||||
if (cleanedUp) return;
|
||||
cleanedUp = true;
|
||||
if (poll) clearInterval(poll);
|
||||
if (heartbeat) clearInterval(heartbeat);
|
||||
activeStreamResponses.delete(res);
|
||||
const remaining = Math.max(0, (activeReaderStreams.get(req.readerBinding.id) || 1) - 1);
|
||||
if (remaining) activeReaderStreams.set(req.readerBinding.id, remaining);
|
||||
else activeReaderStreams.delete(req.readerBinding.id);
|
||||
};
|
||||
req.once("close", cleanup);
|
||||
res.once("close", cleanup);
|
||||
|
||||
try {
|
||||
let cursor = parseCursor(req.get("last-event-id") || req.query.after || "0");
|
||||
const initialEvents = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
if (res.destroyed || cleanedUp) return;
|
||||
res.status(200);
|
||||
res.set({
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Content-Type": "text/event-stream",
|
||||
Connection: "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
});
|
||||
res.flushHeaders();
|
||||
await writeSseFrame(res, ": nodedc-data-product-stream\n\n");
|
||||
await writeSseFrame(res, `event: nodedc.data-product.ready.v1\ndata: ${JSON.stringify({
|
||||
schemaVersion: "nodedc.data-product.ready/v1",
|
||||
dataProductId,
|
||||
cursor: cursor.toString(),
|
||||
emittedAt: new Date().toISOString(),
|
||||
})}\n\n`);
|
||||
for (const event of initialEvents) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
|
||||
const pump = async () => {
|
||||
if (polling || res.writableEnded || res.destroyed || shuttingDown) return;
|
||||
polling = true;
|
||||
try {
|
||||
await assertReaderStreamAccess(req.readerBinding, dataProductId);
|
||||
const events = await readPatchEvents(pool, req.readerBinding, definition, cursor);
|
||||
for (const event of events) {
|
||||
await writePatchEvent(res, event);
|
||||
cursor = BigInt(event.cursor);
|
||||
}
|
||||
} catch (error) {
|
||||
const code = safeErrorCode(error);
|
||||
if (!res.writableEnded && !res.destroyed) {
|
||||
await writeSseFrame(res, `event: error\ndata: ${JSON.stringify({ ok: false, error: code })}\n\n`).catch(() => {});
|
||||
res.end();
|
||||
}
|
||||
} finally {
|
||||
polling = false;
|
||||
}
|
||||
};
|
||||
poll = setInterval(() => { void pump(); }, config.streamPollMs);
|
||||
poll.unref();
|
||||
heartbeat = setInterval(() => {
|
||||
if (polling || heartbeatWriting || res.writableEnded || res.destroyed) return;
|
||||
heartbeatWriting = true;
|
||||
void writeSseFrame(res, `: heartbeat ${Date.now()}\n\n`)
|
||||
.catch(() => { if (!res.writableEnded) res.end(); })
|
||||
.finally(() => { heartbeatWriting = false; });
|
||||
}, config.streamHeartbeatMs);
|
||||
heartbeat.unref();
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
if (res.headersSent) {
|
||||
if (!res.writableEnded) res.end();
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/data-products/:dataProductId/current", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const dataProductId = String(req.params.dataProductId || "");
|
||||
if (!isIdentifier(dataProductId)) throw httpError(400, "data_product_id_invalid");
|
||||
const limit = boundedLimit(req.query.limit, 200, 1, 1000);
|
||||
const rows = await pool.query(
|
||||
`select
|
||||
provider_id as "providerId", data_product_id as "dataProductId",
|
||||
source_id as "sourceId", semantic_type as "semanticType",
|
||||
observed_at as "observedAt", received_at as "receivedAt", attributes,
|
||||
case when geometry is null then null
|
||||
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
|
||||
ST_X(geometry::geometry), ST_Y(geometry::geometry)
|
||||
)) end as geometry
|
||||
from external_data_plane_current
|
||||
where tenant_id = $1 and connection_id = $2 and data_product_id = $3
|
||||
order by observed_at desc, source_id asc
|
||||
limit $4`,
|
||||
[scope.tenantId, scope.connectionId, dataProductId, limit],
|
||||
);
|
||||
res.json({
|
||||
ok: true,
|
||||
dataProductId,
|
||||
tenantId: scope.tenantId,
|
||||
connectionId: scope.connectionId,
|
||||
facts: rows.rows,
|
||||
});
|
||||
}));
|
||||
|
||||
app.get("/internal/data-plane/v1/status", requireLegacyIntake, requireInternalApi, asyncRoute(async (req, res) => {
|
||||
const scope = requireScope(req);
|
||||
const result = await pool.query(
|
||||
`select count(*)::integer as "currentFactCount", max(received_at) as "lastReceivedAt"
|
||||
from external_data_plane_current where tenant_id = $1 and connection_id = $2`,
|
||||
[scope.tenantId, scope.connectionId],
|
||||
);
|
||||
res.json({ ok: true, ...scope, ...result.rows[0], providerLogic: "absent", commandTransport: "absent" });
|
||||
}));
|
||||
|
||||
app.use((error, _req, res, _next) => {
|
||||
const status = Number(error?.status || 500);
|
||||
const publicStatus = status >= 400 && status < 600 ? status : 500;
|
||||
console.error(JSON.stringify({ event: "external_data_plane_error", error: safeErrorCode(error), status: publicStatus }));
|
||||
res.status(publicStatus).json({ ok: false, error: publicStatus >= 500 ? "internal_error" : safeErrorCode(error) });
|
||||
});
|
||||
|
||||
await migrate(pool);
|
||||
const bundledDataProducts = await reconcileDataProductDefinitions(pool);
|
||||
retentionSweepTimer = setInterval(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
}, config.retentionSweepMs);
|
||||
retentionSweepTimer.unref();
|
||||
|
||||
httpServer.listen(config.port, "0.0.0.0", () => {
|
||||
console.log(`NODE.DC External Data Plane listening on http://0.0.0.0:${config.port}`);
|
||||
console.log(JSON.stringify({ event: "external_data_plane_definitions_ready", count: bundledDataProducts.length }));
|
||||
setImmediate(() => {
|
||||
void sweepRetention().catch((error) => {
|
||||
console.error(JSON.stringify({ event: "external_data_plane_retention_sweep_failed", error: safeErrorCode(error) }));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
process.on("SIGTERM", shutdown);
|
||||
process.on("SIGINT", shutdown);
|
||||
|
||||
async function persistBatch(batch) {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("begin");
|
||||
const existing = await client.query(
|
||||
`select id, fact_count as "factCount", inserted_fact_count as "insertedFactCount"
|
||||
from external_data_plane_batches
|
||||
where tenant_id = $1 and connection_id = $2 and provider_id = $3
|
||||
and data_product_id = $4 and idempotency_key = $5
|
||||
for update`,
|
||||
[
|
||||
batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.idempotencyKey,
|
||||
],
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("commit");
|
||||
return { batchId: existing.rows[0].id, idempotent: true, ...existing.rows[0] };
|
||||
}
|
||||
|
||||
const batchId = randomUUID();
|
||||
await client.query(
|
||||
`insert into external_data_plane_batches (
|
||||
id, tenant_id, connection_id, provider_id, data_product_id, contract_version,
|
||||
ontology_revision, run_id, sequence, idempotency_key, received_at, fact_count
|
||||
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.contract.version, batch.contract.ontologyRevision,
|
||||
batch.batch.runId, batch.batch.sequence, batch.batch.idempotencyKey,
|
||||
batch.batch.receivedAt, batch.facts.length,
|
||||
],
|
||||
);
|
||||
|
||||
if (batch.raw) await persistRawEnvelope(client, batchId, batch);
|
||||
|
||||
const records = batch.facts.map((fact) => ({
|
||||
id: randomUUID(),
|
||||
source_id: fact.sourceId,
|
||||
semantic_type: fact.semanticType,
|
||||
observed_at: fact.observedAt,
|
||||
attributes: fact.attributes || {},
|
||||
geometry: fact.geometry || null,
|
||||
fingerprint: hash(fact),
|
||||
}));
|
||||
const inserted = await client.query(
|
||||
`insert into external_data_plane_facts (
|
||||
id, batch_id, tenant_id, connection_id, provider_id, data_product_id,
|
||||
source_id, semantic_type, observed_at, received_at, attributes, geometry, fingerprint
|
||||
)
|
||||
select item.id::uuid, $1, $2, $3, $4, $5,
|
||||
item.source_id, item.semantic_type, item.observed_at, $6,
|
||||
coalesce(item.attributes, '{}'::jsonb),
|
||||
case when item.geometry is null then null
|
||||
else ST_SetSRID(ST_MakePoint(
|
||||
(item.geometry->'coordinates'->>0)::double precision,
|
||||
(item.geometry->'coordinates'->>1)::double precision
|
||||
), 4326)::geography end,
|
||||
item.fingerprint
|
||||
from jsonb_to_recordset($7::jsonb) as item(
|
||||
id text, source_id text, semantic_type text, observed_at timestamptz,
|
||||
attributes jsonb, geometry jsonb, fingerprint text
|
||||
)
|
||||
on conflict do nothing
|
||||
returning id`,
|
||||
[
|
||||
batchId, batch.source.tenantId, batch.source.connectionId, batch.source.providerId,
|
||||
batch.contract.dataProductId, batch.batch.receivedAt, JSON.stringify(records),
|
||||
],
|
||||
);
|
||||
await client.query(
|
||||
`insert into external_data_plane_current (
|
||||
tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, updated_at
|
||||
)
|
||||
select tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type,
|
||||
observed_at, received_at, attributes, geometry, fingerprint, now()
|
||||
from external_data_plane_facts where batch_id = $1
|
||||
on conflict (tenant_id, connection_id, provider_id, data_product_id, source_id, semantic_type)
|
||||
do update set
|
||||
observed_at = excluded.observed_at,
|
||||
received_at = excluded.received_at,
|
||||
attributes = excluded.attributes,
|
||||
geometry = excluded.geometry,
|
||||
fingerprint = excluded.fingerprint,
|
||||
updated_at = now()
|
||||
where excluded.observed_at >= external_data_plane_current.observed_at`,
|
||||
[batchId],
|
||||
);
|
||||
await client.query(
|
||||
"update external_data_plane_batches set inserted_fact_count = $2 where id = $1",
|
||||
[batchId, inserted.rowCount],
|
||||
);
|
||||
await client.query("commit");
|
||||
return { batchId, idempotent: false, factCount: batch.facts.length, insertedFactCount: inserted.rowCount };
|
||||
} catch (error) {
|
||||
await client.query("rollback");
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
async function persistRawEnvelope(client, batchId, batch) {
|
||||
const serialized = batch.raw.payload === undefined ? null : JSON.stringify(batch.raw.payload);
|
||||
const expiresAt = rawRetentionExpiry({ rawRetentionDays: config.rawRetentionDays });
|
||||
await client.query(
|
||||
`insert into external_data_plane_raw_envelopes (
|
||||
id, batch_id, payload_hash, content_type, payload, payload_ref, payload_bytes, received_at, expires_at
|
||||
) values ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9)`,
|
||||
[
|
||||
randomUUID(), batchId, batch.raw.hash || hash(batch.raw.payload), batch.raw.contentType, serialized, batch.raw.ref || null,
|
||||
serialized ? Buffer.byteLength(serialized) : null, batch.batch.receivedAt, expiresAt,
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
function requireInternalApi(req, _res, next) {
|
||||
if (!config.internalAccessToken) return next(httpError(503, "internal_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.internalAccessToken)) return next(httpError(401, "unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireLegacyIntake(_req, _res, next) {
|
||||
if (!config.legacyIntakeEnabled) return next(httpError(410, "legacy_intake_disabled"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireProvisionerApi(req, _res, next) {
|
||||
if (!config.provisionerApiEnabled) return next(httpError(503, "provisioner_api_disabled"));
|
||||
if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured"));
|
||||
const value = bearerToken(req);
|
||||
if (!value || !safeEqual(value, config.provisionerAccessToken)) return next(httpError(401, "provisioner_unauthorized"));
|
||||
return next();
|
||||
}
|
||||
|
||||
function requireWriterBinding(req, _res, next) {
|
||||
return resolveWriterBinding(req)
|
||||
.then((binding) => {
|
||||
req.writerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
function requireReaderBinding(req, _res, next) {
|
||||
return resolveReaderBinding(req)
|
||||
.then((binding) => {
|
||||
req.readerBinding = binding;
|
||||
next();
|
||||
})
|
||||
.catch(next);
|
||||
}
|
||||
|
||||
async function resolveWriterBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "writer_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_writer_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashWriterToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function resolveReaderBinding(req) {
|
||||
const token = bearerToken(req);
|
||||
if (!token) throw httpError(401, "reader_binding_unauthorized");
|
||||
const result = await pool.query(
|
||||
`select id, token_hash as "tokenHash", tenant_id as "tenantId", connection_id as "connectionId",
|
||||
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
|
||||
expires_at as "expiresAt", active, created_at as "createdAt",
|
||||
rotated_at as "rotatedAt", revoked_at as "revokedAt"
|
||||
from external_data_plane_reader_bindings
|
||||
where token_hash = $1 and active = true and expires_at > now()`,
|
||||
[hashReaderToken(token)],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized");
|
||||
return result.rows[0];
|
||||
}
|
||||
|
||||
async function assertReaderStreamAccess(binding, dataProductId) {
|
||||
const result = await pool.query(
|
||||
`select binding.id
|
||||
from external_data_plane_reader_bindings as binding
|
||||
join external_data_plane_products as product
|
||||
on product.id = $3 and product.active = true
|
||||
where binding.id = $1 and binding.token_hash = $2
|
||||
and binding.active = true and binding.expires_at > now()
|
||||
and binding.allowed_data_product_ids ? $3`,
|
||||
[binding.id, binding.tokenHash, dataProductId],
|
||||
);
|
||||
if (!result.rowCount) throw httpError(401, "reader_stream_access_revoked");
|
||||
}
|
||||
|
||||
async function assertLegacyBatchProduct(batch) {
|
||||
const definition = await loadDataProductDefinition(pool, batch.contract.dataProductId);
|
||||
if (!definition) throw httpError(422, "legacy_data_product_not_registered");
|
||||
if (definition.version !== batch.contract.version || definition.ontologyRevision !== batch.contract.ontologyRevision) {
|
||||
throw httpError(422, "legacy_data_product_contract_mismatch");
|
||||
}
|
||||
assertPublishMatchesDefinition(batch, definition);
|
||||
}
|
||||
|
||||
async function listGrantedProducts(binding) {
|
||||
const allowed = Array.isArray(binding.allowedDataProductIds) ? binding.allowedDataProductIds : [];
|
||||
if (!allowed.length) return [];
|
||||
const result = await pool.query(
|
||||
`select id, version, ontology_revision as "ontologyRevision",
|
||||
delivery_mode as "deliveryMode", semantic_types as "semanticTypes",
|
||||
fields, history_policy as "historyPolicy", active,
|
||||
created_at as "createdAt", updated_at as "updatedAt"
|
||||
from external_data_plane_products
|
||||
where active = true and id = any($1::text[])
|
||||
order by id asc`,
|
||||
[allowed],
|
||||
);
|
||||
return result.rows.map(safeDataProductDefinition);
|
||||
}
|
||||
|
||||
async function assertRegisteredProductIds(dataProductIds) {
|
||||
const result = await pool.query(
|
||||
"select id from external_data_plane_products where active = true and id = any($1::text[])",
|
||||
[dataProductIds],
|
||||
);
|
||||
const registered = new Set(result.rows.map((row) => row.id));
|
||||
if (dataProductIds.some((id) => !registered.has(id))) throw httpError(422, "binding_data_product_not_registered");
|
||||
}
|
||||
|
||||
function requireScope(req) {
|
||||
const tenantId = String(req.get("x-nodedc-tenant-id") || "");
|
||||
const connectionId = String(req.get("x-nodedc-connection-id") || "");
|
||||
if (!isIdentifier(tenantId) || !isIdentifier(connectionId)) throw httpError(400, "scope_headers_required");
|
||||
return { tenantId, connectionId };
|
||||
}
|
||||
|
||||
function hasScopeHeaders(req) {
|
||||
return Object.hasOwn(req.headers, "x-nodedc-tenant-id") || Object.hasOwn(req.headers, "x-nodedc-connection-id");
|
||||
}
|
||||
|
||||
function bearerToken(req) {
|
||||
return String(req.get("authorization") || "").replace(/^Bearer\s+/i, "");
|
||||
}
|
||||
|
||||
function requireUuid(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(normalized)) {
|
||||
throw httpError(400, code);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function requireIdentifier(value, code) {
|
||||
const normalized = String(value || "");
|
||||
if (!isIdentifier(normalized)) throw httpError(400, code);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function parseCursor(value) {
|
||||
const normalized = String(value ?? "");
|
||||
if (!/^(?:0|[1-9]\d*)$/.test(normalized)) throw httpError(400, "data_product_cursor_invalid");
|
||||
const cursor = BigInt(normalized);
|
||||
if (cursor > 9_223_372_036_854_775_807n) throw httpError(400, "data_product_cursor_invalid");
|
||||
return cursor;
|
||||
}
|
||||
|
||||
function writePatchEvent(res, event) {
|
||||
return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
||||
function writeSseFrame(res, frame) {
|
||||
if (res.writableEnded || res.destroyed) return Promise.reject(httpError(499, "reader_stream_closed"));
|
||||
if (res.write(frame)) return Promise.resolve();
|
||||
return new Promise((resolve, reject) => {
|
||||
const cleanup = () => {
|
||||
res.off("drain", onDrain);
|
||||
res.off("close", onClose);
|
||||
res.off("error", onError);
|
||||
};
|
||||
const onDrain = () => { cleanup(); resolve(); };
|
||||
const onClose = () => { cleanup(); reject(httpError(499, "reader_stream_closed")); };
|
||||
const onError = (error) => { cleanup(); reject(error); };
|
||||
res.once("drain", onDrain);
|
||||
res.once("close", onClose);
|
||||
res.once("error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function sendOneTimeCapability(res, status, payload) {
|
||||
// Create/rotate is the only boundary where a capability exists in
|
||||
// plaintext. A trusted provisioner must consume the response in memory and
|
||||
// place it directly into its opaque destination; intermediaries must never
|
||||
// cache or persist it.
|
||||
res.set({
|
||||
"Cache-Control": "no-store, max-age=0",
|
||||
Pragma: "no-cache",
|
||||
Expires: "0",
|
||||
});
|
||||
return res.status(status).json(payload);
|
||||
}
|
||||
|
||||
function safeEqual(left, right) {
|
||||
const leftBuffer = Buffer.from(left);
|
||||
const rightBuffer = Buffer.from(right);
|
||||
return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer);
|
||||
}
|
||||
|
||||
function boundedLimit(value, fallback, min, max) {
|
||||
const candidate = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(candidate)) return fallback;
|
||||
return Math.max(min, Math.min(max, candidate));
|
||||
}
|
||||
|
||||
function hash(value) {
|
||||
return createHash("sha256").update(JSON.stringify(value)).digest("hex");
|
||||
}
|
||||
|
||||
function isIdentifier(value) {
|
||||
return /^[a-z][a-z0-9._:-]{2,127}$/i.test(value);
|
||||
}
|
||||
|
||||
function asyncRoute(handler) {
|
||||
return (req, res, next) => Promise.resolve(handler(req, res, next)).catch(next);
|
||||
}
|
||||
|
||||
function httpError(status, code) {
|
||||
return Object.assign(new Error(code), { status, code });
|
||||
}
|
||||
|
||||
function safeErrorCode(error) {
|
||||
return String(error?.code || error?.message || "internal_error").replace(/[^a-z0-9_.:-]/gi, "_").slice(0, 120);
|
||||
}
|
||||
|
||||
async function sweepExpiredRawEnvelopes() {
|
||||
const result = await pool.query(
|
||||
`with expired as (
|
||||
select id from external_data_plane_raw_envelopes
|
||||
where expires_at < now()
|
||||
order by expires_at asc
|
||||
limit $1
|
||||
)
|
||||
delete from external_data_plane_raw_envelopes as target
|
||||
using expired where target.id = expired.id`,
|
||||
[config.retentionDeleteLimit],
|
||||
);
|
||||
return result.rowCount;
|
||||
}
|
||||
|
||||
async function sweepRetention() {
|
||||
const rawEnvelopeCount = await sweepExpiredRawEnvelopes();
|
||||
const patchEventCount = await prunePatchOutbox(pool, {
|
||||
retentionMs: config.patchRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
const historyFactCount = await pruneDataProductHistory(pool, { limit: config.retentionDeleteLimit });
|
||||
const batchReceiptCount = await pruneBatchReceipts(pool, {
|
||||
retentionMs: config.receiptRetentionMs,
|
||||
limit: config.retentionDeleteLimit,
|
||||
});
|
||||
lastRetentionSweepAt = new Date().toISOString();
|
||||
return { rawEnvelopeCount, patchEventCount, historyFactCount, batchReceiptCount };
|
||||
}
|
||||
|
||||
async function shutdown() {
|
||||
if (shutdownPromise) return shutdownPromise;
|
||||
shutdownPromise = (async () => {
|
||||
shuttingDown = true;
|
||||
if (retentionSweepTimer) clearInterval(retentionSweepTimer);
|
||||
for (const response of activeStreamResponses) {
|
||||
if (!response.writableEnded) response.end();
|
||||
}
|
||||
const closed = new Promise((resolve) => httpServer.close(resolve));
|
||||
httpServer.closeIdleConnections?.();
|
||||
const forceClose = setTimeout(() => httpServer.closeAllConnections?.(), 250);
|
||||
await Promise.race([closed, new Promise((resolve) => setTimeout(resolve, 2_000))]);
|
||||
clearTimeout(forceClose);
|
||||
httpServer.closeAllConnections?.();
|
||||
await pool.end();
|
||||
})();
|
||||
return shutdownPromise;
|
||||
}
|
||||
Reference in New Issue
Block a user