SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): add managed data product history plane

Browse Source
This commit is contained in:
Codex
2026-07-18 14:38:06 +03:00
parent 02816c4352
commit 3c5d8f6cef
34 changed files with 2091 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/external-data-plane/src/server.mjs
+183 -16
View File
@@ -12,6 +12,7 @@ import {
pruneBatchReceipts,
pruneDataProductHistory,
prunePatchOutbox,
readDataProductHistory,
readDataProductSnapshot,
readPatchEvents,
} from "./data-product-delivery.mjs";
@@ -26,12 +27,15 @@ import {
assertReaderProduct,
createReaderToken,
hashReaderToken,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeReaderBinding,
} from "./reader-binding.mjs";
import { migrate } from "./schema.mjs";
import {
createWriterToken,
canMigrateManagedWriterBindingToDurable,
hashWriterToken,
materializeDataProductPublish,
materializeWriterBoundBatch,
@@ -90,12 +94,16 @@ app.get("/healthz", asyncRoute(async (_req, res) => {
providerLogic: "absent",
commandTransport: "absent",
writerBindings: "supported",
managedWriterBindings: "digest+idempotent-generation",
managedWriterBindings: "digest+idempotent-generation+explicit-revoke",
readerBindings: "supported",
dataProductDelivery: "snapshot+durable-patch",
managedReaderBindings: "digest+idempotent-generation+explicit-revoke",
dataProductDelivery: "snapshot+history+durable-patch",
legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled",
legacyCapabilityProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled",
managedWriterBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled",
managedWriterBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
managedReaderBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled",
managedReaderBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
rawRetentionSweep: {
mode: "server-scheduled",
lastSweepAt: lastRetentionSweepAt,
@@ -169,9 +177,7 @@ app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, req
app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_writer_binding_key_invalid");
const policy = normalizeManagedWriterBindingRequest(req.body, {
maxTtlDays: config.writerBindingMaxTtlDays,
});
const policy = normalizeManagedWriterBindingRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
const requestHash = writerBindingRequestHash(policy);
const client = await pool.connect();
@@ -180,7 +186,7 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", request_hash as "requestHash", generation,
`select id, token_hash as "capabilityDigest", binding_key as "bindingKey", request_hash as "requestHash", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
@@ -192,12 +198,27 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
if (existing.rowCount) {
const binding = existing.rows[0];
if (binding.requestHash !== requestHash) {
throw httpError(409, "managed_writer_binding_request_conflict");
const legacyCanMigrate = canMigrateManagedWriterBindingToDurable(binding, policy);
if (!legacyCanMigrate) throw httpError(409, "managed_writer_binding_request_conflict");
const migrated = await client.query(
`update external_data_plane_writer_bindings
set request_hash = $3, expires_at = null, rotated_at = now()
where binding_key = $1 and generation = $2 and token_hash = $4
and active = true and expires_at > now()
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[bindingKey, policy.generation, requestHash, policy.capabilityDigest],
);
if (!migrated.rowCount) throw httpError(409, "managed_writer_binding_generation_inactive");
response = { status: 200, idempotent: false, migrated: true, binding: migrated.rows[0] };
} else {
if (binding.active !== true || binding.expiresAt !== null) {
throw httpError(409, "managed_writer_binding_generation_inactive");
}
response = { status: 200, idempotent: true, migrated: false, binding };
}
if (binding.active !== true || new Date(binding.expiresAt) <= new Date()) {
throw httpError(409, "managed_writer_binding_generation_inactive");
}
response = { status: 200, idempotent: true, binding };
} else {
const inserted = await client.query(
`insert into external_data_plane_writer_bindings (
@@ -237,6 +258,7 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
res.status(response.status).json({
ok: true,
idempotent: response.idempotent,
migrated: response.migrated === true,
writerBinding: safeWriterBinding(response.binding),
});
}));
@@ -354,6 +376,126 @@ app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requirePro
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
}));
app.put("/internal/data-plane/v1/reader-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_reader_binding_key_invalid");
const policy = normalizeManagedReaderBindingRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
const requestHash = readerBindingRequestHash(policy);
const client = await pool.connect();
let response;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", request_hash as "requestHash", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, policy.generation],
);
if (existing.rowCount) {
const binding = existing.rows[0];
if (binding.requestHash !== requestHash) {
throw httpError(409, "managed_reader_binding_request_conflict");
}
if (binding.active !== true || binding.expiresAt !== null) {
throw httpError(409, "managed_reader_binding_generation_inactive");
}
response = { status: 200, idempotent: true, binding };
} else {
const inserted = await client.query(
`insert into external_data_plane_reader_bindings (
id, token_hash, binding_key, request_hash, generation,
tenant_id, connection_id, provider_id, allowed_data_product_ids, expires_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10)
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[
randomUUID(),
policy.capabilityDigest,
bindingKey,
requestHash,
policy.generation,
policy.tenantId,
policy.connectionId,
policy.providerId,
JSON.stringify(policy.allowedDataProductIds),
policy.expiresAt,
],
);
response = { status: 201, idempotent: false, binding: inserted.rows[0] };
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
if (error?.code === "23505") {
throw httpError(409, "managed_reader_binding_capability_digest_conflict");
}
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.status(response.status).json({
ok: true,
idempotent: response.idempotent,
readerBinding: safeReaderBinding(response.binding),
});
}));
app.post("/internal/data-plane/v1/reader-bindings/by-key/:bindingKey/generations/:generation/revoke", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_reader_binding_key_invalid");
const generation = requirePositiveInteger(req.params.generation, "managed_reader_binding_generation_invalid");
const client = await pool.connect();
let binding;
let idempotent;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, generation],
);
if (!existing.rowCount) throw httpError(404, "managed_reader_binding_not_found");
if (existing.rows[0].active === true) {
const revoked = await client.query(
`update external_data_plane_reader_bindings
set active = false, revoked_at = now()
where binding_key = $1 and generation = $2 and active = true
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[bindingKey, generation],
);
binding = revoked.rows[0];
idempotent = false;
} else {
binding = existing.rows[0];
idempotent = true;
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.json({ ok: true, idempotent, readerBinding: safeReaderBinding(binding) });
}));
app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
const policy = normalizeReaderBindingRequest(req.body, {
maxTtlDays: config.writerBindingMaxTtlDays,
@@ -389,7 +531,7 @@ app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requirePro
const result = await pool.query(
`update external_data_plane_reader_bindings
set token_hash = $2, rotated_at = now()
where id = $1 and active = true and expires_at > now()
where id = $1 and binding_key is null and active = true and expires_at > now()
returning id, tenant_id as "tenantId", connection_id as "connectionId",
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
expires_at as "expiresAt", active, created_at as "createdAt",
@@ -405,7 +547,7 @@ app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requirePro
const result = await pool.query(
`update external_data_plane_reader_bindings
set active = false, revoked_at = now()
where id = $1 and active = true
where id = $1 and binding_key is null and active = true
returning id, tenant_id as "tenantId", connection_id as "connectionId",
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
expires_at as "expiresAt", active, created_at as "createdAt",
@@ -426,6 +568,22 @@ app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", require
res.json(snapshot);
}));
app.get("/internal/data-plane/v1/data-products/:dataProductId/history", requireReaderBinding, asyncRoute(async (req, res) => {
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
const definition = await loadDataProductDefinition(pool, dataProductId);
if (!definition) throw httpError(404, "data_product_not_found");
const history = await readDataProductHistory(pool, req.readerBinding, definition, {
from: req.query.from,
to: req.query.to,
resolutionMs: req.query.resolutionMs,
sourceIds: parseSourceIds(req.query.sourceIds),
limit: req.query.limit,
cursor: req.query.cursor,
});
res.json(history);
}));
app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => {
if (shuttingDown) throw httpError(503, "service_shutting_down");
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
@@ -767,7 +925,8 @@ async function resolveWriterBinding(req) {
expires_at as "expiresAt", active, created_at as "createdAt",
rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_writer_bindings
where token_hash = $1 and active = true and expires_at > now()`,
where token_hash = $1 and active = true
and (expires_at is null or expires_at > now())`,
[hashWriterToken(token)],
);
if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized");
@@ -783,7 +942,8 @@ async function resolveReaderBinding(req) {
expires_at as "expiresAt", active, created_at as "createdAt",
rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where token_hash = $1 and active = true and expires_at > now()`,
where token_hash = $1 and active = true
and (expires_at is null or expires_at > now())`,
[hashReaderToken(token)],
);
if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized");
@@ -797,7 +957,8 @@ async function assertReaderStreamAccess(binding, dataProductId) {
join external_data_plane_products as product
on product.id = $3 and product.active = true
where binding.id = $1 and binding.token_hash = $2
and binding.active = true and binding.expires_at > now()
and binding.active = true
and (binding.expires_at is null or binding.expires_at > now())
and binding.allowed_data_product_ids ? $3`,
[binding.id, binding.tokenHash, dataProductId],
);
@@ -883,6 +1044,12 @@ function parseCursor(value) {
return cursor;
}
function parseSourceIds(value) {
if (value === undefined || value === null || value === "") return [];
const values = Array.isArray(value) ? value : String(value).split(",");
return values.map((item) => String(item || "").trim()).filter(Boolean);
}
function writePatchEvent(res, event) {
return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`);
}