SILVER/NODEDC_PLATFORM
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(platform): add managed data product history plane

Browse Source
This commit is contained in:
Codex
2026-07-18 14:38:06 +03:00
parent 02816c4352
commit 3c5d8f6cef
34 changed files with 2091 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/external-data-plane/src/data-product-delivery.mjs
+150
View File
@@ -1,9 +1,14 @@
import { createHash, randomUUID } from "node:crypto";
import {
DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
DATA_PRODUCT_PATCH_SCHEMA_VERSION,
DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION,
} from "@nodedc/external-provider-contract/data-plane";
const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const MAX_HISTORY_RESOLUTION_MS = 24 * 60 * 60 * 1000;
const MAX_HISTORY_SOURCE_IDS = 1000;
export async function loadDataProductDefinition(db, dataProductId, { activeOnly = true } = {}) {
const result = await db.query(
`select id, version, ontology_revision as "ontologyRevision",
@@ -216,6 +221,121 @@ export async function readDataProductSnapshot(pool, binding, definition, { limit
}
}
export function normalizeHistoryReadOptions(value, definition) {
const policy = definition?.historyPolicy || {};
if (policy.mode === "none") throw deliveryError("data_product_history_not_supported", 409);
const from = new Date(String(value?.from || ""));
const to = new Date(String(value?.to || ""));
if (Number.isNaN(from.getTime()) || Number.isNaN(to.getTime()) || from >= to) {
throw deliveryError("data_product_history_range_invalid", 400);
}
const retentionDays = Math.max(1, Number(policy.retentionDays) || 1);
if (to.getTime() - from.getTime() > retentionDays * 24 * 60 * 60 * 1000) {
throw deliveryError("data_product_history_range_exceeds_retention", 400);
}
const nativeResolutionMs = policy.mode === "sampled" ? Number(policy.intervalMs) : 1000;
const resolutionMs = value?.resolutionMs === undefined || value?.resolutionMs === ""
? nativeResolutionMs
: Number(value.resolutionMs);
if (
!Number.isInteger(resolutionMs)
|| resolutionMs < nativeResolutionMs
|| resolutionMs > MAX_HISTORY_RESOLUTION_MS
|| resolutionMs % nativeResolutionMs !== 0
) throw deliveryError("data_product_history_resolution_invalid", 400);
const sourceIds = [...new Set((Array.isArray(value?.sourceIds) ? value.sourceIds : [])
.map((item) => String(item || "").trim())
.filter(Boolean))];
if (sourceIds.length > MAX_HISTORY_SOURCE_IDS || sourceIds.some((item) => !IDENTIFIER.test(item))) {
throw deliveryError("data_product_history_source_ids_invalid", 400);
}
const limit = Number(value?.limit ?? 1000);
if (!Number.isInteger(limit) || limit < 1 || limit > 5000) {
throw deliveryError("data_product_history_limit_invalid", 400);
}
const query = {
from: from.toISOString(),
to: to.toISOString(),
resolutionMs,
sourceIds: sourceIds.sort(),
order: "asc",
};
const queryHash = createHash("sha256").update(JSON.stringify(query)).digest("hex");
const cursor = decodeHistoryCursor(value?.cursor, queryHash);
return Object.freeze({ ...query, limit, queryHash, cursor });
}
export async function readDataProductHistory(pool, binding, definition, options) {
const query = normalizeHistoryReadOptions(options, definition);
const cursor = query.cursor || {};
const rows = await pool.query(
`with sampled as (
select
to_timestamp(
floor(extract(epoch from bucket_start) * 1000 / $8::double precision)
* $8::double precision / 1000
) as query_bucket,
source_id, semantic_type, observed_at, received_at, attributes, geometry, fingerprint
from external_data_plane_history
where tenant_id = $1 and connection_id = $2 and provider_id = $3 and data_product_id = $4
and bucket_start >= $5 and bucket_start < $6
and (cardinality($7::text[]) = 0 or source_id = any($7::text[]))
), ranked as (
select *, row_number() over (
partition by query_bucket, source_id, semantic_type
order by observed_at desc, received_at desc, fingerprint desc
) as sample_rank
from sampled
)
select query_bucket as "bucketStart", source_id as "sourceId", semantic_type as "semanticType",
observed_at as "observedAt", received_at as "receivedAt", attributes,
case when geometry is null then null
else jsonb_build_object('type', 'Point', 'coordinates', jsonb_build_array(
ST_X(geometry::geometry), ST_Y(geometry::geometry)
)) end as geometry
from ranked
where sample_rank = 1
and (
$9::boolean = false
or (query_bucket, source_id, semantic_type) > ($10::timestamptz, $11::text, $12::text)
)
order by query_bucket asc, source_id asc, semantic_type asc
limit $13`,
[
binding.tenantId, binding.connectionId, binding.providerId, definition.id,
query.from, query.to, query.sourceIds, query.resolutionMs,
Boolean(query.cursor), cursor.bucketStart || query.from, cursor.sourceId || "", cursor.semanticType || "",
query.limit + 1,
],
);
const page = rows.rows.slice(0, query.limit);
const last = page.at(-1);
return {
schemaVersion: DATA_PRODUCT_HISTORY_SCHEMA_VERSION,
dataProduct: { id: definition.id, version: definition.version },
generatedAt: new Date().toISOString(),
query: {
from: query.from,
to: query.to,
resolutionMs: query.resolutionMs,
sourceIds: query.sourceIds,
order: query.order,
},
facts: page.map((fact) => ({
...publicFact(fact),
bucketStart: new Date(fact.bucketStart).toISOString(),
})),
...(rows.rowCount > query.limit && last ? {
nextCursor: encodeHistoryCursor({
queryHash: query.queryHash,
bucketStart: new Date(last.bucketStart).toISOString(),
sourceId: last.sourceId,
semanticType: last.semanticType,
}),
} : {}),
};
}
export async function readPatchEvents(pool, binding, definition, after, { limit = 100 } = {}) {
const client = await pool.connect();
try {
@@ -450,6 +570,36 @@ function publicFact(fact) {
return value;
}
function encodeHistoryCursor(value) {
return Buffer.from(JSON.stringify(value), "utf8").toString("base64url");
}
function decodeHistoryCursor(value, queryHash) {
const raw = String(value || "").trim();
if (!raw) return null;
if (raw.length > 1024 || !/^[A-Za-z0-9_-]+$/.test(raw)) {
throw deliveryError("data_product_history_cursor_invalid", 400);
}
let cursor;
try {
cursor = JSON.parse(Buffer.from(raw, "base64url").toString("utf8"));
} catch {
throw deliveryError("data_product_history_cursor_invalid", 400);
}
if (
!cursor || typeof cursor !== "object" || Array.isArray(cursor)
|| cursor.queryHash !== queryHash
|| Number.isNaN(Date.parse(cursor.bucketStart))
|| !IDENTIFIER.test(String(cursor.sourceId || ""))
|| !IDENTIFIER.test(String(cursor.semanticType || ""))
) throw deliveryError("data_product_history_cursor_invalid", 400);
return Object.freeze({
bucketStart: new Date(cursor.bucketStart).toISOString(),
sourceId: cursor.sourceId,
semanticType: cursor.semanticType,
});
}
function chunkOperations(values, maxOperations, maxBytes) {
const result = [];
let current = [];
services/external-data-plane/src/reader-binding.mjs
+65 -2
View File
@@ -4,7 +4,15 @@ const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/;
const TOKEN_PREFIX = "ndc_edprb_";
const SECRET_LIKE_KEY = /(token|secret|password|authorization|access[_-]?token|refresh[_-]?token|api[_-]?key)/i;
const REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]);
const MANAGED_REQUEST_KEYS = new Set([
"source",
"allowedDataProductIds",
"expiresAt",
"generation",
"capabilityDigest",
]);
const SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
const SHA256_DIGEST = /^[a-f0-9]{64}$/;
export function createReaderToken() {
return `${TOKEN_PREFIX}${randomBytes(32).toString("base64url")}`;
@@ -35,8 +43,61 @@ export function normalizeReaderBindingRequest(value, { now = new Date(), maxTtlD
return Object.freeze({ tenantId, connectionId, providerId, allowedDataProductIds, expiresAt: expiresAt.toISOString() });
}
export function normalizeManagedReaderBindingRequest(value) {
if (!isPlainObject(value) || !isPlainObject(value.source)) {
throw readerError("managed_reader_binding_request_invalid");
}
if (containsSecretLikeKey(value)) {
throw readerError("managed_reader_binding_secret_material_forbidden");
}
if (!hasOnlyKeys(value, MANAGED_REQUEST_KEYS) || !hasOnlyKeys(value.source, SOURCE_KEYS)) {
throw readerError("managed_reader_binding_request_fields_invalid");
}
const tenantId = identifier(value.source.tenantId);
const connectionId = identifier(value.source.connectionId);
const providerId = identifier(value.source.providerId);
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
if (!tenantId || !connectionId || !providerId || !allowedDataProductIds.length) {
throw readerError("managed_reader_binding_scope_invalid");
}
if (value.expiresAt !== null) {
throw readerError("managed_reader_binding_must_be_durable");
}
const generation = Number(value.generation);
if (!Number.isSafeInteger(generation) || generation < 1 || generation > 2_147_483_647) {
throw readerError("managed_reader_binding_generation_invalid");
}
const capabilityDigest = String(value.capabilityDigest || "").toLowerCase();
if (!SHA256_DIGEST.test(capabilityDigest)) {
throw readerError("managed_reader_binding_capability_digest_invalid");
}
return Object.freeze({
tenantId,
connectionId,
providerId,
allowedDataProductIds: Object.freeze([...allowedDataProductIds].sort()),
expiresAt: null,
generation,
capabilityDigest,
});
}
export function readerBindingRequestHash(policy) {
const canonical = JSON.stringify({
tenantId: policy.tenantId,
connectionId: policy.connectionId,
providerId: policy.providerId,
allowedDataProductIds: [...policy.allowedDataProductIds].sort(),
expiresAt: null,
generation: policy.generation,
capabilityDigest: policy.capabilityDigest,
});
return createHash("sha256").update(canonical, "utf8").digest("hex");
}
export function assertReaderProduct(binding, dataProductId, now = new Date()) {
if (!isPlainObject(binding) || binding.active !== true || new Date(binding.expiresAt) <= now) {
const expired = binding?.expiresAt !== null && new Date(binding?.expiresAt) <= now;
if (!isPlainObject(binding) || binding.active !== true || expired) {
throw readerError("reader_binding_inactive", 401);
}
const normalized = identifier(dataProductId);
@@ -49,12 +110,14 @@ export function assertReaderProduct(binding, dataProductId, now = new Date()) {
export function safeReaderBinding(binding) {
return {
id: binding.id,
...(binding.bindingKey ? { bindingKey: binding.bindingKey } : {}),
...(Number.isInteger(Number(binding.generation)) ? { generation: Number(binding.generation) } : {}),
tenantId: binding.tenantId,
connectionId: binding.connectionId,
providerId: binding.providerId,
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
active: binding.active === true,
expiresAt: new Date(binding.expiresAt).toISOString(),
expiresAt: binding.expiresAt === null ? null : new Date(binding.expiresAt).toISOString(),
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
services/external-data-plane/src/schema.mjs
+64 -3
View File
@@ -55,7 +55,7 @@ export async function migrate(pool) {
payload_ref text,
payload_bytes integer,
received_at timestamptz not null,
expires_at timestamptz not null,
expires_at timestamptz,
check (payload is not null or payload_ref is not null)
)
`);
@@ -72,7 +72,7 @@ export async function migrate(pool) {
connection_id text not null,
provider_id text not null,
allowed_data_product_ids jsonb not null,
expires_at timestamptz not null,
expires_at timestamptz,
active boolean not null default true,
created_at timestamptz not null default now(),
rotated_at timestamptz,
@@ -89,6 +89,7 @@ export async function migrate(pool) {
await pool.query("alter table external_data_plane_writer_bindings add column if not exists binding_key text");
await pool.query("alter table external_data_plane_writer_bindings add column if not exists request_hash text");
await pool.query("alter table external_data_plane_writer_bindings add column if not exists generation integer not null default 1");
await pool.query("alter table external_data_plane_writer_bindings alter column expires_at drop not null");
await pool.query(`
do $$
begin
@@ -113,6 +114,18 @@ export async function migrate(pool) {
or (binding_key is not null and request_hash ~ '^[a-f0-9]{64}$')
);
end if;
if not exists (
select 1 from pg_constraint
where conrelid = 'external_data_plane_writer_bindings'::regclass
and conname = 'external_data_plane_writer_bindings_lifecycle_ck'
) then
alter table external_data_plane_writer_bindings
add constraint external_data_plane_writer_bindings_lifecycle_ck
check (
(binding_key is null and expires_at is not null)
or (binding_key is not null and expires_at is null)
) not valid;
end if;
end
$$
`);
@@ -123,15 +136,19 @@ export async function migrate(pool) {
create table if not exists external_data_plane_reader_bindings (
id uuid primary key,
token_hash text not null unique,
binding_key text,
request_hash text,
generation integer not null default 1,
tenant_id text not null,
connection_id text not null,
provider_id text not null,
allowed_data_product_ids jsonb not null,
expires_at timestamptz not null,
expires_at timestamptz,
active boolean not null default true,
created_at timestamptz not null default now(),
rotated_at timestamptz,
revoked_at timestamptz,
check (generation > 0),
check (
case when jsonb_typeof(allowed_data_product_ids) = 'array'
then jsonb_array_length(allowed_data_product_ids) > 0
@@ -140,6 +157,50 @@ export async function migrate(pool) {
)
)
`);
await pool.query("alter table external_data_plane_reader_bindings add column if not exists binding_key text");
await pool.query("alter table external_data_plane_reader_bindings add column if not exists request_hash text");
await pool.query("alter table external_data_plane_reader_bindings add column if not exists generation integer not null default 1");
await pool.query("alter table external_data_plane_reader_bindings alter column expires_at drop not null");
await pool.query(`
do $$
begin
if not exists (
select 1 from pg_constraint
where conrelid = 'external_data_plane_reader_bindings'::regclass
and conname = 'external_data_plane_reader_bindings_generation_positive_ck'
) then
alter table external_data_plane_reader_bindings
add constraint external_data_plane_reader_bindings_generation_positive_ck
check (generation > 0);
end if;
if not exists (
select 1 from pg_constraint
where conrelid = 'external_data_plane_reader_bindings'::regclass
and conname = 'external_data_plane_reader_bindings_managed_metadata_ck'
) then
alter table external_data_plane_reader_bindings
add constraint external_data_plane_reader_bindings_managed_metadata_ck
check (
(binding_key is null and request_hash is null)
or (binding_key is not null and request_hash ~ '^[a-f0-9]{64}$')
);
end if;
if not exists (
select 1 from pg_constraint
where conrelid = 'external_data_plane_reader_bindings'::regclass
and conname = 'external_data_plane_reader_bindings_lifecycle_ck'
) then
alter table external_data_plane_reader_bindings
add constraint external_data_plane_reader_bindings_lifecycle_ck
check (
(binding_key is null and expires_at is not null)
or (binding_key is not null and expires_at is null)
);
end if;
end
$$
`);
await pool.query("create unique index if not exists external_data_plane_reader_bindings_managed_key_idx on external_data_plane_reader_bindings (binding_key, generation) where binding_key is not null");
await pool.query("create index if not exists external_data_plane_reader_bindings_active_idx on external_data_plane_reader_bindings (active, expires_at)");
await pool.query(`
services/external-data-plane/src/server.mjs
+183 -16
View File
@@ -12,6 +12,7 @@ import {
pruneBatchReceipts,
pruneDataProductHistory,
prunePatchOutbox,
readDataProductHistory,
readDataProductSnapshot,
readPatchEvents,
} from "./data-product-delivery.mjs";
@@ -26,12 +27,15 @@ import {
assertReaderProduct,
createReaderToken,
hashReaderToken,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeReaderBinding,
} from "./reader-binding.mjs";
import { migrate } from "./schema.mjs";
import {
createWriterToken,
canMigrateManagedWriterBindingToDurable,
hashWriterToken,
materializeDataProductPublish,
materializeWriterBoundBatch,
@@ -90,12 +94,16 @@ app.get("/healthz", asyncRoute(async (_req, res) => {
providerLogic: "absent",
commandTransport: "absent",
writerBindings: "supported",
managedWriterBindings: "digest+idempotent-generation",
managedWriterBindings: "digest+idempotent-generation+explicit-revoke",
readerBindings: "supported",
dataProductDelivery: "snapshot+durable-patch",
managedReaderBindings: "digest+idempotent-generation+explicit-revoke",
dataProductDelivery: "snapshot+history+durable-patch",
legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled",
legacyCapabilityProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled",
managedWriterBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled",
managedWriterBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
managedReaderBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled",
managedReaderBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
rawRetentionSweep: {
mode: "server-scheduled",
lastSweepAt: lastRetentionSweepAt,
@@ -169,9 +177,7 @@ app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, req
app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_writer_binding_key_invalid");
const policy = normalizeManagedWriterBindingRequest(req.body, {
maxTtlDays: config.writerBindingMaxTtlDays,
});
const policy = normalizeManagedWriterBindingRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
const requestHash = writerBindingRequestHash(policy);
const client = await pool.connect();
@@ -180,7 +186,7 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", request_hash as "requestHash", generation,
`select id, token_hash as "capabilityDigest", binding_key as "bindingKey", request_hash as "requestHash", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
@@ -192,12 +198,27 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
if (existing.rowCount) {
const binding = existing.rows[0];
if (binding.requestHash !== requestHash) {
throw httpError(409, "managed_writer_binding_request_conflict");
const legacyCanMigrate = canMigrateManagedWriterBindingToDurable(binding, policy);
if (!legacyCanMigrate) throw httpError(409, "managed_writer_binding_request_conflict");
const migrated = await client.query(
`update external_data_plane_writer_bindings
set request_hash = $3, expires_at = null, rotated_at = now()
where binding_key = $1 and generation = $2 and token_hash = $4
and active = true and expires_at > now()
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[bindingKey, policy.generation, requestHash, policy.capabilityDigest],
);
if (!migrated.rowCount) throw httpError(409, "managed_writer_binding_generation_inactive");
response = { status: 200, idempotent: false, migrated: true, binding: migrated.rows[0] };
} else {
if (binding.active !== true || binding.expiresAt !== null) {
throw httpError(409, "managed_writer_binding_generation_inactive");
}
response = { status: 200, idempotent: true, migrated: false, binding };
}
if (binding.active !== true || new Date(binding.expiresAt) <= new Date()) {
throw httpError(409, "managed_writer_binding_generation_inactive");
}
response = { status: 200, idempotent: true, binding };
} else {
const inserted = await client.query(
`insert into external_data_plane_writer_bindings (
@@ -237,6 +258,7 @@ app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireMan
res.status(response.status).json({
ok: true,
idempotent: response.idempotent,
migrated: response.migrated === true,
writerBinding: safeWriterBinding(response.binding),
});
}));
@@ -354,6 +376,126 @@ app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requirePro
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
}));
app.put("/internal/data-plane/v1/reader-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_reader_binding_key_invalid");
const policy = normalizeManagedReaderBindingRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
const requestHash = readerBindingRequestHash(policy);
const client = await pool.connect();
let response;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", request_hash as "requestHash", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, policy.generation],
);
if (existing.rowCount) {
const binding = existing.rows[0];
if (binding.requestHash !== requestHash) {
throw httpError(409, "managed_reader_binding_request_conflict");
}
if (binding.active !== true || binding.expiresAt !== null) {
throw httpError(409, "managed_reader_binding_generation_inactive");
}
response = { status: 200, idempotent: true, binding };
} else {
const inserted = await client.query(
`insert into external_data_plane_reader_bindings (
id, token_hash, binding_key, request_hash, generation,
tenant_id, connection_id, provider_id, allowed_data_product_ids, expires_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10)
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[
randomUUID(),
policy.capabilityDigest,
bindingKey,
requestHash,
policy.generation,
policy.tenantId,
policy.connectionId,
policy.providerId,
JSON.stringify(policy.allowedDataProductIds),
policy.expiresAt,
],
);
response = { status: 201, idempotent: false, binding: inserted.rows[0] };
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
if (error?.code === "23505") {
throw httpError(409, "managed_reader_binding_capability_digest_conflict");
}
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.status(response.status).json({
ok: true,
idempotent: response.idempotent,
readerBinding: safeReaderBinding(response.binding),
});
}));
app.post("/internal/data-plane/v1/reader-bindings/by-key/:bindingKey/generations/:generation/revoke", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_reader_binding_key_invalid");
const generation = requirePositiveInteger(req.params.generation, "managed_reader_binding_generation_invalid");
const client = await pool.connect();
let binding;
let idempotent;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, generation],
);
if (!existing.rowCount) throw httpError(404, "managed_reader_binding_not_found");
if (existing.rows[0].active === true) {
const revoked = await client.query(
`update external_data_plane_reader_bindings
set active = false, revoked_at = now()
where binding_key = $1 and generation = $2 and active = true
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[bindingKey, generation],
);
binding = revoked.rows[0];
idempotent = false;
} else {
binding = existing.rows[0];
idempotent = true;
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.json({ ok: true, idempotent, readerBinding: safeReaderBinding(binding) });
}));
app.post("/internal/data-plane/v1/reader-bindings", requireProvisionerApi, asyncRoute(async (req, res) => {
const policy = normalizeReaderBindingRequest(req.body, {
maxTtlDays: config.writerBindingMaxTtlDays,
@@ -389,7 +531,7 @@ app.post("/internal/data-plane/v1/reader-bindings/:bindingId/rotate", requirePro
const result = await pool.query(
`update external_data_plane_reader_bindings
set token_hash = $2, rotated_at = now()
where id = $1 and active = true and expires_at > now()
where id = $1 and binding_key is null and active = true and expires_at > now()
returning id, tenant_id as "tenantId", connection_id as "connectionId",
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
expires_at as "expiresAt", active, created_at as "createdAt",
@@ -405,7 +547,7 @@ app.post("/internal/data-plane/v1/reader-bindings/:bindingId/revoke", requirePro
const result = await pool.query(
`update external_data_plane_reader_bindings
set active = false, revoked_at = now()
where id = $1 and active = true
where id = $1 and binding_key is null and active = true
returning id, tenant_id as "tenantId", connection_id as "connectionId",
provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds",
expires_at as "expiresAt", active, created_at as "createdAt",
@@ -426,6 +568,22 @@ app.get("/internal/data-plane/v1/data-products/:dataProductId/snapshot", require
res.json(snapshot);
}));
app.get("/internal/data-plane/v1/data-products/:dataProductId/history", requireReaderBinding, asyncRoute(async (req, res) => {
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
const dataProductId = assertReaderProduct(req.readerBinding, req.params.dataProductId);
const definition = await loadDataProductDefinition(pool, dataProductId);
if (!definition) throw httpError(404, "data_product_not_found");
const history = await readDataProductHistory(pool, req.readerBinding, definition, {
from: req.query.from,
to: req.query.to,
resolutionMs: req.query.resolutionMs,
sourceIds: parseSourceIds(req.query.sourceIds),
limit: req.query.limit,
cursor: req.query.cursor,
});
res.json(history);
}));
app.get("/internal/data-plane/v1/data-products/:dataProductId/stream", requireReaderBinding, asyncRoute(async (req, res) => {
if (shuttingDown) throw httpError(503, "service_shutting_down");
if (hasScopeHeaders(req)) throw httpError(400, "data_product_read_scope_headers_forbidden");
@@ -767,7 +925,8 @@ async function resolveWriterBinding(req) {
expires_at as "expiresAt", active, created_at as "createdAt",
rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_writer_bindings
where token_hash = $1 and active = true and expires_at > now()`,
where token_hash = $1 and active = true
and (expires_at is null or expires_at > now())`,
[hashWriterToken(token)],
);
if (!result.rowCount) throw httpError(401, "writer_binding_unauthorized");
@@ -783,7 +942,8 @@ async function resolveReaderBinding(req) {
expires_at as "expiresAt", active, created_at as "createdAt",
rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where token_hash = $1 and active = true and expires_at > now()`,
where token_hash = $1 and active = true
and (expires_at is null or expires_at > now())`,
[hashReaderToken(token)],
);
if (!result.rowCount) throw httpError(401, "reader_binding_unauthorized");
@@ -797,7 +957,8 @@ async function assertReaderStreamAccess(binding, dataProductId) {
join external_data_plane_products as product
on product.id = $3 and product.active = true
where binding.id = $1 and binding.token_hash = $2
and binding.active = true and binding.expires_at > now()
and binding.active = true
and (binding.expires_at is null or binding.expires_at > now())
and binding.allowed_data_product_ids ? $3`,
[binding.id, binding.tokenHash, dataProductId],
);
@@ -883,6 +1044,12 @@ function parseCursor(value) {
return cursor;
}
function parseSourceIds(value) {
if (value === undefined || value === null || value === "") return [];
const values = Array.isArray(value) ? value : String(value).split(",");
return values.map((item) => String(item || "").trim()).filter(Boolean);
}
function writePatchEvent(res, event) {
return writeSseFrame(res, `id: ${event.cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify(event)}\n\n`);
}
services/external-data-plane/src/writer-binding.mjs
+32 -10
View File
@@ -72,7 +72,7 @@ export function normalizeWriterBindingRequest(value, { now = new Date(), maxTtlD
* generated and stored inside Engine; EDP receives only its SHA-256 digest.
* `generation` makes a retry address the same immutable binding generation.
*/
export function normalizeManagedWriterBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) {
export function normalizeManagedWriterBindingRequest(value) {
if (!isPlainObject(value) || !isPlainObject(value.source)) {
throw writerBindingError("managed_writer_binding_request_invalid");
}
@@ -83,11 +83,16 @@ export function normalizeManagedWriterBindingRequest(value, { now = new Date(),
throw writerBindingError("managed_writer_binding_request_fields_invalid");
}
const scope = normalizeWriterBindingRequest({
source: value.source,
allowedDataProductIds: value.allowedDataProductIds,
expiresAt: value.expiresAt,
}, { now, maxTtlDays });
const tenantId = normalizeIdentifier(value.source.tenantId);
const connectionId = normalizeIdentifier(value.source.connectionId);
const providerId = normalizeIdentifier(value.source.providerId);
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
if (!tenantId || !connectionId || !providerId || !allowedDataProductIds.length) {
throw writerBindingError("managed_writer_binding_scope_invalid");
}
if (value.expiresAt !== null) {
throw writerBindingError("managed_writer_binding_must_be_durable");
}
const generation = Number(value.generation);
if (!Number.isSafeInteger(generation) || generation < 1 || generation > 2_147_483_647) {
throw writerBindingError("managed_writer_binding_generation_invalid");
@@ -98,8 +103,11 @@ export function normalizeManagedWriterBindingRequest(value, { now = new Date(),
}
return Object.freeze({
...scope,
allowedDataProductIds: Object.freeze([...scope.allowedDataProductIds].sort()),
tenantId,
connectionId,
providerId,
allowedDataProductIds: Object.freeze([...allowedDataProductIds].sort()),
expiresAt: null,
generation,
capabilityDigest,
});
@@ -111,13 +119,26 @@ export function writerBindingRequestHash(policy) {
connectionId: policy.connectionId,
providerId: policy.providerId,
allowedDataProductIds: [...policy.allowedDataProductIds].sort(),
expiresAt: new Date(policy.expiresAt).toISOString(),
expiresAt: null,
generation: policy.generation,
capabilityDigest: policy.capabilityDigest,
});
return createHash("sha256").update(canonical, "utf8").digest("hex");
}
export function canMigrateManagedWriterBindingToDurable(binding, policy, now = new Date()) {
if (!isPlainObject(binding) || !isPlainObject(policy) || binding.active !== true) return false;
const expiresAt = new Date(binding.expiresAt);
if (binding.expiresAt === null || Number.isNaN(expiresAt.getTime()) || expiresAt <= now) return false;
return binding.tenantId === policy.tenantId
&& binding.connectionId === policy.connectionId
&& binding.providerId === policy.providerId
&& binding.capabilityDigest === policy.capabilityDigest
&& JSON.stringify(uniqueIdentifiers(binding.allowedDataProductIds).sort())
=== JSON.stringify(uniqueIdentifiers(policy.allowedDataProductIds).sort())
&& policy.expiresAt === null;
}
/**
* Converts a caller-provided, deliberately unscoped intake envelope into the
* canonical scoped form. Caller scope is rejected, never trusted or merged.
@@ -202,7 +223,7 @@ export function safeWriterBinding(binding) {
providerId: binding.providerId,
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
active: binding.active === true,
expiresAt: new Date(binding.expiresAt).toISOString(),
expiresAt: binding.expiresAt === null ? null : new Date(binding.expiresAt).toISOString(),
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
@@ -214,6 +235,7 @@ export function writerBindingError(code) {
}
function bindingIsCurrent(binding, now) {
if (binding.expiresAt === null) return true;
const expiresAt = new Date(binding.expiresAt);
return !Number.isNaN(expiresAt.getTime()) && expiresAt > now;
}
services/external-data-plane/test/api.integration.test.mjs
+51 -2
View File
@@ -5,7 +5,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises";
import net from "node:net";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import { validateDataProductHistory, validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import {
MANAGED_PROVISIONER_HEADERS,
managedProvisionerSigningPayload,
@@ -77,7 +77,7 @@ try {
const managedBody = {
source: scope,
allowedDataProductIds: [productId],
expiresAt: expiry,
expiresAt: null,
generation: 1,
capabilityDigest: createHash("sha256").update(managedToken, "utf8").digest("hex"),
};
@@ -194,6 +194,48 @@ try {
});
assert.equal(rejectedManagedGeneration1.status, 401);
const managedReaderToken = "ndc_edprb_managed_api_test_0123456789abcdefghijklmnopqrstuvwxyz";
const managedReaderBody = {
source: scope,
allowedDataProductIds: [productId],
expiresAt: null,
generation: 1,
capabilityDigest: createHash("sha256").update(managedReaderToken, "utf8").digest("hex"),
};
const managedReaderPath = "/internal/data-plane/v1/reader-bindings/by-key/engine.api-connection.positions-reader";
const managedReaderResults = await Promise.all([
rawJsonRequest(managedReaderPath, { method: "PUT", managedSignature: true, body: managedReaderBody }),
rawJsonRequest(managedReaderPath, { method: "PUT", managedSignature: true, body: managedReaderBody }),
]);
assert.deepEqual(managedReaderResults.map(({ value }) => value.idempotent).sort(), [false, true]);
assert.equal(managedReaderResults[0].value.readerBinding.id, managedReaderResults[1].value.readerBinding.id);
assert.equal("token" in managedReaderResults[0].value, false);
assert.equal(JSON.stringify(managedReaderResults[0].value).includes(managedReaderBody.capabilityDigest), false);
assert.deepEqual(
(await jsonRequest("/internal/data-plane/v1/reader/data-products", { token: managedReaderToken })).dataProducts.map((value) => value.id),
[productId],
);
const legacyManagedReaderRevoke = await fetch(
`${baseUrl}/internal/data-plane/v1/reader-bindings/${managedReaderResults[0].value.readerBinding.id}/revoke`,
{ method: "POST", headers: { Authorization: `Bearer ${provisionerSecret}` } },
);
assert.equal(legacyManagedReaderRevoke.status, 404);
const managedReaderRevoke = await jsonRequest(
`${managedReaderPath}/generations/1/revoke`,
{ method: "POST", managedSignature: true },
);
assert.equal(managedReaderRevoke.idempotent, false);
assert.equal(managedReaderRevoke.readerBinding.active, false);
const managedReaderRevokeRetry = await jsonRequest(
`${managedReaderPath}/generations/1/revoke`,
{ method: "POST", managedSignature: true },
);
assert.equal(managedReaderRevokeRetry.idempotent, true);
const rejectedManagedReader = await fetch(`${baseUrl}/internal/data-plane/v1/reader/data-products`, {
headers: { Authorization: `Bearer ${managedReaderToken}` },
});
assert.equal(rejectedManagedReader.status, 401);
const manualWriterBody = { source: scope, allowedDataProductIds: [productId], expiresAt: expiry };
const signedOnlyManualProvisioning = await signedFetch("/internal/data-plane/v1/writer-bindings", {
method: "POST",
@@ -245,6 +287,13 @@ try {
const snapshot = await jsonRequest(`/internal/data-plane/v1/data-products/${productId}/snapshot`, { token: reader.token });
assert.equal(validateDataProductSnapshot(snapshot).ok, true);
assert.equal(snapshot.facts.length, 1);
const history = await jsonRequest(
`/internal/data-plane/v1/data-products/${productId}/history?from=2026-07-15T12%3A00%3A00.000Z&to=2026-07-15T12%3A02%3A00.000Z&resolutionMs=60000&sourceIds=unit-01&limit=100`,
{ token: reader.token },
);
assert.equal(validateDataProductHistory(history).ok, true);
assert.equal(history.facts.length, 1);
assert.equal(history.facts[0].bucketStart, "2026-07-15T12:00:00.000Z");
const controller = new AbortController();
const response = await fetch(`${baseUrl}/internal/data-plane/v1/data-products/${productId}/stream?after=${snapshot.cursor}`, {
services/external-data-plane/test/data-product-delivery.integration.test.mjs
+43 -1
View File
@@ -1,10 +1,11 @@
import assert from "node:assert/strict";
import { Pool } from "pg";
import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import { validateDataProductHistory, validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane";
import {
loadDataProductDefinition,
persistDataProductDefinition,
persistDataProductPublish,
readDataProductHistory,
readDataProductSnapshot,
readPatchEvents,
} from "../src/data-product-delivery.mjs";
@@ -154,6 +155,47 @@ try {
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:01:30.000Z"), true);
assert.equal(unitOneHistory.some((row) => new Date(row.observed_at).toISOString() === "2026-07-15T10:02:00.000Z"), true);
const historyPageOne = await readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-01"],
limit: 2,
});
assert.equal(validateDataProductHistory(historyPageOne).ok, true);
assert.equal(historyPageOne.facts.length, 2);
assert.equal(typeof historyPageOne.nextCursor, "string");
const historyPageTwo = await readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-01"],
limit: 2,
cursor: historyPageOne.nextCursor,
});
assert.equal(validateDataProductHistory(historyPageTwo).ok, true);
assert.equal(historyPageTwo.facts.length, 1);
assert.equal(historyPageTwo.nextCursor, undefined);
assert.deepEqual(
[...historyPageOne.facts, ...historyPageTwo.facts].map((value) => value.bucketStart),
[
"2026-07-15T10:00:00.000Z",
"2026-07-15T10:01:00.000Z",
"2026-07-15T10:02:00.000Z",
],
);
await assert.rejects(
readDataProductHistory(pool, binding, storedDefinition, {
from: "2026-07-15T10:00:00.000Z",
to: "2026-07-15T10:04:00.000Z",
resolutionMs: 60_000,
sourceIds: ["unit-02"],
limit: 2,
cursor: historyPageOne.nextCursor,
}),
(error) => error?.status === 400 && error?.code === "data_product_history_cursor_invalid",
);
console.log("external-data-plane delivery integration: ok");
} finally {
await pool.end();
services/external-data-plane/test/reader-binding.test.mjs
+42
View File
@@ -3,7 +3,9 @@ import {
assertReaderProduct,
createReaderToken,
hashReaderToken,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeReaderBinding,
} from "../src/reader-binding.mjs";
@@ -22,4 +24,44 @@ const safe = safeReaderBinding({ id: "reader-01", ...policy, active: true, token
assert.equal("token" in safe, false);
assert.equal("tokenHash" in safe, false);
const managed = normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: ["fleet.positions.current.v1"],
expiresAt: null,
generation: 2,
capabilityDigest: hashReaderToken(token),
}, { now });
assert.equal(managed.generation, 2);
assert.match(readerBindingRequestHash(managed), /^[a-f0-9]{64}$/);
const safeManaged = safeReaderBinding({
id: "reader-managed-01",
bindingKey: "dprg-reader-managed",
generation: 2,
...managed,
active: true,
});
assert.equal(safeManaged.bindingKey, "dprg-reader-managed");
assert.equal(safeManaged.generation, 2);
assert.equal(safeManaged.expiresAt, null);
assert.equal(
assertReaderProduct(safeManaged, "fleet.positions.current.v1", new Date("2036-07-15T12:00:00.000Z")),
"fleet.positions.current.v1",
);
assert.equal(JSON.stringify(safeManaged).includes(managed.capabilityDigest), false);
assert.throws(() => normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: managed.allowedDataProductIds,
expiresAt: managed.expiresAt,
generation: managed.generation,
capabilityDigest: managed.capabilityDigest,
token: "forbidden",
}, { now }), /managed_reader_binding_secret_material_forbidden/);
assert.throws(() => normalizeManagedReaderBindingRequest({
source: { tenantId: policy.tenantId, connectionId: policy.connectionId, providerId: policy.providerId },
allowedDataProductIds: managed.allowedDataProductIds,
expiresAt: policy.expiresAt,
generation: managed.generation,
capabilityDigest: managed.capabilityDigest,
}), /managed_reader_binding_must_be_durable/);
console.log("external-data-plane reader bindings: ok");
services/external-data-plane/test/writer-binding.test.mjs
+39
View File
@@ -1,6 +1,7 @@
import assert from "node:assert/strict";
import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/data-plane.mjs";
import {
canMigrateManagedWriterBindingToDurable,
createWriterToken,
hashWriterToken,
materializeDataProductPublish,
@@ -38,6 +39,7 @@ assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`));
const managedPolicy = normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
allowedDataProductIds: ["fleet.positions.current.v1", "asset.status.current.v1"],
generation: 1,
capabilityDigest: hashWriterToken(token),
@@ -48,26 +50,47 @@ assert.deepEqual(managedPolicy.allowedDataProductIds, [
]);
assert.equal(managedPolicy.generation, 1);
assert.equal(managedPolicy.capabilityDigest, hashWriterToken(token));
assert.equal(managedPolicy.expiresAt, null);
assert.equal(canMigrateManagedWriterBindingToDurable({
...managedPolicy,
expiresAt: "2026-08-01T12:00:00.000Z",
active: true,
capabilityDigest: managedPolicy.capabilityDigest,
}, managedPolicy, now), true);
assert.equal(canMigrateManagedWriterBindingToDurable({
...managedPolicy,
expiresAt: "2026-07-01T12:00:00.000Z",
active: true,
capabilityDigest: managedPolicy.capabilityDigest,
}, managedPolicy, now), false);
assert.equal(writerBindingRequestHash(managedPolicy), writerBindingRequestHash({
...managedPolicy,
allowedDataProductIds: [...managedPolicy.allowedDataProductIds].reverse(),
}));
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 0,
capabilityDigest: hashWriterToken(token),
}, { now, maxTtlDays: 90 }), /managed_writer_binding_generation_invalid/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 1,
capabilityDigest: "not-a-digest",
}, { now, maxTtlDays: 90 }), /managed_writer_binding_capability_digest_invalid/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
expiresAt: null,
generation: 1,
capabilityDigest: hashWriterToken(token),
token,
}, { now, maxTtlDays: 90 }), /managed_writer_binding_secret_material_forbidden/);
assert.throws(() => normalizeManagedWriterBindingRequest({
...request,
generation: 1,
capabilityDigest: hashWriterToken(token),
}), /managed_writer_binding_must_be_durable/);
const unscopedBatch = {
schemaVersion: "nodedc.external-provider-contract/v1",
@@ -148,5 +171,21 @@ const safeBinding = safeWriterBinding({
});
assert.equal("token" in safeBinding, false);
assert.equal("tokenHash" in safeBinding, false);
const safeManagedBinding = safeWriterBinding({
id: "binding-managed-01",
...managedPolicy,
active: true,
createdAt: now,
});
assert.equal(safeManagedBinding.expiresAt, null);
assert.equal(materializeDataProductPublish({
schemaVersion: "nodedc.data-product.publish/v1",
batch: { runId: "run-durable", sequence: 0, idempotencyKey: "run-durable.batch-0" },
facts: unscopedBatch.facts,
}, safeManagedBinding, {
id: "fleet.positions.current.v1",
version: "1.0.0",
ontologyRevision: "ontology.example-fleet.v1",
}, "fleet.positions.current.v1", { now: new Date("2036-07-15T12:00:00.000Z") }).batch.runId, "run-durable");
console.log("external-data-plane writer bindings: ok");