diff --git a/services/external-data-plane/Dockerfile b/services/external-data-plane/Dockerfile index 0670a12..53805af 100644 --- a/services/external-data-plane/Dockerfile +++ b/services/external-data-plane/Dockerfile @@ -2,7 +2,12 @@ FROM node:20-alpine AS deps WORKDIR /workspace -COPY packages/external-provider-contract ./packages/external-provider-contract +COPY packages/external-provider-contract/package.json ./packages/external-provider-contract/package.json +COPY packages/external-provider-contract/src/contract-version.mjs ./packages/external-provider-contract/src/contract-version.mjs +COPY packages/external-provider-contract/src/data-plane.mjs ./packages/external-provider-contract/src/data-plane.mjs +COPY packages/external-provider-contract/src/data-product.mjs ./packages/external-provider-contract/src/data-product.mjs +COPY packages/external-provider-contract/src/intake-batch.mjs ./packages/external-provider-contract/src/intake-batch.mjs +COPY packages/external-provider-contract/src/sensitive-field-policy.mjs ./packages/external-provider-contract/src/sensitive-field-policy.mjs COPY services/external-data-plane/package.json services/external-data-plane/package-lock.json ./services/external-data-plane/ WORKDIR /workspace/services/external-data-plane @@ -21,8 +26,8 @@ COPY --from=deps /workspace/packages/external-provider-contract /packages/extern COPY services/external-data-plane/src ./src COPY services/external-data-plane/definitions ./definitions -# A dedicated numeric identity can read only the EDP provisioning secret mount; -# it is intentionally not the shared Node/Map Gateway uid/gid (1000). +# A dedicated numeric identity owns the legacy EDP bearer boundary and reads the +# separate public Engine trust mount; it is not the shared service uid/gid. RUN addgroup -S -g 11006 nodedc-edp && adduser -S -D -H -u 11006 -G nodedc-edp nodedc-edp USER 11006:11006 diff --git a/services/external-data-plane/package.json b/services/external-data-plane/package.json index 6d6aa36..9127d8f 100644 --- a/services/external-data-plane/package.json +++ b/services/external-data-plane/package.json @@ -6,9 +6,10 @@ "scripts": { "start": "node src/server.mjs", "dev": "node --watch src/server.mjs", - "check": "node --check src/server.mjs && node --check src/schema.mjs && node --check src/config.mjs && node --check src/intake-policy.mjs && node --check src/writer-binding.mjs && node --check src/reader-binding.mjs && node --check src/data-product-policy.mjs && node --check src/data-product-delivery.mjs && node --check src/definitions.mjs", - "test": "node test/config.test.mjs && node test/intake-policy.test.mjs && node test/writer-binding.test.mjs && node test/reader-binding.test.mjs && node test/data-product-policy.test.mjs && node test/definitions.test.mjs", - "test:integration": "node test/data-product-delivery.integration.test.mjs && node test/api.integration.test.mjs" + "check": "node --check src/server.mjs && node --check src/schema.mjs && node --check src/config.mjs && node --check src/intake-policy.mjs && node --check src/writer-binding.mjs && node --check src/reader-binding.mjs && node --check src/data-product-policy.mjs && node --check src/data-product-delivery.mjs && node --check src/definitions.mjs && node --check src/managed-provisioner-auth.mjs", + "test": "node test/config.test.mjs && node test/managed-provisioner-auth.test.mjs && node test/intake-policy.test.mjs && node test/writer-binding.test.mjs && node test/reader-binding.test.mjs && node test/data-product-policy.test.mjs && node test/definitions.test.mjs", + "test:integration": "node test/data-product-delivery.integration.test.mjs && node test/api.integration.test.mjs", + "test:all": "npm test && npm run test:integration" }, "dependencies": { "@nodedc/external-provider-contract": "file:../../packages/external-provider-contract", diff --git a/services/external-data-plane/src/config.mjs b/services/external-data-plane/src/config.mjs index 7185cc4..b155bf0 100644 --- a/services/external-data-plane/src/config.mjs +++ b/services/external-data-plane/src/config.mjs @@ -1,14 +1,54 @@ import { readFileSync } from "node:fs"; +import { + loadManagedProvisionerPublicKeyFile, + validateManagedProvisionerAudience, + validateManagedProvisionerIdentity, +} from "./managed-provisioner-auth.mjs"; export function readConfig(env = process.env) { const provisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED, false); + const managedProvisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED, false); + const managedProvisionerMaxSkewSeconds = integer( + env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS, + 60, + 5, + 300, + ); const config = { port: integer(env.PORT, 18106, 1, 65535), databaseUrl: required(env.EXTERNAL_DATA_PLANE_DATABASE_URL, "EXTERNAL_DATA_PLANE_DATABASE_URL"), databasePoolSize: integer(env.EXTERNAL_DATA_PLANE_DATABASE_POOL_SIZE, 10, 1, 50), internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN), provisionerApiEnabled, + managedProvisionerApiEnabled, provisionerAccessToken: provisionerApiEnabled ? secretFile(env.EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE) : "", + managedProvisionerPublicKey: managedProvisionerApiEnabled + ? loadManagedProvisionerPublicKeyFile(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE) + : null, + managedProvisionerServiceId: managedProvisionerApiEnabled + ? validateManagedProvisionerIdentity( + required(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID, "EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID"), + "service_id", + ) + : "", + managedProvisionerKeyId: managedProvisionerApiEnabled + ? validateManagedProvisionerIdentity( + required(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID, "EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID"), + "key_id", + ) + : "", + managedProvisionerAudience: managedProvisionerApiEnabled + ? validateManagedProvisionerAudience( + required(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE, "EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE"), + ) + : "", + managedProvisionerMaxSkewMs: managedProvisionerMaxSkewSeconds * 1000, + managedProvisionerReplayCacheMaxEntries: integer( + env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES, + 10_000, + 100, + 100_000, + ), rawRetentionDays: integer(env.EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS, 14, 1, 3650), maxBatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES, 5 * 1024 * 1024, 1024, 50 * 1024 * 1024), maxFactsPerPublish: integer(env.EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH, 5000, 1, 100_000), diff --git a/services/external-data-plane/src/data-product-delivery.mjs b/services/external-data-plane/src/data-product-delivery.mjs index 159669c..646d31e 100644 --- a/services/external-data-plane/src/data-product-delivery.mjs +++ b/services/external-data-plane/src/data-product-delivery.mjs @@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto"; import { DATA_PRODUCT_PATCH_SCHEMA_VERSION, DATA_PRODUCT_SNAPSHOT_SCHEMA_VERSION, -} from "@nodedc/external-provider-contract"; +} from "@nodedc/external-provider-contract/data-plane"; export async function loadDataProductDefinition(db, dataProductId, { activeOnly = true } = {}) { const result = await db.query( diff --git a/services/external-data-plane/src/managed-provisioner-auth.mjs b/services/external-data-plane/src/managed-provisioner-auth.mjs new file mode 100644 index 0000000..ba5dfcf --- /dev/null +++ b/services/external-data-plane/src/managed-provisioner-auth.mjs @@ -0,0 +1,318 @@ +import { createHash, createPublicKey, timingSafeEqual, verify as verifySignature } from "node:crypto"; +import { closeSync, constants as fsConstants, fstatSync, lstatSync, openSync, readFileSync } from "node:fs"; +import { isAbsolute } from "node:path"; + +export const MANAGED_PROVISIONER_SIGNATURE_SCHEMA = "nodedc.external-data-plane.managed-provisioner-request/v1"; + +export const MANAGED_PROVISIONER_HEADERS = Object.freeze({ + serviceId: "x-nodedc-engine-service-id", + keyId: "x-nodedc-engine-key-id", + audience: "x-nodedc-request-audience", + timestamp: "x-nodedc-request-timestamp", + nonce: "x-nodedc-request-nonce", + bodySha256: "x-nodedc-content-sha256", + signature: "x-nodedc-request-signature", +}); + +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const BASE64URL = /^[A-Za-z0-9_-]+$/; +const IDENTITY = /^[a-z][a-z0-9._:-]{2,127}$/i; +const AUDIENCE = /^[a-z][a-z0-9._:/-]{2,255}$/i; + +export function loadManagedProvisionerPublicKeyFile(pathValue) { + const path = String(pathValue ?? "").trim(); + if (!path) throw configError("external_data_plane_managed_provisioner_public_key_file_required"); + if (!isAbsolute(path)) throw configError("external_data_plane_managed_provisioner_public_key_file_must_be_absolute"); + + let stat; + try { + stat = lstatSync(path); + } catch { + throw configError("external_data_plane_managed_provisioner_public_key_file_unreadable"); + } + if (!stat.isFile() || stat.isSymbolicLink()) { + throw configError("external_data_plane_managed_provisioner_public_key_file_must_be_regular"); + } + if ((stat.mode & 0o022) !== 0) { + throw configError("external_data_plane_managed_provisioner_public_key_file_permissions_invalid"); + } + if (stat.size < 1 || stat.size > 8192) { + throw configError("external_data_plane_managed_provisioner_public_key_file_size_invalid"); + } + + let descriptor; + try { + descriptor = openSync(path, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW); + } catch { + throw configError("external_data_plane_managed_provisioner_public_key_file_unreadable"); + } + let pem; + try { + const openedStat = fstatSync(descriptor); + if (!openedStat.isFile()) { + throw configError("external_data_plane_managed_provisioner_public_key_file_must_be_regular"); + } + if ((openedStat.mode & 0o022) !== 0) { + throw configError("external_data_plane_managed_provisioner_public_key_file_permissions_invalid"); + } + if (openedStat.size < 1 || openedStat.size > 8192) { + throw configError("external_data_plane_managed_provisioner_public_key_file_size_invalid"); + } + pem = readFileSync(descriptor, "utf8"); + } catch (error) { + if (String(error?.code || "").startsWith("external_data_plane_managed_provisioner_")) throw error; + throw configError("external_data_plane_managed_provisioner_public_key_file_unreadable"); + } finally { + closeSync(descriptor); + } + const normalizedPem = pem.replace(/\r\n/g, "\n").trim(); + if (!/^-----BEGIN PUBLIC KEY-----\n[A-Za-z0-9+/=\n]+\n-----END PUBLIC KEY-----$/.test(normalizedPem)) { + throw configError("external_data_plane_managed_provisioner_public_key_file_format_invalid"); + } + + let publicKey; + try { + publicKey = createPublicKey(normalizedPem); + } catch { + throw configError("external_data_plane_managed_provisioner_public_key_file_format_invalid"); + } + if (publicKey.type !== "public" || publicKey.asymmetricKeyType !== "ed25519") { + throw configError("external_data_plane_managed_provisioner_public_key_algorithm_invalid"); + } + return publicKey; +} + +export function validateManagedProvisionerIdentity(value, name) { + const normalized = String(value ?? "").trim(); + if (!IDENTITY.test(normalized)) throw configError(`external_data_plane_managed_provisioner_${name}_invalid`); + return normalized; +} + +export function validateManagedProvisionerAudience(value) { + const normalized = String(value ?? "").trim(); + if (!AUDIENCE.test(normalized)) throw configError("external_data_plane_managed_provisioner_audience_invalid"); + return normalized; +} + +export function managedProvisionerSigningPayload({ + audience, + serviceId, + keyId, + method, + path, + timestamp, + nonce, + bodySha256, +}) { + return JSON.stringify({ + schemaVersion: MANAGED_PROVISIONER_SIGNATURE_SCHEMA, + audience, + serviceId, + keyId, + method, + path, + timestamp, + nonce, + bodySha256, + }); +} + +export function sha256RawBody(rawBody) { + if (!Buffer.isBuffer(rawBody)) throw new TypeError("raw_body_buffer_required"); + return createHash("sha256").update(rawBody).digest("hex"); +} + +export class ManagedProvisionerReplayCache { + constructor({ maxEntries }) { + if (!Number.isInteger(maxEntries) || maxEntries < 1) throw new TypeError("replay_cache_max_entries_invalid"); + this.maxEntries = maxEntries; + this.entries = new Map(); + } + + consume(key, expiresAt, now = Date.now()) { + this.prune(now); + const existing = this.entries.get(key); + if (existing !== undefined && existing > now) { + throw authError(409, "managed_provisioner_request_replayed"); + } + if (this.entries.size >= this.maxEntries) { + throw authError(503, "managed_provisioner_replay_cache_exhausted"); + } + this.entries.set(key, expiresAt); + } + + prune(now = Date.now()) { + for (const [key, expiresAt] of this.entries) { + if (expiresAt <= now) this.entries.delete(key); + } + } +} + +export function createManagedProvisionerRequestVerifier({ + publicKey, + serviceId, + keyId, + audience, + maxSkewMs, + replayCache, + now = Date.now, +}) { + if (publicKey?.type !== "public" || publicKey?.asymmetricKeyType !== "ed25519") { + throw configError("external_data_plane_managed_provisioner_public_key_algorithm_invalid"); + } + if (!IDENTITY.test(serviceId)) throw configError("external_data_plane_managed_provisioner_service_id_invalid"); + if (!IDENTITY.test(keyId)) throw configError("external_data_plane_managed_provisioner_key_id_invalid"); + if (!AUDIENCE.test(audience)) throw configError("external_data_plane_managed_provisioner_audience_invalid"); + if (!Number.isInteger(maxSkewMs) || maxSkewMs < 1) { + throw configError("external_data_plane_managed_provisioner_max_skew_invalid"); + } + if (!(replayCache instanceof ManagedProvisionerReplayCache)) { + throw configError("external_data_plane_managed_provisioner_replay_cache_invalid"); + } + + return function verifyManagedProvisionerRequest(req) { + const requestServiceId = exactHeader(req, MANAGED_PROVISIONER_HEADERS.serviceId); + const requestKeyId = exactHeader(req, MANAGED_PROVISIONER_HEADERS.keyId); + const requestAudience = exactHeader(req, MANAGED_PROVISIONER_HEADERS.audience); + const timestamp = exactHeader(req, MANAGED_PROVISIONER_HEADERS.timestamp); + const nonce = exactHeader(req, MANAGED_PROVISIONER_HEADERS.nonce); + const declaredBodySha256 = exactHeader(req, MANAGED_PROVISIONER_HEADERS.bodySha256); + const encodedSignature = exactHeader(req, MANAGED_PROVISIONER_HEADERS.signature); + if (hasHeader(req, "authorization")) { + throw authError(401, "managed_provisioner_authorization_header_forbidden"); + } + + if (!safeEqualText(requestServiceId, serviceId) + || !safeEqualText(requestKeyId, keyId) + || !safeEqualText(requestAudience, audience)) { + throw authError(401, "managed_provisioner_identity_mismatch"); + } + + const timestampMs = parseTimestamp(timestamp); + const verificationTime = Number(now()); + if (!Number.isFinite(verificationTime) || Math.abs(verificationTime - timestampMs) >= maxSkewMs) { + throw authError(401, "managed_provisioner_timestamp_outside_window"); + } + const nonceBytes = decodeCanonicalBase64Url(nonce, null, "managed_provisioner_nonce_invalid"); + if (nonceBytes.length < 16 || nonceBytes.length > 96) throw authError(401, "managed_provisioner_nonce_invalid"); + if (!/^[a-f0-9]{64}$/.test(declaredBodySha256)) { + throw authError(401, "managed_provisioner_body_hash_invalid"); + } + + const method = exactMethod(req); + const path = exactPath(req); + const rawBody = exactRawBody(req); + const actualBodySha256 = sha256RawBody(rawBody); + if (!safeEqualText(declaredBodySha256, actualBodySha256)) { + throw authError(401, "managed_provisioner_body_hash_mismatch"); + } + + const signature = decodeCanonicalBase64Url(encodedSignature, 64, "managed_provisioner_signature_invalid"); + const payload = managedProvisionerSigningPayload({ + audience: requestAudience, + serviceId: requestServiceId, + keyId: requestKeyId, + method, + path, + timestamp, + nonce, + bodySha256: declaredBodySha256, + }); + if (!verifySignature(null, Buffer.from(payload, "utf8"), publicKey, signature)) { + throw authError(401, "managed_provisioner_signature_invalid"); + } + + const replayKey = `${requestServiceId}\0${requestKeyId}\0${nonce}`; + replayCache.consume(replayKey, timestampMs + maxSkewMs, verificationTime); + return Object.freeze({ serviceId: requestServiceId, keyId: requestKeyId }); + }; +} + +function exactHeader(req, name) { + const rawHeaders = req?.rawHeaders; + if (Array.isArray(rawHeaders)) { + const values = []; + for (let index = 0; index < rawHeaders.length; index += 2) { + if (String(rawHeaders[index]).toLowerCase() === name) values.push(rawHeaders[index + 1]); + } + if (values.length !== 1 || typeof values[0] !== "string" || !values[0]) { + throw authError(401, "managed_provisioner_header_invalid"); + } + return values[0]; + } + + const value = req?.headers?.[name]; + if (typeof value !== "string" || !value) throw authError(401, "managed_provisioner_header_invalid"); + return value; +} + +function hasHeader(req, name) { + if (Array.isArray(req?.rawHeaders)) { + for (let index = 0; index < req.rawHeaders.length; index += 2) { + if (String(req.rawHeaders[index]).toLowerCase() === name) return true; + } + return false; + } + return Object.hasOwn(req?.headers || {}, name); +} + +function exactMethod(req) { + const method = String(req?.method ?? ""); + if (!/^[A-Z]{3,10}$/.test(method)) throw authError(401, "managed_provisioner_method_invalid"); + return method; +} + +function exactPath(req) { + const path = String(req?.originalUrl ?? ""); + if (!path.startsWith("/") || path.length > 2048 || /[\u0000-\u0020#]/.test(path)) { + throw authError(401, "managed_provisioner_path_invalid"); + } + return path; +} + +function exactRawBody(req) { + if (req?.rawBodyCaptured === true && Buffer.isBuffer(req.rawBody)) return req.rawBody; + const contentLength = req?.headers?.["content-length"]; + const transferEncoding = req?.headers?.["transfer-encoding"]; + if ((contentLength === undefined || contentLength === "0") && transferEncoding === undefined) { + return Buffer.alloc(0); + } + throw authError(401, "managed_provisioner_raw_body_unavailable"); +} + +function parseTimestamp(value) { + if (!ISO_TIMESTAMP.test(value)) throw authError(401, "managed_provisioner_timestamp_invalid"); + const timestampMs = Date.parse(value); + if (!Number.isFinite(timestampMs) || new Date(timestampMs).toISOString() !== value) { + throw authError(401, "managed_provisioner_timestamp_invalid"); + } + return timestampMs; +} + +function decodeCanonicalBase64Url(value, expectedBytes, errorCode) { + if (!BASE64URL.test(value)) throw authError(401, errorCode); + let decoded; + try { + decoded = Buffer.from(value, "base64url"); + } catch { + throw authError(401, errorCode); + } + if ((expectedBytes !== null && decoded.length !== expectedBytes) || decoded.toString("base64url") !== value) { + throw authError(401, errorCode); + } + return decoded; +} + +function safeEqualText(left, right) { + const leftBuffer = Buffer.from(String(left), "utf8"); + const rightBuffer = Buffer.from(String(right), "utf8"); + return leftBuffer.length === rightBuffer.length && timingSafeEqual(leftBuffer, rightBuffer); +} + +function authError(status, code) { + return Object.assign(new Error(code), { status, code }); +} + +function configError(code) { + return Object.assign(new Error(code), { code }); +} diff --git a/services/external-data-plane/src/schema.mjs b/services/external-data-plane/src/schema.mjs index adc880b..17d715d 100644 --- a/services/external-data-plane/src/schema.mjs +++ b/services/external-data-plane/src/schema.mjs @@ -65,6 +65,9 @@ export async function migrate(pool) { create table if not exists external_data_plane_writer_bindings ( id uuid primary key, token_hash text not null unique, + binding_key text, + request_hash text, + generation integer not null default 1, tenant_id text not null, connection_id text not null, provider_id text not null, @@ -74,6 +77,7 @@ export async function migrate(pool) { created_at timestamptz not null default now(), rotated_at timestamptz, revoked_at timestamptz, + check (generation > 0), check ( case when jsonb_typeof(allowed_data_product_ids) = 'array' then jsonb_array_length(allowed_data_product_ids) > 0 @@ -82,6 +86,37 @@ export async function migrate(pool) { ) ) `); + await pool.query("alter table external_data_plane_writer_bindings add column if not exists binding_key text"); + await pool.query("alter table external_data_plane_writer_bindings add column if not exists request_hash text"); + await pool.query("alter table external_data_plane_writer_bindings add column if not exists generation integer not null default 1"); + await pool.query(` + do $$ + begin + if not exists ( + select 1 from pg_constraint + where conrelid = 'external_data_plane_writer_bindings'::regclass + and conname = 'external_data_plane_writer_bindings_generation_positive_ck' + ) then + alter table external_data_plane_writer_bindings + add constraint external_data_plane_writer_bindings_generation_positive_ck + check (generation > 0); + end if; + if not exists ( + select 1 from pg_constraint + where conrelid = 'external_data_plane_writer_bindings'::regclass + and conname = 'external_data_plane_writer_bindings_managed_metadata_ck' + ) then + alter table external_data_plane_writer_bindings + add constraint external_data_plane_writer_bindings_managed_metadata_ck + check ( + (binding_key is null and request_hash is null) + or (binding_key is not null and request_hash ~ '^[a-f0-9]{64}$') + ); + end if; + end + $$ + `); + await pool.query("create unique index if not exists external_data_plane_writer_bindings_managed_key_idx on external_data_plane_writer_bindings (binding_key, generation) where binding_key is not null"); await pool.query("create index if not exists external_data_plane_writer_bindings_active_idx on external_data_plane_writer_bindings (active, expires_at)"); await pool.query(` diff --git a/services/external-data-plane/src/server.mjs b/services/external-data-plane/src/server.mjs index ba3b8ef..d0fa260 100644 --- a/services/external-data-plane/src/server.mjs +++ b/services/external-data-plane/src/server.mjs @@ -2,7 +2,7 @@ import express from "express"; import { createHash, randomUUID, timingSafeEqual } from "node:crypto"; import { createServer } from "node:http"; import { Pool } from "pg"; -import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract"; +import { validateDataProductPublish, validateIntakeBatch } from "@nodedc/external-provider-contract/data-plane"; import { readConfig } from "./config.mjs"; import { loadDataProductDefinition, @@ -18,6 +18,10 @@ import { import { normalizeDataProductDefinition, safeDataProductDefinition } from "./data-product-policy.mjs"; import { reconcileDataProductDefinitions } from "./definitions.mjs"; import { assertBatchTimeBounds, rawRetentionExpiry } from "./intake-policy.mjs"; +import { + createManagedProvisionerRequestVerifier, + ManagedProvisionerReplayCache, +} from "./managed-provisioner-auth.mjs"; import { assertReaderProduct, createReaderToken, @@ -31,11 +35,26 @@ import { hashWriterToken, materializeDataProductPublish, materializeWriterBoundBatch, + normalizeManagedWriterBindingRequest, normalizeWriterBindingRequest, safeWriterBinding, + writerBindingRequestHash, } from "./writer-binding.mjs"; const config = readConfig(); +const managedProvisionerReplayCache = config.managedProvisionerApiEnabled + ? new ManagedProvisionerReplayCache({ maxEntries: config.managedProvisionerReplayCacheMaxEntries }) + : null; +const verifyManagedProvisionerRequest = config.managedProvisionerApiEnabled + ? createManagedProvisionerRequestVerifier({ + publicKey: config.managedProvisionerPublicKey, + serviceId: config.managedProvisionerServiceId, + keyId: config.managedProvisionerKeyId, + audience: config.managedProvisionerAudience, + maxSkewMs: config.managedProvisionerMaxSkewMs, + replayCache: managedProvisionerReplayCache, + }) + : null; const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize }); const app = express(); const httpServer = createServer(app); @@ -47,7 +66,19 @@ let shuttingDown = false; let shutdownPromise = null; app.disable("x-powered-by"); -app.use(express.json({ limit: config.maxBatchBytes })); +app.use((req, _res, next) => { + req.rawBody = Buffer.alloc(0); + req.rawBodyCaptured = false; + next(); +}); +app.use(express.json({ + limit: config.maxBatchBytes, + inflate: false, + verify(req, _res, buffer) { + req.rawBody = Buffer.from(buffer); + req.rawBodyCaptured = true; + }, +})); app.get("/healthz", asyncRoute(async (_req, res) => { await pool.query("select 1"); @@ -59,10 +90,12 @@ app.get("/healthz", asyncRoute(async (_req, res) => { providerLogic: "absent", commandTransport: "absent", writerBindings: "supported", + managedWriterBindings: "digest+idempotent-generation", readerBindings: "supported", dataProductDelivery: "snapshot+durable-patch", legacyIntake: config.legacyIntakeEnabled ? "migration-only" : "disabled", - writerBindingProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled", + legacyCapabilityProvisioning: config.provisionerApiEnabled ? "enabled" : "disabled", + managedWriterBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled", rawRetentionSweep: { mode: "server-scheduled", lastSweepAt: lastRetentionSweepAt, @@ -134,6 +167,128 @@ app.post("/internal/data-plane/v1/intake/writer-bound", requireLegacyIntake, req res.status(result.idempotent ? 200 : 201).json({ ok: true, ...result }); })); +app.put("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => { + const bindingKey = requireIdentifier(req.params.bindingKey, "managed_writer_binding_key_invalid"); + const policy = normalizeManagedWriterBindingRequest(req.body, { + maxTtlDays: config.writerBindingMaxTtlDays, + }); + await assertRegisteredProductIds(policy.allowedDataProductIds); + const requestHash = writerBindingRequestHash(policy); + const client = await pool.connect(); + let response; + try { + await client.query("begin"); + await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]); + const existing = await client.query( + `select id, binding_key as "bindingKey", request_hash as "requestHash", generation, + tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", + allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", + active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt" + from external_data_plane_writer_bindings + where binding_key = $1 and generation = $2 + for update`, + [bindingKey, policy.generation], + ); + if (existing.rowCount) { + const binding = existing.rows[0]; + if (binding.requestHash !== requestHash) { + throw httpError(409, "managed_writer_binding_request_conflict"); + } + if (binding.active !== true || new Date(binding.expiresAt) <= new Date()) { + throw httpError(409, "managed_writer_binding_generation_inactive"); + } + response = { status: 200, idempotent: true, binding }; + } else { + const inserted = await client.query( + `insert into external_data_plane_writer_bindings ( + id, token_hash, binding_key, request_hash, generation, + tenant_id, connection_id, provider_id, allowed_data_product_ids, expires_at + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10) + returning id, binding_key as "bindingKey", generation, + tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", + allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", + active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [ + randomUUID(), + policy.capabilityDigest, + bindingKey, + requestHash, + policy.generation, + policy.tenantId, + policy.connectionId, + policy.providerId, + JSON.stringify(policy.allowedDataProductIds), + policy.expiresAt, + ], + ); + response = { status: 201, idempotent: false, binding: inserted.rows[0] }; + } + await client.query("commit"); + } catch (error) { + await client.query("rollback"); + if (error?.code === "23505") { + throw httpError(409, "managed_writer_binding_capability_digest_conflict"); + } + throw error; + } finally { + client.release(); + } + res.set("Cache-Control", "no-store, max-age=0"); + res.status(response.status).json({ + ok: true, + idempotent: response.idempotent, + writerBinding: safeWriterBinding(response.binding), + }); +})); + +app.post("/internal/data-plane/v1/writer-bindings/by-key/:bindingKey/generations/:generation/revoke", requireManagedProvisionerApi, asyncRoute(async (req, res) => { + const bindingKey = requireIdentifier(req.params.bindingKey, "managed_writer_binding_key_invalid"); + const generation = requirePositiveInteger(req.params.generation, "managed_writer_binding_generation_invalid"); + const client = await pool.connect(); + let binding; + let idempotent; + try { + await client.query("begin"); + await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]); + const existing = await client.query( + `select id, binding_key as "bindingKey", generation, + tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", + allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", + active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt" + from external_data_plane_writer_bindings + where binding_key = $1 and generation = $2 + for update`, + [bindingKey, generation], + ); + if (!existing.rowCount) throw httpError(404, "managed_writer_binding_not_found"); + if (existing.rows[0].active === true) { + const revoked = await client.query( + `update external_data_plane_writer_bindings + set active = false, revoked_at = now() + where binding_key = $1 and generation = $2 and active = true + returning id, binding_key as "bindingKey", generation, + tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", + allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", + active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`, + [bindingKey, generation], + ); + binding = revoked.rows[0]; + idempotent = false; + } else { + binding = existing.rows[0]; + idempotent = true; + } + await client.query("commit"); + } catch (error) { + await client.query("rollback"); + throw error; + } finally { + client.release(); + } + res.set("Cache-Control", "no-store, max-age=0"); + res.json({ ok: true, idempotent, writerBinding: safeWriterBinding(binding) }); +})); + app.post("/internal/data-plane/v1/writer-bindings", requireProvisionerApi, asyncRoute(async (req, res) => { const policy = normalizeWriterBindingRequest(req.body, { maxTtlDays: config.writerBindingMaxTtlDays, @@ -172,7 +327,7 @@ app.post("/internal/data-plane/v1/writer-bindings/:bindingId/rotate", requirePro const result = await pool.query( `update external_data_plane_writer_bindings set token_hash = $2, rotated_at = now() - where id = $1 and active = true and expires_at > now() + where id = $1 and binding_key is null and active = true and expires_at > now() returning id, tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", active, created_at as "createdAt", @@ -188,7 +343,7 @@ app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requirePro const result = await pool.query( `update external_data_plane_writer_bindings set active = false, revoked_at = now() - where id = $1 and active = true + where id = $1 and binding_key is null and active = true returning id, tenant_id as "tenantId", connection_id as "connectionId", provider_id as "providerId", allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt", active, created_at as "createdAt", @@ -564,6 +719,21 @@ function requireLegacyIntake(_req, _res, next) { function requireProvisionerApi(req, _res, next) { if (!config.provisionerApiEnabled) return next(httpError(503, "provisioner_api_disabled")); + return requireProvisionerCredential(req, next); +} + +function requireManagedProvisionerApi(req, _res, next) { + if (!config.managedProvisionerApiEnabled) return next(httpError(503, "managed_provisioner_api_disabled")); + if (!verifyManagedProvisionerRequest) return next(httpError(503, "managed_provisioner_api_not_configured")); + try { + req.managedProvisionerIdentity = verifyManagedProvisionerRequest(req); + return next(); + } catch (error) { + return next(error); + } +} + +function requireProvisionerCredential(req, next) { if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured")); const value = bearerToken(req); if (!value || !safeEqual(value, config.provisionerAccessToken)) return next(httpError(401, "provisioner_unauthorized")); @@ -697,6 +867,14 @@ function requireIdentifier(value, code) { return normalized; } +function requirePositiveInteger(value, code) { + const normalized = String(value ?? ""); + if (!/^[1-9]\d*$/.test(normalized)) throw httpError(400, code); + const number = Number(normalized); + if (!Number.isSafeInteger(number) || number > 2_147_483_647) throw httpError(400, code); + return number; +} + function parseCursor(value) { const normalized = String(value ?? ""); if (!/^(?:0|[1-9]\d*)$/.test(normalized)) throw httpError(400, "data_product_cursor_invalid"); diff --git a/services/external-data-plane/src/writer-binding.mjs b/services/external-data-plane/src/writer-binding.mjs index 386eb16..bf5322f 100644 --- a/services/external-data-plane/src/writer-binding.mjs +++ b/services/external-data-plane/src/writer-binding.mjs @@ -4,7 +4,15 @@ const IDENTIFIER = /^[a-z][a-z0-9._:-]{2,127}$/; const TOKEN_PREFIX = "ndc_edpwb_"; const SECRET_LIKE_KEY = /(token|secret|password|access[_-]?token|refresh[_-]?token|api[_-]?key)/i; const BINDING_REQUEST_KEYS = new Set(["source", "allowedDataProductIds", "expiresAt"]); +const MANAGED_BINDING_REQUEST_KEYS = new Set([ + "source", + "allowedDataProductIds", + "expiresAt", + "generation", + "capabilityDigest", +]); const BINDING_SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]); +const SHA256_DIGEST = /^[a-f0-9]{64}$/; /** * Produces an opaque, high-entropy capability. The plaintext is returned only @@ -59,6 +67,57 @@ export function normalizeWriterBindingRequest(value, { now = new Date(), maxTtlD }); } +/** + * Validates the zero-touch control-plane form. The native credential secret is + * generated and stored inside Engine; EDP receives only its SHA-256 digest. + * `generation` makes a retry address the same immutable binding generation. + */ +export function normalizeManagedWriterBindingRequest(value, { now = new Date(), maxTtlDays = 90 } = {}) { + if (!isPlainObject(value) || !isPlainObject(value.source)) { + throw writerBindingError("managed_writer_binding_request_invalid"); + } + if (containsSecretLikeKey(value)) { + throw writerBindingError("managed_writer_binding_secret_material_forbidden"); + } + if (!hasOnlyKeys(value, MANAGED_BINDING_REQUEST_KEYS) || !hasOnlyKeys(value.source, BINDING_SOURCE_KEYS)) { + throw writerBindingError("managed_writer_binding_request_fields_invalid"); + } + + const scope = normalizeWriterBindingRequest({ + source: value.source, + allowedDataProductIds: value.allowedDataProductIds, + expiresAt: value.expiresAt, + }, { now, maxTtlDays }); + const generation = Number(value.generation); + if (!Number.isSafeInteger(generation) || generation < 1 || generation > 2_147_483_647) { + throw writerBindingError("managed_writer_binding_generation_invalid"); + } + const capabilityDigest = String(value.capabilityDigest || "").toLowerCase(); + if (!SHA256_DIGEST.test(capabilityDigest)) { + throw writerBindingError("managed_writer_binding_capability_digest_invalid"); + } + + return Object.freeze({ + ...scope, + allowedDataProductIds: Object.freeze([...scope.allowedDataProductIds].sort()), + generation, + capabilityDigest, + }); +} + +export function writerBindingRequestHash(policy) { + const canonical = JSON.stringify({ + tenantId: policy.tenantId, + connectionId: policy.connectionId, + providerId: policy.providerId, + allowedDataProductIds: [...policy.allowedDataProductIds].sort(), + expiresAt: new Date(policy.expiresAt).toISOString(), + generation: policy.generation, + capabilityDigest: policy.capabilityDigest, + }); + return createHash("sha256").update(canonical, "utf8").digest("hex"); +} + /** * Converts a caller-provided, deliberately unscoped intake envelope into the * canonical scoped form. Caller scope is rejected, never trusted or merged. @@ -136,6 +195,8 @@ export function materializeDataProductPublish(value, binding, definition, dataPr export function safeWriterBinding(binding) { return { id: binding.id, + ...(binding.bindingKey ? { bindingKey: binding.bindingKey } : {}), + ...(Number.isInteger(Number(binding.generation)) ? { generation: Number(binding.generation) } : {}), tenantId: binding.tenantId, connectionId: binding.connectionId, providerId: binding.providerId, diff --git a/services/external-data-plane/test/api.integration.test.mjs b/services/external-data-plane/test/api.integration.test.mjs index a0e97c8..c95f37b 100644 --- a/services/external-data-plane/test/api.integration.test.mjs +++ b/services/external-data-plane/test/api.integration.test.mjs @@ -1,10 +1,16 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; +import { createHash, generateKeyPairSync, randomBytes, sign } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import net from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract"; +import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane"; +import { + MANAGED_PROVISIONER_HEADERS, + managedProvisionerSigningPayload, + sha256RawBody, +} from "../src/managed-provisioner-auth.mjs"; const databaseUrl = process.env.EXTERNAL_DATA_PLANE_TEST_DATABASE_URL; if (!databaseUrl) throw new Error("EXTERNAL_DATA_PLANE_TEST_DATABASE_URL_required"); @@ -13,7 +19,17 @@ const baseUrl = `http://127.0.0.1:${port}`; const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-api-test-")); const provisionerSecret = "nodedc_edp_integration_provisioner_secret_0123456789abcdef"; const secretPath = join(directory, "provisioner-token"); +const managedProvisionerServiceId = "nodedc-engine"; +const managedProvisionerKeyId = "engine-edp-managed-provisioner-v1"; +const managedProvisionerAudience = "nodedc-external-data-plane.managed-provisioning.v1"; +const managedProvisionerPublicKeyPath = join(directory, "engine-public-key.pem"); +const { privateKey: managedProvisionerPrivateKey, publicKey: managedProvisionerPublicKey } = generateKeyPairSync("ed25519"); await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 }); +await writeFile( + managedProvisionerPublicKeyPath, + managedProvisionerPublicKey.export({ type: "spki", format: "pem" }), + { mode: 0o600 }, +); const child = spawn(process.execPath, ["src/server.mjs"], { cwd: new URL("..", import.meta.url), @@ -22,7 +38,12 @@ const child = spawn(process.execPath, ["src/server.mjs"], { PORT: String(port), EXTERNAL_DATA_PLANE_DATABASE_URL: databaseUrl, EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "true", EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: secretPath, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: managedProvisionerPublicKeyPath, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID: managedProvisionerServiceId, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID: managedProvisionerKeyId, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: managedProvisionerAudience, EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000", EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250", EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000", @@ -52,10 +73,139 @@ try { const expiry = new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(); const scope = { tenantId: "api-tenant", connectionId: "api-connection", providerId: "api-provider" }; + const managedToken = "ndc_edpwb_managed_api_test_0123456789abcdefghijklmnopqrstuvwxyz"; + const managedBody = { + source: scope, + allowedDataProductIds: [productId], + expiresAt: expiry, + generation: 1, + capabilityDigest: createHash("sha256").update(managedToken, "utf8").digest("hex"), + }; + const managedPath = "/internal/data-plane/v1/writer-bindings/by-key/engine.api-connection.positions"; + const bearerOnlyManaged = await fetch(`${baseUrl}${managedPath}`, { + method: "PUT", + headers: { Authorization: `Bearer ${provisionerSecret}`, "Content-Type": "application/json" }, + body: JSON.stringify(managedBody), + }); + assert.equal(bearerOnlyManaged.status, 401); + assert.equal((await bearerOnlyManaged.json()).error, "managed_provisioner_header_invalid"); + + const managedResults = await Promise.all([ + rawJsonRequest(managedPath, { + method: "PUT", + managedSignature: true, + body: managedBody, + }), + rawJsonRequest(managedPath, { + method: "PUT", + managedSignature: true, + body: managedBody, + }), + ]); + assert.deepEqual(managedResults.map(({ value }) => value.idempotent).sort(), [false, true]); + assert.equal(managedResults[0].value.writerBinding.id, managedResults[1].value.writerBinding.id); + assert.equal("token" in managedResults[0].value, false); + assert.equal(JSON.stringify(managedResults[0].value).includes(managedBody.capabilityDigest), false); + const managedCatalog = await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: managedToken }); + assert.deepEqual(managedCatalog.dataProducts.map((value) => value.id), [productId]); + const replayBody = JSON.stringify(managedBody); + const replayHeaders = signedManagedHeaders(managedPath, "PUT", Buffer.from(replayBody, "utf8")); + const firstReplayAttempt = await fetch(`${baseUrl}${managedPath}`, { + method: "PUT", headers: { ...replayHeaders, "Content-Type": "application/json" }, body: replayBody, + }); + assert.equal(firstReplayAttempt.status, 200); + const rejectedReplay = await fetch(`${baseUrl}${managedPath}`, { + method: "PUT", headers: { ...replayHeaders, "Content-Type": "application/json" }, body: replayBody, + }); + assert.equal(rejectedReplay.status, 409); + assert.equal((await rejectedReplay.json()).error, "managed_provisioner_request_replayed"); + + const tamperHeaders = signedManagedHeaders(managedPath, "PUT", Buffer.from(replayBody, "utf8")); + const tamperedBody = await fetch(`${baseUrl}${managedPath}`, { + method: "PUT", + headers: { ...tamperHeaders, "Content-Type": "application/json" }, + body: `${replayBody} `, + }); + assert.equal(tamperedBody.status, 401); + assert.equal((await tamperedBody.json()).error, "managed_provisioner_body_hash_mismatch"); + + const pathHeaders = signedManagedHeaders(managedPath, "PUT", Buffer.from(replayBody, "utf8")); + const tamperedPath = await fetch(`${baseUrl}${managedPath}?unexpected=true`, { + method: "PUT", headers: { ...pathHeaders, "Content-Type": "application/json" }, body: replayBody, + }); + assert.equal(tamperedPath.status, 401); + assert.equal((await tamperedPath.json()).error, "managed_provisioner_signature_invalid"); + + const audienceHeaders = signedManagedHeaders(managedPath, "PUT", Buffer.from(replayBody, "utf8")); + audienceHeaders[MANAGED_PROVISIONER_HEADERS.audience] = "other-audience"; + const wrongAudience = await fetch(`${baseUrl}${managedPath}`, { + method: "PUT", headers: { ...audienceHeaders, "Content-Type": "application/json" }, body: replayBody, + }); + assert.equal(wrongAudience.status, 401); + assert.equal((await wrongAudience.json()).error, "managed_provisioner_identity_mismatch"); + + const managedConflict = await signedFetch(managedPath, { + method: "PUT", + body: { + ...managedBody, + capabilityDigest: createHash("sha256").update(`${managedToken}-different`, "utf8").digest("hex"), + }, + }); + assert.equal(managedConflict.status, 409); + assert.equal((await managedConflict.json()).error, "managed_writer_binding_request_conflict"); + + const legacyManagedRevoke = await fetch( + `${baseUrl}/internal/data-plane/v1/writer-bindings/${managedResults[0].value.writerBinding.id}/revoke`, + { method: "POST", headers: { Authorization: `Bearer ${provisionerSecret}` } }, + ); + assert.equal(legacyManagedRevoke.status, 404); + + const managedTokenGeneration2 = `${managedToken}_generation_2`; + const managedGeneration2 = await jsonRequest( + "/internal/data-plane/v1/writer-bindings/by-key/engine.api-connection.positions", + { + method: "PUT", + managedSignature: true, + body: { + ...managedBody, + generation: 2, + capabilityDigest: createHash("sha256").update(managedTokenGeneration2, "utf8").digest("hex"), + }, + }, + ); + assert.equal(managedGeneration2.writerBinding.generation, 2); + assert.deepEqual( + (await jsonRequest("/internal/data-plane/v1/writer/data-products", { token: managedTokenGeneration2 })).dataProducts.map((value) => value.id), + [productId], + ); + const managedRevoke = await jsonRequest( + "/internal/data-plane/v1/writer-bindings/by-key/engine.api-connection.positions/generations/1/revoke", + { method: "POST", managedSignature: true }, + ); + assert.equal(managedRevoke.idempotent, false); + assert.equal(managedRevoke.writerBinding.active, false); + const managedRevokeRetry = await jsonRequest( + "/internal/data-plane/v1/writer-bindings/by-key/engine.api-connection.positions/generations/1/revoke", + { method: "POST", managedSignature: true }, + ); + assert.equal(managedRevokeRetry.idempotent, true); + const rejectedManagedGeneration1 = await fetch(`${baseUrl}/internal/data-plane/v1/writer/data-products`, { + headers: { Authorization: `Bearer ${managedToken}` }, + }); + assert.equal(rejectedManagedGeneration1.status, 401); + + const manualWriterBody = { source: scope, allowedDataProductIds: [productId], expiresAt: expiry }; + const signedOnlyManualProvisioning = await signedFetch("/internal/data-plane/v1/writer-bindings", { + method: "POST", + body: manualWriterBody, + }); + assert.equal(signedOnlyManualProvisioning.status, 401); + assert.equal((await signedOnlyManualProvisioning.json()).error, "provisioner_unauthorized"); + const writerIssuance = await rawJsonRequest("/internal/data-plane/v1/writer-bindings", { method: "POST", token: provisionerSecret, - body: { source: scope, allowedDataProductIds: [productId], expiresAt: expiry }, + body: manualWriterBody, }); const readerIssuance = await rawJsonRequest("/internal/data-plane/v1/reader-bindings", { method: "POST", @@ -176,26 +326,66 @@ function publish(runId, observedAt, longitude) { }; } -async function jsonRequest(path, { method = "GET", token, body } = {}) { - const { response, value } = await rawJsonRequest(path, { method, token, body }); +async function jsonRequest(path, { method = "GET", token, managedSignature = false, body } = {}) { + const { response, value } = await rawJsonRequest(path, { method, token, managedSignature, body }); if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`); return value; } -async function rawJsonRequest(path, { method = "GET", token, body } = {}) { +async function rawJsonRequest(path, { method = "GET", token, managedSignature = false, body } = {}) { + const rawBody = body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(body), "utf8"); const response = await fetch(`${baseUrl}${path}`, { method, headers: { ...(token ? { Authorization: `Bearer ${token}` } : {}), - ...(body ? { "Content-Type": "application/json" } : {}), + ...(managedSignature ? signedManagedHeaders(path, method, rawBody) : {}), + ...(body === undefined ? {} : { "Content-Type": "application/json" }), }, - body: body ? JSON.stringify(body) : undefined, + body: body === undefined ? undefined : rawBody, }); const value = await response.json(); if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`); return { response, value }; } +async function signedFetch(path, { method, body } = {}) { + const rawBody = body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(body), "utf8"); + return fetch(`${baseUrl}${path}`, { + method, + headers: { + ...signedManagedHeaders(path, method, rawBody), + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : rawBody, + }); +} + +function signedManagedHeaders(path, method, rawBody) { + const timestamp = new Date().toISOString(); + const nonce = randomBytes(24).toString("base64url"); + const bodySha256 = sha256RawBody(rawBody); + const payload = managedProvisionerSigningPayload({ + audience: managedProvisionerAudience, + serviceId: managedProvisionerServiceId, + keyId: managedProvisionerKeyId, + method, + path, + timestamp, + nonce, + bodySha256, + }); + const signature = sign(null, Buffer.from(payload, "utf8"), managedProvisionerPrivateKey).toString("base64url"); + return { + [MANAGED_PROVISIONER_HEADERS.serviceId]: managedProvisionerServiceId, + [MANAGED_PROVISIONER_HEADERS.keyId]: managedProvisionerKeyId, + [MANAGED_PROVISIONER_HEADERS.audience]: managedProvisionerAudience, + [MANAGED_PROVISIONER_HEADERS.timestamp]: timestamp, + [MANAGED_PROVISIONER_HEADERS.nonce]: nonce, + [MANAGED_PROVISIONER_HEADERS.bodySha256]: bodySha256, + [MANAGED_PROVISIONER_HEADERS.signature]: signature, + }; +} + function assertOneTimeCapabilityResponse(response) { assert.equal(response.headers.get("cache-control"), "no-store, max-age=0"); assert.equal(response.headers.get("pragma"), "no-cache"); diff --git a/services/external-data-plane/test/config.test.mjs b/services/external-data-plane/test/config.test.mjs index 5a5f4ac..ee36477 100644 --- a/services/external-data-plane/test/config.test.mjs +++ b/services/external-data-plane/test/config.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { generateKeyPairSync } from "node:crypto"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -11,9 +12,12 @@ const base = { const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-config-")); const secretPath = join(directory, "provisioner-token"); +const publicKeyPath = join(directory, "engine-public-key.pem"); const secret = "provisioner_secret_is_separate_from_legacy_internal_token_123456"; try { await writeFile(secretPath, `${secret}\n`, { encoding: "utf8", mode: 0o600 }); + const { publicKey } = generateKeyPairSync("ed25519"); + await writeFile(publicKeyPath, publicKey.export({ type: "spki", format: "pem" }), { mode: 0o600 }); const config = readConfig({ ...base, EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", @@ -22,6 +26,8 @@ try { EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000", }); assert.equal(config.provisionerAccessToken, secret); + assert.equal(config.provisionerApiEnabled, true); + assert.equal(config.managedProvisionerApiEnabled, false); assert.equal(config.maxFutureSkewSeconds, 120); assert.equal(config.retentionSweepMs, 60000); assert.equal(config.maxPatchBytes, 262144); @@ -39,10 +45,49 @@ try { EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "true", EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"), }), /external_data_plane_provisioner_token_file_unreadable/); + const managedConfig = readConfig({ + ...base, + EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing-token-is-ignored"), + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: publicKeyPath, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID: "nodedc-engine", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID: "engine-edp-managed-provisioner-v1", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: "nodedc-external-data-plane.managed-provisioning.v1", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS: "45", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES: "500", + }); + assert.equal(managedConfig.provisionerApiEnabled, false); + assert.equal(managedConfig.managedProvisionerApiEnabled, true); + assert.equal(managedConfig.provisionerAccessToken, ""); + assert.equal(managedConfig.managedProvisionerPublicKey.asymmetricKeyType, "ed25519"); + assert.equal(managedConfig.managedProvisionerServiceId, "nodedc-engine"); + assert.equal(managedConfig.managedProvisionerKeyId, "engine-edp-managed-provisioner-v1"); + assert.equal(managedConfig.managedProvisionerAudience, "nodedc-external-data-plane.managed-provisioning.v1"); + assert.equal(managedConfig.managedProvisionerMaxSkewMs, 45_000); + assert.equal(managedConfig.managedProvisionerReplayCacheMaxEntries, 500); + assert.throws(() => readConfig({ + ...base, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: join(directory, "missing"), + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID: "nodedc-engine", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID: "engine-edp-managed-provisioner-v1", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: "nodedc-external-data-plane.managed-provisioning.v1", + }), /external_data_plane_managed_provisioner_public_key_file_unreadable/); + assert.throws(() => readConfig({ + ...base, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "true", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: publicKeyPath, + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID: "invalid service id", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID: "engine-edp-managed-provisioner-v1", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: "nodedc-external-data-plane.managed-provisioning.v1", + }), /external_data_plane_managed_provisioner_service_id_invalid/); assert.equal(readConfig({ ...base, EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED: "false", + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "false", EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"), + EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: join(directory, "missing"), }).provisionerAccessToken, ""); } finally { await rm(directory, { recursive: true, force: true }); diff --git a/services/external-data-plane/test/data-product-delivery.integration.test.mjs b/services/external-data-plane/test/data-product-delivery.integration.test.mjs index 2e99f16..4c5ed5c 100644 --- a/services/external-data-plane/test/data-product-delivery.integration.test.mjs +++ b/services/external-data-plane/test/data-product-delivery.integration.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { Pool } from "pg"; -import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract"; +import { validateDataProductPatch, validateDataProductSnapshot } from "@nodedc/external-provider-contract/data-plane"; import { loadDataProductDefinition, persistDataProductDefinition, diff --git a/services/external-data-plane/test/managed-provisioner-auth.test.mjs b/services/external-data-plane/test/managed-provisioner-auth.test.mjs new file mode 100644 index 0000000..baf80db --- /dev/null +++ b/services/external-data-plane/test/managed-provisioner-auth.test.mjs @@ -0,0 +1,217 @@ +import assert from "node:assert/strict"; +import { + generateKeyPairSync, + randomBytes, + sign, +} from "node:crypto"; +import { chmod, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + createManagedProvisionerRequestVerifier, + loadManagedProvisionerPublicKeyFile, + MANAGED_PROVISIONER_HEADERS, + managedProvisionerSigningPayload, + ManagedProvisionerReplayCache, + sha256RawBody, +} from "../src/managed-provisioner-auth.mjs"; + +const serviceId = "nodedc-engine"; +const keyId = "engine-edp-managed-provisioner-v1"; +const audience = "nodedc-external-data-plane.managed-provisioning.v1"; +const nowMs = Date.parse("2026-07-16T12:00:00.000Z"); +const { privateKey, publicKey } = generateKeyPairSync("ed25519"); +let nonceSequence = 0; + +const replayCache = new ManagedProvisionerReplayCache({ maxEntries: 100 }); +const verify = createManagedProvisionerRequestVerifier({ + publicKey, + serviceId, + keyId, + audience, + maxSkewMs: 60_000, + replayCache, + now: () => nowMs, +}); + +const valid = signedRequest(); +assert.deepEqual(verify(valid), { serviceId, keyId }); +assert.throws(() => verify(valid), hasCode("managed_provisioner_request_replayed", 409)); + +const whitespaceTamper = signedRequest(); +whitespaceTamper.rawBody = Buffer.from(`${whitespaceTamper.rawBody.toString("utf8")} `, "utf8"); +assert.throws(() => verify(whitespaceTamper), hasCode("managed_provisioner_body_hash_mismatch", 401)); + +const methodTamper = signedRequest(); +methodTamper.method = "POST"; +assert.throws(() => verify(methodTamper), hasCode("managed_provisioner_signature_invalid", 401)); + +const pathTamper = signedRequest(); +pathTamper.originalUrl = `${pathTamper.originalUrl}?unexpected=true`; +assert.throws(() => verify(pathTamper), hasCode("managed_provisioner_signature_invalid", 401)); + +const audienceTamper = signedRequest(); +setHeader(audienceTamper, MANAGED_PROVISIONER_HEADERS.audience, "other-audience"); +assert.throws(() => verify(audienceTamper), hasCode("managed_provisioner_identity_mismatch", 401)); + +const serviceTamper = signedRequest(); +setHeader(serviceTamper, MANAGED_PROVISIONER_HEADERS.serviceId, "other-engine"); +assert.throws(() => verify(serviceTamper), hasCode("managed_provisioner_identity_mismatch", 401)); +const keyTamper = signedRequest(); +setHeader(keyTamper, MANAGED_PROVISIONER_HEADERS.keyId, "other-key-v1"); +assert.throws(() => verify(keyTamper), hasCode("managed_provisioner_identity_mismatch", 401)); + +const forbiddenBearer = signedRequest(); +forbiddenBearer.headers.authorization = "Bearer value-that-must-not-appear-in-errors"; +forbiddenBearer.rawHeaders.push("Authorization", forbiddenBearer.headers.authorization); +assert.throws( + () => verify(forbiddenBearer), + (error) => hasCode("managed_provisioner_authorization_header_forbidden", 401)(error) + && !error.message.includes("value-that-must-not-appear-in-errors"), +); + +const stale = signedRequest({ timestamp: new Date(nowMs - 60_000).toISOString() }); +assert.throws(() => verify(stale), hasCode("managed_provisioner_timestamp_outside_window", 401)); +const future = signedRequest({ timestamp: new Date(nowMs + 60_000).toISOString() }); +assert.throws(() => verify(future), hasCode("managed_provisioner_timestamp_outside_window", 401)); +const invalidTimestamp = signedRequest(); +setHeader(invalidTimestamp, MANAGED_PROVISIONER_HEADERS.timestamp, "2026-07-16T12:00:00Z"); +assert.throws(() => verify(invalidTimestamp), hasCode("managed_provisioner_timestamp_invalid", 401)); + +const duplicateHeader = signedRequest(); +duplicateHeader.rawHeaders.push(MANAGED_PROVISIONER_HEADERS.nonce, header(duplicateHeader, MANAGED_PROVISIONER_HEADERS.nonce)); +assert.throws(() => verify(duplicateHeader), hasCode("managed_provisioner_header_invalid", 401)); + +const uncaptured = signedRequest(); +uncaptured.rawBodyCaptured = false; +uncaptured.headers["content-length"] = String(uncaptured.rawBody.length); +assert.throws(() => verify(uncaptured), hasCode("managed_provisioner_raw_body_unavailable", 401)); + +const badSignature = signedRequest(); +setHeader(badSignature, MANAGED_PROVISIONER_HEADERS.signature, randomBytes(64).toString("base64url")); +assert.throws(() => verify(badSignature), hasCode("managed_provisioner_signature_invalid", 401)); +const paddedSignature = signedRequest(); +setHeader( + paddedSignature, + MANAGED_PROVISIONER_HEADERS.signature, + `${header(paddedSignature, MANAGED_PROVISIONER_HEADERS.signature)}=`, +); +assert.throws(() => verify(paddedSignature), hasCode("managed_provisioner_signature_invalid", 401)); +const uppercaseHash = signedRequest(); +setHeader( + uppercaseHash, + MANAGED_PROVISIONER_HEADERS.bodySha256, + header(uppercaseHash, MANAGED_PROVISIONER_HEADERS.bodySha256).toUpperCase(), +); +assert.throws(() => verify(uppercaseHash), hasCode("managed_provisioner_body_hash_invalid", 401)); + +const cache = new ManagedProvisionerReplayCache({ maxEntries: 1 }); +cache.consume("first", nowMs + 1000, nowMs); +assert.throws(() => cache.consume("second", nowMs + 1000, nowMs), hasCode("managed_provisioner_replay_cache_exhausted", 503)); +assert.throws(() => cache.consume("first", nowMs + 1000, nowMs), hasCode("managed_provisioner_request_replayed", 409)); +cache.consume("second", nowMs + 2000, nowMs + 1000); + +const directory = await mkdtemp(join(tmpdir(), "nodedc-edp-managed-auth-")); +try { + const publicKeyPath = join(directory, "engine-public-key.pem"); + const rsaPublicKeyPath = join(directory, "rsa-public-key.pem"); + const privateKeyPath = join(directory, "engine-private-key.pem"); + const symlinkPath = join(directory, "engine-public-key-link.pem"); + await writeFile(publicKeyPath, publicKey.export({ type: "spki", format: "pem" }), { mode: 0o600 }); + assert.equal(loadManagedProvisionerPublicKeyFile(publicKeyPath).asymmetricKeyType, "ed25519"); + assert.throws( + () => loadManagedProvisionerPublicKeyFile("relative-public-key.pem"), + /external_data_plane_managed_provisioner_public_key_file_must_be_absolute/, + ); + assert.throws( + () => loadManagedProvisionerPublicKeyFile(join(directory, "missing.pem")), + /external_data_plane_managed_provisioner_public_key_file_unreadable/, + ); + + await symlink(publicKeyPath, symlinkPath); + assert.throws( + () => loadManagedProvisionerPublicKeyFile(symlinkPath), + /external_data_plane_managed_provisioner_public_key_file_must_be_regular/, + ); + + await chmod(publicKeyPath, 0o622); + assert.throws( + () => loadManagedProvisionerPublicKeyFile(publicKeyPath), + /external_data_plane_managed_provisioner_public_key_file_permissions_invalid/, + ); + await chmod(publicKeyPath, 0o600); + + const { publicKey: rsaPublicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); + await writeFile(rsaPublicKeyPath, rsaPublicKey.export({ type: "spki", format: "pem" }), { mode: 0o600 }); + assert.throws( + () => loadManagedProvisionerPublicKeyFile(rsaPublicKeyPath), + /external_data_plane_managed_provisioner_public_key_algorithm_invalid/, + ); + + await writeFile(privateKeyPath, privateKey.export({ type: "pkcs8", format: "pem" }), { mode: 0o600 }); + assert.throws( + () => loadManagedProvisionerPublicKeyFile(privateKeyPath), + /external_data_plane_managed_provisioner_public_key_file_format_invalid/, + ); +} finally { + await rm(directory, { recursive: true, force: true }); +} + +console.log("external-data-plane managed provisioner auth: ok"); + +function signedRequest({ + method = "PUT", + path = "/internal/data-plane/v1/writer-bindings/by-key/engine.test.positions", + rawBody = Buffer.from('{"generation":1}', "utf8"), + timestamp = new Date(nowMs).toISOString(), + nonce = nextNonce(), +} = {}) { + const bodySha256 = sha256RawBody(rawBody); + const signature = sign(null, Buffer.from(managedProvisionerSigningPayload({ + audience, + serviceId, + keyId, + method, + path, + timestamp, + nonce, + bodySha256, + }), "utf8"), privateKey).toString("base64url"); + const headers = { + [MANAGED_PROVISIONER_HEADERS.serviceId]: serviceId, + [MANAGED_PROVISIONER_HEADERS.keyId]: keyId, + [MANAGED_PROVISIONER_HEADERS.audience]: audience, + [MANAGED_PROVISIONER_HEADERS.timestamp]: timestamp, + [MANAGED_PROVISIONER_HEADERS.nonce]: nonce, + [MANAGED_PROVISIONER_HEADERS.bodySha256]: bodySha256, + [MANAGED_PROVISIONER_HEADERS.signature]: signature, + "content-length": String(rawBody.length), + }; + return { + method, + originalUrl: path, + headers, + rawHeaders: Object.entries(headers).flat(), + rawBody, + rawBodyCaptured: true, + }; +} + +function nextNonce() { + nonceSequence += 1; + return Buffer.from(`managed-auth-test-${String(nonceSequence).padStart(6, "0")}`).toString("base64url"); +} + +function header(req, name) { + return req.headers[name]; +} + +function setHeader(req, name, value) { + req.headers[name] = value; + const index = req.rawHeaders.findIndex((candidate) => String(candidate).toLowerCase() === name); + req.rawHeaders[index + 1] = value; +} + +function hasCode(code, status) { + return (error) => error?.code === code && error?.status === status; +} diff --git a/services/external-data-plane/test/writer-binding.test.mjs b/services/external-data-plane/test/writer-binding.test.mjs index 726515f..b49b940 100644 --- a/services/external-data-plane/test/writer-binding.test.mjs +++ b/services/external-data-plane/test/writer-binding.test.mjs @@ -1,12 +1,14 @@ import assert from "node:assert/strict"; -import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/index.mjs"; +import { validateIntakeBatch } from "../../../packages/external-provider-contract/src/data-plane.mjs"; import { createWriterToken, hashWriterToken, materializeDataProductPublish, materializeWriterBoundBatch, + normalizeManagedWriterBindingRequest, normalizeWriterBindingRequest, safeWriterBinding, + writerBindingRequestHash, } from "../src/writer-binding.mjs"; const now = new Date("2026-07-15T12:00:00.000Z"); @@ -34,6 +36,39 @@ assert.match(token, /^ndc_edpwb_[A-Za-z0-9_-]{40,}$/); assert.equal(hashWriterToken(token), hashWriterToken(token)); assert.notEqual(hashWriterToken(token), hashWriterToken(`${token}x`)); +const managedPolicy = normalizeManagedWriterBindingRequest({ + ...request, + allowedDataProductIds: ["fleet.positions.current.v1", "asset.status.current.v1"], + generation: 1, + capabilityDigest: hashWriterToken(token), +}, { now, maxTtlDays: 90 }); +assert.deepEqual(managedPolicy.allowedDataProductIds, [ + "asset.status.current.v1", + "fleet.positions.current.v1", +]); +assert.equal(managedPolicy.generation, 1); +assert.equal(managedPolicy.capabilityDigest, hashWriterToken(token)); +assert.equal(writerBindingRequestHash(managedPolicy), writerBindingRequestHash({ + ...managedPolicy, + allowedDataProductIds: [...managedPolicy.allowedDataProductIds].reverse(), +})); +assert.throws(() => normalizeManagedWriterBindingRequest({ + ...request, + generation: 0, + capabilityDigest: hashWriterToken(token), +}, { now, maxTtlDays: 90 }), /managed_writer_binding_generation_invalid/); +assert.throws(() => normalizeManagedWriterBindingRequest({ + ...request, + generation: 1, + capabilityDigest: "not-a-digest", +}, { now, maxTtlDays: 90 }), /managed_writer_binding_capability_digest_invalid/); +assert.throws(() => normalizeManagedWriterBindingRequest({ + ...request, + generation: 1, + capabilityDigest: hashWriterToken(token), + token, +}, { now, maxTtlDays: 90 }), /managed_writer_binding_secret_material_forbidden/); + const unscopedBatch = { schemaVersion: "nodedc.external-provider-contract/v1", source: { providerId: "example-provider" },