feat(edp): provision Foundry reader grants

This commit is contained in:
Codex 2026-07-19 15:03:48 +03:00
parent 847a08da93
commit def9a24e0d
17 changed files with 891 additions and 7 deletions

View File

@ -89,6 +89,12 @@ EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID=engine-edp-managed-provisioner-v1
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE=nodedc-external-data-plane.managed-provisioning.v1
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS=60
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES=10000
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED=true
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID=nodedc-module-foundry
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID=foundry-edp-managed-provisioner-v1
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE=nodedc-external-data-plane.managed-provisioning.v1
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS=60
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES=10000
# notification core
NOTIFICATION_PG_DB=nodedc_notifications

View File

@ -182,13 +182,14 @@ force-recreates only `external-data-plane`; the already healthy
independent deploy prerequisite and are never selected by an EDP application
artifact. The reviewed Compose source pins the Timescale image, named volume
and target, internal database-only network, absence of database host ports,
healthy dependency, localhost-only EDP bind and the two read-only
healthy dependency, localhost-only EDP bind and the three read-only
provisioner/trust mounts in the reviewed Compose source. The runner does not
reinterpret version-dependent `docker compose config` JSON as a second deploy
schema. Its canonical enforcement remains the artifact/path allowlist plus
hard-coded build command, selected service set, runtime-secret preparation and
health acceptance. Post-apply acceptance also requires
`database=ready`; a first-rollout failure removes only the candidate EDP
`database=ready`; the managed Foundry slice additionally requires
`foundryReaderBindingProvisioning=digest+server-resolved-source`. A first-rollout failure removes only the candidate EDP
container without volumes and restores the source overlay. It never contains a
provider credential, provider endpoint,
collection schedule or command
@ -218,6 +219,21 @@ symlinks and permissive modes fail closed. `plan` discloses both paths without
printing key material. The private key must not be broadened into an L2 graph,
MCP surface, artifact or shared-token boundary.
Module Foundry has a separate Ed25519 managed-provisioner identity for
target-scoped Data Product consumer grants. On a relevant `platform` or
`module-foundry` apply, the runner creates or validates the Foundry-only private
key at
`/volume1/docker/nodedc-platform/secrets/foundry-edp-managed-provisioner/private-key.pem`
as `root:root 0400` and the matching EDP trust copy at
`/volume1/docker/nodedc-platform/trust/foundry-managed-provisioner/public-key.pem`
as `root:11006 0440`. Foundry generates the opaque reader token only inside its
persistent private runtime; its signed EDP request contains only a SHA-256
digest and Data Product id. EDP resolves the unique active writer source scope
server-side and fails closed on missing or ambiguous coverage. No provider,
tenant, connection, token, private key or endpoint is admitted to the Foundry
MCP plan, application state or browser response. The Engine signing identity,
native n8n credentials and legacy issuance bearer cannot call this endpoint.
The reviewed Engine source candidate has dedicated server-derived MCP
plan/apply handling for the exact `ndcDataProductWriterApi` + Data Product
Publish tuple. It is separate from the generic HTTP safe-ref path and accepts no
@ -340,6 +356,12 @@ never enters an artifact. The component reuses the existing internal platform
credential for Launcher handoff validation and requires that runtime
configuration before its first `apply`.
When the managed reader-grant source is present, Module Foundry acceptance also
requires `/healthz` to report the dedicated Ed25519 provisioner as configured.
The check is source-aware: if an apply rolls back to the prior source, rollback
acceptance uses that prior health contract instead of falsely requiring a
feature which the restored generation does not contain.
The Foundry ↔ Map Gateway signing key is not an application or `.env` setting.
On the first relevant `platform` or `module-foundry` apply, the root-owned
runner creates `/volume1/docker/nodedc-platform/secrets/map-gateway-admin-secret`

View File

@ -77,6 +77,10 @@ async function assertSourceBoundary() {
"source: /volume1/docker/nodedc-platform/trust/engine-managed-provisioner",
"target: /run/nodedc-trust/engine-managed-provisioner",
"EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: /run/nodedc-trust/engine-managed-provisioner/public-key.pem",
"source: /volume1/docker/nodedc-platform/trust/foundry-managed-provisioner",
"target: /run/nodedc-trust/foundry-managed-provisioner",
"EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: /run/nodedc-trust/foundry-managed-provisioner/public-key.pem",
"EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED:-true}",
"create_host_path: false",
]) {
if (!compose.includes(fragment)) throw new Error(`platform_compose_boundary_missing:${fragment}`);
@ -97,6 +101,9 @@ async function assertSourceBoundary() {
'managedReaderBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled"',
'managedReaderBindings: "digest+idempotent-generation+explicit-revoke"',
'readerSourceScope: "writer-resolved+fail-closed-ambiguity"',
'app.post("/internal/data-plane/v1/consumer-reader-bindings/plan"',
'app.put("/internal/data-plane/v1/consumer-reader-bindings/by-key/:bindingKey"',
'foundryReaderBindingProvisioning: config.foundryProvisionerApiEnabled ? "digest+server-resolved-source" : "disabled"',
'source_connection_id as "sourceConnectionId"',
'where id = $1 and binding_key is null and active = true',
]) {
@ -109,6 +116,8 @@ async function assertSourceBoundary() {
for (const marker of [
"managed_reader_source_scope_not_found",
"managed_reader_source_scope_ambiguous",
"managed_consumer_reader_source_scope_not_found",
"managed_consumer_reader_source_scope_ambiguous",
"external_data_plane_writer_bindings",
]) {
if (!readerSourceScope.includes(marker)) throw new Error(`reader_source_scope_boundary_missing:${marker}`);

View File

@ -38,6 +38,8 @@ const stage = await mkdtemp(join(tmpdir(), "nodedc-module-foundry-artifact-"));
const payload = join(stage, "payload");
const target = join(artifactDir, `nodedc-module-foundry-${patchId}.tgz`);
await assertSourceBoundary();
try {
await mkdir(payload, { recursive: true });
for (const sourceRelative of files) {
@ -60,6 +62,31 @@ try {
await rm(stage, { recursive: true, force: true });
}
async function assertSourceBoundary() {
const compose = await readFile(resolve(foundryRoot, "infra/docker-compose.module-foundry.yml"), "utf8");
for (const fragment of [
"source: /volume1/docker/nodedc-platform/secrets/foundry-edp-managed-provisioner/private-key.pem",
"target: /run/nodedc-secrets/foundry-edp-managed-provisioner/private-key.pem",
"NODEDC_EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PRIVATE_KEY_FILE: /run/nodedc-secrets/foundry-edp-managed-provisioner/private-key.pem",
"create_host_path: false",
]) {
if (!compose.includes(fragment)) throw new Error(`foundry_reader_grant_boundary_missing:${fragment}`);
}
const provisioner = await readFile(resolve(foundryRoot, "server/foundry-reader-grant-provisioner.mjs"), "utf8");
for (const marker of [
'"/internal/data-plane/v1/consumer-reader-bindings/plan"',
"consumer-reader-bindings/by-key/${encodeURIComponent(bindingKey)}",
"capabilityDigest: createHash(\"sha256\").update(token",
"sourceScope: \"resolved-server-side\"",
"O_NOFOLLOW",
]) {
if (!provisioner.includes(marker)) throw new Error(`foundry_reader_grant_boundary_missing:${marker}`);
}
if (/providerId|tenantId|connectionId/.test(provisioner)) {
throw new Error("foundry_reader_grant_source_scope_boundary_violation");
}
}
function canonicalTarScript() {
return [
"import gzip,io,pathlib,sys,tarfile",
@ -88,6 +115,7 @@ async function copySafe(source, destination) {
await mkdir(destination, { recursive: true });
for (const entry of await readdir(source, { withFileTypes: true })) {
if (entry.name === "private-key.pem") throw new Error(`private_key_source_rejected:${relative(foundryRoot, join(source, entry.name))}`);
if (ignoredBasenames.has(entry.name) || entry.name.startsWith(".env") || entry.name.endsWith(".tsbuildinfo")) continue;
const childSource = join(source, entry.name);
const childDestination = join(destination, entry.name);

View File

@ -63,6 +63,10 @@ ENGINE_EDP_MANAGED_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "engine-edp
ENGINE_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE = ENGINE_EDP_MANAGED_PROVISIONER_SECRET_DIR / "private-key.pem"
ENGINE_EDP_MANAGED_PROVISIONER_TRUST_DIR = Path("/volume1/docker/nodedc-platform/trust/engine-managed-provisioner")
ENGINE_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE = ENGINE_EDP_MANAGED_PROVISIONER_TRUST_DIR / "public-key.pem"
FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR = MAP_GATEWAY_SECRET_DIR / "foundry-edp-managed-provisioner"
FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE = FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR / "private-key.pem"
FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR = Path("/volume1/docker/nodedc-platform/trust/foundry-managed-provisioner")
FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE = FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR / "public-key.pem"
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "external-data-plane-reader-grants"
FOUNDRY_BINDING_GRANTS_DIR = MAP_GATEWAY_SECRET_DIR / "foundry-binding-grants"
N8N_PRIVATE_EXTENSION_RELEASES_ROOT = Path("/volume1/docker/nodedc-platform/n8n-private-extensions")
@ -1159,6 +1163,168 @@ def ensure_engine_edp_managed_provisioner_keypair():
if temporary_public.exists():
temporary_public.unlink()
return "created"
return "created"
def ensure_foundry_edp_managed_provisioner_keypair():
# Foundry receives its own signing identity. It must never reuse or read
# the Engine private key; EDP trusts the matching public key separately.
openssl = shutil.which("openssl")
if not openssl:
die("openssl is required to manage the Foundry EDP signing key")
try:
secret_parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
except FileNotFoundError:
MAP_GATEWAY_SECRET_DIR.mkdir(parents=True, exist_ok=False)
secret_parent_stat = MAP_GATEWAY_SECRET_DIR.lstat()
if stat.S_ISLNK(secret_parent_stat.st_mode) or not stat.S_ISDIR(secret_parent_stat.st_mode):
die("Foundry EDP signing key parent directory is unsafe")
os.chown(MAP_GATEWAY_SECRET_DIR, 0, MAP_GATEWAY_RUNTIME_GID)
MAP_GATEWAY_SECRET_DIR.chmod(0o710)
try:
private_dir_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR.lstat()
except FileNotFoundError:
FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR.mkdir(parents=False, exist_ok=False)
private_dir_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR.lstat()
if stat.S_ISLNK(private_dir_stat.st_mode) or not stat.S_ISDIR(private_dir_stat.st_mode):
die("Foundry EDP signing key directory is unsafe")
os.chown(FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR, 0, 0)
FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR.chmod(0o700)
trust_parent = FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR.parent
try:
trust_parent_stat = trust_parent.lstat()
except FileNotFoundError:
trust_parent.mkdir(parents=True, exist_ok=False)
trust_parent_stat = trust_parent.lstat()
if stat.S_ISLNK(trust_parent_stat.st_mode) or not stat.S_ISDIR(trust_parent_stat.st_mode):
die("Foundry EDP trust parent directory is unsafe")
os.chown(trust_parent, 0, EXTERNAL_DATA_PLANE_RUNTIME_GID)
trust_parent.chmod(0o710)
try:
trust_dir_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR.lstat()
except FileNotFoundError:
FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR.mkdir(parents=False, exist_ok=False)
trust_dir_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR.lstat()
if stat.S_ISLNK(trust_dir_stat.st_mode) or not stat.S_ISDIR(trust_dir_stat.st_mode):
die("Foundry EDP trust directory is unsafe")
os.chown(FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR, 0, EXTERNAL_DATA_PLANE_RUNTIME_GID)
FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR.chmod(0o550)
try:
private_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE.lstat()
except FileNotFoundError:
private_stat = None
try:
public_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE.lstat()
except FileNotFoundError:
public_stat = None
if private_stat is None and public_stat is not None:
die("Foundry EDP public key exists without its private key")
private_created = False
if private_stat is None:
temporary_private = (
FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR
/ f".private-key.{os.getpid()}.{time.time_ns()}.tmp"
)
try:
subprocess.run(
[openssl, "genpkey", "-algorithm", "ED25519", "-out", str(temporary_private)],
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
text=True,
)
temporary_stat = temporary_private.lstat()
if (stat.S_ISLNK(temporary_stat.st_mode) or not stat.S_ISREG(temporary_stat.st_mode)
or temporary_stat.st_size < 80 or temporary_stat.st_size > 8192):
die("generated Foundry EDP private key is invalid")
os.chown(temporary_private, 0, 0)
temporary_private.chmod(0o400)
with temporary_private.open("rb") as handle:
os.fsync(handle.fileno())
os.replace(temporary_private, FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE)
fsync_directory(FOUNDRY_EDP_MANAGED_PROVISIONER_SECRET_DIR)
private_created = True
except (OSError, subprocess.CalledProcessError) as error:
die(f"failed to generate Foundry EDP signing key: {type(error).__name__}")
finally:
if temporary_private.exists():
temporary_private.unlink()
private_stat = FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE.lstat()
if (stat.S_ISLNK(private_stat.st_mode) or not stat.S_ISREG(private_stat.st_mode)
or private_stat.st_uid != 0 or private_stat.st_gid != 0
or stat.S_IMODE(private_stat.st_mode) != 0o400
or private_stat.st_size < 80 or private_stat.st_size > 8192):
die("Foundry EDP private key has unsafe ownership, mode, or size")
try:
derived = subprocess.run(
[openssl, "pkey", "-in", str(FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE), "-pubout"],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
).stdout
derived_der = subprocess.run(
[
openssl,
"pkey",
"-in",
str(FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE),
"-pubout",
"-outform",
"DER",
],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
).stdout
except (OSError, subprocess.CalledProcessError) as error:
die(f"Foundry EDP private key validation failed: {type(error).__name__}")
if (not derived.startswith(b"-----BEGIN PUBLIC KEY-----\n")
or not derived.rstrip().endswith(b"-----END PUBLIC KEY-----")
or len(derived) > 8192
or len(derived_der) != 44
or not derived_der.startswith(bytes.fromhex("302a300506032b6570032100"))):
die("Foundry EDP signing key must be Ed25519")
if public_stat is not None:
if (stat.S_ISLNK(public_stat.st_mode) or not stat.S_ISREG(public_stat.st_mode)
or public_stat.st_uid != 0
or public_stat.st_gid != EXTERNAL_DATA_PLANE_RUNTIME_GID
or stat.S_IMODE(public_stat.st_mode) != 0o440
or public_stat.st_size < 80 or public_stat.st_size > 8192):
die("Foundry EDP public key has unsafe ownership, mode, or size")
if FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE.read_bytes() != derived:
die("Foundry EDP public key does not match the installed private key")
return "created" if private_created else "reused"
temporary_public = (
FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR
/ f".public-key.{os.getpid()}.{time.time_ns()}.tmp"
)
descriptor = None
try:
descriptor = os.open(str(temporary_public), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o440)
os.write(descriptor, derived)
os.fsync(descriptor)
os.fchown(descriptor, 0, EXTERNAL_DATA_PLANE_RUNTIME_GID)
os.fchmod(descriptor, 0o440)
os.close(descriptor)
descriptor = None
os.replace(temporary_public, FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE)
fsync_directory(FOUNDRY_EDP_MANAGED_PROVISIONER_TRUST_DIR)
finally:
if descriptor is not None:
os.close(descriptor)
if temporary_public.exists():
temporary_public.unlink()
return "created"
def ensure_engine_data_product_grant_private_state(include_reader=False):
@ -6394,9 +6560,12 @@ def plan_artifact(artifact):
if component == "module-foundry":
print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}")
print(f"runtime_grants=runner-managed:{FOUNDRY_BINDING_GRANTS_DIR}")
print(f"runtime_private_key=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PRIVATE_KEY_FILE}")
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
if touches_external_data_plane:
print(f"runtime_secret=runner-managed:{EXTERNAL_DATA_PLANE_PROVISIONER_SECRET_FILE}")
print(f"runtime_grants=runner-managed:{EXTERNAL_DATA_PLANE_READER_GRANTS_DIR}")
print(f"runtime_public_trust=runner-managed:{FOUNDRY_EDP_MANAGED_PROVISIONER_PUBLIC_KEY_FILE}")
if touches_engine_credential_sink(component, entries):
print(f"runtime_identity_key_id={ENGINE_CREDENTIAL_PROVISIONER_KEY_ID}")
print(f"runtime_private_key=runner-managed:{ENGINE_CREDENTIAL_PROVISIONER_PRIVATE_KEY_FILE}")
@ -7136,6 +7305,7 @@ def prepare_component_runtime(component, entries=None):
if component == "module-foundry":
ensure_map_gateway_admin_secret()
ensure_foundry_edp_managed_provisioner_keypair()
ensure_root_owned_grant_directory(
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
@ -7195,6 +7365,7 @@ def prepare_component_runtime(component, entries=None):
if touches_external_data_plane:
ensure_external_data_plane_provisioner_secret()
ensure_engine_edp_managed_provisioner_keypair()
ensure_foundry_edp_managed_provisioner_keypair()
ensure_root_owned_grant_directory(
EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
@ -7292,12 +7463,37 @@ def external_data_plane_healthcheck(require_managed=True):
expected_json["managedWriterBindingLifetime"] = "explicit-revoke"
if 'managedReaderBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled"' in server_text:
expected_json["managedReaderBindingLifetime"] = "explicit-revoke"
if 'foundryReaderBindingProvisioning: config.foundryProvisionerApiEnabled ? "digest+server-resolved-source" : "disabled"' in server_text:
expected_json["foundryReaderBindingProvisioning"] = "digest+server-resolved-source"
expected_json["foundryReaderBindingLifetime"] = "explicit-revoke"
return {
"url": "http://127.0.0.1:18106/healthz",
"expected_json": expected_json,
}
def module_foundry_healthcheck():
expected_json = {
"status": "ok",
"service": "nodedc-module-foundry",
}
server_source = component_root("module-foundry") / "server/catalog-server.mjs"
try:
server_text = server_source.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
server_text = ""
if "dataProductConsumerProvisioner" in server_text:
expected_json["dataProductConsumerProvisioner"] = {
"configured": True,
"auth": "dedicated-ed25519-service-identity",
"sourceScope": "external-data-plane-resolved",
}
return {
"url": "http://172.22.0.222:9920/healthz",
"expected_json": expected_json,
}
def component_healthchecks(component, entries=None, services=None):
if is_engine_n8n_transition(component, entries):
# The transition does not restart the Engine UI/backend generation.
@ -7311,6 +7507,8 @@ def component_healthchecks(component, entries=None, services=None):
return ("http://127.0.0.1:3001/health",)
if component == "platform" and entries is not None and is_platform_provider_catalog_only(entries):
return ()
if component == "module-foundry":
return (module_foundry_healthcheck(),)
if (
(
is_engine_data_product_publish_grant_slice(component, entries)

View File

@ -81,6 +81,14 @@ class ExternalDataPlaneArtifactTest(unittest.TestCase):
"source: /volume1/docker/nodedc-platform/trust/engine-managed-provisioner",
compose,
)
self.assertIn(
"source: /volume1/docker/nodedc-platform/trust/foundry-managed-provisioner",
compose,
)
self.assertIn(
"EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: /run/nodedc-trust/foundry-managed-provisioner/public-key.pem",
compose,
)
self.assertNotIn("private-key.pem", compose)
self.assertNotIn(b"-----BEGIN PRIVATE KEY-----", b"\n".join(regular_payloads))

View File

@ -293,6 +293,48 @@ class CanonicalPlatformRegistryTest(unittest.TestCase):
keypair.assert_called_once_with()
private_state.assert_called_once_with()
def test_module_foundry_runtime_prepares_separate_edp_identity_and_grant_roots(self):
with (
mock.patch.object(RUNNER, "ensure_map_gateway_admin_secret") as map_secret,
mock.patch.object(RUNNER, "ensure_foundry_edp_managed_provisioner_keypair") as foundry_keypair,
mock.patch.object(RUNNER, "ensure_root_owned_grant_directory") as grant_directory,
):
RUNNER.prepare_component_runtime("module-foundry", ("server/catalog-server.mjs",))
map_secret.assert_called_once_with()
foundry_keypair.assert_called_once_with()
self.assertEqual(
grant_directory.call_args_list,
[
mock.call(
RUNNER.EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
),
mock.call(
RUNNER.FOUNDRY_BINDING_GRANTS_DIR,
"foundry binding grants",
),
],
)
def test_edp_runtime_prepares_engine_and_foundry_public_trust_independently(self):
entries = ("platform/services/external-data-plane/src/server.mjs",)
with (
mock.patch.object(RUNNER, "ensure_external_data_plane_provisioner_secret") as legacy_secret,
mock.patch.object(RUNNER, "ensure_engine_edp_managed_provisioner_keypair") as engine_keypair,
mock.patch.object(RUNNER, "ensure_foundry_edp_managed_provisioner_keypair") as foundry_keypair,
mock.patch.object(RUNNER, "ensure_root_owned_grant_directory") as grant_directory,
):
RUNNER.prepare_component_runtime("platform", entries)
legacy_secret.assert_called_once_with()
engine_keypair.assert_called_once_with()
foundry_keypair.assert_called_once_with()
grant_directory.assert_called_once_with(
RUNNER.EXTERNAL_DATA_PLANE_READER_GRANTS_DIR,
"external data plane reader grants",
)
def test_partial_publish_path_has_no_publish_runtime_side_effects(self):
entries = ("nodedc-source/server/dataProductPublishGrant/store.js",)
self.assertTrue(RUNNER.touches_engine_data_product_publish_grant(entries))
@ -441,6 +483,46 @@ class CanonicalPlatformRegistryTest(unittest.TestCase):
(RUNNER.external_data_plane_healthcheck(),),
)
def test_edp_healthcheck_requires_foundry_provisioner_for_the_new_source(self):
with tempfile.TemporaryDirectory(prefix="nodedc-edp-foundry-health-") as directory:
root = Path(directory)
source = root / "platform/services/external-data-plane/src/server.mjs"
source.parent.mkdir(parents=True)
source.write_text(
'foundryReaderBindingProvisioning: config.foundryProvisionerApiEnabled ? "digest+server-resolved-source" : "disabled"\n',
encoding="utf-8",
)
original_root = RUNNER.COMPONENTS["platform"]["payload_root"]
RUNNER.COMPONENTS["platform"]["payload_root"] = root
try:
expected = RUNNER.external_data_plane_healthcheck()["expected_json"]
finally:
RUNNER.COMPONENTS["platform"]["payload_root"] = original_root
self.assertEqual(
expected["foundryReaderBindingProvisioning"],
"digest+server-resolved-source",
)
self.assertEqual(expected["foundryReaderBindingLifetime"], "explicit-revoke")
def test_module_foundry_healthcheck_tracks_candidate_and_rollback_source(self):
with tempfile.TemporaryDirectory(prefix="nodedc-foundry-health-") as directory:
root = Path(directory)
source = root / "server/catalog-server.mjs"
source.parent.mkdir(parents=True)
original_root = RUNNER.COMPONENTS["module-foundry"]["payload_root"]
RUNNER.COMPONENTS["module-foundry"]["payload_root"] = root
try:
source.write_text("const dataProductConsumerProvisioner = true;\n", encoding="utf-8")
candidate = RUNNER.module_foundry_healthcheck()["expected_json"]
source.write_text("const previous = true;\n", encoding="utf-8")
rollback = RUNNER.module_foundry_healthcheck()["expected_json"]
finally:
RUNNER.COMPONENTS["module-foundry"]["payload_root"] = original_root
self.assertEqual(candidate["dataProductConsumerProvisioner"]["configured"], True)
self.assertNotIn("dataProductConsumerProvisioner", rollback)
def test_edp_runtime_never_probes_unselected_platform_services(self):
entries = ("platform/services/external-data-plane/src/server.mjs",)
services = ("external-data-plane",)

View File

@ -111,6 +111,12 @@ EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID=engine-edp-managed-provisioner-v1
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE=nodedc-external-data-plane.managed-provisioning.v1
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS=60
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES=10000
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED=true
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID=nodedc-module-foundry
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID=foundry-edp-managed-provisioner-v1
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE=nodedc-external-data-plane.managed-provisioning.v1
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS=60
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES=10000
NOTIFICATION_PG_DB=nodedc_notifications
NOTIFICATION_PG_USER=nodedc_notifications

View File

@ -68,6 +68,16 @@ services:
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: ${EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE:-nodedc-external-data-plane.managed-provisioning.v1}
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS: ${EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS:-60}
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES: ${EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES:-10000}
# Foundry owns a separate managed-provisioner identity. It submits only
# a consumer target digest and Data Product id; EDP resolves the active
# writer scope internally and returns no provider identity.
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED:-true}
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: /run/nodedc-trust/foundry-managed-provisioner/public-key.pem
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID:-nodedc-module-foundry}
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID:-foundry-edp-managed-provisioner-v1}
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE:-nodedc-external-data-plane.managed-provisioning.v1}
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS:-60}
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES: ${EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES:-10000}
volumes:
- type: bind
source: /volume1/docker/nodedc-platform/secrets/external-data-plane-provisioner/token
@ -81,6 +91,12 @@ services:
read_only: true
bind:
create_host_path: false
- type: bind
source: /volume1/docker/nodedc-platform/trust/foundry-managed-provisioner
target: /run/nodedc-trust/foundry-managed-provisioner
read_only: true
bind:
create_host_path: false
expose:
- "18106"
ports:

View File

@ -8,6 +8,7 @@ import {
export function readConfig(env = process.env) {
const provisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_PROVISIONING_ENABLED, false);
const managedProvisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED, false);
const foundryProvisionerApiEnabled = boolean(env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED, false);
const managedProvisionerMaxSkewSeconds = integer(
env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_MAX_SKEW_SECONDS,
60,
@ -21,6 +22,7 @@ export function readConfig(env = process.env) {
internalAccessToken: optional(env.NODEDC_INTERNAL_ACCESS_TOKEN),
provisionerApiEnabled,
managedProvisionerApiEnabled,
foundryProvisionerApiEnabled,
provisionerAccessToken: provisionerApiEnabled ? secretFile(env.EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE) : "",
managedProvisionerPublicKey: managedProvisionerApiEnabled
? loadManagedProvisionerPublicKeyFile(env.EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE)
@ -49,6 +51,38 @@ export function readConfig(env = process.env) {
100,
100_000,
),
foundryProvisionerPublicKey: foundryProvisionerApiEnabled
? loadManagedProvisionerPublicKeyFile(env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE)
: null,
foundryProvisionerServiceId: foundryProvisionerApiEnabled
? validateManagedProvisionerIdentity(
required(env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID, "EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID"),
"service_id",
)
: "",
foundryProvisionerKeyId: foundryProvisionerApiEnabled
? validateManagedProvisionerIdentity(
required(env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID, "EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID"),
"key_id",
)
: "",
foundryProvisionerAudience: foundryProvisionerApiEnabled
? validateManagedProvisionerAudience(
required(env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE, "EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE"),
)
: "",
foundryProvisionerMaxSkewMs: integer(
env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS,
60,
5,
300,
) * 1000,
foundryProvisionerReplayCacheMaxEntries: integer(
env.EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES,
10_000,
100,
100_000,
),
rawRetentionDays: integer(env.EXTERNAL_DATA_PLANE_RAW_RETENTION_DAYS, 14, 1, 3650),
maxBatchBytes: integer(env.EXTERNAL_DATA_PLANE_MAX_BATCH_BYTES, 5 * 1024 * 1024, 1024, 50 * 1024 * 1024),
maxFactsPerPublish: integer(env.EXTERNAL_DATA_PLANE_MAX_FACTS_PER_PUBLISH, 5000, 1, 100_000),

View File

@ -11,6 +11,13 @@ const MANAGED_REQUEST_KEYS = new Set([
"generation",
"capabilityDigest",
]);
const MANAGED_CONSUMER_REQUEST_KEYS = new Set([
"allowedDataProductIds",
"expiresAt",
"generation",
"capabilityDigest",
]);
const MANAGED_CONSUMER_PLAN_KEYS = new Set(["allowedDataProductIds"]);
const SOURCE_KEYS = new Set(["tenantId", "connectionId", "providerId"]);
const SHA256_DIGEST = /^[a-f0-9]{64}$/;
@ -82,6 +89,38 @@ export function normalizeManagedReaderBindingRequest(value) {
});
}
export function normalizeManagedConsumerReaderPlanRequest(value) {
if (!isPlainObject(value) || containsSecretLikeKey(value) || !hasOnlyKeys(value, MANAGED_CONSUMER_PLAN_KEYS)) {
throw readerError("managed_consumer_reader_plan_request_invalid");
}
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
if (!allowedDataProductIds.length) throw readerError("managed_consumer_reader_scope_invalid");
return Object.freeze({ allowedDataProductIds: Object.freeze([...allowedDataProductIds].sort()) });
}
export function normalizeManagedConsumerReaderBindingRequest(value) {
if (!isPlainObject(value) || containsSecretLikeKey(value) || !hasOnlyKeys(value, MANAGED_CONSUMER_REQUEST_KEYS)) {
throw readerError("managed_consumer_reader_binding_request_invalid");
}
const allowedDataProductIds = uniqueIdentifiers(value.allowedDataProductIds);
if (!allowedDataProductIds.length) throw readerError("managed_consumer_reader_scope_invalid");
if (value.expiresAt !== null) throw readerError("managed_consumer_reader_must_be_durable");
const generation = Number(value.generation);
if (!Number.isSafeInteger(generation) || generation < 1 || generation > 2_147_483_647) {
throw readerError("managed_consumer_reader_generation_invalid");
}
const capabilityDigest = String(value.capabilityDigest || "").toLowerCase();
if (!SHA256_DIGEST.test(capabilityDigest)) {
throw readerError("managed_consumer_reader_capability_digest_invalid");
}
return Object.freeze({
allowedDataProductIds: Object.freeze([...allowedDataProductIds].sort()),
expiresAt: null,
generation,
capabilityDigest,
});
}
export function readerBindingRequestHash(policy) {
const canonical = JSON.stringify({
tenantId: policy.tenantId,
@ -95,6 +134,15 @@ export function readerBindingRequestHash(policy) {
return createHash("sha256").update(canonical, "utf8").digest("hex");
}
export function consumerReaderBindingRequestHash(policy) {
return createHash("sha256").update(JSON.stringify({
allowedDataProductIds: [...policy.allowedDataProductIds].sort(),
expiresAt: null,
generation: policy.generation,
capabilityDigest: policy.capabilityDigest,
}), "utf8").digest("hex");
}
export function assertReaderProduct(binding, dataProductId, now = new Date()) {
const expired = binding?.expiresAt !== null && new Date(binding?.expiresAt) <= now;
if (!isPlainObject(binding) || binding.active !== true || expired) {
@ -124,6 +172,21 @@ export function safeReaderBinding(binding) {
};
}
export function safeManagedConsumerReaderBinding(binding) {
return {
id: binding.id,
bindingKey: binding.bindingKey,
generation: Number(binding.generation),
allowedDataProductIds: uniqueIdentifiers(binding.allowedDataProductIds),
active: binding.active === true,
expiresAt: binding.expiresAt === null ? null : new Date(binding.expiresAt).toISOString(),
createdAt: binding.createdAt ? new Date(binding.createdAt).toISOString() : undefined,
rotatedAt: binding.rotatedAt ? new Date(binding.rotatedAt).toISOString() : undefined,
revokedAt: binding.revokedAt ? new Date(binding.revokedAt).toISOString() : undefined,
sourceScope: "resolved-server-side",
};
}
function readerError(code, status = 400) {
return Object.assign(new Error(code), { status, code });
}

View File

@ -41,6 +41,47 @@ export async function resolveManagedReaderSourceConnection(db, policy) {
throw sourceScopeError("managed_reader_source_scope_ambiguous", 409);
}
export async function resolveManagedReaderScope(db, allowedDataProductIdsValue) {
const allowedDataProductIds = uniqueIdentifiers(allowedDataProductIdsValue);
if (!allowedDataProductIds.length) {
throw sourceScopeError("managed_consumer_reader_scope_request_invalid", 400);
}
const result = await db.query(
`select distinct candidate.tenant_id as "tenantId",
candidate.connection_id as "connectionId", candidate.provider_id as "providerId"
from external_data_plane_writer_bindings as candidate
where candidate.active = true
and (candidate.expires_at is null or candidate.expires_at > now())
and not exists (
select 1
from unnest($1::text[]) as requested(data_product_id)
where not exists (
select 1
from external_data_plane_writer_bindings as coverage
where coverage.tenant_id = candidate.tenant_id
and coverage.connection_id = candidate.connection_id
and coverage.provider_id = candidate.provider_id
and coverage.active = true
and (coverage.expires_at is null or coverage.expires_at > now())
and coverage.allowed_data_product_ids ? requested.data_product_id
)
)
order by candidate.tenant_id asc, candidate.provider_id asc, candidate.connection_id asc
limit 3`,
[allowedDataProductIds],
);
const candidates = result.rows
.map((row) => ({
tenantId: identifier(row.tenantId),
connectionId: identifier(row.connectionId),
providerId: identifier(row.providerId),
}))
.filter((row) => row.tenantId && row.connectionId && row.providerId);
if (candidates.length === 1) return Object.freeze(candidates[0]);
if (!candidates.length) throw sourceScopeError("managed_consumer_reader_source_scope_not_found", 409);
throw sourceScopeError("managed_consumer_reader_source_scope_ambiguous", 409);
}
export async function backfillManagedReaderSourceConnections(db) {
const unresolved = await db.query(
`select id, tenant_id as "tenantId", connection_id as "connectionId",

View File

@ -25,14 +25,18 @@ import {
} from "./managed-provisioner-auth.mjs";
import {
assertReaderProduct,
consumerReaderBindingRequestHash,
createReaderToken,
hashReaderToken,
normalizeManagedConsumerReaderBindingRequest,
normalizeManagedConsumerReaderPlanRequest,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeManagedConsumerReaderBinding,
safeReaderBinding,
} from "./reader-binding.mjs";
import { resolveManagedReaderSourceConnection } from "./reader-source-scope.mjs";
import { resolveManagedReaderScope, resolveManagedReaderSourceConnection } from "./reader-source-scope.mjs";
import { migrate } from "./schema.mjs";
import {
createWriterToken,
@ -60,6 +64,19 @@ const verifyManagedProvisionerRequest = config.managedProvisionerApiEnabled
replayCache: managedProvisionerReplayCache,
})
: null;
const foundryProvisionerReplayCache = config.foundryProvisionerApiEnabled
? new ManagedProvisionerReplayCache({ maxEntries: config.foundryProvisionerReplayCacheMaxEntries })
: null;
const verifyFoundryProvisionerRequest = config.foundryProvisionerApiEnabled
? createManagedProvisionerRequestVerifier({
publicKey: config.foundryProvisionerPublicKey,
serviceId: config.foundryProvisionerServiceId,
keyId: config.foundryProvisionerKeyId,
audience: config.foundryProvisionerAudience,
maxSkewMs: config.foundryProvisionerMaxSkewMs,
replayCache: foundryProvisionerReplayCache,
})
: null;
const pool = new Pool({ connectionString: config.databaseUrl, max: config.databasePoolSize });
const app = express();
const httpServer = createServer(app);
@ -106,6 +123,8 @@ app.get("/healthz", asyncRoute(async (_req, res) => {
managedWriterBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
managedReaderBindingProvisioning: config.managedProvisionerApiEnabled ? "enabled" : "disabled",
managedReaderBindingLifetime: config.managedProvisionerApiEnabled ? "explicit-revoke" : "disabled",
foundryReaderBindingProvisioning: config.foundryProvisionerApiEnabled ? "digest+server-resolved-source" : "disabled",
foundryReaderBindingLifetime: config.foundryProvisionerApiEnabled ? "explicit-revoke" : "disabled",
rawRetentionSweep: {
mode: "server-scheduled",
lastSweepAt: lastRetentionSweepAt,
@ -378,6 +397,143 @@ app.post("/internal/data-plane/v1/writer-bindings/:bindingId/revoke", requirePro
res.json({ ok: true, writerBinding: safeWriterBinding(result.rows[0]) });
}));
app.post("/internal/data-plane/v1/consumer-reader-bindings/plan", requireFoundryProvisionerApi, asyncRoute(async (req, res) => {
const policy = normalizeManagedConsumerReaderPlanRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
await resolveManagedReaderScope(pool, policy.allowedDataProductIds);
const dataProducts = [];
for (const dataProductId of policy.allowedDataProductIds) {
const definition = await loadDataProductDefinition(pool, dataProductId);
if (!definition || definition.active === false) throw httpError(404, "data_product_not_found");
dataProducts.push(safeDataProductDefinition(definition));
}
res.set("Cache-Control", "no-store, max-age=0");
res.json({ ok: true, sourceScope: "resolved-server-side", dataProducts });
}));
app.put("/internal/data-plane/v1/consumer-reader-bindings/by-key/:bindingKey", requireFoundryProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_consumer_reader_binding_key_invalid");
const policy = normalizeManagedConsumerReaderBindingRequest(req.body);
await assertRegisteredProductIds(policy.allowedDataProductIds);
const requestHash = consumerReaderBindingRequestHash(policy);
const client = await pool.connect();
let response;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", request_hash as "requestHash", generation,
tenant_id as "tenantId", connection_id as "connectionId",
source_connection_id as "sourceConnectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, policy.generation],
);
if (existing.rowCount) {
const binding = existing.rows[0];
if (binding.requestHash !== requestHash) throw httpError(409, "managed_consumer_reader_binding_request_conflict");
if (binding.active !== true || binding.expiresAt !== null) {
throw httpError(409, "managed_consumer_reader_binding_generation_inactive");
}
response = { status: 200, idempotent: true, binding };
} else {
const source = await resolveManagedReaderScope(client, policy.allowedDataProductIds);
const inserted = await client.query(
`insert into external_data_plane_reader_bindings (
id, token_hash, binding_key, request_hash, generation,
tenant_id, connection_id, source_connection_id, provider_id,
allowed_data_product_ids, expires_at
) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::jsonb, $11)
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId",
source_connection_id as "sourceConnectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[
randomUUID(),
policy.capabilityDigest,
bindingKey,
requestHash,
policy.generation,
source.tenantId,
source.connectionId,
source.connectionId,
source.providerId,
JSON.stringify(policy.allowedDataProductIds),
policy.expiresAt,
],
);
response = { status: 201, idempotent: false, binding: inserted.rows[0] };
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
if (error?.code === "23505") throw httpError(409, "managed_consumer_reader_capability_digest_conflict");
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.status(response.status).json({
ok: true,
idempotent: response.idempotent,
readerBinding: safeManagedConsumerReaderBinding(response.binding),
});
}));
app.post("/internal/data-plane/v1/consumer-reader-bindings/by-key/:bindingKey/generations/:generation/revoke", requireFoundryProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_consumer_reader_binding_key_invalid");
const generation = requirePositiveInteger(req.params.generation, "managed_consumer_reader_generation_invalid");
const client = await pool.connect();
let binding;
let idempotent;
try {
await client.query("begin");
await client.query("select pg_advisory_xact_lock(hashtextextended($1, 0))", [bindingKey]);
const existing = await client.query(
`select id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId",
source_connection_id as "sourceConnectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"
from external_data_plane_reader_bindings
where binding_key = $1 and generation = $2
for update`,
[bindingKey, generation],
);
if (!existing.rowCount) throw httpError(404, "managed_consumer_reader_binding_not_found");
if (existing.rows[0].active === true) {
const revoked = await client.query(
`update external_data_plane_reader_bindings
set active = false, revoked_at = now()
where binding_key = $1 and generation = $2 and active = true
returning id, binding_key as "bindingKey", generation,
tenant_id as "tenantId", connection_id as "connectionId",
source_connection_id as "sourceConnectionId", provider_id as "providerId",
allowed_data_product_ids as "allowedDataProductIds", expires_at as "expiresAt",
active, created_at as "createdAt", rotated_at as "rotatedAt", revoked_at as "revokedAt"`,
[bindingKey, generation],
);
binding = revoked.rows[0];
idempotent = false;
} else {
binding = existing.rows[0];
idempotent = true;
}
await client.query("commit");
} catch (error) {
await client.query("rollback");
throw error;
} finally {
client.release();
}
res.set("Cache-Control", "no-store, max-age=0");
res.json({ ok: true, idempotent, readerBinding: safeManagedConsumerReaderBinding(binding) });
}));
app.put("/internal/data-plane/v1/reader-bindings/by-key/:bindingKey", requireManagedProvisionerApi, asyncRoute(async (req, res) => {
const bindingKey = requireIdentifier(req.params.bindingKey, "managed_reader_binding_key_invalid");
const policy = normalizeManagedReaderBindingRequest(req.body);
@ -918,6 +1074,17 @@ function requireManagedProvisionerApi(req, _res, next) {
}
}
function requireFoundryProvisionerApi(req, _res, next) {
if (!config.foundryProvisionerApiEnabled) return next(httpError(503, "foundry_provisioner_api_disabled"));
if (!verifyFoundryProvisionerRequest) return next(httpError(503, "foundry_provisioner_api_not_configured"));
try {
req.managedProvisionerIdentity = verifyFoundryProvisionerRequest(req);
return next();
} catch (error) {
return next(error);
}
}
function requireProvisionerCredential(req, next) {
if (!config.provisionerAccessToken) return next(httpError(503, "provisioner_api_not_configured"));
const value = bearerToken(req);

View File

@ -24,12 +24,22 @@ const managedProvisionerKeyId = "engine-edp-managed-provisioner-v1";
const managedProvisionerAudience = "nodedc-external-data-plane.managed-provisioning.v1";
const managedProvisionerPublicKeyPath = join(directory, "engine-public-key.pem");
const { privateKey: managedProvisionerPrivateKey, publicKey: managedProvisionerPublicKey } = generateKeyPairSync("ed25519");
const foundryProvisionerServiceId = "nodedc-module-foundry";
const foundryProvisionerKeyId = "foundry-edp-managed-provisioner-v1";
const foundryProvisionerAudience = "nodedc-external-data-plane.managed-provisioning.v1";
const foundryProvisionerPublicKeyPath = join(directory, "foundry-public-key.pem");
const { privateKey: foundryProvisionerPrivateKey, publicKey: foundryProvisionerPublicKey } = generateKeyPairSync("ed25519");
await writeFile(secretPath, `${provisionerSecret}\n`, { mode: 0o600 });
await writeFile(
managedProvisionerPublicKeyPath,
managedProvisionerPublicKey.export({ type: "spki", format: "pem" }),
{ mode: 0o600 },
);
await writeFile(
foundryProvisionerPublicKeyPath,
foundryProvisionerPublicKey.export({ type: "spki", format: "pem" }),
{ mode: 0o600 },
);
const child = spawn(process.execPath, ["src/server.mjs"], {
cwd: new URL("..", import.meta.url),
@ -44,6 +54,11 @@ const child = spawn(process.execPath, ["src/server.mjs"], {
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_SERVICE_ID: managedProvisionerServiceId,
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_KEY_ID: managedProvisionerKeyId,
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_AUDIENCE: managedProvisionerAudience,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED: "true",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: foundryProvisionerPublicKeyPath,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID: foundryProvisionerServiceId,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID: foundryProvisionerKeyId,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE: foundryProvisionerAudience,
EXTERNAL_DATA_PLANE_RETENTION_SWEEP_MS: "60000",
EXTERNAL_DATA_PLANE_STREAM_POLL_MS: "250",
EXTERNAL_DATA_PLANE_STREAM_HEARTBEAT_MS: "5000",
@ -249,6 +264,62 @@ try {
});
assert.equal(rejectedManagedReader.status, 401);
const foundryPlanPath = "/internal/data-plane/v1/consumer-reader-bindings/plan";
const foundryPlanBody = { allowedDataProductIds: [productId] };
const engineCannotProvisionFoundryGrant = await signedFetch(foundryPlanPath, {
method: "POST",
body: foundryPlanBody,
});
assert.equal(engineCannotProvisionFoundryGrant.status, 401);
assert.equal((await engineCannotProvisionFoundryGrant.json()).error, "managed_provisioner_identity_mismatch");
const foundryPlan = await jsonRequest(foundryPlanPath, {
method: "POST",
foundrySignature: true,
body: foundryPlanBody,
});
assert.equal(foundryPlan.sourceScope, "resolved-server-side");
assert.deepEqual(foundryPlan.dataProducts.map((value) => value.id), [productId]);
assert.equal(/tenant|provider|connection/i.test(JSON.stringify(foundryPlan)), false);
const foundryReaderToken = "ndc_edprb_foundry_api_test_0123456789abcdefghijklmnopqrstuvwxyz";
const foundryReaderBody = {
allowedDataProductIds: [productId],
expiresAt: null,
generation: 1,
capabilityDigest: createHash("sha256").update(foundryReaderToken, "utf8").digest("hex"),
};
const foundryReaderPath = "/internal/data-plane/v1/consumer-reader-bindings/by-key/fndrc-api-test-positions";
const foundryReaderResults = await Promise.all([
rawJsonRequest(foundryReaderPath, { method: "PUT", foundrySignature: true, body: foundryReaderBody }),
rawJsonRequest(foundryReaderPath, { method: "PUT", foundrySignature: true, body: foundryReaderBody }),
]);
assert.deepEqual(foundryReaderResults.map(({ value }) => value.idempotent).sort(), [false, true]);
assert.equal(foundryReaderResults[0].value.readerBinding.id, foundryReaderResults[1].value.readerBinding.id);
assert.equal(JSON.stringify(foundryReaderResults[0].value).includes(foundryReaderBody.capabilityDigest), false);
assert.equal(/tenant|provider|connection/i.test(JSON.stringify(foundryReaderResults[0].value)), false);
assert.equal("token" in foundryReaderResults[0].value, false);
const foundrySnapshot = await jsonRequest(
`/internal/data-plane/v1/data-products/${productId}/snapshot`,
{ token: foundryReaderToken },
);
assert.equal(validateDataProductSnapshot(foundrySnapshot).ok, true);
assert.equal(foundrySnapshot.facts.length, 1);
const foundryRevoke = await jsonRequest(`${foundryReaderPath}/generations/1/revoke`, {
method: "POST",
foundrySignature: true,
});
assert.equal(foundryRevoke.idempotent, false);
assert.equal(foundryRevoke.readerBinding.active, false);
const foundryRevokeRetry = await jsonRequest(`${foundryReaderPath}/generations/1/revoke`, {
method: "POST",
foundrySignature: true,
});
assert.equal(foundryRevokeRetry.idempotent, true);
const rejectedFoundryReader = await fetch(`${baseUrl}/internal/data-plane/v1/reader/data-products`, {
headers: { Authorization: `Bearer ${foundryReaderToken}` },
});
assert.equal(rejectedFoundryReader.status, 401);
const secondWriterToken = "ndc_edpwb_second_source_api_test_0123456789abcdefghijklmnopqrstuvwxyz";
await jsonRequest("/internal/data-plane/v1/writer-bindings/by-key/engine.api-connection-2.positions", {
method: "PUT",
@ -259,6 +330,12 @@ try {
capabilityDigest: createHash("sha256").update(secondWriterToken, "utf8").digest("hex"),
},
});
const ambiguousFoundryPlan = await foundrySignedFetch(foundryPlanPath, {
method: "POST",
body: foundryPlanBody,
});
assert.equal(ambiguousFoundryPlan.status, 409);
assert.equal((await ambiguousFoundryPlan.json()).error, "managed_consumer_reader_source_scope_ambiguous");
const ambiguousReaderToken = "ndc_edprb_ambiguous_api_test_0123456789abcdefghijklmnopqrstuvwxyz";
const ambiguousReader = await signedFetch(
"/internal/data-plane/v1/reader-bindings/by-key/engine.api-ambiguous.positions-reader",
@ -413,19 +490,20 @@ function publish(runId, observedAt, longitude) {
};
}
async function jsonRequest(path, { method = "GET", token, managedSignature = false, body } = {}) {
const { response, value } = await rawJsonRequest(path, { method, token, managedSignature, body });
async function jsonRequest(path, { method = "GET", token, managedSignature = false, foundrySignature = false, body } = {}) {
const { response, value } = await rawJsonRequest(path, { method, token, managedSignature, foundrySignature, body });
if (!response.ok) throw new Error(`request_failed:${response.status}:${value.error}`);
return value;
}
async function rawJsonRequest(path, { method = "GET", token, managedSignature = false, body } = {}) {
async function rawJsonRequest(path, { method = "GET", token, managedSignature = false, foundrySignature = false, body } = {}) {
const rawBody = body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(body), "utf8");
const response = await fetch(`${baseUrl}${path}`, {
method,
headers: {
...(token ? { Authorization: `Bearer ${token}` } : {}),
...(managedSignature ? signedManagedHeaders(path, method, rawBody) : {}),
...(foundrySignature ? signedFoundryHeaders(path, method, rawBody) : {}),
...(body === undefined ? {} : { "Content-Type": "application/json" }),
},
body: body === undefined ? undefined : rawBody,
@ -447,6 +525,18 @@ async function signedFetch(path, { method, body } = {}) {
});
}
async function foundrySignedFetch(path, { method, body } = {}) {
const rawBody = body === undefined ? Buffer.alloc(0) : Buffer.from(JSON.stringify(body), "utf8");
return fetch(`${baseUrl}${path}`, {
method,
headers: {
...signedFoundryHeaders(path, method, rawBody),
...(body === undefined ? {} : { "Content-Type": "application/json" }),
},
body: body === undefined ? undefined : rawBody,
});
}
function signedManagedHeaders(path, method, rawBody) {
const timestamp = new Date().toISOString();
const nonce = randomBytes(24).toString("base64url");
@ -473,6 +563,44 @@ function signedManagedHeaders(path, method, rawBody) {
};
}
function signedFoundryHeaders(path, method, rawBody) {
return signedHeaders({
path,
method,
rawBody,
audience: foundryProvisionerAudience,
serviceId: foundryProvisionerServiceId,
keyId: foundryProvisionerKeyId,
privateKey: foundryProvisionerPrivateKey,
});
}
function signedHeaders({ path, method, rawBody, audience, serviceId, keyId, privateKey }) {
const timestamp = new Date().toISOString();
const nonce = randomBytes(24).toString("base64url");
const bodySha256 = sha256RawBody(rawBody);
const payload = managedProvisionerSigningPayload({
audience,
serviceId,
keyId,
method,
path,
timestamp,
nonce,
bodySha256,
});
const signature = sign(null, Buffer.from(payload, "utf8"), privateKey).toString("base64url");
return {
[MANAGED_PROVISIONER_HEADERS.serviceId]: serviceId,
[MANAGED_PROVISIONER_HEADERS.keyId]: keyId,
[MANAGED_PROVISIONER_HEADERS.audience]: audience,
[MANAGED_PROVISIONER_HEADERS.timestamp]: timestamp,
[MANAGED_PROVISIONER_HEADERS.nonce]: nonce,
[MANAGED_PROVISIONER_HEADERS.bodySha256]: bodySha256,
[MANAGED_PROVISIONER_HEADERS.signature]: signature,
};
}
function assertOneTimeCapabilityResponse(response) {
assert.equal(response.headers.get("cache-control"), "no-store, max-age=0");
assert.equal(response.headers.get("pragma"), "no-cache");

View File

@ -66,6 +66,22 @@ try {
assert.equal(managedConfig.managedProvisionerAudience, "nodedc-external-data-plane.managed-provisioning.v1");
assert.equal(managedConfig.managedProvisionerMaxSkewMs, 45_000);
assert.equal(managedConfig.managedProvisionerReplayCacheMaxEntries, 500);
const foundryConfig = readConfig({
...base,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONING_ENABLED: "true",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: publicKeyPath,
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID: "nodedc-module-foundry",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID: "foundry-edp-managed-provisioner-v1",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE: "nodedc-external-data-plane.managed-provisioning.v1",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_MAX_SKEW_SECONDS: "30",
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_REPLAY_CACHE_MAX_ENTRIES: "600",
});
assert.equal(foundryConfig.foundryProvisionerApiEnabled, true);
assert.equal(foundryConfig.foundryProvisionerPublicKey.asymmetricKeyType, "ed25519");
assert.equal(foundryConfig.foundryProvisionerServiceId, "nodedc-module-foundry");
assert.equal(foundryConfig.foundryProvisionerKeyId, "foundry-edp-managed-provisioner-v1");
assert.equal(foundryConfig.foundryProvisionerMaxSkewMs, 30_000);
assert.equal(foundryConfig.foundryProvisionerReplayCacheMaxEntries, 600);
assert.throws(() => readConfig({
...base,
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "true",
@ -88,6 +104,7 @@ try {
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONING_ENABLED: "false",
EXTERNAL_DATA_PLANE_PROVISIONER_TOKEN_FILE: join(directory, "missing"),
EXTERNAL_DATA_PLANE_MANAGED_PROVISIONER_PUBLIC_KEY_FILE: join(directory, "missing"),
EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PUBLIC_KEY_FILE: join(directory, "missing"),
}).provisionerAccessToken, "");
} finally {
await rm(directory, { recursive: true, force: true });

View File

@ -1,11 +1,15 @@
import assert from "node:assert/strict";
import {
assertReaderProduct,
consumerReaderBindingRequestHash,
createReaderToken,
hashReaderToken,
normalizeManagedConsumerReaderBindingRequest,
normalizeManagedConsumerReaderPlanRequest,
normalizeManagedReaderBindingRequest,
normalizeReaderBindingRequest,
readerBindingRequestHash,
safeManagedConsumerReaderBinding,
safeReaderBinding,
} from "../src/reader-binding.mjs";
@ -64,4 +68,31 @@ assert.throws(() => normalizeManagedReaderBindingRequest({
capabilityDigest: managed.capabilityDigest,
}), /managed_reader_binding_must_be_durable/);
const consumerPlan = normalizeManagedConsumerReaderPlanRequest({
allowedDataProductIds: ["fleet.positions.current.v1", "fleet.positions.current.v1"],
});
assert.deepEqual(consumerPlan.allowedDataProductIds, ["fleet.positions.current.v1"]);
const consumerManaged = normalizeManagedConsumerReaderBindingRequest({
allowedDataProductIds: consumerPlan.allowedDataProductIds,
expiresAt: null,
generation: 1,
capabilityDigest: hashReaderToken(token),
});
assert.match(consumerReaderBindingRequestHash(consumerManaged), /^[a-f0-9]{64}$/);
const safeConsumer = safeManagedConsumerReaderBinding({
id: "reader-foundry-01",
bindingKey: "fndrc-reader-managed",
...consumerManaged,
active: true,
});
assert.equal(safeConsumer.sourceScope, "resolved-server-side");
assert.equal(JSON.stringify(safeConsumer).includes(consumerManaged.capabilityDigest), false);
assert.equal("tenantId" in safeConsumer, false);
assert.equal("connectionId" in safeConsumer, false);
assert.equal("providerId" in safeConsumer, false);
assert.throws(() => normalizeManagedConsumerReaderBindingRequest({
...consumerManaged,
capability: "forbidden",
}), /managed_consumer_reader_binding_request_invalid/);
console.log("external-data-plane reader bindings: ok");

View File

@ -1,5 +1,5 @@
import assert from "node:assert/strict";
import { resolveManagedReaderSourceConnection } from "../src/reader-source-scope.mjs";
import { resolveManagedReaderScope, resolveManagedReaderSourceConnection } from "../src/reader-source-scope.mjs";
const policy = {
tenantId: "tenant-01",
@ -20,6 +20,24 @@ await assert.rejects(
resolveManagedReaderSourceConnection(db([]), policy),
(error) => error?.status === 409 && error?.code === "managed_reader_source_scope_not_found",
);
assert.deepEqual(
await resolveManagedReaderScope(scopeDb([
{ tenantId: "tenant-01", connectionId: "producer-connection", providerId: "provider-01" },
]), policy.allowedDataProductIds),
{ tenantId: "tenant-01", connectionId: "producer-connection", providerId: "provider-01" },
);
await assert.rejects(
resolveManagedReaderScope(scopeDb([]), policy.allowedDataProductIds),
(error) => error?.status === 409 && error?.code === "managed_consumer_reader_source_scope_not_found",
);
await assert.rejects(
resolveManagedReaderScope(scopeDb([
{ tenantId: "tenant-01", connectionId: "producer-a", providerId: "provider-01" },
{ tenantId: "tenant-01", connectionId: "producer-b", providerId: "provider-01" },
]), policy.allowedDataProductIds),
(error) => error?.status === 409 && error?.code === "managed_consumer_reader_source_scope_ambiguous",
);
await assert.rejects(
resolveManagedReaderSourceConnection(db(["producer-a", "producer-b"]), policy),
(error) => error?.status === 409 && error?.code === "managed_reader_source_scope_ambiguous",
@ -39,4 +57,14 @@ function db(connectionIds) {
};
}
function scopeDb(rows) {
return {
async query(sql, values) {
assert.match(sql, /external_data_plane_writer_bindings/);
assert.deepEqual(values, [policy.allowedDataProductIds]);
return { rows };
},
};
}
console.log("external-data-plane reader source scope: ok");