feat(foundry): add managed data consumers and agent settings

2026-07-19 15:03:54 +03:00
parent 5f583caa05
commit aac44d057f
35 changed files with 4534 additions and 243 deletions
@@ -1,6 +1,7 @@
 import { constants as fsConstants, createReadStream, createWriteStream } from "node:fs";
 import { lstat, mkdir, open, readFile, readdir, rename, stat, writeFile } from "node:fs/promises";
-import { createServer } from "node:http";
+import { createServer, request as httpRequest } from "node:http";
+import { request as httpsRequest } from "node:https";
 import { createHash, createHmac, randomUUID } from "node:crypto";
 import { basename, extname, join, normalize, resolve } from "node:path";
 import { Readable, Transform } from "node:stream";
@@ -11,6 +12,10 @@ import {
  FOUNDRY_BINDING_UPSERT_PATH,
  handleFoundryBindingApiRequest,
 } from "./foundry-binding-api.mjs";
+import { createFoundryAgentGateway } from "./foundry-agent-gateway.mjs";
+import { createFoundryAgentStore } from "./foundry-agent-store.mjs";
+import { createFoundryDataProductConsumerManager } from "./foundry-data-product-consumer.mjs";
+import { createFoundryReaderGrantProvisioner } from "./foundry-reader-grant-provisioner.mjs";
 import { handleFoundryEntitlementRequest, handleFoundryMcpRequest } from "./foundry-mcp.mjs";
 import { createFoundryAuth } from "./nodedc-auth.mjs";

@@ -25,6 +30,9 @@ const designProfilesDir = join(runtimeDir, "design-profiles");
 const designProfileReleasesDir = join(runtimeDir, "design-profile-releases");
 const pageLayoutsDir = join(runtimeDir, "page-layouts");
 const foundryMcpOperationsDir = join(runtimeDir, "foundry-mcp-operations");
+const foundryDataProductConsumersDir = join(runtimeDir, "data-product-consumers");
+const foundryManagedReaderGrantsDir = join(runtimeDir, "runtime-secrets", "external-data-plane-reader-grants");
+const foundryAgentDataDir = join(runtimeDir, "foundry-agent-gateway");
 const layoutPath = join(runtimeDir, "layout.json");
 const pageRegistryPath = join(root, "registry", "pages.json");
 const port = Number(process.env.PORT || 3333);
@@ -35,7 +43,10 @@ await mkdir(designProfilesDir, { recursive: true });
 await mkdir(designProfileReleasesDir, { recursive: true });
 await mkdir(pageLayoutsDir, { recursive: true });
 await mkdir(foundryMcpOperationsDir, { recursive: true });
+await mkdir(foundryDataProductConsumersDir, { recursive: true });
+await mkdir(foundryAgentDataDir, { recursive: true });
 const pageRegistry = JSON.parse(await readFile(pageRegistryPath, "utf8"));
+const dataProductConsumerPolicyRegistry = JSON.parse(await readFile(join(root, "registry", "data-product-consumer-policies.json"), "utf8"));
 const foundryAuth = createFoundryAuth();
 const mapGatewayHeadersTimeoutMs = boundedMapGatewayTimeout(
  process.env.NODEDC_MAP_GATEWAY_HEADERS_TIMEOUT_MS,
@@ -54,12 +65,28 @@ const mapGatewayInternalUrl = String(
 // a persisted application/page/binding target through the same-origin BFF.
 const externalDataPlaneInternalUrl = String(process.env.NODEDC_EXTERNAL_DATA_PLANE_INTERNAL_URL || "").trim().replace(/\/$/, "");
 const externalDataPlaneReaderGrantsDir = String(process.env.NODEDC_EXTERNAL_DATA_PLANE_READER_GRANTS_DIR || "").trim();
+const foundryReaderGrantProvisioner = createFoundryReaderGrantProvisioner({
+  dataPlaneUrl: externalDataPlaneInternalUrl,
+  privateKeyFile: process.env.NODEDC_EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_PRIVATE_KEY_FILE,
+  grantsDir: foundryManagedReaderGrantsDir,
+  serviceId: process.env.NODEDC_EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_SERVICE_ID || "nodedc-module-foundry",
+  keyId: process.env.NODEDC_EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_KEY_ID || "foundry-edp-managed-provisioner-v1",
+  audience: process.env.NODEDC_EXTERNAL_DATA_PLANE_FOUNDRY_PROVISIONER_AUDIENCE || "nodedc-external-data-plane.managed-provisioning.v1",
+});
 const foundryBindingGrantsDir = String(process.env.NODEDC_FOUNDRY_BINDING_GRANTS_DIR || "").trim();
 const activeRuntimeStreams = new Set();
 // A dedicated server-only signing value shared with Map Gateway. In production
 // the runner supplies it as a read-only file, never an env value or browser API.
 const mapGatewayAdminSecret = await readMapGatewayAdminSecret();
 const foundryPublicUrl = normalizeFoundryPublicUrl(process.env.FOUNDRY_PUBLIC_URL);
+const foundryAgentStore = createFoundryAgentStore({ dataRoot: foundryAgentDataDir });
+const foundryAgentGateway = createFoundryAgentGateway({
+  store: foundryAgentStore,
+  configuredPublicOrigin: foundryPublicUrl?.origin || "",
+  installerPackagePath: join(root, "server", "assets", "nodedc-foundry-codex-agent-0.1.0.tgz"),
+  ontologyCoreUrl: process.env.NODEDC_ONTOLOGY_CORE_URL,
+  ontologyCoreAccessToken: process.env.NODEDC_INTERNAL_ACCESS_TOKEN,
+});

 function boundedMapGatewayTimeout(value, fallback) {
  const parsed = Number.parseInt(String(value || ""), 10);
@@ -826,12 +853,14 @@ async function resolveRuntimeMapDataProductBinding(applicationId, pageId, bindin
 }

 async function resolveRuntimeDataProductReaderToken(applicationId, pageId, bindingId) {
-  if (!externalDataPlaneInternalUrl || !externalDataPlaneReaderGrantsDir) {
+  if (!externalDataPlaneInternalUrl || (!externalDataPlaneReaderGrantsDir && !foundryReaderGrantProvisioner.configured)) {
    throw applicationError("data_product_runtime_not_configured", 503);
  }
-  // The name is deterministic but opaque to the browser. The runner creates
-  // this root-owned, read-only file after it provisions an EDP reader grant.
-  // There is no fallback to one shared Platform token.
+  const managedToken = await foundryReaderGrantProvisioner.readToken({ applicationId, pageId, bindingId });
+  if (managedToken) return managedToken;
+  if (!externalDataPlaneReaderGrantsDir) throw applicationError("data_product_reader_grant_not_found", 403);
+  // Compatibility fallback for grants issued before Foundry-owned managed
+  // provisioning. There is no fallback to one shared Platform token.
  const grantPath = join(resolve(externalDataPlaneReaderGrantsDir), runtimeTargetGrantKey(applicationId, pageId, bindingId));
  let handle;
  try {
@@ -875,6 +904,15 @@ async function preflightFoundryBindingReaderGrant({ applicationId, pageId, bindi
    if (["data_product_reader_grant_not_found", "data_product_reader_grant_invalid"].includes(error?.message)) return false;
    throw error;
  }
+  try {
+    return Boolean(await readRuntimeReaderProduct(readerToken, dataProductId));
+  } catch (error) {
+    if (error?.message === "data_product_access_denied") return false;
+    throw error;
+  }
+}
+
+async function readRuntimeReaderProduct(readerToken, dataProductId) {
  let upstream;
  try {
    upstream = await fetch(new URL("/internal/data-plane/v1/reader/data-products", `${externalDataPlaneInternalUrl}/`), {
@@ -884,13 +922,28 @@ async function preflightFoundryBindingReaderGrant({ applicationId, pageId, bindi
  } catch {
    throw applicationError("data_product_reader_grant_preflight_unavailable", 503);
  }
-  if (upstream.status === 401 || upstream.status === 403) return false;
+  if (upstream.status === 401 || upstream.status === 403) throw applicationError("data_product_access_denied", 403);
  if (!upstream.ok) throw applicationError("data_product_reader_grant_preflight_unavailable", 503);
  const payload = await upstream.json().catch(() => null);
  const product = Array.isArray(payload?.dataProducts)
    ? payload.dataProducts.find((candidate) => candidate?.id === dataProductId)
    : null;
-  return Boolean(product && product.active !== false && product.deliveryMode === "snapshot+patch");
+  return product && product.active !== false && product.deliveryMode === "snapshot+patch" ? product : null;
+}
+
+async function inspectFoundryConsumerReaderGrant(target) {
+  try {
+    const readerToken = await resolveRuntimeDataProductReaderToken(target.application.id, target.page.id, target.binding.id);
+    const product = await readRuntimeReaderProduct(readerToken, target.binding.dataProductId);
+    if (product) return { product, readerGrantAction: "reuse" };
+  } catch (error) {
+    if (!new Set(["data_product_reader_grant_not_found", "data_product_access_denied"]).has(error?.message)) throw error;
+  }
+  if (!foundryReaderGrantProvisioner.configured) {
+    throw applicationError("data_product_reader_grant_not_found", 403);
+  }
+  const planned = await foundryReaderGrantProvisioner.plan(target);
+  return { product: planned.product, readerGrantAction: "ensure" };
 }

 function runtimePointGeometry(value) {
@@ -950,13 +1003,88 @@ function sanitizeRuntimeSnapshot(value, binding) {
  };
 }

+function sanitizeRuntimeHistory(value, binding) {
+  if (!isObject(value) || value.schemaVersion !== "nodedc.data-product.history/v1" || !isObject(value.dataProduct) || !isObject(value.query)) {
+    throw applicationError("data_product_history_contract_invalid", 502);
+  }
+  const query = value.query;
+  if (
+    value.dataProduct.id !== binding.dataProductId
+    || !isRuntimeVersion(value.dataProduct.version)
+    || !isRuntimeTimestamp(value.generatedAt)
+    || !isRuntimeTimestamp(query.from)
+    || !isRuntimeTimestamp(query.to)
+    || Date.parse(query.from) >= Date.parse(query.to)
+    || !Number.isInteger(query.resolutionMs)
+    || query.resolutionMs < 1000
+    || query.resolutionMs > 86_400_000
+    || query.order !== "asc"
+    || !Array.isArray(query.sourceIds)
+    || query.sourceIds.some((sourceId) => !isRuntimeIdentifier(sourceId))
+    || JSON.stringify(query.sourceIds) !== JSON.stringify([...new Set(query.sourceIds)].sort())
+    || !Array.isArray(value.facts)
+    || (value.nextCursor !== undefined && (typeof value.nextCursor !== "string" || !/^[A-Za-z0-9_-]{1,1024}$/.test(value.nextCursor)))
+  ) throw applicationError("data_product_history_contract_invalid", 502);
+  let previousOrderKey = null;
+  const facts = value.facts.flatMap((valueFact) => {
+    if (
+      !isRuntimeTimestamp(valueFact?.bucketStart)
+      || Date.parse(valueFact.bucketStart) < Date.parse(query.from)
+      || Date.parse(valueFact.bucketStart) >= Date.parse(query.to)
+    ) throw applicationError("data_product_history_contract_invalid", 502);
+    const orderKey = `${valueFact.bucketStart}\u0000${valueFact.sourceId}\u0000${valueFact.semanticType}`;
+    if (previousOrderKey !== null && orderKey <= previousOrderKey) {
+      throw applicationError("data_product_history_contract_invalid", 502);
+    }
+    previousOrderKey = orderKey;
+    const fact = sanitizeRuntimeFact(valueFact, binding);
+    return fact ? [{ ...fact, bucketStart: valueFact.bucketStart }] : [];
+  });
+  return {
+    schemaVersion: "nodedc.data-product.history/v1",
+    dataProduct: { id: binding.dataProductId, version: value.dataProduct.version },
+    generatedAt: value.generatedAt,
+    query: {
+      from: query.from,
+      to: query.to,
+      resolutionMs: query.resolutionMs,
+      sourceIds: [...query.sourceIds],
+      order: "asc",
+    },
+    facts,
+    ...(value.nextCursor ? { nextCursor: value.nextCursor } : {}),
+  };
+}
+
 function sanitizeRuntimePatch(value, binding) {
  if (!isObject(value) || value.schemaVersion !== "nodedc.data-product.patch/v1" || !isObject(value.dataProduct)) return null;
  if (value.dataProduct.id !== binding.dataProductId || !isRuntimeVersion(value.dataProduct.version) || !isRuntimeCursor(value.cursor) || !isRuntimeCursor(value.previousCursor) || !isRuntimeTimestamp(value.emittedAt) || !Array.isArray(value.operations)) return null;
  const operations = value.operations.flatMap((operation) => {
-    if (!isObject(operation) || operation.op !== "upsert") return [];
-    const fact = sanitizeRuntimeFact(operation.fact, binding);
-    return fact ? [{ op: "upsert", fact }] : [];
+    if (!isObject(operation)) return [];
+    if (operation.op === "upsert") {
+      const fact = sanitizeRuntimeFact(operation.fact, binding);
+      return fact ? [{ op: "upsert", fact }] : [];
+    }
+    // Current EDP v1 emits only upserts. Foundry already understands the
+    // canonical removal shape so a future versioned product can revoke a
+    // subject without turning a transient stream failure into deletion.
+    if (
+      operation.op === "remove"
+      && isRuntimeIdentifier(operation.sourceId)
+      && isRuntimeIdentifier(operation.semanticType)
+      && binding.semanticTypes.includes(operation.semanticType)
+      && isRuntimeTimestamp(operation.removedAt)
+      && ["tombstone", "revoked"].includes(operation.reason)
+    ) {
+      return [{
+        op: "remove",
+        sourceId: operation.sourceId,
+        semanticType: operation.semanticType,
+        removedAt: operation.removedAt,
+        reason: operation.reason,
+      }];
+    }
+    return [];
  });
  return {
    schemaVersion: "nodedc.data-product.patch/v1",
@@ -968,6 +1096,15 @@ function sanitizeRuntimePatch(value, binding) {
  };
 }

+function resolveDataProductConsumerPolicy(product) {
+  if (dataProductConsumerPolicyRegistry?.schemaVersion !== "nodedc.foundry.data-product-consumer-policies/v1") {
+    throw applicationError("data_product_consumer_policy_registry_invalid", 500);
+  }
+  return dataProductConsumerPolicyRegistry.policies?.find((policy) => (
+    policy?.dataProductId === product.id && policy?.productVersion === product.version
+  )) || null;
+}
+
 function runtimeUpstreamUrl(dataProductId, resource, after) {
  const target = new URL(`/internal/data-plane/v1/data-products/${encodeURIComponent(dataProductId)}/${resource}`, `${externalDataPlaneInternalUrl}/`);
  if (after) target.searchParams.set("after", after);
@@ -1019,25 +1156,127 @@ async function writeSse(response, { event, data, id }) {
  return writeRuntimeStreamChunk(response, frame);
 }

-function parseSseBlock(block) {
-  const fields = { event: "message", id: "", data: [] };
-  for (const line of block.split("\n")) {
-    if (!line || line.startsWith(":")) continue;
-    const separator = line.indexOf(":");
-    const field = separator === -1 ? line : line.slice(0, separator);
-    const value = separator === -1 ? "" : line.slice(separator + 1).replace(/^ /, "");
-    if (field === "event") fields.event = value;
-    if (field === "id") fields.id = value;
-    if (field === "data") fields.data.push(value);
-  }
-  return { event: fields.event, id: fields.id, data: fields.data.join("\n") };
+function openRuntimeDataProductConsumerStream({ url, token, signal }) {
+  return new Promise((resolveStream, rejectStream) => {
+    const transport = url.protocol === "https:" ? httpsRequest : url.protocol === "http:" ? httpRequest : null;
+    if (!transport) return rejectStream(applicationError("data_product_runtime_url_invalid", 500));
+    let incoming = null;
+    let settled = false;
+    const request = transport(url, {
+      method: "GET",
+      headers: {
+        authorization: `Bearer ${token}`,
+        accept: "text/event-stream",
+        connection: "close",
+      },
+    });
+    const abort = () => {
+      incoming?.destroy();
+      request.destroy();
+    };
+    signal.addEventListener("abort", abort, { once: true });
+    request.once("response", (response) => {
+      incoming = response;
+      settled = true;
+      response.once("close", () => signal.removeEventListener("abort", abort));
+      resolveStream({
+        status: response.statusCode || 500,
+        ok: (response.statusCode || 500) >= 200 && (response.statusCode || 500) < 300,
+        headers: new Headers(response.headers),
+        body: Readable.toWeb(response),
+      });
+    });
+    request.once("error", (error) => {
+      signal.removeEventListener("abort", abort);
+      if (!settled) rejectStream(signal.aborted ? applicationError("data_product_stream_closed", 503) : error);
+    });
+    if (signal.aborted) abort();
+    else request.end();
+  });
 }

+const dataProductConsumerManager = createFoundryDataProductConsumerManager({
+  stateDir: foundryDataProductConsumersDir,
+  dataPlaneUrl: externalDataPlaneInternalUrl,
+  resolveTarget: resolveRuntimeMapDataProductBinding,
+  readReaderToken: resolveRuntimeDataProductReaderToken,
+  inspectReaderGrant: inspectFoundryConsumerReaderGrant,
+  ensureReaderGrant: (target) => foundryReaderGrantProvisioner.ensure(target),
+  resolvePolicy: resolveDataProductConsumerPolicy,
+  sanitizeSnapshot: sanitizeRuntimeSnapshot,
+  sanitizePatch: sanitizeRuntimePatch,
+  openStream: openRuntimeDataProductConsumerStream,
+});
+await dataProductConsumerManager.resumePersisted();
+
 async function proxyRuntimeDataProductSnapshot(response, target) {
+  json(response, 200, await dataProductConsumerManager.snapshot(target));
+}
+
+async function readRuntimeJsonResponse(upstream, maxBytes = 64 * 1024 * 1024) {
+  const contentLength = Number(upstream.headers.get("content-length") || 0);
+  if (Number.isFinite(contentLength) && contentLength > maxBytes) {
+    throw applicationError("data_product_runtime_response_too_large", 502);
+  }
+  if (!upstream.body) throw applicationError("data_product_runtime_invalid_response", 502);
+  const reader = upstream.body.getReader();
+  const chunks = [];
+  let bytes = 0;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      bytes += value.byteLength;
+      if (bytes > maxBytes) {
+        await reader.cancel();
+        throw applicationError("data_product_runtime_response_too_large", 502);
+      }
+      chunks.push(Buffer.from(value));
+    }
+    return JSON.parse(Buffer.concat(chunks, bytes).toString("utf8"));
+  } catch (error) {
+    if (error?.statusCode) throw error;
+    throw applicationError("data_product_runtime_invalid_response", 502);
+  }
+}
+
+function runtimeHistoryQuery(url) {
+  const from = String(url.searchParams.get("from") || "");
+  const to = String(url.searchParams.get("to") || "");
+  const resolutionMs = Number(url.searchParams.get("resolutionMs") || 60_000);
+  const limit = Number(url.searchParams.get("limit") || 1000);
+  const sourceIds = [...new Set(String(url.searchParams.get("sourceIds") || "").split(",").map((value) => value.trim()).filter(Boolean))];
+  const cursor = String(url.searchParams.get("cursor") || "");
+  if (
+    !isRuntimeTimestamp(from)
+    || !isRuntimeTimestamp(to)
+    || Date.parse(from) >= Date.parse(to)
+    || !Number.isInteger(resolutionMs)
+    || resolutionMs < 1000
+    || resolutionMs > 86_400_000
+    || !Number.isInteger(limit)
+    || limit < 1
+    || limit > 5000
+    || sourceIds.length > 1000
+    || sourceIds.some((sourceId) => !isRuntimeIdentifier(sourceId))
+    || (cursor && !/^[A-Za-z0-9_-]{1,1024}$/.test(cursor))
+  ) throw applicationError("data_product_history_query_invalid", 400);
+  return { from, to, resolutionMs, limit, sourceIds, cursor };
+}
+
+async function proxyRuntimeDataProductHistory(response, url, target) {
  const readerToken = await resolveRuntimeDataProductReaderToken(target.application.id, target.page.id, target.binding.id);
+  const query = runtimeHistoryQuery(url);
+  const upstreamUrl = runtimeUpstreamUrl(target.binding.dataProductId, "history");
+  upstreamUrl.searchParams.set("from", query.from);
+  upstreamUrl.searchParams.set("to", query.to);
+  upstreamUrl.searchParams.set("resolutionMs", String(query.resolutionMs));
+  upstreamUrl.searchParams.set("limit", String(query.limit));
+  if (query.sourceIds.length) upstreamUrl.searchParams.set("sourceIds", query.sourceIds.join(","));
+  if (query.cursor) upstreamUrl.searchParams.set("cursor", query.cursor);
  let upstream;
  try {
-    upstream = await fetch(runtimeUpstreamUrl(target.binding.dataProductId, "snapshot"), {
+    upstream = await fetch(upstreamUrl, {
      headers: { authorization: `Bearer ${readerToken}`, accept: "application/json" },
      signal: AbortSignal.timeout(10_000),
    });
@@ -1045,12 +1284,11 @@ async function proxyRuntimeDataProductSnapshot(response, target) {
    throw applicationError("data_product_runtime_unavailable", 503);
  }
  if (!upstream.ok) throw runtimeUpstreamFailure(upstream.status);
-  const payload = await upstream.json().catch(() => null);
-  json(response, 200, sanitizeRuntimeSnapshot(payload, target.binding));
+  const payload = await readRuntimeJsonResponse(upstream);
+  json(response, 200, sanitizeRuntimeHistory(payload, target.binding));
 }

 async function proxyRuntimeDataProductStream(request, response, url, target) {
-  const readerToken = await resolveRuntimeDataProductReaderToken(target.application.id, target.page.id, target.binding.id);
  const requestedAfter = String(url.searchParams.get("after") || "");
  const lastEventId = String(request.headers["last-event-id"] || "");
  if (requestedAfter && !isRuntimeCursor(requestedAfter)) throw applicationError("stream_cursor_invalid");
@@ -1060,39 +1298,39 @@ async function proxyRuntimeDataProductStream(request, response, url, target) {
  // cursor; using it avoids replaying a stale query-string cursor forever.
  const after = lastEventId || requestedAfter;
  const controller = new AbortController();
-  const abort = () => controller.abort();
+  let lease = null;
+  let heartbeat = null;
+  let headersReady = false;
+  const bufferedEvents = [];
+  let pendingEventCount = 0;
+  let writeChain = Promise.resolve(true);
+  const enqueue = (event) => {
+    if (!headersReady) {
+      bufferedEvents.push(event);
+      return;
+    }
+    pendingEventCount += 1;
+    if (pendingEventCount > 256) {
+      abort();
+      response.destroy();
+      return;
+    }
+    writeChain = writeChain
+      .then((open) => open && writeSse(response, event))
+      .catch(() => false)
+      .finally(() => { pendingEventCount = Math.max(0, pendingEventCount - 1); });
+  };
+  const abort = () => {
+    if (!controller.signal.aborted) controller.abort();
+    lease?.release();
+  };
  const activeStream = { controller, response };
  activeRuntimeStreams.add(activeStream);
  request.once("aborted", abort);
  response.once("close", abort);
-  let upstream;
+  request.socket?.once("close", abort);
  try {
-    upstream = await fetch(runtimeUpstreamUrl(target.binding.dataProductId, "stream", after), {
-      headers: {
-        authorization: `Bearer ${readerToken}`,
-        accept: "text/event-stream",
-        ...(lastEventId ? { "last-event-id": lastEventId } : {}),
-      },
-      signal: controller.signal,
-    });
-    if (upstream.status === 409) {
-      response.writeHead(200, {
-        "content-type": "text/event-stream; charset=utf-8",
-        "cache-control": "no-store, no-transform",
-        connection: "keep-alive",
-        "x-accel-buffering": "no",
-        vary: "Cookie",
-      });
-      await writeSse(response, {
-        event: "nodedc.data-product.resync-required.v1",
-        data: { schemaVersion: "nodedc.data-product.resync-required/v1", dataProductId: target.binding.dataProductId },
-      });
-      if (!response.destroyed && !response.writableEnded) response.end();
-      return;
-    }
-    if (!upstream.ok || !upstream.body || !String(upstream.headers.get("content-type") || "").startsWith("text/event-stream")) {
-      throw runtimeUpstreamFailure(upstream.status);
-    }
+    lease = await dataProductConsumerManager.subscribe({ ...target, after }, enqueue);
    response.writeHead(200, {
      "content-type": "text/event-stream; charset=utf-8",
      "cache-control": "no-store, no-transform",
@@ -1100,54 +1338,31 @@ async function proxyRuntimeDataProductStream(request, response, url, target) {
      "x-accel-buffering": "no",
      vary: "Cookie",
    });
-    const reader = upstream.body.getReader();
-    const decoder = new TextDecoder();
-    let pending = "";
-    try {
-      streamLoop: while (!response.destroyed && !controller.signal.aborted) {
-        const next = await reader.read();
-        if (next.done) break;
-        pending += decoder.decode(next.value, { stream: true }).replace(/\r\n/g, "\n");
-        let separator;
-        while ((separator = pending.indexOf("\n\n")) !== -1) {
-          const block = pending.slice(0, separator);
-          pending = pending.slice(separator + 2);
-          const event = parseSseBlock(block);
-          if (event.event === "nodedc.data-product.patch.v1") {
-            const patch = sanitizeRuntimePatch(JSON.parse(event.data), target.binding);
-            if (patch?.operations.length && !(await writeSse(response, { event: event.event, id: patch.cursor, data: patch }))) {
-              abort();
-              break streamLoop;
-            }
-          } else if (event.event === "nodedc.data-product.ready.v1") {
-            let ready = null;
-            try { ready = JSON.parse(event.data); } catch { ready = null; }
-            if (isObject(ready) && ready.schemaVersion === "nodedc.data-product.ready/v1" && ready.dataProductId === target.binding.dataProductId && isRuntimeCursor(ready.cursor) && isRuntimeTimestamp(ready.emittedAt)) {
-              if (!(await writeSse(response, { event: event.event, data: { schemaVersion: ready.schemaVersion, dataProductId: ready.dataProductId, cursor: ready.cursor, emittedAt: ready.emittedAt } }))) {
-                abort();
-                break streamLoop;
-              }
-            }
-          } else if (!event.data) {
-            // Preserve a heartbeat without forwarding unknown upstream fields.
-            if (!(await writeRuntimeStreamChunk(response, ": keepalive\n\n"))) {
-              abort();
-              break streamLoop;
-            }
-          }
-        }
-      }
-    } finally {
-      if (controller.signal.aborted) await reader.cancel().catch(() => undefined);
-      reader.releaseLock();
+    headersReady = true;
+    if (lease.resyncRequired) {
+      await writeSse(response, {
+        event: "nodedc.data-product.resync-required.v1",
+        data: { schemaVersion: "nodedc.data-product.resync-required/v1", dataProductId: target.binding.dataProductId },
+      });
+      response.end();
+      return;
    }
-  } catch (error) {
-    if (!response.headersSent && !controller.signal.aborted) throw error;
+    for (const event of bufferedEvents.splice(0)) enqueue(event);
+    heartbeat = setInterval(() => {
+      writeChain = writeChain.then((open) => open && writeRuntimeStreamChunk(response, ": keepalive\n\n")).catch(() => false);
+    }, 15_000);
+    heartbeat.unref?.();
+    await new Promise((resolve) => {
+      if (controller.signal.aborted || response.destroyed) return resolve();
+      controller.signal.addEventListener("abort", resolve, { once: true });
+    });
  } finally {
+    if (heartbeat) clearInterval(heartbeat);
    abort();
    activeRuntimeStreams.delete(activeStream);
    request.off("aborted", abort);
    response.off("close", abort);
+    request.socket?.off("close", abort);
    if (!response.writableEnded && !response.destroyed) response.end();
  }
 }
@@ -1334,14 +1549,25 @@ const foundryMcpOperations = {
    const config = foundryMcpConfig();
    return {
      module: "NDC Module Foundry",
-      schemaVersion: "nodedc.module-foundry.mcp.v0.1",
+      schemaVersion: "nodedc.module-foundry.mcp.v0.2",
      status: config.capabilitySecret && config.mcpUrl ? "ready" : "configuration_required",
      canonicalPageLibrary: "read-only",
      applicationInstances: "read-write",
      destructiveApplicationDelete: false,
      supportedPageTemplates: pageRegistry.templates.map((template) => ({ id: template.id, version: template.version, title: template.title })),
-      supportedActions: ["application.create", "application.metadata.update", "page-instance.create", "map-pin.upsert"],
-      persistence: { runtimeDir: "persistent runtime volume required", idempotentWrites: true },
+      supportedActions: [
+        "application.create",
+        "application.metadata.update",
+        "page-instance.create",
+        "map-pin.upsert",
+        "map-data-product.upsert",
+        "map-data-product-consumer.plan",
+        "map-data-product-consumer.apply",
+        "map-data-product-consumer.status",
+        "map-data-product-consumer.accept",
+        "map-data-product-consumer.rollback",
+      ],
+      persistence: { runtimeDir: "persistent runtime volume required", idempotentWrites: true, dataProductConsumerState: true },
    };
  },
  async listApplications() {
@@ -1468,6 +1694,31 @@ const foundryMcpOperations = {
      },
    });
  },
+  async planMapDataProductConsumer(input) {
+    return dataProductConsumerManager.plan(input);
+  },
+  async applyMapDataProductConsumer(input, actor) {
+    return executeFoundryMcpWrite({
+      tool: "foundry_apply_map_data_product_consumer",
+      actor,
+      input,
+      action: () => dataProductConsumerManager.apply(input),
+    });
+  },
+  async getMapDataProductConsumerStatus(input) {
+    return dataProductConsumerManager.status(input);
+  },
+  async acceptMapDataProductConsumer(input) {
+    return dataProductConsumerManager.accept(input);
+  },
+  async rollbackMapDataProductConsumer(input, actor) {
+    return executeFoundryMcpWrite({
+      tool: "foundry_rollback_map_data_product_consumer",
+      actor,
+      input,
+      action: () => dataProductConsumerManager.rollback(input),
+    });
+  },
 };

 async function readJsonBody(request, maxBytes = 2 * 1024 * 1024) {
@@ -1762,6 +2013,9 @@ const server = createServer(async (request, response) => {
  try {
    const url = new URL(request.url || "/", `http://${request.headers.host || "127.0.0.1"}`);

+    if (await foundryAgentGateway.handlePublicRequest(request, response, url)) return;
+    if (await foundryAgentGateway.handleOntologyMcp(request, response, url)) return;
+
    if (url.pathname === "/auth/nodedc/handoff" && request.method === "GET") {
      await foundryAuth.handleLauncherHandoff(request, response, url);
      return;
@@ -1781,10 +2035,16 @@ const server = createServer(async (request, response) => {
          configured: Boolean(config.capabilitySecret && config.mcpUrl),
          entitlementConfigured: Boolean(config.internalAccessToken),
        },
+        codexAgent: foundryAgentGateway.health(),
        dataProductBindingApi: {
          configured: Boolean(foundryBindingGrantsDir && externalDataPlaneInternalUrl && externalDataPlaneReaderGrantsDir),
          auth: "scoped-workload-grant",
        },
+        dataProductConsumerProvisioner: {
+          configured: foundryReaderGrantProvisioner.configured,
+          auth: "dedicated-ed25519-service-identity",
+          sourceScope: "external-data-plane-resolved",
+        },
      });
      return;
    }
@@ -1793,6 +2053,7 @@ const server = createServer(async (request, response) => {
      await handleFoundryMcpRequest(request, response, {
        getConfig: foundryMcpConfig,
        operations: foundryMcpOperations,
+        authenticateAgentToken: foundryAgentGateway.authenticateFoundryToken,
      });
      return;
    }
@@ -1816,6 +2077,16 @@ const server = createServer(async (request, response) => {
      return;
    }

+    const foundryUserProfile = foundryAuth.currentUserProfile(request)
+      || (!foundryAuth.authRequired ? {
+        id: "local_foundry_user",
+        email: "local-foundry@nodedc.local",
+        displayName: "Local Foundry User",
+        avatarUrl: null,
+        initials: "LF",
+      } : null);
+    if (await foundryAgentGateway.handleManagementRequest(request, response, url, foundryUserProfile)) return;
+
    if (url.pathname === "/api/session/profile" && request.method === "GET") {
      const access = foundryAuth.currentUserAccess(request);
      json(response, 200, { user: foundryAuth.currentUserProfile(request), profileUrl: foundryAuth.profileUrl, access: { role: access.role } });
@@ -1858,11 +2129,12 @@ const server = createServer(async (request, response) => {
      return;
    }

-    const runtimeDataProductMatch = url.pathname.match(/^\/api\/applications\/([0-9a-f-]{36})\/pages\/([a-z0-9-]+)\/data-bindings\/([A-Za-z0-9._:-]{1,160})\/(snapshot|stream)$/i);
+    const runtimeDataProductMatch = url.pathname.match(/^\/api\/applications\/([0-9a-f-]{36})\/pages\/([a-z0-9-]+)\/data-bindings\/([A-Za-z0-9._:-]{1,160})\/(snapshot|history|stream)$/i);
    if (runtimeDataProductMatch && request.method === "GET") {
      const [, applicationId, pageId, bindingId, resource] = runtimeDataProductMatch;
      const target = await resolveRuntimeMapDataProductBinding(applicationId, pageId, bindingId);
      if (resource === "snapshot") await proxyRuntimeDataProductSnapshot(response, target);
+      else if (resource === "history") await proxyRuntimeDataProductHistory(response, url, target);
      else await proxyRuntimeDataProductStream(request, response, url, target);
      return;
    }
@@ -2125,28 +2397,39 @@ server.on("connection", (socket) => {
 let shutdownPromise = null;
 function shutdownServer(signal) {
  if (shutdownPromise) return shutdownPromise;
-  shutdownPromise = new Promise((resolveShutdown) => {
+  shutdownPromise = (async () => {
    for (const activeStream of activeRuntimeStreams) {
      activeStream.controller.abort();
      activeStream.response.destroy();
    }
-    const timeout = setTimeout(() => {
-      for (const socket of serverSockets) socket.destroy();
-    }, 5_000);
-    server.close((error) => {
-      clearTimeout(timeout);
-      if (error) {
-        process.exitCode = 1;
-        console.error(`NDC Module Foundry shutdown error (${signal}): ${error.message}`);
-      }
-      resolveShutdown();
+    await dataProductConsumerManager.shutdown();
+    await new Promise((resolveShutdown) => {
+      const timeout = setTimeout(() => {
+        for (const socket of serverSockets) socket.destroy();
+      }, 5_000);
+      server.close((error) => {
+        clearTimeout(timeout);
+        if (error) {
+          process.exitCode = 1;
+          console.error(`NDC Module Foundry shutdown error (${signal}): ${error.message}`);
+        }
+        resolveShutdown();
+      });
    });
-  });
+  })();
  return shutdownPromise;
 }

-process.once("SIGTERM", () => { void shutdownServer("SIGTERM"); });
-process.once("SIGINT", () => { void shutdownServer("SIGINT"); });
+async function shutdownAndExit(signal) {
+  await shutdownServer(signal);
+  // A canceled upstream fetch may leave an idle keep-alive socket owned by
+  // Node's global HTTP dispatcher. All Foundry state and inbound sockets are
+  // already closed at this point, so finish the container stop deterministically.
+  process.exit(process.exitCode || 0);
+}
+
+process.once("SIGTERM", () => { void shutdownAndExit("SIGTERM"); });
+process.once("SIGINT", () => { void shutdownAndExit("SIGINT"); });

 server.listen(port, host, () => {
  console.log(`NDC Module Foundry: http://${host}:${port}`);