SILVER/NODEDC_DESIGN_GUIDELINE
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feat(foundry): add managed data consumers and agent settings

Browse Source
This commit is contained in:
Codex
2026-07-19 15:03:54 +03:00
parent 5f583caa05
commit aac44d057f
35 changed files with 4534 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/smoke-data-product-runtime.mjs
+178 -73
View File
@@ -28,8 +28,9 @@ const bindingGrantDir = await mkdtemp(join(tmpdir(), "nodedc-foundry-binding-gra
const bindingGrantToken = createFoundryBindingGrantToken();
let foundry;
let edp;
const pressure = { sent: 0, backpressured: false, closed: false };
const firstStream = { opened: false, closed: false };
const shutdownStream = { opened: false, closed: false };
const internalAccessToken = "foundry-runtime-smoke-internal-access";
function listen(server) {
return new Promise((resolve) => {
@@ -54,39 +55,64 @@ async function waitFor(url) {
async function waitUntil(predicate, label, timeoutMs = 5_000) {
const deadline = Date.now() + timeoutMs;
while (Date.now() < deadline) {
if (predicate()) return;
if (await predicate()) return;
await new Promise((resolve) => setTimeout(resolve, 25));
}
throw new Error(`${label}_timeout`);
}
function waitForDrainOrClose(response) {
return new Promise((resolve) => {
let settled = false;
const settle = (value) => {
if (settled) return;
settled = true;
response.off("drain", onDrain);
response.off("close", onClose);
response.off("error", onError);
resolve(value);
};
const onDrain = () => settle(true);
const onClose = () => settle(false);
const onError = () => settle(false);
response.once("drain", onDrain);
response.once("close", onClose);
response.once("error", onError);
});
async function readStreamUntil(response, pattern, timeoutMs = 5_000) {
assert.equal(response.status, 200);
assert.ok(response.body);
const reader = response.body.getReader();
const decoder = new TextDecoder();
let text = "";
const deadline = Date.now() + timeoutMs;
try {
while (Date.now() < deadline) {
const next = await Promise.race([
reader.read(),
new Promise((_, reject) => setTimeout(() => reject(new Error("stream_read_timeout")), 250)),
]);
if (next.done) break;
text += decoder.decode(next.value, { stream: true });
if (pattern.test(text)) return text;
}
throw new Error("stream_pattern_timeout");
} finally {
await reader.cancel().catch(() => undefined);
reader.releaseLock();
}
}
function openPausedStream(url) {
function readHttpStreamUntil(url, pattern, timeoutMs = 5_000) {
return new Promise((resolve, reject) => {
let text = "";
const timer = setTimeout(() => {
request.destroy();
reject(new Error("http_stream_pattern_timeout"));
}, timeoutMs);
const request = httpRequest(url, { headers: { accept: "text/event-stream" } }, (response) => {
response.pause();
resolve({ request, response });
response.setEncoding("utf8");
response.on("data", (chunk) => {
text += chunk;
if (!pattern.test(text)) return;
clearTimeout(timer);
response.destroy();
request.destroy();
resolve(text);
});
response.once("error", (error) => {
if (pattern.test(text)) return;
clearTimeout(timer);
reject(error);
});
});
request.once("error", (error) => {
if (pattern.test(text)) return;
clearTimeout(timer);
reject(error);
});
request.once("error", reject);
request.end();
});
}
@@ -181,13 +207,37 @@ try {
}));
return;
}
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/stream?after=1") {
response.writeHead(409, { "content-type": "application/json" });
response.end(JSON.stringify({ ok: false, error: "resync_required" }));
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/history?from=2026-07-15T12%3A00%3A00.000Z&to=2026-07-15T12%3A05%3A00.000Z&resolutionMs=60000&limit=1000&sourceIds=vehicle-001") {
response.writeHead(200, { "content-type": "application/json" });
response.end(JSON.stringify({
schemaVersion: "nodedc.data-product.history/v1",
dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" },
generatedAt: "2026-07-15T12:05:01.000Z",
query: {
from: "2026-07-15T12:00:00.000Z",
to: "2026-07-15T12:05:00.000Z",
resolutionMs: 60000,
sourceIds: ["vehicle-001"],
order: "asc",
},
facts: [{
...canonicalFact({ longitude: 37.61, name: "Vehicle 001 historical" }),
bucketStart: "2026-07-15T12:00:00.000Z",
}],
nextCursor: "opaque_history_cursor",
}));
return;
}
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/stream?after=7") {
response.writeHead(200, { "content-type": "text/event-stream" });
firstStream.opened = true;
const heartbeat = setInterval(() => response.write(": keepalive\n\n"), 50);
const closeFirstStream = () => {
clearInterval(heartbeat);
firstStream.closed = true;
};
response.once("close", closeFirstStream);
request.socket.once("close", closeFirstStream);
response.write("event: nodedc.data-product.ready.v1\n");
response.write(`data: ${JSON.stringify({ schemaVersion: "nodedc.data-product.ready/v1", dataProductId: "fleet.positions.current.v1", cursor: "7", emittedAt: "2026-07-15T12:00:01.000Z" })}\n\n`);
response.write("id: 8\n");
@@ -200,44 +250,18 @@ try {
emittedAt: "2026-07-15T12:00:02.000Z",
operations: [{ op: "upsert", fact: canonicalFact({ longitude: 37.62, name: "Vehicle 001 updated" }) }],
})}\n\n`);
response.end();
return;
}
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/stream?after=9") {
response.writeHead(200, { "content-type": "text/event-stream" });
response.once("close", () => { pressure.closed = true; });
void (async () => {
const limit = 2_048;
const largeName = "vehicle-position-".padEnd(16 * 1024, "x");
for (let index = 0; index < limit && !response.destroyed; index += 1) {
const cursor = String(10 + index);
const previousCursor = String(9 + index);
const frame = `id: ${cursor}\nevent: nodedc.data-product.patch.v1\ndata: ${JSON.stringify({
schemaVersion: "nodedc.data-product.patch/v1",
dataProduct: { id: "fleet.positions.current.v1", version: "1.0.0" },
cursor,
previousCursor,
emittedAt: "2026-07-15T12:00:02.000Z",
operations: [{ op: "upsert", fact: canonicalFact({ longitude: 37.62, name: largeName }) }],
})}\n\n`;
pressure.sent += 1;
if (!response.write(frame)) {
pressure.backpressured = true;
if (!(await waitForDrainOrClose(response))) return;
}
}
if (!response.destroyed) response.end();
})();
return;
}
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/stream?after=10") {
if (request.url === "/internal/data-plane/v1/data-products/fleet.positions.current.v1/stream?after=8") {
response.writeHead(200, { "content-type": "text/event-stream" });
shutdownStream.opened = true;
const heartbeat = setInterval(() => response.write(": keepalive\n\n"), 50);
response.once("close", () => {
const closeShutdownStream = () => {
clearInterval(heartbeat);
shutdownStream.closed = true;
});
};
response.once("close", closeShutdownStream);
request.socket.once("close", closeShutdownStream);
response.write(": keepalive\n\n");
return;
}
@@ -258,6 +282,8 @@ try {
PORT: String(foundryPort),
FOUNDRY_RUNTIME_DIR: runtimeDir,
NODEDC_FOUNDRY_AUTH_REQUIRED: "false",
NODEDC_INTERNAL_ACCESS_TOKEN: internalAccessToken,
FOUNDRY_MCP_URL: `http://127.0.0.1:${foundryPort}/api/mcp`,
NODEDC_EXTERNAL_DATA_PLANE_INTERNAL_URL: `http://127.0.0.1:${edpPort}`,
NODEDC_EXTERNAL_DATA_PLANE_READER_GRANTS_DIR: grantDir,
NODEDC_FOUNDRY_BINDING_GRANTS_DIR: bindingGrantDir,
@@ -268,6 +294,40 @@ try {
foundry.stderr.on("data", (chunk) => { stderr += String(chunk); });
await waitFor(`http://127.0.0.1:${foundryPort}/healthz`);
const entitlementResponse = await fetch(`http://127.0.0.1:${foundryPort}/api/ai-workspace/entitlements`, {
method: "POST",
headers: { authorization: `Bearer ${internalAccessToken}`, "content-type": "application/json" },
body: JSON.stringify({
schemaVersion: "ai-workspace.entitlement-request.v1",
appId: "module-foundry",
owner: { id: "user_root", key: "user_root" },
}),
});
assert.equal(entitlementResponse.status, 200);
const entitlement = await entitlementResponse.json();
const mcpAuthorization = entitlement.appGrants["module-foundry"].mcpServers[0].httpHeaders.Authorization;
let mcpRequestId = 0;
const mcpRequest = async (method, params = {}) => {
const response = await fetch(`http://127.0.0.1:${foundryPort}/api/mcp`, {
method: "POST",
headers: {
authorization: mcpAuthorization,
"content-type": "application/json",
"mcp-protocol-version": "2025-06-18",
},
body: JSON.stringify({ jsonrpc: "2.0", id: ++mcpRequestId, method, params }),
});
assert.equal(response.status, 200);
const payload = await response.json();
assert.equal(payload.error, undefined);
return payload.result;
};
const initialized = await mcpRequest("initialize", { protocolVersion: "2025-06-18", capabilities: {}, clientInfo: { name: "runtime-smoke", version: "1" } });
assert.equal(initialized.serverInfo.version, "0.2.0");
const listedTools = await mcpRequest("tools/list");
assert.equal(listedTools.tools.length, 13);
assert.ok(listedTools.tools.some((tool) => tool.name === "foundry_apply_map_data_product_consumer"));
const workloadHeaders = {
authorization: `Bearer ${bindingGrantToken}`,
"content-type": "application/json",
@@ -308,6 +368,23 @@ try {
assert.equal(replayResponse.status, 200);
assert.equal((await replayResponse.json()).idempotency.replayed, true);
const consumerTarget = { applicationId, pageId, bindingId };
const planCall = await mcpRequest("tools/call", {
name: "foundry_plan_map_data_product_consumer",
arguments: consumerTarget,
});
const consumerPlan = planCall.structuredContent;
assert.equal(consumerPlan.action, "create");
assert.match(consumerPlan.planId, /^fcp1_/);
assert.equal(JSON.stringify(consumerPlan).includes(readerToken), false);
assert.equal(JSON.stringify(consumerPlan).includes(`127.0.0.1:${edpPort}`), false);
const applyCall = await mcpRequest("tools/call", {
name: "foundry_apply_map_data_product_consumer",
arguments: { ...consumerTarget, planId: consumerPlan.planId, idempotencyKey: "runtime-smoke-consumer-apply" },
});
assert.equal(applyCall.structuredContent.consumer.cursor, "7");
assert.equal(applyCall.structuredContent.consumer.subjectCount, 1);
const base = `http://127.0.0.1:${foundryPort}/api/applications/${applicationId}/pages/${pageId}/data-bindings/${bindingId}`;
const snapshotResponse = await fetch(`${base}/snapshot`);
assert.equal(snapshotResponse.status, 200);
@@ -317,18 +394,55 @@ try {
assert.deepEqual(snapshot.facts[0].attributes, { name: "Vehicle 001" });
assert.equal(JSON.stringify(snapshot).includes("must-not-reach-browser"), false);
const historyResponse = await fetch(`${base}/history?from=2026-07-15T12%3A00%3A00.000Z&to=2026-07-15T12%3A05%3A00.000Z&resolutionMs=60000&sourceIds=vehicle-001&limit=1000`);
assert.equal(historyResponse.status, 200);
const history = await historyResponse.json();
assert.equal(history.schemaVersion, "nodedc.data-product.history/v1");
assert.equal(history.facts.length, 1);
assert.equal(history.facts[0].bucketStart, "2026-07-15T12:00:00.000Z");
assert.deepEqual(history.facts[0].attributes, { name: "Vehicle 001 historical" });
assert.equal(history.nextCursor, "opaque_history_cursor");
assert.equal(JSON.stringify(history).includes("must-not-reach-browser"), false);
const resyncResponse = await fetch(`${base}/stream?after=1`, { headers: { accept: "text/event-stream" } });
assert.equal(resyncResponse.status, 200);
assert.match(await resyncResponse.text(), /nodedc\.data-product\.resync-required\.v1/);
const streamResponse = await fetch(`${base}/stream?after=7`, { headers: { accept: "text/event-stream" } });
assert.equal(streamResponse.status, 200);
const streamText = await streamResponse.text();
const streamText = await readHttpStreamUntil(`${base}/stream?after=7`, /"cursor":"8"/);
assert.match(streamText, /event: nodedc\.data-product\.ready\.v1/);
assert.match(streamText, /event: nodedc\.data-product\.patch\.v1/);
assert.match(streamText, /"cursor":"8"/);
assert.match(streamText, /Vehicle 001 updated/);
assert.equal(streamText.includes("must-not-reach-browser"), false);
await waitUntil(async () => {
const call = await mcpRequest("tools/call", {
name: "foundry_get_map_data_product_consumer_status",
arguments: consumerTarget,
});
return call.structuredContent.consumer.viewerCount === 0;
}, "viewer_lease_release");
await waitUntil(async () => {
const call = await mcpRequest("tools/call", {
name: "foundry_get_map_data_product_consumer_status",
arguments: consumerTarget,
});
return call.structuredContent.consumer.upstreamStreamCount === 0;
}, "shared_upstream_manager_release");
await waitUntil(() => firstStream.closed, "shared_upstream_close_after_last_viewer");
const statusCall = await mcpRequest("tools/call", {
name: "foundry_get_map_data_product_consumer_status",
arguments: consumerTarget,
});
assert.equal(statusCall.structuredContent.consumer.cursor, "8");
assert.equal(statusCall.structuredContent.consumer.subjectCount, 1);
assert.equal(statusCall.structuredContent.consumer.metrics.patchCommits, 1);
assert.equal(statusCall.structuredContent.consumer.viewerCount, 0);
const acceptanceCall = await mcpRequest("tools/call", {
name: "foundry_accept_map_data_product_consumer",
arguments: { ...consumerTarget, timeoutMs: 1000, minSubjectCount: 1, minPatchCount: 0 },
});
assert.equal(acceptanceCall.structuredContent.accepted, true);
assert.equal(stderr, "");
await rm(join(grantDir, targetKey));
@@ -338,16 +452,8 @@ try {
await writeFile(join(grantDir, targetKey), `${readerToken}\n`, "utf8");
await chmod(join(grantDir, targetKey), 0o400);
const paused = await openPausedStream(`${base}/stream?after=9`);
await waitUntil(() => pressure.backpressured, "upstream_backpressure");
await waitUntil(() => pressure.sent > 0 && pressure.sent < 2_048, "bounded_upstream_read");
await new Promise((resolve) => setTimeout(resolve, 150));
assert.ok(pressure.sent < 2_048, "Foundry must stop draining the upstream while the browser is backpressured");
paused.response.destroy();
paused.request.destroy();
await waitUntil(() => pressure.closed, "upstream_close_after_browser_disconnect");
const shutdownClient = await openPausedStream(`${base}/stream?after=10`);
const shutdownClient = await fetch(`${base}/stream?after=8`, { headers: { accept: "text/event-stream" } });
assert.equal(shutdownClient.status, 200);
await waitUntil(() => shutdownStream.opened, "shutdown_stream_open");
const exit = once(foundry, "exit");
foundry.kill("SIGTERM");
@@ -358,8 +464,7 @@ try {
assert.equal(exitCode, 0);
assert.equal(exitSignal, null);
await waitUntil(() => shutdownStream.closed, "upstream_close_after_shutdown");
shutdownClient.response.destroy();
shutdownClient.request.destroy();
await shutdownClient.body?.cancel().catch(() => undefined);
assert.equal(stderr, "");
console.log("foundry data-product runtime BFF: ok");
} finally {