import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { once } from "node:events"; import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { createServer, get as httpGet } from "node:http"; import { createServer as createNetServer } from "node:net"; import { tmpdir } from "node:os"; import { join } from "node:path"; const root = await mkdtemp(join(tmpdir(), "nodedc-map-gateway-streaming-smoke-")); const cacheDir = join(root, "cache"); const upstreamPort = await freePort(); const gatewayPort = await freePort(); const distinctCount = 12; const state = { sharedRequests: 0, sharedEnded: false, abortRequests: 0, abortEnded: false, brokenRequests: 0, distinctRequests: 0, distinctResponses: [], }; let upstream; let gateway; try { upstream = createServer((request, response) => { const requestUrl = new URL(request.url || "/", `http://127.0.0.1:${upstreamPort}`); if (requestUrl.pathname === "/slow/shared") { state.sharedRequests += 1; response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); response.write("first-"); setTimeout(() => { state.sharedEnded = true; response.end("second"); }, 250); return; } if (requestUrl.pathname === "/slow/abort") { state.abortRequests += 1; response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); response.write("abort-first-"); setTimeout(() => { state.abortEnded = true; response.end("abort-second"); }, 250); return; } if (requestUrl.pathname === "/broken") { state.brokenRequests += 1; response.writeHead(200, { "content-type": "application/octet-stream", "cache-control": "max-age=3600" }); response.write("partial-must-not-commit"); setTimeout(() => response.destroy(new Error("intentional_upstream_truncation")), 50); return; } if (requestUrl.pathname.startsWith("/distinct/")) { state.distinctRequests += 1; state.distinctResponses.push({ id: requestUrl.pathname.split("/").pop(), response }); if (state.distinctResponses.length === distinctCount) { for (const item of state.distinctResponses) { item.response.writeHead(200, { "content-type": "text/plain", "cache-control": "max-age=3600" }); item.response.end(`distinct-${item.id}`); } } return; } response.writeHead(404).end(); }); upstream.listen(upstreamPort, "127.0.0.1"); await once(upstream, "listening"); gateway = spawn(process.execPath, ["src/server.mjs"], { cwd: new URL("..", import.meta.url), env: { ...process.env, NODE_ENV: "test", PORT: String(gatewayPort), MAP_CACHE_DIR: cacheDir, MAP_CACHE_MODE: "readwrite", MAP_GATEWAY_ALLOW_ANONYMOUS: "true", MAP_GATEWAY_UPSTREAM_ALLOWLIST: "127.0.0.1", MAP_GATEWAY_TEST_ALLOW_HTTP_LOOPBACK: "true", }, stdio: ["ignore", "pipe", "pipe"], }); await waitForGateway(gatewayPort, gateway); const sharedSource = `http://127.0.0.1:${upstreamPort}/slow/shared`; const sharedUrl = gatewayUrl(gatewayPort, sharedSource); const leader = await fetch(sharedUrl); assert.equal(leader.status, 200); const reader = leader.body.getReader(); const first = await reader.read(); assert.equal(Buffer.from(first.value).toString("utf8"), "first-"); assert.equal(state.sharedEnded, false, "first byte must reach the leader before upstream end"); assert.equal(await indexEntryCount(cacheDir), 0, "partial body must not be published in index"); const followerPromise = fetch(sharedUrl).then(async (response) => ({ status: response.status, body: await response.text() })); const leaderBody = `first-${await readRemaining(reader)}`; const follower = await followerPromise; assert.equal(leaderBody, "first-second"); assert.deepEqual(follower, { status: 200, body: "first-second" }); assert.equal(state.sharedRequests, 1, "same-key followers must share one upstream"); assert.equal(await indexEntryCount(cacheDir), 1, "complete body must be committed"); const abortSource = `http://127.0.0.1:${upstreamPort}/slow/abort`; const abortUrl = gatewayUrl(gatewayPort, abortSource); assert.equal(await abortAfterFirstChunk(abortUrl), "abort-first-"); const afterAbort = await fetch(abortUrl); assert.equal(await afterAbort.text(), "abort-first-abort-second"); assert.equal(state.abortEnded, true); assert.equal(state.abortRequests, 1, "client abort must not cancel the server-owned cache fill"); assert.equal(await indexEntryCount(cacheDir), 2); const beforeBroken = await indexEntryCount(cacheDir); const brokenSource = `http://127.0.0.1:${upstreamPort}/broken`; let brokenRejected = false; try { const broken = await fetch(gatewayUrl(gatewayPort, brokenSource)); await broken.arrayBuffer(); } catch { brokenRejected = true; } assert.equal(brokenRejected, true); await waitFor(async () => (await tempFiles(cacheDir)).length === 0); assert.equal(await indexEntryCount(cacheDir), beforeBroken, "truncated upstream must not publish an index entry"); assert.equal((await tempFiles(cacheDir)).length, 0, "truncated upstream must remove temp files"); assert.equal((await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).status, 200, "stream failure must not stop Gateway"); const diagnosticsBefore = (await (await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).json()).diagnostics; const distinctBodies = await Promise.all(Array.from({ length: distinctCount }, async (_, index) => { const response = await fetch(gatewayUrl(gatewayPort, `http://127.0.0.1:${upstreamPort}/distinct/${index}`)); return response.text(); })); assert.deepEqual(distinctBodies.sort(), Array.from({ length: distinctCount }, (_, index) => `distinct-${index}`).sort()); await waitFor(async () => await indexEntryCount(cacheDir) === beforeBroken + distinctCount); const diagnosticsAfter = (await (await fetch(`http://127.0.0.1:${gatewayPort}/healthz`)).json()).diagnostics; const snapshotDelta = diagnosticsAfter.indexSnapshots - diagnosticsBefore.indexSnapshots; assert.equal(state.distinctRequests, distinctCount); assert.equal(snapshotDelta >= 1 && snapshotDelta <= 2, true, `expected <=2 coalesced snapshots, got ${snapshotDelta}`); console.log("ok: cold miss streams immediately, singleflight survives abort, partial bodies stay private, and index snapshots coalesce"); } finally { await stop(gateway); if (upstream?.listening) { upstream.close(); await once(upstream, "close"); } await rm(root, { recursive: true, force: true }); } function gatewayUrl(port, source) { return `http://127.0.0.1:${port}/api/map/cache?url=${encodeURIComponent(source)}`; } async function readRemaining(reader) { const chunks = []; while (true) { const next = await reader.read(); if (next.done) break; chunks.push(Buffer.from(next.value)); } return Buffer.concat(chunks).toString("utf8"); } function abortAfterFirstChunk(url) { return new Promise((resolve, reject) => { let resolved = false; const request = httpGet(url, (response) => { response.once("data", (chunk) => { resolved = true; resolve(String(chunk)); response.destroy(); }); response.once("error", (error) => { if (!resolved) reject(error); }); }); request.once("error", (error) => { if (!resolved) reject(error); }); }); } async function indexEntryCount(dir) { try { const index = JSON.parse(await readFile(join(dir, "index.json"), "utf8")); return Object.keys(index.entries || {}).length; } catch (error) { if (error?.code === "ENOENT") return 0; throw error; } } async function tempFiles(dir) { const found = []; async function walk(current) { let entries; try { entries = await readdir(current, { withFileTypes: true }); } catch (error) { if (error?.code === "ENOENT") return; throw error; } for (const entry of entries) { const path = join(current, entry.name); if (entry.isDirectory()) await walk(path); else if (entry.name.includes(".tmp")) found.push(path); } } await walk(dir); return found; } async function waitFor(predicate) { for (let attempt = 0; attempt < 200; attempt += 1) { if (await predicate()) return; await new Promise((resolve) => setTimeout(resolve, 20)); } throw new Error("condition_timeout"); } async function freePort() { const server = createNetServer(); server.listen(0, "127.0.0.1"); await once(server, "listening"); const address = server.address(); assert(address && typeof address === "object"); const port = address.port; server.close(); await once(server, "close"); return port; } async function waitForGateway(port, child) { let output = ""; child.stderr.on("data", (chunk) => { output += String(chunk); }); for (let attempt = 0; attempt < 80; attempt += 1) { if (child.exitCode !== null) throw new Error(`gateway_exited:${child.exitCode}:${output}`); try { const response = await fetch(`http://127.0.0.1:${port}/healthz`); if (response.ok) return; } catch { /* service is still starting */ } await new Promise((resolve) => setTimeout(resolve, 50)); } throw new Error(`gateway_start_timeout:${output}`); } async function stop(child) { if (child && !child.killed) { child.kill("SIGTERM"); await once(child, "exit").catch(() => undefined); } }